linux - mm/swapfile.c

3958 lines · 104.4 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/mm/swapfile.c
4	*
5	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6	* Swap reorganised 29.12.95, Stephen Tweedie
7	*/
8
9	#include <linux/blkdev.h>
10	#include <linux/mm.h>
11	#include <linux/sched/mm.h>
12	#include <linux/sched/task.h>
13	#include <linux/hugetlb.h>
14	#include <linux/mman.h>
15	#include <linux/slab.h>
16	#include <linux/kernel_stat.h>
17	#include <linux/swap.h>
18	#include <linux/vmalloc.h>
19	#include <linux/pagemap.h>
20	#include <linux/namei.h>
21	#include <linux/shmem_fs.h>
22	#include <linux/blk-cgroup.h>
23	#include <linux/random.h>
24	#include <linux/writeback.h>
25	#include <linux/proc_fs.h>
26	#include <linux/seq_file.h>
27	#include <linux/init.h>
28	#include <linux/ksm.h>
29	#include <linux/rmap.h>
30	#include <linux/security.h>
31	#include <linux/backing-dev.h>
32	#include <linux/mutex.h>
33	#include <linux/capability.h>
34	#include <linux/syscalls.h>
35	#include <linux/memcontrol.h>
36	#include <linux/poll.h>
37	#include <linux/oom.h>
38	#include <linux/swapfile.h>
39	#include <linux/export.h>
40	#include <linux/sort.h>
41	#include <linux/completion.h>
42	#include <linux/suspend.h>
43	#include <linux/zswap.h>
44	#include <linux/plist.h>
45
46	#include <asm/tlbflush.h>
47	#include <linux/leafops.h>
48	#include "swap_table.h"
49	#include "internal.h"
50	#include "swap.h"
51
52	static void swap_range_alloc(struct swap_info_struct *si,
53	unsigned int nr_entries);
54	static bool folio_swapcache_freeable(struct folio *folio);
55	static void move_cluster(struct swap_info_struct *si,
56	struct swap_cluster_info ci, struct list_head list,
57	enum swap_cluster_flags new_flags);
58
59	/*
60	* Protects the swap_info array, and the SWP_USED flag. swap_info contains
61	* lazily allocated & freed swap device info struts, and SWP_USED indicates
62	* which device is used, ~SWP_USED devices and can be reused.
63	*
64	* Also protects swap_active_head total_swap_pages, and the SWP_WRITEOK flag.
65	*/
66	static DEFINE_SPINLOCK(swap_lock);
67	static unsigned int nr_swapfiles;
68	atomic_long_t nr_swap_pages;
69	/*
70	* Some modules use swappable objects and may try to swap them out under
71	* memory pressure (via the shrinker). Before doing so, they may wish to
72	* check to see if any swap space is available.
73	*/
74	EXPORT_SYMBOL_GPL(nr_swap_pages);
75	/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
76	long total_swap_pages;
77	#define DEF_SWAP_PRIO -1
78	unsigned long swapfile_maximum_size;
79	#ifdef CONFIG_MIGRATION
80	bool swap_migration_ad_supported;
81	#endif /* CONFIG_MIGRATION */
82
83	static const char Bad_file[] = "Bad swap file entry ";
84	static const char Bad_offset[] = "Bad swap offset entry ";
85
86	/*
87	* all active swap_info_structs
88	* protected with swap_lock, and ordered by priority.
89	*/
90	static PLIST_HEAD(swap_active_head);
91
92	/*
93	* all available (active, not full) swap_info_structs
94	* protected with swap_avail_lock, ordered by priority.
95	* This is used by folio_alloc_swap() instead of swap_active_head
96	* because swap_active_head includes all swap_info_structs,
97	* but folio_alloc_swap() doesn't need to look at full ones.
98	* This uses its own lock instead of swap_lock because when a
99	* swap_info_struct changes between not-full/full, it needs to
100	* add/remove itself to/from this list, but the swap_info_struct->lock
101	* is held and the locking order requires swap_lock to be taken
102	* before any swap_info_struct->lock.
103	*/
104	static PLIST_HEAD(swap_avail_head);
105	static DEFINE_SPINLOCK(swap_avail_lock);
106
107	struct swap_info_struct *swap_info[MAX_SWAPFILES];
108
109	static struct kmem_cache *swap_table_cachep;
110
111	/* Protects si->swap_file for /proc/swaps usage */
112	static DEFINE_MUTEX(swapon_mutex);
113
114	static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
115	/* Activity counter to indicate that a swapon or swapoff has occurred */
116	static atomic_t proc_poll_event = ATOMIC_INIT(0);
117
118	atomic_t nr_rotate_swap = ATOMIC_INIT(0);
119
120	struct percpu_swap_cluster {
121	struct swap_info_struct *si[SWAP_NR_ORDERS];
122	unsigned long offset[SWAP_NR_ORDERS];
123	local_lock_t lock;
124	};
125
126	static DEFINE_PER_CPU(struct percpu_swap_cluster, percpu_swap_cluster) = {
127	.si = { NULL },
128	.offset = { SWAP_ENTRY_INVALID },
129	.lock = INIT_LOCAL_LOCK(),
130	};
131
132	/* May return NULL on invalid type, caller must check for NULL return */
133	static struct swap_info_struct *swap_type_to_info(int type)
134	{
135	if (type < 0 \|\| type >= MAX_SWAPFILES)
136	return NULL;
137	return READ_ONCE(swap_info[type]); /* rcu_dereference() */
138	}
139
140	/* May return NULL on invalid entry, caller must check for NULL return */
141	static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry)
142	{
143	return swap_type_to_info(swp_type(entry));
144	}
145
146	/*
147	* Use the second highest bit of inuse_pages counter as the indicator
148	* if one swap device is on the available plist, so the atomic can
149	* still be updated arithmetically while having special data embedded.
150	*
151	* inuse_pages counter is the only thing indicating if a device should
152	* be on avail_lists or not (except swapon / swapoff). By embedding the
153	* off-list bit in the atomic counter, updates no longer need any lock
154	* to check the list status.
155	*
156	* This bit will be set if the device is not on the plist and not
157	* usable, will be cleared if the device is on the plist.
158	*/
159	#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_long_t) - 2))
160	#define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT)
161	static long swap_usage_in_pages(struct swap_info_struct *si)
162	{
163	return atomic_long_read(&si->inuse_pages) & SWAP_USAGE_COUNTER_MASK;
164	}
165
166	/* Reclaim the swap entry anyway if possible */
167	#define TTRS_ANYWAY 0x1
168	/*
169	* Reclaim the swap entry if there are no more mappings of the
170	* corresponding page
171	*/
172	#define TTRS_UNMAPPED 0x2
173	/* Reclaim the swap entry if swap is getting full */
174	#define TTRS_FULL 0x4
175
176	static bool swap_only_has_cache(struct swap_cluster_info *ci,
177	unsigned long offset, int nr_pages)
178	{
179	unsigned int ci_off = offset % SWAPFILE_CLUSTER;
180	unsigned int ci_end = ci_off + nr_pages;
181	unsigned long swp_tb;
182
183	do {
184	swp_tb = __swap_table_get(ci, ci_off);
185	VM_WARN_ON_ONCE(!swp_tb_is_folio(swp_tb));
186	if (swp_tb_get_count(swp_tb))
187	return false;
188	} while (++ci_off < ci_end);
189
190	return true;
191	}
192
193	/*
194	* returns number of pages in the folio that backs the swap entry. If positive,
195	* the folio was reclaimed. If negative, the folio was not reclaimed. If 0, no
196	* folio was associated with the swap entry.
197	*/
198	static int __try_to_reclaim_swap(struct swap_info_struct *si,
199	unsigned long offset, unsigned long flags)
200	{
201	const swp_entry_t entry = swp_entry(si->type, offset);
202	struct swap_cluster_info *ci;
203	struct folio *folio;
204	int ret, nr_pages;
205	bool need_reclaim;
206
207	again:
208	folio = swap_cache_get_folio(entry);
209	if (!folio)
210	return 0;
211
212	nr_pages = folio_nr_pages(folio);
213	ret = -nr_pages;
214
215	/*
216	* We hold a folio lock here. We have to use trylock for
217	* avoiding deadlock. This is a special case and you should
218	* use folio_free_swap() with explicit folio_lock() in usual
219	* operations.
220	*/
221	if (!folio_trylock(folio))
222	goto out;
223
224	/*
225	* Offset could point to the middle of a large folio, or folio
226	* may no longer point to the expected offset before it's locked.
227	*/
228	if (!folio_matches_swap_entry(folio, entry)) {
229	folio_unlock(folio);
230	folio_put(folio);
231	goto again;
232	}
233	offset = swp_offset(folio->swap);
234
235	need_reclaim = ((flags & TTRS_ANYWAY) \|\|
236	((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) \|\|
237	((flags & TTRS_FULL) && mem_cgroup_swap_full(folio)));
238	if (!need_reclaim \|\| !folio_swapcache_freeable(folio))
239	goto out_unlock;
240
241	/*
242	* It's safe to delete the folio from swap cache only if the folio
243	* is in swap cache with swap count == 0. The slots have no page table
244	* reference or pending writeback, and can't be allocated to others.
245	*/
246	ci = swap_cluster_lock(si, offset);
247	need_reclaim = swap_only_has_cache(ci, offset, nr_pages);
248	swap_cluster_unlock(ci);
249	if (!need_reclaim)
250	goto out_unlock;
251
252	swap_cache_del_folio(folio);
253	folio_set_dirty(folio);
254	ret = nr_pages;
255	out_unlock:
256	folio_unlock(folio);
257	out:
258	folio_put(folio);
259	return ret;
260	}
261
262	static inline struct swap_extent first_se(struct swap_info_struct sis)
263	{
264	struct rb_node *rb = rb_first(&sis->swap_extent_root);
265	return rb_entry(rb, struct swap_extent, rb_node);
266	}
267
268	static inline struct swap_extent next_se(struct swap_extent se)
269	{
270	struct rb_node *rb = rb_next(&se->rb_node);
271	return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL;
272	}
273
274	/*
275	* swapon tell device that all the old swap contents can be discarded,
276	* to allow the swap device to optimize its wear-levelling.
277	*/
278	static int discard_swap(struct swap_info_struct *si)
279	{
280	struct swap_extent *se;
281	sector_t start_block;
282	sector_t nr_blocks;
283	int err = 0;
284
285	/* Do not discard the swap header page! */
286	se = first_se(si);
287	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
288	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
289	if (nr_blocks) {
290	err = blkdev_issue_discard(si->bdev, start_block,
291	nr_blocks, GFP_KERNEL);
292	if (err)
293	return err;
294	cond_resched();
295	}
296
297	for (se = next_se(se); se; se = next_se(se)) {
298	start_block = se->start_block << (PAGE_SHIFT - 9);
299	nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
300
301	err = blkdev_issue_discard(si->bdev, start_block,
302	nr_blocks, GFP_KERNEL);
303	if (err)
304	break;
305
306	cond_resched();
307	}
308	return err; /* That will often be -EOPNOTSUPP */
309	}
310
311	static struct swap_extent *
312	offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)
313	{
314	struct swap_extent *se;
315	struct rb_node *rb;
316
317	rb = sis->swap_extent_root.rb_node;
318	while (rb) {
319	se = rb_entry(rb, struct swap_extent, rb_node);
320	if (offset < se->start_page)
321	rb = rb->rb_left;
322	else if (offset >= se->start_page + se->nr_pages)
323	rb = rb->rb_right;
324	else
325	return se;
326	}
327	/* It must be present */
328	BUG();
329	}
330
331	sector_t swap_folio_sector(struct folio *folio)
332	{
333	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
334	struct swap_extent *se;
335	sector_t sector;
336	pgoff_t offset;
337
338	offset = swp_offset(folio->swap);
339	se = offset_to_swap_extent(sis, offset);
340	sector = se->start_block + (offset - se->start_page);
341	return sector << (PAGE_SHIFT - 9);
342	}
343
344	/*
345	* swap allocation tell device that a cluster of swap can now be discarded,
346	* to allow the swap device to optimize its wear-levelling.
347	*/
348	static void discard_swap_cluster(struct swap_info_struct *si,
349	pgoff_t start_page, pgoff_t nr_pages)
350	{
351	struct swap_extent *se = offset_to_swap_extent(si, start_page);
352
353	while (nr_pages) {
354	pgoff_t offset = start_page - se->start_page;
355	sector_t start_block = se->start_block + offset;
356	sector_t nr_blocks = se->nr_pages - offset;
357
358	if (nr_blocks > nr_pages)
359	nr_blocks = nr_pages;
360	start_page += nr_blocks;
361	nr_pages -= nr_blocks;
362
363	start_block <<= PAGE_SHIFT - 9;
364	nr_blocks <<= PAGE_SHIFT - 9;
365	if (blkdev_issue_discard(si->bdev, start_block,
366	nr_blocks, GFP_NOIO))
367	break;
368
369	se = next_se(se);
370	}
371	}
372
373	#define LATENCY_LIMIT 256
374
375	static inline bool cluster_is_empty(struct swap_cluster_info *info)
376	{
377	return info->count == 0;
378	}
379
380	static inline bool cluster_is_discard(struct swap_cluster_info *info)
381	{
382	return info->flags == CLUSTER_FLAG_DISCARD;
383	}
384
385	static inline bool cluster_table_is_alloced(struct swap_cluster_info *ci)
386	{
387	return rcu_dereference_protected(ci->table, lockdep_is_held(&ci->lock));
388	}
389
390	static inline bool cluster_is_usable(struct swap_cluster_info *ci, int order)
391	{
392	if (unlikely(ci->flags > CLUSTER_FLAG_USABLE))
393	return false;
394	if (!cluster_table_is_alloced(ci))
395	return false;
396	if (!order)
397	return true;
398	return cluster_is_empty(ci) \|\| order == ci->order;
399	}
400
401	static inline unsigned int cluster_index(struct swap_info_struct *si,
402	struct swap_cluster_info *ci)
403	{
404	return ci - si->cluster_info;
405	}
406
407	static inline unsigned int cluster_offset(struct swap_info_struct *si,
408	struct swap_cluster_info *ci)
409	{
410	return cluster_index(si, ci) * SWAPFILE_CLUSTER;
411	}
412
413	static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head)
414	{
415	struct folio *folio;
416
417	folio = page_folio(container_of(head, struct page, rcu_head));
418	folio_put(folio);
419	}
420
421	static void swap_cluster_free_table(struct swap_cluster_info *ci)
422	{
423	struct swap_table *table;
424
425	#ifdef CONFIG_MEMCG
426	kfree(ci->memcg_table);
427	ci->memcg_table = NULL;
428	#endif
429
430	#if !SWAP_TABLE_HAS_ZEROFLAG
431	kfree(ci->zero_bitmap);
432	ci->zero_bitmap = NULL;
433	#endif
434
435	table = (struct swap_table *)rcu_access_pointer(ci->table);
436	if (!table)
437	return;
438
439	rcu_assign_pointer(ci->table, NULL);
440	if (!SWP_TABLE_USE_PAGE) {
441	kmem_cache_free(swap_table_cachep, table);
442	return;
443	}
444
445	call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
446	swap_cluster_free_table_folio_rcu_cb);
447	}
448
449	static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
450	{
451	struct swap_table *table = NULL;
452	struct folio *folio;
453
454	/* The cluster must be empty and not on any list during allocation. */
455	VM_WARN_ON_ONCE(ci->flags \|\| !cluster_is_empty(ci));
456	if (rcu_access_pointer(ci->table))
457	return 0;
458
459	if (SWP_TABLE_USE_PAGE) {
460	folio = folio_alloc(gfp \| __GFP_ZERO, 0);
461	if (folio)
462	table = folio_address(folio);
463	} else {
464	table = kmem_cache_zalloc(swap_table_cachep, gfp);
465	}
466	if (!table)
467	return -ENOMEM;
468
469	rcu_assign_pointer(ci->table, table);
470
471	#ifdef CONFIG_MEMCG
472	if (!mem_cgroup_disabled()) {
473	VM_WARN_ON_ONCE(ci->memcg_table);
474	ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp);
475	if (!ci->memcg_table) {
476	swap_cluster_free_table(ci);
477	return -ENOMEM;
478	}
479	}
480	#endif
481
482	#if !SWAP_TABLE_HAS_ZEROFLAG
483	VM_WARN_ON_ONCE(ci->zero_bitmap);
484	ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp);
485	if (!ci->zero_bitmap) {
486	swap_cluster_free_table(ci);
487	return -ENOMEM;
488	}
489	#endif
490	return 0;
491	}
492
493	/*
494	* Sanity check to ensure nothing leaked, and the specified range is empty.
495	* One special case is that bad slots can't be freed, so check the number of
496	* bad slots for swapoff, and non-swapoff path must never free bad slots.
497	*/
498	static void swap_cluster_assert_empty(struct swap_cluster_info *ci,
499	unsigned int ci_off, unsigned int nr,
500	bool swapoff)
501	{
502	unsigned int ci_end = ci_off + nr;
503	unsigned long swp_tb;
504	int bad_slots = 0;
505
506	if (!IS_ENABLED(CONFIG_DEBUG_VM) && !swapoff)
507	return;
508
509	do {
510	swp_tb = __swap_table_get(ci, ci_off);
511	if (swp_tb_is_bad(swp_tb))
512	bad_slots++;
513	else
514	WARN_ON_ONCE(!swp_tb_is_null(swp_tb));
515	WARN_ON_ONCE(__swap_cgroup_get(ci, ci_off));
516	} while (++ci_off < ci_end);
517
518	WARN_ON_ONCE(bad_slots != (swapoff ? ci->count : 0));
519	WARN_ON_ONCE(nr == SWAPFILE_CLUSTER && ci->extend_table);
520	}
521
522	/*
523	* Allocate swap table for one cluster. Attempt an atomic allocation first,
524	* then fallback to sleeping allocation.
525	*/
526	static struct swap_cluster_info *
527	swap_cluster_populate(struct swap_info_struct *si,
528	struct swap_cluster_info *ci)
529	{
530	int ret;
531
532	/*
533	* Only cluster isolation from the allocator does table allocation.
534	* Swap allocator uses percpu clusters and holds the local lock.
535	*/
536	lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock);
537	if (!(si->flags & SWP_SOLIDSTATE))
538	lockdep_assert_held(&si->global_cluster_lock);
539	lockdep_assert_held(&ci->lock);
540
541	if (!swap_cluster_alloc_table(ci, __GFP_HIGH \| __GFP_NOMEMALLOC \|
542	__GFP_NOWARN))
543	return ci;
544
545	/*
546	* Try a sleep allocation. Each isolated free cluster may cause
547	* a sleep allocation, but there is a limited number of them, so
548	* the potential recursive allocation is limited.
549	*/
550	spin_unlock(&ci->lock);
551	if (!(si->flags & SWP_SOLIDSTATE))
552	spin_unlock(&si->global_cluster_lock);
553	local_unlock(&percpu_swap_cluster.lock);
554
555	ret = swap_cluster_alloc_table(ci, __GFP_HIGH \| __GFP_NOMEMALLOC \|
556	GFP_KERNEL);
557
558	/*
559	* Back to atomic context. We might have migrated to a new CPU with a
560	* usable percpu cluster. But just keep using the isolated cluster to
561	* make things easier. Migration indicates a slight change of workload
562	* so using a new free cluster might not be a bad idea, and the worst
563	* could happen with ignoring the percpu cluster is fragmentation,
564	* which is acceptable since this fallback and race is rare.
565	*/
566	local_lock(&percpu_swap_cluster.lock);
567	if (!(si->flags & SWP_SOLIDSTATE))
568	spin_lock(&si->global_cluster_lock);
569	spin_lock(&ci->lock);
570
571	if (ret) {
572	move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
573	spin_unlock(&ci->lock);
574	return NULL;
575	}
576	return ci;
577	}
578
579	static void move_cluster(struct swap_info_struct *si,
580	struct swap_cluster_info ci, struct list_head list,
581	enum swap_cluster_flags new_flags)
582	{
583	VM_WARN_ON(ci->flags == new_flags);
584
585	BUILD_BUG_ON(1 << sizeof(ci->flags) * BITS_PER_BYTE < CLUSTER_FLAG_MAX);
586	lockdep_assert_held(&ci->lock);
587
588	spin_lock(&si->lock);
589	if (ci->flags == CLUSTER_FLAG_NONE)
590	list_add_tail(&ci->list, list);
591	else
592	list_move_tail(&ci->list, list);
593	spin_unlock(&si->lock);
594	ci->flags = new_flags;
595	}
596
597	/* Add a cluster to discard list and schedule it to do discard */
598	static void swap_cluster_schedule_discard(struct swap_info_struct *si,
599	struct swap_cluster_info *ci)
600	{
601	VM_BUG_ON(ci->flags == CLUSTER_FLAG_FREE);
602	move_cluster(si, ci, &si->discard_clusters, CLUSTER_FLAG_DISCARD);
603	schedule_work(&si->discard_work);
604	}
605
606	static void __free_cluster(struct swap_info_struct si, struct swap_cluster_info ci)
607	{
608	swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, false);
609	swap_cluster_free_table(ci);
610	move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
611	ci->order = 0;
612	}
613
614	/*
615	* Isolate and lock the first cluster that is not contented on a list,
616	* clean its flag before taken off-list. Cluster flag must be in sync
617	* with list status, so cluster updaters can always know the cluster
618	* list status without touching si lock.
619	*
620	* Note it's possible that all clusters on a list are contented so
621	* this returns NULL for an non-empty list.
622	*/
623	static struct swap_cluster_info *isolate_lock_cluster(
624	struct swap_info_struct si, struct list_head list)
625	{
626	struct swap_cluster_info ci, found = NULL;
627	u8 flags = CLUSTER_FLAG_NONE;
628
629	spin_lock(&si->lock);
630	list_for_each_entry(ci, list, list) {
631	if (!spin_trylock(&ci->lock))
632	continue;
633
634	/* We may only isolate and clear flags of following lists */
635	VM_BUG_ON(!ci->flags);
636	VM_BUG_ON(ci->flags > CLUSTER_FLAG_USABLE &&
637	ci->flags != CLUSTER_FLAG_FULL);
638
639	list_del(&ci->list);
640	flags = ci->flags;
641	ci->flags = CLUSTER_FLAG_NONE;
642	found = ci;
643	break;
644	}
645	spin_unlock(&si->lock);
646
647	/* Cluster's table is freed when and only when it's on the free list. */
648	if (found && flags == CLUSTER_FLAG_FREE) {
649	VM_WARN_ON_ONCE(list != &si->free_clusters);
650	VM_WARN_ON_ONCE(cluster_table_is_alloced(found));
651	return swap_cluster_populate(si, found);
652	}
653
654	return found;
655	}
656
657	/*
658	* Doing discard actually. After a cluster discard is finished, the cluster
659	* will be added to free cluster list. Discard cluster is a bit special as
660	* they don't participate in allocation or reclaim, so clusters marked as
661	* CLUSTER_FLAG_DISCARD must remain off-list or on discard list.
662	*/
663	static bool swap_do_scheduled_discard(struct swap_info_struct *si)
664	{
665	struct swap_cluster_info *ci;
666	bool ret = false;
667	unsigned int idx;
668
669	spin_lock(&si->lock);
670	while (!list_empty(&si->discard_clusters)) {
671	ci = list_first_entry(&si->discard_clusters, struct swap_cluster_info, list);
672	/*
673	* Delete the cluster from list to prepare for discard, but keep
674	* the CLUSTER_FLAG_DISCARD flag, percpu_swap_cluster could be
675	* pointing to it, or ran into by relocate_cluster.
676	*/
677	list_del(&ci->list);
678	idx = cluster_index(si, ci);
679	spin_unlock(&si->lock);
680	discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
681	SWAPFILE_CLUSTER);
682
683	spin_lock(&ci->lock);
684	/*
685	* Discard is done, clear its flags as it's off-list, then
686	* return the cluster to allocation list.
687	*/
688	ci->flags = CLUSTER_FLAG_NONE;
689	__free_cluster(si, ci);
690	spin_unlock(&ci->lock);
691	ret = true;
692	spin_lock(&si->lock);
693	}
694	spin_unlock(&si->lock);
695	return ret;
696	}
697
698	static void swap_discard_work(struct work_struct *work)
699	{
700	struct swap_info_struct *si;
701
702	si = container_of(work, struct swap_info_struct, discard_work);
703
704	swap_do_scheduled_discard(si);
705	}
706
707	static void swap_users_ref_free(struct percpu_ref *ref)
708	{
709	struct swap_info_struct *si;
710
711	si = container_of(ref, struct swap_info_struct, users);
712	complete(&si->comp);
713	}
714
715	/*
716	* Must be called after freeing if ci->count == 0, moves the cluster to free
717	* or discard list.
718	*/
719	static void free_cluster(struct swap_info_struct si, struct swap_cluster_info ci)
720	{
721	VM_BUG_ON(ci->count != 0);
722	VM_BUG_ON(ci->flags == CLUSTER_FLAG_FREE);
723	lockdep_assert_held(&ci->lock);
724
725	/*
726	* If the swap is discardable, prepare discard the cluster
727	* instead of free it immediately. The cluster will be freed
728	* after discard.
729	*/
730	if ((si->flags & (SWP_WRITEOK \| SWP_PAGE_DISCARD)) ==
731	(SWP_WRITEOK \| SWP_PAGE_DISCARD)) {
732	swap_cluster_schedule_discard(si, ci);
733	return;
734	}
735
736	__free_cluster(si, ci);
737	}
738
739	/*
740	* Must be called after freeing if ci->count != 0, moves the cluster to
741	* nonfull list.
742	*/
743	static void partial_free_cluster(struct swap_info_struct *si,
744	struct swap_cluster_info *ci)
745	{
746	VM_BUG_ON(!ci->count \|\| ci->count == SWAPFILE_CLUSTER);
747	lockdep_assert_held(&ci->lock);
748
749	if (ci->flags != CLUSTER_FLAG_NONFULL)
750	move_cluster(si, ci, &si->nonfull_clusters[ci->order],
751	CLUSTER_FLAG_NONFULL);
752	}
753
754	/*
755	* Must be called after allocation, moves the cluster to full or frag list.
756	* Note: allocation doesn't acquire si lock, and may drop the ci lock for
757	* reclaim, so the cluster could be any where when called.
758	*/
759	static void relocate_cluster(struct swap_info_struct *si,
760	struct swap_cluster_info *ci)
761	{
762	lockdep_assert_held(&ci->lock);
763
764	/* Discard cluster must remain off-list or on discard list */
765	if (cluster_is_discard(ci))
766	return;
767
768	if (!ci->count) {
769	if (ci->flags != CLUSTER_FLAG_FREE)
770	free_cluster(si, ci);
771	} else if (ci->count != SWAPFILE_CLUSTER) {
772	if (ci->flags != CLUSTER_FLAG_FRAG)
773	move_cluster(si, ci, &si->frag_clusters[ci->order],
774	CLUSTER_FLAG_FRAG);
775	} else {
776	if (ci->flags != CLUSTER_FLAG_FULL)
777	move_cluster(si, ci, &si->full_clusters,
778	CLUSTER_FLAG_FULL);
779	}
780	}
781
782	/*
783	* The cluster corresponding to @offset will be accounted as having one bad
784	* slot. The cluster will not be added to the free cluster list, and its
785	* usage counter will be increased by 1. Only used for initialization.
786	*/
787	static int swap_cluster_setup_bad_slot(struct swap_info_struct *si,
788	struct swap_cluster_info *cluster_info,
789	unsigned int offset, bool mask)
790	{
791	unsigned int ci_off = offset % SWAPFILE_CLUSTER;
792	unsigned long idx = offset / SWAPFILE_CLUSTER;
793	struct swap_cluster_info *ci;
794	int ret = 0;
795
796	/* si->max may got shrunk by swap swap_activate() */
797	if (offset >= si->max && !mask) {
798	pr_debug("Ignoring bad slot %u (max: %u)\n", offset, si->max);
799	return 0;
800	}
801	/*
802	* Account it, skip header slot: si->pages is initiated as
803	* si->max - 1. Also skip the masking of last cluster,
804	* si->pages doesn't include that part.
805	*/
806	if (offset && !mask)
807	si->pages -= 1;
808	if (!si->pages) {
809	pr_warn("Empty swap-file\n");
810	return -EINVAL;
811	}
812
813	ci = cluster_info + idx;
814	/* Need to allocate swap table first for initial bad slot marking. */
815	if (!ci->count && swap_cluster_alloc_table(ci, GFP_KERNEL))
816	return -ENOMEM;
817	spin_lock(&ci->lock);
818	/* Check for duplicated bad swap slots. */
819	if (__swap_table_xchg(ci, ci_off, SWP_TB_BAD) != SWP_TB_NULL) {
820	pr_warn("Duplicated bad slot offset %d\n", offset);
821	ret = -EINVAL;
822	} else {
823	ci->count++;
824	}
825	spin_unlock(&ci->lock);
826
827	WARN_ON(ci->count > SWAPFILE_CLUSTER);
828	WARN_ON(ci->flags);
829
830	return ret;
831	}
832
833	/*
834	* Reclaim drops the ci lock, so the cluster may become unusable (freed or
835	* stolen by a lower order). @usable will be set to false if that happens.
836	*/
837	static bool cluster_reclaim_range(struct swap_info_struct *si,
838	struct swap_cluster_info *ci,
839	unsigned long start, unsigned int order,
840	bool *usable)
841	{
842	unsigned int nr_pages = 1 << order;
843	unsigned long offset = start, end = start + nr_pages;
844	unsigned long swp_tb;
845
846	spin_unlock(&ci->lock);
847	do {
848	swp_tb = swap_table_get(ci, offset % SWAPFILE_CLUSTER);
849	if (swp_tb_get_count(swp_tb))
850	break;
851	if (swp_tb_is_folio(swp_tb))
852	if (__try_to_reclaim_swap(si, offset, TTRS_ANYWAY) < 0)
853	break;
854	} while (++offset < end);
855	spin_lock(&ci->lock);
856
857	/*
858	* We just dropped ci->lock so cluster could be used by another
859	* order or got freed, check if it's still usable or empty.
860	*/
861	if (!cluster_is_usable(ci, order)) {
862	*usable = false;
863	return false;
864	}
865	*usable = true;
866
867	/* Fast path, no need to scan if the whole cluster is empty */
868	if (cluster_is_empty(ci))
869	return true;
870
871	/*
872	* Recheck the range no matter reclaim succeeded or not, the slot
873	* could have been be freed while we are not holding the lock.
874	*/
875	for (offset = start; offset < end; offset++) {
876	swp_tb = __swap_table_get(ci, offset % SWAPFILE_CLUSTER);
877	if (!swp_tb_is_null(swp_tb))
878	return false;
879	}
880
881	return true;
882	}
883
884	static bool cluster_scan_range(struct swap_info_struct *si,
885	struct swap_cluster_info *ci,
886	unsigned long offset, unsigned int nr_pages,
887	bool *need_reclaim)
888	{
889	unsigned int ci_off = offset % SWAPFILE_CLUSTER;
890	unsigned int ci_end = ci_off + nr_pages;
891	unsigned long swp_tb;
892
893	do {
894	swp_tb = __swap_table_get(ci, ci_off);
895	if (swp_tb_is_null(swp_tb))
896	continue;
897	if (swp_tb_is_folio(swp_tb) && !__swp_tb_get_count(swp_tb)) {
898	if (!vm_swap_full())
899	return false;
900	*need_reclaim = true;
901	continue;
902	}
903	/* Slot with zero count can only be NULL or folio */
904	VM_WARN_ON(!swp_tb_get_count(swp_tb));
905	return false;
906	} while (++ci_off < ci_end);
907
908	return true;
909	}
910
911	static bool __swap_cluster_alloc_entries(struct swap_info_struct *si,
912	struct swap_cluster_info *ci,
913	struct folio *folio,
914	unsigned int ci_off)
915	{
916	unsigned int order;
917	unsigned long nr_pages;
918
919	lockdep_assert_held(&ci->lock);
920
921	if (!(si->flags & SWP_WRITEOK))
922	return false;
923
924	/*
925	* All mm swap allocation starts with a folio (folio_alloc_swap),
926	* it's also the only allocation path for large orders allocation.
927	* Such swap slots starts with count == 0 and will be increased
928	* upon folio unmap.
929	*
930	* Else, it's a exclusive order 0 allocation for hibernation.
931	* The slot starts with count == 1 and never increases.
932	*/
933	if (likely(folio)) {
934	order = folio_order(folio);
935	nr_pages = 1 << order;
936	swap_cluster_assert_empty(ci, ci_off, nr_pages, false);
937	__swap_cache_add_folio(ci, folio, swp_entry(si->type,
938	ci_off + cluster_offset(si, ci)));
939	} else if (IS_ENABLED(CONFIG_HIBERNATION)) {
940	order = 0;
941	nr_pages = 1;
942	swap_cluster_assert_empty(ci, ci_off, 1, false);
943	/* Fake shadow placeholder with no flag, hibernation does not use the zeromap */
944	__swap_table_set(ci, ci_off, __swp_tb_mk_count(shadow_to_swp_tb(NULL, 0), 1));
945	} else {
946	/* Allocation without folio is only possible with hibernation */
947	WARN_ON_ONCE(1);
948	return false;
949	}
950
951	/*
952	* The first allocation in a cluster makes the
953	* cluster exclusive to this order
954	*/
955	if (cluster_is_empty(ci))
956	ci->order = order;
957	ci->count += nr_pages;
958	swap_range_alloc(si, nr_pages);
959
960	return true;
961	}
962
963	/* Try use a new cluster for current CPU and allocate from it. */
964	static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
965	struct swap_cluster_info *ci,
966	struct folio *folio, unsigned long offset)
967	{
968	unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
969	unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER);
970	unsigned int order = likely(folio) ? folio_order(folio) : 0;
971	unsigned long end = start + SWAPFILE_CLUSTER;
972	unsigned int nr_pages = 1 << order;
973	bool need_reclaim, ret, usable;
974
975	lockdep_assert_held(&ci->lock);
976	VM_WARN_ON(!cluster_is_usable(ci, order));
977
978	if (end < nr_pages \|\| ci->count + nr_pages > SWAPFILE_CLUSTER)
979	goto out;
980
981	for (end -= nr_pages; offset <= end; offset += nr_pages) {
982	need_reclaim = false;
983	if (!cluster_scan_range(si, ci, offset, nr_pages, &need_reclaim))
984	continue;
985	if (need_reclaim) {
986	ret = cluster_reclaim_range(si, ci, offset, order, &usable);
987	if (!usable)
988	goto out;
989	if (cluster_is_empty(ci))
990	offset = start;
991	/* Reclaim failed but cluster is usable, try next */
992	if (!ret)
993	continue;
994	}
995	if (!__swap_cluster_alloc_entries(si, ci, folio, offset % SWAPFILE_CLUSTER))
996	break;
997	found = offset;
998	offset += nr_pages;
999	if (ci->count < SWAPFILE_CLUSTER && offset <= end)
1000	next = offset;
1001	break;
1002	}
1003	out:
1004	relocate_cluster(si, ci);
1005	swap_cluster_unlock(ci);
1006	if (si->flags & SWP_SOLIDSTATE) {
1007	this_cpu_write(percpu_swap_cluster.offset[order], next);
1008	this_cpu_write(percpu_swap_cluster.si[order], si);
1009	} else {
1010	si->global_cluster->next[order] = next;
1011	}
1012	return found;
1013	}
1014
1015	static unsigned int alloc_swap_scan_list(struct swap_info_struct *si,
1016	struct list_head *list,
1017	struct folio *folio,
1018	bool scan_all)
1019	{
1020	unsigned int found = SWAP_ENTRY_INVALID;
1021
1022	do {
1023	struct swap_cluster_info *ci = isolate_lock_cluster(si, list);
1024	unsigned long offset;
1025
1026	if (!ci)
1027	break;
1028	offset = cluster_offset(si, ci);
1029	found = alloc_swap_scan_cluster(si, ci, folio, offset);
1030	if (found)
1031	break;
1032	} while (scan_all);
1033
1034	return found;
1035	}
1036
1037	static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force)
1038	{
1039	long to_scan = 1;
1040	unsigned long offset, end;
1041	struct swap_cluster_info *ci;
1042	unsigned long swp_tb;
1043	int nr_reclaim;
1044
1045	if (force)
1046	to_scan = swap_usage_in_pages(si) / SWAPFILE_CLUSTER;
1047
1048	while ((ci = isolate_lock_cluster(si, &si->full_clusters))) {
1049	offset = cluster_offset(si, ci);
1050	end = min(si->max, offset + SWAPFILE_CLUSTER);
1051	to_scan--;
1052
1053	while (offset < end) {
1054	swp_tb = swap_table_get(ci, offset % SWAPFILE_CLUSTER);
1055	if (swp_tb_is_folio(swp_tb) && !__swp_tb_get_count(swp_tb)) {
1056	spin_unlock(&ci->lock);
1057	nr_reclaim = __try_to_reclaim_swap(si, offset,
1058	TTRS_ANYWAY);
1059	spin_lock(&ci->lock);
1060	if (nr_reclaim) {
1061	offset += abs(nr_reclaim);
1062	continue;
1063	}
1064	}
1065	offset++;
1066	}
1067
1068	/* in case no swap cache is reclaimed */
1069	if (ci->flags == CLUSTER_FLAG_NONE)
1070	relocate_cluster(si, ci);
1071
1072	swap_cluster_unlock(ci);
1073	if (to_scan <= 0)
1074	break;
1075
1076	/*
1077	* When 'force' is false, 'to_scan' is initialized to 1.
1078	* The loop breaks above, making this cond_resched() unreachable
1079	* in atomic contexts.
1080	*/
1081	cond_resched();
1082	}
1083	}
1084
1085	static void swap_reclaim_work(struct work_struct *work)
1086	{
1087	struct swap_info_struct *si;
1088
1089	si = container_of(work, struct swap_info_struct, reclaim_work);
1090
1091	swap_reclaim_full_clusters(si, true);
1092	}
1093
1094	/*
1095	* Try to allocate swap entries with specified order and try set a new
1096	* cluster for current CPU too.
1097	*/
1098	static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si,
1099	struct folio *folio)
1100	{
1101	struct swap_cluster_info *ci;
1102	unsigned int order = likely(folio) ? folio_order(folio) : 0;
1103	unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
1104
1105	/*
1106	* Swapfile is not block device so unable
1107	* to allocate large entries.
1108	*/
1109	if (order && !(si->flags & SWP_BLKDEV))
1110	return 0;
1111
1112	if (!(si->flags & SWP_SOLIDSTATE)) {
1113	/* Serialize HDD SWAP allocation for each device. */
1114	spin_lock(&si->global_cluster_lock);
1115	offset = si->global_cluster->next[order];
1116	if (offset == SWAP_ENTRY_INVALID)
1117	goto new_cluster;
1118
1119	ci = swap_cluster_lock(si, offset);
1120	/* Cluster could have been used by another order */
1121	if (cluster_is_usable(ci, order)) {
1122	if (cluster_is_empty(ci))
1123	offset = cluster_offset(si, ci);
1124	found = alloc_swap_scan_cluster(si, ci, folio, offset);
1125	} else {
1126	swap_cluster_unlock(ci);
1127	}
1128	if (found)
1129	goto done;
1130	}
1131
1132	new_cluster:
1133	/*
1134	* If the device need discard, prefer new cluster over nonfull
1135	* to spread out the writes.
1136	*/
1137	if (si->flags & SWP_PAGE_DISCARD) {
1138	found = alloc_swap_scan_list(si, &si->free_clusters, folio, false);
1139	if (found)
1140	goto done;
1141	}
1142
1143	if (order < PMD_ORDER) {
1144	found = alloc_swap_scan_list(si, &si->nonfull_clusters[order], folio, true);
1145	if (found)
1146	goto done;
1147	}
1148
1149	if (!(si->flags & SWP_PAGE_DISCARD)) {
1150	found = alloc_swap_scan_list(si, &si->free_clusters, folio, false);
1151	if (found)
1152	goto done;
1153	}
1154
1155	/* Try reclaim full clusters if free and nonfull lists are drained */
1156	if (vm_swap_full())
1157	swap_reclaim_full_clusters(si, false);
1158
1159	if (order < PMD_ORDER) {
1160	/*
1161	* Scan only one fragment cluster is good enough. Order 0
1162	* allocation will surely success, and large allocation
1163	* failure is not critical. Scanning one cluster still
1164	* keeps the list rotated and reclaimed (for clean swap cache).
1165	*/
1166	found = alloc_swap_scan_list(si, &si->frag_clusters[order], folio, false);
1167	if (found)
1168	goto done;
1169	}
1170
1171	if (order)
1172	goto done;
1173
1174	/* Order 0 stealing from higher order */
1175	for (int o = 1; o < SWAP_NR_ORDERS; o++) {
1176	/*
1177	* Clusters here have at least one usable slots and can't fail order 0
1178	* allocation, but reclaim may drop si->lock and race with another user.
1179	*/
1180	found = alloc_swap_scan_list(si, &si->frag_clusters[o], folio, true);
1181	if (found)
1182	goto done;
1183
1184	found = alloc_swap_scan_list(si, &si->nonfull_clusters[o], folio, true);
1185	if (found)
1186	goto done;
1187	}
1188	done:
1189	if (!(si->flags & SWP_SOLIDSTATE))
1190	spin_unlock(&si->global_cluster_lock);
1191
1192	return found;
1193	}
1194
1195	/* SWAP_USAGE_OFFLIST_BIT can only be set by this helper. */
1196	static void del_from_avail_list(struct swap_info_struct *si, bool swapoff)
1197	{
1198	unsigned long pages;
1199
1200	spin_lock(&swap_avail_lock);
1201
1202	if (swapoff) {
1203	/*
1204	* Forcefully remove it. Clear the SWP_WRITEOK flags for
1205	* swapoff here so it's synchronized by both si->lock and
1206	* swap_avail_lock, to ensure the result can be seen by
1207	* add_to_avail_list.
1208	*/
1209	lockdep_assert_held(&si->lock);
1210	si->flags &= ~SWP_WRITEOK;
1211	atomic_long_or(SWAP_USAGE_OFFLIST_BIT, &si->inuse_pages);
1212	} else {
1213	/*
1214	* If not called by swapoff, take it off-list only if it's
1215	* full and SWAP_USAGE_OFFLIST_BIT is not set (strictly
1216	* si->inuse_pages == pages), any concurrent slot freeing,
1217	* or device already removed from plist by someone else
1218	* will make this return false.
1219	*/
1220	pages = si->pages;
1221	if (!atomic_long_try_cmpxchg(&si->inuse_pages, &pages,
1222	pages \| SWAP_USAGE_OFFLIST_BIT))
1223	goto skip;
1224	}
1225
1226	plist_del(&si->avail_list, &swap_avail_head);
1227
1228	skip:
1229	spin_unlock(&swap_avail_lock);
1230	}
1231
1232	/* SWAP_USAGE_OFFLIST_BIT can only be cleared by this helper. */
1233	static void add_to_avail_list(struct swap_info_struct *si, bool swapon)
1234	{
1235	long val;
1236	unsigned long pages;
1237
1238	spin_lock(&swap_avail_lock);
1239
1240	/* Corresponding to SWP_WRITEOK clearing in del_from_avail_list */
1241	if (swapon) {
1242	lockdep_assert_held(&si->lock);
1243	si->flags \|= SWP_WRITEOK;
1244	} else {
1245	if (!(READ_ONCE(si->flags) & SWP_WRITEOK))
1246	goto skip;
1247	}
1248
1249	if (!(atomic_long_read(&si->inuse_pages) & SWAP_USAGE_OFFLIST_BIT))
1250	goto skip;
1251
1252	val = atomic_long_fetch_and_relaxed(~SWAP_USAGE_OFFLIST_BIT, &si->inuse_pages);
1253
1254	/*
1255	* When device is full and device is on the plist, only one updater will
1256	* see (inuse_pages == si->pages) and will call del_from_avail_list. If
1257	* that updater happen to be here, just skip adding.
1258	*/
1259	pages = si->pages;
1260	if (val == pages) {
1261	/* Just like the cmpxchg in del_from_avail_list */
1262	if (atomic_long_try_cmpxchg(&si->inuse_pages, &pages,
1263	pages \| SWAP_USAGE_OFFLIST_BIT))
1264	goto skip;
1265	}
1266
1267	plist_add(&si->avail_list, &swap_avail_head);
1268
1269	skip:
1270	spin_unlock(&swap_avail_lock);
1271	}
1272
1273	/*
1274	* swap_usage_add / swap_usage_sub of each slot are serialized by ci->lock
1275	* within each cluster, so the total contribution to the global counter should
1276	* always be positive and cannot exceed the total number of usable slots.
1277	*/
1278	static bool swap_usage_add(struct swap_info_struct *si, unsigned int nr_entries)
1279	{
1280	long val = atomic_long_add_return_relaxed(nr_entries, &si->inuse_pages);
1281
1282	/*
1283	* If device is full, and SWAP_USAGE_OFFLIST_BIT is not set,
1284	* remove it from the plist.
1285	*/
1286	if (unlikely(val == si->pages)) {
1287	del_from_avail_list(si, false);
1288	return true;
1289	}
1290
1291	return false;
1292	}
1293
1294	static void swap_usage_sub(struct swap_info_struct *si, unsigned int nr_entries)
1295	{
1296	long val = atomic_long_sub_return_relaxed(nr_entries, &si->inuse_pages);
1297
1298	/*
1299	* If device is not full, and SWAP_USAGE_OFFLIST_BIT is set,
1300	* add it to the plist.
1301	*/
1302	if (unlikely(val & SWAP_USAGE_OFFLIST_BIT))
1303	add_to_avail_list(si, false);
1304	}
1305
1306	static void swap_range_alloc(struct swap_info_struct *si,
1307	unsigned int nr_entries)
1308	{
1309	if (swap_usage_add(si, nr_entries)) {
1310	if (vm_swap_full())
1311	schedule_work(&si->reclaim_work);
1312	}
1313	atomic_long_sub(nr_entries, &nr_swap_pages);
1314	}
1315
1316	static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
1317	unsigned int nr_entries)
1318	{
1319	unsigned long end = offset + nr_entries - 1;
1320	void (swap_slot_free_notify)(struct block_device , unsigned long);
1321	unsigned int i;
1322
1323	for (i = 0; i < nr_entries; i++)
1324	zswap_invalidate(swp_entry(si->type, offset + i));
1325
1326	if (si->flags & SWP_BLKDEV)
1327	swap_slot_free_notify =
1328	si->bdev->bd_disk->fops->swap_slot_free_notify;
1329	else
1330	swap_slot_free_notify = NULL;
1331	while (offset <= end) {
1332	arch_swap_invalidate_page(si->type, offset);
1333	if (swap_slot_free_notify)
1334	swap_slot_free_notify(si->bdev, offset);
1335	offset++;
1336	}
1337
1338	/*
1339	* Make sure that try_to_unuse() observes si->inuse_pages reaching 0
1340	* only after the above cleanups are done.
1341	*/
1342	smp_wmb();
1343	atomic_long_add(nr_entries, &nr_swap_pages);
1344	swap_usage_sub(si, nr_entries);
1345	}
1346
1347	static bool get_swap_device_info(struct swap_info_struct *si)
1348	{
1349	if (!percpu_ref_tryget_live(&si->users))
1350	return false;
1351	/*
1352	* Guarantee the si->users are checked before accessing other
1353	* fields of swap_info_struct, and si->flags (SWP_WRITEOK) is
1354	* up to dated.
1355	*
1356	* Paired with the spin_unlock() after setup_swap_info() in
1357	* enable_swap_info(), and smp_wmb() in swapoff.
1358	*/
1359	smp_rmb();
1360	return true;
1361	}
1362
1363	/*
1364	* Fast path try to get swap entries with specified order from current
1365	* CPU's swap entry pool (a cluster).
1366	*/
1367	static bool swap_alloc_fast(struct folio *folio)
1368	{
1369	unsigned int order = folio_order(folio);
1370	struct swap_cluster_info *ci;
1371	struct swap_info_struct *si;
1372	unsigned int offset;
1373
1374	/*
1375	* Once allocated, swap_info_struct will never be completely freed,
1376	* so checking it's liveness by get_swap_device_info is enough.
1377	*/
1378	si = this_cpu_read(percpu_swap_cluster.si[order]);
1379	offset = this_cpu_read(percpu_swap_cluster.offset[order]);
1380	if (!si \|\| !offset \|\| !get_swap_device_info(si))
1381	return false;
1382
1383	ci = swap_cluster_lock(si, offset);
1384	if (cluster_is_usable(ci, order)) {
1385	if (cluster_is_empty(ci))
1386	offset = cluster_offset(si, ci);
1387	alloc_swap_scan_cluster(si, ci, folio, offset);
1388	} else {
1389	swap_cluster_unlock(ci);
1390	}
1391
1392	put_swap_device(si);
1393	return folio_test_swapcache(folio);
1394	}
1395
1396	/* Rotate the device and switch to a new cluster */
1397	static void swap_alloc_slow(struct folio *folio)
1398	{
1399	struct swap_info_struct si, next;
1400
1401	spin_lock(&swap_avail_lock);
1402	start_over:
1403	plist_for_each_entry_safe(si, next, &swap_avail_head, avail_list) {
1404	/* Rotate the device and switch to a new cluster */
1405	plist_requeue(&si->avail_list, &swap_avail_head);
1406	spin_unlock(&swap_avail_lock);
1407	if (get_swap_device_info(si)) {
1408	cluster_alloc_swap_entry(si, folio);
1409	put_swap_device(si);
1410	if (folio_test_swapcache(folio))
1411	return;
1412	if (folio_test_large(folio))
1413	return;
1414	}
1415
1416	spin_lock(&swap_avail_lock);
1417	/*
1418	* if we got here, it's likely that si was almost full before,
1419	* multiple callers probably all tried to get a page from the
1420	* same si and it filled up before we could get one; or, the si
1421	* filled up between us dropping swap_avail_lock.
1422	* Since we dropped the swap_avail_lock, the swap_avail_list
1423	* may have been modified; so if next is still in the
1424	* swap_avail_head list then try it, otherwise start over if we
1425	* have not gotten any slots.
1426	*/
1427	if (plist_node_empty(&next->avail_list))
1428	goto start_over;
1429	}
1430	spin_unlock(&swap_avail_lock);
1431	}
1432
1433	/*
1434	* Discard pending clusters in a synchronized way when under high pressure.
1435	* Return: true if any cluster is discarded.
1436	*/
1437	static bool swap_sync_discard(void)
1438	{
1439	bool ret = false;
1440	struct swap_info_struct si, next;
1441
1442	spin_lock(&swap_lock);
1443	start_over:
1444	plist_for_each_entry_safe(si, next, &swap_active_head, list) {
1445	spin_unlock(&swap_lock);
1446	if (get_swap_device_info(si)) {
1447	if (si->flags & SWP_PAGE_DISCARD)
1448	ret = swap_do_scheduled_discard(si);
1449	put_swap_device(si);
1450	}
1451	if (ret)
1452	return true;
1453
1454	spin_lock(&swap_lock);
1455	if (plist_node_empty(&next->list))
1456	goto start_over;
1457	}
1458	spin_unlock(&swap_lock);
1459
1460	return false;
1461	}
1462
1463	static int swap_extend_table_alloc(struct swap_info_struct *si,
1464	struct swap_cluster_info *ci,
1465	unsigned int ci_off, gfp_t gfp)
1466	{
1467	int count;
1468	void *table;
1469
1470	table = kzalloc(sizeof(ci->extend_table[0]) * SWAPFILE_CLUSTER, gfp);
1471	if (!table)
1472	return -ENOMEM;
1473
1474	spin_lock(&ci->lock);
1475	/*
1476	* Extend table allocation requires releasing ci lock first so it's
1477	* possible that the slot has been freed, no longer overflowed, or
1478	* a concurrent extend table allocation has already succeeded, so
1479	* the allocation is no longer needed.
1480	*/
1481	if (!cluster_table_is_alloced(ci))
1482	goto out_free;
1483	count = swp_tb_get_count(__swap_table_get(ci, ci_off));
1484	if (count < (SWP_TB_COUNT_MAX - 1))
1485	goto out_free;
1486	if (ci->extend_table)
1487	goto out_free;
1488
1489	ci->extend_table = table;
1490	spin_unlock(&ci->lock);
1491	return 0;
1492
1493	out_free:
1494	spin_unlock(&ci->lock);
1495	kfree(table);
1496	return 0;
1497	}
1498
1499	int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
1500	{
1501	int ret;
1502	struct swap_info_struct *si;
1503	struct swap_cluster_info *ci;
1504	unsigned long offset = swp_offset(entry);
1505
1506	si = get_swap_device(entry);
1507	if (!si)
1508	return 0;
1509
1510	ci = __swap_offset_to_cluster(si, offset);
1511	ret = swap_extend_table_alloc(si, ci, swp_cluster_offset(entry), gfp);
1512
1513	put_swap_device(si);
1514	return ret;
1515	}
1516
1517	static void swap_extend_table_try_free(struct swap_cluster_info *ci)
1518	{
1519	unsigned long i;
1520	bool can_free = true;
1521
1522	if (!ci->extend_table)
1523	return;
1524
1525	for (i = 0; i < SWAPFILE_CLUSTER; i++) {
1526	if (ci->extend_table[i])
1527	can_free = false;
1528	}
1529
1530	if (can_free) {
1531	kfree(ci->extend_table);
1532	ci->extend_table = NULL;
1533	}
1534	}
1535
1536	/* Decrease the swap count of one slot, without freeing it */
1537	static void __swap_cluster_put_entry(struct swap_cluster_info *ci,
1538	unsigned int ci_off)
1539	{
1540	int count;
1541	unsigned long swp_tb;
1542
1543	lockdep_assert_held(&ci->lock);
1544	swp_tb = __swap_table_get(ci, ci_off);
1545	count = __swp_tb_get_count(swp_tb);
1546
1547	VM_WARN_ON_ONCE(count <= 0);
1548	VM_WARN_ON_ONCE(count > SWP_TB_COUNT_MAX);
1549
1550	if (count == SWP_TB_COUNT_MAX) {
1551	count = ci->extend_table[ci_off];
1552	/* Overflow starts with SWP_TB_COUNT_MAX */
1553	VM_WARN_ON_ONCE(count < SWP_TB_COUNT_MAX);
1554	count--;
1555	if (count == (SWP_TB_COUNT_MAX - 1)) {
1556	ci->extend_table[ci_off] = 0;
1557	__swap_table_set(ci, ci_off, __swp_tb_mk_count(swp_tb, count));
1558	} else {
1559	ci->extend_table[ci_off] = count;
1560	}
1561	} else {
1562	__swap_table_set(ci, ci_off, __swp_tb_mk_count(swp_tb, --count));
1563	}
1564
1565	/*
1566	* `SWP_TB_COUNT_MAX - 1` triggers extend table allocation. If the
1567	* count was above that, then the extend table is no longer needed,
1568	* so free it. And if we just put the count value from MAX - 1, it's
1569	* also possible that a pending dup just attached an extend table.
1570	*/
1571	if (unlikely(count == SWP_TB_COUNT_MAX - 2 \|\| count == SWP_TB_COUNT_MAX - 1))
1572	swap_extend_table_try_free(ci);
1573	}
1574
1575	/**
1576	* swap_put_entries_cluster - Decrease the swap count of slots within one cluster
1577	* @si: The swap device.
1578	* @offset: start offset of slots.
1579	* @nr: number of slots.
1580	* @reclaim_cache: if true, also reclaim the swap cache if slots are freed.
1581	*
1582	* This helper decreases the swap count of a set of slots and tries to
1583	* batch free them. Also reclaims the swap cache if @reclaim_cache is true.
1584	*
1585	* Context: The specified slots must be pinned by existing swap count or swap
1586	* cache reference, so they won't be released until this helper returns.
1587	*/
1588	static void swap_put_entries_cluster(struct swap_info_struct *si,
1589	pgoff_t offset, int nr,
1590	bool reclaim_cache)
1591	{
1592	struct swap_cluster_info *ci;
1593	unsigned int ci_off, ci_end;
1594	pgoff_t end = offset + nr;
1595	bool need_reclaim = false;
1596	unsigned int nr_reclaimed;
1597	unsigned long swp_tb;
1598	int ci_batch = -1;
1599
1600	ci = swap_cluster_lock(si, offset);
1601	ci_off = offset % SWAPFILE_CLUSTER;
1602	ci_end = ci_off + nr;
1603	do {
1604	swp_tb = __swap_table_get(ci, ci_off);
1605	if (swp_tb_get_count(swp_tb) == 1) {
1606	/* count == 1 and non-cached slots will be batch freed. */
1607	if (!swp_tb_is_folio(swp_tb)) {
1608	if (ci_batch == -1)
1609	ci_batch = ci_off;
1610	continue;
1611	}
1612	/* count will be 0 after put, slot can be reclaimed */
1613	need_reclaim = true;
1614	}
1615	/*
1616	* A count != 1 or cached slot can't be freed. Put its swap
1617	* count and then free the interrupted pending batch. Cached
1618	* slots will be freed when folio is removed from swap cache
1619	* (__swap_cache_del_folio).
1620	*/
1621	__swap_cluster_put_entry(ci, ci_off);
1622	if (ci_batch != -1) {
1623	__swap_cluster_free_entries(si, ci, ci_batch, ci_off - ci_batch);
1624	ci_batch = -1;
1625	}
1626	} while (++ci_off < ci_end);
1627
1628	if (ci_batch != -1)
1629	__swap_cluster_free_entries(si, ci, ci_batch, ci_off - ci_batch);
1630	swap_cluster_unlock(ci);
1631
1632	if (!need_reclaim \|\| !reclaim_cache)
1633	return;
1634
1635	do {
1636	nr_reclaimed = __try_to_reclaim_swap(si, offset,
1637	TTRS_UNMAPPED \| TTRS_FULL);
1638	offset++;
1639	if (nr_reclaimed)
1640	offset = round_up(offset, abs(nr_reclaimed));
1641	} while (offset < end);
1642	}
1643
1644	/* Increase the swap count of one slot. */
1645	static int __swap_cluster_dup_entry(struct swap_cluster_info *ci,
1646	unsigned int ci_off)
1647	{
1648	int count;
1649	unsigned long swp_tb;
1650
1651	lockdep_assert_held(&ci->lock);
1652	swp_tb = __swap_table_get(ci, ci_off);
1653	/* Bad or special slots can't be handled */
1654	if (WARN_ON_ONCE(swp_tb_is_bad(swp_tb)))
1655	return -EINVAL;
1656	count = __swp_tb_get_count(swp_tb);
1657	/* Must be either cached or have a count already */
1658	if (WARN_ON_ONCE(!count && !swp_tb_is_folio(swp_tb)))
1659	return -ENOENT;
1660
1661	if (likely(count < (SWP_TB_COUNT_MAX - 1))) {
1662	__swap_table_set(ci, ci_off, __swp_tb_mk_count(swp_tb, count + 1));
1663	VM_WARN_ON_ONCE(ci->extend_table && ci->extend_table[ci_off]);
1664	} else if (count == (SWP_TB_COUNT_MAX - 1)) {
1665	if (ci->extend_table) {
1666	VM_WARN_ON_ONCE(ci->extend_table[ci_off]);
1667	ci->extend_table[ci_off] = SWP_TB_COUNT_MAX;
1668	__swap_table_set(ci, ci_off, __swp_tb_mk_count(swp_tb, SWP_TB_COUNT_MAX));
1669	} else {
1670	return -ENOMEM;
1671	}
1672	} else if (count == SWP_TB_COUNT_MAX) {
1673	VM_WARN_ON_ONCE(ci->extend_table[ci_off] >=
1674	type_max(typeof(ci->extend_table[0])));
1675	++ci->extend_table[ci_off];
1676	} else {
1677	/* Never happens unless counting went wrong */
1678	WARN_ON_ONCE(1);
1679	}
1680
1681	return 0;
1682	}
1683
1684	/**
1685	* swap_dup_entries_cluster: Increase the swap count of slots within one cluster.
1686	* @si: The swap device.
1687	* @offset: start offset of slots.
1688	* @nr: number of slots.
1689	*
1690	* Context: The specified slots must be pinned by existing swap count or swap
1691	* cache reference, so they won't be released until this helper returns.
1692	* Return: 0 on success. -ENOMEM if the swap count maxed out (SWP_TB_COUNT_MAX)
1693	* and failed to allocate an extended table, -EINVAL if any entry is bad entry.
1694	*/
1695	static int swap_dup_entries_cluster(struct swap_info_struct *si,
1696	pgoff_t offset, int nr)
1697	{
1698	int err;
1699	struct swap_cluster_info *ci;
1700	unsigned int ci_start, ci_off, ci_end;
1701
1702	ci_start = offset % SWAPFILE_CLUSTER;
1703	ci_end = ci_start + nr;
1704	ci_off = ci_start;
1705	ci = swap_cluster_lock(si, offset);
1706	restart:
1707	do {
1708	err = __swap_cluster_dup_entry(ci, ci_off);
1709	if (unlikely(err)) {
1710	if (err == -ENOMEM) {
1711	spin_unlock(&ci->lock);
1712	err = swap_extend_table_alloc(si, ci, ci_off, GFP_ATOMIC);
1713	spin_lock(&ci->lock);
1714	if (!err)
1715	goto restart;
1716	}
1717	goto failed;
1718	}
1719	} while (++ci_off < ci_end);
1720	swap_cluster_unlock(ci);
1721	return 0;
1722	failed:
1723	while (ci_off-- > ci_start)
1724	__swap_cluster_put_entry(ci, ci_off);
1725	swap_extend_table_try_free(ci);
1726	swap_cluster_unlock(ci);
1727	return err;
1728	}
1729
1730	/**
1731	* folio_alloc_swap - allocate swap space for a folio
1732	* @folio: folio we want to move to swap
1733	*
1734	* Allocate swap space for the folio and add the folio to the
1735	* swap cache.
1736	*
1737	* Context: Caller needs to hold the folio lock.
1738	* Return: Whether the folio was added to the swap cache.
1739	*/
1740	int folio_alloc_swap(struct folio *folio)
1741	{
1742	unsigned int order = folio_order(folio);
1743	unsigned int size = 1 << order;
1744
1745	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1746	VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio);
1747
1748	if (order) {
1749	/*
1750	* Reject large allocation when THP_SWAP is disabled,
1751	* the caller should split the folio and try again.
1752	*/
1753	if (!IS_ENABLED(CONFIG_THP_SWAP))
1754	return -EAGAIN;
1755
1756	/*
1757	* Allocation size should never exceed cluster size
1758	* (HPAGE_PMD_SIZE).
1759	*/
1760	if (size > SWAPFILE_CLUSTER) {
1761	VM_WARN_ON_ONCE(1);
1762	return -EINVAL;
1763	}
1764	}
1765
1766	again:
1767	local_lock(&percpu_swap_cluster.lock);
1768	if (!swap_alloc_fast(folio))
1769	swap_alloc_slow(folio);
1770	local_unlock(&percpu_swap_cluster.lock);
1771
1772	if (!order && unlikely(!folio_test_swapcache(folio))) {
1773	if (swap_sync_discard())
1774	goto again;
1775	}
1776
1777	/* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */
1778	if (unlikely(mem_cgroup_try_charge_swap(folio)))
1779	swap_cache_del_folio(folio);
1780
1781	if (unlikely(!folio_test_swapcache(folio)))
1782	return -ENOMEM;
1783
1784	return 0;
1785	}
1786
1787	/**
1788	* folio_dup_swap() - Increase swap count of swap entries of a folio.
1789	* @folio: folio with swap entries bounded.
1790	* @page: if not NULL, only increase the swap count of this page.
1791	*
1792	* Typically called when the folio is unmapped and have its swap entry to
1793	* take its place: Swap entries allocated to a folio has count == 0 and pinned
1794	* by swap cache. The swap cache pin doesn't increase the swap count. This
1795	* helper sets the initial count == 1 and increases the count as the folio is
1796	* unmapped and swap entries referencing the slots are generated to replace
1797	* the folio.
1798	*
1799	* Context: Caller must ensure the folio is locked and in the swap cache.
1800	* NOTE: The caller also has to ensure there is no raced call to
1801	* swap_put_entries_direct on its swap entry before this helper returns, or
1802	* the swap count may underflow.
1803	*/
1804	int folio_dup_swap(struct folio folio, struct page page)
1805	{
1806	swp_entry_t entry = folio->swap;
1807	unsigned long nr_pages = folio_nr_pages(folio);
1808
1809	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
1810	VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio);
1811
1812	if (page) {
1813	entry.val += folio_page_idx(folio, page);
1814	nr_pages = 1;
1815	}
1816
1817	return swap_dup_entries_cluster(swap_entry_to_info(entry),
1818	swp_offset(entry), nr_pages);
1819	}
1820
1821	/**
1822	* folio_put_swap() - Decrease swap count of swap entries of a folio.
1823	* @folio: folio with swap entries bounded, must be in swap cache and locked.
1824	* @page: if not NULL, only decrease the swap count of this page.
1825	*
1826	* This won't free the swap slots even if swap count drops to zero, they are
1827	* still pinned by the swap cache. User may call folio_free_swap to free them.
1828	* Context: Caller must ensure the folio is locked and in the swap cache.
1829	*/
1830	void folio_put_swap(struct folio folio, struct page page)
1831	{
1832	swp_entry_t entry = folio->swap;
1833	unsigned long nr_pages = folio_nr_pages(folio);
1834	struct swap_info_struct *si = __swap_entry_to_info(entry);
1835
1836	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
1837	VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio);
1838
1839	if (page) {
1840	entry.val += folio_page_idx(folio, page);
1841	nr_pages = 1;
1842	}
1843
1844	swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false);
1845	}
1846
1847	/*
1848	* When we get a swap entry, if there aren't some other ways to
1849	* prevent swapoff, such as the folio in swap cache is locked, RCU
1850	* reader side is locked, etc., the swap entry may become invalid
1851	* because of swapoff. Then, we need to enclose all swap related
1852	* functions with get_swap_device() and put_swap_device(), unless the
1853	* swap functions call get/put_swap_device() by themselves.
1854	*
1855	* RCU reader side lock (including any spinlock) is sufficient to
1856	* prevent swapoff, because synchronize_rcu() is called in swapoff()
1857	* before freeing data structures.
1858	*
1859	* Check whether swap entry is valid in the swap device. If so,
1860	* return pointer to swap_info_struct, and keep the swap entry valid
1861	* via preventing the swap device from being swapoff, until
1862	* put_swap_device() is called. Otherwise return NULL.
1863	*
1864	* Notice that swapoff or swapoff+swapon can still happen before the
1865	* percpu_ref_tryget_live() in get_swap_device() or after the
1866	* percpu_ref_put() in put_swap_device() if there isn't any other way
1867	* to prevent swapoff. The caller must be prepared for that. For
1868	* example, the following situation is possible.
1869	*
1870	* CPU1 CPU2
1871	* do_swap_page()
1872	* ... swapoff+swapon
1873	* swap_cache_alloc_folio()
1874	* // check swap_map
1875	* // verify PTE not changed
1876	*
1877	* In __swap_duplicate(), the swap_map need to be checked before
1878	* changing partly because the specified swap entry may be for another
1879	* swap device which has been swapoff. And in do_swap_page(), after
1880	* the page is read from the swap device, the PTE is verified not
1881	* changed with the page table locked to check whether the swap device
1882	* has been swapoff or swapoff+swapon.
1883	*/
1884	struct swap_info_struct *get_swap_device(swp_entry_t entry)
1885	{
1886	struct swap_info_struct *si;
1887	unsigned long offset;
1888
1889	if (!entry.val)
1890	goto out;
1891	si = swap_entry_to_info(entry);
1892	if (!si)
1893	goto bad_nofile;
1894	if (!get_swap_device_info(si))
1895	goto out;
1896	offset = swp_offset(entry);
1897	if (offset >= si->max)
1898	goto put_out;
1899
1900	return si;
1901	bad_nofile:
1902	pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1903	out:
1904	return NULL;
1905	put_out:
1906	pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1907	percpu_ref_put(&si->users);
1908	return NULL;
1909	}
1910
1911	/*
1912	* Free a set of swap slots after their swap count dropped to zero, or will be
1913	* zero after putting the last ref (saves one __swap_cluster_put_entry call).
1914	*/
1915	void __swap_cluster_free_entries(struct swap_info_struct *si,
1916	struct swap_cluster_info *ci,
1917	unsigned int ci_start, unsigned int nr_pages)
1918	{
1919	unsigned long old_tb;
1920	unsigned short batch_id = 0, id_cur;
1921	unsigned int ci_off = ci_start, ci_end = ci_start + nr_pages;
1922	unsigned long ci_head = cluster_offset(si, ci);
1923	unsigned int batch_off = ci_off;
1924
1925	VM_WARN_ON(ci->count < nr_pages);
1926
1927	ci->count -= nr_pages;
1928	do {
1929	old_tb = __swap_table_get(ci, ci_off);
1930	/*
1931	* Freeing is done after release of the last swap count
1932	* ref, or after swap cache is dropped
1933	*/
1934	VM_WARN_ON(!swp_tb_is_shadow(old_tb) \|\| __swp_tb_get_count(old_tb) > 1);
1935
1936	/* Resetting the slot to NULL also clears the inline flags. */
1937	__swap_table_set(ci, ci_off, null_to_swp_tb());
1938	if (!SWAP_TABLE_HAS_ZEROFLAG)
1939	__swap_table_clear_zero(ci, ci_off);
1940
1941	/*
1942	* Uncharge swap slots by memcg in batches. Consecutive
1943	* slots with the same cgroup id are uncharged together.
1944	*/
1945	id_cur = __swap_cgroup_clear(ci, ci_off, 1);
1946	if (batch_id != id_cur) {
1947	if (batch_id)
1948	mem_cgroup_uncharge_swap(batch_id, ci_off - batch_off);
1949	batch_id = id_cur;
1950	batch_off = ci_off;
1951	}
1952	} while (++ci_off < ci_end);
1953
1954	if (batch_id)
1955	mem_cgroup_uncharge_swap(batch_id, ci_off - batch_off);
1956
1957	swap_range_free(si, ci_head + ci_start, nr_pages);
1958	swap_cluster_assert_empty(ci, ci_start, nr_pages, false);
1959
1960	if (!ci->count)
1961	free_cluster(si, ci);
1962	else
1963	partial_free_cluster(si, ci);
1964	}
1965
1966	int __swap_count(swp_entry_t entry)
1967	{
1968	struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
1969	unsigned int ci_off = swp_cluster_offset(entry);
1970
1971	return swp_tb_get_count(__swap_table_get(ci, ci_off));
1972	}
1973
1974	/**
1975	* swap_entry_swapped - Check if the swap entry is swapped.
1976	* @si: the swap device.
1977	* @entry: the swap entry.
1978	*/
1979	bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry)
1980	{
1981	pgoff_t offset = swp_offset(entry);
1982	struct swap_cluster_info *ci;
1983	unsigned long swp_tb;
1984
1985	ci = swap_cluster_lock(si, offset);
1986	swp_tb = swap_table_get(ci, offset % SWAPFILE_CLUSTER);
1987	swap_cluster_unlock(ci);
1988
1989	return swp_tb_get_count(swp_tb) > 0;
1990	}
1991
1992	/*
1993	* How many references to @entry are currently swapped out?
1994	* This returns exact answer.
1995	*/
1996	int swp_swapcount(swp_entry_t entry)
1997	{
1998	struct swap_info_struct *si;
1999	struct swap_cluster_info *ci;
2000	unsigned long swp_tb;
2001	int count;
2002
2003	si = get_swap_device(entry);
2004	if (!si)
2005	return 0;
2006
2007	ci = swap_cluster_lock(si, swp_offset(entry));
2008	swp_tb = __swap_table_get(ci, swp_cluster_offset(entry));
2009	count = swp_tb_get_count(swp_tb);
2010	if (count == SWP_TB_COUNT_MAX)
2011	count = ci->extend_table[swp_cluster_offset(entry)];
2012	swap_cluster_unlock(ci);
2013	put_swap_device(si);
2014
2015	return count < 0 ? 0 : count;
2016	}
2017
2018	/*
2019	* folio_maybe_swapped - Test if a folio covers any swap slot with count > 0.
2020	*
2021	* Check if a folio is swapped. Holding the folio lock ensures the folio won't
2022	* go from not-swapped to swapped because the initial swap count increment can
2023	* only be done by folio_dup_swap, which also locks the folio. But a concurrent
2024	* decrease of swap count is possible through swap_put_entries_direct, so this
2025	* may return a false positive.
2026	*
2027	* Context: Caller must ensure the folio is locked and in the swap cache.
2028	*/
2029	static bool folio_maybe_swapped(struct folio *folio)
2030	{
2031	swp_entry_t entry = folio->swap;
2032	struct swap_cluster_info *ci;
2033	unsigned int ci_off, ci_end;
2034	bool ret = false;
2035
2036	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
2037	VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
2038
2039	ci = __swap_entry_to_cluster(entry);
2040	ci_off = swp_cluster_offset(entry);
2041	ci_end = ci_off + folio_nr_pages(folio);
2042	/*
2043	* Extra locking not needed, folio lock ensures its swap entries
2044	* won't be released, the backing data won't be gone either.
2045	*/
2046	rcu_read_lock();
2047	do {
2048	if (__swp_tb_get_count(__swap_table_get(ci, ci_off))) {
2049	ret = true;
2050	break;
2051	}
2052	} while (++ci_off < ci_end);
2053	rcu_read_unlock();
2054
2055	return ret;
2056	}
2057
2058	static bool folio_swapcache_freeable(struct folio *folio)
2059	{
2060	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
2061
2062	if (!folio_test_swapcache(folio))
2063	return false;
2064	if (folio_test_writeback(folio))
2065	return false;
2066
2067	/*
2068	* Once hibernation has begun to create its image of memory,
2069	* there's a danger that one of the calls to folio_free_swap()
2070	* - most probably a call from __try_to_reclaim_swap() while
2071	* hibernation is allocating its own swap pages for the image,
2072	* but conceivably even a call from memory reclaim - will free
2073	* the swap from a folio which has already been recorded in the
2074	* image as a clean swapcache folio, and then reuse its swap for
2075	* another page of the image. On waking from hibernation, the
2076	* original folio might be freed under memory pressure, then
2077	* later read back in from swap, now with the wrong data.
2078	*
2079	* Hibernation suspends storage while it is writing the image
2080	* to disk so check that here.
2081	*/
2082	if (pm_suspended_storage())
2083	return false;
2084
2085	return true;
2086	}
2087
2088	/**
2089	* folio_free_swap() - Free the swap space used for this folio.
2090	* @folio: The folio to remove.
2091	*
2092	* If swap is getting full, or if there are no more mappings of this folio,
2093	* then call folio_free_swap to free its swap space.
2094	*
2095	* Return: true if we were able to release the swap space.
2096	*/
2097	bool folio_free_swap(struct folio *folio)
2098	{
2099	if (!folio_swapcache_freeable(folio))
2100	return false;
2101	if (folio_maybe_swapped(folio))
2102	return false;
2103
2104	swap_cache_del_folio(folio);
2105	folio_set_dirty(folio);
2106	return true;
2107	}
2108
2109	/**
2110	* swap_put_entries_direct() - Release reference on range of swap entries and
2111	* reclaim their cache if no more references remain.
2112	* @entry: First entry of range.
2113	* @nr: Number of entries in range.
2114	*
2115	* For each swap entry in the contiguous range, release a reference. If any swap
2116	* entries become free, try to reclaim their underlying folios, if present. The
2117	* offset range is defined by [entry.offset, entry.offset + nr).
2118	*
2119	* Context: Caller must ensure there is no race condition on the reference
2120	* owner. e.g., locking the PTL of a PTE containing the entry being released.
2121	*/
2122	void swap_put_entries_direct(swp_entry_t entry, int nr)
2123	{
2124	const unsigned long start_offset = swp_offset(entry);
2125	const unsigned long end_offset = start_offset + nr;
2126	unsigned long offset, cluster_end;
2127	struct swap_info_struct *si;
2128
2129	si = get_swap_device(entry);
2130	if (WARN_ON_ONCE(!si))
2131	return;
2132	if (WARN_ON_ONCE(end_offset > si->max))
2133	goto out;
2134
2135	/* Put entries and reclaim cache in each cluster */
2136	offset = start_offset;
2137	do {
2138	cluster_end = min(round_up(offset + 1, SWAPFILE_CLUSTER), end_offset);
2139	swap_put_entries_cluster(si, offset, cluster_end - offset, true);
2140	offset = cluster_end;
2141	} while (offset < end_offset);
2142	out:
2143	put_swap_device(si);
2144	}
2145
2146	#ifdef CONFIG_HIBERNATION
2147	/**
2148	* swap_alloc_hibernation_slot() - Allocate a swap slot for hibernation.
2149	* @type: swap device type index to allocate from.
2150	*
2151	* The caller must ensure the swap device is stable, either by pinning
2152	* it (SWP_HIBERNATION) or by freezing user-space.
2153	*
2154	* Return: a valid swp_entry_t on success, or an empty entry (val == 0)
2155	* on failure.
2156	*/
2157	swp_entry_t swap_alloc_hibernation_slot(int type)
2158	{
2159	struct swap_info_struct pcp_si, si = swap_type_to_info(type);
2160	unsigned long pcp_offset, offset = SWAP_ENTRY_INVALID;
2161	struct swap_cluster_info *ci;
2162	swp_entry_t entry = {0};
2163
2164	if (!si)
2165	goto fail;
2166
2167	/*
2168	* Try the local cluster first if it matches the device. If
2169	* not, try grab a new cluster and override local cluster.
2170	*/
2171	local_lock(&percpu_swap_cluster.lock);
2172	pcp_si = this_cpu_read(percpu_swap_cluster.si[0]);
2173	pcp_offset = this_cpu_read(percpu_swap_cluster.offset[0]);
2174	if (pcp_si == si && pcp_offset) {
2175	ci = swap_cluster_lock(si, pcp_offset);
2176	if (cluster_is_usable(ci, 0))
2177	offset = alloc_swap_scan_cluster(si, ci, NULL, pcp_offset);
2178	else
2179	swap_cluster_unlock(ci);
2180	}
2181	if (!offset)
2182	offset = cluster_alloc_swap_entry(si, NULL);
2183	local_unlock(&percpu_swap_cluster.lock);
2184	if (offset)
2185	entry = swp_entry(si->type, offset);
2186
2187	fail:
2188	return entry;
2189	}
2190
2191	/**
2192	* swap_free_hibernation_slot() - Free a swap slot allocated for hibernation.
2193	* @entry: swap entry to free.
2194	*
2195	* The caller must ensure the swap device is stable.
2196	*/
2197	void swap_free_hibernation_slot(swp_entry_t entry)
2198	{
2199	struct swap_info_struct *si = __swap_entry_to_info(entry);
2200	struct swap_cluster_info *ci;
2201	pgoff_t offset = swp_offset(entry);
2202
2203	ci = swap_cluster_lock(si, offset);
2204	__swap_cluster_put_entry(ci, offset % SWAPFILE_CLUSTER);
2205	/*
2206	* A slot with a folio in the swap cache is freed when the folio
2207	* leaves the cache, the same rule swap_put_entries_cluster() follows.
2208	* Readahead can put a folio here, and freeing the slot now would
2209	* leave that folio with no entry behind it.
2210	*/
2211	if (!swp_tb_is_folio(__swap_table_get(ci, offset % SWAPFILE_CLUSTER)))
2212	__swap_cluster_free_entries(si, ci, offset % SWAPFILE_CLUSTER, 1);
2213	swap_cluster_unlock(ci);
2214
2215	/* In theory readahead might add it to the swap cache by accident */
2216	__try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
2217	}
2218
2219	static int __find_hibernation_swap_type(dev_t device, sector_t offset)
2220	{
2221	int type;
2222
2223	lockdep_assert_held(&swap_lock);
2224
2225	if (!device)
2226	return -EINVAL;
2227
2228	for (type = 0; type < nr_swapfiles; type++) {
2229	struct swap_info_struct *sis = swap_info[type];
2230
2231	if (!(sis->flags & SWP_WRITEOK))
2232	continue;
2233
2234	if (device == sis->bdev->bd_dev) {
2235	struct swap_extent *se = first_se(sis);
2236
2237	if (se->start_block == offset)
2238	return type;
2239	}
2240	}
2241	return -ENODEV;
2242	}
2243
2244	/**
2245	* pin_hibernation_swap_type - Pin the swap device for hibernation
2246	* @device: Block device containing the resume image
2247	* @offset: Offset identifying the swap area
2248	*
2249	* Locate the swap device for @device/@offset and mark it as pinned
2250	* for hibernation. While pinned, swapoff() is prevented.
2251	*
2252	* Only one uswsusp context may pin a swap device at a time.
2253	* If already pinned, this function returns -EBUSY.
2254	*
2255	* Return:
2256	* >= 0 on success (swap type).
2257	* -EINVAL if @device is invalid.
2258	* -ENODEV if the swap device is not found.
2259	* -EBUSY if the device is already pinned for hibernation.
2260	*/
2261	int pin_hibernation_swap_type(dev_t device, sector_t offset)
2262	{
2263	int type;
2264	struct swap_info_struct *si;
2265
2266	spin_lock(&swap_lock);
2267
2268	type = __find_hibernation_swap_type(device, offset);
2269	if (type < 0) {
2270	spin_unlock(&swap_lock);
2271	return type;
2272	}
2273
2274	si = swap_type_to_info(type);
2275	if (WARN_ON_ONCE(!si)) {
2276	spin_unlock(&swap_lock);
2277	return -ENODEV;
2278	}
2279
2280	/*
2281	* hibernate_acquire() prevents concurrent hibernation sessions.
2282	* This check additionally guards against double-pinning within
2283	* the same session.
2284	*/
2285	if (WARN_ON_ONCE(si->flags & SWP_HIBERNATION)) {
2286	spin_unlock(&swap_lock);
2287	return -EBUSY;
2288	}
2289
2290	si->flags \|= SWP_HIBERNATION;
2291
2292	spin_unlock(&swap_lock);
2293	return type;
2294	}
2295
2296	/**
2297	* unpin_hibernation_swap_type - Unpin the swap device for hibernation
2298	* @type: Swap type previously returned by pin_hibernation_swap_type()
2299	*
2300	* Clear the hibernation pin on the given swap device, allowing
2301	* swapoff() to proceed normally.
2302	*
2303	* If @type does not refer to a valid swap device, this function
2304	* does nothing.
2305	*/
2306	void unpin_hibernation_swap_type(int type)
2307	{
2308	struct swap_info_struct *si;
2309
2310	spin_lock(&swap_lock);
2311	si = swap_type_to_info(type);
2312	if (!si) {
2313	spin_unlock(&swap_lock);
2314	return;
2315	}
2316	si->flags &= ~SWP_HIBERNATION;
2317	spin_unlock(&swap_lock);
2318	}
2319
2320	/**
2321	* find_hibernation_swap_type - Find swap type for hibernation
2322	* @device: Block device containing the resume image
2323	* @offset: Offset within the device identifying the swap area
2324	*
2325	* Locate the swap device corresponding to @device and @offset.
2326	*
2327	* Unlike pin_hibernation_swap_type(), this function only performs a
2328	* lookup and does not mark the swap device as pinned for hibernation.
2329	*
2330	* This is safe in the sysfs-based hibernation path where user space
2331	* is already frozen and swapoff() cannot run concurrently.
2332	*
2333	* Return:
2334	* A non-negative swap type on success.
2335	* -EINVAL if @device is invalid.
2336	* -ENODEV if no matching swap device is found.
2337	*/
2338	int find_hibernation_swap_type(dev_t device, sector_t offset)
2339	{
2340	int type;
2341
2342	spin_lock(&swap_lock);
2343	type = __find_hibernation_swap_type(device, offset);
2344	spin_unlock(&swap_lock);
2345
2346	return type;
2347	}
2348
2349	int find_first_swap(dev_t *device)
2350	{
2351	int type;
2352
2353	spin_lock(&swap_lock);
2354	for (type = 0; type < nr_swapfiles; type++) {
2355	struct swap_info_struct *sis = swap_info[type];
2356
2357	if (!(sis->flags & SWP_WRITEOK))
2358	continue;
2359	*device = sis->bdev->bd_dev;
2360	spin_unlock(&swap_lock);
2361	return type;
2362	}
2363	spin_unlock(&swap_lock);
2364	return -ENODEV;
2365	}
2366
2367	/*
2368	* Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
2369	* corresponding to given index in swap_info (swap type).
2370	*/
2371	sector_t swapdev_block(int type, pgoff_t offset)
2372	{
2373	struct swap_info_struct *si = swap_type_to_info(type);
2374	struct swap_extent *se;
2375
2376	if (!si \|\| !(si->flags & SWP_WRITEOK))
2377	return 0;
2378	se = offset_to_swap_extent(si, offset);
2379	return se->start_block + (offset - se->start_page);
2380	}
2381
2382	/*
2383	* Return either the total number of swap pages of given type, or the number
2384	* of free pages of that type (depending on @free)
2385	*
2386	* This is needed for software suspend
2387	*/
2388	unsigned int count_swap_pages(int type, int free)
2389	{
2390	unsigned int n = 0;
2391
2392	spin_lock(&swap_lock);
2393	if ((unsigned int)type < nr_swapfiles) {
2394	struct swap_info_struct *sis = swap_info[type];
2395
2396	spin_lock(&sis->lock);
2397	if (sis->flags & SWP_WRITEOK) {
2398	n = sis->pages;
2399	if (free)
2400	n -= swap_usage_in_pages(sis);
2401	}
2402	spin_unlock(&sis->lock);
2403	}
2404	spin_unlock(&swap_lock);
2405	return n;
2406	}
2407	#endif /* CONFIG_HIBERNATION */
2408
2409	static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
2410	{
2411	return pte_same(pte_swp_clear_flags(pte), swp_pte);
2412	}
2413
2414	/*
2415	* No need to decide whether this PTE shares the swap entry with others,
2416	* just let do_wp_page work it out if a write is requested later - to
2417	* force COW, vm_page_prot omits write permission from any private vma.
2418	*/
2419	static int unuse_pte(struct vm_area_struct vma, pmd_t pmd,
2420	unsigned long addr, swp_entry_t entry, struct folio *folio)
2421	{
2422	struct page *page;
2423	struct folio *swapcache;
2424	spinlock_t *ptl;
2425	pte_t *pte, new_pte, old_pte;
2426	bool hwpoisoned = false;
2427	int ret = 1;
2428
2429	/*
2430	* If the folio is removed from swap cache by others, continue to
2431	* unuse other PTEs. try_to_unuse may try again if we missed this one.
2432	*/
2433	if (!folio_matches_swap_entry(folio, entry))
2434	return 0;
2435
2436	swapcache = folio;
2437	folio = ksm_might_need_to_copy(folio, vma, addr);
2438	if (unlikely(!folio))
2439	return -ENOMEM;
2440	else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
2441	hwpoisoned = true;
2442	folio = swapcache;
2443	}
2444
2445	page = folio_file_page(folio, swp_offset(entry));
2446	if (PageHWPoison(page))
2447	hwpoisoned = true;
2448
2449	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
2450	if (unlikely(!pte \|\| !pte_same_as_swp(ptep_get(pte),
2451	swp_entry_to_pte(entry)))) {
2452	ret = 0;
2453	goto out;
2454	}
2455
2456	old_pte = ptep_get(pte);
2457
2458	if (unlikely(hwpoisoned \|\| !folio_test_uptodate(folio))) {
2459	swp_entry_t swp_entry;
2460
2461	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
2462	if (hwpoisoned) {
2463	swp_entry = make_hwpoison_entry(page);
2464	} else {
2465	swp_entry = make_poisoned_swp_entry();
2466	}
2467	new_pte = swp_entry_to_pte(swp_entry);
2468	ret = 0;
2469	goto setpte;
2470	}
2471
2472	/*
2473	* Some architectures may have to restore extra metadata to the page
2474	* when reading from swap. This metadata may be indexed by swap entry
2475	* so this must be called before folio_put_swap().
2476	*/
2477	arch_swap_restore(folio_swap(entry, folio), folio);
2478
2479	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
2480	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
2481	folio_get(folio);
2482	if (folio == swapcache) {
2483	rmap_t rmap_flags = RMAP_NONE;
2484
2485	/*
2486	* See do_swap_page(): writeback would be problematic.
2487	* However, we do a folio_wait_writeback() just before this
2488	* call and have the folio locked.
2489	*/
2490	VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
2491	if (pte_swp_exclusive(old_pte))
2492	rmap_flags \|= RMAP_EXCLUSIVE;
2493	/*
2494	* We currently only expect small !anon folios, which are either
2495	* fully exclusive or fully shared. If we ever get large folios
2496	* here, we have to be careful.
2497	*/
2498	if (!folio_test_anon(folio)) {
2499	VM_WARN_ON_ONCE(folio_test_large(folio));
2500	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
2501	folio_add_new_anon_rmap(folio, vma, addr, rmap_flags);
2502	} else {
2503	folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags);
2504	}
2505	} else { /* ksm created a completely new copy */
2506	folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
2507	folio_add_lru_vma(folio, vma);
2508	}
2509	new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot));
2510	if (pte_swp_soft_dirty(old_pte))
2511	new_pte = pte_mksoft_dirty(new_pte);
2512	if (pte_swp_uffd(old_pte))
2513	new_pte = pte_mkuffd(new_pte);
2514
2515	/* See do_swap_page(): restore PAGE_NONE for RWP */
2516	if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
2517	new_pte = pte_modify(new_pte, PAGE_NONE);
2518
2519	setpte:
2520	set_pte_at(vma->vm_mm, addr, pte, new_pte);
2521	folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry)));
2522	out:
2523	if (pte)
2524	pte_unmap_unlock(pte, ptl);
2525	if (folio != swapcache) {
2526	folio_unlock(folio);
2527	folio_put(folio);
2528	}
2529	return ret;
2530	}
2531
2532	static int unuse_pte_range(struct vm_area_struct vma, pmd_t pmd,
2533	unsigned long addr, unsigned long end,
2534	unsigned int type)
2535	{
2536	pte_t *pte = NULL;
2537
2538	do {
2539	struct folio *folio;
2540	unsigned long swp_tb;
2541	softleaf_t entry;
2542	int ret;
2543	pte_t ptent;
2544
2545	if (!pte++) {
2546	pte = pte_offset_map(pmd, addr);
2547	if (!pte)
2548	break;
2549	}
2550
2551	ptent = ptep_get_lockless(pte);
2552	entry = softleaf_from_pte(ptent);
2553
2554	if (!softleaf_is_swap(entry))
2555	continue;
2556	if (swp_type(entry) != type)
2557	continue;
2558
2559	pte_unmap(pte);
2560	pte = NULL;
2561
2562	folio = swap_cache_get_folio(entry);
2563	if (!folio) {
2564	struct vm_fault vmf = {
2565	.vma = vma,
2566	.address = addr,
2567	.real_address = addr,
2568	.pmd = pmd,
2569	};
2570
2571	folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
2572	&vmf);
2573	}
2574	if (!folio) {
2575	swp_tb = swap_table_get(__swap_entry_to_cluster(entry),
2576	swp_cluster_offset(entry));
2577	if (swp_tb_get_count(swp_tb) <= 0)
2578	continue;
2579	return -ENOMEM;
2580	}
2581
2582	folio_lock(folio);
2583	folio_wait_writeback(folio);
2584	ret = unuse_pte(vma, pmd, addr, entry, folio);
2585	if (ret < 0) {
2586	folio_unlock(folio);
2587	folio_put(folio);
2588	return ret;
2589	}
2590
2591	folio_free_swap(folio);
2592	folio_unlock(folio);
2593	folio_put(folio);
2594	} while (addr += PAGE_SIZE, addr != end);
2595
2596	if (pte)
2597	pte_unmap(pte);
2598	return 0;
2599	}
2600
2601	static inline int unuse_pmd_range(struct vm_area_struct vma, pud_t pud,
2602	unsigned long addr, unsigned long end,
2603	unsigned int type)
2604	{
2605	pmd_t *pmd;
2606	unsigned long next;
2607	int ret;
2608
2609	pmd = pmd_offset(pud, addr);
2610	do {
2611	cond_resched();
2612	next = pmd_addr_end(addr, end);
2613	ret = unuse_pte_range(vma, pmd, addr, next, type);
2614	if (ret)
2615	return ret;
2616	} while (pmd++, addr = next, addr != end);
2617	return 0;
2618	}
2619
2620	static inline int unuse_pud_range(struct vm_area_struct vma, p4d_t p4d,
2621	unsigned long addr, unsigned long end,
2622	unsigned int type)
2623	{
2624	pud_t *pud;
2625	unsigned long next;
2626	int ret;
2627
2628	pud = pud_offset(p4d, addr);
2629	do {
2630	next = pud_addr_end(addr, end);
2631	if (pud_none_or_clear_bad(pud))
2632	continue;
2633	ret = unuse_pmd_range(vma, pud, addr, next, type);
2634	if (ret)
2635	return ret;
2636	} while (pud++, addr = next, addr != end);
2637	return 0;
2638	}
2639
2640	static inline int unuse_p4d_range(struct vm_area_struct vma, pgd_t pgd,
2641	unsigned long addr, unsigned long end,
2642	unsigned int type)
2643	{
2644	p4d_t *p4d;
2645	unsigned long next;
2646	int ret;
2647
2648	p4d = p4d_offset(pgd, addr);
2649	do {
2650	next = p4d_addr_end(addr, end);
2651	if (p4d_none_or_clear_bad(p4d))
2652	continue;
2653	ret = unuse_pud_range(vma, p4d, addr, next, type);
2654	if (ret)
2655	return ret;
2656	} while (p4d++, addr = next, addr != end);
2657	return 0;
2658	}
2659
2660	static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
2661	{
2662	pgd_t *pgd;
2663	unsigned long addr, end, next;
2664	int ret;
2665
2666	addr = vma->vm_start;
2667	end = vma->vm_end;
2668
2669	pgd = pgd_offset(vma->vm_mm, addr);
2670	do {
2671	next = pgd_addr_end(addr, end);
2672	if (pgd_none_or_clear_bad(pgd))
2673	continue;
2674	ret = unuse_p4d_range(vma, pgd, addr, next, type);
2675	if (ret)
2676	return ret;
2677	} while (pgd++, addr = next, addr != end);
2678	return 0;
2679	}
2680
2681	static int unuse_mm(struct mm_struct *mm, unsigned int type)
2682	{
2683	struct vm_area_struct *vma;
2684	int ret = 0;
2685	VMA_ITERATOR(vmi, mm, 0);
2686
2687	mmap_read_lock(mm);
2688	if (check_stable_address_space(mm))
2689	goto unlock;
2690	for_each_vma(vmi, vma) {
2691	if (vma->anon_vma && !is_vm_hugetlb_page(vma)) {
2692	ret = unuse_vma(vma, type);
2693	if (ret)
2694	break;
2695	}
2696
2697	cond_resched();
2698	}
2699	unlock:
2700	mmap_read_unlock(mm);
2701	return ret;
2702	}
2703
2704	/*
2705	* Scan swap table from current position to next entry still in use.
2706	* Return 0 if there are no inuse entries after prev till end of
2707	* the map.
2708	*/
2709	static unsigned int find_next_to_unuse(struct swap_info_struct *si,
2710	unsigned int prev)
2711	{
2712	unsigned int i;
2713	unsigned long swp_tb;
2714
2715	/*
2716	* No need for swap_lock here: we're just looking
2717	* for whether an entry is in use, not modifying it; false
2718	* hits are okay, and sys_swapoff() has already prevented new
2719	* allocations from this area (while holding swap_lock).
2720	*/
2721	for (i = prev + 1; i < si->max; i++) {
2722	swp_tb = swap_table_get(__swap_offset_to_cluster(si, i),
2723	i % SWAPFILE_CLUSTER);
2724	if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb))
2725	break;
2726	if ((i % LATENCY_LIMIT) == 0)
2727	cond_resched();
2728	}
2729
2730	if (i == si->max)
2731	i = 0;
2732
2733	return i;
2734	}
2735
2736	static int try_to_unuse(unsigned int type)
2737	{
2738	struct mm_struct *prev_mm;
2739	struct mm_struct *mm;
2740	struct list_head *p;
2741	int retval = 0;
2742	struct swap_info_struct *si = swap_info[type];
2743	struct folio *folio;
2744	swp_entry_t entry;
2745	unsigned int i;
2746
2747	if (!swap_usage_in_pages(si))
2748	goto success;
2749
2750	retry:
2751	retval = shmem_unuse(type);
2752	if (retval)
2753	return retval;
2754
2755	prev_mm = &init_mm;
2756	mmget(prev_mm);
2757
2758	spin_lock(&mmlist_lock);
2759	p = &init_mm.mmlist;
2760	while (swap_usage_in_pages(si) &&
2761	!signal_pending(current) &&
2762	(p = p->next) != &init_mm.mmlist) {
2763
2764	mm = list_entry(p, struct mm_struct, mmlist);
2765	if (!mmget_not_zero(mm))
2766	continue;
2767	spin_unlock(&mmlist_lock);
2768	mmput(prev_mm);
2769	prev_mm = mm;
2770	retval = unuse_mm(mm, type);
2771	if (retval) {
2772	mmput(prev_mm);
2773	return retval;
2774	}
2775
2776	/*
2777	* Make sure that we aren't completely killing
2778	* interactive performance.
2779	*/
2780	cond_resched();
2781	spin_lock(&mmlist_lock);
2782	}
2783	spin_unlock(&mmlist_lock);
2784
2785	mmput(prev_mm);
2786
2787	i = 0;
2788	while (swap_usage_in_pages(si) &&
2789	!signal_pending(current) &&
2790	(i = find_next_to_unuse(si, i)) != 0) {
2791
2792	entry = swp_entry(type, i);
2793	folio = swap_cache_get_folio(entry);
2794	if (!folio)
2795	continue;
2796
2797	/*
2798	* It is conceivable that a racing task removed this folio from
2799	* swap cache just before we acquired the page lock. The folio
2800	* might even be back in swap cache on another swap area. But
2801	* that is okay, folio_free_swap() only removes stale folios.
2802	*/
2803	folio_lock(folio);
2804	folio_wait_writeback(folio);
2805	folio_free_swap(folio);
2806	folio_unlock(folio);
2807	folio_put(folio);
2808	}
2809
2810	/*
2811	* Lets check again to see if there are still swap entries in the map.
2812	* If yes, we would need to do retry the unuse logic again.
2813	* Under global memory pressure, swap entries can be reinserted back
2814	* into process space after the mmlist loop above passes over them.
2815	*
2816	* Limit the number of retries? No: when mmget_not_zero()
2817	* above fails, that mm is likely to be freeing swap from
2818	* exit_mmap(), which proceeds at its own independent pace;
2819	* and even shmem_writeout() could have been preempted after
2820	* folio_alloc_swap(), temporarily hiding that swap. It's easy
2821	* and robust (though cpu-intensive) just to keep retrying.
2822	*/
2823	if (swap_usage_in_pages(si)) {
2824	if (!signal_pending(current))
2825	goto retry;
2826	return -EINTR;
2827	}
2828
2829	success:
2830	/*
2831	* Make sure that further cleanups after try_to_unuse() returns happen
2832	* after swap_range_free() reduces si->inuse_pages to 0.
2833	*/
2834	smp_mb();
2835	return 0;
2836	}
2837
2838	/*
2839	* After a successful try_to_unuse, if no swap is now in use, we know
2840	* we can empty the mmlist. swap_lock must be held on entry and exit.
2841	* Note that mmlist_lock nests inside swap_lock, and an mm must be
2842	* added to the mmlist just after page_duplicate - before would be racy.
2843	*/
2844	static void drain_mmlist(void)
2845	{
2846	struct list_head p, next;
2847	unsigned int type;
2848
2849	for (type = 0; type < nr_swapfiles; type++)
2850	if (swap_usage_in_pages(swap_info[type]))
2851	return;
2852	spin_lock(&mmlist_lock);
2853	list_for_each_safe(p, next, &init_mm.mmlist)
2854	list_del_init(p);
2855	spin_unlock(&mmlist_lock);
2856	}
2857
2858	/*
2859	* Free all of a swapdev's extent information
2860	*/
2861	static void destroy_swap_extents(struct swap_info_struct *sis,
2862	struct file *swap_file)
2863	{
2864	while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) {
2865	struct rb_node *rb = sis->swap_extent_root.rb_node;
2866	struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node);
2867
2868	rb_erase(rb, &sis->swap_extent_root);
2869	kfree(se);
2870	}
2871
2872	if (sis->flags & SWP_ACTIVATED) {
2873	struct address_space *mapping = swap_file->f_mapping;
2874
2875	sis->flags &= ~SWP_ACTIVATED;
2876	if (mapping->a_ops->swap_deactivate)
2877	mapping->a_ops->swap_deactivate(swap_file);
2878	}
2879	}
2880
2881	/*
2882	* Add a block range (and the corresponding page range) into this swapdev's
2883	* extent tree.
2884	*
2885	* This function rather assumes that it is called in ascending page order.
2886	*/
2887	int
2888	add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
2889	unsigned long nr_pages, sector_t start_block)
2890	{
2891	struct rb_node *link = &sis->swap_extent_root.rb_node, parent = NULL;
2892	struct swap_extent *se;
2893	struct swap_extent *new_se;
2894
2895	/*
2896	* place the new node at the right most since the
2897	* function is called in ascending page order.
2898	*/
2899	while (*link) {
2900	parent = *link;
2901	link = &parent->rb_right;
2902	}
2903
2904	if (parent) {
2905	se = rb_entry(parent, struct swap_extent, rb_node);
2906	BUG_ON(se->start_page + se->nr_pages != start_page);
2907	if (se->start_block + se->nr_pages == start_block) {
2908	/* Merge it */
2909	se->nr_pages += nr_pages;
2910	return 0;
2911	}
2912	}
2913
2914	/* No merge, insert a new extent. */
2915	new_se = kmalloc_obj(*se);
2916	if (new_se == NULL)
2917	return -ENOMEM;
2918	new_se->start_page = start_page;
2919	new_se->nr_pages = nr_pages;
2920	new_se->start_block = start_block;
2921
2922	rb_link_node(&new_se->rb_node, parent, link);
2923	rb_insert_color(&new_se->rb_node, &sis->swap_extent_root);
2924	return 1;
2925	}
2926	EXPORT_SYMBOL_GPL(add_swap_extent);
2927
2928	/*
2929	* A `swap extent' is a simple thing which maps a contiguous range of pages
2930	* onto a contiguous range of disk blocks. A rbtree of swap extents is
2931	* built at swapon time and is then used at swap_writepage/swap_read_folio
2932	* time for locating where on disk a page belongs.
2933	*
2934	* If the swapfile is an S_ISBLK block device, a single extent is installed.
2935	* This is done so that the main operating code can treat S_ISBLK and S_ISREG
2936	* swap files identically.
2937	*
2938	* Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2939	* extent rbtree operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
2940	* swapfiles are handled identically after swapon time.
2941	*
2942	* For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
2943	* and will parse them into a rbtree, in PAGE_SIZE chunks. If some stray
2944	* blocks are found which do not fall within the PAGE_SIZE alignment
2945	* requirements, they are simply tossed out - we will never use those blocks
2946	* for swapping.
2947	*
2948	* For all swap devices we set S_SWAPFILE across the life of the swapon. This
2949	* prevents users from writing to the swap device, which will corrupt memory.
2950	*
2951	* The amount of disk space which a single swap extent represents varies.
2952	* Typically it is in the 1-4 megabyte range. So we can have hundreds of
2953	* extents in the rbtree. - akpm.
2954	*/
2955	static int setup_swap_extents(struct swap_info_struct *sis,
2956	struct file swap_file, sector_t span)
2957	{
2958	struct address_space *mapping = swap_file->f_mapping;
2959	struct inode *inode = mapping->host;
2960	int ret;
2961
2962	ret = sio_pool_init();
2963	if (ret)
2964	return ret;
2965
2966	sis->ops = &swap_bdev_ops;
2967
2968	if (S_ISBLK(inode->i_mode)) {
2969	ret = add_swap_extent(sis, 0, sis->max, 0);
2970	*span = sis->pages;
2971	return ret;
2972	}
2973
2974	if (mapping->a_ops->swap_activate) {
2975	ret = mapping->a_ops->swap_activate(sis, swap_file, span);
2976	if (ret < 0)
2977	return ret;
2978	sis->flags \|= SWP_ACTIVATED;
2979	return ret;
2980	}
2981
2982	return generic_swapfile_activate(sis, swap_file, span);
2983	}
2984
2985	static void _enable_swap_info(struct swap_info_struct *si)
2986	{
2987	atomic_long_add(si->pages, &nr_swap_pages);
2988	total_swap_pages += si->pages;
2989
2990	assert_spin_locked(&swap_lock);
2991
2992	plist_add(&si->list, &swap_active_head);
2993
2994	/* Add back to available list */
2995	add_to_avail_list(si, true);
2996	}
2997
2998	/*
2999	* Called after the swap device is ready, resurrect its percpu ref, it's now
3000	* safe to reference it. Add it to the list to expose it to the allocator.
3001	*/
3002	static void enable_swap_info(struct swap_info_struct *si)
3003	{
3004	percpu_ref_resurrect(&si->users);
3005	spin_lock(&swap_lock);
3006	spin_lock(&si->lock);
3007	_enable_swap_info(si);
3008	spin_unlock(&si->lock);
3009	spin_unlock(&swap_lock);
3010	}
3011
3012	static void reinsert_swap_info(struct swap_info_struct *si)
3013	{
3014	spin_lock(&swap_lock);
3015	spin_lock(&si->lock);
3016	_enable_swap_info(si);
3017	spin_unlock(&si->lock);
3018	spin_unlock(&swap_lock);
3019	}
3020
3021	/*
3022	* Called after clearing SWP_WRITEOK, ensures cluster_alloc_range
3023	* see the updated flags, so there will be no more allocations.
3024	*/
3025	static void wait_for_allocation(struct swap_info_struct *si)
3026	{
3027	unsigned long offset;
3028	unsigned long end = ALIGN(si->max, SWAPFILE_CLUSTER);
3029	struct swap_cluster_info *ci;
3030
3031	BUG_ON(si->flags & SWP_WRITEOK);
3032
3033	for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
3034	ci = swap_cluster_lock(si, offset);
3035	swap_cluster_unlock(ci);
3036	}
3037	}
3038
3039	static void free_swap_cluster_info(struct swap_cluster_info *cluster_info,
3040	unsigned long maxpages)
3041	{
3042	struct swap_cluster_info *ci;
3043	int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3044
3045	if (!cluster_info)
3046	return;
3047	for (i = 0; i < nr_clusters; i++) {
3048	ci = cluster_info + i;
3049	/* Cluster with bad marks count will have a remaining table */
3050	spin_lock(&ci->lock);
3051	if (cluster_table_is_alloced(ci)) {
3052	swap_cluster_assert_empty(ci, 0, SWAPFILE_CLUSTER, true);
3053	swap_cluster_free_table(ci);
3054	}
3055	spin_unlock(&ci->lock);
3056	}
3057	kvfree(cluster_info);
3058	}
3059
3060	/*
3061	* Called after swap device's reference count is dead, so
3062	* neither scan nor allocation will use it.
3063	*/
3064	static void flush_percpu_swap_cluster(struct swap_info_struct *si)
3065	{
3066	int cpu, i;
3067	struct swap_info_struct **pcp_si;
3068
3069	for_each_possible_cpu(cpu) {
3070	pcp_si = per_cpu_ptr(percpu_swap_cluster.si, cpu);
3071	/*
3072	* Invalidate the percpu swap cluster cache, si->users
3073	* is dead, so no new user will point to it, just flush
3074	* any existing user.
3075	*/
3076	for (i = 0; i < SWAP_NR_ORDERS; i++)
3077	cmpxchg(&pcp_si[i], si, NULL);
3078	}
3079	}
3080
3081
3082	SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
3083	{
3084	struct swap_info_struct *p = NULL;
3085	struct swap_cluster_info *cluster_info;
3086	struct file swap_file, victim;
3087	struct address_space *mapping;
3088	struct inode *inode;
3089	unsigned int maxpages;
3090	int err, found = 0;
3091
3092	if (!capable(CAP_SYS_ADMIN))
3093	return -EPERM;
3094
3095	BUG_ON(!current->mm);
3096
3097	CLASS(filename, pathname)(specialfile);
3098	victim = file_open_name(pathname, O_RDWR\|O_LARGEFILE, 0);
3099	if (IS_ERR(victim))
3100	return PTR_ERR(victim);
3101
3102	mapping = victim->f_mapping;
3103	spin_lock(&swap_lock);
3104	plist_for_each_entry(p, &swap_active_head, list) {
3105	if (p->flags & SWP_WRITEOK) {
3106	if (p->swap_file->f_mapping == mapping) {
3107	found = 1;
3108	break;
3109	}
3110	}
3111	}
3112	if (!found) {
3113	err = -EINVAL;
3114	spin_unlock(&swap_lock);
3115	goto out_dput;
3116	}
3117
3118	/* Refuse swapoff while the device is pinned for hibernation */
3119	if (p->flags & SWP_HIBERNATION) {
3120	err = -EBUSY;
3121	spin_unlock(&swap_lock);
3122	goto out_dput;
3123	}
3124
3125	if (!security_vm_enough_memory_mm(current->mm, p->pages))
3126	vm_unacct_memory(p->pages);
3127	else {
3128	err = -ENOMEM;
3129	spin_unlock(&swap_lock);
3130	goto out_dput;
3131	}
3132	spin_lock(&p->lock);
3133	del_from_avail_list(p, true);
3134	plist_del(&p->list, &swap_active_head);
3135	atomic_long_sub(p->pages, &nr_swap_pages);
3136	total_swap_pages -= p->pages;
3137	spin_unlock(&p->lock);
3138	spin_unlock(&swap_lock);
3139
3140	wait_for_allocation(p);
3141
3142	set_current_oom_origin();
3143	err = try_to_unuse(p->type);
3144	clear_current_oom_origin();
3145
3146	if (err) {
3147	/* re-insert swap space back into swap_list */
3148	reinsert_swap_info(p);
3149	goto out_dput;
3150	}
3151
3152	/*
3153	* Wait for swap operations protected by get/put_swap_device()
3154	* to complete. Because of synchronize_rcu() here, all swap
3155	* operations protected by RCU reader side lock (including any
3156	* spinlock) will be waited too. This makes it easy to
3157	* prevent folio_test_swapcache() and the following swap cache
3158	* operations from racing with swapoff.
3159	*/
3160	percpu_ref_kill(&p->users);
3161	synchronize_rcu();
3162	wait_for_completion(&p->comp);
3163
3164	flush_work(&p->discard_work);
3165	flush_work(&p->reclaim_work);
3166	flush_percpu_swap_cluster(p);
3167
3168	destroy_swap_extents(p, p->swap_file);
3169
3170	if (!(p->flags & SWP_SOLIDSTATE))
3171	atomic_dec(&nr_rotate_swap);
3172
3173	mutex_lock(&swapon_mutex);
3174	spin_lock(&swap_lock);
3175	spin_lock(&p->lock);
3176	drain_mmlist();
3177
3178	swap_file = p->swap_file;
3179	p->swap_file = NULL;
3180	maxpages = p->max;
3181	cluster_info = p->cluster_info;
3182	p->max = 0;
3183	p->cluster_info = NULL;
3184	spin_unlock(&p->lock);
3185	spin_unlock(&swap_lock);
3186	arch_swap_invalidate_area(p->type);
3187	zswap_swapoff(p->type);
3188	mutex_unlock(&swapon_mutex);
3189	kfree(p->global_cluster);
3190	p->global_cluster = NULL;
3191	free_swap_cluster_info(cluster_info, maxpages);
3192
3193	inode = mapping->host;
3194
3195	inode_lock(inode);
3196	inode->i_flags &= ~S_SWAPFILE;
3197	inode_unlock(inode);
3198	filp_close(swap_file, NULL);
3199
3200	/*
3201	* Clear the SWP_USED flag after all resources are freed so that swapon
3202	* can reuse this swap_info in alloc_swap_info() safely. It is ok to
3203	* not hold p->lock after we cleared its SWP_WRITEOK.
3204	*/
3205	spin_lock(&swap_lock);
3206	p->flags = 0;
3207	spin_unlock(&swap_lock);
3208
3209	err = 0;
3210	atomic_inc(&proc_poll_event);
3211	wake_up_interruptible(&proc_poll_wait);
3212
3213	out_dput:
3214	filp_close(victim, NULL);
3215	return err;
3216	}
3217
3218	#ifdef CONFIG_PROC_FS
3219	static __poll_t swaps_poll(struct file file, poll_table wait)
3220	{
3221	struct seq_file *seq = file->private_data;
3222
3223	poll_wait(file, &proc_poll_wait, wait);
3224
3225	if (seq->poll_event != atomic_read(&proc_poll_event)) {
3226	seq->poll_event = atomic_read(&proc_poll_event);
3227	return EPOLLIN \| EPOLLRDNORM \| EPOLLERR \| EPOLLPRI;
3228	}
3229
3230	return EPOLLIN \| EPOLLRDNORM;
3231	}
3232
3233	/* iterator */
3234	static void swap_start(struct seq_file swap, loff_t *pos)
3235	{
3236	struct swap_info_struct *si;
3237	int type;
3238	loff_t l = *pos;
3239
3240	mutex_lock(&swapon_mutex);
3241
3242	if (!l)
3243	return SEQ_START_TOKEN;
3244
3245	for (type = 0; (si = swap_type_to_info(type)); type++) {
3246	if (!(si->swap_file))
3247	continue;
3248	if (!--l)
3249	return si;
3250	}
3251
3252	return NULL;
3253	}
3254
3255	static void swap_next(struct seq_file swap, void v, loff_t pos)
3256	{
3257	struct swap_info_struct *si = v;
3258	int type;
3259
3260	if (v == SEQ_START_TOKEN)
3261	type = 0;
3262	else
3263	type = si->type + 1;
3264
3265	++(*pos);
3266	for (; (si = swap_type_to_info(type)); type++) {
3267	if (!(si->swap_file))
3268	continue;
3269	return si;
3270	}
3271
3272	return NULL;
3273	}
3274
3275	static void swap_stop(struct seq_file swap, void v)
3276	{
3277	mutex_unlock(&swapon_mutex);
3278	}
3279
3280	static int swap_show(struct seq_file swap, void v)
3281	{
3282	struct swap_info_struct *si = v;
3283	struct file *file;
3284	int len;
3285	unsigned long bytes, inuse;
3286
3287	if (si == SEQ_START_TOKEN) {
3288	seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");
3289	return 0;
3290	}
3291
3292	bytes = K(si->pages);
3293	inuse = K(swap_usage_in_pages(si));
3294
3295	file = si->swap_file;
3296	len = seq_file_path(swap, file, " \t\n\\");
3297	seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n",
3298	len < 40 ? 40 - len : 1, " ",
3299	S_ISBLK(file_inode(file)->i_mode) ?
3300	"partition" : "file\t",
3301	bytes, bytes < 10000000 ? "\t" : "",
3302	inuse, inuse < 10000000 ? "\t" : "",
3303	si->prio);
3304	return 0;
3305	}
3306
3307	static const struct seq_operations swaps_op = {
3308	.start = swap_start,
3309	.next = swap_next,
3310	.stop = swap_stop,
3311	.show = swap_show
3312	};
3313
3314	static int swaps_open(struct inode inode, struct file file)
3315	{
3316	struct seq_file *seq;
3317	int ret;
3318
3319	ret = seq_open(file, &swaps_op);
3320	if (ret)
3321	return ret;
3322
3323	seq = file->private_data;
3324	seq->poll_event = atomic_read(&proc_poll_event);
3325	return 0;
3326	}
3327
3328	static const struct proc_ops swaps_proc_ops = {
3329	.proc_flags = PROC_ENTRY_PERMANENT,
3330	.proc_open = swaps_open,
3331	.proc_read = seq_read,
3332	.proc_lseek = seq_lseek,
3333	.proc_release = seq_release,
3334	.proc_poll = swaps_poll,
3335	};
3336
3337	static int __init procswaps_init(void)
3338	{
3339	proc_create("swaps", 0, NULL, &swaps_proc_ops);
3340	return 0;
3341	}
3342	__initcall(procswaps_init);
3343	#endif /* CONFIG_PROC_FS */
3344
3345	#ifdef MAX_SWAPFILES_CHECK
3346	static int __init max_swapfiles_check(void)
3347	{
3348	MAX_SWAPFILES_CHECK();
3349	return 0;
3350	}
3351	late_initcall(max_swapfiles_check);
3352	#endif
3353
3354	static struct swap_info_struct *alloc_swap_info(void)
3355	{
3356	struct swap_info_struct *p;
3357	struct swap_info_struct *defer = NULL;
3358	unsigned int type;
3359
3360	p = kvzalloc_obj(struct swap_info_struct);
3361	if (!p)
3362	return ERR_PTR(-ENOMEM);
3363
3364	if (percpu_ref_init(&p->users, swap_users_ref_free,
3365	PERCPU_REF_INIT_DEAD, GFP_KERNEL)) {
3366	kvfree(p);
3367	return ERR_PTR(-ENOMEM);
3368	}
3369
3370	spin_lock(&swap_lock);
3371	for (type = 0; type < nr_swapfiles; type++) {
3372	if (!(swap_info[type]->flags & SWP_USED))
3373	break;
3374	}
3375	if (type >= MAX_SWAPFILES) {
3376	spin_unlock(&swap_lock);
3377	percpu_ref_exit(&p->users);
3378	kvfree(p);
3379	return ERR_PTR(-EPERM);
3380	}
3381	if (type >= nr_swapfiles) {
3382	p->type = type;
3383	/*
3384	* Publish the swap_info_struct after initializing it.
3385	* Note that kvzalloc() above zeroes all its fields.
3386	*/
3387	smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */
3388	nr_swapfiles++;
3389	} else {
3390	defer = p;
3391	p = swap_info[type];
3392	/*
3393	* Do not memset this entry: a racing procfs swap_next()
3394	* would be relying on p->type to remain valid.
3395	*/
3396	}
3397	p->swap_extent_root = RB_ROOT;
3398	plist_node_init(&p->list, 0);
3399	plist_node_init(&p->avail_list, 0);
3400	p->flags = SWP_USED;
3401	spin_unlock(&swap_lock);
3402	if (defer) {
3403	percpu_ref_exit(&defer->users);
3404	kvfree(defer);
3405	}
3406	spin_lock_init(&p->lock);
3407	atomic_long_set(&p->inuse_pages, SWAP_USAGE_OFFLIST_BIT);
3408	init_completion(&p->comp);
3409
3410	return p;
3411	}
3412
3413	static int claim_swapfile(struct swap_info_struct si, struct inode inode)
3414	{
3415	if (S_ISBLK(inode->i_mode)) {
3416	si->bdev = I_BDEV(inode);
3417	/*
3418	* Zoned block devices contain zones that have a sequential
3419	* write only restriction. Hence zoned block devices are not
3420	* suitable for swapping. Disallow them here.
3421	*/
3422	if (bdev_is_zoned(si->bdev))
3423	return -EINVAL;
3424	si->flags \|= SWP_BLKDEV;
3425	} else if (S_ISREG(inode->i_mode)) {
3426	si->bdev = inode->i_sb->s_bdev;
3427	}
3428
3429	return 0;
3430	}
3431
3432
3433	/*
3434	* Find out how many pages are allowed for a single swap device. There
3435	* are two limiting factors:
3436	* 1) the number of bits for the swap offset in the swp_entry_t type, and
3437	* 2) the number of bits in the swap pte, as defined by the different
3438	* architectures.
3439	*
3440	* In order to find the largest possible bit mask, a swap entry with
3441	* swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
3442	* decoded to a swp_entry_t again, and finally the swap offset is
3443	* extracted.
3444	*
3445	* This will mask all the bits from the initial ~0UL mask that can't
3446	* be encoded in either the swp_entry_t or the architecture definition
3447	* of a swap pte.
3448	*/
3449	unsigned long generic_max_swapfile_size(void)
3450	{
3451	swp_entry_t entry = swp_entry(0, ~0UL);
3452	const pte_t pte = softleaf_to_pte(entry);
3453
3454	/*
3455	* Since the PTE can be an invalid softleaf entry (e.g. the none PTE),
3456	* we need to do this manually.
3457	*/
3458	entry = __pte_to_swp_entry(pte);
3459	entry = swp_entry(__swp_type(entry), __swp_offset(entry));
3460
3461	return swp_offset(entry) + 1;
3462	}
3463
3464	/* Can be overridden by an architecture for additional checks. */
3465	__weak unsigned long arch_max_swapfile_size(void)
3466	{
3467	return generic_max_swapfile_size();
3468	}
3469
3470	static unsigned long read_swap_header(struct swap_info_struct *si,
3471	union swap_header *swap_header,
3472	struct inode *inode)
3473	{
3474	int i;
3475	unsigned long maxpages;
3476	unsigned long swapfilepages;
3477	unsigned long last_page;
3478
3479	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
3480	pr_err("Unable to find swap-space signature\n");
3481	return 0;
3482	}
3483
3484	/* swap partition endianness hack... */
3485	if (swab32(swap_header->info.version) == 1) {
3486	swab32s(&swap_header->info.version);
3487	swab32s(&swap_header->info.last_page);
3488	swab32s(&swap_header->info.nr_badpages);
3489	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3490	return 0;
3491	for (i = 0; i < swap_header->info.nr_badpages; i++)
3492	swab32s(&swap_header->info.badpages[i]);
3493	}
3494	/* Check the swap header's sub-version */
3495	if (swap_header->info.version != 1) {
3496	pr_warn("Unable to handle swap header version %d\n",
3497	swap_header->info.version);
3498	return 0;
3499	}
3500
3501	maxpages = swapfile_maximum_size;
3502	last_page = swap_header->info.last_page;
3503	if (!last_page) {
3504	pr_warn("Empty swap-file\n");
3505	return 0;
3506	}
3507	if (last_page > maxpages) {
3508	pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
3509	K(maxpages), K(last_page));
3510	}
3511	if (maxpages > last_page) {
3512	maxpages = last_page + 1;
3513	/* p->max is an unsigned int: don't overflow it */
3514	if ((unsigned int)maxpages == 0)
3515	maxpages = UINT_MAX;
3516	}
3517
3518	if (!maxpages)
3519	return 0;
3520	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
3521	if (swapfilepages && maxpages > swapfilepages) {
3522	pr_warn("Swap area shorter than signature indicates\n");
3523	return 0;
3524	}
3525	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
3526	return 0;
3527	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3528	return 0;
3529
3530	return maxpages;
3531	}
3532
3533	static int setup_swap_clusters_info(struct swap_info_struct *si,
3534	union swap_header *swap_header,
3535	unsigned long maxpages)
3536	{
3537	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3538	struct swap_cluster_info *cluster_info;
3539	int err = -ENOMEM;
3540	unsigned long i;
3541
3542	cluster_info = kvzalloc_objs(*cluster_info, nr_clusters);
3543	if (!cluster_info)
3544	goto err;
3545
3546	for (i = 0; i < nr_clusters; i++)
3547	spin_lock_init(&cluster_info[i].lock);
3548
3549	if (!(si->flags & SWP_SOLIDSTATE)) {
3550	si->global_cluster = kmalloc_obj(*si->global_cluster);
3551	if (!si->global_cluster)
3552	goto err;
3553	for (i = 0; i < SWAP_NR_ORDERS; i++)
3554	si->global_cluster->next[i] = SWAP_ENTRY_INVALID;
3555	spin_lock_init(&si->global_cluster_lock);
3556	}
3557
3558	/*
3559	* Mark unusable pages (header page, bad pages, and the EOF part of
3560	* the last cluster) as unavailable. The clusters aren't marked free
3561	* yet, so no list operations are involved yet.
3562	*/
3563	err = swap_cluster_setup_bad_slot(si, cluster_info, 0, false);
3564	if (err)
3565	goto err;
3566	for (i = 0; i < swap_header->info.nr_badpages; i++) {
3567	unsigned int page_nr = swap_header->info.badpages[i];
3568
3569	if (!page_nr \|\| page_nr > swap_header->info.last_page) {
3570	pr_warn("Bad slot offset is out of border: %d (last_page: %d)\n",
3571	page_nr, swap_header->info.last_page);
3572	err = -EINVAL;
3573	goto err;
3574	}
3575	err = swap_cluster_setup_bad_slot(si, cluster_info, page_nr, false);
3576	if (err)
3577	goto err;
3578	}
3579	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) {
3580	err = swap_cluster_setup_bad_slot(si, cluster_info, i, true);
3581	if (err)
3582	goto err;
3583	}
3584
3585	INIT_LIST_HEAD(&si->free_clusters);
3586	INIT_LIST_HEAD(&si->full_clusters);
3587	INIT_LIST_HEAD(&si->discard_clusters);
3588
3589	for (i = 0; i < SWAP_NR_ORDERS; i++) {
3590	INIT_LIST_HEAD(&si->nonfull_clusters[i]);
3591	INIT_LIST_HEAD(&si->frag_clusters[i]);
3592	}
3593
3594	for (i = 0; i < nr_clusters; i++) {
3595	struct swap_cluster_info *ci = &cluster_info[i];
3596
3597	if (ci->count) {
3598	ci->flags = CLUSTER_FLAG_NONFULL;
3599	list_add_tail(&ci->list, &si->nonfull_clusters[0]);
3600	} else {
3601	ci->flags = CLUSTER_FLAG_FREE;
3602	list_add_tail(&ci->list, &si->free_clusters);
3603	}
3604	}
3605
3606	si->cluster_info = cluster_info;
3607	return 0;
3608	err:
3609	free_swap_cluster_info(cluster_info, maxpages);
3610	return err;
3611	}
3612
3613	SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
3614	{
3615	struct swap_info_struct *si;
3616	struct file *swap_file = NULL;
3617	struct address_space *mapping;
3618	struct dentry *dentry;
3619	int prio;
3620	int error;
3621	union swap_header *swap_header;
3622	int nr_extents;
3623	sector_t span;
3624	unsigned long maxpages;
3625	struct folio *folio = NULL;
3626	struct inode *inode = NULL;
3627	bool inced_nr_rotate_swap = false;
3628
3629	if (swap_flags & ~SWAP_FLAGS_VALID)
3630	return -EINVAL;
3631
3632	if (!capable(CAP_SYS_ADMIN))
3633	return -EPERM;
3634
3635	/*
3636	* Allocate or reuse existing !SWP_USED swap_info. The returned
3637	* si will stay in a dying status, so nothing will access its content
3638	* until enable_swap_info resurrects its percpu ref and expose it.
3639	*/
3640	si = alloc_swap_info();
3641	if (IS_ERR(si))
3642	return PTR_ERR(si);
3643
3644	INIT_WORK(&si->discard_work, swap_discard_work);
3645	INIT_WORK(&si->reclaim_work, swap_reclaim_work);
3646
3647	CLASS(filename, name)(specialfile);
3648	swap_file = file_open_name(name, O_RDWR \| O_LARGEFILE \| O_EXCL, 0);
3649	if (IS_ERR(swap_file)) {
3650	error = PTR_ERR(swap_file);
3651	swap_file = NULL;
3652	goto bad_swap;
3653	}
3654
3655	mapping = swap_file->f_mapping;
3656	dentry = swap_file->f_path.dentry;
3657	inode = mapping->host;
3658
3659	error = claim_swapfile(si, inode);
3660	if (unlikely(error))
3661	goto bad_swap;
3662
3663	inode_lock(inode);
3664	if (d_unlinked(dentry) \|\| cant_mount(dentry)) {
3665	error = -ENOENT;
3666	goto bad_swap_unlock_inode;
3667	}
3668	if (IS_SWAPFILE(inode)) {
3669	error = -EBUSY;
3670	goto bad_swap_unlock_inode;
3671	}
3672	if (IS_ENCRYPTED(inode)) {
3673	pr_warn_once(
3674	"Filesystem-level encrypted swapfile '%s' is unsupported. Create a loop device over it, or use dm-crypt\n",
3675	name->name);
3676	error = -EINVAL;
3677	goto bad_swap_unlock_inode;
3678	}
3679
3680	/*
3681	* The swap subsystem needs a major overhaul to support this.
3682	* It doesn't work yet so just disable it for now.
3683	*/
3684	if (mapping_min_folio_order(mapping) > 0) {
3685	error = -EINVAL;
3686	goto bad_swap_unlock_inode;
3687	}
3688
3689	/*
3690	* Read the swap header.
3691	*/
3692	if (!mapping->a_ops->read_folio) {
3693	error = -EINVAL;
3694	goto bad_swap_unlock_inode;
3695	}
3696	folio = read_mapping_folio(mapping, 0, swap_file);
3697	if (IS_ERR(folio)) {
3698	error = PTR_ERR(folio);
3699	goto bad_swap_unlock_inode;
3700	}
3701	swap_header = kmap_local_folio(folio, 0);
3702
3703	maxpages = read_swap_header(si, swap_header, inode);
3704	if (unlikely(!maxpages)) {
3705	error = -EINVAL;
3706	goto bad_swap_unlock_inode;
3707	}
3708
3709	si->max = maxpages;
3710	si->pages = maxpages - 1;
3711	nr_extents = setup_swap_extents(si, swap_file, &span);
3712	if (nr_extents < 0) {
3713	error = nr_extents;
3714	goto bad_swap_unlock_inode;
3715	}
3716	if (si->pages != si->max - 1) {
3717	pr_err("swap:%u != (max:%u - 1)\n", si->pages, si->max);
3718	error = -EINVAL;
3719	goto bad_swap_unlock_inode;
3720	}
3721
3722	maxpages = si->max;
3723
3724	/* Set up the swap cluster info */
3725	error = setup_swap_clusters_info(si, swap_header, maxpages);
3726	if (error)
3727	goto bad_swap_unlock_inode;
3728
3729	if (si->bdev && bdev_stable_writes(si->bdev))
3730	si->flags \|= SWP_STABLE_WRITES;
3731
3732	if (si->bdev && bdev_synchronous(si->bdev))
3733	si->flags \|= SWP_SYNCHRONOUS_IO;
3734
3735	if (si->bdev && !bdev_rot(si->bdev)) {
3736	si->flags \|= SWP_SOLIDSTATE;
3737	} else {
3738	atomic_inc(&nr_rotate_swap);
3739	inced_nr_rotate_swap = true;
3740	}
3741
3742	if ((swap_flags & SWAP_FLAG_DISCARD) &&
3743	si->bdev && bdev_max_discard_sectors(si->bdev)) {
3744	/*
3745	* When discard is enabled for swap with no particular
3746	* policy flagged, we set all swap discard flags here in
3747	* order to sustain backward compatibility with older
3748	* swapon(8) releases.
3749	*/
3750	si->flags \|= (SWP_DISCARDABLE \| SWP_AREA_DISCARD \|
3751	SWP_PAGE_DISCARD);
3752
3753	/*
3754	* By flagging sys_swapon, a sysadmin can tell us to
3755	* either do single-time area discards only, or to just
3756	* perform discards for released swap page-clusters.
3757	* Now it's time to adjust the p->flags accordingly.
3758	*/
3759	if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
3760	si->flags &= ~SWP_PAGE_DISCARD;
3761	else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
3762	si->flags &= ~SWP_AREA_DISCARD;
3763
3764	/* issue a swapon-time discard if it's still required */
3765	if (si->flags & SWP_AREA_DISCARD) {
3766	int err = discard_swap(si);
3767	if (unlikely(err))
3768	pr_err("swapon: discard_swap(%p): %d\n",
3769	si, err);
3770	}
3771	}
3772
3773	error = zswap_swapon(si->type, maxpages);
3774	if (error)
3775	goto bad_swap_unlock_inode;
3776
3777	/*
3778	* Flush any pending IO and dirty mappings before we start using this
3779	* swap device.
3780	*/
3781	inode->i_flags \|= S_SWAPFILE;
3782	error = inode_drain_writes(inode);
3783	if (error) {
3784	inode->i_flags &= ~S_SWAPFILE;
3785	goto free_swap_zswap;
3786	}
3787
3788	mutex_lock(&swapon_mutex);
3789	prio = DEF_SWAP_PRIO;
3790	if (swap_flags & SWAP_FLAG_PREFER)
3791	prio = swap_flags & SWAP_FLAG_PRIO_MASK;
3792
3793	/*
3794	* The plist prio is negated because plist ordering is
3795	* low-to-high, while swap ordering is high-to-low
3796	*/
3797	si->prio = prio;
3798	si->list.prio = -si->prio;
3799	si->avail_list.prio = -si->prio;
3800	si->swap_file = swap_file;
3801
3802	/* Sets SWP_WRITEOK, resurrect the percpu ref, expose the swap device */
3803	enable_swap_info(si);
3804
3805	pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s\n",
3806	K(si->pages), name->name, si->prio, nr_extents,
3807	K((unsigned long long)span),
3808	(si->flags & SWP_SOLIDSTATE) ? "SS" : "",
3809	(si->flags & SWP_DISCARDABLE) ? "D" : "",
3810	(si->flags & SWP_AREA_DISCARD) ? "s" : "",
3811	(si->flags & SWP_PAGE_DISCARD) ? "c" : "");
3812
3813	mutex_unlock(&swapon_mutex);
3814	atomic_inc(&proc_poll_event);
3815	wake_up_interruptible(&proc_poll_wait);
3816
3817	error = 0;
3818	goto out;
3819	free_swap_zswap:
3820	zswap_swapoff(si->type);
3821	bad_swap_unlock_inode:
3822	inode_unlock(inode);
3823	bad_swap:
3824	kfree(si->global_cluster);
3825	si->global_cluster = NULL;
3826	inode = NULL;
3827	destroy_swap_extents(si, swap_file);
3828	free_swap_cluster_info(si->cluster_info, si->max);
3829	si->cluster_info = NULL;
3830	/*
3831	* Clear the SWP_USED flag after all resources are freed so
3832	* alloc_swap_info can reuse this si safely.
3833	*/
3834	spin_lock(&swap_lock);
3835	si->flags = 0;
3836	spin_unlock(&swap_lock);
3837	if (inced_nr_rotate_swap)
3838	atomic_dec(&nr_rotate_swap);
3839	if (swap_file)
3840	filp_close(swap_file, NULL);
3841	out:
3842	if (!IS_ERR_OR_NULL(folio))
3843	folio_release_kmap(folio, swap_header);
3844	if (inode)
3845	inode_unlock(inode);
3846	return error;
3847	}
3848
3849	void si_swapinfo(struct sysinfo *val)
3850	{
3851	unsigned int type;
3852	unsigned long nr_to_be_unused = 0;
3853
3854	spin_lock(&swap_lock);
3855	for (type = 0; type < nr_swapfiles; type++) {
3856	struct swap_info_struct *si = swap_info[type];
3857
3858	if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
3859	nr_to_be_unused += swap_usage_in_pages(si);
3860	}
3861	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
3862	val->totalswap = total_swap_pages + nr_to_be_unused;
3863	spin_unlock(&swap_lock);
3864	}
3865
3866	/*
3867	* swap_dup_entry_direct() - Increase reference count of a swap entry by one.
3868	* @entry: first swap entry from which we want to increase the refcount.
3869	*
3870	* Returns 0 for success, or -ENOMEM if the extend table is required
3871	* but could not be atomically allocated. Returns -EINVAL if the swap
3872	* entry is invalid, which might occur if a page table entry has got
3873	* corrupted.
3874	*
3875	* Context: Caller must ensure there is no race condition on the reference
3876	* owner. e.g., locking the PTL of a PTE containing the entry being increased.
3877	* Also the swap entry must have a count >= 1. Otherwise folio_dup_swap should
3878	* be used.
3879	*/
3880	int swap_dup_entry_direct(swp_entry_t entry)
3881	{
3882	struct swap_info_struct *si;
3883
3884	si = swap_entry_to_info(entry);
3885	if (WARN_ON_ONCE(!si)) {
3886	pr_err_ratelimited("%s%08lx\n", Bad_file, entry.val);
3887	return -EINVAL;
3888	}
3889
3890	/*
3891	* The caller must be increasing the swap count from a direct
3892	* reference of the swap slot (e.g. a swap entry in page table).
3893	* So the swap count must be >= 1.
3894	*/
3895	VM_WARN_ON_ONCE(!swap_entry_swapped(si, entry));
3896
3897	return swap_dup_entries_cluster(si, swp_offset(entry), 1);
3898	}
3899
3900	#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
3901	static bool __has_usable_swap(void)
3902	{
3903	return !plist_head_empty(&swap_active_head);
3904	}
3905
3906	void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
3907	{
3908	struct swap_info_struct *si;
3909
3910	if (!(gfp & __GFP_IO))
3911	return;
3912
3913	if (!__has_usable_swap())
3914	return;
3915
3916	if (!blk_cgroup_congested())
3917	return;
3918
3919	/*
3920	* We've already scheduled a throttle, avoid taking the global swap
3921	* lock.
3922	*/
3923	if (current->throttle_disk)
3924	return;
3925
3926	spin_lock(&swap_avail_lock);
3927	plist_for_each_entry(si, &swap_avail_head, avail_list) {
3928	if (si->bdev) {
3929	blkcg_schedule_throttle(si->bdev->bd_disk, true);
3930	break;
3931	}
3932	}
3933	spin_unlock(&swap_avail_lock);
3934	}
3935	#endif
3936
3937	static int __init swapfile_init(void)
3938	{
3939	swapfile_maximum_size = arch_max_swapfile_size();
3940
3941	/*
3942	* Once a cluster is freed, it's swap table content is read
3943	* only, and all swap cache readers (swap_cache_*) verifies
3944	* the content before use. So it's safe to use RCU slab here.
3945	*/
3946	if (!SWP_TABLE_USE_PAGE)
3947	swap_table_cachep = kmem_cache_create("swap_table",
3948	sizeof(struct swap_table),
3949	0, SLAB_PANIC \| SLAB_TYPESAFE_BY_RCU, NULL);
3950
3951	#ifdef CONFIG_MIGRATION
3952	if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
3953	swap_migration_ad_supported = true;
3954	#endif /* CONFIG_MIGRATION */
3955
3956	return 0;
3957	}
3958	subsys_initcall(swapfile_init);