projects/linux·blob 2c455ddc · refs

471 lines · 14.2 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _MM_SWAP_H
3#define _MM_SWAP_H
4
5#include <linux/atomic.h> /* for atomic_long_t */
6#include <linux/mm.h> /* for PAGE_SHIFT */
7#include <linux/memcontrol.h> /* for mem_cgroup_swappiness() */
8#include <linux/swap.h> /* for MAX_SWAPFILES_SHIFT, struct swap_info_struct */
9
10struct mempolicy;
11struct swap_iocb;
12struct swap_memcg_table;
13struct swap_io_ctx;
14
15#if defined(MAX_POSSIBLE_PHYSMEM_BITS)
16#define SWAP_CACHE_PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
17#elif defined(MAX_PHYSMEM_BITS)
18#define SWAP_CACHE_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT)
19#else
20#define SWAP_CACHE_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT)
21#endif
22
23/* Swap table marker, 0x1 means shadow, 0x2 means PFN (SWP_TB_PFN_MARK) */
24#define SWAP_CACHE_PFN_MARK_BITS 2
25/* At least 2 bits are needed to distinguish SWP_TB_COUNT_MAX, 1 and 0 */
26#define SWAP_COUNT_MIN_BITS 2
27/* If there are enough bits besides PFN and marker, store zero flag inline */
28#define SWAP_TABLE_HAS_ZEROFLAG ((BITS_PER_LONG - SWAP_CACHE_PFN_MARK_BITS - \
29 SWAP_CACHE_PFN_BITS) > SWAP_COUNT_MIN_BITS)
30
31#ifdef CONFIG_THP_SWAP
32#define SWAPFILE_CLUSTER HPAGE_PMD_NR
33#define swap_entry_order(order) (order)
34#else
35#define SWAPFILE_CLUSTER 256
36#define swap_entry_order(order) 0
37#endif
38
39extern struct swap_info_struct *swap_info[];
40
41/*
42 * We use this to track usage of a cluster. A cluster is a block of swap disk
43 * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All
44 * free clusters are organized into a list. We fetch an entry from the list to
45 * get a free cluster.
46 *
47 * The flags field determines if a cluster is free. This is
48 * protected by cluster lock.
49 */
50struct swap_cluster_info {
51 spinlock_t lock; /*
52 * Protect swap_cluster_info fields
53 * other than list, and swap_info_struct->swap_map
54 * elements corresponding to the swap cluster.
55 */
56 u16 count;
57 u8 flags;
58 u8 order;
59 atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */
60 unsigned int *extend_table; /* For large swap count, protected by ci->lock */
61#ifdef CONFIG_MEMCG
62 struct swap_memcg_table *memcg_table; /* Swap table entries' cgroup record */
63#endif
64#if !SWAP_TABLE_HAS_ZEROFLAG
65 unsigned long *zero_bitmap;
66#endif
67 struct list_head list;
68};
69
70/* All on-list cluster must have a non-zero flag. */
71enum swap_cluster_flags {
72 CLUSTER_FLAG_NONE = 0, /* For temporary off-list cluster */
73 CLUSTER_FLAG_FREE,
74 CLUSTER_FLAG_NONFULL,
75 CLUSTER_FLAG_FRAG,
76 /* Clusters with flags above are allocatable */
77 CLUSTER_FLAG_USABLE = CLUSTER_FLAG_FRAG,
78 CLUSTER_FLAG_FULL,
79 CLUSTER_FLAG_DISCARD,
80 CLUSTER_FLAG_MAX,
81};
82
83extern int vm_swappiness;
84
85static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
86{
87#ifdef CONFIG_MEMCG_V1
88 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
89 !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
90 return READ_ONCE(memcg->swappiness);
91#endif
92 return READ_ONCE(vm_swappiness);
93}
94
95#ifdef CONFIG_SWAP
96#include <linux/swapops.h> /* for swp_offset */
97#include <linux/blk_types.h> /* for bio_end_io_t */
98
99static inline unsigned int swp_cluster_offset(swp_entry_t entry)
100{
101 return swp_offset(entry) % SWAPFILE_CLUSTER;
102}
103
104/*
105 * Callers of all helpers below must ensure the entry, type, or offset is
106 * valid, and protect the swap device with reference count or locks.
107 */
108static inline struct swap_info_struct *__swap_type_to_info(int type)
109{
110 struct swap_info_struct *si;
111
112 si = READ_ONCE(swap_info[type]); /* rcu_dereference() */
113 VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
114 return si;
115}
116
117static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry)
118{
119 return __swap_type_to_info(swp_type(entry));
120}
121
122static inline struct swap_cluster_info *__swap_offset_to_cluster(
123 struct swap_info_struct *si, pgoff_t offset)
124{
125 VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
126 VM_WARN_ON_ONCE(offset >= roundup(si->max, SWAPFILE_CLUSTER));
127 return &si->cluster_info[offset / SWAPFILE_CLUSTER];
128}
129
130static inline struct swap_cluster_info *__swap_entry_to_cluster(swp_entry_t entry)
131{
132 return __swap_offset_to_cluster(__swap_entry_to_info(entry),
133 swp_offset(entry));
134}
135
136static __always_inline struct swap_cluster_info *__swap_cluster_lock(
137 struct swap_info_struct *si, unsigned long offset, bool irq)
138{
139 struct swap_cluster_info *ci = __swap_offset_to_cluster(si, offset);
140
141 /*
142 * Nothing modifies swap cache in an IRQ context. All access to
143 * swap cache is wrapped by swap_cache_* helpers, and swap cache
144 * writeback is handled outside of IRQs. Swapin or swapout never
145 * occurs in IRQ, and neither does in-place split or replace.
146 *
147 * Besides, modifying swap cache requires synchronization with
148 * swap_map, which was never IRQ safe.
149 */
150 VM_WARN_ON_ONCE(!in_task());
151 VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
152 if (irq)
153 spin_lock_irq(&ci->lock);
154 else
155 spin_lock(&ci->lock);
156 return ci;
157}
158
159/**
160 * swap_cluster_lock - Lock and return the swap cluster of given offset.
161 * @si: swap device the cluster belongs to.
162 * @offset: the swap entry offset, pointing to a valid slot.
163 *
164 * Context: The caller must ensure the offset is in the valid range and
165 * protect the swap device with reference count or locks.
166 */
167static inline struct swap_cluster_info *swap_cluster_lock(
168 struct swap_info_struct *si, unsigned long offset)
169{
170 return __swap_cluster_lock(si, offset, false);
171}
172
173static inline struct swap_cluster_info *__swap_cluster_get_and_lock(
174 const struct folio *folio, bool irq)
175{
176 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
177 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
178 return __swap_cluster_lock(__swap_entry_to_info(folio->swap),
179 swp_offset(folio->swap), irq);
180}
181
182/*
183 * swap_cluster_get_and_lock - Locks the cluster that holds a folio's entries.
184 * @folio: The folio.
185 *
186 * This locks and returns the swap cluster that contains a folio's swap
187 * entries. The swap entries of a folio are always in one single cluster.
188 * The folio has to be locked so its swap entries won't change and the
189 * cluster won't be freed.
190 *
191 * Context: Caller must ensure the folio is locked and in the swap cache.
192 * Return: Pointer to the swap cluster.
193 */
194static inline struct swap_cluster_info *swap_cluster_get_and_lock(
195 const struct folio *folio)
196{
197 return __swap_cluster_get_and_lock(folio, false);
198}
199
200/*
201 * swap_cluster_get_and_lock_irq - Locks the cluster that holds a folio's entries.
202 * @folio: The folio.
203 *
204 * Same as swap_cluster_get_and_lock but also disable IRQ.
205 *
206 * Context: Caller must ensure the folio is locked and in the swap cache.
207 * Return: Pointer to the swap cluster.
208 */
209static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq(
210 const struct folio *folio)
211{
212 return __swap_cluster_get_and_lock(folio, true);
213}
214
215static inline void swap_cluster_unlock(struct swap_cluster_info *ci)
216{
217 spin_unlock(&ci->lock);
218}
219
220static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci)
221{
222 spin_unlock_irq(&ci->lock);
223}
224
225extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp);
226
227/*
228 * Below are the core routines for doing swap for a folio.
229 * All helpers requires the folio to be locked, and a locked folio
230 * in the swap cache pins the swap entries / slots allocated to the
231 * folio, swap relies heavily on the swap cache and folio lock for
232 * synchronization.
233 *
234 * folio_alloc_swap(): the entry point for a folio to be swapped
235 * out. It allocates swap slots and pins the slots with swap cache.
236 * The slots start with a swap count of zero. The slots are pinned
237 * by swap cache reference which doesn't contribute to swap count.
238 *
239 * folio_dup_swap(): increases the swap count of a folio, usually
240 * during it gets unmapped and a swap entry is installed to replace
241 * it (e.g., swap entry in page table). A swap slot with swap
242 * count == 0 can only be increased by this helper.
243 *
244 * folio_put_swap(): does the opposite thing of folio_dup_swap().
245 */
246int folio_alloc_swap(struct folio *folio);
247int folio_dup_swap(struct folio *folio, struct page *page);
248void folio_put_swap(struct folio *folio, struct page *page);
249
250/* For internal use */
251extern void __swap_cluster_free_entries(struct swap_info_struct *si,
252 struct swap_cluster_info *ci,
253 unsigned int ci_off, unsigned int nr_pages);
254
255/* linux/mm/page_io.c */
256int sio_pool_init(void);
257void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio);
258void swap_read_submit(struct swap_io_ctx *ctx);
259void swap_write_submit(struct swap_io_ctx *ctx);
260int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio);
261void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio);
262
263/* linux/mm/swap_state.c */
264extern struct address_space swap_space __read_mostly;
265static inline struct address_space *swap_address_space(swp_entry_t entry)
266{
267 return &swap_space;
268}
269
270/*
271 * Return the swap device position of the swap entry.
272 */
273static inline loff_t swap_dev_pos(swp_entry_t entry)
274{
275 return ((loff_t)swp_offset(entry)) << PAGE_SHIFT;
276}
277
278/**
279 * folio_matches_swap_entry - Check if a folio matches a given swap entry.
280 * @folio: The folio.
281 * @entry: The swap entry to check against.
282 *
283 * Context: The caller should have the folio locked to ensure it's stable
284 * and nothing will move it in or out of the swap cache.
285 * Return: true or false.
286 */
287static inline bool folio_matches_swap_entry(const struct folio *folio,
288 swp_entry_t entry)
289{
290 swp_entry_t folio_entry = folio->swap;
291 long nr_pages = folio_nr_pages(folio);
292
293 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
294 if (!folio_test_swapcache(folio))
295 return false;
296 VM_WARN_ON_ONCE_FOLIO(!IS_ALIGNED(folio_entry.val, nr_pages), folio);
297 return folio_entry.val == round_down(entry.val, nr_pages);
298}
299
300/*
301 * All swap cache helpers below require the caller to ensure the swap entries
302 * used are valid and stabilize the device by any of the following ways:
303 * - Hold a reference by get_swap_device(): this ensures a single entry is
304 * valid and increases the swap device's refcount.
305 * - Locking a folio in the swap cache: this ensures the folio's swap entries
306 * are valid and pinned, also implies reference to the device.
307 * - Locking anything referencing the swap entry: e.g. PTL that protects
308 * swap entries in the page table, similar to locking swap cache folio.
309 * - See the comment of get_swap_device() for more complex usage.
310 */
311bool swap_cache_has_folio(swp_entry_t entry);
312struct folio *swap_cache_get_folio(swp_entry_t entry);
313void *swap_cache_get_shadow(swp_entry_t entry);
314void swap_cache_del_folio(struct folio *folio);
315struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask,
316 unsigned long orders, struct vm_fault *vmf,
317 struct mempolicy *mpol, pgoff_t ilx);
318/* Below helpers require the caller to lock and pass in the swap cluster. */
319void __swap_cache_add_folio(struct swap_cluster_info *ci,
320 struct folio *folio, swp_entry_t entry);
321void __swap_cache_del_folio(struct swap_cluster_info *ci,
322 struct folio *folio, swp_entry_t entry, void *shadow);
323void __swap_cache_replace_folio(struct swap_cluster_info *ci,
324 struct folio *old, struct folio *new);
325
326void show_swap_cache_info(void);
327void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
328struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
329 gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr);
330struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
331 struct mempolicy *mpol, pgoff_t ilx);
332struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
333 struct vm_fault *vmf);
334struct folio *swapin_sync(swp_entry_t entry, gfp_t flag, unsigned long orders,
335 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx);
336void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma,
337 unsigned long addr);
338
339#else /* CONFIG_SWAP */
340
341static inline struct swap_cluster_info *swap_cluster_get_and_lock(
342 struct folio *folio)
343{
344 return NULL;
345}
346
347static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq(
348 struct folio *folio)
349{
350 return NULL;
351}
352
353static inline void swap_cluster_unlock(struct swap_cluster_info *ci)
354{
355}
356
357static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci)
358{
359}
360
361static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry)
362{
363 return NULL;
364}
365
366static inline int folio_alloc_swap(struct folio *folio)
367{
368 return -EINVAL;
369}
370
371static inline int folio_dup_swap(struct folio *folio, struct page *page)
372{
373 return -EINVAL;
374}
375
376static inline void folio_put_swap(struct folio *folio, struct page *page)
377{
378}
379
380static inline void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio)
381{
382}
383
384static inline void swap_write_submit(struct swap_io_ctx *ctx)
385{
386}
387
388static inline struct address_space *swap_address_space(swp_entry_t entry)
389{
390 return NULL;
391}
392
393static inline bool folio_matches_swap_entry(const struct folio *folio, swp_entry_t entry)
394{
395 return false;
396}
397
398static inline void show_swap_cache_info(void)
399{
400}
401
402static inline struct folio *swap_cluster_readahead(swp_entry_t entry,
403 gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx)
404{
405 return NULL;
406}
407
408static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask,
409 struct vm_fault *vmf)
410{
411 return NULL;
412}
413
414static inline struct folio *swapin_sync(
415 swp_entry_t entry, gfp_t flag, unsigned long orders,
416 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx)
417{
418 return NULL;
419}
420
421static inline void swap_update_readahead(struct folio *folio,
422 struct vm_area_struct *vma, unsigned long addr)
423{
424}
425
426static inline int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
427{
428 return 0;
429}
430
431static inline int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
432{
433 return -EINVAL;
434}
435
436static inline bool swap_cache_has_folio(swp_entry_t entry)
437{
438 return false;
439}
440
441static inline struct folio *swap_cache_get_folio(swp_entry_t entry)
442{
443 return NULL;
444}
445
446static inline void *swap_cache_get_shadow(swp_entry_t entry)
447{
448 return NULL;
449}
450
451static inline void swap_cache_del_folio(struct folio *folio)
452{
453}
454
455static inline void __swap_cache_del_folio(struct swap_cluster_info *ci,
456 struct folio *folio, swp_entry_t entry, void *shadow)
457{
458}
459
460static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci,
461 struct folio *old, struct folio *new)
462{
463}
464#endif /* CONFIG_SWAP */
465
466extern const struct swap_ops swap_bdev_ops;
467
468int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
469 struct list_head *folio_list);
470
471#endif /* _MM_SWAP_H */