projects/linux·blob master · refs

312 lines · 10.6 KiB

1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * mm-internal API for the page (buddy) allocator. Public API lives in
4 * include/linux/gfp.h.
5 */
6#ifndef __MM_PAGE_ALLOC_H
7#define __MM_PAGE_ALLOC_H
8
9#include <linux/mm.h>
10#include <linux/mmzone.h>
11#include <linux/nodemask.h>
12#include <linux/types.h>
13
14#define ALLOC_DEFAULT 0
15/* The ALLOC_WMARK bits are used as an index to zone->watermark */
16#define ALLOC_WMARK_MIN WMARK_MIN
17#define ALLOC_WMARK_LOW WMARK_LOW
18#define ALLOC_WMARK_HIGH WMARK_HIGH
19#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
20
21/* Mask to get the watermark bits */
22#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
23
24/*
25 * Only MMU archs have async oom victim reclaim - aka oom_reaper so we
26 * cannot assume a reduced access to memory reserves is sufficient for
27 * !MMU
28 */
29#ifdef CONFIG_MMU
30#define ALLOC_OOM 0x08
31#else
32#define ALLOC_OOM ALLOC_NO_WATERMARKS
33#endif
34
35#define ALLOC_NON_BLOCK 0x10 /* Caller cannot block. Allow access
36 * to 25% of the min watermark or
37 * 62.5% if __GFP_HIGH is set.
38 */
39#define ALLOC_MIN_RESERVE 0x20 /* __GFP_HIGH set. Allow access to 50%
40 * of the min watermark.
41 */
42#define ALLOC_CPUSET 0x40 /* check for correct cpuset */
43#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */
44#ifdef CONFIG_ZONE_DMA32
45#define ALLOC_NOFRAGMENT 0x100 /* avoid mixing pageblock types */
46#else
47#define ALLOC_NOFRAGMENT 0x0
48#endif
49#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */
50#define ALLOC_NOLOCK 0x400 /* Only use spin_trylock in allocation path */
51#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
52/*
53 * Avoid alloc_tag recursion for internal allocations.
54 *
55 * Callers must clear_page_tag_ref() before freeing to avoid warnings from
56 * alloc_tag_sub_check().
57 */
58#define ALLOC_NO_CODETAG 0x1000
59
60/* Flags that allow allocations below the min watermark. */
61#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
62
63/*
64 * Structure for holding the mostly immutable allocation parameters passed
65 * between functions involved in allocations, including the alloc_pages*
66 * family of functions.
67 *
68 * nodemask, migratetype and highest_zoneidx are initialized only once in
69 * __alloc_pages() and then never change.
70 *
71 * zonelist, preferred_zone and highest_zoneidx are set first in
72 * __alloc_pages() for the fast path, and might be later changed
73 * in __alloc_pages_slowpath(). All other functions pass the whole structure
74 * by a const pointer.
75 */
76struct alloc_context {
77 struct zonelist *zonelist;
78 const nodemask_t *nodemask;
79 struct zoneref *preferred_zoneref;
80 int migratetype;
81
82 /*
83 * highest_zoneidx represents highest usable zone index of
84 * the allocation request. Due to the nature of the zone,
85 * memory on lower zone than the highest_zoneidx will be
86 * protected by lowmem_reserve[highest_zoneidx].
87 *
88 * highest_zoneidx is also used by reclaim/compaction to limit
89 * the target zone since higher zone than this index cannot be
90 * usable for this allocation request.
91 */
92 enum zone_type highest_zoneidx;
93 bool spread_dirty_pages;
94 /* Only flags that are global to the whole allocation go here. */
95 unsigned int alloc_flags;
96};
97
98/*
99 * This function returns the order of a free page in the buddy system. In
100 * general, page_zone(page)->lock must be held by the caller to prevent the
101 * page from being allocated in parallel and returning garbage as the order.
102 * If a caller does not hold page_zone(page)->lock, it must guarantee that the
103 * page cannot be allocated or merged in parallel. Alternatively, it must
104 * handle invalid values gracefully, and use buddy_order_unsafe() below.
105 */
106static inline unsigned int buddy_order(struct page *page)
107{
108 /* PageBuddy() must be checked by the caller */
109 return page_private(page);
110}
111
112/*
113 * Like buddy_order(), but for callers who cannot afford to hold the zone lock.
114 * PageBuddy() should be checked first by the caller to minimize race window,
115 * and invalid values must be handled gracefully.
116 *
117 * READ_ONCE is used so that if the caller assigns the result into a local
118 * variable and e.g. tests it for valid range before using, the compiler cannot
119 * decide to remove the variable and inline the page_private(page) multiple
120 * times, potentially observing different values in the tests and the actual
121 * use of the result.
122 */
123#define buddy_order_unsafe(page) READ_ONCE(page_private(page))
124
125/*
126 * This function checks whether a page is free && is the buddy
127 * we can coalesce a page and its buddy if
128 * (a) the buddy is not in a hole (check before calling!) &&
129 * (b) the buddy is in the buddy system &&
130 * (c) a page and its buddy have the same order &&
131 * (d) a page and its buddy are in the same zone.
132 *
133 * For recording whether a page is in the buddy system, we set PageBuddy.
134 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
135 *
136 * For recording page's order, we use page_private(page).
137 */
138static inline bool page_is_buddy(struct page *page, struct page *buddy,
139 unsigned int order)
140{
141 if (!page_is_guard(buddy) && !PageBuddy(buddy))
142 return false;
143
144 if (buddy_order(buddy) != order)
145 return false;
146
147 /*
148 * zone check is done late to avoid uselessly calculating
149 * zone/node ids for pages that could never merge.
150 */
151 if (page_zone_id(page) != page_zone_id(buddy))
152 return false;
153
154 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
155
156 return true;
157}
158
159/*
160 * Locate the struct page for both the matching buddy in our
161 * pair (buddy1) and the combined O(n+1) page they form (page).
162 *
163 * 1) Any buddy B1 will have an order O twin B2 which satisfies
164 * the following equation:
165 * B2 = B1 ^ (1 << O)
166 * For example, if the starting buddy (buddy2) is #8 its order
167 * 1 buddy is #10:
168 * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
169 *
170 * 2) Any buddy B will have an order O+1 parent P which
171 * satisfies the following equation:
172 * P = B & ~(1 << O)
173 *
174 * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER
175 */
176static inline unsigned long
177__find_buddy_pfn(unsigned long page_pfn, unsigned int order)
178{
179 return page_pfn ^ (1 << order);
180}
181
182/*
183 * Find the buddy of @page and validate it.
184 * @page: The input page
185 * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the
186 * function is used in the performance-critical __free_one_page().
187 * @order: The order of the page
188 * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to
189 * page_to_pfn().
190 *
191 * The found buddy can be a non PageBuddy, out of @page's zone, or its order is
192 * not the same as @page. The validation is necessary before use it.
193 *
194 * Return: the found buddy page or NULL if not found.
195 */
196static inline struct page *find_buddy_page_pfn(struct page *page,
197 unsigned long pfn, unsigned int order, unsigned long *buddy_pfn)
198{
199 unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order);
200 struct page *buddy;
201
202 buddy = page + (__buddy_pfn - pfn);
203 if (buddy_pfn)
204 *buddy_pfn = __buddy_pfn;
205
206 if (page_is_buddy(page, buddy, order))
207 return buddy;
208 return NULL;
209}
210
211extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
212 unsigned long end_pfn, struct zone *zone);
213
214static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
215 unsigned long end_pfn, struct zone *zone)
216{
217 if (zone->contiguous)
218 return pfn_to_page(start_pfn);
219
220 return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
221}
222
223extern void __free_pages_core(struct page *page, unsigned int order,
224 enum meminit_context context);
225
226void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags,
227 unsigned int alloc_flags);
228extern bool free_pages_prepare(struct page *page, unsigned int order);
229
230extern int user_min_free_kbytes;
231
232struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order, int nid,
233 nodemask_t *nodemask, unsigned int alloc_flags);
234#define __alloc_frozen_pages(...) \
235 alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
236void free_frozen_pages(struct page *page, unsigned int order);
237void free_unref_folios(struct folio_batch *fbatch);
238
239#ifdef CONFIG_NUMA
240struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
241#else
242static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
243{
244 return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL,
245 ALLOC_DEFAULT);
246}
247#endif
248
249#define alloc_frozen_pages(...) \
250 alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__))
251
252struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
253#define alloc_frozen_pages_nolock(...) \
254 alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))
255void free_frozen_pages_nolock(struct page *page, unsigned int order);
256
257struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
258 nodemask_t *nodemask, unsigned int alloc_flags);
259#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__))
260
261extern void zone_pcp_reset(struct zone *zone);
262extern void zone_pcp_disable(struct zone *zone);
263extern void zone_pcp_enable(struct zone *zone);
264extern void zone_pcp_init(struct zone *zone);
265
266enum fallback_result {
267 /* Found suitable migratetype, *mt_out is valid. */
268 FALLBACK_FOUND,
269 /* No fallback found in requested order. */
270 FALLBACK_EMPTY,
271 /* Passed @claimable, but claiming whole block is a bad idea. */
272 FALLBACK_NOCLAIM,
273};
274enum fallback_result
275find_suitable_fallback(struct free_area *area, unsigned int order,
276 int migratetype, bool claimable, int *mt_out);
277
278static inline bool free_area_empty(struct free_area *area, int migratetype)
279{
280 return list_empty(&area->free_list[migratetype]);
281}
282
283/* Convert GFP flags to their corresponding migrate type */
284#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
285#define GFP_MOVABLE_SHIFT 3
286
287static inline int gfp_migratetype(const gfp_t gfp_flags)
288{
289 VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
290 BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
291 BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
292 BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE);
293 BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >>
294 GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC);
295
296 if (unlikely(page_group_by_mobility_disabled))
297 return MIGRATE_UNMOVABLE;
298
299 /* Group based on mobility */
300 return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
301}
302#undef GFP_MOVABLE_MASK
303#undef GFP_MOVABLE_SHIFT
304
305bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp);
306void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp);
307void drain_all_pages(struct zone *zone);
308
309void page_alloc_init_cpuhp(void);
310void page_alloc_sysctl_init(void);
311
312#endif /* __MM_PAGE_ALLOC_H */