projects/linux·blob master · refs

411 lines · 9.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/mm.h>
4#include <linux/cma.h>
5#include <linux/compiler.h>
6#include <linux/cpuset.h>
7#include <linux/mm_inline.h>
8
9#include <asm/page.h>
10#include <asm/setup.h>
11
12#include <linux/hugetlb.h>
13#include <linux/memblock.h>
14#include <linux/math.h>
15#include <linux/math64.h>
16#include "internal.h"
17#include "hugetlb_cma.h"
18
19
20static struct cma *hugetlb_cma[MAX_NUMNODES] __ro_after_init;
21static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
22static bool hugetlb_cma_only __ro_after_init;
23static unsigned long hugetlb_cma_size __ro_after_init;
24
25static unsigned int hugetlb_cma_percent __initdata;
26static unsigned int hugetlb_cma_percent_in_node[MAX_NUMNODES] __initdata;
27
28#ifdef CONFIG_NUMA
29static phys_addr_t __init memblock_node_memory_size(int nid)
30{
31 struct memblock_region *reg;
32 phys_addr_t size = 0;
33
34 for_each_mem_region(reg) {
35 if (reg->nid == nid)
36 size += reg->size;
37 }
38 return size;
39}
40#else
41static phys_addr_t __init memblock_node_memory_size(int nid)
42{
43 return memblock_phys_mem_size();
44}
45#endif
46
47void hugetlb_cma_free_frozen_folio(struct folio *folio)
48{
49 WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],
50 &folio->page, folio_nr_pages(folio)));
51}
52
53struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
54 int nid, nodemask_t *nodemask)
55{
56 int node;
57 struct folio *folio;
58 struct page *page = NULL;
59 const nodemask_t *nmask;
60 unsigned int cpuset_mems_cookie;
61
62 if (!hugetlb_cma_size)
63 return NULL;
64
65retry_cpuset:
66 if (!nodemask) {
67 cpuset_mems_cookie = read_mems_allowed_begin();
68 nmask = &cpuset_current_mems_allowed;
69 } else {
70 nmask = nodemask;
71 }
72
73 if (hugetlb_cma[nid] && node_isset(nid, *nmask))
74 page = cma_alloc_frozen_compound(hugetlb_cma[nid], order);
75
76 if (!page && !(gfp_mask & __GFP_THISNODE)) {
77 for_each_node_mask(node, *nmask) {
78 if (node == nid || !hugetlb_cma[node])
79 continue;
80
81 page = cma_alloc_frozen_compound(hugetlb_cma[node], order);
82 if (page)
83 break;
84 }
85 }
86
87 if (!page) {
88 if (!nodemask &&
89 unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
90 goto retry_cpuset;
91 return NULL;
92 }
93
94 folio = page_folio(page);
95 folio_set_hugetlb_cma(folio);
96 return folio;
97}
98
99void * __init hugetlb_cma_alloc_bootmem(struct hstate *h, int nid, bool node_exact)
100{
101 struct cma *cma;
102 void *m;
103 int node;
104
105 cma = hugetlb_cma[nid];
106 m = cma_reserve_early(cma, huge_page_size(h));
107 if (m || node_exact)
108 return m;
109
110 for_each_node_mask(node, hugetlb_bootmem_nodes) {
111 cma = hugetlb_cma[node];
112 if (!cma || node == nid)
113 continue;
114 m = cma_reserve_early(cma, huge_page_size(h));
115 if (m)
116 return m;
117 }
118
119 return NULL;
120}
121
122static int __init cmdline_parse_hugetlb_cma(char *p)
123{
124 int nid, count = 0;
125 unsigned long tmp;
126 char *s = p;
127
128 while (*s) {
129 if (sscanf(s, "%lu%n", &tmp, &count) != 1)
130 break;
131
132 if (s[count] == ':') {
133 char *next;
134
135 if (tmp >= MAX_NUMNODES)
136 break;
137 nid = array_index_nospec(tmp, MAX_NUMNODES);
138
139 hugetlb_cma_size = 0;
140 hugetlb_cma_percent = 0;
141
142 s += count + 1;
143 tmp = memparse(s, &next);
144 if (*next == '%') {
145 if (tmp > 100) {
146 pr_warn("hugetlb_cma: invalid percentage %lu for node %d\n",
147 tmp, nid);
148 break;
149 }
150 hugetlb_cma_percent_in_node[nid] = tmp;
151 hugetlb_cma_size_in_node[nid] = 0;
152 s = next + 1;
153 } else {
154 hugetlb_cma_size_in_node[nid] = tmp;
155 hugetlb_cma_percent_in_node[nid] = 0;
156 s = next;
157 }
158
159 /*
160 * Skip the separator if have one, otherwise
161 * break the parsing.
162 */
163 if (*s == ',')
164 s++;
165 else
166 break;
167 } else {
168 char *next;
169
170 tmp = memparse(p, &next);
171 if (*next == '%') {
172 if (tmp > 100) {
173 pr_warn("hugetlb_cma: invalid percentage %lu\n", tmp);
174 } else {
175 hugetlb_cma_percent = tmp;
176 hugetlb_cma_size = 0;
177 for (nid = 0; nid < MAX_NUMNODES; nid++) {
178 hugetlb_cma_size_in_node[nid] = 0;
179 hugetlb_cma_percent_in_node[nid] = 0;
180 }
181 }
182 } else {
183 hugetlb_cma_size = tmp;
184 hugetlb_cma_percent = 0;
185 for (nid = 0; nid < MAX_NUMNODES; nid++) {
186 hugetlb_cma_size_in_node[nid] = 0;
187 hugetlb_cma_percent_in_node[nid] = 0;
188 }
189 }
190 break;
191 }
192 }
193
194 return 0;
195}
196
197early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
198
199static int __init cmdline_parse_hugetlb_cma_only(char *p)
200{
201 return kstrtobool(p, &hugetlb_cma_only);
202}
203
204early_param("hugetlb_cma_only", cmdline_parse_hugetlb_cma_only);
205
206unsigned int __weak arch_hugetlb_cma_order(void)
207{
208 return 0;
209}
210
211void __init hugetlb_cma_reserve(void)
212{
213 unsigned long size, reserved, per_node, order, gigantic_page_size;
214 bool node_specific_cma_alloc = false;
215 bool has_node_specific_param = false;
216 int nid;
217
218 for (nid = 0; nid < MAX_NUMNODES; nid++) {
219 if (hugetlb_cma_size_in_node[nid] || hugetlb_cma_percent_in_node[nid]) {
220 has_node_specific_param = true;
221 break;
222 }
223 }
224
225 if (has_node_specific_param) {
226 hugetlb_cma_size = 0;
227 for (nid = 0; nid < MAX_NUMNODES; nid++) {
228 if (hugetlb_cma_percent_in_node[nid]) {
229 phys_addr_t node_gfp_mem = memblock_node_memory_size(nid);
230 u64 s;
231
232 s = mul_u64_u32_div((u64)node_gfp_mem,
233 hugetlb_cma_percent_in_node[nid],
234 100);
235
236 hugetlb_cma_size_in_node[nid] = s;
237 }
238 hugetlb_cma_size += hugetlb_cma_size_in_node[nid];
239 }
240 } else if (hugetlb_cma_percent) {
241 hugetlb_cma_size = mul_u64_u32_div((u64)memblock_phys_mem_size(),
242 hugetlb_cma_percent, 100);
243 }
244
245 if (!hugetlb_cma_size)
246 return;
247
248 order = arch_hugetlb_cma_order();
249 if (!order) {
250 pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
251 return;
252 }
253
254 /*
255 * HugeTLB CMA reservation is required for gigantic
256 * huge pages which could not be allocated via the
257 * page allocator. Just warn if there is any change
258 * breaking this assumption.
259 */
260 VM_WARN_ON(order <= MAX_PAGE_ORDER);
261 gigantic_page_size = PAGE_SIZE << order;
262
263 if (hugetlb_cma_percent) {
264 unsigned long orig_size = hugetlb_cma_size;
265
266 hugetlb_cma_size = ALIGN_DOWN(hugetlb_cma_size, PAGE_SIZE << order);
267 if (orig_size && !hugetlb_cma_size)
268 pr_warn("hugetlb_cma: reservation size rounded down to 0 from %lu MiB (%u%%)\n",
269 orig_size / SZ_1M, hugetlb_cma_percent);
270 } else if (has_node_specific_param) {
271 hugetlb_cma_size = 0;
272 for (nid = 0; nid < MAX_NUMNODES; nid++) {
273 if (hugetlb_cma_percent_in_node[nid]) {
274 unsigned long orig_size = hugetlb_cma_size_in_node[nid];
275
276 hugetlb_cma_size_in_node[nid] =
277 ALIGN_DOWN(hugetlb_cma_size_in_node[nid],
278 PAGE_SIZE << order);
279 if (orig_size && !hugetlb_cma_size_in_node[nid])
280 pr_warn("hugetlb_cma: reservation size rounded down to 0 from %lu MiB (%u%%) on node %d\n",
281 orig_size / SZ_1M,
282 hugetlb_cma_percent_in_node[nid],
283 nid);
284 }
285 hugetlb_cma_size += hugetlb_cma_size_in_node[nid];
286 }
287 }
288
289 hugetlb_bootmem_set_nodes();
290
291 for (nid = 0; nid < MAX_NUMNODES; nid++) {
292 size = hugetlb_cma_size_in_node[nid];
293 if (size == 0)
294 continue;
295
296 if (!node_isset(nid, hugetlb_bootmem_nodes)) {
297 pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
298 } else if (!IS_ALIGNED(size, gigantic_page_size)) {
299 pr_warn("hugetlb_cma: cma area of node %d must be a multiple of %lu MiB\n",
300 nid, gigantic_page_size / SZ_1M);
301 } else {
302 node_specific_cma_alloc = true;
303 continue;
304 }
305
306 hugetlb_cma_size -= size;
307 hugetlb_cma_size_in_node[nid] = 0;
308 }
309
310 /* Validate the CMA size again in case some invalid nodes specified. */
311 if (!hugetlb_cma_size)
312 return;
313
314 if (!IS_ALIGNED(hugetlb_cma_size, gigantic_page_size)) {
315 pr_warn("hugetlb_cma: cma area must be a multiple of %lu MiB\n",
316 gigantic_page_size / SZ_1M);
317 hugetlb_cma_size = 0;
318 return;
319 }
320
321 if (!node_specific_cma_alloc) {
322 /*
323 * If 3 GB area is requested on a machine with 4 numa nodes,
324 * let's allocate 1 GB on first three nodes and ignore the last one.
325 */
326 per_node = DIV_ROUND_UP(hugetlb_cma_size,
327 nodes_weight(hugetlb_bootmem_nodes));
328 per_node = round_up(per_node, gigantic_page_size);
329 if (hugetlb_cma_percent)
330 pr_info("hugetlb_cma: reserve %lu MiB (%u%%), up to %lu MiB per node\n",
331 hugetlb_cma_size / SZ_1M, hugetlb_cma_percent,
332 per_node / SZ_1M);
333 else
334 pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
335 hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
336 }
337
338 reserved = 0;
339 for_each_node_mask(nid, hugetlb_bootmem_nodes) {
340 int res;
341 char name[CMA_MAX_NAME];
342
343 if (node_specific_cma_alloc) {
344 if (hugetlb_cma_size_in_node[nid] == 0)
345 continue;
346
347 size = hugetlb_cma_size_in_node[nid];
348 } else {
349 size = min(per_node, hugetlb_cma_size - reserved);
350 }
351
352 snprintf(name, sizeof(name), "hugetlb%d", nid);
353 /*
354 * Note that 'order per bit' is based on smallest size that
355 * may be returned to CMA allocator in the case of
356 * huge page demotion.
357 */
358 res = cma_declare_contiguous_multi(size, gigantic_page_size,
359 HUGETLB_PAGE_ORDER, name,
360 &hugetlb_cma[nid], nid);
361 if (res || !cma_validate_zones(hugetlb_cma[nid])) {
362 pr_warn("hugetlb_cma: %s: err %d, node %d\n",
363 res ? "reservation failed" : "reserved area spans zones",
364 res, nid);
365 hugetlb_cma[nid] = NULL;
366 continue;
367 }
368
369 reserved += size;
370 if (hugetlb_cma_percent_in_node[nid])
371 pr_info("hugetlb_cma: reserved %lu MiB (%u%%) on node %d\n",
372 size / SZ_1M, hugetlb_cma_percent_in_node[nid], nid);
373 else
374 pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
375 size / SZ_1M, nid);
376
377 if (reserved >= hugetlb_cma_size)
378 break;
379 }
380
381 if (!reserved)
382 /*
383 * hugetlb_cma_size is used to determine if allocations from
384 * cma are possible. Set to zero if no cma regions are set up.
385 */
386 hugetlb_cma_size = 0;
387}
388
389bool hugetlb_cma_exclusive_alloc(void)
390{
391 return hugetlb_cma_only;
392}
393
394unsigned long __init hugetlb_cma_total_size(void)
395{
396 return hugetlb_cma_size;
397}
398
399void __init hugetlb_cma_validate_params(void)
400{
401 if (!hugetlb_cma_size)
402 hugetlb_cma_only = false;
403}
404
405bool __init hugetlb_early_cma(struct hstate *h)
406{
407 if (arch_has_huge_bootmem_alloc())
408 return false;
409
410 return hstate_is_gigantic(h) && hugetlb_cma_only;
411}