projects/linux·blob master · refs

2529 lines · 70.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/memory_hotplug.c
4 *
5 * Copyright (C)
6 */
7
8#include <linux/stddef.h>
9#include <linux/mm.h>
10#include <linux/sched/signal.h>
11#include <linux/swap.h>
12#include <linux/interrupt.h>
13#include <linux/pagemap.h>
14#include <linux/compiler.h>
15#include <linux/export.h>
16#include <linux/writeback.h>
17#include <linux/slab.h>
18#include <linux/sysctl.h>
19#include <linux/cpu.h>
20#include <linux/memory.h>
21#include <linux/memremap.h>
22#include <linux/memory_hotplug.h>
23#include <linux/vmalloc.h>
24#include <linux/ioport.h>
25#include <linux/delay.h>
26#include <linux/migrate.h>
27#include <linux/page-isolation.h>
28#include <linux/pfn.h>
29#include <linux/suspend.h>
30#include <linux/mm_inline.h>
31#include <linux/firmware-map.h>
32#include <linux/stop_machine.h>
33#include <linux/hugetlb.h>
34#include <linux/memblock.h>
35#include <linux/compaction.h>
36#include <linux/rmap.h>
37#include <linux/module.h>
38#include <linux/node.h>
39
40#include <asm/tlbflush.h>
41
42#include "internal.h"
43#include "mm_init.h"
44#include "page_alloc.h"
45#include "shuffle.h"
46
47enum {
48 MEMMAP_ON_MEMORY_DISABLE = 0,
49 MEMMAP_ON_MEMORY_ENABLE,
50 MEMMAP_ON_MEMORY_FORCE,
51};
52
53static int memmap_mode __read_mostly = MEMMAP_ON_MEMORY_DISABLE;
54
55static inline unsigned long memory_block_memmap_size(void)
56{
57 return PHYS_PFN(memory_block_size_bytes()) * sizeof(struct page);
58}
59
60static inline unsigned long memory_block_memmap_on_memory_pages(void)
61{
62 unsigned long nr_pages = PFN_UP(memory_block_memmap_size());
63
64 /*
65 * In "forced" memmap_on_memory mode, we add extra pages to align the
66 * vmemmap size to cover full pageblocks. That way, we can add memory
67 * even if the vmemmap size is not properly aligned, however, we might waste
68 * memory.
69 */
70 if (memmap_mode == MEMMAP_ON_MEMORY_FORCE)
71 return pageblock_align(nr_pages);
72 return nr_pages;
73}
74
75#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY
76/*
77 * memory_hotplug.memmap_on_memory parameter
78 */
79static int set_memmap_mode(const char *val, const struct kernel_param *kp)
80{
81 int ret, mode;
82 bool enabled;
83
84 if (sysfs_streq(val, "force") || sysfs_streq(val, "FORCE")) {
85 mode = MEMMAP_ON_MEMORY_FORCE;
86 } else {
87 ret = kstrtobool(val, &enabled);
88 if (ret < 0)
89 return ret;
90 if (enabled)
91 mode = MEMMAP_ON_MEMORY_ENABLE;
92 else
93 mode = MEMMAP_ON_MEMORY_DISABLE;
94 }
95 *((int *)kp->arg) = mode;
96 if (mode == MEMMAP_ON_MEMORY_FORCE) {
97 unsigned long memmap_pages = memory_block_memmap_on_memory_pages();
98
99 pr_info_once("Memory hotplug will waste %ld pages in each memory block\n",
100 memmap_pages - PFN_UP(memory_block_memmap_size()));
101 }
102 return 0;
103}
104
105static int get_memmap_mode(char *buffer, const struct kernel_param *kp)
106{
107 int mode = *((int *)kp->arg);
108
109 if (mode == MEMMAP_ON_MEMORY_FORCE)
110 return sprintf(buffer, "force\n");
111 return sprintf(buffer, "%c\n", mode ? 'Y' : 'N');
112}
113
114static const struct kernel_param_ops memmap_mode_ops = {
115 .set = set_memmap_mode,
116 .get = get_memmap_mode,
117};
118module_param_cb(memmap_on_memory, &memmap_mode_ops, &memmap_mode, 0444);
119MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug\n"
120 "With value \"force\" it could result in memory wastage due "
121 "to memmap size limitations (Y/N/force)");
122
123static inline bool mhp_memmap_on_memory(void)
124{
125 return memmap_mode != MEMMAP_ON_MEMORY_DISABLE;
126}
127#else
128static inline bool mhp_memmap_on_memory(void)
129{
130 return false;
131}
132#endif
133
134enum {
135 ONLINE_POLICY_CONTIG_ZONES = 0,
136 ONLINE_POLICY_AUTO_MOVABLE,
137};
138
139static const char * const online_policy_to_str[] = {
140 [ONLINE_POLICY_CONTIG_ZONES] = "contig-zones",
141 [ONLINE_POLICY_AUTO_MOVABLE] = "auto-movable",
142};
143
144static int set_online_policy(const char *val, const struct kernel_param *kp)
145{
146 int ret = sysfs_match_string(online_policy_to_str, val);
147
148 if (ret < 0)
149 return ret;
150 *((int *)kp->arg) = ret;
151 return 0;
152}
153
154static int get_online_policy(char *buffer, const struct kernel_param *kp)
155{
156 return sprintf(buffer, "%s\n", online_policy_to_str[*((int *)kp->arg)]);
157}
158
159/*
160 * memory_hotplug.online_policy: configure online behavior when onlining without
161 * specifying a zone (MMOP_ONLINE)
162 *
163 * "contig-zones": keep zone contiguous
164 * "auto-movable": online memory to ZONE_MOVABLE if the configuration
165 * (auto_movable_ratio, auto_movable_numa_aware) allows for it
166 */
167static int online_policy __read_mostly = ONLINE_POLICY_CONTIG_ZONES;
168static const struct kernel_param_ops online_policy_ops = {
169 .set = set_online_policy,
170 .get = get_online_policy,
171};
172module_param_cb(online_policy, &online_policy_ops, &online_policy, 0644);
173MODULE_PARM_DESC(online_policy,
174 "Set the online policy (\"contig-zones\", \"auto-movable\") "
175 "Default: \"contig-zones\"");
176
177/*
178 * memory_hotplug.auto_movable_ratio: specify maximum MOVABLE:KERNEL ratio
179 *
180 * The ratio represent an upper limit and the kernel might decide to not
181 * online some memory to ZONE_MOVABLE -- e.g., because hotplugged KERNEL memory
182 * doesn't allow for more MOVABLE memory.
183 */
184static unsigned int auto_movable_ratio __read_mostly = 301;
185module_param(auto_movable_ratio, uint, 0644);
186MODULE_PARM_DESC(auto_movable_ratio,
187 "Set the maximum ratio of MOVABLE:KERNEL memory in the system "
188 "in percent for \"auto-movable\" online policy. Default: 301");
189
190/*
191 * memory_hotplug.auto_movable_numa_aware: consider numa node stats
192 */
193#ifdef CONFIG_NUMA
194static bool auto_movable_numa_aware __read_mostly = true;
195module_param(auto_movable_numa_aware, bool, 0644);
196MODULE_PARM_DESC(auto_movable_numa_aware,
197 "Consider numa node stats in addition to global stats in "
198 "\"auto-movable\" online policy. Default: true");
199#endif /* CONFIG_NUMA */
200
201/*
202 * online_page_callback contains pointer to current page onlining function.
203 * Initially it is generic_online_page(). If it is required it could be
204 * changed by calling set_online_page_callback() for callback registration
205 * and restore_online_page_callback() for generic callback restore.
206 */
207
208static online_page_callback_t online_page_callback = generic_online_page;
209static DEFINE_MUTEX(online_page_callback_lock);
210
211DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock);
212
213void get_online_mems(void)
214{
215 percpu_down_read(&mem_hotplug_lock);
216}
217
218void put_online_mems(void)
219{
220 percpu_up_read(&mem_hotplug_lock);
221}
222
223bool movable_node_enabled = false;
224
225static int mhp_default_online_type = -1;
226enum mmop mhp_get_default_online_type(void)
227{
228 if (mhp_default_online_type >= 0)
229 return mhp_default_online_type;
230
231 if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_OFFLINE))
232 mhp_default_online_type = MMOP_OFFLINE;
233 else if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_AUTO))
234 mhp_default_online_type = MMOP_ONLINE;
235 else if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_KERNEL))
236 mhp_default_online_type = MMOP_ONLINE_KERNEL;
237 else if (IS_ENABLED(CONFIG_MHP_DEFAULT_ONLINE_TYPE_ONLINE_MOVABLE))
238 mhp_default_online_type = MMOP_ONLINE_MOVABLE;
239 else
240 mhp_default_online_type = MMOP_OFFLINE;
241
242 return mhp_default_online_type;
243}
244EXPORT_SYMBOL_GPL(mhp_get_default_online_type);
245
246void mhp_set_default_online_type(enum mmop online_type)
247{
248 mhp_default_online_type = online_type;
249}
250
251static int __init setup_memhp_default_state(char *str)
252{
253 const int online_type = mhp_online_type_from_str(str);
254
255 if (online_type >= 0)
256 mhp_default_online_type = online_type;
257
258 return 1;
259}
260__setup("memhp_default_state=", setup_memhp_default_state);
261
262void mem_hotplug_begin(void)
263{
264 cpus_read_lock();
265 percpu_down_write(&mem_hotplug_lock);
266}
267
268void mem_hotplug_done(void)
269{
270 percpu_up_write(&mem_hotplug_lock);
271 cpus_read_unlock();
272}
273
274u64 max_mem_size = U64_MAX;
275
276/* add this memory to iomem resource */
277static struct resource *register_memory_resource(u64 start, u64 size,
278 const char *resource_name)
279{
280 struct resource *res;
281 unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
282
283 if (strcmp(resource_name, "System RAM"))
284 flags |= IORESOURCE_SYSRAM_DRIVER_MANAGED;
285
286 if (!mhp_range_allowed(start, size, true))
287 return ERR_PTR(-E2BIG);
288
289 /*
290 * Make sure value parsed from 'mem=' only restricts memory adding
291 * while booting, so that memory hotplug won't be impacted. Please
292 * refer to document of 'mem=' in kernel-parameters.txt for more
293 * details.
294 */
295 if (start + size > max_mem_size && system_state < SYSTEM_RUNNING)
296 return ERR_PTR(-E2BIG);
297
298 /*
299 * Request ownership of the new memory range. This might be
300 * a child of an existing resource that was present but
301 * not marked as busy.
302 */
303 res = __request_region(&iomem_resource, start, size,
304 resource_name, flags);
305
306 if (!res) {
307 pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",
308 start, start + size);
309 return ERR_PTR(-EEXIST);
310 }
311 return res;
312}
313
314static void release_memory_resource(struct resource *res)
315{
316 if (!res)
317 return;
318 release_resource(res);
319 kfree(res);
320}
321
322static int check_pfn_span(unsigned long pfn, unsigned long nr_pages)
323{
324 /*
325 * Disallow all operations smaller than a sub-section.
326 * Note that check_hotplug_memory_range() enforces a larger
327 * memory_block_size_bytes() granularity for memory that will be marked
328 * online, so this check should only fire for direct
329 * arch_{add,remove}_memory() users outside of add_memory_resource().
330 */
331 if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION))
332 return -EINVAL;
333 return 0;
334}
335
336/*
337 * Return page for the valid pfn only if the page is online. All pfn
338 * walkers which rely on the fully initialized page->flags and others
339 * should use this rather than pfn_valid && pfn_to_page
340 */
341struct page *pfn_to_online_page(unsigned long pfn)
342{
343 unsigned long nr = pfn_to_section_nr(pfn);
344 struct dev_pagemap *pgmap;
345 struct mem_section *ms;
346
347 if (nr >= NR_MEM_SECTIONS)
348 return NULL;
349
350 ms = __nr_to_section(nr);
351 if (!online_section(ms))
352 return NULL;
353
354 /*
355 * Save some code text when online_section() +
356 * pfn_section_valid() are sufficient.
357 */
358 if (IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) && !pfn_valid(pfn))
359 return NULL;
360
361 if (!pfn_section_valid(ms, pfn))
362 return NULL;
363
364 if (!online_device_section(ms))
365 return pfn_to_page(pfn);
366
367 /*
368 * Slowpath: when ZONE_DEVICE collides with
369 * ZONE_{NORMAL,MOVABLE} within the same section some pfns in
370 * the section may be 'offline' but 'valid'. Only
371 * get_dev_pagemap() can determine sub-section online status.
372 */
373 pgmap = get_dev_pagemap(pfn);
374 put_dev_pagemap(pgmap);
375
376 /* The presence of a pgmap indicates ZONE_DEVICE offline pfn */
377 if (pgmap)
378 return NULL;
379
380 return pfn_to_page(pfn);
381}
382EXPORT_SYMBOL_GPL(pfn_to_online_page);
383
384int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages,
385 struct mhp_params *params)
386{
387 const unsigned long end_pfn = pfn + nr_pages;
388 unsigned long cur_nr_pages;
389 int err;
390 struct vmem_altmap *altmap = params->altmap;
391
392 if (WARN_ON_ONCE(!pgprot_val(params->pgprot)))
393 return -EINVAL;
394
395 VM_BUG_ON(!mhp_range_allowed(PFN_PHYS(pfn), nr_pages * PAGE_SIZE, false));
396
397 if (altmap) {
398 /*
399 * Validate altmap is within bounds of the total request
400 */
401 if (altmap->base_pfn != pfn
402 || vmem_altmap_offset(altmap) > nr_pages) {
403 pr_warn_once("memory add fail, invalid altmap\n");
404 return -EINVAL;
405 }
406 altmap->alloc = 0;
407 }
408
409 if (check_pfn_span(pfn, nr_pages)) {
410 WARN(1, "Misaligned %s start: %#lx end: %#lx\n", __func__, pfn, pfn + nr_pages - 1);
411 return -EINVAL;
412 }
413
414 for (; pfn < end_pfn; pfn += cur_nr_pages) {
415 /* Select all remaining pages up to the next section boundary */
416 cur_nr_pages = min(end_pfn - pfn,
417 SECTION_ALIGN_UP(pfn + 1) - pfn);
418 err = sparse_add_section(nid, pfn, cur_nr_pages, altmap,
419 params->pgmap);
420 if (err)
421 break;
422 cond_resched();
423 }
424 vmemmap_populate_print_last();
425 return err;
426}
427
428/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
429static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,
430 unsigned long start_pfn,
431 unsigned long end_pfn)
432{
433 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {
434 if (unlikely(!pfn_to_online_page(start_pfn)))
435 continue;
436
437 if (unlikely(pfn_to_nid(start_pfn) != nid))
438 continue;
439
440 if (zone != page_zone(pfn_to_page(start_pfn)))
441 continue;
442
443 return start_pfn;
444 }
445
446 return 0;
447}
448
449/* find the biggest valid pfn in the range [start_pfn, end_pfn). */
450static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,
451 unsigned long start_pfn,
452 unsigned long end_pfn)
453{
454 unsigned long pfn;
455
456 /* pfn is the end pfn of a memory section. */
457 pfn = end_pfn - 1;
458 for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) {
459 if (unlikely(!pfn_to_online_page(pfn)))
460 continue;
461
462 if (unlikely(pfn_to_nid(pfn) != nid))
463 continue;
464
465 if (zone != page_zone(pfn_to_page(pfn)))
466 continue;
467
468 return pfn;
469 }
470
471 return 0;
472}
473
474static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
475 unsigned long end_pfn)
476{
477 unsigned long pfn;
478 int nid = zone_to_nid(zone);
479
480 if (zone->zone_start_pfn == start_pfn) {
481 /*
482 * If the section is smallest section in the zone, it need
483 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
484 * In this case, we find second smallest valid mem_section
485 * for shrinking zone.
486 */
487 pfn = find_smallest_section_pfn(nid, zone, end_pfn,
488 zone_end_pfn(zone));
489 if (pfn) {
490 zone->spanned_pages = zone_end_pfn(zone) - pfn;
491 zone->zone_start_pfn = pfn;
492 } else {
493 zone->zone_start_pfn = 0;
494 zone->spanned_pages = 0;
495 }
496 } else if (zone_end_pfn(zone) == end_pfn) {
497 /*
498 * If the section is biggest section in the zone, it need
499 * shrink zone->spanned_pages.
500 * In this case, we find second biggest valid mem_section for
501 * shrinking zone.
502 */
503 pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn,
504 start_pfn);
505 if (pfn)
506 zone->spanned_pages = pfn - zone->zone_start_pfn + 1;
507 else {
508 zone->zone_start_pfn = 0;
509 zone->spanned_pages = 0;
510 }
511 }
512}
513
514static void update_pgdat_span(struct pglist_data *pgdat)
515{
516 unsigned long node_start_pfn = 0, node_end_pfn = 0;
517 struct zone *zone;
518
519 for (zone = pgdat->node_zones;
520 zone < pgdat->node_zones + MAX_NR_ZONES; zone++) {
521 unsigned long end_pfn = zone_end_pfn(zone);
522
523 /* No need to lock the zones, they can't change. */
524 if (!zone->spanned_pages)
525 continue;
526 if (!node_end_pfn) {
527 node_start_pfn = zone->zone_start_pfn;
528 node_end_pfn = end_pfn;
529 continue;
530 }
531
532 if (end_pfn > node_end_pfn)
533 node_end_pfn = end_pfn;
534 if (zone->zone_start_pfn < node_start_pfn)
535 node_start_pfn = zone->zone_start_pfn;
536 }
537
538 pgdat->node_start_pfn = node_start_pfn;
539 pgdat->node_spanned_pages = node_end_pfn - node_start_pfn;
540}
541
542void remove_pfn_range_from_zone(struct zone *zone,
543 unsigned long start_pfn,
544 unsigned long nr_pages)
545{
546 const unsigned long end_pfn = start_pfn + nr_pages;
547 struct pglist_data *pgdat = zone->zone_pgdat;
548 unsigned long pfn, cur_nr_pages;
549
550 /* Poison struct pages because they are now uninitialized again. */
551 for (pfn = start_pfn; pfn < end_pfn; pfn += cur_nr_pages) {
552 cond_resched();
553
554 /* Select all remaining pages up to the next section boundary */
555 cur_nr_pages =
556 min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn);
557 page_init_poison(pfn_to_page(pfn),
558 sizeof(struct page) * cur_nr_pages);
559 }
560
561 /*
562 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So
563 * we will not try to shrink the zones - which is okay as
564 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way.
565 */
566 if (zone_is_zone_device(zone))
567 return;
568
569 clear_zone_contiguous(zone);
570
571 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
572 update_pgdat_span(pgdat);
573
574 set_zone_contiguous(zone);
575}
576
577/**
578 * __remove_pages() - remove sections of pages
579 * @pfn: starting pageframe (must be aligned to start of a section)
580 * @nr_pages: number of pages to remove (must be multiple of section size)
581 * @altmap: alternative device page map or %NULL if default memmap is used
582 * @pgmap: device page map or %NULL if not ZONE_DEVICE
583 *
584 * Generic helper function to remove section mappings and sysfs entries
585 * for the section of the memory we are removing. Caller needs to make
586 * sure that pages are marked reserved and zones are adjust properly by
587 * calling offline_pages().
588 */
589void __remove_pages(unsigned long pfn, unsigned long nr_pages,
590 struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
591{
592 const unsigned long end_pfn = pfn + nr_pages;
593 unsigned long cur_nr_pages;
594
595 if (check_pfn_span(pfn, nr_pages)) {
596 WARN(1, "Misaligned %s start: %#lx end: %#lx\n", __func__, pfn, pfn + nr_pages - 1);
597 return;
598 }
599
600 for (; pfn < end_pfn; pfn += cur_nr_pages) {
601 cond_resched();
602 /* Select all remaining pages up to the next section boundary */
603 cur_nr_pages = min(end_pfn - pfn,
604 SECTION_ALIGN_UP(pfn + 1) - pfn);
605 sparse_remove_section(pfn, cur_nr_pages, altmap, pgmap);
606 }
607}
608
609int set_online_page_callback(online_page_callback_t callback)
610{
611 int rc = -EINVAL;
612
613 get_online_mems();
614 mutex_lock(&online_page_callback_lock);
615
616 if (online_page_callback == generic_online_page) {
617 online_page_callback = callback;
618 rc = 0;
619 }
620
621 mutex_unlock(&online_page_callback_lock);
622 put_online_mems();
623
624 return rc;
625}
626EXPORT_SYMBOL_GPL(set_online_page_callback);
627
628int restore_online_page_callback(online_page_callback_t callback)
629{
630 int rc = -EINVAL;
631
632 get_online_mems();
633 mutex_lock(&online_page_callback_lock);
634
635 if (online_page_callback == callback) {
636 online_page_callback = generic_online_page;
637 rc = 0;
638 }
639
640 mutex_unlock(&online_page_callback_lock);
641 put_online_mems();
642
643 return rc;
644}
645EXPORT_SYMBOL_GPL(restore_online_page_callback);
646
647/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
648void generic_online_page(struct page *page, unsigned int order)
649{
650 __free_pages_core(page, order, MEMINIT_HOTPLUG);
651}
652EXPORT_SYMBOL_GPL(generic_online_page);
653
654static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages)
655{
656 const unsigned long end_pfn = start_pfn + nr_pages;
657 unsigned long pfn;
658
659 /*
660 * Online the pages in MAX_PAGE_ORDER aligned chunks. The callback might
661 * decide to not expose all pages to the buddy (e.g., expose them
662 * later). We account all pages as being online and belonging to this
663 * zone ("present").
664 * When using memmap_on_memory, the range might not be aligned to
665 * MAX_ORDER_NR_PAGES - 1, but pageblock aligned. __ffs() will detect
666 * this and the first chunk to online will be pageblock_nr_pages.
667 */
668 for (pfn = start_pfn; pfn < end_pfn;) {
669 struct page *page = pfn_to_page(pfn);
670 int order;
671
672 /*
673 * Free to online pages in the largest chunks alignment allows.
674 *
675 * __ffs() behaviour is undefined for 0. start == 0 is
676 * MAX_PAGE_ORDER-aligned, Set order to MAX_PAGE_ORDER for
677 * the case.
678 */
679 if (pfn)
680 order = min_t(int, MAX_PAGE_ORDER, __ffs(pfn));
681 else
682 order = MAX_PAGE_ORDER;
683
684 /*
685 * Exposing the page to the buddy by freeing can cause
686 * issues with debug_pagealloc enabled: some archs don't
687 * like double-unmappings. So treat them like any pages that
688 * were allocated from the buddy.
689 */
690 debug_pagealloc_map_pages(page, 1 << order);
691 (*online_page_callback)(page, order);
692 pfn += (1UL << order);
693 }
694
695 /* mark all involved sections as online */
696 online_mem_sections(start_pfn, end_pfn);
697}
698
699static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,
700 unsigned long nr_pages)
701{
702 unsigned long old_end_pfn = zone_end_pfn(zone);
703
704 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
705 zone->zone_start_pfn = start_pfn;
706
707 zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;
708}
709
710static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn,
711 unsigned long nr_pages)
712{
713 unsigned long old_end_pfn = pgdat_end_pfn(pgdat);
714
715 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
716 pgdat->node_start_pfn = start_pfn;
717
718 pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn;
719
720}
721
722#ifdef CONFIG_ZONE_DEVICE
723static void section_taint_zone_device(unsigned long pfn)
724{
725 struct mem_section *ms = __pfn_to_section(pfn);
726
727 ms->section_mem_map |= SECTION_TAINT_ZONE_DEVICE;
728}
729#else
730static inline void section_taint_zone_device(unsigned long pfn)
731{
732}
733#endif
734
735/*
736 * Associate the pfn range with the given zone, initializing the memmaps
737 * and resizing the pgdat/zone data to span the added pages. After this
738 * call, all affected pages are PageOffline().
739 *
740 * All aligned pageblocks are initialized to the specified migratetype
741 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
742 * zone stats (e.g., nr_isolate_pageblock) are touched.
743 */
744void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
745 unsigned long nr_pages,
746 struct vmem_altmap *altmap, int migratetype,
747 bool isolate_pageblock)
748{
749 struct pglist_data *pgdat = zone->zone_pgdat;
750 int nid = pgdat->node_id;
751
752 clear_zone_contiguous(zone);
753
754 if (zone_is_empty(zone))
755 init_currently_empty_zone(zone, start_pfn, nr_pages);
756 resize_zone_range(zone, start_pfn, nr_pages);
757 resize_pgdat_range(pgdat, start_pfn, nr_pages);
758
759 /*
760 * Subsection population requires care in pfn_to_online_page().
761 * Set the taint to enable the slow path detection of
762 * ZONE_DEVICE pages in an otherwise ZONE_{NORMAL,MOVABLE}
763 * section.
764 */
765 if (zone_is_zone_device(zone)) {
766 if (!IS_ALIGNED(start_pfn, PAGES_PER_SECTION))
767 section_taint_zone_device(start_pfn);
768 if (!IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION))
769 section_taint_zone_device(start_pfn + nr_pages);
770 }
771
772 /*
773 * TODO now we have a visible range of pages which are not associated
774 * with their zone properly. Not nice but set_pfnblock_migratetype()
775 * expects the zone spans the pfn range. All the pages in the range
776 * are reserved so nobody should be touching them so we should be safe
777 */
778 memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0,
779 MEMINIT_HOTPLUG, altmap, migratetype,
780 isolate_pageblock);
781
782 set_zone_contiguous(zone);
783}
784
785struct auto_movable_stats {
786 unsigned long kernel_early_pages;
787 unsigned long movable_pages;
788};
789
790static void auto_movable_stats_account_zone(struct auto_movable_stats *stats,
791 struct zone *zone)
792{
793 if (zone_idx(zone) == ZONE_MOVABLE) {
794 stats->movable_pages += zone->present_pages;
795 } else {
796 stats->kernel_early_pages += zone->present_early_pages;
797#ifdef CONFIG_CMA
798 /*
799 * CMA pages (never on hotplugged memory) behave like
800 * ZONE_MOVABLE.
801 */
802 stats->movable_pages += zone->cma_pages;
803 stats->kernel_early_pages -= zone->cma_pages;
804#endif /* CONFIG_CMA */
805 }
806}
807struct auto_movable_group_stats {
808 unsigned long movable_pages;
809 unsigned long req_kernel_early_pages;
810};
811
812static int auto_movable_stats_account_group(struct memory_group *group,
813 void *arg)
814{
815 const int ratio = READ_ONCE(auto_movable_ratio);
816 struct auto_movable_group_stats *stats = arg;
817 long pages;
818
819 /*
820 * We don't support modifying the config while the auto-movable online
821 * policy is already enabled. Just avoid the division by zero below.
822 */
823 if (!ratio)
824 return 0;
825
826 /*
827 * Calculate how many early kernel pages this group requires to
828 * satisfy the configured zone ratio.
829 */
830 pages = group->present_movable_pages * 100 / ratio;
831 pages -= group->present_kernel_pages;
832
833 if (pages > 0)
834 stats->req_kernel_early_pages += pages;
835 stats->movable_pages += group->present_movable_pages;
836 return 0;
837}
838
839static bool auto_movable_can_online_movable(int nid, struct memory_group *group,
840 unsigned long nr_pages)
841{
842 unsigned long kernel_early_pages, movable_pages;
843 struct auto_movable_group_stats group_stats = {};
844 struct auto_movable_stats stats = {};
845 struct zone *zone;
846 int i;
847
848 /* Walk all relevant zones and collect MOVABLE vs. KERNEL stats. */
849 if (nid == NUMA_NO_NODE) {
850 /* TODO: cache values */
851 for_each_populated_zone(zone)
852 auto_movable_stats_account_zone(&stats, zone);
853 } else {
854 for (i = 0; i < MAX_NR_ZONES; i++) {
855 pg_data_t *pgdat = NODE_DATA(nid);
856
857 zone = pgdat->node_zones + i;
858 if (populated_zone(zone))
859 auto_movable_stats_account_zone(&stats, zone);
860 }
861 }
862
863 kernel_early_pages = stats.kernel_early_pages;
864 movable_pages = stats.movable_pages;
865
866 /*
867 * Kernel memory inside dynamic memory group allows for more MOVABLE
868 * memory within the same group. Remove the effect of all but the
869 * current group from the stats.
870 */
871 walk_dynamic_memory_groups(nid, auto_movable_stats_account_group,
872 group, &group_stats);
873 if (kernel_early_pages <= group_stats.req_kernel_early_pages)
874 return false;
875 kernel_early_pages -= group_stats.req_kernel_early_pages;
876 movable_pages -= group_stats.movable_pages;
877
878 if (group && group->is_dynamic)
879 kernel_early_pages += group->present_kernel_pages;
880
881 /*
882 * Test if we could online the given number of pages to ZONE_MOVABLE
883 * and still stay in the configured ratio.
884 */
885 movable_pages += nr_pages;
886 return movable_pages <= (auto_movable_ratio * kernel_early_pages) / 100;
887}
888
889/*
890 * Returns a default kernel memory zone for the given pfn range.
891 * If no kernel zone covers this pfn range it will automatically go
892 * to the ZONE_NORMAL.
893 */
894static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,
895 unsigned long nr_pages)
896{
897 struct pglist_data *pgdat = NODE_DATA(nid);
898 int zid;
899
900 for (zid = 0; zid < ZONE_NORMAL; zid++) {
901 struct zone *zone = &pgdat->node_zones[zid];
902
903 if (zone_intersects(zone, start_pfn, nr_pages))
904 return zone;
905 }
906
907 return &pgdat->node_zones[ZONE_NORMAL];
908}
909
910/*
911 * Determine to which zone to online memory dynamically based on user
912 * configuration and system stats. We care about the following ratio:
913 *
914 * MOVABLE : KERNEL
915 *
916 * Whereby MOVABLE is memory in ZONE_MOVABLE and KERNEL is memory in
917 * one of the kernel zones. CMA pages inside one of the kernel zones really
918 * behaves like ZONE_MOVABLE, so we treat them accordingly.
919 *
920 * We don't allow for hotplugged memory in a KERNEL zone to increase the
921 * amount of MOVABLE memory we can have, so we end up with:
922 *
923 * MOVABLE : KERNEL_EARLY
924 *
925 * Whereby KERNEL_EARLY is memory in one of the kernel zones, available since
926 * boot. We base our calculation on KERNEL_EARLY internally, because:
927 *
928 * a) Hotplugged memory in one of the kernel zones can sometimes still get
929 * hotunplugged, especially when hot(un)plugging individual memory blocks.
930 * There is no coordination across memory devices, therefore "automatic"
931 * hotunplugging, as implemented in hypervisors, could result in zone
932 * imbalances.
933 * b) Early/boot memory in one of the kernel zones can usually not get
934 * hotunplugged again (e.g., no firmware interface to unplug, fragmented
935 * with unmovable allocations). While there are corner cases where it might
936 * still work, it is barely relevant in practice.
937 *
938 * Exceptions are dynamic memory groups, which allow for more MOVABLE
939 * memory within the same memory group -- because in that case, there is
940 * coordination within the single memory device managed by a single driver.
941 *
942 * We rely on "present pages" instead of "managed pages", as the latter is
943 * highly unreliable and dynamic in virtualized environments, and does not
944 * consider boot time allocations. For example, memory ballooning adjusts the
945 * managed pages when inflating/deflating the balloon, and balloon page
946 * migration can even migrate inflated pages between zones.
947 *
948 * Using "present pages" is better but some things to keep in mind are:
949 *
950 * a) Some memblock allocations, such as for the crashkernel area, are
951 * effectively unused by the kernel, yet they account to "present pages".
952 * Fortunately, these allocations are comparatively small in relevant setups
953 * (e.g., fraction of system memory).
954 * b) Some hotplugged memory blocks in virtualized environments, especially
955 * hotplugged by virtio-mem, look like they are completely present, however,
956 * only parts of the memory block are actually currently usable.
957 * "present pages" is an upper limit that can get reached at runtime. As
958 * we base our calculations on KERNEL_EARLY, this is not an issue.
959 */
960static struct zone *auto_movable_zone_for_pfn(int nid,
961 struct memory_group *group,
962 unsigned long pfn,
963 unsigned long nr_pages)
964{
965 unsigned long online_pages = 0, max_pages, end_pfn;
966 struct page *page;
967
968 if (!auto_movable_ratio)
969 goto kernel_zone;
970
971 if (group && !group->is_dynamic) {
972 max_pages = group->s.max_pages;
973 online_pages = group->present_movable_pages;
974
975 /* If anything is !MOVABLE online the rest !MOVABLE. */
976 if (group->present_kernel_pages)
977 goto kernel_zone;
978 } else if (!group || group->d.unit_pages == nr_pages) {
979 max_pages = nr_pages;
980 } else {
981 max_pages = group->d.unit_pages;
982 /*
983 * Take a look at all online sections in the current unit.
984 * We can safely assume that all pages within a section belong
985 * to the same zone, because dynamic memory groups only deal
986 * with hotplugged memory.
987 */
988 pfn = ALIGN_DOWN(pfn, group->d.unit_pages);
989 end_pfn = pfn + group->d.unit_pages;
990 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
991 page = pfn_to_online_page(pfn);
992 if (!page)
993 continue;
994 /* If anything is !MOVABLE online the rest !MOVABLE. */
995 if (!is_zone_movable_page(page))
996 goto kernel_zone;
997 online_pages += PAGES_PER_SECTION;
998 }
999 }
1000
1001 /*
1002 * Online MOVABLE if we could *currently* online all remaining parts
1003 * MOVABLE. We expect to (add+) online them immediately next, so if
1004 * nobody interferes, all will be MOVABLE if possible.
1005 */
1006 nr_pages = max_pages - online_pages;
1007 if (!auto_movable_can_online_movable(NUMA_NO_NODE, group, nr_pages))
1008 goto kernel_zone;
1009
1010#ifdef CONFIG_NUMA
1011 if (auto_movable_numa_aware &&
1012 !auto_movable_can_online_movable(nid, group, nr_pages))
1013 goto kernel_zone;
1014#endif /* CONFIG_NUMA */
1015
1016 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
1017kernel_zone:
1018 return default_kernel_zone_for_pfn(nid, pfn, nr_pages);
1019}
1020
1021static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,
1022 unsigned long nr_pages)
1023{
1024 struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,
1025 nr_pages);
1026 struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
1027 bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages);
1028 bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages);
1029
1030 /*
1031 * We inherit the existing zone in a simple case where zones do not
1032 * overlap in the given range
1033 */
1034 if (in_kernel ^ in_movable)
1035 return (in_kernel) ? kernel_zone : movable_zone;
1036
1037 /*
1038 * If the range doesn't belong to any zone or two zones overlap in the
1039 * given range then we use movable zone only if movable_node is
1040 * enabled because we always online to a kernel zone by default.
1041 */
1042 return movable_node_enabled ? movable_zone : kernel_zone;
1043}
1044
1045struct zone *zone_for_pfn_range(enum mmop online_type, int nid,
1046 struct memory_group *group, unsigned long start_pfn,
1047 unsigned long nr_pages)
1048{
1049 if (online_type == MMOP_ONLINE_KERNEL)
1050 return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages);
1051
1052 if (online_type == MMOP_ONLINE_MOVABLE)
1053 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
1054
1055 if (online_policy == ONLINE_POLICY_AUTO_MOVABLE)
1056 return auto_movable_zone_for_pfn(nid, group, start_pfn, nr_pages);
1057
1058 return default_zone_for_pfn(nid, start_pfn, nr_pages);
1059}
1060
1061/*
1062 * This function should only be called by memory_block_{online,offline},
1063 * and {online,offline}_pages.
1064 */
1065void adjust_present_page_count(struct page *page, struct memory_group *group,
1066 long nr_pages)
1067{
1068 struct zone *zone = page_zone(page);
1069 const bool movable = zone_idx(zone) == ZONE_MOVABLE;
1070
1071 /*
1072 * We only support onlining/offlining/adding/removing of complete
1073 * memory blocks; therefore, either all is either early or hotplugged.
1074 */
1075 if (early_section(__pfn_to_section(page_to_pfn(page))))
1076 zone->present_early_pages += nr_pages;
1077 zone->present_pages += nr_pages;
1078 zone->zone_pgdat->node_present_pages += nr_pages;
1079
1080 if (group && movable)
1081 group->present_movable_pages += nr_pages;
1082 else if (group && !movable)
1083 group->present_kernel_pages += nr_pages;
1084}
1085
1086int mhp_init_memmap_on_memory(unsigned long pfn, unsigned long nr_pages,
1087 struct zone *zone)
1088{
1089 unsigned long end_pfn = pfn + nr_pages;
1090 int ret, i;
1091
1092 ret = kasan_add_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages));
1093 if (ret)
1094 return ret;
1095
1096 move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_UNMOVABLE,
1097 false);
1098
1099 for (i = 0; i < nr_pages; i++) {
1100 struct page *page = pfn_to_page(pfn + i);
1101
1102 __ClearPageOffline(page);
1103 SetPageVmemmapSelfHosted(page);
1104 }
1105
1106 /*
1107 * It might be that the vmemmap_pages fully span sections. If that is
1108 * the case, mark those sections online here as otherwise they will be
1109 * left offline.
1110 */
1111 if (nr_pages >= PAGES_PER_SECTION)
1112 online_mem_sections(pfn, ALIGN_DOWN(end_pfn, PAGES_PER_SECTION));
1113
1114 return ret;
1115}
1116
1117void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages)
1118{
1119 unsigned long end_pfn = pfn + nr_pages;
1120
1121 /*
1122 * It might be that the vmemmap_pages fully span sections. If that is
1123 * the case, mark those sections offline here as otherwise they will be
1124 * left online.
1125 */
1126 if (nr_pages >= PAGES_PER_SECTION)
1127 offline_mem_sections(pfn, ALIGN_DOWN(end_pfn, PAGES_PER_SECTION));
1128
1129 /*
1130 * The pages associated with this vmemmap have been offlined, so
1131 * we can reset its state here.
1132 */
1133 remove_pfn_range_from_zone(page_zone(pfn_to_page(pfn)), pfn, nr_pages);
1134 kasan_remove_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages));
1135}
1136
1137/*
1138 * Must be called with mem_hotplug_lock in write mode.
1139 */
1140int online_pages(unsigned long pfn, unsigned long nr_pages,
1141 struct zone *zone, struct memory_group *group)
1142{
1143 struct memory_notify mem_arg = {
1144 .start_pfn = pfn,
1145 .nr_pages = nr_pages,
1146 };
1147 struct node_notify node_arg = {
1148 .nid = NUMA_NO_NODE,
1149 };
1150 const int nid = zone_to_nid(zone);
1151 int need_zonelists_rebuild = 0;
1152 unsigned long flags;
1153 int ret;
1154
1155 /*
1156 * {on,off}lining is constrained to full memory sections (or more
1157 * precisely to memory blocks from the user space POV).
1158 * memmap_on_memory is an exception because it reserves initial part
1159 * of the physical memory space for vmemmaps. That space is pageblock
1160 * aligned.
1161 */
1162 if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(pfn) ||
1163 !IS_ALIGNED(pfn + nr_pages, PAGES_PER_SECTION)))
1164 return -EINVAL;
1165
1166
1167 /* associate pfn range with the zone */
1168 move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_MOVABLE,
1169 true);
1170
1171 if (!node_state(nid, N_MEMORY)) {
1172 /* Adding memory to the node for the first time */
1173 node_arg.nid = nid;
1174 ret = node_notify(NODE_ADDING_FIRST_MEMORY, &node_arg);
1175 ret = notifier_to_errno(ret);
1176 if (ret)
1177 goto failed_addition;
1178 }
1179
1180 ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
1181 ret = notifier_to_errno(ret);
1182 if (ret)
1183 goto failed_addition;
1184
1185 /*
1186 * Fixup the number of isolated pageblocks before marking the sections
1187 * onlining, such that undo_isolate_page_range() works correctly.
1188 */
1189 spin_lock_irqsave(&zone->lock, flags);
1190 zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages;
1191 spin_unlock_irqrestore(&zone->lock, flags);
1192
1193 /*
1194 * If this zone is not populated, then it is not in zonelist.
1195 * This means the page allocator ignores this zone.
1196 * So, zonelist must be updated after online.
1197 */
1198 if (!populated_zone(zone)) {
1199 need_zonelists_rebuild = 1;
1200 setup_zone_pageset(zone);
1201 }
1202
1203 online_pages_range(pfn, nr_pages);
1204 adjust_present_page_count(pfn_to_page(pfn), group, nr_pages);
1205
1206 if (node_arg.nid >= 0)
1207 node_set_state(nid, N_MEMORY);
1208 /*
1209 * Check whether we are adding normal memory to the node for the first
1210 * time.
1211 */
1212 if (!node_state(nid, N_NORMAL_MEMORY) && zone_idx(zone) <= ZONE_NORMAL)
1213 node_set_state(nid, N_NORMAL_MEMORY);
1214
1215 if (need_zonelists_rebuild)
1216 build_all_zonelists(NULL);
1217
1218 /* Basic onlining is complete, allow allocation of onlined pages. */
1219 undo_isolate_page_range(pfn, pfn + nr_pages);
1220
1221 /*
1222 * Freshly onlined pages aren't shuffled (e.g., all pages are placed to
1223 * the tail of the freelist when undoing isolation). Shuffle the whole
1224 * zone to make sure the just onlined pages are properly distributed
1225 * across the whole freelist - to create an initial shuffle.
1226 */
1227 shuffle_zone(zone);
1228
1229 /* reinitialise watermarks and update pcp limits */
1230 init_per_zone_wmark_min();
1231
1232 kswapd_run(nid);
1233 kcompactd_run(nid);
1234
1235 if (node_arg.nid >= 0)
1236 /* First memory added successfully. Notify consumers. */
1237 node_notify(NODE_ADDED_FIRST_MEMORY, &node_arg);
1238
1239 writeback_set_ratelimit();
1240
1241 memory_notify(MEM_ONLINE, &mem_arg);
1242 return 0;
1243
1244failed_addition:
1245 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
1246 (unsigned long long) pfn << PAGE_SHIFT,
1247 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
1248 memory_notify(MEM_CANCEL_ONLINE, &mem_arg);
1249 if (node_arg.nid != NUMA_NO_NODE)
1250 node_notify(NODE_CANCEL_ADDING_FIRST_MEMORY, &node_arg);
1251 remove_pfn_range_from_zone(zone, pfn, nr_pages);
1252 return ret;
1253}
1254
1255/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
1256static pg_data_t *hotadd_init_pgdat(int nid)
1257{
1258 struct pglist_data *pgdat;
1259
1260 /*
1261 * NODE_DATA is preallocated (free_area_init) but its internal
1262 * state is not allocated completely. Add missing pieces.
1263 * Completely offline nodes stay around and they just need
1264 * reinitialization.
1265 */
1266 pgdat = NODE_DATA(nid);
1267
1268 /* init node's zones as empty zones, we don't have any present pages.*/
1269 if (free_area_init_core_hotplug(pgdat))
1270 return NULL;
1271
1272 /*
1273 * The node we allocated has no zone fallback lists. For avoiding
1274 * to access not-initialized zonelist, build here.
1275 */
1276 build_all_zonelists(pgdat);
1277
1278 return pgdat;
1279}
1280
1281/*
1282 * __try_online_node - online a node if offlined
1283 * @nid: the node ID
1284 * @set_node_online: Whether we want to online the node
1285 * called by cpu_up() to online a node without onlined memory.
1286 *
1287 * Returns:
1288 * 1 -> a new node has been allocated
1289 * 0 -> the node is already online
1290 * -ENOMEM -> the node could not be allocated
1291 */
1292static int __try_online_node(int nid, bool set_node_online)
1293{
1294 pg_data_t *pgdat;
1295 int ret = 1;
1296
1297 if (node_online(nid))
1298 return 0;
1299
1300 pgdat = hotadd_init_pgdat(nid);
1301 if (!pgdat) {
1302 pr_err("Cannot online node %d due to NULL pgdat\n", nid);
1303 ret = -ENOMEM;
1304 goto out;
1305 }
1306
1307 if (set_node_online) {
1308 node_set_online(nid);
1309 ret = register_node(nid);
1310 BUG_ON(ret);
1311 }
1312out:
1313 return ret;
1314}
1315
1316/*
1317 * Users of this function always want to online/register the node
1318 */
1319int try_online_node(int nid)
1320{
1321 int ret;
1322
1323 mem_hotplug_begin();
1324 ret = __try_online_node(nid, true);
1325 mem_hotplug_done();
1326 return ret;
1327}
1328
1329static int check_hotplug_memory_range(u64 start, u64 size)
1330{
1331 /* memory range must be block size aligned */
1332 if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) ||
1333 !IS_ALIGNED(size, memory_block_size_bytes())) {
1334 pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",
1335 memory_block_size_bytes(), start, size);
1336 return -EINVAL;
1337 }
1338
1339 return 0;
1340}
1341
1342static int online_memory_block(struct memory_block *mem, void *arg)
1343{
1344 enum mmop *online_type = arg;
1345
1346 mem->online_type = *online_type;
1347 return device_online(&mem->dev);
1348}
1349
1350#ifndef arch_supports_memmap_on_memory
1351static inline bool arch_supports_memmap_on_memory(unsigned long vmemmap_size)
1352{
1353 /*
1354 * As default, we want the vmemmap to span a complete PMD such that we
1355 * can map the vmemmap using a single PMD if supported by the
1356 * architecture.
1357 */
1358 return IS_ALIGNED(vmemmap_size, PMD_SIZE);
1359}
1360#endif
1361
1362bool mhp_supports_memmap_on_memory(void)
1363{
1364 unsigned long vmemmap_size = memory_block_memmap_size();
1365 unsigned long memmap_pages = memory_block_memmap_on_memory_pages();
1366
1367 /*
1368 * Besides having arch support and the feature enabled at runtime, we
1369 * need a few more assumptions to hold true:
1370 *
1371 * a) The vmemmap pages span complete PMDs: We don't want vmemmap code
1372 * to populate memory from the altmap for unrelated parts (i.e.,
1373 * other memory blocks)
1374 *
1375 * b) The vmemmap pages (and thereby the pages that will be exposed to
1376 * the buddy) have to cover full pageblocks: memory onlining/offlining
1377 * code requires applicable ranges to be page-aligned, for example, to
1378 * set the migratetypes properly.
1379 *
1380 * TODO: Although we have a check here to make sure that vmemmap pages
1381 * fully populate a PMD, it is not the right place to check for
1382 * this. A much better solution involves improving vmemmap code
1383 * to fallback to base pages when trying to populate vmemmap using
1384 * altmap as an alternative source of memory, and we do not exactly
1385 * populate a single PMD.
1386 */
1387 if (!mhp_memmap_on_memory())
1388 return false;
1389
1390 /*
1391 * Make sure the vmemmap allocation is fully contained
1392 * so that we always allocate vmemmap memory from altmap area.
1393 */
1394 if (!IS_ALIGNED(vmemmap_size, PAGE_SIZE))
1395 return false;
1396
1397 /*
1398 * start pfn should be pageblock_nr_pages aligned for correctly
1399 * setting migrate types
1400 */
1401 if (!pageblock_aligned(memmap_pages))
1402 return false;
1403
1404 if (memmap_pages == PHYS_PFN(memory_block_size_bytes()))
1405 /* No effective hotplugged memory doesn't make sense. */
1406 return false;
1407
1408 return arch_supports_memmap_on_memory(vmemmap_size);
1409}
1410EXPORT_SYMBOL_GPL(mhp_supports_memmap_on_memory);
1411
1412static void altmap_free(struct vmem_altmap *altmap)
1413{
1414 WARN_ONCE(altmap->alloc, "Altmap not fully unmapped");
1415 kfree(altmap);
1416}
1417
1418static void remove_memory_blocks_and_altmaps(u64 start, u64 size)
1419{
1420 unsigned long memblock_size = memory_block_size_bytes();
1421 u64 cur_start;
1422
1423 /*
1424 * For memmap_on_memory, the altmaps were added on a per-memblock
1425 * basis; we have to process each individual memory block.
1426 */
1427 for (cur_start = start; cur_start < start + size;
1428 cur_start += memblock_size) {
1429 struct vmem_altmap *altmap = NULL;
1430 struct memory_block *mem;
1431
1432 mem = memory_block_get(phys_to_block_id(cur_start));
1433 if (WARN_ON_ONCE(!mem))
1434 continue;
1435
1436 altmap = mem->altmap;
1437 mem->altmap = NULL;
1438 memory_block_put(mem);
1439
1440 remove_memory_block_devices(cur_start, memblock_size);
1441 arch_remove_memory(cur_start, memblock_size, altmap, NULL);
1442 altmap_free(altmap);
1443 }
1444}
1445
1446static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
1447 u64 start, u64 size)
1448{
1449 unsigned long memblock_size = memory_block_size_bytes();
1450 u64 cur_start;
1451 int ret;
1452
1453 for (cur_start = start; cur_start < start + size;
1454 cur_start += memblock_size) {
1455 struct mhp_params params = { .pgprot =
1456 pgprot_mhp(PAGE_KERNEL) };
1457 struct vmem_altmap mhp_altmap = {
1458 .base_pfn = PHYS_PFN(cur_start),
1459 .end_pfn = PHYS_PFN(cur_start + memblock_size - 1),
1460 };
1461
1462 mhp_altmap.free = memory_block_memmap_on_memory_pages();
1463 params.altmap = kmemdup(&mhp_altmap, sizeof(struct vmem_altmap),
1464 GFP_KERNEL);
1465 if (!params.altmap) {
1466 ret = -ENOMEM;
1467 goto out;
1468 }
1469
1470 /* call arch's memory hotadd */
1471 ret = arch_add_memory(nid, cur_start, memblock_size, &params);
1472 if (ret < 0) {
1473 altmap_free(params.altmap);
1474 goto out;
1475 }
1476
1477 /* create memory block devices after memory was added */
1478 ret = create_memory_block_devices(cur_start, memblock_size, nid,
1479 params.altmap, group);
1480 if (ret) {
1481 arch_remove_memory(cur_start, memblock_size, params.altmap, NULL);
1482 altmap_free(params.altmap);
1483 goto out;
1484 }
1485 }
1486
1487 return 0;
1488out:
1489 if (ret && cur_start != start)
1490 remove_memory_blocks_and_altmaps(start, cur_start - start);
1491 return ret;
1492}
1493
1494/*
1495 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1496 * and online/offline operations (triggered e.g. by sysfs).
1497 *
1498 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG
1499 */
1500static int __add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags,
1501 enum mmop online_type)
1502{
1503 struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) };
1504 enum memblock_flags memblock_flags = MEMBLOCK_NONE;
1505 struct memory_group *group = NULL;
1506 u64 start, size;
1507 bool new_node = false;
1508 int ret;
1509
1510 start = res->start;
1511 size = resource_size(res);
1512
1513 ret = check_hotplug_memory_range(start, size);
1514 if (ret)
1515 return ret;
1516
1517 if (mhp_flags & MHP_NID_IS_MGID) {
1518 group = memory_group_find_by_id(nid);
1519 if (!group)
1520 return -EINVAL;
1521 nid = group->nid;
1522 }
1523
1524 if (!node_possible(nid)) {
1525 WARN(1, "node %d was absent from the node_possible_map\n", nid);
1526 return -EINVAL;
1527 }
1528
1529 mem_hotplug_begin();
1530
1531 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) {
1532 if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED)
1533 memblock_flags = MEMBLOCK_DRIVER_MANAGED;
1534 ret = memblock_add_node(start, size, nid, memblock_flags);
1535 if (ret)
1536 goto error_mem_hotplug_end;
1537 }
1538
1539 ret = __try_online_node(nid, false);
1540 if (ret < 0)
1541 goto error_memblock_remove;
1542 if (ret) {
1543 node_set_online(nid);
1544 ret = register_node(nid);
1545 if (WARN_ON(ret)) {
1546 node_set_offline(nid);
1547 goto error_memblock_remove;
1548 }
1549 new_node = true;
1550 }
1551
1552 /*
1553 * Self hosted memmap array
1554 */
1555 if ((mhp_flags & MHP_MEMMAP_ON_MEMORY) &&
1556 mhp_supports_memmap_on_memory()) {
1557 ret = create_altmaps_and_memory_blocks(nid, group, start, size);
1558 if (ret)
1559 goto error;
1560 } else {
1561 ret = arch_add_memory(nid, start, size, &params);
1562 if (ret < 0)
1563 goto error;
1564
1565 /* create memory block devices after memory was added */
1566 ret = create_memory_block_devices(start, size, nid, NULL, group);
1567 if (ret) {
1568 arch_remove_memory(start, size, params.altmap, NULL);
1569 goto error;
1570 }
1571 }
1572
1573 register_memory_blocks_under_node_hotplug(nid, PFN_DOWN(start),
1574 PFN_UP(start + size - 1));
1575
1576 /* create new memmap entry */
1577 if (!strcmp(res->name, "System RAM"))
1578 firmware_map_add_hotplug(start, start + size, "System RAM");
1579
1580 /* device_online() will take the lock when calling online_pages() */
1581 mem_hotplug_done();
1582
1583 /*
1584 * In case we're allowed to merge the resource, flag it and trigger
1585 * merging now that adding succeeded.
1586 */
1587 if (mhp_flags & MHP_MERGE_RESOURCE)
1588 merge_system_ram_resource(res);
1589
1590 /* online pages if requested */
1591 if (online_type != MMOP_OFFLINE)
1592 walk_memory_blocks(start, size, &online_type,
1593 online_memory_block);
1594
1595 return ret;
1596error:
1597 if (new_node) {
1598 node_set_offline(nid);
1599 unregister_node(nid);
1600 }
1601error_memblock_remove:
1602 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
1603 memblock_remove(start, size);
1604error_mem_hotplug_end:
1605 mem_hotplug_done();
1606 return ret;
1607}
1608
1609int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
1610{
1611 return __add_memory_resource(nid, res, mhp_flags,
1612 mhp_get_default_online_type());
1613}
1614
1615/* requires device_hotplug_lock, see __add_memory_resource() */
1616int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags)
1617{
1618 struct resource *res;
1619 int ret;
1620
1621 res = register_memory_resource(start, size, "System RAM");
1622 if (IS_ERR(res))
1623 return PTR_ERR(res);
1624
1625 ret = add_memory_resource(nid, res, mhp_flags);
1626 if (ret < 0)
1627 release_memory_resource(res);
1628 return ret;
1629}
1630
1631int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags)
1632{
1633 int rc;
1634
1635 lock_device_hotplug();
1636 rc = __add_memory(nid, start, size, mhp_flags);
1637 unlock_device_hotplug();
1638
1639 return rc;
1640}
1641EXPORT_SYMBOL_GPL(add_memory);
1642
1643/**
1644 * __add_memory_driver_managed - add driver-managed memory with explicit online_type
1645 * @nid: NUMA node ID where the memory will be added
1646 * @start: Start physical address of the memory range
1647 * @size: Size of the memory range in bytes
1648 * @resource_name: Resource name in format "System RAM ($DRIVER)"
1649 * @mhp_flags: Memory hotplug flags
1650 * @online_type: Auto-Online behavior (offline, online, kernel, movable)
1651 *
1652 * Add special, driver-managed memory to the system as system RAM. Such
1653 * memory is not exposed via the raw firmware-provided memmap as system
1654 * RAM, instead, it is detected and added by a driver - during cold boot,
1655 * after a reboot, and after kexec.
1656 *
1657 * Reasons why this memory should not be used for the initial memmap of a
1658 * kexec kernel or for placing kexec images:
1659 *
1660 * - The booting kernel is in charge of determining how this memory will be
1661 * used (e.g., use persistent memory as system RAM)
1662 * - Coordination with a hypervisor is required before this memory
1663 * can be used (e.g., inaccessible parts).
1664 *
1665 * For this memory, no entries in /sys/firmware/memmap ("raw firmware-provided
1666 * memory map") are created. Also, the created memory resource is flagged
1667 * with IORESOURCE_SYSRAM_DRIVER_MANAGED, so in-kernel users can special-case
1668 * this memory as well (esp., not place kexec images onto it).
1669 *
1670 * The resource_name (visible via /proc/iomem) has to have the format
1671 * "System RAM ($DRIVER)".
1672 *
1673 * Return: 0 on success, negative error code on failure.
1674 */
1675int __add_memory_driver_managed(int nid, u64 start, u64 size,
1676 const char *resource_name, mhp_t mhp_flags,
1677 enum mmop online_type)
1678{
1679 struct resource *res;
1680 int rc;
1681
1682 if (!resource_name ||
1683 strstr(resource_name, "System RAM (") != resource_name ||
1684 resource_name[strlen(resource_name) - 1] != ')')
1685 return -EINVAL;
1686
1687 if (online_type < MMOP_OFFLINE || online_type > MMOP_ONLINE_MOVABLE)
1688 return -EINVAL;
1689
1690 lock_device_hotplug();
1691
1692 res = register_memory_resource(start, size, resource_name);
1693 if (IS_ERR(res)) {
1694 rc = PTR_ERR(res);
1695 goto out_unlock;
1696 }
1697
1698 rc = __add_memory_resource(nid, res, mhp_flags, online_type);
1699 if (rc < 0)
1700 release_memory_resource(res);
1701
1702out_unlock:
1703 unlock_device_hotplug();
1704 return rc;
1705}
1706EXPORT_SYMBOL_FOR_MODULES(__add_memory_driver_managed, "kmem");
1707
1708/**
1709 * add_memory_driver_managed - add driver-managed memory
1710 * @nid: NUMA node ID where the memory will be added
1711 * @start: Start physical address of the memory range
1712 * @size: Size of the memory range in bytes
1713 * @resource_name: Resource name in format "System RAM ($DRIVER)"
1714 * @mhp_flags: Memory hotplug flags
1715 *
1716 * Add driver-managed memory with the system default online type set by
1717 * build config or kernel boot parameter.
1718 *
1719 * See __add_memory_driver_managed for more details.
1720 *
1721 * Return: 0 on success, negative error code on failure.
1722 */
1723int add_memory_driver_managed(int nid, u64 start, u64 size,
1724 const char *resource_name, mhp_t mhp_flags)
1725{
1726 return __add_memory_driver_managed(nid, start, size, resource_name,
1727 mhp_flags,
1728 mhp_get_default_online_type());
1729}
1730EXPORT_SYMBOL_GPL(add_memory_driver_managed);
1731
1732/*
1733 * Platforms should define arch_get_mappable_range() that provides
1734 * maximum possible addressable physical memory range for which the
1735 * linear mapping could be created. The platform returned address
1736 * range must adhere to these following semantics.
1737 *
1738 * - range.start <= range.end
1739 * - Range includes both end points [range.start..range.end]
1740 *
1741 * There is also a fallback definition provided here, allowing the
1742 * entire possible physical address range in case any platform does
1743 * not define arch_get_mappable_range().
1744 */
1745struct range __weak arch_get_mappable_range(void)
1746{
1747 struct range mhp_range = {
1748 .start = 0UL,
1749 .end = -1ULL,
1750 };
1751 return mhp_range;
1752}
1753
1754struct range mhp_get_pluggable_range(bool need_mapping)
1755{
1756 const u64 max_phys = DIRECT_MAP_PHYSMEM_END;
1757 struct range mhp_range;
1758
1759 if (need_mapping) {
1760 mhp_range = arch_get_mappable_range();
1761 if (mhp_range.start > max_phys) {
1762 mhp_range.start = 0;
1763 mhp_range.end = 0;
1764 }
1765 mhp_range.end = min_t(u64, mhp_range.end, max_phys);
1766 } else {
1767 mhp_range.start = 0;
1768 mhp_range.end = max_phys;
1769 }
1770 return mhp_range;
1771}
1772EXPORT_SYMBOL_GPL(mhp_get_pluggable_range);
1773
1774bool mhp_range_allowed(u64 start, u64 size, bool need_mapping)
1775{
1776 struct range mhp_range = mhp_get_pluggable_range(need_mapping);
1777 u64 end = start + size;
1778
1779 if (start < end && start >= mhp_range.start && (end - 1) <= mhp_range.end)
1780 return true;
1781
1782 pr_warn("Hotplug memory [%#llx-%#llx] exceeds maximum addressable range [%#llx-%#llx]\n",
1783 start, end, mhp_range.start, mhp_range.end);
1784 return false;
1785}
1786
1787#ifdef CONFIG_MEMORY_HOTREMOVE
1788/*
1789 * Scan pfn range [start,end) to find movable/migratable pages (LRU and
1790 * hugetlb folio, movable_ops pages). Will skip over most unmovable
1791 * pages (esp., pages that can be skipped when offlining), but bail out on
1792 * definitely unmovable pages.
1793 *
1794 * Returns:
1795 * 0 in case a movable page is found and movable_pfn was updated.
1796 * -ENOENT in case no movable page was found.
1797 * -EBUSY in case a definitely unmovable page was found.
1798 */
1799static int scan_movable_pages(unsigned long start, unsigned long end,
1800 unsigned long *movable_pfn)
1801{
1802 unsigned long pfn;
1803
1804 for (pfn = start; pfn < end; pfn++) {
1805 unsigned long nr_pages;
1806 struct page *page;
1807 struct folio *folio;
1808
1809 page = pfn_to_page(pfn);
1810 if (PageLRU(page) || page_has_movable_ops(page))
1811 goto found;
1812
1813 /*
1814 * PageOffline() pages that do not have movable_ops and
1815 * have a reference count > 0 (after MEM_GOING_OFFLINE) are
1816 * definitely unmovable. If their reference count would be 0,
1817 * they could at least be skipped when offlining memory.
1818 */
1819 if (PageOffline(page) && page_count(page))
1820 return -EBUSY;
1821
1822 folio = page_folio(page);
1823 if (!folio_test_hugetlb(folio))
1824 continue;
1825 /*
1826 * This test is racy as we hold no reference or lock. The
1827 * hugetlb page could have been free'ed and head is no longer
1828 * a hugetlb page before the following check. In such unlikely
1829 * cases false positives and negatives are possible. Calling
1830 * code must deal with these scenarios.
1831 */
1832 if (folio_test_hugetlb_migratable(folio))
1833 goto found;
1834 nr_pages = folio_nr_pages(folio);
1835 if (unlikely(nr_pages < 1 || nr_pages > MAX_FOLIO_NR_PAGES ||
1836 !is_power_of_2(nr_pages)))
1837 continue;
1838 pfn |= nr_pages - 1;
1839 }
1840 return -ENOENT;
1841found:
1842 *movable_pfn = pfn;
1843 return 0;
1844}
1845
1846static void do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
1847{
1848 struct folio *folio;
1849 unsigned long pfn;
1850 LIST_HEAD(source);
1851 static DEFINE_RATELIMIT_STATE(migrate_rs, DEFAULT_RATELIMIT_INTERVAL,
1852 DEFAULT_RATELIMIT_BURST);
1853
1854 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1855 struct page *page;
1856
1857 page = pfn_to_page(pfn);
1858 folio = page_folio(page);
1859
1860 if (!folio_try_get(folio))
1861 continue;
1862
1863 if (unlikely(page_folio(page) != folio))
1864 goto put_folio;
1865
1866 if (folio_test_large(folio))
1867 pfn = folio_pfn(folio) + folio_nr_pages(folio) - 1;
1868
1869 if (folio_contain_hwpoisoned_page(folio)) {
1870 /*
1871 * unmap_poisoned_folio() cannot handle large folios
1872 * in all cases yet.
1873 */
1874 if (folio_test_large(folio) && !folio_test_hugetlb(folio))
1875 goto put_folio;
1876 if (folio_test_lru(folio) && !folio_isolate_lru(folio))
1877 goto put_folio;
1878 if (folio_mapped(folio)) {
1879 folio_lock(folio);
1880 unmap_poisoned_folio(folio, pfn, false);
1881 folio_unlock(folio);
1882 }
1883
1884 goto put_folio;
1885 }
1886
1887 if (!isolate_folio_to_list(folio, &source)) {
1888 if (__ratelimit(&migrate_rs)) {
1889 pr_warn("failed to isolate pfn %lx\n",
1890 page_to_pfn(page));
1891 dump_page(page, "isolation failed");
1892 }
1893 }
1894put_folio:
1895 folio_put(folio);
1896 }
1897 if (!list_empty(&source)) {
1898 nodemask_t nmask = node_states[N_MEMORY];
1899 struct migration_target_control mtc = {
1900 .nmask = &nmask,
1901 .gfp_mask = GFP_KERNEL | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
1902 .reason = MR_MEMORY_HOTPLUG,
1903 };
1904 int ret;
1905
1906 /*
1907 * We have checked that migration range is on a single zone so
1908 * we can use the nid of the first page to all the others.
1909 */
1910 mtc.nid = folio_nid(list_first_entry(&source, struct folio, lru));
1911
1912 /*
1913 * try to allocate from a different node but reuse this node
1914 * if there are no other online nodes to be used (e.g. we are
1915 * offlining a part of the only existing node)
1916 */
1917 node_clear(mtc.nid, nmask);
1918 if (nodes_empty(nmask))
1919 node_set(mtc.nid, nmask);
1920 ret = migrate_pages(&source, alloc_migration_target, NULL,
1921 (unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_HOTPLUG, NULL);
1922 if (ret) {
1923 list_for_each_entry(folio, &source, lru) {
1924 if (__ratelimit(&migrate_rs)) {
1925 pr_warn("migrating pfn %lx failed ret:%d\n",
1926 folio_pfn(folio), ret);
1927 dump_page(&folio->page,
1928 "migration failure");
1929 }
1930 }
1931 putback_movable_pages(&source);
1932 }
1933 }
1934}
1935
1936static int __init cmdline_parse_movable_node(char *p)
1937{
1938 movable_node_enabled = true;
1939 return 0;
1940}
1941early_param("movable_node", cmdline_parse_movable_node);
1942
1943static int count_system_ram_pages_cb(unsigned long start_pfn,
1944 unsigned long nr_pages, void *data)
1945{
1946 unsigned long *nr_system_ram_pages = data;
1947
1948 *nr_system_ram_pages += nr_pages;
1949 return 0;
1950}
1951
1952/*
1953 * Must be called with mem_hotplug_lock in write mode.
1954 */
1955int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
1956 struct zone *zone, struct memory_group *group)
1957{
1958 unsigned long pfn, managed_pages, system_ram_pages = 0;
1959 const unsigned long end_pfn = start_pfn + nr_pages;
1960 struct pglist_data *pgdat = zone->zone_pgdat;
1961 const int node = zone_to_nid(zone);
1962 struct memory_notify mem_arg = {
1963 .start_pfn = start_pfn,
1964 .nr_pages = nr_pages,
1965 };
1966 struct node_notify node_arg = {
1967 .nid = NUMA_NO_NODE,
1968 };
1969 unsigned long flags;
1970 char *reason;
1971 int ret;
1972 unsigned long normal_pages = 0;
1973 enum zone_type zt;
1974
1975 /*
1976 * {on,off}lining is constrained to full memory sections (or more
1977 * precisely to memory blocks from the user space POV).
1978 * memmap_on_memory is an exception because it reserves initial part
1979 * of the physical memory space for vmemmaps. That space is pageblock
1980 * aligned.
1981 */
1982 if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(start_pfn) ||
1983 !IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION)))
1984 return -EINVAL;
1985
1986 /*
1987 * Don't allow to offline memory blocks that contain holes.
1988 * Consequently, memory blocks with holes can never get onlined
1989 * via the hotplug path - online_pages() - as hotplugged memory has
1990 * no holes. This way, we don't have to worry about memory holes,
1991 * don't need pfn_valid() checks, and can avoid using
1992 * walk_system_ram_range() later.
1993 */
1994 walk_system_ram_range(start_pfn, nr_pages, &system_ram_pages,
1995 count_system_ram_pages_cb);
1996 if (system_ram_pages != nr_pages) {
1997 ret = -EINVAL;
1998 reason = "memory holes";
1999 goto failed_removal;
2000 }
2001
2002 /*
2003 * We only support offlining of memory blocks managed by a single zone,
2004 * checked by calling code. This is just a sanity check that we might
2005 * want to remove in the future.
2006 */
2007 if (WARN_ON_ONCE(page_zone(pfn_to_page(start_pfn)) != zone ||
2008 page_zone(pfn_to_page(end_pfn - 1)) != zone)) {
2009 ret = -EINVAL;
2010 reason = "multizone range";
2011 goto failed_removal;
2012 }
2013
2014 /*
2015 * Disable pcplists so that page isolation cannot race with freeing
2016 * in a way that pages from isolated pageblock are left on pcplists.
2017 */
2018 zone_pcp_disable(zone);
2019 lru_cache_disable();
2020
2021 /* set above range as isolated */
2022 ret = start_isolate_page_range(start_pfn, end_pfn,
2023 PB_ISOLATE_MODE_MEM_OFFLINE);
2024 if (ret) {
2025 reason = "failure to isolate range";
2026 goto failed_removal_pcplists_disabled;
2027 }
2028
2029 /*
2030 * Check whether the node will have no present pages after we offline
2031 * 'nr_pages' more. If so, we know that the node will become empty, and
2032 * so we will clear N_MEMORY for it.
2033 */
2034 if (nr_pages >= pgdat->node_present_pages) {
2035 node_arg.nid = node;
2036 ret = node_notify(NODE_REMOVING_LAST_MEMORY, &node_arg);
2037 ret = notifier_to_errno(ret);
2038 if (ret) {
2039 reason = "node notifier failure";
2040 goto failed_removal_isolated;
2041 }
2042 }
2043
2044 ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
2045 ret = notifier_to_errno(ret);
2046 if (ret) {
2047 reason = "notifier failure";
2048 goto failed_removal_isolated;
2049 }
2050
2051 do {
2052 pfn = start_pfn;
2053 do {
2054 /*
2055 * Historically we always checked for any signal and
2056 * can't limit it to fatal signals without eventually
2057 * breaking user space.
2058 */
2059 if (signal_pending(current)) {
2060 ret = -EINTR;
2061 reason = "signal backoff";
2062 goto failed_removal_isolated;
2063 }
2064
2065 cond_resched();
2066
2067 ret = scan_movable_pages(pfn, end_pfn, &pfn);
2068 if (!ret) {
2069 /*
2070 * TODO: fatal migration failures should bail
2071 * out
2072 */
2073 do_migrate_range(pfn, end_pfn);
2074 }
2075 } while (!ret);
2076
2077 if (ret != -ENOENT) {
2078 reason = "unmovable page";
2079 goto failed_removal_isolated;
2080 }
2081
2082 /*
2083 * Dissolve free hugetlb folios in the memory block before doing
2084 * offlining actually in order to make hugetlbfs's object
2085 * counting consistent.
2086 */
2087 ret = dissolve_free_hugetlb_folios(start_pfn, end_pfn);
2088 if (ret) {
2089 reason = "failure to dissolve huge pages";
2090 goto failed_removal_isolated;
2091 }
2092
2093 ret = test_pages_isolated(start_pfn, end_pfn,
2094 PB_ISOLATE_MODE_MEM_OFFLINE);
2095
2096 } while (ret);
2097
2098 /* Mark all sections offline and remove free pages from the buddy. */
2099 managed_pages = __offline_isolated_pages(start_pfn, end_pfn);
2100 pr_debug("Offlined Pages %ld\n", nr_pages);
2101
2102 /*
2103 * The memory sections are marked offline, and the pageblock flags
2104 * effectively stale; nobody should be touching them. Fixup the number
2105 * of isolated pageblocks, memory onlining will properly revert this.
2106 */
2107 spin_lock_irqsave(&zone->lock, flags);
2108 zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages;
2109 spin_unlock_irqrestore(&zone->lock, flags);
2110
2111 lru_cache_enable();
2112 zone_pcp_enable(zone);
2113
2114 /* removal success */
2115 adjust_managed_page_count(pfn_to_page(start_pfn), -managed_pages);
2116 adjust_present_page_count(pfn_to_page(start_pfn), group, -nr_pages);
2117
2118 /* reinitialise watermarks and update pcp limits */
2119 init_per_zone_wmark_min();
2120
2121 /*
2122 * Check whether this operation removes the last normal memory from
2123 * the node. We do this before clearing N_MEMORY to avoid the possible
2124 * transient "!N_MEMORY && N_NORMAL_MEMORY" state.
2125 */
2126 if (zone_idx(zone) <= ZONE_NORMAL) {
2127 for (zt = 0; zt <= ZONE_NORMAL; zt++)
2128 normal_pages += pgdat->node_zones[zt].present_pages;
2129 if (!normal_pages)
2130 node_clear_state(node, N_NORMAL_MEMORY);
2131 }
2132 /*
2133 * Make sure to mark the node as memory-less before rebuilding the zone
2134 * list. Otherwise this node would still appear in the fallback lists.
2135 */
2136 if (node_arg.nid >= 0)
2137 node_clear_state(node, N_MEMORY);
2138 if (!populated_zone(zone)) {
2139 zone_pcp_reset(zone);
2140 build_all_zonelists(NULL);
2141 }
2142
2143 if (node_arg.nid >= 0) {
2144 kcompactd_stop(node);
2145 kswapd_stop(node);
2146 /* Node went memoryless. Notify consumers */
2147 node_notify(NODE_REMOVED_LAST_MEMORY, &node_arg);
2148 }
2149
2150 writeback_set_ratelimit();
2151
2152 memory_notify(MEM_OFFLINE, &mem_arg);
2153 remove_pfn_range_from_zone(zone, start_pfn, nr_pages);
2154 return 0;
2155
2156failed_removal_isolated:
2157 /* pushback to free area */
2158 undo_isolate_page_range(start_pfn, end_pfn);
2159 memory_notify(MEM_CANCEL_OFFLINE, &mem_arg);
2160 if (node_arg.nid != NUMA_NO_NODE)
2161 node_notify(NODE_CANCEL_REMOVING_LAST_MEMORY, &node_arg);
2162failed_removal_pcplists_disabled:
2163 lru_cache_enable();
2164 zone_pcp_enable(zone);
2165failed_removal:
2166 pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n",
2167 (unsigned long long) start_pfn << PAGE_SHIFT,
2168 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1,
2169 reason);
2170 return ret;
2171}
2172
2173static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
2174{
2175 int *nid = arg;
2176
2177 *nid = mem->nid;
2178 if (unlikely(mem->state != MEM_OFFLINE)) {
2179 phys_addr_t beginpa, endpa;
2180
2181 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
2182 endpa = beginpa + memory_block_size_bytes() - 1;
2183 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
2184 &beginpa, &endpa);
2185
2186 return -EBUSY;
2187 }
2188 return 0;
2189}
2190
2191static int count_memory_range_altmaps_cb(struct memory_block *mem, void *arg)
2192{
2193 u64 *num_altmaps = (u64 *)arg;
2194
2195 if (mem->altmap)
2196 *num_altmaps += 1;
2197
2198 return 0;
2199}
2200
2201static int check_cpu_on_node(int nid)
2202{
2203 int cpu;
2204
2205 for_each_present_cpu(cpu) {
2206 if (cpu_to_node(cpu) == nid)
2207 /*
2208 * the cpu on this node isn't removed, and we can't
2209 * offline this node.
2210 */
2211 return -EBUSY;
2212 }
2213
2214 return 0;
2215}
2216
2217static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg)
2218{
2219 int nid = *(int *)arg;
2220
2221 /*
2222 * If a memory block belongs to multiple nodes, the stored nid is not
2223 * reliable. However, such blocks are always online (e.g., cannot get
2224 * offlined) and, therefore, are still spanned by the node.
2225 */
2226 return mem->nid == nid ? -EEXIST : 0;
2227}
2228
2229/**
2230 * try_offline_node
2231 * @nid: the node ID
2232 *
2233 * Offline a node if all memory sections and cpus of the node are removed.
2234 *
2235 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
2236 * and online/offline operations before this call.
2237 */
2238void try_offline_node(int nid)
2239{
2240 int rc;
2241
2242 /*
2243 * If the node still spans pages (especially ZONE_DEVICE), don't
2244 * offline it. A node spans memory after move_pfn_range_to_zone(),
2245 * e.g., after the memory block was onlined.
2246 */
2247 if (node_spanned_pages(nid))
2248 return;
2249
2250 /*
2251 * Especially offline memory blocks might not be spanned by the
2252 * node. They will get spanned by the node once they get onlined.
2253 * However, they link to the node in sysfs and can get onlined later.
2254 */
2255 rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb);
2256 if (rc)
2257 return;
2258
2259 if (check_cpu_on_node(nid))
2260 return;
2261
2262 /*
2263 * all memory/cpu of this node are removed, we can offline this
2264 * node now.
2265 */
2266 node_set_offline(nid);
2267 unregister_node(nid);
2268}
2269EXPORT_SYMBOL(try_offline_node);
2270
2271static int memory_blocks_have_altmaps(u64 start, u64 size)
2272{
2273 u64 num_memblocks = size / memory_block_size_bytes();
2274 u64 num_altmaps = 0;
2275
2276 if (!mhp_memmap_on_memory())
2277 return 0;
2278
2279 walk_memory_blocks(start, size, &num_altmaps,
2280 count_memory_range_altmaps_cb);
2281
2282 if (num_altmaps == 0)
2283 return 0;
2284
2285 if (WARN_ON_ONCE(num_memblocks != num_altmaps))
2286 return -EINVAL;
2287
2288 return 1;
2289}
2290
2291static int try_remove_memory(u64 start, u64 size)
2292{
2293 int rc, nid = NUMA_NO_NODE;
2294
2295 BUG_ON(check_hotplug_memory_range(start, size));
2296
2297 /*
2298 * All memory blocks must be offlined before removing memory. Check
2299 * whether all memory blocks in question are offline and return error
2300 * if this is not the case.
2301 *
2302 * While at it, determine the nid. Note that if we'd have mixed nodes,
2303 * we'd only try to offline the last determined one -- which is good
2304 * enough for the cases we care about.
2305 */
2306 rc = walk_memory_blocks(start, size, &nid, check_memblock_offlined_cb);
2307 if (rc)
2308 return rc;
2309
2310 /* remove memmap entry */
2311 firmware_map_remove(start, start + size, "System RAM");
2312
2313 mem_hotplug_begin();
2314
2315 rc = memory_blocks_have_altmaps(start, size);
2316 if (rc < 0) {
2317 mem_hotplug_done();
2318 return rc;
2319 } else if (!rc) {
2320 /*
2321 * Memory block device removal under the device_hotplug_lock is
2322 * a barrier against racing online attempts.
2323 * No altmaps present, do the removal directly
2324 */
2325 remove_memory_block_devices(start, size);
2326 arch_remove_memory(start, size, NULL, NULL);
2327 } else {
2328 /* all memblocks in the range have altmaps */
2329 remove_memory_blocks_and_altmaps(start, size);
2330 }
2331
2332 if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
2333 memblock_remove(start, size);
2334
2335 release_mem_region_adjustable(start, size);
2336
2337 if (nid != NUMA_NO_NODE)
2338 try_offline_node(nid);
2339
2340 mem_hotplug_done();
2341 return 0;
2342}
2343
2344/**
2345 * __remove_memory - Remove memory if every memory block is offline
2346 * @start: physical address of the region to remove
2347 * @size: size of the region to remove
2348 *
2349 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
2350 * and online/offline operations before this call, as required by
2351 * try_offline_node().
2352 */
2353void __remove_memory(u64 start, u64 size)
2354{
2355
2356 /*
2357 * trigger BUG() if some memory is not offlined prior to calling this
2358 * function
2359 */
2360 if (try_remove_memory(start, size))
2361 BUG();
2362}
2363
2364/*
2365 * Remove memory if every memory block is offline, otherwise return -EBUSY is
2366 * some memory is not offline
2367 */
2368int remove_memory(u64 start, u64 size)
2369{
2370 int rc;
2371
2372 lock_device_hotplug();
2373 rc = try_remove_memory(start, size);
2374 unlock_device_hotplug();
2375
2376 return rc;
2377}
2378EXPORT_SYMBOL_GPL(remove_memory);
2379
2380static int try_offline_memory_block(struct memory_block *mem, void *arg)
2381{
2382 enum mmop online_type = MMOP_ONLINE_KERNEL;
2383 uint8_t **online_types = arg;
2384 struct page *page;
2385 int rc;
2386
2387 /*
2388 * Sense the online_type via the zone of the memory block. Offlining
2389 * with multiple zones within one memory block will be rejected
2390 * by offlining code ... so we don't care about that.
2391 */
2392 page = pfn_to_online_page(section_nr_to_pfn(mem->start_section_nr));
2393 if (page && page_zonenum(page) == ZONE_MOVABLE)
2394 online_type = MMOP_ONLINE_MOVABLE;
2395
2396 rc = device_offline(&mem->dev);
2397 /*
2398 * Default is MMOP_OFFLINE - change it only if offlining succeeded,
2399 * so try_reonline_memory_block() can do the right thing.
2400 */
2401 if (!rc)
2402 **online_types = online_type;
2403
2404 (*online_types)++;
2405 /* Ignore if already offline. */
2406 return rc < 0 ? rc : 0;
2407}
2408
2409static int try_reonline_memory_block(struct memory_block *mem, void *arg)
2410{
2411 uint8_t **online_types = arg;
2412 int rc;
2413
2414 if (**online_types != MMOP_OFFLINE) {
2415 mem->online_type = (enum mmop)**online_types;
2416 rc = device_online(&mem->dev);
2417 if (rc < 0)
2418 pr_warn("%s: Failed to re-online memory: %d",
2419 __func__, rc);
2420 }
2421
2422 /* Continue processing all remaining memory blocks. */
2423 (*online_types)++;
2424 return 0;
2425}
2426
2427/*
2428 * Try to offline and remove memory. Might take a long time to finish in case
2429 * memory is still in use. Primarily useful for memory devices that logically
2430 * unplugged all memory (so it's no longer in use) and want to offline + remove
2431 * that memory.
2432 */
2433int offline_and_remove_memory(u64 start, u64 size)
2434{
2435 struct range range = {
2436 .start = start,
2437 .end = start + size - 1,
2438 };
2439
2440 return offline_and_remove_memory_ranges(&range, 1);
2441}
2442EXPORT_SYMBOL_GPL(offline_and_remove_memory);
2443
2444/**
2445 * offline_and_remove_memory_ranges - offline and remove multiple memory ranges
2446 * @ranges: array of physical address ranges to offline and remove
2447 * @nr_ranges: number of entries in @ranges
2448 *
2449 * Offline and remove several memory ranges as one operation, serialized
2450 * against other hotplug operations by a single lock_device_hotplug().
2451 *
2452 * This offlines all ranges before removing any of them. If offlining any
2453 * range fails, the entire process is reverted and nothing is removed.
2454 * This provides a fully atomic semantic for unplugging an entire device.
2455 *
2456 * Each range must be memory-block aligned in start and size.
2457 *
2458 * Return: 0 on success, negative errno on failure (never positive). On
2459 * failure no range has been removed.
2460 */
2461int offline_and_remove_memory_ranges(const struct range *ranges,
2462 unsigned int nr_ranges)
2463{
2464 unsigned long mb_count = 0;
2465 uint8_t *online_types, *tmp;
2466 unsigned int i;
2467 int rc = 0;
2468
2469 if (!ranges || !nr_ranges)
2470 return -EINVAL;
2471
2472 for (i = 0; i < nr_ranges; i++) {
2473 const u64 start = ranges[i].start;
2474 const u64 size = range_len(&ranges[i]);
2475
2476 if (!IS_ALIGNED(start, memory_block_size_bytes()) ||
2477 !IS_ALIGNED(size, memory_block_size_bytes()) || !size)
2478 return -EINVAL;
2479 mb_count += size / memory_block_size_bytes();
2480 }
2481
2482 /*
2483 * Remember the old online type of every memory block across all ranges,
2484 * so we can revert if offlining a later block fails. All entries start
2485 * as MMOP_OFFLINE so blocks we never touched are skipped on rollback.
2486 */
2487 online_types = kmalloc_array(mb_count, sizeof(*online_types),
2488 GFP_KERNEL);
2489 if (!online_types)
2490 return -ENOMEM;
2491 memset(online_types, MMOP_OFFLINE, mb_count);
2492
2493 lock_device_hotplug();
2494
2495 /*
2496 * Phase 1: offline every block in every range. An already-offline
2497 * block folds to success, so out-of-band offlining never blocks unplug.
2498 */
2499 tmp = online_types;
2500 for (i = 0; i < nr_ranges; i++) {
2501 rc = walk_memory_blocks(ranges[i].start, range_len(&ranges[i]),
2502 &tmp, try_offline_memory_block);
2503 if (rc)
2504 break;
2505 }
2506
2507 /* If any failure occurred at all, rollback any changes and bail */
2508 if (rc) {
2509 tmp = online_types;
2510 for (i = 0; i < nr_ranges; i++)
2511 walk_memory_blocks(ranges[i].start,
2512 range_len(&ranges[i]), &tmp,
2513 try_reonline_memory_block);
2514 goto out_unlock;
2515 }
2516
2517 /* Phase 2: Remove. This should never fail holding the hotplug lock */
2518 for (i = 0; i < nr_ranges; i++)
2519 WARN_ON_ONCE(try_remove_memory(ranges[i].start,
2520 range_len(&ranges[i])));
2521
2522out_unlock:
2523 unlock_device_hotplug();
2524
2525 kfree(online_types);
2526 return rc;
2527}
2528EXPORT_SYMBOL_GPL(offline_and_remove_memory_ranges);
2529#endif /* CONFIG_MEMORY_HOTREMOVE */