projects/linux·blob master · refs

2098 lines · 53.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kexec_handover.c - kexec handover metadata processing
4 * Copyright (C) 2023 Alexander Graf <[email protected]>
5 * Copyright (C) 2025 Microsoft Corporation, Mike Rapoport <[email protected]>
6 * Copyright (C) 2025 Google LLC, Changyuan Lyu <[email protected]>
7 * Copyright (C) 2025 Pasha Tatashin <[email protected]>
8 * Copyright (C) 2026 Google LLC, Jason Miu <[email protected]>
9 */
10
11#define pr_fmt(fmt) "KHO: " fmt
12
13#include <linux/cleanup.h>
14#include <linux/cma.h>
15#include <linux/kmemleak.h>
16#include <linux/count_zeros.h>
17#include <linux/kasan.h>
18#include <linux/kexec.h>
19#include <linux/kexec_handover.h>
20#include <linux/kho_radix_tree.h>
21#include <linux/utsname.h>
22#include <linux/kho/abi/kexec_handover.h>
23#include <linux/kho/abi/kexec_metadata.h>
24#include <linux/libfdt.h>
25#include <linux/list.h>
26#include <linux/memblock.h>
27#include <linux/page-isolation.h>
28#include <linux/unaligned.h>
29#include <linux/vmalloc.h>
30
31#include <asm/early_ioremap.h>
32
33/*
34 * KHO is tightly coupled with mm init and needs access to some of mm
35 * internal APIs.
36 */
37#include "../../mm/mm_init.h"
38#include "../../mm/vmalloc.h"
39#include "../kexec_internal.h"
40#include "kexec_handover_internal.h"
41
42/*
43 * This is the minimal alignment required by deferred struct page init.
44 * deferred_init_memmap_chunk frees memory to the buddy allocator, which looks
45 * at the neighboring pages (up to MAX_PAGE_ORDER) to merge them.
46 * If KHO scratch is not aligned to that value, buddy can access uninitialized
47 * struct pages, which can cause a crash.
48 */
49#define SCRATCH_ALIGNMENT_BYTES (PAGE_SIZE * MAX_ORDER_NR_PAGES)
50static_assert(SCRATCH_ALIGNMENT_BYTES >= CMA_MIN_ALIGNMENT_BYTES);
51
52/* The magic token for preserved pages */
53#define KHO_PAGE_MAGIC 0x4b484f50U /* ASCII for 'KHOP' */
54
55/*
56 * KHO uses page->private, which is an unsigned long, to store page metadata.
57 * Use it to store both the magic and the order.
58 */
59union kho_page_info {
60 unsigned long page_private;
61 struct {
62 unsigned int order;
63 unsigned int magic;
64 };
65};
66
67static_assert(sizeof(union kho_page_info) == sizeof(((struct page *)0)->private));
68
69static bool kho_enable __ro_after_init = IS_ENABLED(CONFIG_KEXEC_HANDOVER_ENABLE_DEFAULT);
70
71bool kho_is_enabled(void)
72{
73 return kho_enable;
74}
75EXPORT_SYMBOL_GPL(kho_is_enabled);
76
77static int __init kho_parse_enable(char *p)
78{
79 return kstrtobool(p, &kho_enable);
80}
81early_param("kho", kho_parse_enable);
82
83struct kho_out {
84 void *fdt;
85 struct mutex lock; /* protects KHO FDT */
86
87 struct kho_radix_tree radix_tree;
88 struct kho_debugfs dbg;
89};
90
91static struct kho_out kho_out = {
92 .lock = __MUTEX_INITIALIZER(kho_out.lock),
93 .radix_tree = {
94 .lock = __MUTEX_INITIALIZER(kho_out.radix_tree.lock),
95 },
96};
97
98struct kho_in {
99 phys_addr_t fdt_phys;
100 phys_addr_t scratch_phys;
101 char previous_release[__NEW_UTS_LEN + 1];
102 u32 kexec_count;
103 struct kho_debugfs dbg;
104 struct kho_radix_tree radix_tree;
105};
106
107static struct kho_in kho_in = {
108};
109
110static const void *kho_get_fdt(void)
111{
112 return kho_in.fdt_phys ? phys_to_virt(kho_in.fdt_phys) : NULL;
113}
114
115/**
116 * kho_encode_radix_key - Encodes a physical address and order into a radix key.
117 * @phys: The physical address of the page.
118 * @order: The order of the page.
119 *
120 * This function combines a page's physical address and its order into a
121 * single unsigned long, which is used as a key for all radix tree
122 * operations.
123 *
124 * Return: The encoded unsigned long radix key.
125 */
126static unsigned long kho_encode_radix_key(phys_addr_t phys, unsigned int order)
127{
128 /* The physical address is encoded by shifting the PFN by its order. */
129 unsigned long shift = PAGE_SHIFT + order;
130 /* Order bit goes right before the shifted PFN. */
131 unsigned long h = 1UL << (64 - shift);
132 /* Shifted PFN. */
133 unsigned long l = phys >> shift;
134
135 return h | l;
136}
137
138/**
139 * kho_decode_radix_key - Decodes a radix key back into a physical address and order.
140 * @key: The unsigned long key to decode.
141 * @order: An output parameter, a pointer to an unsigned int where the decoded
142 * page order will be stored.
143 *
144 * This function reverses the encoding performed by kho_encode_radix_key(),
145 * extracting the original physical address and page order from a given key.
146 *
147 * Return: The decoded physical address.
148 */
149static phys_addr_t kho_decode_radix_key(unsigned long key, unsigned int *order)
150{
151 /* fls64() indexes starting from 1. */
152 unsigned int order_bit = fls64(key) - 1;
153 phys_addr_t phys;
154
155 /* order bit goes right before the shifted PFN. */
156 *order = 64 - (PAGE_SHIFT + order_bit);
157 /* The order bit is discarded by the shift */
158 phys = key << (PAGE_SHIFT + *order);
159
160 return phys;
161}
162
163static unsigned long kho_radix_get_bitmap_index(unsigned long key)
164{
165 return key % (1 << KHO_BITMAP_SIZE_LOG2);
166}
167
168static unsigned long kho_radix_get_table_index(unsigned long key,
169 unsigned int level)
170{
171 int s;
172
173 s = ((level - 1) * KHO_TABLE_SIZE_LOG2) + KHO_BITMAP_SIZE_LOG2;
174 return (key >> s) % (1 << KHO_TABLE_SIZE_LOG2);
175}
176
177static void __ref *kho_radix_alloc_node(void)
178{
179 struct kho_radix_node *node;
180
181 if (slab_is_available())
182 node = (struct kho_radix_node *)get_zeroed_page(GFP_KERNEL);
183 else
184 node = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
185
186 return node;
187}
188
189static void __ref kho_radix_free_node(struct kho_radix_node *node)
190{
191 if (slab_is_available())
192 free_page((unsigned long)node);
193 else
194 memblock_free(node, PAGE_SIZE);
195}
196
197/**
198 * kho_radix_add_key - Add a key to the radix tree.
199 * @tree: The KHO radix tree.
200 * @key: The key to add.
201 *
202 * This function traverses the radix tree based on the @key provided. It sets the
203 * corresponding bit in the leaf bitmap to mark the @key as present. If
204 * intermediate nodes do not exist along the path, they are allocated and added
205 * to the tree.
206 *
207 * NOTE: Currently only keys of width up to %KHO_RADIX_KEY_WIDTH are supported.
208 * This limit only exists because current users of the radix tree don't use more
209 * than that. Changing the maximum width requires changing the tree depth, which
210 * needs bumping the ABI version.
211 *
212 * Return: 0 on success, or a negative error code on failure.
213 */
214int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key)
215{
216 /* Newly allocated nodes for error cleanup */
217 struct kho_radix_node *intermediate_nodes[KHO_TREE_MAX_DEPTH] = { 0 };
218 struct kho_radix_node *anchor_node = NULL;
219 struct kho_radix_node *node = tree->root;
220 struct kho_radix_node *new_node;
221 unsigned int i, idx, anchor_idx;
222 struct kho_radix_leaf *leaf;
223 int err = 0;
224
225 if (WARN_ON_ONCE(!tree->root))
226 return -EINVAL;
227
228 if (unlikely(fls64(key) > KHO_RADIX_KEY_WIDTH))
229 return -ERANGE;
230
231 might_sleep();
232
233 guard(mutex)(&tree->lock);
234
235 /* Go from high levels to low levels */
236 for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
237 idx = kho_radix_get_table_index(key, i);
238
239 if (node->table[idx]) {
240 node = phys_to_virt(node->table[idx]);
241 continue;
242 }
243
244 /* Next node is empty, create a new node for it */
245 new_node = kho_radix_alloc_node();
246 if (!new_node) {
247 err = -ENOMEM;
248 goto err_free_nodes;
249 }
250
251 node->table[idx] = virt_to_phys(new_node);
252
253 /*
254 * Capture the node where the new branch starts for cleanup
255 * if allocation fails.
256 */
257 if (!anchor_node) {
258 anchor_node = node;
259 anchor_idx = idx;
260 }
261 intermediate_nodes[i] = new_node;
262
263 node = new_node;
264 }
265
266 /* Handle the leaf level bitmap (level 0) */
267 idx = kho_radix_get_bitmap_index(key);
268 leaf = (struct kho_radix_leaf *)node;
269 __set_bit(idx, leaf->bitmap);
270
271 return 0;
272
273err_free_nodes:
274 for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
275 if (intermediate_nodes[i])
276 kho_radix_free_node(intermediate_nodes[i]);
277 }
278 if (anchor_node)
279 anchor_node->table[anchor_idx] = 0;
280
281 return err;
282}
283EXPORT_SYMBOL_GPL(kho_radix_add_key);
284
285/**
286 * kho_radix_del_key - Removes the key from the radix tree.
287 * @tree: The KHO radix tree.
288 * @key: The key to remove.
289 *
290 * This function traverses the radix tree and clears the bit corresponding to
291 * the @key, effectively removing it from the tree. It does not free the tree's
292 * intermediate nodes, even if they become empty.
293 */
294void kho_radix_del_key(struct kho_radix_tree *tree, unsigned long key)
295{
296 struct kho_radix_node *node = tree->root;
297 struct kho_radix_leaf *leaf;
298 unsigned int i, idx;
299
300 if (WARN_ON_ONCE(!tree->root))
301 return;
302
303 /* Keys wider than KHO_RADIX_KEY_WIDTH are not allowed to be added. */
304 if (unlikely(fls64(key) > KHO_RADIX_KEY_WIDTH))
305 return;
306
307 might_sleep();
308
309 guard(mutex)(&tree->lock);
310
311 /* Go from high levels to low levels */
312 for (i = KHO_TREE_MAX_DEPTH - 1; i > 0; i--) {
313 idx = kho_radix_get_table_index(key, i);
314
315 /*
316 * Attempting to delete a page that has not been preserved,
317 * return with a warning.
318 */
319 if (WARN_ON(!node->table[idx]))
320 return;
321
322 node = phys_to_virt(node->table[idx]);
323 }
324
325 /* Handle the leaf level bitmap (level 0) */
326 leaf = (struct kho_radix_leaf *)node;
327 idx = kho_radix_get_bitmap_index(key);
328 __clear_bit(idx, leaf->bitmap);
329}
330EXPORT_SYMBOL_GPL(kho_radix_del_key);
331
332static void __kho_radix_destroy_tree(struct kho_radix_node *root,
333 unsigned int level)
334{
335 unsigned long i;
336
337 if (level == 0) {
338 kho_radix_free_node(root);
339 return;
340 }
341
342 for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) {
343 if (root->table[i])
344 __kho_radix_destroy_tree(phys_to_virt(root->table[i]),
345 level - 1);
346 }
347
348 kho_radix_free_node(root);
349}
350
351/**
352 * kho_radix_init_tree - initialize the radix tree.
353 * @tree: the tree to initialize.
354 * @root: root table of the radix tree.
355 *
356 * Initialize the radix tree with the given root node. If root is %NULL, an
357 * empty root table is allocated. If root is not %NULL, it is the caller's
358 * responsibility to make sure the root is valid and in the correct format.
359 *
360 * Return: 0 on success, -errno on failure.
361 */
362int kho_radix_init_tree(struct kho_radix_tree *tree, struct kho_radix_node *root)
363{
364 if (!root)
365 root = kho_radix_alloc_node();
366 if (!root)
367 return -ENOMEM;
368
369 tree->root = root;
370 mutex_init(&tree->lock);
371 return 0;
372}
373EXPORT_SYMBOL_GPL(kho_radix_init_tree);
374
375/**
376 * kho_radix_destroy_tree - Destroy the radix tree
377 * @tree: The radix tree to destroy
378 *
379 * Walk @tree and free all its nodes.
380 */
381void kho_radix_destroy_tree(struct kho_radix_tree *tree)
382{
383 if (!tree->root)
384 return;
385
386 __kho_radix_destroy_tree(tree->root, KHO_TREE_MAX_DEPTH - 1);
387 tree->root = NULL;
388}
389EXPORT_SYMBOL_GPL(kho_radix_destroy_tree);
390
391static int kho_radix_walk_leaf(struct kho_radix_leaf *leaf, unsigned long key,
392 const struct kho_radix_walk_cb *cb, void *data)
393{
394 unsigned long *bitmap = (unsigned long *)leaf;
395 unsigned int i;
396 int err;
397
398 if (cb->node) {
399 err = cb->node(virt_to_phys(leaf), data);
400 if (err)
401 return err;
402 }
403
404 if (!cb->leaf)
405 return 0;
406
407 for_each_set_bit(i, bitmap, PAGE_SIZE * BITS_PER_BYTE) {
408 err = cb->leaf(key | i, data);
409 if (err)
410 return err;
411 }
412
413 return 0;
414}
415
416static int __kho_radix_walk_tree(struct kho_radix_node *root,
417 unsigned int level, unsigned long start,
418 const struct kho_radix_walk_cb *cb, void *data)
419{
420 struct kho_radix_node *node;
421 struct kho_radix_leaf *leaf;
422 unsigned long key, i;
423 unsigned int shift;
424 int err;
425
426 if (cb->node) {
427 err = cb->node(virt_to_phys(root), data);
428 if (err)
429 return err;
430 }
431
432 for (i = 0; i < PAGE_SIZE / sizeof(phys_addr_t); i++) {
433 if (!root->table[i])
434 continue;
435
436 shift = ((level - 1) * KHO_TABLE_SIZE_LOG2) +
437 KHO_BITMAP_SIZE_LOG2;
438 key = start | (i << shift);
439
440 node = phys_to_virt(root->table[i]);
441
442 if (level == 1) {
443 /*
444 * we are at level 1,
445 * node is pointing to the level 0 bitmap.
446 */
447 leaf = (struct kho_radix_leaf *)node;
448 err = kho_radix_walk_leaf(leaf, key, cb, data);
449 } else {
450 err = __kho_radix_walk_tree(node, level - 1,
451 key, cb, data);
452 }
453
454 if (err)
455 return err;
456 }
457
458 return 0;
459}
460
461/**
462 * kho_radix_walk_tree - Traverses the radix tree and calls a callback for each key.
463 * @tree: A pointer to the KHO radix tree to walk.
464 * @cb: Set of callbacks to be invoked during the tree walk.
465 * @data: Opaque data pointer passed to each callback in @cb.
466 *
467 * This function walks the radix tree, searching from the top level down to the
468 * lowest level (level 0), invoking the appropriate callbacks.
469 *
470 * Return: 0 if the walk completed the specified tree, or the non-zero return
471 * value from the callback that stopped the walk.
472 */
473int kho_radix_walk_tree(struct kho_radix_tree *tree,
474 const struct kho_radix_walk_cb *cb, void *data)
475{
476 if (WARN_ON_ONCE(!tree->root))
477 return -EINVAL;
478
479 guard(mutex)(&tree->lock);
480
481 return __kho_radix_walk_tree(tree->root, KHO_TREE_MAX_DEPTH - 1, 0, cb,
482 data);
483}
484EXPORT_SYMBOL_GPL(kho_radix_walk_tree);
485
486/* For physically contiguous 0-order pages. */
487static void kho_init_pages(struct page *page, unsigned long nr_pages)
488{
489 for (unsigned long i = 0; i < nr_pages; i++) {
490 set_page_count(page + i, 1);
491 /* Clear each page's codetag to avoid accounting mismatch. */
492 clear_page_tag_ref(page + i);
493 }
494}
495
496static void kho_init_folio(struct page *page, unsigned int order)
497{
498 unsigned long nr_pages = (1 << order);
499
500 /* Head page gets refcount of 1. */
501 set_page_count(page, 1);
502 /* Clear head page's codetag to avoid accounting mismatch. */
503 clear_page_tag_ref(page);
504
505 /* For higher order folios, tail pages get a page count of zero. */
506 for (unsigned long i = 1; i < nr_pages; i++)
507 set_page_count(page + i, 0);
508
509 if (order > 0)
510 prep_compound_page(page, order);
511}
512
513static struct page *kho_restore_page(phys_addr_t phys, bool is_folio)
514{
515 struct page *page = pfn_to_online_page(PHYS_PFN(phys));
516 unsigned long nr_pages;
517 union kho_page_info info;
518
519 if (!page)
520 return NULL;
521
522 info.page_private = page->private;
523 /*
524 * deserialize_bitmap() only sets the magic on the head page. This magic
525 * check also implicitly makes sure phys is order-aligned since for
526 * non-order-aligned phys addresses, magic will never be set.
527 */
528 if (WARN_ON_ONCE(info.magic != KHO_PAGE_MAGIC))
529 return NULL;
530 nr_pages = (1 << info.order);
531
532 /* Clear private to make sure later restores on this page error out. */
533 page->private = 0;
534
535 if (is_folio)
536 kho_init_folio(page, info.order);
537 else
538 kho_init_pages(page, nr_pages);
539
540 adjust_managed_page_count(page, nr_pages);
541 return page;
542}
543
544/**
545 * kho_restore_folio - recreates the folio from the preserved memory.
546 * @phys: physical address of the folio.
547 *
548 * Return: pointer to the struct folio on success, NULL on failure.
549 */
550struct folio *kho_restore_folio(phys_addr_t phys)
551{
552 struct page *page = kho_restore_page(phys, true);
553
554 return page ? page_folio(page) : NULL;
555}
556EXPORT_SYMBOL_GPL(kho_restore_folio);
557
558/**
559 * kho_restore_pages - restore list of contiguous order 0 pages.
560 * @phys: physical address of the first page.
561 * @nr_pages: number of pages.
562 *
563 * Restore a contiguous list of order 0 pages that was preserved with
564 * kho_preserve_pages().
565 *
566 * Return: the first page on success, NULL on failure.
567 */
568struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages)
569{
570 const unsigned long start_pfn = PHYS_PFN(phys);
571 const unsigned long end_pfn = start_pfn + nr_pages;
572 unsigned long pfn = start_pfn;
573
574 while (pfn < end_pfn) {
575 const unsigned int order =
576 min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
577 struct page *page = kho_restore_page(PFN_PHYS(pfn), false);
578
579 if (!page)
580 return NULL;
581 pfn += 1 << order;
582 }
583
584 return pfn_to_page(start_pfn);
585}
586EXPORT_SYMBOL_GPL(kho_restore_pages);
587
588/*
589 * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, struct pages in higher memory regions
590 * may not be initialized yet at the time KHO deserializes preserved memory.
591 * KHO uses the struct page to store metadata and a later initialization would
592 * overwrite it.
593 * Ensure all the struct pages in the preservation are
594 * initialized. kho_preserved_memory_reserve() marks the reservation as noinit
595 * to make sure they don't get re-initialized later.
596 */
597static struct page *__init kho_get_preserved_page(phys_addr_t phys,
598 unsigned int order)
599{
600 unsigned long pfn = PHYS_PFN(phys);
601 int nid;
602
603 if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT))
604 return pfn_to_page(pfn);
605
606 nid = early_pfn_to_nid(pfn);
607 for (unsigned long i = 0; i < (1UL << order); i++)
608 init_deferred_page(pfn + i, nid);
609
610 return pfn_to_page(pfn);
611}
612
613static int __init kho_preserved_memory_reserve(unsigned long key, void *data)
614{
615 union kho_page_info info;
616 struct page *page;
617 unsigned int order;
618 phys_addr_t phys;
619 u64 sz;
620
621 phys = kho_decode_radix_key(key, &order);
622
623 sz = 1UL << (order + PAGE_SHIFT);
624 page = kho_get_preserved_page(phys, order);
625
626 /* Reserve the memory preserved in KHO in memblock */
627 memblock_reserve(phys, sz);
628 memblock_reserved_mark_noinit(phys, sz);
629 info.magic = KHO_PAGE_MAGIC;
630 info.order = order;
631 page->private = info.page_private;
632
633 return 0;
634}
635
636/* Returns physical address of the preserved memory map from FDT */
637static phys_addr_t __init kho_get_mem_map_phys(const void *fdt)
638{
639 const void *mem_ptr;
640 int len;
641
642 mem_ptr = fdt_getprop(fdt, 0, KHO_FDT_MEMORY_MAP_PROP_NAME, &len);
643 if (!mem_ptr || len != sizeof(u64)) {
644 pr_err("failed to get preserved memory map\n");
645 return 0;
646 }
647
648 return get_unaligned((const u64 *)mem_ptr);
649}
650
651static void __init *kho_get_mem_map(const void *fdt)
652{
653 phys_addr_t phys = kho_get_mem_map_phys(fdt);
654
655 return phys ? phys_to_virt(phys) : NULL;
656}
657
658/*
659 * With KHO enabled, memory can become fragmented because KHO regions may
660 * be anywhere in physical address space. The scratch regions give us a
661 * safe zones that we will never see KHO allocations from. This is where we
662 * can later safely load our new kexec images into and then use the scratch
663 * area for early allocations that happen before page allocator is
664 * initialized.
665 */
666struct kho_scratch *kho_scratch;
667unsigned int kho_scratch_cnt;
668
669/*
670 * The scratch areas are scaled by default as percent of memory allocated from
671 * memblock. A user can override the scale with command line parameter:
672 *
673 * kho_scratch=N%
674 *
675 * It is also possible to explicitly define size for a lowmem, a global and
676 * per-node scratch areas:
677 *
678 * kho_scratch=l[KMG],n[KMG],m[KMG]
679 *
680 * The explicit size definition takes precedence over scale definition.
681 */
682static unsigned int scratch_scale __initdata = 200;
683static phys_addr_t scratch_size_global __initdata;
684static phys_addr_t scratch_size_pernode __initdata;
685static phys_addr_t scratch_size_lowmem __initdata;
686
687static int __init kho_parse_scratch_size(char *p)
688{
689 size_t len;
690 unsigned long sizes[3];
691 size_t total_size = 0;
692 int i;
693
694 if (!p)
695 return -EINVAL;
696
697 len = strlen(p);
698 if (!len)
699 return -EINVAL;
700
701 /* parse nn% */
702 if (p[len - 1] == '%') {
703 /* unsigned int max is 4,294,967,295, 10 chars */
704 char s_scale[11] = {};
705 int ret = 0;
706
707 if (len > ARRAY_SIZE(s_scale))
708 return -EINVAL;
709
710 memcpy(s_scale, p, len - 1);
711 ret = kstrtouint(s_scale, 10, &scratch_scale);
712 if (!ret)
713 pr_notice("scratch scale is %d%%\n", scratch_scale);
714 return ret;
715 }
716
717 /* parse ll[KMG],mm[KMG],nn[KMG] */
718 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
719 char *endp = p;
720
721 if (i > 0) {
722 if (*p != ',')
723 return -EINVAL;
724 p += 1;
725 }
726
727 sizes[i] = memparse(p, &endp);
728 if (endp == p)
729 return -EINVAL;
730 p = endp;
731 total_size += sizes[i];
732 }
733
734 if (!total_size)
735 return -EINVAL;
736
737 /* The string should be fully consumed by now. */
738 if (*p)
739 return -EINVAL;
740
741 scratch_size_lowmem = sizes[0];
742 scratch_size_global = sizes[1];
743 scratch_size_pernode = sizes[2];
744 scratch_scale = 0;
745
746 pr_notice("scratch areas: lowmem: %lluMiB global: %lluMiB pernode: %lldMiB\n",
747 (u64)(scratch_size_lowmem >> 20),
748 (u64)(scratch_size_global >> 20),
749 (u64)(scratch_size_pernode >> 20));
750
751 return 0;
752}
753early_param("kho_scratch", kho_parse_scratch_size);
754
755static void __init scratch_size_update(void)
756{
757 /*
758 * If fixed sizes are not provided via command line, calculate them now.
759 * Remove HugeTLB allocations from it because they never get allocated
760 * from scratch.
761 */
762 if (scratch_scale) {
763 phys_addr_t size;
764
765 size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT,
766 NUMA_NO_NODE);
767 size -= memblock_reserved_hugetlb_size(ARCH_LOW_ADDRESS_LIMIT,
768 NUMA_NO_NODE);
769 size = size * scratch_scale / 100;
770 scratch_size_lowmem = size;
771
772 size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
773 NUMA_NO_NODE);
774 size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE,
775 NUMA_NO_NODE);
776 size = size * scratch_scale / 100 - scratch_size_lowmem;
777 scratch_size_global = size;
778 }
779
780 /*
781 * Scratch areas are released as MIGRATE_CMA. Round them up to the right
782 * size.
783 */
784 scratch_size_lowmem = round_up(scratch_size_lowmem, SCRATCH_ALIGNMENT_BYTES);
785 scratch_size_global = round_up(scratch_size_global, SCRATCH_ALIGNMENT_BYTES);
786}
787
788static phys_addr_t __init scratch_size_node(int nid)
789{
790 phys_addr_t size;
791
792 if (scratch_scale) {
793 size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
794 nid);
795 /* Do not count HugeTLB pages. */
796 size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE,
797 nid);
798 size = size * scratch_scale / 100;
799 } else {
800 size = scratch_size_pernode;
801 }
802
803 return round_up(size, SCRATCH_ALIGNMENT_BYTES);
804}
805
806bool kho_scratch_overlap(phys_addr_t phys, size_t size)
807{
808 phys_addr_t scratch_start, scratch_end;
809 unsigned int i;
810
811 for (i = 0; i < kho_scratch_cnt; i++) {
812 scratch_start = kho_scratch[i].addr;
813 scratch_end = kho_scratch[i].addr + kho_scratch[i].size;
814
815 if (phys < scratch_end && (phys + size) > scratch_start)
816 return true;
817 }
818
819 return false;
820}
821
822/**
823 * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
824 *
825 * With KHO we can preserve arbitrary pages in the system. To ensure we still
826 * have a large contiguous region of memory when we search the physical address
827 * space for target memory, let's make sure we always have a large CMA region
828 * active. This CMA region will only be used for movable pages which are not a
829 * problem for us during KHO because we can just move them somewhere else.
830 */
831static void __init kho_reserve_scratch(void)
832{
833 phys_addr_t addr, size;
834 int nid, i = 0;
835
836 if (!kho_enable)
837 return;
838
839 scratch_size_update();
840
841 /* FIXME: deal with node hot-plug/remove */
842 kho_scratch_cnt = nodes_weight(node_states[N_MEMORY]) + 2;
843 size = kho_scratch_cnt * sizeof(*kho_scratch);
844 kho_scratch = memblock_alloc(size, PAGE_SIZE);
845 if (!kho_scratch) {
846 pr_err("Failed to reserve scratch array\n");
847 goto err_disable_kho;
848 }
849
850 /*
851 * reserve scratch area in low memory for lowmem allocations in the
852 * next kernel
853 */
854 size = scratch_size_lowmem;
855 addr = memblock_phys_alloc_range(size, SCRATCH_ALIGNMENT_BYTES, 0,
856 ARCH_LOW_ADDRESS_LIMIT);
857 if (!addr) {
858 pr_err("Failed to reserve lowmem scratch buffer\n");
859 goto err_free_scratch_desc;
860 }
861
862 kho_scratch[i].addr = addr;
863 kho_scratch[i].size = size;
864 i++;
865
866 /* reserve large contiguous area for allocations without nid */
867 size = scratch_size_global;
868 addr = memblock_phys_alloc(size, SCRATCH_ALIGNMENT_BYTES);
869 if (!addr) {
870 pr_err("Failed to reserve global scratch buffer\n");
871 goto err_free_scratch_areas;
872 }
873
874 kho_scratch[i].addr = addr;
875 kho_scratch[i].size = size;
876 i++;
877
878 /*
879 * Loop over nodes that have both memory and are online. Skip
880 * memoryless nodes, as we can not allocate scratch areas there.
881 */
882 for_each_node_state(nid, N_MEMORY) {
883 size = scratch_size_node(nid);
884 addr = memblock_alloc_range_nid(size, SCRATCH_ALIGNMENT_BYTES,
885 0, MEMBLOCK_ALLOC_ACCESSIBLE,
886 nid, true);
887 if (!addr) {
888 pr_err("Failed to reserve nid %d scratch buffer\n", nid);
889 goto err_free_scratch_areas;
890 }
891
892 kho_scratch[i].addr = addr;
893 kho_scratch[i].size = size;
894 i++;
895 }
896
897 return;
898
899err_free_scratch_areas:
900 for (i--; i >= 0; i--)
901 memblock_phys_free(kho_scratch[i].addr, kho_scratch[i].size);
902err_free_scratch_desc:
903 memblock_free(kho_scratch, kho_scratch_cnt * sizeof(*kho_scratch));
904err_disable_kho:
905 pr_warn("Failed to reserve scratch area, disabling kexec handover\n");
906 kho_enable = false;
907}
908
909/*
910 * Look for free blocks of 1G. This is a heuristic chosen to work efficiently
911 * with large systems with hundreds of gigabytes of memory. It will work poorly
912 * on smaller systems. The algorithm itself doesn't depend on the actual value,
913 * so it can be changed to a different heuristic later if needed.
914 */
915#define KHO_SCRATCH_EXT_BLKSIZE SZ_1G
916#define KHO_SCRATCH_EXT_BLKSHIFT const_ilog2(KHO_SCRATCH_EXT_BLKSIZE)
917
918/* Called for the KHO preserved memory radix tree. */
919static int __init kho_ext_walk_leaf(unsigned long key, void *data)
920{
921 struct kho_radix_tree *busy_blocks = data;
922 phys_addr_t start, end;
923 unsigned int order;
924 int err;
925
926 /*
927 * The key is from the KHO preserved memory radix tree. It is decoded to
928 * a physical address of a preservation and its order.
929 */
930 start = kho_decode_radix_key(key, &order);
931 end = start + (1UL << (order + PAGE_SHIFT));
932
933 while (start < end) {
934 err = kho_radix_add_key(busy_blocks, start >> KHO_SCRATCH_EXT_BLKSHIFT);
935 if (err)
936 return err;
937
938 start += (1UL << KHO_SCRATCH_EXT_BLKSHIFT);
939 }
940
941 return 0;
942}
943
944/* Called for the KHO preserved memory radix tree. */
945static int __init kho_ext_walk_node(phys_addr_t phys, void *data)
946{
947 struct kho_radix_tree *busy_blocks = data;
948
949 return kho_radix_add_key(busy_blocks, phys >> KHO_SCRATCH_EXT_BLKSHIFT);
950}
951
952/* Called for the busy block radix tree. */
953static int __init kho_ext_mark_scratch(unsigned long key, void *data)
954{
955 phys_addr_t *prev_end = data;
956 phys_addr_t start = key << KHO_SCRATCH_EXT_BLKSHIFT;
957 int err;
958
959 if (start > *prev_end) {
960 err = memblock_mark_kho_scratch(*prev_end, start - *prev_end);
961 if (err)
962 return err;
963 }
964
965 *prev_end = start + (1UL << KHO_SCRATCH_EXT_BLKSHIFT);
966 return 0;
967}
968
969/*
970 * kho_extend_scratch - Extend the scratch regions
971 *
972 * The KHO preserved memory radix tree mixes both physical address and order
973 * into a single key. This makes it hard to look for free ranges directly. This
974 * function first walks the radix tree and digests it down into another radix
975 * tree, whose keys identify blocks of size KHO_SCRATCH_EXT_BLKSIZE which
976 * contain preserved memory.
977 *
978 * Then it walks the digested radix tree and marks everything that doesn't have
979 * preserved memory as scratch.
980 *
981 * NOTE: This function allocates memory so it should be called when scratch has
982 * available space.
983 *
984 * NOTE: The pages of the KHO preserved memory radix tree tables are not marked
985 * as preserved in the preserved memory tree. But they are expected to remain
986 * untouched until the tree is fully parsed. So this function also considers
987 * them to be "preserved memory" and marks their blocks as busy.
988 *
989 * NOTE: efi_init()::reserve_regions() removes all regions except
990 * MEMBLOCK_KHO_SCRATCH. This function adds such regions but they are not KHO
991 * scratch memory, so they should not be removed. This function should always be
992 * called after reserve_regions().
993 */
994static void __init kho_extend_scratch(void)
995{
996 const struct kho_radix_walk_cb kho_cb = {
997 .leaf = kho_ext_walk_leaf,
998 .node = kho_ext_walk_node,
999 };
1000 const struct kho_radix_walk_cb ext_cb = {
1001 .leaf = kho_ext_mark_scratch,
1002 };
1003 static struct lock_class_key busy_radix_class;
1004 struct kho_radix_tree busy_blocks;
1005 phys_addr_t prev_end = 0;
1006 int err = 0;
1007
1008 err = kho_radix_init_tree(&busy_blocks, NULL);
1009 if (err)
1010 goto print;
1011
1012 /*
1013 * The walk of kho_in.radix_tree adds keys to busy_blocks. The walk
1014 * takes the kho_in radix tree lock and adding the key takes busy_blocks
1015 * lock. Since both are struct kho_radix_tree and share the same lock
1016 * class, lockdep gets confused. Set a different class for
1017 * busy_blocks.lock to make lockdep happy.
1018 */
1019 lockdep_set_class(&busy_blocks.lock, &busy_radix_class);
1020
1021 /* Walk the KHO radix tree to find busy blocks. */
1022 err = kho_radix_walk_tree(&kho_in.radix_tree, &kho_cb, &busy_blocks);
1023 if (err)
1024 goto out;
1025
1026 /* Walk the busy blocks and mark everything between keys as scratch. */
1027 err = kho_radix_walk_tree(&busy_blocks, &ext_cb, &prev_end);
1028 if (err)
1029 goto out;
1030
1031 /* Mark everything from last busy block to end of DRAM. */
1032 if (prev_end < memblock_end_of_DRAM())
1033 err = memblock_mark_kho_scratch(prev_end, memblock_end_of_DRAM() - prev_end);
1034
1035 /* fallthrough */
1036out:
1037 kho_radix_destroy_tree(&busy_blocks);
1038print:
1039 if (err)
1040 pr_err("Failed to extend scratch: %pe\n", ERR_PTR(err));
1041}
1042
1043/**
1044 * kho_add_subtree - record the physical address of a sub blob in KHO root tree.
1045 * @name: name of the sub tree.
1046 * @blob: the sub tree blob.
1047 * @size: size of the blob in bytes.
1048 *
1049 * Creates a new child node named @name in KHO root FDT and records
1050 * the physical address of @blob. The pages of @blob must also be preserved
1051 * by KHO for the new kernel to retrieve it after kexec.
1052 *
1053 * A debugfs blob entry is also created at
1054 * ``/sys/kernel/debug/kho/out/sub_fdts/@name`` when kernel is configured with
1055 * CONFIG_KEXEC_HANDOVER_DEBUGFS
1056 *
1057 * Return: 0 on success, error code on failure
1058 */
1059int kho_add_subtree(const char *name, void *blob, size_t size)
1060{
1061 phys_addr_t phys = virt_to_phys(blob);
1062 void *root_fdt = kho_out.fdt;
1063 u64 size_u64 = size;
1064 int err = -ENOMEM;
1065 int off, fdt_err;
1066
1067 guard(mutex)(&kho_out.lock);
1068
1069 fdt_err = fdt_open_into(root_fdt, root_fdt, PAGE_SIZE);
1070 if (fdt_err < 0)
1071 return err;
1072
1073 off = fdt_add_subnode(root_fdt, 0, name);
1074 if (off < 0) {
1075 if (off == -FDT_ERR_EXISTS)
1076 err = -EEXIST;
1077 goto out_pack;
1078 }
1079
1080 fdt_err = fdt_setprop(root_fdt, off, KHO_SUB_TREE_PROP_NAME,
1081 &phys, sizeof(phys));
1082 if (fdt_err < 0)
1083 goto out_del_node;
1084
1085 fdt_err = fdt_setprop(root_fdt, off, KHO_SUB_TREE_SIZE_PROP_NAME,
1086 &size_u64, sizeof(size_u64));
1087 if (fdt_err < 0)
1088 goto out_del_node;
1089
1090 WARN_ON_ONCE(kho_debugfs_blob_add(&kho_out.dbg, name, blob,
1091 size, false));
1092
1093 err = 0;
1094 goto out_pack;
1095
1096out_del_node:
1097 fdt_del_node(root_fdt, off);
1098out_pack:
1099 fdt_pack(root_fdt);
1100
1101 return err;
1102}
1103EXPORT_SYMBOL_GPL(kho_add_subtree);
1104
1105void kho_remove_subtree(void *blob)
1106{
1107 phys_addr_t target_phys = virt_to_phys(blob);
1108 void *root_fdt = kho_out.fdt;
1109 int off;
1110 int err;
1111
1112 guard(mutex)(&kho_out.lock);
1113
1114 err = fdt_open_into(root_fdt, root_fdt, PAGE_SIZE);
1115 if (err < 0)
1116 return;
1117
1118 for (off = fdt_first_subnode(root_fdt, 0); off >= 0;
1119 off = fdt_next_subnode(root_fdt, off)) {
1120 const u64 *val;
1121 int len;
1122
1123 val = fdt_getprop(root_fdt, off, KHO_SUB_TREE_PROP_NAME, &len);
1124 if (!val || len != sizeof(phys_addr_t))
1125 continue;
1126
1127 if ((phys_addr_t)*val == target_phys) {
1128 fdt_del_node(root_fdt, off);
1129 kho_debugfs_blob_remove(&kho_out.dbg, blob);
1130 break;
1131 }
1132 }
1133
1134 fdt_pack(root_fdt);
1135}
1136EXPORT_SYMBOL_GPL(kho_remove_subtree);
1137
1138/**
1139 * kho_preserve_folio - preserve a folio across kexec.
1140 * @folio: folio to preserve.
1141 *
1142 * Instructs KHO to preserve the whole folio across kexec. The order
1143 * will be preserved as well.
1144 *
1145 * Return: 0 on success, error code on failure
1146 */
1147int kho_preserve_folio(struct folio *folio)
1148{
1149 struct kho_radix_tree *tree = &kho_out.radix_tree;
1150 const unsigned long pfn = folio_pfn(folio);
1151 const unsigned int order = folio_order(folio);
1152
1153 if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) &&
1154 WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
1155 return -EINVAL;
1156
1157 return kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
1158 order));
1159}
1160EXPORT_SYMBOL_GPL(kho_preserve_folio);
1161
1162/**
1163 * kho_unpreserve_folio - unpreserve a folio.
1164 * @folio: folio to unpreserve.
1165 *
1166 * Instructs KHO to unpreserve a folio that was preserved by
1167 * kho_preserve_folio() before. The provided @folio (pfn and order)
1168 * must exactly match a previously preserved folio.
1169 */
1170void kho_unpreserve_folio(struct folio *folio)
1171{
1172 struct kho_radix_tree *tree = &kho_out.radix_tree;
1173 const unsigned long pfn = folio_pfn(folio);
1174 const unsigned int order = folio_order(folio);
1175
1176 kho_radix_del_key(tree, kho_encode_radix_key(PFN_PHYS(pfn), order));
1177}
1178EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
1179
1180static unsigned int __kho_preserve_pages_order(unsigned long start_pfn,
1181 unsigned long end_pfn)
1182{
1183 unsigned int order = min(count_trailing_zeros(start_pfn),
1184 ilog2(end_pfn - start_pfn));
1185
1186 /*
1187 * Make sure all the pages in a single preservation are in the same NUMA
1188 * node. The restore machinery can not cope with a preservation spanning
1189 * multiple NUMA nodes.
1190 */
1191 while (pfn_to_nid(start_pfn) != pfn_to_nid(start_pfn + (1UL << order) - 1))
1192 order--;
1193
1194 return order;
1195}
1196
1197static void __kho_unpreserve(struct kho_radix_tree *tree,
1198 unsigned long pfn, unsigned long end_pfn)
1199{
1200 unsigned int order;
1201
1202 while (pfn < end_pfn) {
1203 order = __kho_preserve_pages_order(pfn, end_pfn);
1204
1205 kho_radix_del_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
1206 order));
1207
1208 pfn += 1 << order;
1209 }
1210}
1211
1212/**
1213 * kho_preserve_pages - preserve contiguous pages across kexec
1214 * @page: first page in the list.
1215 * @nr_pages: number of pages.
1216 *
1217 * Preserve a contiguous list of order 0 pages. Must be restored using
1218 * kho_restore_pages() to ensure the pages are restored properly as order 0.
1219 *
1220 * Return: 0 on success, error code on failure
1221 */
1222int kho_preserve_pages(struct page *page, unsigned long nr_pages)
1223{
1224 struct kho_radix_tree *tree = &kho_out.radix_tree;
1225 const unsigned long start_pfn = page_to_pfn(page);
1226 const unsigned long end_pfn = start_pfn + nr_pages;
1227 unsigned long pfn = start_pfn;
1228 unsigned long failed_pfn = 0;
1229 int err = 0;
1230
1231 if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) &&
1232 WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT,
1233 nr_pages << PAGE_SHIFT))) {
1234 return -EINVAL;
1235 }
1236
1237 while (pfn < end_pfn) {
1238 unsigned int order = __kho_preserve_pages_order(pfn, end_pfn);
1239
1240 err = kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
1241 order));
1242 if (err) {
1243 failed_pfn = pfn;
1244 break;
1245 }
1246
1247 pfn += 1 << order;
1248 }
1249
1250 if (err)
1251 __kho_unpreserve(tree, start_pfn, failed_pfn);
1252
1253 return err;
1254}
1255EXPORT_SYMBOL_GPL(kho_preserve_pages);
1256
1257/**
1258 * kho_unpreserve_pages - unpreserve contiguous pages.
1259 * @page: first page in the list.
1260 * @nr_pages: number of pages.
1261 *
1262 * Instructs KHO to unpreserve @nr_pages contiguous pages starting from @page.
1263 * This must be called with the same @page and @nr_pages as the corresponding
1264 * kho_preserve_pages() call. Unpreserving arbitrary sub-ranges of larger
1265 * preserved blocks is not supported.
1266 */
1267void kho_unpreserve_pages(struct page *page, unsigned long nr_pages)
1268{
1269 struct kho_radix_tree *tree = &kho_out.radix_tree;
1270 const unsigned long start_pfn = page_to_pfn(page);
1271 const unsigned long end_pfn = start_pfn + nr_pages;
1272
1273 __kho_unpreserve(tree, start_pfn, end_pfn);
1274}
1275EXPORT_SYMBOL_GPL(kho_unpreserve_pages);
1276
1277/* vmalloc flags KHO supports */
1278#define KHO_VMALLOC_SUPPORTED_FLAGS (VM_ALLOC | VM_ALLOW_HUGE_VMAP)
1279
1280/* KHO internal flags for vmalloc preservations */
1281#define KHO_VMALLOC_ALLOC 0x0001
1282#define KHO_VMALLOC_HUGE_VMAP 0x0002
1283
1284static unsigned short vmalloc_flags_to_kho(unsigned int vm_flags)
1285{
1286 unsigned short kho_flags = 0;
1287
1288 if (vm_flags & VM_ALLOC)
1289 kho_flags |= KHO_VMALLOC_ALLOC;
1290 if (vm_flags & VM_ALLOW_HUGE_VMAP)
1291 kho_flags |= KHO_VMALLOC_HUGE_VMAP;
1292
1293 return kho_flags;
1294}
1295
1296static unsigned int kho_flags_to_vmalloc(unsigned short kho_flags)
1297{
1298 unsigned int vm_flags = 0;
1299
1300 if (kho_flags & KHO_VMALLOC_ALLOC)
1301 vm_flags |= VM_ALLOC;
1302 if (kho_flags & KHO_VMALLOC_HUGE_VMAP)
1303 vm_flags |= VM_ALLOW_HUGE_VMAP;
1304
1305 return vm_flags;
1306}
1307
1308static struct kho_vmalloc_chunk *new_vmalloc_chunk(struct kho_vmalloc_chunk *cur)
1309{
1310 struct kho_vmalloc_chunk *chunk;
1311 int err;
1312
1313 chunk = (struct kho_vmalloc_chunk *)get_zeroed_page(GFP_KERNEL);
1314 if (!chunk)
1315 return NULL;
1316
1317 err = kho_preserve_pages(virt_to_page(chunk), 1);
1318 if (err)
1319 goto err_free;
1320 if (cur)
1321 KHOSER_STORE_PTR(cur->hdr.next, chunk);
1322 return chunk;
1323
1324err_free:
1325 free_page((unsigned long)chunk);
1326 return NULL;
1327}
1328
1329static void kho_vmalloc_unpreserve_chunk(struct kho_vmalloc_chunk *chunk,
1330 unsigned short order)
1331{
1332 struct kho_radix_tree *tree = &kho_out.radix_tree;
1333 unsigned long pfn = PHYS_PFN(virt_to_phys(chunk));
1334
1335 __kho_unpreserve(tree, pfn, pfn + 1);
1336
1337 for (int i = 0; i < ARRAY_SIZE(chunk->phys) && chunk->phys[i]; i++) {
1338 pfn = PHYS_PFN(chunk->phys[i]);
1339 __kho_unpreserve(tree, pfn, pfn + (1 << order));
1340 }
1341}
1342
1343/**
1344 * kho_preserve_vmalloc - preserve memory allocated with vmalloc() across kexec
1345 * @ptr: pointer to the area in vmalloc address space
1346 * @preservation: placeholder for preservation metadata
1347 *
1348 * Instructs KHO to preserve the area in vmalloc address space at @ptr. The
1349 * physical pages mapped at @ptr will be preserved and on successful return
1350 * @preservation will hold the physical address of a structure that describes
1351 * the preservation.
1352 *
1353 * NOTE: The memory allocated with vmalloc_node() variants cannot be reliably
1354 * restored on the same node
1355 *
1356 * Return: 0 on success, error code on failure
1357 */
1358int kho_preserve_vmalloc(void *ptr, struct kho_vmalloc *preservation)
1359{
1360 struct kho_vmalloc_chunk *chunk;
1361 struct vm_struct *vm = find_vm_area(ptr);
1362 unsigned int order, flags, nr_contig_pages;
1363 unsigned int idx = 0;
1364 int err;
1365
1366 if (!vm)
1367 return -EINVAL;
1368
1369 if (vm->flags & ~KHO_VMALLOC_SUPPORTED_FLAGS)
1370 return -EOPNOTSUPP;
1371
1372 flags = vmalloc_flags_to_kho(vm->flags);
1373 order = get_vm_area_page_order(vm);
1374
1375 chunk = new_vmalloc_chunk(NULL);
1376 if (!chunk)
1377 return -ENOMEM;
1378 KHOSER_STORE_PTR(preservation->first, chunk);
1379
1380 nr_contig_pages = (1 << order);
1381 for (int i = 0; i < vm->nr_pages; i += nr_contig_pages) {
1382 phys_addr_t phys = page_to_phys(vm->pages[i]);
1383
1384 err = kho_preserve_pages(vm->pages[i], nr_contig_pages);
1385 if (err)
1386 goto err_free;
1387
1388 chunk->phys[idx++] = phys;
1389 if (idx == ARRAY_SIZE(chunk->phys)) {
1390 chunk = new_vmalloc_chunk(chunk);
1391 if (!chunk) {
1392 err = -ENOMEM;
1393 goto err_free;
1394 }
1395 idx = 0;
1396 }
1397 }
1398
1399 preservation->total_pages = vm->nr_pages;
1400 preservation->flags = flags;
1401 preservation->order = order;
1402
1403 return 0;
1404
1405err_free:
1406 kho_unpreserve_vmalloc(preservation);
1407 return err;
1408}
1409EXPORT_SYMBOL_GPL(kho_preserve_vmalloc);
1410
1411/**
1412 * kho_unpreserve_vmalloc - unpreserve memory allocated with vmalloc()
1413 * @preservation: preservation metadata returned by kho_preserve_vmalloc()
1414 *
1415 * Instructs KHO to unpreserve the area in vmalloc address space that was
1416 * previously preserved with kho_preserve_vmalloc().
1417 */
1418void kho_unpreserve_vmalloc(struct kho_vmalloc *preservation)
1419{
1420 struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(preservation->first);
1421
1422 while (chunk) {
1423 struct kho_vmalloc_chunk *tmp = chunk;
1424
1425 kho_vmalloc_unpreserve_chunk(chunk, preservation->order);
1426
1427 chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
1428 free_page((unsigned long)tmp);
1429 }
1430}
1431EXPORT_SYMBOL_GPL(kho_unpreserve_vmalloc);
1432
1433/**
1434 * kho_restore_vmalloc - recreates and populates an area in vmalloc address
1435 * space from the preserved memory.
1436 * @preservation: preservation metadata.
1437 *
1438 * Recreates an area in vmalloc address space and populates it with memory that
1439 * was preserved using kho_preserve_vmalloc().
1440 *
1441 * Return: pointer to the area in the vmalloc address space, NULL on failure.
1442 */
1443void *kho_restore_vmalloc(const struct kho_vmalloc *preservation)
1444{
1445 struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(preservation->first);
1446 kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_PROT_NORMAL;
1447 unsigned int align, order, shift, vm_flags;
1448 unsigned long total_pages, contig_pages;
1449 unsigned long addr, size;
1450 struct vm_struct *area;
1451 struct page **pages;
1452 unsigned int idx = 0;
1453 int err;
1454
1455 vm_flags = kho_flags_to_vmalloc(preservation->flags);
1456 if (vm_flags & ~KHO_VMALLOC_SUPPORTED_FLAGS)
1457 return NULL;
1458
1459 total_pages = preservation->total_pages;
1460 pages = kvmalloc_objs(*pages, total_pages);
1461 if (!pages)
1462 return NULL;
1463 order = preservation->order;
1464 contig_pages = (1 << order);
1465 shift = PAGE_SHIFT + order;
1466 align = 1 << shift;
1467
1468 while (chunk) {
1469 struct page *page;
1470
1471 for (int i = 0; i < ARRAY_SIZE(chunk->phys) && chunk->phys[i]; i++) {
1472 phys_addr_t phys = chunk->phys[i];
1473
1474 if (idx + contig_pages > total_pages)
1475 goto err_free_pages_array;
1476
1477 page = kho_restore_pages(phys, contig_pages);
1478 if (!page)
1479 goto err_free_pages_array;
1480
1481 for (int j = 0; j < contig_pages; j++)
1482 pages[idx++] = page + j;
1483
1484 phys += contig_pages * PAGE_SIZE;
1485 }
1486
1487 page = kho_restore_pages(virt_to_phys(chunk), 1);
1488 if (!page)
1489 goto err_free_pages_array;
1490 chunk = KHOSER_LOAD_PTR(chunk->hdr.next);
1491 __free_page(page);
1492 }
1493
1494 if (idx != total_pages)
1495 goto err_free_pages_array;
1496
1497 area = __get_vm_area_node(total_pages * PAGE_SIZE, align, shift,
1498 vm_flags | VM_UNINITIALIZED,
1499 VMALLOC_START, VMALLOC_END,
1500 NUMA_NO_NODE, GFP_KERNEL,
1501 __builtin_return_address(0));
1502 if (!area)
1503 goto err_free_pages_array;
1504
1505 addr = (unsigned long)area->addr;
1506 size = get_vm_area_size(area);
1507 err = vmap_pages_range(addr, addr + size, PAGE_KERNEL, pages, shift);
1508 if (err)
1509 goto err_free_vm_area;
1510
1511 area->nr_pages = total_pages;
1512 area->pages = pages;
1513
1514 if (vm_flags & VM_ALLOC)
1515 kasan_flags |= KASAN_VMALLOC_VM_ALLOC;
1516
1517 area->addr = kasan_unpoison_vmalloc(area->addr, total_pages * PAGE_SIZE,
1518 kasan_flags);
1519 clear_vm_uninitialized_flag(area);
1520
1521 return area->addr;
1522
1523err_free_vm_area:
1524 free_vm_area(area);
1525err_free_pages_array:
1526 kvfree(pages);
1527 return NULL;
1528}
1529EXPORT_SYMBOL_GPL(kho_restore_vmalloc);
1530
1531/**
1532 * kho_alloc_preserve - Allocate, zero, and preserve memory.
1533 * @size: The number of bytes to allocate.
1534 *
1535 * Allocates a physically contiguous block of zeroed pages that is large
1536 * enough to hold @size bytes. The allocated memory is then registered with
1537 * KHO for preservation across a kexec.
1538 *
1539 * Note: The actual allocated size will be rounded up to the nearest
1540 * power-of-two page boundary.
1541 *
1542 * @return A virtual pointer to the allocated and preserved memory on success,
1543 * or an ERR_PTR() encoded error on failure.
1544 */
1545void *kho_alloc_preserve(size_t size)
1546{
1547 struct folio *folio;
1548 int order, ret;
1549
1550 if (!size)
1551 return ERR_PTR(-EINVAL);
1552
1553 order = get_order(size);
1554 if (order > MAX_PAGE_ORDER)
1555 return ERR_PTR(-E2BIG);
1556
1557 folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, order);
1558 if (!folio)
1559 return ERR_PTR(-ENOMEM);
1560
1561 ret = kho_preserve_folio(folio);
1562 if (ret) {
1563 folio_put(folio);
1564 return ERR_PTR(ret);
1565 }
1566
1567 return folio_address(folio);
1568}
1569EXPORT_SYMBOL_GPL(kho_alloc_preserve);
1570
1571/**
1572 * kho_unpreserve_free - Unpreserve and free memory.
1573 * @mem: Pointer to the memory allocated by kho_alloc_preserve().
1574 *
1575 * Unregisters the memory from KHO preservation and frees the underlying
1576 * pages back to the system. This function should be called to clean up
1577 * memory allocated with kho_alloc_preserve().
1578 */
1579void kho_unpreserve_free(void *mem)
1580{
1581 struct folio *folio;
1582
1583 if (!mem)
1584 return;
1585
1586 folio = virt_to_folio(mem);
1587 kho_unpreserve_folio(folio);
1588 folio_put(folio);
1589}
1590EXPORT_SYMBOL_GPL(kho_unpreserve_free);
1591
1592/**
1593 * kho_restore_free - Restore and free memory after kexec.
1594 * @mem: Pointer to the memory (in the new kernel's address space)
1595 * that was allocated by the old kernel.
1596 *
1597 * This function is intended to be called in the new kernel (post-kexec)
1598 * to take ownership of and free a memory region that was preserved by the
1599 * old kernel using kho_alloc_preserve().
1600 *
1601 * It first restores the pages from KHO (using their physical address)
1602 * and then frees the pages back to the new kernel's page allocator.
1603 */
1604void kho_restore_free(void *mem)
1605{
1606 struct folio *folio;
1607
1608 if (!mem)
1609 return;
1610
1611 folio = kho_restore_folio(__pa(mem));
1612 if (!WARN_ON(!folio))
1613 folio_put(folio);
1614}
1615EXPORT_SYMBOL_GPL(kho_restore_free);
1616
1617/**
1618 * is_kho_boot - check if current kernel was booted via KHO-enabled
1619 * kexec
1620 *
1621 * This function checks if the current kernel was loaded through a kexec
1622 * operation with KHO enabled, by verifying that a valid KHO FDT
1623 * was passed.
1624 *
1625 * Note: This function returns reliable results only after
1626 * kho_populate() has been called during early boot. Before that,
1627 * it may return false even if KHO data is present.
1628 *
1629 * Return: true if booted via KHO-enabled kexec, false otherwise
1630 */
1631bool is_kho_boot(void)
1632{
1633 return !!kho_get_fdt();
1634}
1635EXPORT_SYMBOL_GPL(is_kho_boot);
1636
1637/**
1638 * kho_retrieve_subtree - retrieve a preserved sub blob by its name.
1639 * @name: the name of the sub blob passed to kho_add_subtree().
1640 * @phys: if found, the physical address of the sub blob is stored in @phys.
1641 * @size: if not NULL and found, the size of the sub blob is stored in @size.
1642 *
1643 * Retrieve a preserved sub blob named @name and store its physical
1644 * address in @phys and optionally its size in @size.
1645 *
1646 * Return: 0 on success, error code on failure
1647 */
1648int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size)
1649{
1650 const void *fdt = kho_get_fdt();
1651 const u64 *val;
1652 int offset, len;
1653
1654 if (!fdt)
1655 return -ENOENT;
1656
1657 if (!phys)
1658 return -EINVAL;
1659
1660 offset = fdt_subnode_offset(fdt, 0, name);
1661 if (offset < 0)
1662 return -ENOENT;
1663
1664 val = fdt_getprop(fdt, offset, KHO_SUB_TREE_PROP_NAME, &len);
1665 if (!val || len != sizeof(*val))
1666 return -EINVAL;
1667
1668 *phys = (phys_addr_t)*val;
1669
1670 val = fdt_getprop(fdt, offset, KHO_SUB_TREE_SIZE_PROP_NAME, &len);
1671 if (!val || len != sizeof(*val)) {
1672 pr_warn("broken KHO subnode '%s': missing or invalid blob-size property\n",
1673 name);
1674 return -EINVAL;
1675 }
1676
1677 if (size)
1678 *size = (size_t)*val;
1679
1680 return 0;
1681}
1682EXPORT_SYMBOL_GPL(kho_retrieve_subtree);
1683
1684static void __init kho_mem_retrieve(void)
1685{
1686 const struct kho_radix_walk_cb cb = {
1687 .leaf = kho_preserved_memory_reserve,
1688 };
1689
1690 if (kho_radix_walk_tree(&kho_in.radix_tree, &cb, NULL))
1691 goto err;
1692
1693 return;
1694
1695err:
1696 /*
1697 * Failed to initialize preserved memory. Clear FDT and radix so KHO
1698 * users don't treat it as a KHO boot.
1699 */
1700 kho_in.fdt_phys = 0;
1701 kho_in.radix_tree.root = NULL;
1702}
1703
1704static __init int kho_out_fdt_setup(void)
1705{
1706 struct kho_radix_tree *tree = &kho_out.radix_tree;
1707 void *root = kho_out.fdt;
1708 u64 preserved_mem_tree_pa;
1709 int err;
1710
1711 err = fdt_create(root, PAGE_SIZE);
1712 err |= fdt_finish_reservemap(root);
1713 err |= fdt_begin_node(root, "");
1714 err |= fdt_property_string(root, "compatible", KHO_FDT_COMPATIBLE);
1715
1716 preserved_mem_tree_pa = virt_to_phys(tree->root);
1717
1718 err |= fdt_property(root, KHO_FDT_MEMORY_MAP_PROP_NAME,
1719 &preserved_mem_tree_pa,
1720 sizeof(preserved_mem_tree_pa));
1721
1722 err |= fdt_end_node(root);
1723 err |= fdt_finish(root);
1724
1725 return err;
1726}
1727
1728static void __init kho_in_kexec_metadata(void)
1729{
1730 struct kho_kexec_metadata *metadata;
1731 phys_addr_t metadata_phys;
1732 size_t blob_size;
1733 int err;
1734
1735 err = kho_retrieve_subtree(KHO_METADATA_NODE_NAME, &metadata_phys,
1736 &blob_size);
1737 if (err)
1738 /* This is fine, previous kernel didn't export metadata */
1739 return;
1740
1741 /* Check that, at least, "version" is present */
1742 if (blob_size < sizeof(u32)) {
1743 pr_warn("kexec-metadata blob too small (%zu bytes)\n",
1744 blob_size);
1745 return;
1746 }
1747
1748 metadata = phys_to_virt(metadata_phys);
1749
1750 if (metadata->version != KHO_KEXEC_METADATA_VERSION) {
1751 pr_warn("kexec-metadata version %u not supported (expected %u)\n",
1752 metadata->version, KHO_KEXEC_METADATA_VERSION);
1753 return;
1754 }
1755
1756 if (blob_size < sizeof(*metadata)) {
1757 pr_warn("kexec-metadata blob too small for v%u (%zu < %zu)\n",
1758 metadata->version, blob_size, sizeof(*metadata));
1759 return;
1760 }
1761
1762 /*
1763 * Copy data to the kernel structure that will persist during
1764 * kernel lifetime.
1765 */
1766 kho_in.kexec_count = metadata->kexec_count;
1767 strscpy(kho_in.previous_release, metadata->previous_release,
1768 sizeof(kho_in.previous_release));
1769
1770 pr_info("exec from: %s (count %u)\n",
1771 kho_in.previous_release, kho_in.kexec_count);
1772}
1773
1774/*
1775 * Create kexec metadata to pass kernel version and boot count to the
1776 * next kernel. This keeps the core KHO ABI minimal and allows the
1777 * metadata format to evolve independently.
1778 */
1779static __init int kho_out_kexec_metadata(void)
1780{
1781 struct kho_kexec_metadata *metadata;
1782 int err;
1783
1784 metadata = kho_alloc_preserve(sizeof(*metadata));
1785 if (IS_ERR(metadata))
1786 return PTR_ERR(metadata);
1787
1788 metadata->version = KHO_KEXEC_METADATA_VERSION;
1789 strscpy(metadata->previous_release, init_uts_ns.name.release,
1790 sizeof(metadata->previous_release));
1791 /* kho_in.kexec_count is set to 0 on cold boot */
1792 metadata->kexec_count = kho_in.kexec_count + 1;
1793
1794 err = kho_add_subtree(KHO_METADATA_NODE_NAME, metadata,
1795 sizeof(*metadata));
1796 if (err)
1797 kho_unpreserve_free(metadata);
1798
1799 return err;
1800}
1801
1802static int __init kho_kexec_metadata_init(const void *fdt)
1803{
1804 int err;
1805
1806 if (fdt)
1807 kho_in_kexec_metadata();
1808
1809 /* Populate kexec metadata for the possible next kexec */
1810 err = kho_out_kexec_metadata();
1811 if (err)
1812 pr_warn("failed to initialize kexec-metadata subtree: %d\n",
1813 err);
1814
1815 return err;
1816}
1817
1818static __init int kho_init(void)
1819{
1820 struct kho_radix_tree *tree = &kho_out.radix_tree;
1821 const void *fdt = kho_get_fdt();
1822 int err = 0;
1823
1824 if (!kho_enable)
1825 return 0;
1826
1827 err = kho_radix_init_tree(tree, NULL);
1828 if (err)
1829 goto err_free_scratch;
1830
1831 kho_out.fdt = kho_alloc_preserve(PAGE_SIZE);
1832 if (IS_ERR(kho_out.fdt)) {
1833 err = PTR_ERR(kho_out.fdt);
1834 goto err_free_kho_radix_tree;
1835 }
1836
1837 err = kho_debugfs_init();
1838 if (err)
1839 goto err_free_fdt;
1840
1841 err = kho_out_debugfs_init(&kho_out.dbg);
1842 if (err)
1843 goto err_free_fdt;
1844
1845 err = kho_out_fdt_setup();
1846 if (err)
1847 goto err_free_fdt;
1848
1849 err = kho_kexec_metadata_init(fdt);
1850 if (err)
1851 goto err_free_fdt;
1852
1853 if (fdt) {
1854 kho_in_debugfs_init(&kho_in.dbg, fdt);
1855 return 0;
1856 }
1857
1858 for (int i = 0; i < kho_scratch_cnt; i++) {
1859 unsigned long base_pfn = PHYS_PFN(kho_scratch[i].addr);
1860 unsigned long count = kho_scratch[i].size >> PAGE_SHIFT;
1861 unsigned long pfn;
1862
1863 /*
1864 * When debug_pagealloc is enabled, __free_pages() clears the
1865 * corresponding PRESENT bit in the kernel page table.
1866 * Subsequent kmemleak scans of these pages cause the
1867 * non-PRESENT page faults.
1868 * Mark scratch areas with kmemleak_ignore_phys() to exclude
1869 * them from kmemleak scanning.
1870 */
1871 kmemleak_ignore_phys(kho_scratch[i].addr);
1872 for (pfn = base_pfn; pfn < base_pfn + count;
1873 pfn += pageblock_nr_pages)
1874 init_cma_reserved_pageblock(pfn_to_page(pfn));
1875 }
1876
1877 WARN_ON_ONCE(kho_debugfs_blob_add(&kho_out.dbg, "fdt",
1878 kho_out.fdt,
1879 fdt_totalsize(kho_out.fdt), true));
1880
1881 return 0;
1882
1883err_free_fdt:
1884 kho_unpreserve_free(kho_out.fdt);
1885err_free_kho_radix_tree:
1886 kho_radix_destroy_tree(tree);
1887err_free_scratch:
1888 kho_out.fdt = NULL;
1889 for (int i = 0; i < kho_scratch_cnt; i++) {
1890 void *start = __va(kho_scratch[i].addr);
1891 void *end = start + kho_scratch[i].size;
1892
1893 free_reserved_area(start, end, -1, "");
1894 }
1895 kho_enable = false;
1896 return err;
1897}
1898fs_initcall(kho_init);
1899
1900void __init kho_memory_init_early(void)
1901{
1902 const void *fdt = kho_get_fdt();
1903 void *mem_map;
1904
1905 if (!is_kho_boot())
1906 return;
1907
1908 /*
1909 * kho_get_mem_map() should always succeed. If it fails, kho_populate()
1910 * catches that and never sets kho_in.scratch_phys, which stops memory
1911 * retrieval.
1912 */
1913 mem_map = kho_get_mem_map(fdt);
1914 if (WARN_ON(!mem_map))
1915 goto err;
1916
1917 /*
1918 * kho_scratch_overlap() needs kho_scratch to be initialized. It
1919 * is used by free_area_init() on KHO boots, so initialize it
1920 * early.
1921 */
1922 kho_scratch = phys_to_virt(kho_in.scratch_phys);
1923
1924 if (kho_radix_init_tree(&kho_in.radix_tree, mem_map))
1925 goto err;
1926
1927 kho_extend_scratch();
1928
1929 return;
1930
1931err:
1932 /*
1933 * Failed to initialize preserved memory radix tree. Clear FDT
1934 * and scratch so KHO users don't treat it as a KHO boot.
1935 */
1936 kho_in.fdt_phys = 0;
1937 kho_in.scratch_phys = 0;
1938}
1939
1940void __init kho_memory_init(void)
1941{
1942 if (kho_in.scratch_phys)
1943 kho_mem_retrieve();
1944 else
1945 kho_reserve_scratch();
1946}
1947
1948void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
1949 phys_addr_t scratch_phys, u64 scratch_len)
1950{
1951 unsigned int scratch_cnt = scratch_len / sizeof(*kho_scratch);
1952 struct kho_scratch *scratch = NULL;
1953 phys_addr_t mem_map_phys;
1954 void *fdt = NULL;
1955 bool populated = false;
1956 int err;
1957
1958 /* Validate the input FDT */
1959 fdt = early_memremap(fdt_phys, fdt_len);
1960 if (!fdt) {
1961 pr_warn("setup: failed to memremap FDT (0x%llx)\n", fdt_phys);
1962 goto report;
1963 }
1964 err = fdt_check_header(fdt);
1965 if (err) {
1966 pr_warn("setup: handover FDT (0x%llx) is invalid: %d\n",
1967 fdt_phys, err);
1968 goto unmap_fdt;
1969 }
1970 err = fdt_node_check_compatible(fdt, 0, KHO_FDT_COMPATIBLE);
1971 if (err) {
1972 pr_warn("setup: handover FDT (0x%llx) is incompatible with '%s': %d\n",
1973 fdt_phys, KHO_FDT_COMPATIBLE, err);
1974 goto unmap_fdt;
1975 }
1976
1977 mem_map_phys = kho_get_mem_map_phys(fdt);
1978 if (!mem_map_phys)
1979 goto unmap_fdt;
1980
1981 scratch = early_memremap(scratch_phys, scratch_len);
1982 if (!scratch) {
1983 pr_warn("setup: failed to memremap scratch (phys=0x%llx, len=%lld)\n",
1984 scratch_phys, scratch_len);
1985 goto unmap_fdt;
1986 }
1987
1988 /*
1989 * We pass a safe contiguous blocks of memory to use for early boot
1990 * purporses from the previous kernel so that we can resize the
1991 * memblock array as needed.
1992 */
1993 for (int i = 0; i < scratch_cnt; i++) {
1994 struct kho_scratch *area = &scratch[i];
1995 u64 size = area->size;
1996
1997 memblock_add(area->addr, size);
1998 err = memblock_mark_kho_scratch(area->addr, size);
1999 if (err) {
2000 pr_warn("failed to mark the scratch region 0x%pa+0x%pa: %pe",
2001 &area->addr, &size, ERR_PTR(err));
2002 goto unmap_scratch;
2003 }
2004 pr_debug("Marked 0x%pa+0x%pa as scratch", &area->addr, &size);
2005 }
2006
2007 memblock_reserve(scratch_phys, scratch_len);
2008
2009 /*
2010 * Now that we have a viable region of scratch memory, let's tell
2011 * the memblocks allocator to only use that for any allocations.
2012 * That way we ensure that nothing scribbles over in use data while
2013 * we initialize the page tables which we will need to ingest all
2014 * memory reservations from the previous kernel.
2015 */
2016 memblock_set_kho_scratch_only();
2017
2018 kho_in.fdt_phys = fdt_phys;
2019 kho_in.scratch_phys = scratch_phys;
2020 kho_scratch_cnt = scratch_cnt;
2021
2022 populated = true;
2023 pr_info("found kexec handover data.\n");
2024
2025unmap_scratch:
2026 early_memunmap(scratch, scratch_len);
2027unmap_fdt:
2028 early_memunmap(fdt, fdt_len);
2029report:
2030 if (!populated)
2031 pr_warn("disabling KHO revival\n");
2032}
2033
2034/* Helper functions for kexec_file_load */
2035
2036int kho_fill_kimage(struct kimage *image)
2037{
2038 ssize_t scratch_size;
2039 int err = 0;
2040 struct kexec_buf scratch;
2041
2042 if (!kho_enable || image->type == KEXEC_TYPE_CRASH)
2043 return 0;
2044
2045 image->kho.fdt = virt_to_phys(kho_out.fdt);
2046
2047 scratch_size = sizeof(*kho_scratch) * kho_scratch_cnt;
2048 scratch = (struct kexec_buf){
2049 .image = image,
2050 .buffer = kho_scratch,
2051 .bufsz = scratch_size,
2052 .mem = KEXEC_BUF_MEM_UNKNOWN,
2053 .memsz = scratch_size,
2054 .buf_align = SZ_64K, /* Makes it easier to map */
2055 .buf_max = ULONG_MAX,
2056 .top_down = true,
2057 };
2058 err = kexec_add_buffer(&scratch);
2059 if (err)
2060 return err;
2061 image->kho.scratch = &image->segment[image->nr_segments - 1];
2062
2063 return 0;
2064}
2065
2066static int kho_walk_scratch(struct kexec_buf *kbuf,
2067 int (*func)(struct resource *, void *))
2068{
2069 int ret = 0;
2070 int i;
2071
2072 for (i = 0; i < kho_scratch_cnt; i++) {
2073 struct resource res = {
2074 .start = kho_scratch[i].addr,
2075 .end = kho_scratch[i].addr + kho_scratch[i].size - 1,
2076 };
2077
2078 /* Try to fit the kimage into our KHO scratch region */
2079 ret = func(&res, kbuf);
2080 if (ret)
2081 break;
2082 }
2083
2084 return ret;
2085}
2086
2087int kho_locate_mem_hole(struct kexec_buf *kbuf,
2088 int (*func)(struct resource *, void *))
2089{
2090 int ret;
2091
2092 if (!kho_enable || kbuf->image->type == KEXEC_TYPE_CRASH)
2093 return 1;
2094
2095 ret = kho_walk_scratch(kbuf, func);
2096
2097 return ret == 1 ? 0 : -EADDRNOTAVAIL;
2098}