diff --git a/Documentation/core-api/kho/abi.rst b/Documentation/core-api/kho/abi.rst index 799d743..edeb5b3 100644 --- a/Documentation/core-api/kho/abi.rst +++ b/Documentation/core-api/kho/abi.rst @@ -28,6 +28,11 @@ KHO persistent memory tracker ABI .. kernel-doc:: include/linux/kho/abi/kexec_handover.h :doc: KHO persistent memory tracker +KHO serialization block ABI +=========================== + +.. kernel-doc:: include/linux/kho/abi/block.h + See Also ======== diff --git a/Documentation/core-api/kho/index.rst b/Documentation/core-api/kho/index.rst index 0a2dee4..320914a 100644 --- a/Documentation/core-api/kho/index.rst +++ b/Documentation/core-api/kho/index.rst @@ -83,6 +83,17 @@ Public API .. kernel-doc:: kernel/liveupdate/kexec_handover.c :export: +KHO Serialization Blocks API +============================ + +.. kernel-doc:: kernel/liveupdate/kho_block.c + :doc: KHO Serialization Blocks + +.. kernel-doc:: include/linux/kho_block.h + +.. kernel-doc:: kernel/liveupdate/kho_block.c + :internal: + See Also ======== diff --git a/Documentation/mm/memfd_preservation.rst b/Documentation/mm/memfd_preservation.rst index a8a5b47..c908a12 100644 --- a/Documentation/mm/memfd_preservation.rst +++ b/Documentation/mm/memfd_preservation.rst @@ -11,7 +11,7 @@ Memfd Preservation ABI ====================== .. kernel-doc:: include/linux/kho/abi/memfd.h - :doc: DOC: memfd Live Update ABI + :doc: memfd Live Update ABI .. kernel-doc:: include/linux/kho/abi/memfd.h :internal: diff --git a/Documentation/userspace-api/liveupdate.rst b/Documentation/userspace-api/liveupdate.rst index 41c0473..cdd7462 100644 --- a/Documentation/userspace-api/liveupdate.rst +++ b/Documentation/userspace-api/liveupdate.rst @@ -14,6 +14,37 @@ ioctl uAPI =========== .. kernel-doc:: include/uapi/linux/liveupdate.h +Note on Compatibility +===================== + +Note that the Live Update feature is still under development and subject to +change, so compatibility is not guaranteed when kexec rebooting between +different kernel versions. This is expected to change and stabilize in a future +version. + +Userspace Integration +===================== + +systemd (since version v261) uses LUO to preserve its per-service file +descriptor store across a kexec-based live update. Services opt in by setting +``FileDescriptorStoreMax=`` and ``FileDescriptorStorePreserve=`` in their unit, +and push file descriptors with a name into the store via +``sd_pid_notify_with_fds(... "FDSTORE=1\nFDNAME=foo")``. + +Services may also create their own LUO sessions (via ``/dev/liveupdate``) and +push the resulting session fds into their file descriptor store like any other +fd. systemd detects such session fds and handles them accordingly, and +hands the re-retrieved session fd back to the service after kexec, using the +existing file descriptor store service interface. + +For details, see: + +- `File Descriptor Store `_ +- `systemd.service(5) FileDescriptorStorePreserve= + `_ +- `sd_pid_notify_with_fds(3) + `_ + See Also ======== diff --git a/MAINTAINERS b/MAINTAINERS index d95d3ef..c97c254 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14301,6 +14301,7 @@ F: Documentation/admin-guide/mm/kho.rst F: Documentation/core-api/kho/* F: include/linux/kexec_handover.h F: include/linux/kho/ +F: include/linux/kho_block.h F: kernel/liveupdate/kexec_handover* F: lib/test_kho.c F: tools/testing/selftests/kho/ diff --git a/include/linux/kho/abi/block.h b/include/linux/kho/abi/block.h new file mode 100644 index 0000000..d06d64b --- /dev/null +++ b/include/linux/kho/abi/block.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +/** + * DOC: KHO Serialization Blocks ABI + * + * Subsystems using the KHO Serialization Blocks framework rely on the stable + * Application Binary Interface defined below to pass serialized state from a + * pre-update kernel to a post-update kernel. + * + * This interface is a contract. Any modification to the structure fields, + * compatible strings, or the layout of the `__packed` serialization + * structures defined here constitutes a breaking change. Such changes require + * incrementing the version number in the `KHO_FDT_COMPATIBLE` string to + * prevent a new kernel from misinterpreting data from an old kernel. + * + * Changes are allowed provided the compatibility version is incremented; + * however, backward/forward compatibility is only guaranteed for kernels + * supporting the same ABI version. + */ + +#ifndef _LINUX_KHO_ABI_BLOCK_H +#define _LINUX_KHO_ABI_BLOCK_H + +#include +#include + +/** + * KHO_BLOCK_SIZE - The size of each serialization block. + * + * This is defined as PAGE_SIZE. PAGE_SIZE is ABI compliant because live + * update between kernels with different page sizes is not supported by KHO. + */ +#define KHO_BLOCK_SIZE PAGE_SIZE + +/** + * struct kho_block_header_ser - Header for the serialized data block. + * @next: Physical address of the next struct kho_block_header_ser. + * @count: The number of entries that immediately follow this header in the + * memory block. + * + * This structure is located at the beginning of a block of physical memory + * preserved across a kexec. It provides the necessary metadata to interpret + * the array of entries that follow. + */ +struct kho_block_header_ser { + u64 next; + u64 count; +} __packed; + +#endif /* _LINUX_KHO_ABI_BLOCK_H */ diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h index db9bda6..5e2eb85 100644 --- a/include/linux/kho/abi/kexec_handover.h +++ b/include/linux/kho/abi/kexec_handover.h @@ -64,7 +64,7 @@ * Root KHO Node (/): * - compatible: "kho-v3" * - * Indentifies the overall KHO ABI version. + * Identifies the overall KHO ABI version. * * - preserved-memory-map: u64 * @@ -90,7 +90,7 @@ */ /* The compatible string for the KHO FDT root node. */ -#define KHO_FDT_COMPATIBLE "kho-v3" +#define KHO_FDT_COMPATIBLE "kho-v4" /* The FDT property for the preserved memory map. */ #define KHO_FDT_MEMORY_MAP_PROP_NAME "preserved-memory-map" diff --git a/include/linux/kho/abi/luo.h b/include/linux/kho/abi/luo.h index 46750a0..288076d 100644 --- a/include/linux/kho/abi/luo.h +++ b/include/linux/kho/abi/luo.h @@ -10,11 +10,11 @@ * * Live Update Orchestrator uses the stable Application Binary Interface * defined below to pass state from a pre-update kernel to a post-update - * kernel. The ABI is built upon the Kexec HandOver framework and uses a - * Flattened Device Tree to describe the preserved data. + * kernel. The ABI is built upon the Kexec HandOver framework and registers + * the central `struct luo_ser` via the KHO raw subtree API. * - * This interface is a contract. Any modification to the FDT structure, node - * properties, compatible strings, or the layout of the `__packed` serialization + * This interface is a contract. Any modification to the structure fields, + * compatible strings, or the layout of the `__packed` serialization * structures defined here constitutes a breaking change. Such changes require * incrementing the version number in the relevant `_COMPATIBLE` string to * prevent a new kernel from misinterpreting data from an old kernel. @@ -23,68 +23,20 @@ * however, backward/forward compatibility is only guaranteed for kernels * supporting the same ABI version. * - * FDT Structure Overview: + * KHO Structure Overview: * The entire LUO state is encapsulated within a single KHO entry named "LUO". - * This entry contains an FDT with the following layout: - * - * .. code-block:: none - * - * / { - * compatible = "luo-v1"; - * liveupdate-number = <...>; - * - * luo-session { - * compatible = "luo-session-v1"; - * luo-session-header = ; - * }; - * - * luo-flb { - * compatible = "luo-flb-v1"; - * luo-flb-header = ; - * }; - * }; - * - * Main LUO Node (/): - * - * - compatible: "luo-v1" - * Identifies the overall LUO ABI version. - * - liveupdate-number: u64 - * A counter tracking the number of successful live updates performed. - * - * Session Node (luo-session): - * This node describes all preserved user-space sessions. - * - * - compatible: "luo-session-v1" - * Identifies the session ABI version. - * - luo-session-header: u64 - * The physical address of a `struct luo_session_header_ser`. This structure - * is the header for a contiguous block of memory containing an array of - * `struct luo_session_ser`, one for each preserved session. - * - * File-Lifecycle-Bound Node (luo-flb): - * This node describes all preserved global objects whose lifecycle is bound - * to that of the preserved files (e.g., shared IOMMU state). - * - * - compatible: "luo-flb-v1" - * Identifies the FLB ABI version. - * - luo-flb-header: u64 - * The physical address of a `struct luo_flb_header_ser`. This structure is - * the header for a contiguous block of memory containing an array of - * `struct luo_flb_ser`, one for each preserved global object. + * This entry contains the `struct luo_ser` structure. * * Serialization Structures: - * The FDT properties point to memory regions containing arrays of simple, - * `__packed` structures. These structures contain the actual preserved state. - * - * - struct luo_session_header_ser: - * Header for the session array. Contains the total page count of the - * preserved memory block and the number of `struct luo_session_ser` - * entries that follow. + * - struct luo_ser: + * The central ABI structure that contains the overall state of the LUO. + * It includes the compatibility string, the liveupdate-number, and pointers + * to sessions and FLBs. * * - struct luo_session_ser: * Metadata for a single session, including its name and a physical pointer - * to another preserved memory block containing an array of - * `struct luo_file_ser` for all files in that session. + * to the first `struct kho_block_header_ser` for all files in that session. + * Multiple blocks are linked via the `next` field in the header. * * - struct luo_file_ser: * Metadata for a single preserved file. Contains the `compatible` string to @@ -105,17 +57,32 @@ #ifndef _LINUX_KHO_ABI_LUO_H #define _LINUX_KHO_ABI_LUO_H +#include +#include #include /* - * The LUO FDT hooks all LUO state for sessions, fds, etc. - * In the root it also carries "liveupdate-number" 64-bit property that - * corresponds to the number of live-updates performed on this machine. + * The LUO state is registered under this KHO entry name. */ -#define LUO_FDT_SIZE PAGE_SIZE -#define LUO_FDT_KHO_ENTRY_NAME "LUO" -#define LUO_FDT_COMPATIBLE "luo-v1" -#define LUO_FDT_LIVEUPDATE_NUM "liveupdate-number" +#define LUO_KHO_ENTRY_NAME "LUO" +#define LUO_ABI_COMPATIBLE "luo-v5" +#define LUO_ABI_COMPAT_LEN ALIGN(sizeof(LUO_ABI_COMPATIBLE), 8) + +/** + * struct luo_ser - Centralized LUO ABI header. + * @compatible: Compatibility string identifying the LUO ABI version. + * @liveupdate_num: A counter tracking the number of successful live updates. + * @sessions_pa: Physical address of the first session block header. + * @flbs_pa: Physical address of the FLB header. + * + * This structure is the root of all preserved LUO state. + */ +struct luo_ser { + char compatible[LUO_ABI_COMPAT_LEN]; + u64 liveupdate_num; + u64 sessions_pa; + u64 flbs_pa; +} __packed; #define LIVEUPDATE_HNDL_COMPAT_LENGTH 48 @@ -125,7 +92,7 @@ * @data: Private data * @token: User provided token for this file * - * If this structure is modified, LUO_SESSION_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_file_ser { char compatible[LIVEUPDATE_HNDL_COMPAT_LENGTH]; @@ -135,9 +102,10 @@ struct luo_file_ser { /** * struct luo_file_set_ser - Represents the serialized metadata for file set - * @files: The physical address of a contiguous memory block that holds - * the serialized state of files (array of luo_file_ser) in this file - * set. + * @files: The physical address of the first `struct kho_block_header_ser`. + * This structure is the header for a block of memory containing + * an array of `struct luo_file_ser` entries. Multiple blocks are + * linked via the `next` field in the header. * @count: The total number of files that were part of this session during * serialization. Used for iteration and validation during * restoration. @@ -147,30 +115,6 @@ struct luo_file_set_ser { u64 count; } __packed; -/* - * LUO FDT session node - * LUO_FDT_SESSION_HEADER: is a u64 physical address of struct - * luo_session_header_ser - */ -#define LUO_FDT_SESSION_NODE_NAME "luo-session" -#define LUO_FDT_SESSION_COMPATIBLE "luo-session-v2" -#define LUO_FDT_SESSION_HEADER "luo-session-header" - -/** - * struct luo_session_header_ser - Header for the serialized session data block. - * @count: The number of `struct luo_session_ser` entries that immediately - * follow this header in the memory block. - * - * This structure is located at the beginning of a contiguous block of - * physical memory preserved across the kexec. It provides the necessary - * metadata to interpret the array of session entries that follow. - * - * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. - */ -struct luo_session_header_ser { - u64 count; -} __packed; - /** * struct luo_session_ser - Represents the serialized metadata for a LUO session. * @name: The unique name of the session, provided by the userspace at @@ -182,7 +126,7 @@ struct luo_session_header_ser { * session) is created and passed to the new kernel, allowing it to reconstruct * the session context. * - * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_session_ser { char name[LIVEUPDATE_SESSION_NAME_LENGTH]; @@ -192,10 +136,6 @@ struct luo_session_ser { /* The max size is set so it can be reliably used during in serialization */ #define LIVEUPDATE_FLB_COMPAT_LENGTH 48 -#define LUO_FDT_FLB_NODE_NAME "luo-flb" -#define LUO_FDT_FLB_COMPATIBLE "luo-flb-v1" -#define LUO_FDT_FLB_HEADER "luo-flb-header" - /** * struct luo_flb_header_ser - Header for the serialized FLB data block. * @pgcnt: The total number of pages occupied by the entire preserved memory @@ -205,11 +145,9 @@ struct luo_session_ser { * in the memory block. * * This structure is located at the physical address specified by the - * `LUO_FDT_FLB_HEADER` FDT property. It provides the new kernel with the - * necessary information to find and iterate over the array of preserved - * File-Lifecycle-Bound objects and to manage the underlying memory. + * flbs_pa in luo_ser. * - * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_flb_header_ser { u64 pgcnt; @@ -231,7 +169,7 @@ struct luo_flb_header_ser { * passed to the new kernel. Each entry allows the LUO core to restore one * global, shared object. * - * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_flb_ser { char name[LIVEUPDATE_FLB_COMPAT_LENGTH]; diff --git a/include/linux/kho_block.h b/include/linux/kho_block.h new file mode 100644 index 0000000..93a7cc2 --- /dev/null +++ b/include/linux/kho_block.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +#ifndef _LINUX_KHO_BLOCK_H +#define _LINUX_KHO_BLOCK_H + +#include +#include +#include + +/** + * struct kho_block - Internal representation of a serialization block. + * @list: List head for linking blocks in memory. + * @ser: Pointer to the serialized header in preserved memory. + */ +struct kho_block { + struct list_head list; + struct kho_block_header_ser *ser; +}; + +/** + * struct kho_block_set - A set of blocks containing serialized entries of the same type. + * @blocks: The list of serialization blocks (struct kho_block). + * @nblocks: The number of allocated serialization blocks. + * @head_pa: Physical address of the first block header. + * @entry_size: The size of each entry in the blocks. + * @count_per_block: The maximum number of entries each block can hold. + * @incoming: True if this block set was restored from the previous kernel. + * + * Note: Synchronization and locking are the responsibility of the caller. + * The block set structure itself is not internally synchronized. + */ +struct kho_block_set { + struct list_head blocks; + long nblocks; + u64 head_pa; + size_t entry_size; + u64 count_per_block; + bool incoming; +}; + +/** + * struct kho_block_set_it - Iterator for serializing entries into blocks. + * @bs: The block set being iterated. + * @block: The current block. + * @i: The current entry index within @block. + */ +struct kho_block_set_it { + struct kho_block_set *bs; + struct kho_block *block; + u64 i; +}; + +/** + * KHO_BLOCK_SET_INIT - Initialize a static kho_block_set. + * @_name: Name of the kho_block_set variable. + * @_entry_size: The size of each entry in the block set. + */ +#define KHO_BLOCK_SET_INIT(_name, _entry_size) { \ + .blocks = LIST_HEAD_INIT((_name).blocks), \ + .entry_size = _entry_size, \ + .count_per_block = (KHO_BLOCK_SIZE - \ + sizeof(struct kho_block_header_ser)) / \ + (_entry_size), \ +} + +void kho_block_set_init(struct kho_block_set *bs, size_t entry_size); + +int kho_block_set_grow(struct kho_block_set *bs, u64 count); +void kho_block_set_shrink(struct kho_block_set *bs, u64 count); + +int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa); +void kho_block_set_destroy(struct kho_block_set *bs); +void kho_block_set_clear(struct kho_block_set *bs); + +/** + * kho_block_set_head_pa - Get the physical address of the first block header. + * @bs: The block set. + * + * Return: The physical address of the first block header, or 0 if empty. + */ +static inline u64 kho_block_set_head_pa(struct kho_block_set *bs) +{ + return bs->head_pa; +} + +/** + * kho_block_set_is_empty - Check if the block set has no allocated blocks. + * @bs: The block set. + * + * Return: True if there are no blocks in the set, false otherwise. + */ +static inline bool kho_block_set_is_empty(struct kho_block_set *bs) +{ + return list_empty(&bs->blocks); +} + +void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs); +void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it); +void *kho_block_set_it_read_entry(struct kho_block_set_it *it); +void *kho_block_set_it_prev(struct kho_block_set_it *it); + +#endif /* _LINUX_KHO_BLOCK_H */ diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index 30c5a39..88722e5 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -175,7 +176,7 @@ struct liveupdate_flb_ops { * @retrieved: True once the FLB's retrieve() callback has run. */ struct luo_flb_private_state { - long count; + refcount_t count; u64 data; void *obj; struct mutex lock; @@ -239,6 +240,8 @@ void liveupdate_unregister_flb(struct liveupdate_file_handler *fh, struct liveupdate_flb *flb); int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); + int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); #else /* CONFIG_LIVEUPDATE */ @@ -279,6 +282,10 @@ static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) +{ +} + static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) { diff --git a/include/linux/memblock.h b/include/linux/memblock.h index b0f750d..5afcd99 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -613,11 +613,28 @@ static inline void memtest_report_meminfo(struct seq_file *m) { } #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH void memblock_set_kho_scratch_only(void); void memblock_clear_kho_scratch_only(void); -void memmap_init_kho_scratch_pages(void); +bool memblock_is_kho_scratch_memory(phys_addr_t addr); + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + if (memblock_is_kho_scratch_memory(PFN_PHYS(pfn))) + return MIGRATE_CMA; + return mt; +} #else static inline void memblock_set_kho_scratch_only(void) { } static inline void memblock_clear_kho_scratch_only(void) { } -static inline void memmap_init_kho_scratch_pages(void) {} +static inline bool memblock_is_kho_scratch_memory(phys_addr_t addr) +{ + return false; +} + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + return mt; +} #endif #endif /* _LINUX_MEMBLOCK_H */ diff --git a/include/uapi/linux/liveupdate.h b/include/uapi/linux/liveupdate.h index 30bc66e..4043d40 100644 --- a/include/uapi/linux/liveupdate.h +++ b/include/uapi/linux/liveupdate.h @@ -59,6 +59,7 @@ enum { LIVEUPDATE_CMD_SESSION_PRESERVE_FD = LIVEUPDATE_CMD_SESSION_BASE, LIVEUPDATE_CMD_SESSION_RETRIEVE_FD = 0x41, LIVEUPDATE_CMD_SESSION_FINISH = 0x42, + LIVEUPDATE_CMD_SESSION_GET_NAME = 0x43, }; /** @@ -168,7 +169,9 @@ struct liveupdate_session_preserve_fd { * associated with the token and populates the @fd field with a new file * descriptor referencing the restored resource in the current (new) kernel. * This operation must be performed *before* signaling completion via - * %LIVEUPDATE_IOCTL_FINISH. + * %LIVEUPDATE_IOCTL_FINISH. If a retrieve of a token fails, subsequent + * attempts to retrieve the token fail with the same error code. Failed + * retrieves are not retried. * * Return: 0 on success, negative error code on failure (e.g., invalid token). */ @@ -213,4 +216,24 @@ struct liveupdate_session_finish { #define LIVEUPDATE_SESSION_FINISH \ _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_FINISH) +/** + * struct liveupdate_session_get_name - ioctl(LIVEUPDATE_SESSION_GET_NAME) + * @size: Input; sizeof(struct liveupdate_session_get_name) + * @reserved: Input; Must be zero. Reserved for future use. + * @name: Output; A null-terminated string with the full session name. + * + * Retrieves the full name of the session associated with this file descriptor. + * This is useful because the kernel may truncate the name shown in /proc. + * + * Return: 0 on success, negative error code on failure. + */ +struct liveupdate_session_get_name { + __u32 size; + __u32 reserved; + __u8 name[LIVEUPDATE_SESSION_NAME_LENGTH]; +}; + +#define LIVEUPDATE_SESSION_GET_NAME \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_GET_NAME) + #endif /* _UAPI_LIVEUPDATE_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index a43d2da..dc770b9 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1146,9 +1146,11 @@ int kernel_kexec(void) goto Unlock; } - error = liveupdate_reboot(); - if (error) - goto Unlock; + if (!kexec_image->preserve_context) { + error = liveupdate_reboot(); + if (error) + goto Unlock; + } #ifdef CONFIG_KEXEC_JUMP if (kexec_image->preserve_context) { diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig index 1a8513f..c13af38 100644 --- a/kernel/liveupdate/Kconfig +++ b/kernel/liveupdate/Kconfig @@ -1,12 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only menu "Live Update and Kexec HandOver" - depends on !DEFERRED_STRUCT_PAGE_INIT config KEXEC_HANDOVER bool "kexec handover" depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE - depends on !DEFERRED_STRUCT_PAGE_INIT select MEMBLOCK_KHO_SCRATCH select KEXEC_FILE select LIBFDT diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile index d2f779c..eec9d3a 100644 --- a/kernel/liveupdate/Makefile +++ b/kernel/liveupdate/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 luo-y := \ + kho_block.o \ luo_core.o \ luo_file.o \ luo_flb.o \ diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 1b592d8..4834a80 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -459,6 +459,31 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages) } EXPORT_SYMBOL_GPL(kho_restore_pages); +/* + * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, struct pages in higher memory regions + * may not be initialized yet at the time KHO deserializes preserved memory. + * KHO uses the struct page to store metadata and a later initialization would + * overwrite it. + * Ensure all the struct pages in the preservation are + * initialized. kho_preserved_memory_reserve() marks the reservation as noinit + * to make sure they don't get re-initialized later. + */ +static struct page *__init kho_get_preserved_page(phys_addr_t phys, + unsigned int order) +{ + unsigned long pfn = PHYS_PFN(phys); + int nid; + + if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) + return pfn_to_page(pfn); + + nid = early_pfn_to_nid(pfn); + for (unsigned long i = 0; i < (1UL << order); i++) + init_deferred_page(pfn + i, nid); + + return pfn_to_page(pfn); +} + static int __init kho_preserved_memory_reserve(phys_addr_t phys, unsigned int order) { @@ -467,7 +492,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys, u64 sz; sz = 1 << (order + PAGE_SHIFT); - page = phys_to_page(phys); + page = kho_get_preserved_page(phys, order); /* Reserve the memory preserved in KHO in memblock */ memblock_reserve(phys, sz); @@ -593,20 +618,30 @@ early_param("kho_scratch", kho_parse_scratch_size); static void __init scratch_size_update(void) { - phys_addr_t size; + /* + * If fixed sizes are not provided via command line, calculate them + * now. + */ + if (scratch_scale) { + phys_addr_t size; - if (!scratch_scale) - return; + size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, + NUMA_NO_NODE); + size = size * scratch_scale / 100; + scratch_size_lowmem = size; - size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, - NUMA_NO_NODE); - size = size * scratch_scale / 100; - scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, + NUMA_NO_NODE); + size = size * scratch_scale / 100 - scratch_size_lowmem; + scratch_size_global = size; + } - size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, - NUMA_NO_NODE); - size = size * scratch_scale / 100 - scratch_size_lowmem; - scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + /* + * Scratch areas are released as MIGRATE_CMA. Round them up to the right + * size. + */ + scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES); + scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES); } static phys_addr_t __init scratch_size_node(int nid) @@ -1584,35 +1619,10 @@ err_free_scratch: } fs_initcall(kho_init); -static void __init kho_release_scratch(void) -{ - phys_addr_t start, end; - u64 i; - - memmap_init_kho_scratch_pages(); - - /* - * Mark scratch mem as CMA before we return it. That way we - * ensure that no kernel allocations happen on it. That means - * we can reuse it as scratch memory again later. - */ - __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, - MEMBLOCK_KHO_SCRATCH, &start, &end, NULL) { - ulong start_pfn = pageblock_start_pfn(PFN_DOWN(start)); - ulong end_pfn = pageblock_align(PFN_UP(end)); - ulong pfn; - - for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) - init_pageblock_migratetype(pfn_to_page(pfn), - MIGRATE_CMA, false); - } -} - void __init kho_memory_init(void) { if (kho_in.scratch_phys) { kho_scratch = phys_to_virt(kho_in.scratch_phys); - kho_release_scratch(); if (kho_mem_retrieve(kho_get_fdt())) kho_in.fdt_phys = 0; diff --git a/kernel/liveupdate/kho_block.c b/kernel/liveupdate/kho_block.c new file mode 100644 index 0000000..0d2a342 --- /dev/null +++ b/kernel/liveupdate/kho_block.c @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + */ + +/** + * DOC: KHO Serialization Blocks + * + * KHO provides a mechanism to preserve stateful data across a kexec handover + * by serializing it into memory blocks, and provides the common + * infrastructure for managing these blocks. + * + * Each block consists of a header (struct kho_block_header_ser) followed by an + * array of serialized entries. Multiple blocks are linked together via a + * physical pointer in the header, forming a linked list that can be easily + * traversed in both the current and the next kernel. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +/* + * Safeguard limit for the number of serialization blocks. This is used to + * prevent infinite loops and excessive memory allocation in case of memory + * corruption in the preserved state. + * + * With a 4KB page size, 10k blocks is about 40MB. For 32-byte entries + * (e.g. 4 u64s), each block holds up to 127 entries (accounting for the + * 16-byte header), allowing the block set to hold up to 1.27M entries. + */ +#define KHO_MAX_BLOCKS 10000 + +/** + * kho_block_set_init - Initialize a block set. + * @bs: The block set to initialize. + * @entry_size: The size of each entry in the blocks. + */ +void kho_block_set_init(struct kho_block_set *bs, size_t entry_size) +{ + *bs = (struct kho_block_set)KHO_BLOCK_SET_INIT(*bs, entry_size); + WARN_ON_ONCE(!bs->count_per_block); +} + +/* Serialized entries start immediately after the block header */ +static void *kho_block_entries(struct kho_block *block) +{ + return (void *)(block->ser + 1); +} + +/* Get the address of the serialized entry at the specified index */ +static void *kho_block_entry(struct kho_block_set_it *it, u64 index) +{ + return kho_block_entries(it->block) + (index * it->bs->entry_size); +} + +/* Free serialized data */ +static void kho_block_free_ser(struct kho_block_set *bs, + struct kho_block_header_ser *ser) +{ + if (bs->incoming) + kho_restore_free(ser); + else + kho_unpreserve_free(ser); +} + +static struct kho_block_header_ser *kho_block_alloc_ser(struct kho_block_set *bs) +{ + WARN_ON_ONCE(bs->incoming); + return kho_alloc_preserve(KHO_BLOCK_SIZE); +} + +static int kho_block_add(struct kho_block_set *bs, + struct kho_block_header_ser *ser) +{ + struct kho_block *block, *last; + + if (bs->nblocks >= KHO_MAX_BLOCKS) + return -ENOSPC; + + block = kzalloc_obj(*block); + if (!block) + return -ENOMEM; + + block->ser = ser; + last = list_last_entry_or_null(&bs->blocks, struct kho_block, list); + list_add_tail(&block->list, &bs->blocks); + bs->nblocks++; + + if (last) + last->ser->next = virt_to_phys(ser); + else + bs->head_pa = virt_to_phys(ser); + + return 0; +} + +static int kho_block_set_grow_one(struct kho_block_set *bs) +{ + struct kho_block_header_ser *ser; + int err; + + ser = kho_block_alloc_ser(bs); + if (IS_ERR(ser)) + return PTR_ERR(ser); + + err = kho_block_add(bs, ser); + if (err) { + kho_block_free_ser(bs, ser); + return err; + } + + return 0; +} + +static void kho_block_set_shrink_one(struct kho_block_set *bs) +{ + struct kho_block *last, *new_last; + + if (list_empty(&bs->blocks)) + return; + + last = list_last_entry(&bs->blocks, struct kho_block, list); + list_del(&last->list); + bs->nblocks--; + kho_block_free_ser(bs, last->ser); + kfree(last); + + new_last = list_last_entry_or_null(&bs->blocks, struct kho_block, list); + if (new_last) + new_last->ser->next = 0; + else + bs->head_pa = 0; +} + +/** + * kho_block_set_grow - Expand the block set to accommodate the target count. + * @bs: The block set. + * @count: The target number of valid entries to accommodate. + * + * Dynamically preallocates and links preserved memory blocks if the target + * entry count exceeds the current total capacity of the set, ensuring they + * are available during serialization/deserialization. + * + * Context: Caller must hold a lock protecting the block set. + * Return: 0 on success, or a negative errno on failure. + */ +int kho_block_set_grow(struct kho_block_set *bs, u64 count) +{ + long orig_nblocks = bs->nblocks; + int err; + + if (WARN_ON_ONCE(bs->incoming)) + return -EINVAL; + + while (count > bs->nblocks * bs->count_per_block) { + err = kho_block_set_grow_one(bs); + if (err) + goto err_shrink; + } + + return 0; + +err_shrink: + while (bs->nblocks > orig_nblocks) + kho_block_set_shrink_one(bs); + return err; +} + +/** + * kho_block_set_shrink - Shrink the block set to accommodate the target count. + * @bs: The block set. + * @count: The target number of valid entries to accommodate. + * + * Releases and unallocates redundant preserved memory blocks. Checks if the + * last block in the set can be removed because the remaining entry count is + * fully accommodated by the preceding blocks. + * + * Note: It is the caller's responsibility to ensure that entries are removed + * in the reverse order of their insertion. Because shrinking destroys the last + * block in the set, removing entries in any other order would corrupt active + * data. + * + * Context: Caller must hold a lock protecting the block set. + */ +void kho_block_set_shrink(struct kho_block_set *bs, u64 count) +{ + while (bs->nblocks > 0 && count <= (bs->nblocks - 1) * bs->count_per_block) + kho_block_set_shrink_one(bs); +} + +/* + * kho_block_set_is_cyclic - Check for cycles in a linked list of blocks. + * Uses Floyd's cycle-finding algorithm to ensure sanity of the incoming list. + * + * Return: true if a cycle or corruption is detected, false otherwise. + */ +static bool kho_block_set_is_cyclic(struct kho_block_set *bs) +{ + struct kho_block_header_ser *fast; + struct kho_block_header_ser *slow; + int count = 0; + + fast = phys_to_virt(bs->head_pa); + slow = fast; + + while (fast) { + if (count++ >= KHO_MAX_BLOCKS) { + pr_err("Block set is corrupted\n"); + return true; + } + + if (!fast->next) + break; + + fast = phys_to_virt(fast->next); + if (!fast->next) + break; + + fast = phys_to_virt(fast->next); + slow = phys_to_virt(slow->next); + + if (slow == fast) { + pr_err("Block set is corrupted\n"); + return true; + } + } + + return false; +} + +/** + * kho_block_set_restore - Restore a block set from a physical address. + * @bs: The block set to restore. + * @head_pa: Physical address of the first block header. + * + * Restores a serialized block set from a given physical address. The caller is + * responsible for ensuring that the block set @bs has been allocated and + * initialized prior to calling this function. + * + * Return: 0 on success, or a negative errno on failure. + */ +int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa) +{ + struct kho_block_header_ser *ser; + u64 next_pa = head_pa; + int err; + + /* Restored block sets use size from the previous kernel */ + bs->incoming = true; + if (!head_pa) + return 0; + + bs->head_pa = head_pa; + if (kho_block_set_is_cyclic(bs)) { + bs->head_pa = 0; + return -EINVAL; + } + + while (next_pa) { + ser = phys_to_virt(next_pa); + if (!ser->count || ser->count > bs->count_per_block) { + pr_warn("Block contains invalid entry count: %llu\n", + ser->count); + err = -EINVAL; + goto err_destroy; + } + err = kho_block_add(bs, ser); + if (err) + goto err_destroy; + next_pa = ser->next; + } + + return 0; + +err_destroy: + kho_block_set_destroy(bs); + + /* Free the remaining un-restored blocks in the physical chain */ + while (next_pa) { + struct kho_block_header_ser *next_ser = phys_to_virt(next_pa); + + next_pa = next_ser->next; + kho_block_free_ser(bs, next_ser); + } + return err; +} + +/** + * kho_block_set_destroy - Destroy all blocks in a block set. + * @bs: The block set. + */ +void kho_block_set_destroy(struct kho_block_set *bs) +{ + struct kho_block *block, *tmp; + + list_for_each_entry_safe(block, tmp, &bs->blocks, list) { + list_del(&block->list); + kho_block_free_ser(bs, block->ser); + kfree(block); + } + bs->nblocks = 0; + bs->head_pa = 0; +} + +/** + * kho_block_set_clear - Clear all serialized data in a block set. + * @bs: The block set to clear. + */ +void kho_block_set_clear(struct kho_block_set *bs) +{ + struct kho_block *block; + + list_for_each_entry(block, &bs->blocks, list) { + block->ser->count = 0; + memset(block->ser + 1, 0, KHO_BLOCK_SIZE - sizeof(*block->ser)); + } +} + +/** + * kho_block_set_it_init - Initialize a block set iterator. + * @it: The iterator to initialize. + * @bs: The block set to iterate over. + */ +void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs) +{ + it->bs = bs; + it->block = list_first_entry_or_null(&bs->blocks, struct kho_block, list); + it->i = 0; +} + +/** + * kho_block_set_it_reserve_entry - Reserve and return the next available slot for writing. + * @it: The block iterator. + * + * Reserves a slot in the current block during state serialization to add a new + * entry, advancing the internal index. If the current block is full, it + * automatically moves to the next block in the set. + * + * Return: A pointer to the reserved entry slot, or NULL if the block set's + * capacity is fully exhausted. + */ +void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it) +{ + void *entry; + + if (!it->block) + return NULL; + + if (it->i == it->bs->count_per_block) { + if (list_is_last(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_next_entry(it->block, list); + it->i = 0; + } + + entry = kho_block_entry(it, it->i++); + it->block->ser->count = it->i; + return entry; +} + +/** + * kho_block_set_it_read_entry - Read the next serialized entry from the block set. + * @it: The block iterator. + * + * Iterates through previously written entries during state deserialization, + * respecting the actual count stored in each block's header. + * + * Return: A pointer to the next serialized entry, or NULL if all serialized + * entries have been read. + */ +void *kho_block_set_it_read_entry(struct kho_block_set_it *it) +{ + if (!it->block) + return NULL; + + if (it->i == it->block->ser->count) { + if (list_is_last(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_next_entry(it->block, list); + it->i = 0; + } + + return kho_block_entry(it, it->i++); +} + +/** + * kho_block_set_it_prev - Return the previous entry slot in the block set. + * @it: The block iterator. + * + * If the current index is at the start of a block, it automatically moves to + * the end of the previous block. + * + * Return: A pointer to the previous entry slot, or NULL if at the very + * beginning of the block set. + */ +void *kho_block_set_it_prev(struct kho_block_set_it *it) +{ + if (!it->block) + return NULL; + + if (it->i == 0) { + if (list_is_first(&it->block->list, &it->bs->blocks)) + return NULL; + it->block = list_prev_entry(it->block, list); + it->i = it->bs->count_per_block; + } + + return kho_block_entry(it, --it->i); +} diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c index 803f51c..1b2bda2 100644 --- a/kernel/liveupdate/luo_core.c +++ b/kernel/liveupdate/luo_core.c @@ -36,6 +36,10 @@ * * LUO uses Kexec Handover to transfer memory state from the current kernel to * the next kernel. For more details see Documentation/core-api/kho/index.rst. + * + * .. note:: + * To enable LUO, boot the kernel with the ``liveupdate=on`` command line + * parameter. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -50,22 +54,19 @@ #include #include #include -#include #include #include #include #include #include #include -#include #include "kexec_handover_internal.h" #include "luo_internal.h" static struct { bool enabled; - void *fdt_out; - void *fdt_in; + struct luo_ser *luo_ser_out; u64 liveupdate_num; } luo_global; @@ -82,9 +83,10 @@ early_param("liveupdate", early_liveupdate_param); static int __init luo_early_startup(void) { - phys_addr_t fdt_phys; - int err, ln_size; - const void *ptr; + phys_addr_t luo_ser_phys; + struct luo_ser *luo_ser; + size_t len; + int err; if (!kho_is_enabled()) { if (liveupdate_enabled()) @@ -93,48 +95,43 @@ static int __init luo_early_startup(void) return 0; } - /* Retrieve LUO subtree, and verify its format. */ - err = kho_retrieve_subtree(LUO_FDT_KHO_ENTRY_NAME, &fdt_phys, NULL); + /* Retrieve LUO state from KHO. */ + err = kho_retrieve_subtree(LUO_KHO_ENTRY_NAME, &luo_ser_phys, &len); if (err) { if (err != -ENOENT) { - pr_err("failed to retrieve FDT '%s' from KHO: %pe\n", - LUO_FDT_KHO_ENTRY_NAME, ERR_PTR(err)); + pr_err("failed to retrieve LUO state '%s' from KHO: %pe\n", + LUO_KHO_ENTRY_NAME, ERR_PTR(err)); return err; } return 0; } - luo_global.fdt_in = phys_to_virt(fdt_phys); - err = fdt_node_check_compatible(luo_global.fdt_in, 0, - LUO_FDT_COMPATIBLE); - if (err) { - pr_err("FDT '%s' is incompatible with '%s' [%d]\n", - LUO_FDT_KHO_ENTRY_NAME, LUO_FDT_COMPATIBLE, err); - + if (len < sizeof(*luo_ser)) { + pr_err("LUO state is too small (%zu < %zu)\n", len, sizeof(*luo_ser)); return -EINVAL; } - ln_size = 0; - ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_LIVEUPDATE_NUM, - &ln_size); - if (!ptr || ln_size != sizeof(luo_global.liveupdate_num)) { - pr_err("Unable to get live update number '%s' [%d]\n", - LUO_FDT_LIVEUPDATE_NUM, ln_size); - + luo_ser = phys_to_virt(luo_ser_phys); + if (strncmp(luo_ser->compatible, LUO_ABI_COMPATIBLE, LUO_ABI_COMPAT_LEN)) { + pr_err("LUO state is incompatible with '%s'\n", LUO_ABI_COMPATIBLE); return -EINVAL; } - luo_global.liveupdate_num = get_unaligned((u64 *)ptr); + luo_global.liveupdate_num = luo_ser->liveupdate_num; pr_info("Retrieved live update data, liveupdate number: %lld\n", luo_global.liveupdate_num); - err = luo_session_setup_incoming(luo_global.fdt_in); + err = luo_session_setup_incoming(luo_ser->sessions_pa); if (err) - return err; + goto out_free_ser; + + luo_flb_setup_incoming(luo_ser->flbs_pa); - err = luo_flb_setup_incoming(luo_global.fdt_in); + err = 0; +out_free_ser: + kho_restore_free(luo_ser); return err; } @@ -153,42 +150,38 @@ static int __init liveupdate_early_init(void) } early_initcall(liveupdate_early_init); -/* Called during boot to create outgoing LUO fdt tree */ -static int __init luo_fdt_setup(void) +/* Called during boot to create outgoing LUO state */ +static int __init luo_state_setup(void) { - const u64 ln = luo_global.liveupdate_num + 1; - void *fdt_out; + struct luo_ser *luo_ser; int err; - fdt_out = kho_alloc_preserve(LUO_FDT_SIZE); - if (IS_ERR(fdt_out)) { - pr_err("failed to allocate/preserve FDT memory\n"); - return PTR_ERR(fdt_out); + luo_ser = kho_alloc_preserve(sizeof(*luo_ser)); + if (IS_ERR(luo_ser)) { + pr_err("failed to allocate/preserve LUO state memory\n"); + return PTR_ERR(luo_ser); } - err = fdt_create(fdt_out, LUO_FDT_SIZE); - err |= fdt_finish_reservemap(fdt_out); - err |= fdt_begin_node(fdt_out, ""); - err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_LIVEUPDATE_NUM, &ln, sizeof(ln)); - err |= luo_session_setup_outgoing(fdt_out); - err |= luo_flb_setup_outgoing(fdt_out); - err |= fdt_end_node(fdt_out); - err |= fdt_finish(fdt_out); + strscpy(luo_ser->compatible, LUO_ABI_COMPATIBLE, sizeof(luo_ser->compatible)); + luo_ser->liveupdate_num = luo_global.liveupdate_num + 1; + + luo_session_setup_outgoing(&luo_ser->sessions_pa); + + err = luo_flb_setup_outgoing(&luo_ser->flbs_pa); if (err) - goto exit_free; + goto exit_free_luo_ser; - err = kho_add_subtree(LUO_FDT_KHO_ENTRY_NAME, fdt_out, - fdt_totalsize(fdt_out)); + err = kho_add_subtree(LUO_KHO_ENTRY_NAME, luo_ser, sizeof(*luo_ser)); if (err) - goto exit_free; - luo_global.fdt_out = fdt_out; + goto exit_free_luo_ser; + + luo_global.luo_ser_out = luo_ser; return 0; -exit_free: - kho_unpreserve_free(fdt_out); - pr_err("failed to prepare LUO FDT: %d\n", err); +exit_free_luo_ser: + kho_unpreserve_free(luo_ser); + pr_err("failed to prepare LUO state: %d\n", err); return err; } @@ -204,7 +197,7 @@ static int __init luo_late_startup(void) if (!liveupdate_enabled()) return 0; - err = luo_fdt_setup(); + err = luo_state_setup(); if (err) luo_global.enabled = false; diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c index a0a4190..c39f969 100644 --- a/kernel/liveupdate/luo_file.c +++ b/kernel/liveupdate/luo_file.c @@ -118,11 +118,6 @@ static LIST_HEAD(luo_file_handler_list); /* Keep track of files being preserved by LUO */ static DEFINE_XARRAY(luo_preserved_files); -/* 2 4K pages, give space for 128 files per file_set */ -#define LUO_FILE_PGCNT 2ul -#define LUO_FILE_MAX \ - ((LUO_FILE_PGCNT << PAGE_SHIFT) / sizeof(struct luo_file_ser)) - /** * struct luo_file - Represents a single preserved file instance. * @fh: Pointer to the &struct liveupdate_file_handler that manages @@ -174,39 +169,6 @@ struct luo_file { u64 token; }; -static int luo_alloc_files_mem(struct luo_file_set *file_set) -{ - size_t size; - void *mem; - - if (file_set->files) - return 0; - - WARN_ON_ONCE(file_set->count); - - size = LUO_FILE_PGCNT << PAGE_SHIFT; - mem = kho_alloc_preserve(size); - if (IS_ERR(mem)) - return PTR_ERR(mem); - - file_set->files = mem; - - return 0; -} - -static void luo_free_files_mem(struct luo_file_set *file_set) -{ - /* If file_set has files, no need to free preservation memory */ - if (file_set->count) - return; - - if (!file_set->files) - return; - - kho_unpreserve_free(file_set->files); - file_set->files = NULL; -} - static unsigned long luo_get_id(struct liveupdate_file_handler *fh, struct file *file) { @@ -276,16 +238,15 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) if (luo_token_is_used(file_set, token)) return -EEXIST; - if (file_set->count == LUO_FILE_MAX) - return -ENOSPC; + err = kho_block_set_grow(&file_set->block_set, file_set->count + 1); + if (err) + return err; file = fget(fd); - if (!file) - return -EBADF; - - err = luo_alloc_files_mem(file_set); - if (err) - goto err_fput; + if (!file) { + err = -EBADF; + goto err_shrink; + } err = -ENOENT; down_read(&luo_register_rwlock); @@ -300,7 +261,7 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) /* err is still -ENOENT if no handler was found */ if (err) - goto err_free_files_mem; + goto err_fput; err = xa_insert(&luo_preserved_files, luo_get_id(fh, file), file, GFP_KERNEL); @@ -343,10 +304,10 @@ err_erase_xa: xa_erase(&luo_preserved_files, luo_get_id(fh, file)); err_module_put: module_put(fh->ops->owner); -err_free_files_mem: - luo_free_files_mem(file_set); err_fput: fput(file); +err_shrink: + kho_block_set_shrink(&file_set->block_set, file_set->count); return err; } @@ -385,19 +346,21 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set) args.private_data = luo_file->private_data; luo_file->fh->ops->unpreserve(&args); luo_flb_file_unpreserve(luo_file->fh); - module_put(luo_file->fh->ops->owner); xa_erase(&luo_preserved_files, luo_get_id(luo_file->fh, luo_file->file)); + module_put(luo_file->fh->ops->owner); + list_del(&luo_file->list); file_set->count--; + kho_block_set_shrink(&file_set->block_set, file_set->count); fput(luo_file->file); mutex_destroy(&luo_file->mutex); kfree(luo_file); } - luo_free_files_mem(file_set); + kho_block_set_destroy(&file_set->block_set); } static int luo_file_freeze_one(struct luo_file_set *file_set, @@ -453,7 +416,7 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set, luo_file_unfreeze_one(file_set, luo_file); } - memset(file_set->files, 0, LUO_FILE_PGCNT << PAGE_SHIFT); + kho_block_set_clear(&file_set->block_set); } /** @@ -492,19 +455,24 @@ static void __luo_file_unfreeze(struct luo_file_set *file_set, int luo_file_freeze(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { - struct luo_file_ser *file_ser = file_set->files; struct luo_file *luo_file; + struct kho_block_set_it it; int err; - int i; if (!file_set->count) return 0; - if (WARN_ON(!file_ser)) - return -EINVAL; + kho_block_set_it_init(&it, &file_set->block_set); - i = 0; list_for_each_entry(luo_file, &file_set->files_list, list) { + struct luo_file_ser *file_ser = kho_block_set_it_reserve_entry(&it); + + /* This should not fail normally as blocks were pre-allocated */ + if (WARN_ON_ONCE(!file_ser)) { + err = -ENOSPC; + goto err_unfreeze; + } + err = luo_file_freeze_one(file_set, luo_file); if (err < 0) { pr_warn("Freeze failed for token[%#0llx] handler[%s] err[%pe]\n", @@ -513,16 +481,14 @@ int luo_file_freeze(struct luo_file_set *file_set, goto err_unfreeze; } - strscpy(file_ser[i].compatible, luo_file->fh->compatible, - sizeof(file_ser[i].compatible)); - file_ser[i].data = luo_file->serialized_data; - file_ser[i].token = luo_file->token; - i++; + strscpy(file_ser->compatible, luo_file->fh->compatible, + sizeof(file_ser->compatible)); + file_ser->data = luo_file->serialized_data; + file_ser->token = luo_file->token; } file_set_ser->count = file_set->count; - if (file_set->files) - file_set_ser->files = virt_to_phys(file_set->files); + file_set_ser->files = kho_block_set_head_pa(&file_set->block_set); return 0; @@ -677,7 +643,6 @@ static void luo_file_finish_one(struct luo_file_set *file_set, luo_file->fh->ops->finish(&args); luo_flb_file_finish(luo_file->fh); - module_put(luo_file->fh->ops->owner); } /** @@ -738,17 +703,56 @@ int luo_file_finish(struct luo_file_set *file_set) luo_get_id(luo_file->fh, luo_file->file)); fput(luo_file->file); } + module_put(luo_file->fh->ops->owner); list_del(&luo_file->list); file_set->count--; + kho_block_set_shrink(&file_set->block_set, file_set->count); mutex_destroy(&luo_file->mutex); kfree(luo_file); } - if (file_set->files) { - kho_restore_free(file_set->files); - file_set->files = NULL; + kho_block_set_destroy(&file_set->block_set); + + return 0; +} + +static int luo_file_deserialize_one(struct luo_file_set *file_set, + struct luo_file_ser *ser) +{ + struct liveupdate_file_handler *fh; + bool handler_found = false; + struct luo_file *luo_file; + + down_read(&luo_register_rwlock); + list_private_for_each_entry(fh, &luo_file_handler_list, list) { + if (!strcmp(fh->compatible, ser->compatible)) { + if (try_module_get(fh->ops->owner)) + handler_found = true; + break; + } + } + up_read(&luo_register_rwlock); + + if (!handler_found) { + pr_warn("No registered handler for compatible '%.*s'\n", + (int)sizeof(ser->compatible), + ser->compatible); + return -ENOENT; } + luo_file = kzalloc_obj(*luo_file); + if (!luo_file) { + module_put(fh->ops->owner); + return -ENOMEM; + } + + luo_file->fh = fh; + luo_file->file = NULL; + luo_file->serialized_data = ser->data; + luo_file->token = ser->token; + mutex_init(&luo_file->mutex); + list_add_tail(&luo_file->list, &file_set->files_list); + return 0; } @@ -781,15 +785,18 @@ int luo_file_deserialize(struct luo_file_set *file_set, struct luo_file_set_ser *file_set_ser) { struct luo_file_ser *file_ser; - u64 i; + struct kho_block_set_it it; + int err; if (!file_set_ser->files) { WARN_ON(file_set_ser->count); return 0; } - file_set->count = file_set_ser->count; - file_set->files = phys_to_virt(file_set_ser->files); + file_set->count = 0; + err = kho_block_set_restore(&file_set->block_set, file_set_ser->files); + if (err) + return err; /* * Note on error handling: @@ -806,55 +813,50 @@ int luo_file_deserialize(struct luo_file_set *file_set, * userspace to detect the failure and trigger a reboot, which will * reliably reset devices and reclaim memory. */ - file_ser = file_set->files; - for (i = 0; i < file_set->count; i++) { - struct liveupdate_file_handler *fh; - bool handler_found = false; - struct luo_file *luo_file; + kho_block_set_it_init(&it, &file_set->block_set); + while ((file_ser = kho_block_set_it_read_entry(&it))) { + err = luo_file_deserialize_one(file_set, file_ser); + if (err) + goto err_destroy_blocks; + file_set->count++; + } - down_read(&luo_register_rwlock); - list_private_for_each_entry(fh, &luo_file_handler_list, list) { - if (!strcmp(fh->compatible, file_ser[i].compatible)) { - if (try_module_get(fh->ops->owner)) - handler_found = true; - break; - } - } - up_read(&luo_register_rwlock); + if (file_set->count != file_set_ser->count) { + pr_warn("File count mismatch: expected %llu, found %llu\n", + file_set_ser->count, file_set->count); + err = -EINVAL; + goto err_destroy_blocks; + } - if (!handler_found) { - pr_warn("No registered handler for compatible '%.*s'\n", - (int)sizeof(file_ser[i].compatible), - file_ser[i].compatible); - return -ENOENT; - } + return 0; - luo_file = kzalloc_obj(*luo_file); - if (!luo_file) { - module_put(fh->ops->owner); - return -ENOMEM; - } +err_destroy_blocks: + while (!list_empty(&file_set->files_list)) { + struct luo_file *luo_file; - luo_file->fh = fh; - luo_file->file = NULL; - luo_file->serialized_data = file_ser[i].data; - luo_file->token = file_ser[i].token; - mutex_init(&luo_file->mutex); - list_add_tail(&luo_file->list, &file_set->files_list); + luo_file = list_first_entry(&file_set->files_list, + struct luo_file, list); + list_del(&luo_file->list); + module_put(luo_file->fh->ops->owner); + mutex_destroy(&luo_file->mutex); + kfree(luo_file); } - - return 0; + file_set->count = 0; + kho_block_set_destroy(&file_set->block_set); + return err; } void luo_file_set_init(struct luo_file_set *file_set) { INIT_LIST_HEAD(&file_set->files_list); + kho_block_set_init(&file_set->block_set, sizeof(struct luo_file_ser)); } void luo_file_set_destroy(struct luo_file_set *file_set) { WARN_ON(file_set->count); WARN_ON(!list_empty(&file_set->files_list)); + WARN_ON(!kho_block_set_is_empty(&file_set->block_set)); } /** diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 00f5494..5c27134 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -44,13 +44,11 @@ #include #include #include -#include #include #include #include #include #include -#include #include "luo_internal.h" #define LUO_FLB_PGCNT 1ul @@ -111,7 +109,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - if (!private->outgoing.count) { + if (!refcount_read(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; int err; @@ -126,8 +124,10 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) } private->outgoing.data = args.data; private->outgoing.obj = args.obj; + refcount_set(&private->outgoing.count, 1); + } else { + refcount_inc(&private->outgoing.count); } - private->outgoing.count++; } return 0; @@ -138,8 +138,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) struct luo_flb_private *private = luo_flb_get_private(flb); scoped_guard(mutex, &private->outgoing.lock) { - private->outgoing.count--; - if (!private->outgoing.count) { + if (refcount_dec_and_test(&private->outgoing.count)) { struct liveupdate_flb_op_args args = {0}; args.flb = flb; @@ -164,7 +163,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) bool found = false; int err; - guard(mutex)(&private->incoming.lock); + lockdep_assert_held(&private->incoming.lock); if (private->incoming.finished) return -ENODATA; @@ -178,7 +177,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) for (int i = 0; i < fh->header_ser->count; i++) { if (!strcmp(fh->ser[i].name, flb->compatible)) { private->incoming.data = fh->ser[i].data; - private->incoming.count = fh->ser[i].count; + refcount_set(&private->incoming.count, fh->ser[i].count); found = true; break; } @@ -205,16 +204,14 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_finish_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); - u64 count; - - scoped_guard(mutex, &private->incoming.lock) - count = --private->incoming.count; + struct liveupdate_flb_op_args args = {0}; - if (!count) { - struct liveupdate_flb_op_args args = {0}; + scoped_guard(mutex, &private->incoming.lock) { + if (!refcount_dec_and_test(&private->incoming.count)) + return; if (!private->incoming.retrieved) { int err = luo_flb_retrieve_one(flb); @@ -223,16 +220,14 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb) return; } - scoped_guard(mutex, &private->incoming.lock) { - args.flb = flb; - args.obj = private->incoming.obj; - flb->ops->finish(&args); + args.flb = flb; + args.obj = private->incoming.obj; + flb->ops->finish(&args); - private->incoming.data = 0; - private->incoming.obj = NULL; - private->incoming.finished = true; - module_put(flb->ops->owner); - } + private->incoming.data = 0; + private->incoming.obj = NULL; + private->incoming.finished = true; + module_put(flb->ops->owner); } } @@ -315,7 +310,7 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_finish_one(iter->flb); + liveupdate_flb_put_incoming(iter->flb); } static void luo_flb_unregister_one(struct liveupdate_file_handler *fh, @@ -512,6 +507,8 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) if (!liveupdate_enabled()) return -EOPNOTSUPP; + guard(mutex)(&private->incoming.lock); + if (!private->incoming.obj) { int err = luo_flb_retrieve_one(flb); @@ -519,7 +516,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp) return err; } - guard(mutex)(&private->incoming.lock); + refcount_inc(&private->incoming.count); *objp = private->incoming.obj; return 0; @@ -552,27 +549,15 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) return 0; } -int __init luo_flb_setup_outgoing(void *fdt_out) +int __init luo_flb_setup_outgoing(u64 *flbs_pa) { struct luo_flb_header_ser *header_ser; - u64 header_ser_pa; - int err; header_ser = kho_alloc_preserve(LUO_FLB_PGCNT << PAGE_SHIFT); if (IS_ERR(header_ser)) return PTR_ERR(header_ser); - header_ser_pa = virt_to_phys(header_ser); - - err = fdt_begin_node(fdt_out, LUO_FDT_FLB_NODE_NAME); - err |= fdt_property_string(fdt_out, "compatible", - LUO_FDT_FLB_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_FLB_HEADER, &header_ser_pa, - sizeof(header_ser_pa)); - err |= fdt_end_node(fdt_out); - - if (err) - goto err_unpreserve; + *flbs_pa = virt_to_phys(header_ser); header_ser->pgcnt = LUO_FLB_PGCNT; luo_flb_global.outgoing.header_ser = header_ser; @@ -580,53 +565,19 @@ int __init luo_flb_setup_outgoing(void *fdt_out) luo_flb_global.outgoing.active = true; return 0; - -err_unpreserve: - kho_unpreserve_free(header_ser); - - return err; } -int __init luo_flb_setup_incoming(void *fdt_in) +void __init luo_flb_setup_incoming(u64 flbs_pa) { struct luo_flb_header_ser *header_ser; - int err, header_size, offset; - const void *ptr; - u64 header_ser_pa; - offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_FLB_NODE_NAME); - if (offset < 0) { - pr_err("Unable to get FLB node [%s]\n", LUO_FDT_FLB_NODE_NAME); - - return -ENOENT; - } - - err = fdt_node_check_compatible(fdt_in, offset, - LUO_FDT_FLB_COMPATIBLE); - if (err) { - pr_err("FLB node is incompatible with '%s' [%d]\n", - LUO_FDT_FLB_COMPATIBLE, err); - - return -EINVAL; - } - - header_size = 0; - ptr = fdt_getprop(fdt_in, offset, LUO_FDT_FLB_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get FLB header property '%s' [%d]\n", - LUO_FDT_FLB_HEADER, header_size); - - return -EINVAL; - } - - header_ser_pa = get_unaligned((u64 *)ptr); - header_ser = phys_to_virt(header_ser_pa); + if (!flbs_pa) + return; + header_ser = phys_to_virt(flbs_pa); luo_flb_global.incoming.header_ser = header_ser; luo_flb_global.incoming.ser = (void *)(header_ser + 1); luo_flb_global.incoming.active = true; - - return 0; } /** @@ -652,12 +603,13 @@ void luo_flb_serialize(void) guard(rwsem_read)(&luo_register_rwlock); list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) { struct luo_flb_private *private = luo_flb_get_private(gflb); + long count = refcount_read(&private->outgoing.count); - if (private->outgoing.count > 0) { + if (count > 0) { strscpy(fh->ser[i].name, gflb->compatible, sizeof(fh->ser[i].name)); fh->ser[i].data = private->outgoing.data; - fh->ser[i].count = private->outgoing.count; + fh->ser[i].count = count; i++; } } diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h index 875844d..64879ff 100644 --- a/kernel/liveupdate/luo_internal.h +++ b/kernel/liveupdate/luo_internal.h @@ -10,6 +10,7 @@ #include #include +#include struct luo_ucmd { void __user *ubuffer; @@ -44,22 +45,20 @@ static inline int luo_ucmd_respond(struct luo_ucmd *ucmd, * struct luo_file_set - A set of files that belong to the same sessions. * @files_list: An ordered list of files associated with this session, it is * ordered by preservation time. - * @files: The physically contiguous memory block that holds the serialized - * state of files. + * @block_set: The set of serialization blocks. * @count: A counter tracking the number of files currently stored in the * @files_list for this session. */ struct luo_file_set { struct list_head files_list; - struct luo_file_ser *files; - long count; + struct kho_block_set block_set; + u64 count; }; /** * struct luo_session - Represents an active or incoming Live Update session. * @name: A unique name for this session, used for identification and * retrieval. - * @ser: Pointer to the serialized data for this session. * @list: A list_head member used to link this session into a global list * of either outgoing (to be preserved) or incoming (restored from * previous kernel) sessions. @@ -70,7 +69,6 @@ struct luo_file_set { */ struct luo_session { char name[LIVEUPDATE_SESSION_NAME_LENGTH]; - struct luo_session_ser *ser; struct list_head list; bool retrieved; struct luo_file_set file_set; @@ -81,8 +79,8 @@ extern struct rw_semaphore luo_register_rwlock; int luo_session_create(const char *name, struct file **filep); int luo_session_retrieve(const char *name, struct file **filep); -int __init luo_session_setup_outgoing(void *fdt); -int __init luo_session_setup_incoming(void *fdt); +void __init luo_session_setup_outgoing(u64 *sessions_pa); +int __init luo_session_setup_incoming(u64 sessions_pa); int luo_session_serialize(void); int luo_session_deserialize(void); @@ -104,8 +102,8 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh); void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh); void luo_flb_file_finish(struct liveupdate_file_handler *fh); void luo_flb_unregister_all(struct liveupdate_file_handler *fh); -int __init luo_flb_setup_outgoing(void *fdt); -int __init luo_flb_setup_incoming(void *fdt); +int __init luo_flb_setup_outgoing(u64 *flbs_pa); +void __init luo_flb_setup_incoming(u64 flbs_pa); void luo_flb_serialize(void); #ifdef CONFIG_LIVEUPDATE_TEST diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index 7a42385..b79b2a4 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -24,10 +24,10 @@ * ioctls on /dev/liveupdate. * * - Serialization: Session metadata is preserved using the KHO framework. When - * a live update is triggered via kexec, an array of `struct luo_session_ser` - * is populated and placed in a preserved memory region. An FDT node is also - * created, containing the count of sessions and the physical address of this - * array. + * a live update is triggered via kexec, session metadata is serialized into + * a chain of linked-blocks and placed in a preserved memory region. The + * physical address of the first block header is stored in the centralized + * `struct luo_ser` structure. * * Session Lifecycle: * @@ -46,6 +46,38 @@ * 4. Retrieval: A userspace agent in the new kernel can then call * `luo_session_retrieve()` with a session name to get a new file * descriptor and access the preserved state. + * + * Locking: + * + * The LUO session subsystem uses a three-tier locking hierarchy to ensure thread + * safety and prevent deadlocks during concurrent session mutations and kexec + * serialization: + * + * 1. `luo_session_serialize_rwsem` (global rwsem): + * Protects session mutations (creation, retrieval, release, and ioctls) + * against the serialization process during reboot. + * + * - Readers: Taken by any path modifying or accessing session state (e.g., + * `luo_session_create()`, `luo_session_retrieve()`, `luo_session_release()`, + * and `luo_session_ioctl()`). + * - Writer: Taken by the serialization process (`luo_session_serialize()`) + * during reboot. On success, the write lock is held indefinitely to freeze + * the subsystem. On failure, it is released to allow recovery. + * + * 2. `luo_session_header->rwsem` (per-list rwsem): + * Synchronizes list-level operations for the incoming and outgoing session headers. + * + * - Writer: Taken during list mutation operations (inserting or removing a + * session from the list). + * - Reader: Taken when traversing the list (e.g., retrieving a session by name). + * + * 3. `luo_session->mutex` (per-session mutex): + * Protects the internal state and file sets of an individual session. It is + * acquired during per-session operations such as preserving, retrieving, + * or freezing files. + * + * Lock Hierarchy: + * `luo_session_serialize_rwsem` -> `luo_session_header->rwsem` -> `luo_session->mutex` */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -58,41 +90,34 @@ #include #include #include +#include #include -#include #include #include #include #include #include -#include #include #include "luo_internal.h" -/* 16 4K pages, give space for 744 sessions */ -#define LUO_SESSION_PGCNT 16ul -#define LUO_SESSION_MAX (((LUO_SESSION_PGCNT << PAGE_SHIFT) - \ - sizeof(struct luo_session_header_ser)) / \ - sizeof(struct luo_session_ser)) - +static DECLARE_RWSEM(luo_session_serialize_rwsem); /** * struct luo_session_header - Header struct for managing LUO sessions. - * @count: The number of sessions currently tracked in the @list. - * @list: The head of the linked list of `struct luo_session` instances. - * @rwsem: A read-write semaphore providing synchronized access to the - * session list and other fields in this structure. - * @header_ser: The header data of serialization array. - * @ser: The serialized session data (an array of - * `struct luo_session_ser`). - * @active: Set to true when first initialized. If previous kernel did not - * send session data, active stays false for incoming. + * @count: The number of sessions currently tracked in the @list. + * @list: The head of the linked list of `struct luo_session` instances. + * @rwsem: A read-write semaphore providing synchronized access to the + * session list and other fields in this structure. + * @block_set: The set of serialization blocks. + * @sessions_pa: Points to the location of sessions_pa within struct luo_ser. + * @active: Set to true when first initialized. If previous kernel did not + * send session data, active stays false for incoming. */ struct luo_session_header { long count; struct list_head list; struct rw_semaphore rwsem; - struct luo_session_header_ser *header_ser; - struct luo_session_ser *ser; + struct kho_block_set block_set; + u64 *sessions_pa; bool active; }; @@ -110,10 +135,14 @@ static struct luo_session_global luo_session_global = { .incoming = { .list = LIST_HEAD_INIT(luo_session_global.incoming.list), .rwsem = __RWSEM_INITIALIZER(luo_session_global.incoming.rwsem), + .block_set = KHO_BLOCK_SET_INIT(luo_session_global.incoming.block_set, + sizeof(struct luo_session_ser)), }, .outgoing = { .list = LIST_HEAD_INIT(luo_session_global.outgoing.list), .rwsem = __RWSEM_INITIALIZER(luo_session_global.outgoing.rwsem), + .block_set = KHO_BLOCK_SET_INIT(luo_session_global.outgoing.block_set, + sizeof(struct luo_session_ser)), }, }; @@ -144,6 +173,7 @@ static int luo_session_insert(struct luo_session_header *sh, struct luo_session *session) { struct luo_session *it; + int err; guard(rwsem_write)(&sh->rwsem); @@ -152,8 +182,9 @@ static int luo_session_insert(struct luo_session_header *sh, * for new session. */ if (sh == &luo_session_global.outgoing) { - if (sh->count == LUO_SESSION_MAX) - return -ENOMEM; + err = kho_block_set_grow(&sh->block_set, sh->count + 1); + if (err) + return err; } /* @@ -178,6 +209,8 @@ static void luo_session_remove(struct luo_session_header *sh, guard(rwsem_write)(&sh->rwsem); list_del(&session->list); sh->count--; + if (sh == &luo_session_global.outgoing) + kho_block_set_shrink(&sh->block_set, sh->count); } static int luo_session_finish_one(struct luo_session *session) @@ -205,6 +238,7 @@ static int luo_session_release(struct inode *inodep, struct file *filep) struct luo_session *session = filep->private_data; struct luo_session_header *sh; + guard(rwsem_read)(&luo_session_serialize_rwsem); /* If retrieved is set, it means this session is from incoming list */ if (session->retrieved) { int err = luo_session_finish_one(session); @@ -256,10 +290,11 @@ static int luo_session_retrieve_fd(struct luo_session *session, if (argp->fd < 0) return argp->fd; - guard(mutex)(&session->mutex); + mutex_lock(&session->mutex); err = luo_retrieve_file(&session->file_set, argp->token, &file); + mutex_unlock(&session->mutex); if (err < 0) - goto err_put_fd; + goto err_put_fd; err = luo_ucmd_respond(ucmd, sizeof(*argp)); if (err) @@ -289,38 +324,80 @@ static int luo_session_finish(struct luo_session *session, return luo_ucmd_respond(ucmd, sizeof(*argp)); } +static int luo_session_get_name(struct luo_session *session, + struct luo_ucmd *ucmd) +{ + struct liveupdate_session_get_name *argp = ucmd->cmd; + + if (argp->reserved != 0) + return -EINVAL; + + strscpy((char *)argp->name, session->name, sizeof(argp->name)); + + return luo_ucmd_respond(ucmd, sizeof(*argp)); +} + union ucmd_buffer { struct liveupdate_session_finish finish; struct liveupdate_session_preserve_fd preserve; struct liveupdate_session_retrieve_fd retrieve; + struct liveupdate_session_get_name get_name; +}; + +/* Type of sessions the ioctl applies to. */ +enum luo_ioctl_type { + LUO_IOCTL_INCOMING, + LUO_IOCTL_OUTGOING, + LUO_IOCTL_ALL, }; struct luo_ioctl_op { unsigned int size; unsigned int min_size; unsigned int ioctl_num; + enum luo_ioctl_type type; int (*execute)(struct luo_session *session, struct luo_ucmd *ucmd); }; -#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ +#define IOCTL_OP(_ioctl, _fn, _struct, _last, _type) \ [_IOC_NR(_ioctl) - LIVEUPDATE_CMD_SESSION_BASE] = { \ .size = sizeof(_struct) + \ BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ sizeof(_struct)), \ .min_size = offsetofend(_struct, _last), \ .ioctl_num = _ioctl, \ + .type = _type, \ .execute = _fn, \ } static const struct luo_ioctl_op luo_session_ioctl_ops[] = { IOCTL_OP(LIVEUPDATE_SESSION_FINISH, luo_session_finish, - struct liveupdate_session_finish, reserved), + struct liveupdate_session_finish, reserved, LUO_IOCTL_INCOMING), IOCTL_OP(LIVEUPDATE_SESSION_PRESERVE_FD, luo_session_preserve_fd, - struct liveupdate_session_preserve_fd, token), + struct liveupdate_session_preserve_fd, token, LUO_IOCTL_OUTGOING), IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd, - struct liveupdate_session_retrieve_fd, token), + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_INCOMING), + IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name, + struct liveupdate_session_retrieve_fd, token, LUO_IOCTL_ALL), }; +static bool luo_ioctl_type_valid(struct luo_session *session, + const struct luo_ioctl_op *op) +{ + switch (op->type) { + case LUO_IOCTL_INCOMING: + /* Retrieved is only set on incoming sessions */ + return session->retrieved; + case LUO_IOCTL_OUTGOING: + return !session->retrieved; + case LUO_IOCTL_ALL: + return true; + } + + /* Catch-all. */ + return false; +} + static long luo_session_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -345,6 +422,8 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, op = &luo_session_ioctl_ops[nr - LIVEUPDATE_CMD_SESSION_BASE]; if (op->ioctl_num != cmd) return -ENOIOCTLCMD; + if (!luo_ioctl_type_valid(session, op)) + return -EINVAL; if (ucmd.user_size < op->min_size) return -EINVAL; @@ -354,6 +433,7 @@ static long luo_session_ioctl(struct file *filep, unsigned int cmd, if (ret) return ret; + guard(rwsem_read)(&luo_session_serialize_rwsem); return op->execute(session, &ucmd); } @@ -382,21 +462,28 @@ static int luo_session_getfile(struct luo_session *session, struct file **filep) int luo_session_create(const char *name, struct file **filep) { + size_t len = strnlen(name, LIVEUPDATE_SESSION_NAME_LENGTH); struct luo_session *session; int err; + if (len == 0 || len > LIVEUPDATE_SESSION_NAME_LENGTH - 1) + return -EINVAL; + session = luo_session_alloc(name); if (IS_ERR(session)) return PTR_ERR(session); + down_read(&luo_session_serialize_rwsem); err = luo_session_insert(&luo_session_global.outgoing, session); if (err) goto err_free; - scoped_guard(mutex, &session->mutex) - err = luo_session_getfile(session, filep); + mutex_lock(&session->mutex); + err = luo_session_getfile(session, filep); + mutex_unlock(&session->mutex); if (err) goto err_remove; + up_read(&luo_session_serialize_rwsem); return 0; @@ -404,6 +491,7 @@ err_remove: luo_session_remove(&luo_session_global.outgoing, session); err_free: luo_session_free(session); + up_read(&luo_session_serialize_rwsem); return err; } @@ -415,12 +503,12 @@ int luo_session_retrieve(const char *name, struct file **filep) struct luo_session *it; int err; - scoped_guard(rwsem_read, &sh->rwsem) { - list_for_each_entry(it, &sh->list, list) { - if (!strncmp(it->name, name, sizeof(it->name))) { - session = it; - break; - } + guard(rwsem_read)(&luo_session_serialize_rwsem); + guard(rwsem_read)(&sh->rwsem); + list_for_each_entry(it, &sh->list, list) { + if (!strncmp(it->name, name, sizeof(it->name))) { + session = it; + break; } } @@ -438,74 +526,58 @@ int luo_session_retrieve(const char *name, struct file **filep) return err; } -int __init luo_session_setup_outgoing(void *fdt_out) +void __init luo_session_setup_outgoing(u64 *sessions_pa) { - struct luo_session_header_ser *header_ser; - u64 header_ser_pa; - int err; + luo_session_global.outgoing.sessions_pa = sessions_pa; + luo_session_global.outgoing.active = true; +} - header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); - if (IS_ERR(header_ser)) - return PTR_ERR(header_ser); - header_ser_pa = virt_to_phys(header_ser); +int __init luo_session_setup_incoming(u64 sessions_pa) +{ + struct luo_session_header *sh = &luo_session_global.incoming; + int err; - err = fdt_begin_node(fdt_out, LUO_FDT_SESSION_NODE_NAME); - err |= fdt_property_string(fdt_out, "compatible", - LUO_FDT_SESSION_COMPATIBLE); - err |= fdt_property(fdt_out, LUO_FDT_SESSION_HEADER, &header_ser_pa, - sizeof(header_ser_pa)); - err |= fdt_end_node(fdt_out); + if (!sessions_pa) + return 0; + err = kho_block_set_restore(&sh->block_set, sessions_pa); if (err) - goto err_unpreserve; - - luo_session_global.outgoing.header_ser = header_ser; - luo_session_global.outgoing.ser = (void *)(header_ser + 1); - luo_session_global.outgoing.active = true; + return err; + sh->active = true; return 0; - -err_unpreserve: - kho_unpreserve_free(header_ser); - return err; } -int __init luo_session_setup_incoming(void *fdt_in) +static int luo_session_deserialize_one(struct luo_session_header *sh, + struct luo_session_ser *ser) { - struct luo_session_header_ser *header_ser; - int err, header_size, offset; - u64 header_ser_pa; - const void *ptr; - - offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_SESSION_NODE_NAME); - if (offset < 0) { - pr_err("Unable to get session node: [%s]\n", - LUO_FDT_SESSION_NODE_NAME); - return -EINVAL; + struct luo_session *session; + int err; + + session = luo_session_alloc(ser->name); + if (IS_ERR(session)) { + pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n", + (int)sizeof(ser->name), ser->name, session); + return PTR_ERR(session); } - err = fdt_node_check_compatible(fdt_in, offset, - LUO_FDT_SESSION_COMPATIBLE); + err = luo_session_insert(sh, session); if (err) { - pr_err("Session node incompatible [%s]\n", - LUO_FDT_SESSION_COMPATIBLE); - return -EINVAL; + pr_warn("Failed to insert session [%s] %pe\n", + session->name, ERR_PTR(err)); + luo_session_free(session); + return err; } - header_size = 0; - ptr = fdt_getprop(fdt_in, offset, LUO_FDT_SESSION_HEADER, &header_size); - if (!ptr || header_size != sizeof(u64)) { - pr_err("Unable to get session header '%s' [%d]\n", - LUO_FDT_SESSION_HEADER, header_size); - return -EINVAL; + scoped_guard(mutex, &session->mutex) { + err = luo_file_deserialize(&session->file_set, + &ser->file_set_ser); + } + if (err) { + pr_warn("Failed to deserialize files for session [%s] %pe\n", + session->name, ERR_PTR(err)); + return err; } - - header_ser_pa = get_unaligned((u64 *)ptr); - header_ser = phys_to_virt(header_ser_pa); - - luo_session_global.incoming.header_ser = header_ser; - luo_session_global.incoming.ser = (void *)(header_ser + 1); - luo_session_global.incoming.active = true; return 0; } @@ -514,6 +586,8 @@ int luo_session_deserialize(void) { struct luo_session_header *sh = &luo_session_global.incoming; static bool is_deserialized; + struct luo_session_ser *ser; + struct kho_block_set_it it; static int saved_err; int err; @@ -540,43 +614,19 @@ int luo_session_deserialize(void) * userspace to detect the failure and trigger a reboot, which will * reliably reset devices and reclaim memory. */ - for (int i = 0; i < sh->header_ser->count; i++) { - struct luo_session *session; - - session = luo_session_alloc(sh->ser[i].name); - if (IS_ERR(session)) { - pr_warn("Failed to allocate session [%.*s] during deserialization %pe\n", - (int)sizeof(sh->ser[i].name), - sh->ser[i].name, session); - err = PTR_ERR(session); - goto save_err; - } - - err = luo_session_insert(sh, session); - if (err) { - pr_warn("Failed to insert session [%s] %pe\n", - session->name, ERR_PTR(err)); - luo_session_free(session); - goto save_err; - } - - scoped_guard(mutex, &session->mutex) { - err = luo_file_deserialize(&session->file_set, - &sh->ser[i].file_set_ser); - } - if (err) { - pr_warn("Failed to deserialize files for session [%s] %pe\n", - session->name, ERR_PTR(err)); + kho_block_set_it_init(&it, &sh->block_set); + while ((ser = kho_block_set_it_read_entry(&it))) { + err = luo_session_deserialize_one(sh, ser); + if (err) goto save_err; - } } - kho_restore_free(sh->header_ser); - sh->header_ser = NULL; - sh->ser = NULL; + kho_block_set_destroy(&sh->block_set); return 0; + save_err: + kho_block_set_destroy(&sh->block_set); saved_err = err; return err; } @@ -585,30 +635,48 @@ int luo_session_serialize(void) { struct luo_session_header *sh = &luo_session_global.outgoing; struct luo_session *session; - int i = 0; + struct kho_block_set_it it; int err; - guard(rwsem_write)(&sh->rwsem); + down_write(&luo_session_serialize_rwsem); + down_write(&sh->rwsem); + *sh->sessions_pa = 0; + + kho_block_set_it_init(&it, &sh->block_set); + list_for_each_entry(session, &sh->list, list) { - err = luo_session_freeze_one(session, &sh->ser[i]); - if (err) + struct luo_session_ser *ser = kho_block_set_it_reserve_entry(&it); + + /* This should not fail normally as blocks were pre-allocated */ + if (WARN_ON_ONCE(!ser)) { + err = -ENOSPC; + goto err_undo; + } + + err = luo_session_freeze_one(session, ser); + if (err) { + kho_block_set_it_prev(&it); goto err_undo; + } - strscpy(sh->ser[i].name, session->name, - sizeof(sh->ser[i].name)); - i++; + strscpy(ser->name, session->name, sizeof(ser->name)); } - sh->header_ser->count = sh->count; + + if (sh->count > 0) + *sh->sessions_pa = kho_block_set_head_pa(&sh->block_set); + up_write(&sh->rwsem); return 0; err_undo: list_for_each_entry_continue_reverse(session, &sh->list, list) { - i--; - luo_session_unfreeze_one(session, &sh->ser[i]); - memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name)); + struct luo_session_ser *ser = kho_block_set_it_prev(&it); + + luo_session_unfreeze_one(session, ser); + memset(ser->name, 0, sizeof(ser->name)); } + up_write(&sh->rwsem); + up_write(&luo_session_serialize_rwsem); return err; } - diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c index e4b0ecb..4c08a7c 100644 --- a/lib/tests/liveupdate.c +++ b/lib/tests/liveupdate.c @@ -105,6 +105,9 @@ static void liveupdate_test_init(void) pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n", flb->compatible, ERR_PTR(err)); } + + if (!err) + liveupdate_flb_put_incoming(flb); } initialized = true; } diff --git a/mm/memblock.c b/mm/memblock.c index ccd43f3..6349c48 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1028,40 +1028,6 @@ int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) } #endif -#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH -__init void memblock_set_kho_scratch_only(void) -{ - kho_scratch_only = true; -} - -__init void memblock_clear_kho_scratch_only(void) -{ - kho_scratch_only = false; -} - -__init void memmap_init_kho_scratch_pages(void) -{ - phys_addr_t start, end; - unsigned long pfn; - int nid; - u64 i; - - if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) - return; - - /* - * Initialize struct pages for free scratch memory. - * The struct pages for reserved scratch memory will be set up in - * memmap_init_reserved_pages() - */ - __for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE, - MEMBLOCK_KHO_SCRATCH, &start, &end, &nid) { - for (pfn = PFN_UP(start); pfn < PFN_DOWN(end); pfn++) - init_deferred_page(pfn, nid); - } -} -#endif - /** * memblock_setclr_flag - set or clear flag for a memory region * @type: memblock type to set/clear flag for @@ -2535,6 +2501,28 @@ int reserve_mem_release_by_name(const char *name) return 1; } +#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH +__init void memblock_set_kho_scratch_only(void) +{ + kho_scratch_only = true; +} + +__init void memblock_clear_kho_scratch_only(void) +{ + kho_scratch_only = false; +} + +bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr) +{ + int i = memblock_search(&memblock.memory, addr); + + if (i == -1) + return false; + + return memblock_is_kho_scratch(&memblock.memory.regions[i]); +} +#endif + #ifdef CONFIG_KEXEC_HANDOVER static int __init reserved_mem_preserve(void) diff --git a/mm/mm_init.c b/mm/mm_init.c index 65623f9..0f64909 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -706,9 +706,11 @@ void __meminit __init_page_from_nid(unsigned long pfn, int nid) } __init_single_page(pfn_to_page(pfn), pfn, zid, nid); - if (pageblock_aligned(pfn)) - init_pageblock_migratetype(pfn_to_page(pfn), MIGRATE_MOVABLE, - false); + if (pageblock_aligned(pfn)) { + enum migratetype mt = + kho_scratch_migratetype(pfn, MIGRATE_MOVABLE); + init_pageblock_migratetype(pfn_to_page(pfn), mt, false); + } } #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT @@ -941,7 +943,8 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone static void __init memmap_init_zone_range(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn, - unsigned long *hole_pfn) + unsigned long *hole_pfn, + enum migratetype mt) { unsigned long zone_start_pfn = zone->zone_start_pfn; unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; @@ -954,8 +957,7 @@ static void __init memmap_init_zone_range(struct zone *zone, return; memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, - zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE, - false); + zone_end_pfn, MEMINIT_EARLY, NULL, mt, false); if (*hole_pfn < start_pfn) init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); @@ -971,6 +973,8 @@ static void __init memmap_init(void) for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { struct pglist_data *node = NODE_DATA(nid); + enum migratetype mt = + kho_scratch_migratetype(start_pfn, MIGRATE_MOVABLE); for (j = 0; j < MAX_NR_ZONES; j++) { struct zone *zone = node->node_zones + j; @@ -979,7 +983,7 @@ static void __init memmap_init(void) continue; memmap_init_zone_range(zone, start_pfn, end_pfn, - &hole_pfn); + &hole_pfn, mt); zone_id = j; } } @@ -1969,7 +1973,7 @@ unsigned long __init node_map_pfn_alignment(void) #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT static void __init deferred_free_pages(unsigned long pfn, - unsigned long nr_pages) + unsigned long nr_pages, enum migratetype mt) { struct page *page; unsigned long i; @@ -1982,8 +1986,7 @@ static void __init deferred_free_pages(unsigned long pfn, /* Free a large naturally-aligned chunk if possible */ if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { for (i = 0; i < nr_pages; i += pageblock_nr_pages) - init_pageblock_migratetype(page + i, MIGRATE_MOVABLE, - false); + init_pageblock_migratetype(page + i, mt, false); __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); return; } @@ -1993,8 +1996,7 @@ static void __init deferred_free_pages(unsigned long pfn, for (i = 0; i < nr_pages; i++, page++, pfn++) { if (pageblock_aligned(pfn)) - init_pageblock_migratetype(page, MIGRATE_MOVABLE, - false); + init_pageblock_migratetype(page, mt, false); __free_pages_core(page, 0, MEMINIT_EARLY); } } @@ -2052,6 +2054,8 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, for_each_free_mem_range(i, nid, 0, &start, &end, NULL) { unsigned long spfn = PFN_UP(start); unsigned long epfn = PFN_DOWN(end); + enum migratetype mt = + kho_scratch_migratetype(spfn, MIGRATE_MOVABLE); if (spfn >= end_pfn) break; @@ -2064,7 +2068,7 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, unsigned long chunk_end = min(mo_pfn, epfn); nr_pages += deferred_init_pages(zone, spfn, chunk_end); - deferred_free_pages(spfn, chunk_end - spfn); + deferred_free_pages(spfn, chunk_end - spfn, mt); spfn = chunk_end; diff --git a/tools/testing/memblock/linux/mmzone.h b/tools/testing/memblock/linux/mmzone.h index bb68265..8d934ff 100644 --- a/tools/testing/memblock/linux/mmzone.h +++ b/tools/testing/memblock/linux/mmzone.h @@ -35,4 +35,8 @@ typedef struct pglist_data { } pg_data_t; +enum migratetype { + MIGRATE_CMA, +}; + #endif diff --git a/tools/testing/selftests/kho/vmtest.sh b/tools/testing/selftests/kho/vmtest.sh index 49fdac8..0014bd7 100755 --- a/tools/testing/selftests/kho/vmtest.sh +++ b/tools/testing/selftests/kho/vmtest.sh @@ -59,10 +59,14 @@ function build_kernel() { tee "$kconfig" > "$kho_config" < +#include "luo_test_utils.h" #include "../kselftest.h" #include "../kselftest_harness.h" @@ -102,6 +103,22 @@ static int create_session(int lu_fd, const char *name) return args.fd; } +/* Helper function to get a session name via ioctl. */ +static int get_session_name(int session_fd, char *name, size_t name_len) +{ + struct liveupdate_session_get_name args = {}; + + args.size = sizeof(args); + + if (ioctl(session_fd, LIVEUPDATE_SESSION_GET_NAME, &args)) + return -errno; + + strncpy(name, (char *)args.name, name_len - 1); + name[name_len - 1] = '\0'; + + return 0; +} + /* * Test Case: Create Duplicate Session * @@ -386,4 +403,175 @@ TEST_F(liveupdate_device, prevent_double_preservation) ASSERT_EQ(close(session_fd2), 0); } +/* + * Test Case: Create Session with No Null Termination + * + * Verifies that filling the entire 64-byte name field with non-null characters + * (no '\0' terminator) is rejected by the kernel with EINVAL. + */ +TEST_F(liveupdate_device, create_session_no_null_termination) +{ + struct liveupdate_ioctl_create_session args = {}; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + /* Fill entire name field with 'X', no null terminator */ + args.size = sizeof(args); + memset(args.name, 'X', sizeof(args.name)); + + EXPECT_LT(ioctl(self->fd1, LIVEUPDATE_IOCTL_CREATE_SESSION, &args), 0); + EXPECT_EQ(errno, EINVAL); +} + +/* + * Test Case: Create Session with Empty Name + * + * Verifies that creating a session with an empty string name fails + * with EINVAL. + */ +TEST_F(liveupdate_device, create_session_empty_name) +{ + int session_fd; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, ""); + EXPECT_EQ(session_fd, -EINVAL); +} + +/* + * Test Case: Get Session Name + * + * Verifies that the full session name can be retrieved from a session file + * descriptor via ioctl. + */ +TEST_F(liveupdate_device, get_session_name) +{ + char name_buf[LIVEUPDATE_SESSION_NAME_LENGTH] = {}; + const char *session_name = "get-name-test-session"; + int session_fd; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, session_name); + ASSERT_GE(session_fd, 0); + + ASSERT_EQ(get_session_name(session_fd, name_buf, sizeof(name_buf)), 0); + ASSERT_STREQ(name_buf, session_name); + + ASSERT_EQ(close(session_fd), 0); +} + +/* + * Test Case: Get Session Name at Maximum Length + * + * Verifies that a session name using the full LIVEUPDATE_SESSION_NAME_LENGTH + * (minus the null terminator) can be correctly retrieved. + */ +TEST_F(liveupdate_device, get_session_name_max_length) +{ + char name_buf[LIVEUPDATE_SESSION_NAME_LENGTH] = {}; + char long_name[LIVEUPDATE_SESSION_NAME_LENGTH]; + int session_fd; + + memset(long_name, 'A', sizeof(long_name) - 1); + long_name[sizeof(long_name) - 1] = '\0'; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, long_name); + ASSERT_GE(session_fd, 0); + + ASSERT_EQ(get_session_name(session_fd, name_buf, sizeof(name_buf)), 0); + ASSERT_STREQ(name_buf, long_name); + + ASSERT_EQ(close(session_fd), 0); +} + +/* + * Test Case: Manage Many Sessions + * + * Verifies that a large number of sessions can be created and then + * destroyed during normal system operation. This specifically tests the + * dynamic block allocation and reuse logic for session metadata management + * without preserving any files. + */ +TEST_F(liveupdate_device, preserve_many_sessions) +{ +#define MANY_SESSIONS 2000 + int session_fds[MANY_SESSIONS]; + int ret, i; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + ret = luo_ensure_nofile_limit(MANY_SESSIONS); + if (ret == -EPERM) + SKIP(return, "Insufficient privileges to set RLIMIT_NOFILE"); + ASSERT_EQ(ret, 0); + + for (i = 0; i < MANY_SESSIONS; i++) { + char name[64]; + + snprintf(name, sizeof(name), "many-session-%d", i); + session_fds[i] = create_session(self->fd1, name); + ASSERT_GE(session_fds[i], 0); + } + + for (i = 0; i < MANY_SESSIONS; i++) + ASSERT_EQ(close(session_fds[i]), 0); +} + +/* + * Test Case: Preserve Many Files + * + * Verifies that a large number of files can be preserved in a single session + * and then destroyed during normal system operation. This tests the dynamic + * block allocation and management for outgoing files. + */ +TEST_F(liveupdate_device, preserve_many_files) +{ +#define MANY_FILES 500 + int mem_fds[MANY_FILES]; + int session_fd, ret, i; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, "many-files-test"); + ASSERT_GE(session_fd, 0); + + ret = luo_ensure_nofile_limit(MANY_FILES + 10); + if (ret == -EPERM) + SKIP(return, "Insufficient privileges to set RLIMIT_NOFILE"); + ASSERT_EQ(ret, 0); + + for (i = 0; i < MANY_FILES; i++) { + mem_fds[i] = memfd_create("test-memfd", 0); + ASSERT_GE(mem_fds[i], 0); + ASSERT_EQ(preserve_fd(session_fd, mem_fds[i], i), 0); + } + + for (i = 0; i < MANY_FILES; i++) + ASSERT_EQ(close(mem_fds[i]), 0); + + ASSERT_EQ(close(session_fd), 0); +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/liveupdate/luo_stress_files.c b/tools/testing/selftests/liveupdate/luo_stress_files.c new file mode 100644 index 0000000..0cdf9cd --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_stress_files.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + * + * Validate that LUO can handle a large number of files per session across + * a kexec reboot. + */ + +#include +#include +#include "luo_test_utils.h" + +#define NUM_FILES 500 +#define STATE_SESSION_NAME "kexec_many_files_state" +#define STATE_MEMFD_TOKEN 9999 +#define TEST_SESSION_NAME "many_files_session" + +/* Stage 1: Executed before the kexec reboot. */ +static void run_stage_1(int luo_fd) +{ + int session_fd, i; + + ksft_print_msg("[STAGE 1] Creating state file for next stage (2)...\n"); + create_state_file(luo_fd, STATE_SESSION_NAME, STATE_MEMFD_TOKEN, 2); + + ksft_print_msg("[STAGE 1] Creating test session '%s'...\n", TEST_SESSION_NAME); + session_fd = luo_create_session(luo_fd, TEST_SESSION_NAME); + if (session_fd < 0) + fail_exit("luo_create_session"); + + ksft_print_msg("[STAGE 1] Preserving %d files...\n", NUM_FILES); + for (i = 0; i < NUM_FILES; i++) { + char data[64]; + + snprintf(data, sizeof(data), "file-data-%d", i); + if (create_and_preserve_memfd(session_fd, i, data) < 0) + fail_exit("create_and_preserve_memfd for index %d", i); + } + + ksft_print_msg("[STAGE 1] Successfully preserved %d files.\n", NUM_FILES); + + close(luo_fd); + daemonize_and_wait(); +} + +/* Stage 2: Executed after the kexec reboot. */ +static void run_stage_2(int luo_fd, int state_session_fd) +{ + int session_fd; + int i, stage; + + ksft_print_msg("[STAGE 2] Starting post-kexec verification...\n"); + + restore_and_read_stage(state_session_fd, STATE_MEMFD_TOKEN, &stage); + if (stage != 2) { + fail_exit("Expected stage 2, but state file contains %d", + stage); + } + + ksft_print_msg("[STAGE 2] Retrieving test session '%s'...\n", TEST_SESSION_NAME); + session_fd = luo_retrieve_session(luo_fd, TEST_SESSION_NAME); + if (session_fd < 0) + fail_exit("luo_retrieve_session"); + + ksft_print_msg("[STAGE 2] Verifying %d files...\n", NUM_FILES); + for (i = 0; i < NUM_FILES; i++) { + char data[64]; + int fd; + + snprintf(data, sizeof(data), "file-data-%d", i); + fd = restore_and_verify_memfd(session_fd, i, data); + if (fd < 0) + fail_exit("restore_and_verify_memfd for index %d", i); + close(fd); + } + + ksft_print_msg("[STAGE 2] Finishing test session...\n"); + if (luo_session_finish(session_fd) < 0) + fail_exit("luo_session_finish for test session"); + close(session_fd); + + ksft_print_msg("[STAGE 2] Finalizing state session...\n"); + if (luo_session_finish(state_session_fd) < 0) + fail_exit("luo_session_finish for state session"); + close(state_session_fd); + + ksft_print_msg("\n--- MANY-FILES KEXEC TEST PASSED (%d files) ---\n", + NUM_FILES); +} + +int main(int argc, char *argv[]) +{ + return luo_test(argc, argv, STATE_SESSION_NAME, + run_stage_1, run_stage_2); +} diff --git a/tools/testing/selftests/liveupdate/luo_stress_sessions.c b/tools/testing/selftests/liveupdate/luo_stress_sessions.c new file mode 100644 index 0000000..f201b18 --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_stress_sessions.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin + * + * Validate that LUO can handle a large number of sessions across a kexec + * reboot. + */ + +#include +#include +#include "luo_test_utils.h" + +#define NUM_SESSIONS 2000 +#define STATE_SESSION_NAME "kexec_many_state" +#define STATE_MEMFD_TOKEN 999 + +/* Stage 1: Executed before the kexec reboot. */ +static void run_stage_1(int luo_fd) +{ + int ret, i; + + ksft_print_msg("[STAGE 1] Increasing ulimit for open files...\n"); + ret = luo_ensure_nofile_limit(NUM_SESSIONS); + if (ret == -EPERM) + ksft_exit_skip("Insufficient privileges to set RLIMIT_NOFILE\n"); + if (ret < 0) + ksft_exit_fail_msg("luo_ensure_nofile_limit failed: %s\n", strerror(-ret)); + + ksft_print_msg("[STAGE 1] Creating state file for next stage (2)...\n"); + create_state_file(luo_fd, STATE_SESSION_NAME, STATE_MEMFD_TOKEN, 2); + + ksft_print_msg("[STAGE 1] Creating %d sessions...\n", NUM_SESSIONS); + + for (i = 0; i < NUM_SESSIONS; i++) { + char name[LIVEUPDATE_SESSION_NAME_LENGTH]; + int s_fd; + + snprintf(name, sizeof(name), "many-test-%d", i); + s_fd = luo_create_session(luo_fd, name); + if (s_fd < 0) { + fail_exit("luo_create_session for '%s' at index %d", + name, i); + } + } + + ksft_print_msg("[STAGE 1] Successfully created %d sessions.\n", + NUM_SESSIONS); + + close(luo_fd); + daemonize_and_wait(); +} + +/* Stage 2: Executed after the kexec reboot. */ +static void run_stage_2(int luo_fd, int state_session_fd) +{ + int i, stage; + + ksft_print_msg("[STAGE 2] Starting post-kexec verification...\n"); + + restore_and_read_stage(state_session_fd, STATE_MEMFD_TOKEN, &stage); + if (stage != 2) { + fail_exit("Expected stage 2, but state file contains %d", + stage); + } + + ksft_print_msg("[STAGE 2] Retrieving and finishing %d sessions...\n", + NUM_SESSIONS); + + for (i = 0; i < NUM_SESSIONS; i++) { + char name[LIVEUPDATE_SESSION_NAME_LENGTH]; + int s_fd; + + snprintf(name, sizeof(name), "many-test-%d", i); + s_fd = luo_retrieve_session(luo_fd, name); + if (s_fd < 0) { + fail_exit("luo_retrieve_session for '%s' at index %d", + name, i); + } + + if (luo_session_finish(s_fd) < 0) { + fail_exit("luo_session_finish for '%s' at index %d", + name, i); + } + close(s_fd); + } + + ksft_print_msg("[STAGE 2] Finalizing state session...\n"); + if (luo_session_finish(state_session_fd) < 0) + fail_exit("luo_session_finish for state session"); + close(state_session_fd); + + ksft_print_msg("\n--- MANY-SESSIONS KEXEC TEST PASSED (%d sessions) ---\n", + NUM_SESSIONS); +} + +int main(int argc, char *argv[]) +{ + return luo_test(argc, argv, STATE_SESSION_NAME, + run_stage_1, run_stage_2); +} diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.c b/tools/testing/selftests/liveupdate/luo_test_utils.c index 3c8721c..333a353 100644 --- a/tools/testing/selftests/liveupdate/luo_test_utils.c +++ b/tools/testing/selftests/liveupdate/luo_test_utils.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,29 @@ int luo_open_device(void) return open(LUO_DEVICE, O_RDWR); } +int luo_ensure_nofile_limit(long min_limit) +{ + struct rlimit hl; + + /* Allow to extra files to be used by test itself */ + min_limit += 32; + + if (getrlimit(RLIMIT_NOFILE, &hl) < 0) + return -errno; + + if (hl.rlim_cur >= min_limit) + return 0; + + hl.rlim_cur = min_limit; + if (hl.rlim_cur > hl.rlim_max) + hl.rlim_max = hl.rlim_cur; + + if (setrlimit(RLIMIT_NOFILE, &hl) < 0) + return -errno; + + return 0; +} + int luo_create_session(int luo_fd, const char *name) { struct liveupdate_ioctl_create_session arg = { .size = sizeof(arg) }; diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.h b/tools/testing/selftests/liveupdate/luo_test_utils.h index 90099bf..6a0d853 100644 --- a/tools/testing/selftests/liveupdate/luo_test_utils.h +++ b/tools/testing/selftests/liveupdate/luo_test_utils.h @@ -26,6 +26,8 @@ int luo_create_session(int luo_fd, const char *name); int luo_retrieve_session(int luo_fd, const char *name); int luo_session_finish(int session_fd); +int luo_ensure_nofile_limit(long min_limit); + int create_and_preserve_memfd(int session_fd, int token, const char *data); int restore_and_verify_memfd(int session_fd, int token, const char *expected_data);