projects/linux·blob master · refs

626 lines · 18.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright (c) 2025, Google LLC.
5 * Pasha Tatashin <[email protected]>
6 */
7
8/**
9 * DOC: LUO File Lifecycle Bound Global Data
10 *
11 * File-Lifecycle-Bound (FLB) objects provide a mechanism for managing global
12 * state that is shared across multiple live-updatable files. The lifecycle of
13 * this shared state is tied to the preservation of the files that depend on it.
14 *
15 * An FLB represents a global resource, such as the IOMMU core state, that is
16 * required by multiple file descriptors (e.g., all VFIO fds).
17 *
18 * The preservation of the FLB's state is triggered when the *first* file
19 * depending on it is preserved. The cleanup of this state (unpreserve or
20 * finish) is triggered when the *last* file depending on it is unpreserved or
21 * finished.
22 *
23 * Handler Dependency: A file handler declares its dependency on one or more
24 * FLBs by registering them via liveupdate_register_flb().
25 *
26 * Callback Model: Each FLB is defined by a set of operations
27 * (&struct liveupdate_flb_ops) that LUO invokes at key points:
28 *
29 * - .preserve(): Called for the first file. Saves global state.
30 * - .unpreserve(): Called for the last file (if aborted pre-reboot).
31 * - .retrieve(): Called on-demand in the new kernel to restore the state.
32 * - .finish(): Called for the last file in the new kernel for cleanup.
33 *
34 * This reference-counted approach ensures that shared state is saved exactly
35 * once and restored exactly once, regardless of how many files depend on it,
36 * and that its lifecycle is correctly managed across the kexec transition.
37 */
38
39#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
40
41#include <linux/cleanup.h>
42#include <linux/err.h>
43#include <linux/errno.h>
44#include <linux/io.h>
45#include <linux/kexec_handover.h>
46#include <linux/kho/abi/luo.h>
47#include <linux/list_private.h>
48#include <linux/liveupdate.h>
49#include <linux/module.h>
50#include <linux/mutex.h>
51#include <linux/slab.h>
52#include "luo_internal.h"
53
54#define LUO_FLB_PGCNT 1ul
55#define LUO_FLB_MAX (((LUO_FLB_PGCNT << PAGE_SHIFT) - \
56 sizeof(struct luo_flb_header_ser)) / sizeof(struct luo_flb_ser))
57
58struct luo_flb_header {
59 struct luo_flb_header_ser *header_ser;
60 struct luo_flb_ser *ser;
61 bool active;
62};
63
64struct luo_flb_global {
65 struct luo_flb_header incoming;
66 struct luo_flb_header outgoing;
67 struct list_head list;
68 long count;
69};
70
71static struct luo_flb_global luo_flb_global = {
72 .list = LIST_HEAD_INIT(luo_flb_global.list),
73};
74
75/*
76 * struct luo_flb_link - Links an FLB definition to a file handler's internal
77 * list of dependencies.
78 * @flb: A pointer to the registered &struct liveupdate_flb definition.
79 * @list: The list_head for linking.
80 */
81struct luo_flb_link {
82 struct liveupdate_flb *flb;
83 struct list_head list;
84};
85
86/* luo_flb_get_private - Access private field, and if needed initialize it. */
87static struct luo_flb_private *luo_flb_get_private(struct liveupdate_flb *flb)
88{
89 struct luo_flb_private *private = &ACCESS_PRIVATE(flb, private);
90 static DEFINE_SPINLOCK(luo_flb_init_lock);
91
92 if (smp_load_acquire(&private->initialized))
93 return private;
94
95 guard(spinlock)(&luo_flb_init_lock);
96 if (!private->initialized) {
97 mutex_init(&private->incoming.lock);
98 mutex_init(&private->outgoing.lock);
99 INIT_LIST_HEAD(&private->list);
100 private->users = 0;
101 smp_store_release(&private->initialized, true);
102 }
103
104 return private;
105}
106
107static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
108{
109 struct luo_flb_private *private = luo_flb_get_private(flb);
110
111 scoped_guard(mutex, &private->outgoing.lock) {
112 if (!refcount_read(&private->outgoing.count)) {
113 struct liveupdate_flb_op_args args = {0};
114 int err;
115
116 if (!try_module_get(flb->ops->owner))
117 return -ENODEV;
118
119 args.flb = flb;
120 err = flb->ops->preserve(&args);
121 if (err) {
122 module_put(flb->ops->owner);
123 return err;
124 }
125 private->outgoing.data = args.data;
126 private->outgoing.obj = args.obj;
127 refcount_set(&private->outgoing.count, 1);
128 } else {
129 refcount_inc(&private->outgoing.count);
130 }
131 }
132
133 return 0;
134}
135
136void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb)
137{
138 struct luo_flb_private *private = luo_flb_get_private(flb);
139
140 scoped_guard(mutex, &private->outgoing.lock) {
141 if (refcount_dec_and_test(&private->outgoing.count)) {
142 struct liveupdate_flb_op_args args = {0};
143
144 args.flb = flb;
145 args.data = private->outgoing.data;
146 args.obj = private->outgoing.obj;
147
148 if (flb->ops->unpreserve)
149 flb->ops->unpreserve(&args);
150
151 private->outgoing.data = 0;
152 private->outgoing.obj = NULL;
153 module_put(flb->ops->owner);
154 }
155 }
156}
157
158static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
159{
160 struct luo_flb_private *private = luo_flb_get_private(flb);
161 struct luo_flb_header *fh = &luo_flb_global.incoming;
162 struct liveupdate_flb_op_args args = {0};
163 bool found = false;
164 int err;
165
166 lockdep_assert_held(&private->incoming.lock);
167
168 if (private->incoming.finished)
169 return -ENODATA;
170
171 if (private->incoming.retrieve_status < 0)
172 return private->incoming.retrieve_status;
173
174 if (private->incoming.retrieve_status > 0)
175 return 0;
176
177 if (!fh->active)
178 return -ENODATA;
179
180 for (int i = 0; i < fh->header_ser->count; i++) {
181 if (!strcmp(fh->ser[i].name, flb->compatible)) {
182 private->incoming.data = fh->ser[i].data;
183 refcount_set(&private->incoming.count, fh->ser[i].count);
184 found = true;
185 break;
186 }
187 }
188
189 if (!found)
190 return -ENOENT;
191
192 if (!try_module_get(flb->ops->owner))
193 return -ENODEV;
194
195 args.flb = flb;
196 args.data = private->incoming.data;
197
198 err = flb->ops->retrieve(&args);
199 if (err) {
200 private->incoming.retrieve_status = err;
201 module_put(flb->ops->owner);
202 return err;
203 }
204
205 private->incoming.obj = args.obj;
206 private->incoming.retrieve_status = 1;
207
208 return 0;
209}
210
211void liveupdate_flb_put_incoming(struct liveupdate_flb *flb)
212{
213 struct luo_flb_private *private = luo_flb_get_private(flb);
214 struct liveupdate_flb_op_args args = {0};
215
216 scoped_guard(mutex, &private->incoming.lock) {
217 if (!refcount_dec_and_test(&private->incoming.count))
218 return;
219
220 if (private->incoming.retrieve_status <= 0) {
221 int err = luo_flb_retrieve_one(flb);
222
223 if (WARN_ON(err))
224 return;
225 }
226
227 args.flb = flb;
228 args.obj = private->incoming.obj;
229 flb->ops->finish(&args);
230
231 private->incoming.data = 0;
232 private->incoming.obj = NULL;
233 private->incoming.finished = true;
234 module_put(flb->ops->owner);
235 }
236}
237
238/**
239 * luo_flb_file_preserve - Notifies FLBs that a file is about to be preserved.
240 * @fh: The file handler for the preserved file.
241 *
242 * This function iterates through all FLBs associated with the given file
243 * handler. It increments the reference count for each FLB. If the count becomes
244 * 1, it triggers the FLB's .preserve() callback to save the global state.
245 *
246 * This operation is atomic. If any FLB's .preserve() op fails, it will roll
247 * back by calling .unpreserve() on any FLBs that were successfully preserved
248 * during this call.
249 *
250 * Context: Called from luo_preserve_file()
251 * Return: 0 on success, or a negative errno on failure.
252 */
253int luo_flb_file_preserve(struct liveupdate_file_handler *fh)
254{
255 struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
256 struct luo_flb_link *iter;
257 int err = 0;
258
259 down_read(&luo_register_rwlock);
260 list_for_each_entry(iter, flb_list, list) {
261 err = luo_flb_file_preserve_one(iter->flb);
262 if (err)
263 goto exit_err;
264 }
265 up_read(&luo_register_rwlock);
266
267 return 0;
268
269exit_err:
270 list_for_each_entry_continue_reverse(iter, flb_list, list)
271 liveupdate_flb_put_outgoing(iter->flb);
272 up_read(&luo_register_rwlock);
273
274 return err;
275}
276
277/**
278 * luo_flb_file_unpreserve - Notifies FLBs that a dependent file was unpreserved.
279 * @fh: The file handler for the unpreserved file.
280 *
281 * This function iterates through all FLBs associated with the given file
282 * handler, in reverse order of registration. It decrements the reference count
283 * for each FLB. If the count becomes 0, it triggers the FLB's .unpreserve()
284 * callback to clean up the global state.
285 *
286 * Context: Called when a preserved file is being cleaned up before reboot
287 * (e.g., from luo_file_unpreserve_files()).
288 */
289void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh)
290{
291 struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
292 struct luo_flb_link *iter;
293
294 guard(rwsem_read)(&luo_register_rwlock);
295 list_for_each_entry_reverse(iter, flb_list, list)
296 liveupdate_flb_put_outgoing(iter->flb);
297}
298
299/**
300 * luo_flb_file_finish - Notifies FLBs that a dependent file has been finished.
301 * @fh: The file handler for the finished file.
302 *
303 * This function iterates through all FLBs associated with the given file
304 * handler, in reverse order of registration. It decrements the incoming
305 * reference count for each FLB. If the count becomes 0, it triggers the FLB's
306 * .finish() callback for final cleanup in the new kernel.
307 *
308 * Context: Called from luo_file_finish() for each file being finished.
309 */
310void luo_flb_file_finish(struct liveupdate_file_handler *fh)
311{
312 struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
313 struct luo_flb_link *iter;
314
315 guard(rwsem_read)(&luo_register_rwlock);
316 list_for_each_entry_reverse(iter, flb_list, list)
317 liveupdate_flb_put_incoming(iter->flb);
318}
319
320static void luo_flb_unregister_one(struct liveupdate_file_handler *fh,
321 struct liveupdate_flb *flb)
322{
323 struct luo_flb_private *private = luo_flb_get_private(flb);
324 struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
325 struct luo_flb_link *iter;
326 bool found = false;
327
328 /* Find and remove the link from the file handler's list */
329 list_for_each_entry(iter, flb_list, list) {
330 if (iter->flb == flb) {
331 list_del(&iter->list);
332 kfree(iter);
333 found = true;
334 break;
335 }
336 }
337
338 if (!found) {
339 pr_warn("Failed to unregister FLB '%s': not found in file handler '%s'\n",
340 flb->compatible, fh->compatible);
341 return;
342 }
343
344 private->users--;
345
346 /*
347 * If this is the last file-handler with which we are registred, remove
348 * from the global list.
349 */
350 if (!private->users) {
351 list_del_init(&private->list);
352 luo_flb_global.count--;
353 }
354}
355
356/**
357 * luo_flb_unregister_all - Unregister all FLBs associated with a file handler.
358 * @fh: The file handler whose FLBs should be unregistered.
359 *
360 * This function iterates through the list of FLBs associated with the given
361 * file handler and unregisters them all one by one.
362 */
363void luo_flb_unregister_all(struct liveupdate_file_handler *fh)
364{
365 struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
366 struct luo_flb_link *iter, *tmp;
367
368 if (!liveupdate_enabled())
369 return;
370
371 lockdep_assert_held_write(&luo_register_rwlock);
372 list_for_each_entry_safe(iter, tmp, flb_list, list)
373 luo_flb_unregister_one(fh, iter->flb);
374}
375
376/**
377 * liveupdate_register_flb - Associate an FLB with a file handler and register it globally.
378 * @fh: The file handler that will now depend on the FLB.
379 * @flb: The File-Lifecycle-Bound object to associate.
380 *
381 * Establishes a dependency, informing the LUO core that whenever a file of
382 * type @fh is preserved, the state of @flb must also be managed.
383 *
384 * On the first registration of a given @flb object, it is added to a global
385 * registry. This function checks for duplicate registrations, both for a
386 * specific handler and globally, and ensures the total number of unique
387 * FLBs does not exceed the system limit.
388 *
389 * Context: Typically called from a subsystem's module init function after
390 * both the handler and the FLB have been defined and initialized.
391 * Return: 0 on success. Returns a negative errno on failure:
392 * -EINVAL if arguments are NULL or not initialized.
393 * -ENOMEM on memory allocation failure.
394 * -EEXIST if this FLB is already registered with this handler.
395 * -ENOSPC if the maximum number of global FLBs has been reached.
396 * -EOPNOTSUPP if live update is disabled or not configured.
397 */
398int liveupdate_register_flb(struct liveupdate_file_handler *fh,
399 struct liveupdate_flb *flb)
400{
401 struct luo_flb_private *private = luo_flb_get_private(flb);
402 struct list_head *flb_list = &ACCESS_PRIVATE(fh, flb_list);
403 struct luo_flb_link *link __free(kfree) = NULL;
404 struct liveupdate_flb *gflb;
405 struct luo_flb_link *iter;
406
407 if (!liveupdate_enabled())
408 return -EOPNOTSUPP;
409
410 if (WARN_ON(!flb->ops->preserve || !flb->ops->unpreserve ||
411 !flb->ops->retrieve || !flb->ops->finish)) {
412 return -EINVAL;
413 }
414
415 /*
416 * File handler must already be registered, as it initializes the
417 * flb_list
418 */
419 if (WARN_ON(list_empty(&ACCESS_PRIVATE(fh, list))))
420 return -EINVAL;
421
422 link = kzalloc_obj(*link);
423 if (!link)
424 return -ENOMEM;
425
426 guard(rwsem_write)(&luo_register_rwlock);
427
428 /* Check that this FLB is not already linked to this file handler */
429 list_for_each_entry(iter, flb_list, list) {
430 if (iter->flb == flb)
431 return -EEXIST;
432 }
433
434 /*
435 * If this FLB is not linked to global list it's the first time the FLB
436 * is registered
437 */
438 if (!private->users) {
439 if (WARN_ON(!list_empty(&private->list)))
440 return -EINVAL;
441
442 if (luo_flb_global.count == LUO_FLB_MAX)
443 return -ENOSPC;
444
445 /* Check that compatible string is unique in global list */
446 list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) {
447 if (!strcmp(gflb->compatible, flb->compatible))
448 return -EEXIST;
449 }
450
451 list_add_tail(&private->list, &luo_flb_global.list);
452 luo_flb_global.count++;
453 }
454
455 /* Finally, link the FLB to the file handler */
456 private->users++;
457 link->flb = flb;
458 list_add_tail(&no_free_ptr(link)->list, flb_list);
459
460 return 0;
461}
462
463/**
464 * liveupdate_unregister_flb - Remove an FLB dependency from a file handler.
465 * @fh: The file handler that is currently depending on the FLB.
466 * @flb: The File-Lifecycle-Bound object to remove.
467 *
468 * Removes the association between the specified file handler and the FLB
469 * previously established by liveupdate_register_flb().
470 *
471 * This function manages the global lifecycle of the FLB. It decrements the
472 * FLB's usage count. If this was the last file handler referencing this FLB,
473 * the FLB is removed from the global registry and the reference to its
474 * owner module (acquired during registration) is released.
475 *
476 * Context: It is typically called from a subsystem's module exit function.
477 */
478void liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
479 struct liveupdate_flb *flb)
480{
481 if (!liveupdate_enabled())
482 return;
483
484 guard(rwsem_write)(&luo_register_rwlock);
485
486 luo_flb_unregister_one(fh, flb);
487}
488
489/**
490 * liveupdate_flb_get_incoming - Retrieve the incoming FLB object.
491 * @flb: The FLB definition.
492 * @objp: Output parameter; will be populated with the live shared object.
493 *
494 * Returns a pointer to its shared live object for the incoming (post-reboot)
495 * path.
496 *
497 * If this is the first time the object is requested in the new kernel, this
498 * function will trigger the FLB's .retrieve() callback to reconstruct the
499 * object from its preserved state. Subsequent calls will return the same
500 * cached object.
501 *
502 * Return: 0 on success, or a negative errno on failure. -ENODATA means no
503 * incoming FLB data, -ENOENT means specific flb not found in the incoming
504 * data, -ENODEV if the FLB's module is unloading, and -EOPNOTSUPP when
505 * live update is disabled or not configured.
506 */
507int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
508{
509 struct luo_flb_private *private = luo_flb_get_private(flb);
510
511 if (!liveupdate_enabled())
512 return -EOPNOTSUPP;
513
514 guard(mutex)(&private->incoming.lock);
515
516 if (!private->incoming.obj) {
517 int err = luo_flb_retrieve_one(flb);
518
519 if (err)
520 return err;
521 }
522
523 refcount_inc(&private->incoming.count);
524 *objp = private->incoming.obj;
525
526 return 0;
527}
528
529/**
530 * liveupdate_flb_get_outgoing - Retrieve the outgoing FLB object.
531 * @flb: The FLB definition.
532 * @objp: Output parameter; will be populated with the live shared object.
533 *
534 * Returns a pointer to its shared live object for the outgoing (pre-reboot)
535 * path.
536 *
537 * This function assumes the object has already been created by the FLB's
538 * .preserve() callback, which is triggered when the first dependent file
539 * is preserved.
540 *
541 * Return: 0 on success, or a negative errno on failure.
542 */
543int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp)
544{
545 struct luo_flb_private *private = luo_flb_get_private(flb);
546
547 if (!liveupdate_enabled())
548 return -EOPNOTSUPP;
549
550 guard(mutex)(&private->outgoing.lock);
551 if (!private->outgoing.obj)
552 return -ENOENT;
553
554 refcount_inc(&private->outgoing.count);
555 *objp = private->outgoing.obj;
556
557 return 0;
558}
559
560int __init luo_flb_setup_outgoing(u64 *flbs_pa)
561{
562 struct luo_flb_header_ser *header_ser;
563
564 header_ser = kho_alloc_preserve(LUO_FLB_PGCNT << PAGE_SHIFT);
565 if (IS_ERR(header_ser))
566 return PTR_ERR(header_ser);
567
568 *flbs_pa = virt_to_phys(header_ser);
569
570 header_ser->pgcnt = LUO_FLB_PGCNT;
571 luo_flb_global.outgoing.header_ser = header_ser;
572 luo_flb_global.outgoing.ser = (void *)(header_ser + 1);
573 luo_flb_global.outgoing.active = true;
574
575 return 0;
576}
577
578void __init luo_flb_setup_incoming(u64 flbs_pa)
579{
580 struct luo_flb_header_ser *header_ser;
581
582 if (!flbs_pa)
583 return;
584
585 header_ser = phys_to_virt(flbs_pa);
586 luo_flb_global.incoming.header_ser = header_ser;
587 luo_flb_global.incoming.ser = (void *)(header_ser + 1);
588 luo_flb_global.incoming.active = true;
589}
590
591/**
592 * luo_flb_serialize - Serializes all active FLB objects for KHO.
593 *
594 * This function is called from the reboot path. It iterates through all
595 * registered File-Lifecycle-Bound (FLB) objects. For each FLB that has been
596 * preserved (i.e., its reference count is greater than zero), it writes its
597 * metadata into the memory region designated for Kexec Handover.
598 *
599 * The serialized data includes the FLB's compatibility string, its opaque
600 * data handle, and the final reference count. This allows the new kernel to
601 * find the appropriate handler and reconstruct the FLB's state.
602 *
603 * Context: Called from liveupdate_reboot() just before kho_finalize().
604 */
605void luo_flb_serialize(void)
606{
607 struct luo_flb_header *fh = &luo_flb_global.outgoing;
608 struct liveupdate_flb *gflb;
609 int i = 0;
610
611 guard(rwsem_read)(&luo_register_rwlock);
612 list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) {
613 struct luo_flb_private *private = luo_flb_get_private(gflb);
614 long count = refcount_read(&private->outgoing.count);
615
616 if (count > 0) {
617 strscpy(fh->ser[i].name, gflb->compatible,
618 sizeof(fh->ser[i].name));
619 fh->ser[i].data = private->outgoing.data;
620 fh->ser[i].count = count;
621 i++;
622 }
623 }
624
625 fh->header_ser->count = i;
626}