projects/linux·blob v6.5 · refs

1432 lines · 48.3 KiB

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4//!
5//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6//! overflow.
7//!
8//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10//!
11//! # Overview
12//!
13//! To initialize a `struct` with an in-place constructor you will need two things:
14//! - an in-place constructor,
15//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
16//! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]).
17//!
18//! To get an in-place constructor there are generally three options:
19//! - directly creating an in-place constructor using the [`pin_init!`] macro,
20//! - a custom function/macro returning an in-place constructor provided by someone else,
21//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
22//!
23//! Aside from pinned initialization, this API also supports in-place construction without pinning,
24//! the macros/types/functions are generally named like the pinned variants without the `pin`
25//! prefix.
26//!
27//! # Examples
28//!
29//! ## Using the [`pin_init!`] macro
30//!
31//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
32//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
33//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
34//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
35//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
36//!
37//! ```rust
38//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
39//! use kernel::{prelude::*, sync::Mutex, new_mutex};
40//! # use core::pin::Pin;
41//! #[pin_data]
42//! struct Foo {
43//! #[pin]
44//! a: Mutex<usize>,
45//! b: u32,
46//! }
47//!
48//! let foo = pin_init!(Foo {
49//! a <- new_mutex!(42, "Foo::a"),
50//! b: 24,
51//! });
52//! ```
53//!
54//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
55//! (or just the stack) to actually initialize a `Foo`:
56//!
57//! ```rust
58//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
59//! # use kernel::{prelude::*, sync::Mutex, new_mutex};
60//! # use core::pin::Pin;
61//! # #[pin_data]
62//! # struct Foo {
63//! # #[pin]
64//! # a: Mutex<usize>,
65//! # b: u32,
66//! # }
67//! # let foo = pin_init!(Foo {
68//! # a <- new_mutex!(42, "Foo::a"),
69//! # b: 24,
70//! # });
71//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo);
72//! ```
73//!
74//! For more information see the [`pin_init!`] macro.
75//!
76//! ## Using a custom function/macro that returns an initializer
77//!
78//! Many types from the kernel supply a function/macro that returns an initializer, because the
79//! above method only works for types where you can access the fields.
80//!
81//! ```rust
82//! # use kernel::{new_mutex, sync::{Arc, Mutex}};
83//! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx"));
84//! ```
85//!
86//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
87//!
88//! ```rust
89//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
90//! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init};
91//! #[pin_data]
92//! struct DriverData {
93//! #[pin]
94//! status: Mutex<i32>,
95//! buffer: Box<[u8; 1_000_000]>,
96//! }
97//!
98//! impl DriverData {
99//! fn new() -> impl PinInit<Self, Error> {
100//! try_pin_init!(Self {
101//! status <- new_mutex!(0, "DriverData::status"),
102//! buffer: Box::init(kernel::init::zeroed())?,
103//! })
104//! }
105//! }
106//! ```
107//!
108//! ## Manual creation of an initializer
109//!
110//! Often when working with primitives the previous approaches are not sufficient. That is where
111//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
112//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
113//! actually does the initialization in the correct way. Here are the things to look out for
114//! (we are calling the parameter to the closure `slot`):
115//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
116//! `slot` now contains a valid bit pattern for the type `T`,
117//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
118//! you need to take care to clean up anything if your initialization fails mid-way,
119//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
120//! `slot` gets called.
121//!
122//! ```rust
123//! use kernel::{prelude::*, init};
124//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
125//! # mod bindings {
126//! # pub struct foo;
127//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
128//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
129//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
130//! # }
131//! /// # Invariants
132//! ///
133//! /// `foo` is always initialized
134//! #[pin_data(PinnedDrop)]
135//! pub struct RawFoo {
136//! #[pin]
137//! foo: Opaque<bindings::foo>,
138//! #[pin]
139//! _p: PhantomPinned,
140//! }
141//!
142//! impl RawFoo {
143//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
144//! // SAFETY:
145//! // - when the closure returns `Ok(())`, then it has successfully initialized and
146//! // enabled `foo`,
147//! // - when it returns `Err(e)`, then it has cleaned up before
148//! unsafe {
149//! init::pin_init_from_closure(move |slot: *mut Self| {
150//! // `slot` contains uninit memory, avoid creating a reference.
151//! let foo = addr_of_mut!((*slot).foo);
152//!
153//! // Initialize the `foo`
154//! bindings::init_foo(Opaque::raw_get(foo));
155//!
156//! // Try to enable it.
157//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
158//! if err != 0 {
159//! // Enabling has failed, first clean up the foo and then return the error.
160//! bindings::destroy_foo(Opaque::raw_get(foo));
161//! return Err(Error::from_kernel_errno(err));
162//! }
163//!
164//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
165//! Ok(())
166//! })
167//! }
168//! }
169//! }
170//!
171//! #[pinned_drop]
172//! impl PinnedDrop for RawFoo {
173//! fn drop(self: Pin<&mut Self>) {
174//! // SAFETY: Since `foo` is initialized, destroying is safe.
175//! unsafe { bindings::destroy_foo(self.foo.get()) };
176//! }
177//! }
178//! ```
179//!
180//! For the special case where initializing a field is a single FFI-function call that cannot fail,
181//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
182//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
183//! with [`pin_init!`].
184//!
185//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
186//! the `kernel` crate. The [`sync`] module is a good starting point.
187//!
188//! [`sync`]: kernel::sync
189//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
190//! [structurally pinned fields]:
191//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
192//! [stack]: crate::stack_pin_init
193//! [`Arc<T>`]: crate::sync::Arc
194//! [`impl PinInit<Foo>`]: PinInit
195//! [`impl PinInit<T, E>`]: PinInit
196//! [`impl Init<T, E>`]: Init
197//! [`Opaque`]: kernel::types::Opaque
198//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
199//! [`pin_data`]: ::macros::pin_data
200//! [`pin_init!`]: crate::pin_init!
201
202use crate::{
203 error::{self, Error},
204 sync::UniqueArc,
205};
206use alloc::boxed::Box;
207use core::{
208 alloc::AllocError,
209 cell::Cell,
210 convert::Infallible,
211 marker::PhantomData,
212 mem::MaybeUninit,
213 num::*,
214 pin::Pin,
215 ptr::{self, NonNull},
216};
217
218#[doc(hidden)]
219pub mod __internal;
220#[doc(hidden)]
221pub mod macros;
222
223/// Initialize and pin a type directly on the stack.
224///
225/// # Examples
226///
227/// ```rust
228/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
229/// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
230/// # use macros::pin_data;
231/// # use core::pin::Pin;
232/// #[pin_data]
233/// struct Foo {
234/// #[pin]
235/// a: Mutex<usize>,
236/// b: Bar,
237/// }
238///
239/// #[pin_data]
240/// struct Bar {
241/// x: u32,
242/// }
243///
244/// stack_pin_init!(let foo = pin_init!(Foo {
245/// a <- new_mutex!(42),
246/// b: Bar {
247/// x: 64,
248/// },
249/// }));
250/// let foo: Pin<&mut Foo> = foo;
251/// pr_info!("a: {}", &*foo.a.lock());
252/// ```
253///
254/// # Syntax
255///
256/// A normal `let` binding with optional type annotation. The expression is expected to implement
257/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
258/// type, then use [`stack_try_pin_init!`].
259///
260/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
261#[macro_export]
262macro_rules! stack_pin_init {
263 (let $var:ident $(: $t:ty)? = $val:expr) => {
264 let val = $val;
265 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
266 let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
267 Ok(res) => res,
268 Err(x) => {
269 let x: ::core::convert::Infallible = x;
270 match x {}
271 }
272 };
273 };
274}
275
276/// Initialize and pin a type directly on the stack.
277///
278/// # Examples
279///
280/// ```rust
281/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
282/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
283/// # use macros::pin_data;
284/// # use core::{alloc::AllocError, pin::Pin};
285/// #[pin_data]
286/// struct Foo {
287/// #[pin]
288/// a: Mutex<usize>,
289/// b: Box<Bar>,
290/// }
291///
292/// struct Bar {
293/// x: u32,
294/// }
295///
296/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
297/// a <- new_mutex!(42),
298/// b: Box::try_new(Bar {
299/// x: 64,
300/// })?,
301/// }));
302/// let foo = foo.unwrap();
303/// pr_info!("a: {}", &*foo.a.lock());
304/// ```
305///
306/// ```rust
307/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
308/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
309/// # use macros::pin_data;
310/// # use core::{alloc::AllocError, pin::Pin};
311/// #[pin_data]
312/// struct Foo {
313/// #[pin]
314/// a: Mutex<usize>,
315/// b: Box<Bar>,
316/// }
317///
318/// struct Bar {
319/// x: u32,
320/// }
321///
322/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
323/// a <- new_mutex!(42),
324/// b: Box::try_new(Bar {
325/// x: 64,
326/// })?,
327/// }));
328/// pr_info!("a: {}", &*foo.a.lock());
329/// # Ok::<_, AllocError>(())
330/// ```
331///
332/// # Syntax
333///
334/// A normal `let` binding with optional type annotation. The expression is expected to implement
335/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
336/// `=` will propagate this error.
337#[macro_export]
338macro_rules! stack_try_pin_init {
339 (let $var:ident $(: $t:ty)? = $val:expr) => {
340 let val = $val;
341 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
342 let mut $var = $crate::init::__internal::StackInit::init($var, val);
343 };
344 (let $var:ident $(: $t:ty)? =? $val:expr) => {
345 let val = $val;
346 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
347 let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
348 };
349}
350
351/// Construct an in-place, pinned initializer for `struct`s.
352///
353/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
354/// [`try_pin_init!`].
355///
356/// The syntax is almost identical to that of a normal `struct` initializer:
357///
358/// ```rust
359/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
360/// # use kernel::{init, pin_init, macros::pin_data, init::*};
361/// # use core::pin::Pin;
362/// #[pin_data]
363/// struct Foo {
364/// a: usize,
365/// b: Bar,
366/// }
367///
368/// #[pin_data]
369/// struct Bar {
370/// x: u32,
371/// }
372///
373/// # fn demo() -> impl PinInit<Foo> {
374/// let a = 42;
375///
376/// let initializer = pin_init!(Foo {
377/// a,
378/// b: Bar {
379/// x: 64,
380/// },
381/// });
382/// # initializer }
383/// # Box::pin_init(demo()).unwrap();
384/// ```
385///
386/// Arbitrary Rust expressions can be used to set the value of a variable.
387///
388/// The fields are initialized in the order that they appear in the initializer. So it is possible
389/// to read already initialized fields using raw pointers.
390///
391/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
392/// initializer.
393///
394/// # Init-functions
395///
396/// When working with this API it is often desired to let others construct your types without
397/// giving access to all fields. This is where you would normally write a plain function `new`
398/// that would return a new instance of your type. With this API that is also possible.
399/// However, there are a few extra things to keep in mind.
400///
401/// To create an initializer function, simply declare it like this:
402///
403/// ```rust
404/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
405/// # use kernel::{init, pin_init, prelude::*, init::*};
406/// # use core::pin::Pin;
407/// # #[pin_data]
408/// # struct Foo {
409/// # a: usize,
410/// # b: Bar,
411/// # }
412/// # #[pin_data]
413/// # struct Bar {
414/// # x: u32,
415/// # }
416/// impl Foo {
417/// fn new() -> impl PinInit<Self> {
418/// pin_init!(Self {
419/// a: 42,
420/// b: Bar {
421/// x: 64,
422/// },
423/// })
424/// }
425/// }
426/// ```
427///
428/// Users of `Foo` can now create it like this:
429///
430/// ```rust
431/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
432/// # use kernel::{init, pin_init, macros::pin_data, init::*};
433/// # use core::pin::Pin;
434/// # #[pin_data]
435/// # struct Foo {
436/// # a: usize,
437/// # b: Bar,
438/// # }
439/// # #[pin_data]
440/// # struct Bar {
441/// # x: u32,
442/// # }
443/// # impl Foo {
444/// # fn new() -> impl PinInit<Self> {
445/// # pin_init!(Self {
446/// # a: 42,
447/// # b: Bar {
448/// # x: 64,
449/// # },
450/// # })
451/// # }
452/// # }
453/// let foo = Box::pin_init(Foo::new());
454/// ```
455///
456/// They can also easily embed it into their own `struct`s:
457///
458/// ```rust
459/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
460/// # use kernel::{init, pin_init, macros::pin_data, init::*};
461/// # use core::pin::Pin;
462/// # #[pin_data]
463/// # struct Foo {
464/// # a: usize,
465/// # b: Bar,
466/// # }
467/// # #[pin_data]
468/// # struct Bar {
469/// # x: u32,
470/// # }
471/// # impl Foo {
472/// # fn new() -> impl PinInit<Self> {
473/// # pin_init!(Self {
474/// # a: 42,
475/// # b: Bar {
476/// # x: 64,
477/// # },
478/// # })
479/// # }
480/// # }
481/// #[pin_data]
482/// struct FooContainer {
483/// #[pin]
484/// foo1: Foo,
485/// #[pin]
486/// foo2: Foo,
487/// other: u32,
488/// }
489///
490/// impl FooContainer {
491/// fn new(other: u32) -> impl PinInit<Self> {
492/// pin_init!(Self {
493/// foo1 <- Foo::new(),
494/// foo2 <- Foo::new(),
495/// other,
496/// })
497/// }
498/// }
499/// ```
500///
501/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
502/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
503/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
504///
505/// # Syntax
506///
507/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
508/// the following modifications is expected:
509/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
510/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
511/// pointer named `this` inside of the initializer.
512///
513/// For instance:
514///
515/// ```rust
516/// # use kernel::pin_init;
517/// # use macros::pin_data;
518/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
519/// #[pin_data]
520/// struct Buf {
521/// // `ptr` points into `buf`.
522/// ptr: *mut u8,
523/// buf: [u8; 64],
524/// #[pin]
525/// pin: PhantomPinned,
526/// }
527/// pin_init!(&this in Buf {
528/// buf: [0; 64],
529/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
530/// pin: PhantomPinned,
531/// });
532/// ```
533///
534/// [`try_pin_init!`]: kernel::try_pin_init
535/// [`NonNull<Self>`]: core::ptr::NonNull
536// For a detailed example of how this macro works, see the module documentation of the hidden
537// module `__internal` inside of `init/__internal.rs`.
538#[macro_export]
539macro_rules! pin_init {
540 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
541 $($fields:tt)*
542 }) => {
543 $crate::try_pin_init!(
544 @this($($this)?),
545 @typ($t $(::<$($generics),*>)?),
546 @fields($($fields)*),
547 @error(::core::convert::Infallible),
548 )
549 };
550}
551
552/// Construct an in-place, fallible pinned initializer for `struct`s.
553///
554/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
555///
556/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
557/// initialization and return the error.
558///
559/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
560/// initialization fails, the memory can be safely deallocated without any further modifications.
561///
562/// This macro defaults the error to [`Error`].
563///
564/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
565/// after the `struct` initializer to specify the error type you want to use.
566///
567/// # Examples
568///
569/// ```rust
570/// # #![feature(new_uninit)]
571/// use kernel::{init::{self, PinInit}, error::Error};
572/// #[pin_data]
573/// struct BigBuf {
574/// big: Box<[u8; 1024 * 1024 * 1024]>,
575/// small: [u8; 1024 * 1024],
576/// ptr: *mut u8,
577/// }
578///
579/// impl BigBuf {
580/// fn new() -> impl PinInit<Self, Error> {
581/// try_pin_init!(Self {
582/// big: Box::init(init::zeroed())?,
583/// small: [0; 1024 * 1024],
584/// ptr: core::ptr::null_mut(),
585/// }? Error)
586/// }
587/// }
588/// ```
589// For a detailed example of how this macro works, see the module documentation of the hidden
590// module `__internal` inside of `init/__internal.rs`.
591#[macro_export]
592macro_rules! try_pin_init {
593 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
594 $($fields:tt)*
595 }) => {
596 $crate::try_pin_init!(
597 @this($($this)?),
598 @typ($t $(::<$($generics),*>)? ),
599 @fields($($fields)*),
600 @error($crate::error::Error),
601 )
602 };
603 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
604 $($fields:tt)*
605 }? $err:ty) => {
606 $crate::try_pin_init!(
607 @this($($this)?),
608 @typ($t $(::<$($generics),*>)? ),
609 @fields($($fields)*),
610 @error($err),
611 )
612 };
613 (
614 @this($($this:ident)?),
615 @typ($t:ident $(::<$($generics:ty),*>)?),
616 @fields($($fields:tt)*),
617 @error($err:ty),
618 ) => {{
619 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
620 // type and shadow it later when we insert the arbitrary user code. That way there will be
621 // no possibility of returning without `unsafe`.
622 struct __InitOk;
623 // Get the pin data from the supplied type.
624 let data = unsafe {
625 use $crate::init::__internal::HasPinData;
626 $t$(::<$($generics),*>)?::__pin_data()
627 };
628 // Ensure that `data` really is of type `PinData` and help with type inference:
629 let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>(
630 data,
631 move |slot| {
632 {
633 // Shadow the structure so it cannot be used to return early.
634 struct __InitOk;
635 // Create the `this` so it can be referenced by the user inside of the
636 // expressions creating the individual fields.
637 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
638 // Initialize every field.
639 $crate::try_pin_init!(init_slot:
640 @data(data),
641 @slot(slot),
642 @munch_fields($($fields)*,),
643 );
644 // We use unreachable code to ensure that all fields have been mentioned exactly
645 // once, this struct initializer will still be type-checked and complain with a
646 // very natural error message if a field is forgotten/mentioned more than once.
647 #[allow(unreachable_code, clippy::diverging_sub_expression)]
648 if false {
649 $crate::try_pin_init!(make_initializer:
650 @slot(slot),
651 @type_name($t),
652 @munch_fields($($fields)*,),
653 @acc(),
654 );
655 }
656 // Forget all guards, since initialization was a success.
657 $crate::try_pin_init!(forget_guards:
658 @munch_fields($($fields)*,),
659 );
660 }
661 Ok(__InitOk)
662 }
663 );
664 let init = move |slot| -> ::core::result::Result<(), $err> {
665 init(slot).map(|__InitOk| ())
666 };
667 let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) };
668 init
669 }};
670 (init_slot:
671 @data($data:ident),
672 @slot($slot:ident),
673 @munch_fields($(,)?),
674 ) => {
675 // Endpoint of munching, no fields are left.
676 };
677 (init_slot:
678 @data($data:ident),
679 @slot($slot:ident),
680 // In-place initialization syntax.
681 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
682 ) => {
683 let $field = $val;
684 // Call the initializer.
685 //
686 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
687 // return when an error/panic occurs.
688 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
689 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? };
690 // Create the drop guard.
691 //
692 // We only give access to `&DropGuard`, so it cannot be forgotten via safe code.
693 //
694 // SAFETY: We forget the guard later when initialization has succeeded.
695 let $field = &unsafe {
696 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
697 };
698
699 $crate::try_pin_init!(init_slot:
700 @data($data),
701 @slot($slot),
702 @munch_fields($($rest)*),
703 );
704 };
705 (init_slot:
706 @data($data:ident),
707 @slot($slot:ident),
708 // Direct value init, this is safe for every field.
709 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
710 ) => {
711 $(let $field = $val;)?
712 // Initialize the field.
713 //
714 // SAFETY: The memory at `slot` is uninitialized.
715 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
716 // Create the drop guard:
717 //
718 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
719 //
720 // SAFETY: We forget the guard later when initialization has succeeded.
721 let $field = &unsafe {
722 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
723 };
724
725 $crate::try_pin_init!(init_slot:
726 @data($data),
727 @slot($slot),
728 @munch_fields($($rest)*),
729 );
730 };
731 (make_initializer:
732 @slot($slot:ident),
733 @type_name($t:ident),
734 @munch_fields($(,)?),
735 @acc($($acc:tt)*),
736 ) => {
737 // Endpoint, nothing more to munch, create the initializer.
738 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
739 // get the correct type inference here:
740 unsafe {
741 ::core::ptr::write($slot, $t {
742 $($acc)*
743 });
744 }
745 };
746 (make_initializer:
747 @slot($slot:ident),
748 @type_name($t:ident),
749 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
750 @acc($($acc:tt)*),
751 ) => {
752 $crate::try_pin_init!(make_initializer:
753 @slot($slot),
754 @type_name($t),
755 @munch_fields($($rest)*),
756 @acc($($acc)* $field: ::core::panic!(),),
757 );
758 };
759 (make_initializer:
760 @slot($slot:ident),
761 @type_name($t:ident),
762 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
763 @acc($($acc:tt)*),
764 ) => {
765 $crate::try_pin_init!(make_initializer:
766 @slot($slot),
767 @type_name($t),
768 @munch_fields($($rest)*),
769 @acc($($acc)* $field: ::core::panic!(),),
770 );
771 };
772 (forget_guards:
773 @munch_fields($(,)?),
774 ) => {
775 // Munching finished.
776 };
777 (forget_guards:
778 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
779 ) => {
780 unsafe { $crate::init::__internal::DropGuard::forget($field) };
781
782 $crate::try_pin_init!(forget_guards:
783 @munch_fields($($rest)*),
784 );
785 };
786 (forget_guards:
787 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
788 ) => {
789 unsafe { $crate::init::__internal::DropGuard::forget($field) };
790
791 $crate::try_pin_init!(forget_guards:
792 @munch_fields($($rest)*),
793 );
794 };
795}
796
797/// Construct an in-place initializer for `struct`s.
798///
799/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
800/// [`try_init!`].
801///
802/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
803/// - `unsafe` code must guarantee either full initialization or return an error and allow
804/// deallocation of the memory.
805/// - the fields are initialized in the order given in the initializer.
806/// - no references to fields are allowed to be created inside of the initializer.
807///
808/// This initializer is for initializing data in-place that might later be moved. If you want to
809/// pin-initialize, use [`pin_init!`].
810///
811/// [`try_init!`]: crate::try_init!
812// For a detailed example of how this macro works, see the module documentation of the hidden
813// module `__internal` inside of `init/__internal.rs`.
814#[macro_export]
815macro_rules! init {
816 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
817 $($fields:tt)*
818 }) => {
819 $crate::try_init!(
820 @this($($this)?),
821 @typ($t $(::<$($generics),*>)?),
822 @fields($($fields)*),
823 @error(::core::convert::Infallible),
824 )
825 }
826}
827
828/// Construct an in-place fallible initializer for `struct`s.
829///
830/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
831/// [`init!`].
832///
833/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
834/// append `? $type` after the `struct` initializer.
835/// The safety caveats from [`try_pin_init!`] also apply:
836/// - `unsafe` code must guarantee either full initialization or return an error and allow
837/// deallocation of the memory.
838/// - the fields are initialized in the order given in the initializer.
839/// - no references to fields are allowed to be created inside of the initializer.
840///
841/// # Examples
842///
843/// ```rust
844/// use kernel::{init::PinInit, error::Error, InPlaceInit};
845/// struct BigBuf {
846/// big: Box<[u8; 1024 * 1024 * 1024]>,
847/// small: [u8; 1024 * 1024],
848/// }
849///
850/// impl BigBuf {
851/// fn new() -> impl Init<Self, Error> {
852/// try_init!(Self {
853/// big: Box::init(zeroed())?,
854/// small: [0; 1024 * 1024],
855/// }? Error)
856/// }
857/// }
858/// ```
859// For a detailed example of how this macro works, see the module documentation of the hidden
860// module `__internal` inside of `init/__internal.rs`.
861#[macro_export]
862macro_rules! try_init {
863 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
864 $($fields:tt)*
865 }) => {
866 $crate::try_init!(
867 @this($($this)?),
868 @typ($t $(::<$($generics),*>)?),
869 @fields($($fields)*),
870 @error($crate::error::Error),
871 )
872 };
873 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
874 $($fields:tt)*
875 }? $err:ty) => {
876 $crate::try_init!(
877 @this($($this)?),
878 @typ($t $(::<$($generics),*>)?),
879 @fields($($fields)*),
880 @error($err),
881 )
882 };
883 (
884 @this($($this:ident)?),
885 @typ($t:ident $(::<$($generics:ty),*>)?),
886 @fields($($fields:tt)*),
887 @error($err:ty),
888 ) => {{
889 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
890 // type and shadow it later when we insert the arbitrary user code. That way there will be
891 // no possibility of returning without `unsafe`.
892 struct __InitOk;
893 // Get the init data from the supplied type.
894 let data = unsafe {
895 use $crate::init::__internal::HasInitData;
896 $t$(::<$($generics),*>)?::__init_data()
897 };
898 // Ensure that `data` really is of type `InitData` and help with type inference:
899 let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>(
900 data,
901 move |slot| {
902 {
903 // Shadow the structure so it cannot be used to return early.
904 struct __InitOk;
905 // Create the `this` so it can be referenced by the user inside of the
906 // expressions creating the individual fields.
907 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
908 // Initialize every field.
909 $crate::try_init!(init_slot:
910 @slot(slot),
911 @munch_fields($($fields)*,),
912 );
913 // We use unreachable code to ensure that all fields have been mentioned exactly
914 // once, this struct initializer will still be type-checked and complain with a
915 // very natural error message if a field is forgotten/mentioned more than once.
916 #[allow(unreachable_code, clippy::diverging_sub_expression)]
917 if false {
918 $crate::try_init!(make_initializer:
919 @slot(slot),
920 @type_name($t),
921 @munch_fields($($fields)*,),
922 @acc(),
923 );
924 }
925 // Forget all guards, since initialization was a success.
926 $crate::try_init!(forget_guards:
927 @munch_fields($($fields)*,),
928 );
929 }
930 Ok(__InitOk)
931 }
932 );
933 let init = move |slot| -> ::core::result::Result<(), $err> {
934 init(slot).map(|__InitOk| ())
935 };
936 let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) };
937 init
938 }};
939 (init_slot:
940 @slot($slot:ident),
941 @munch_fields( $(,)?),
942 ) => {
943 // Endpoint of munching, no fields are left.
944 };
945 (init_slot:
946 @slot($slot:ident),
947 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
948 ) => {
949 let $field = $val;
950 // Call the initializer.
951 //
952 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
953 // return when an error/panic occurs.
954 unsafe {
955 $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?;
956 }
957 // Create the drop guard.
958 //
959 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
960 //
961 // SAFETY: We forget the guard later when initialization has succeeded.
962 let $field = &unsafe {
963 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
964 };
965
966 $crate::try_init!(init_slot:
967 @slot($slot),
968 @munch_fields($($rest)*),
969 );
970 };
971 (init_slot:
972 @slot($slot:ident),
973 // Direct value init.
974 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
975 ) => {
976 $(let $field = $val;)?
977 // Call the initializer.
978 //
979 // SAFETY: The memory at `slot` is uninitialized.
980 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
981 // Create the drop guard.
982 //
983 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
984 //
985 // SAFETY: We forget the guard later when initialization has succeeded.
986 let $field = &unsafe {
987 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
988 };
989
990 $crate::try_init!(init_slot:
991 @slot($slot),
992 @munch_fields($($rest)*),
993 );
994 };
995 (make_initializer:
996 @slot($slot:ident),
997 @type_name($t:ident),
998 @munch_fields( $(,)?),
999 @acc($($acc:tt)*),
1000 ) => {
1001 // Endpoint, nothing more to munch, create the initializer.
1002 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
1003 // get the correct type inference here:
1004 unsafe {
1005 ::core::ptr::write($slot, $t {
1006 $($acc)*
1007 });
1008 }
1009 };
1010 (make_initializer:
1011 @slot($slot:ident),
1012 @type_name($t:ident),
1013 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1014 @acc($($acc:tt)*),
1015 ) => {
1016 $crate::try_init!(make_initializer:
1017 @slot($slot),
1018 @type_name($t),
1019 @munch_fields($($rest)*),
1020 @acc($($acc)*$field: ::core::panic!(),),
1021 );
1022 };
1023 (make_initializer:
1024 @slot($slot:ident),
1025 @type_name($t:ident),
1026 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1027 @acc($($acc:tt)*),
1028 ) => {
1029 $crate::try_init!(make_initializer:
1030 @slot($slot),
1031 @type_name($t),
1032 @munch_fields($($rest)*),
1033 @acc($($acc)*$field: ::core::panic!(),),
1034 );
1035 };
1036 (forget_guards:
1037 @munch_fields($(,)?),
1038 ) => {
1039 // Munching finished.
1040 };
1041 (forget_guards:
1042 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1043 ) => {
1044 unsafe { $crate::init::__internal::DropGuard::forget($field) };
1045
1046 $crate::try_init!(forget_guards:
1047 @munch_fields($($rest)*),
1048 );
1049 };
1050 (forget_guards:
1051 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1052 ) => {
1053 unsafe { $crate::init::__internal::DropGuard::forget($field) };
1054
1055 $crate::try_init!(forget_guards:
1056 @munch_fields($($rest)*),
1057 );
1058 };
1059}
1060
1061/// A pin-initializer for the type `T`.
1062///
1063/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
1064/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
1065/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
1066///
1067/// Also see the [module description](self).
1068///
1069/// # Safety
1070///
1071/// When implementing this type you will need to take great care. Also there are probably very few
1072/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
1073///
1074/// The [`PinInit::__pinned_init`] function
1075/// - returns `Ok(())` if it initialized every field of `slot`,
1076/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1077/// - `slot` can be deallocated without UB occurring,
1078/// - `slot` does not need to be dropped,
1079/// - `slot` is not partially initialized.
1080/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1081///
1082/// [`Arc<T>`]: crate::sync::Arc
1083/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
1084#[must_use = "An initializer must be used in order to create its value."]
1085pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
1086 /// Initializes `slot`.
1087 ///
1088 /// # Safety
1089 ///
1090 /// - `slot` is a valid pointer to uninitialized memory.
1091 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
1092 /// deallocate.
1093 /// - `slot` will not move until it is dropped, i.e. it will be pinned.
1094 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
1095}
1096
1097/// An initializer for `T`.
1098///
1099/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
1100/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
1101/// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
1102/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
1103///
1104/// Also see the [module description](self).
1105///
1106/// # Safety
1107///
1108/// When implementing this type you will need to take great care. Also there are probably very few
1109/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
1110///
1111/// The [`Init::__init`] function
1112/// - returns `Ok(())` if it initialized every field of `slot`,
1113/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1114/// - `slot` can be deallocated without UB occurring,
1115/// - `slot` does not need to be dropped,
1116/// - `slot` is not partially initialized.
1117/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1118///
1119/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
1120/// code as `__init`.
1121///
1122/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
1123/// move the pointee after initialization.
1124///
1125/// [`Arc<T>`]: crate::sync::Arc
1126#[must_use = "An initializer must be used in order to create its value."]
1127pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized {
1128 /// Initializes `slot`.
1129 ///
1130 /// # Safety
1131 ///
1132 /// - `slot` is a valid pointer to uninitialized memory.
1133 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
1134 /// deallocate.
1135 unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
1136}
1137
1138// SAFETY: Every in-place initializer can also be used as a pin-initializer.
1139unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I
1140where
1141 I: Init<T, E>,
1142{
1143 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1144 // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not
1145 // require `slot` to not move after init.
1146 unsafe { self.__init(slot) }
1147 }
1148}
1149
1150/// Creates a new [`PinInit<T, E>`] from the given closure.
1151///
1152/// # Safety
1153///
1154/// The closure:
1155/// - returns `Ok(())` if it initialized every field of `slot`,
1156/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1157/// - `slot` can be deallocated without UB occurring,
1158/// - `slot` does not need to be dropped,
1159/// - `slot` is not partially initialized.
1160/// - may assume that the `slot` does not move if `T: !Unpin`,
1161/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1162#[inline]
1163pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1164 f: impl FnOnce(*mut T) -> Result<(), E>,
1165) -> impl PinInit<T, E> {
1166 __internal::InitClosure(f, PhantomData)
1167}
1168
1169/// Creates a new [`Init<T, E>`] from the given closure.
1170///
1171/// # Safety
1172///
1173/// The closure:
1174/// - returns `Ok(())` if it initialized every field of `slot`,
1175/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1176/// - `slot` can be deallocated without UB occurring,
1177/// - `slot` does not need to be dropped,
1178/// - `slot` is not partially initialized.
1179/// - the `slot` may move after initialization.
1180/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1181#[inline]
1182pub const unsafe fn init_from_closure<T: ?Sized, E>(
1183 f: impl FnOnce(*mut T) -> Result<(), E>,
1184) -> impl Init<T, E> {
1185 __internal::InitClosure(f, PhantomData)
1186}
1187
1188/// An initializer that leaves the memory uninitialized.
1189///
1190/// The initializer is a no-op. The `slot` memory is not changed.
1191#[inline]
1192pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1193 // SAFETY: The memory is allowed to be uninitialized.
1194 unsafe { init_from_closure(|_| Ok(())) }
1195}
1196
1197// SAFETY: Every type can be initialized by-value.
1198unsafe impl<T, E> Init<T, E> for T {
1199 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1200 unsafe { slot.write(self) };
1201 Ok(())
1202 }
1203}
1204
1205/// Smart pointer that can initialize memory in-place.
1206pub trait InPlaceInit<T>: Sized {
1207 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1208 /// type.
1209 ///
1210 /// If `T: !Unpin` it will not be able to move afterwards.
1211 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1212 where
1213 E: From<AllocError>;
1214
1215 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1216 /// type.
1217 ///
1218 /// If `T: !Unpin` it will not be able to move afterwards.
1219 fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>>
1220 where
1221 Error: From<E>,
1222 {
1223 // SAFETY: We delegate to `init` and only change the error type.
1224 let init = unsafe {
1225 pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1226 };
1227 Self::try_pin_init(init)
1228 }
1229
1230 /// Use the given initializer to in-place initialize a `T`.
1231 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1232 where
1233 E: From<AllocError>;
1234
1235 /// Use the given initializer to in-place initialize a `T`.
1236 fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
1237 where
1238 Error: From<E>,
1239 {
1240 // SAFETY: We delegate to `init` and only change the error type.
1241 let init = unsafe {
1242 init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1243 };
1244 Self::try_init(init)
1245 }
1246}
1247
1248impl<T> InPlaceInit<T> for Box<T> {
1249 #[inline]
1250 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1251 where
1252 E: From<AllocError>,
1253 {
1254 let mut this = Box::try_new_uninit()?;
1255 let slot = this.as_mut_ptr();
1256 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1257 // slot is valid and will not be moved, because we pin it later.
1258 unsafe { init.__pinned_init(slot)? };
1259 // SAFETY: All fields have been initialized.
1260 Ok(unsafe { this.assume_init() }.into())
1261 }
1262
1263 #[inline]
1264 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1265 where
1266 E: From<AllocError>,
1267 {
1268 let mut this = Box::try_new_uninit()?;
1269 let slot = this.as_mut_ptr();
1270 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1271 // slot is valid.
1272 unsafe { init.__init(slot)? };
1273 // SAFETY: All fields have been initialized.
1274 Ok(unsafe { this.assume_init() })
1275 }
1276}
1277
1278impl<T> InPlaceInit<T> for UniqueArc<T> {
1279 #[inline]
1280 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1281 where
1282 E: From<AllocError>,
1283 {
1284 let mut this = UniqueArc::try_new_uninit()?;
1285 let slot = this.as_mut_ptr();
1286 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1287 // slot is valid and will not be moved, because we pin it later.
1288 unsafe { init.__pinned_init(slot)? };
1289 // SAFETY: All fields have been initialized.
1290 Ok(unsafe { this.assume_init() }.into())
1291 }
1292
1293 #[inline]
1294 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1295 where
1296 E: From<AllocError>,
1297 {
1298 let mut this = UniqueArc::try_new_uninit()?;
1299 let slot = this.as_mut_ptr();
1300 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1301 // slot is valid.
1302 unsafe { init.__init(slot)? };
1303 // SAFETY: All fields have been initialized.
1304 Ok(unsafe { this.assume_init() })
1305 }
1306}
1307
1308/// Trait facilitating pinned destruction.
1309///
1310/// Use [`pinned_drop`] to implement this trait safely:
1311///
1312/// ```rust
1313/// # use kernel::sync::Mutex;
1314/// use kernel::macros::pinned_drop;
1315/// use core::pin::Pin;
1316/// #[pin_data(PinnedDrop)]
1317/// struct Foo {
1318/// #[pin]
1319/// mtx: Mutex<usize>,
1320/// }
1321///
1322/// #[pinned_drop]
1323/// impl PinnedDrop for Foo {
1324/// fn drop(self: Pin<&mut Self>) {
1325/// pr_info!("Foo is being dropped!");
1326/// }
1327/// }
1328/// ```
1329///
1330/// # Safety
1331///
1332/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1333///
1334/// [`pinned_drop`]: kernel::macros::pinned_drop
1335pub unsafe trait PinnedDrop: __internal::HasPinData {
1336 /// Executes the pinned destructor of this type.
1337 ///
1338 /// While this function is marked safe, it is actually unsafe to call it manually. For this
1339 /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1340 /// and thus prevents this function from being called where it should not.
1341 ///
1342 /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1343 /// automatically.
1344 fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1345}
1346
1347/// Marker trait for types that can be initialized by writing just zeroes.
1348///
1349/// # Safety
1350///
1351/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1352/// this is not UB:
1353///
1354/// ```rust,ignore
1355/// let val: Self = unsafe { core::mem::zeroed() };
1356/// ```
1357pub unsafe trait Zeroable {}
1358
1359/// Create a new zeroed T.
1360///
1361/// The returned initializer will write `0x00` to every byte of the given `slot`.
1362#[inline]
1363pub fn zeroed<T: Zeroable>() -> impl Init<T> {
1364 // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1365 // and because we write all zeroes, the memory is initialized.
1366 unsafe {
1367 init_from_closure(|slot: *mut T| {
1368 slot.write_bytes(0, 1);
1369 Ok(())
1370 })
1371 }
1372}
1373
1374macro_rules! impl_zeroable {
1375 ($($({$($generics:tt)*})? $t:ty, )*) => {
1376 $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1377 };
1378}
1379
1380impl_zeroable! {
1381 // SAFETY: All primitives that are allowed to be zero.
1382 bool,
1383 char,
1384 u8, u16, u32, u64, u128, usize,
1385 i8, i16, i32, i64, i128, isize,
1386 f32, f64,
1387
1388 // SAFETY: These are ZSTs, there is nothing to zero.
1389 {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (),
1390
1391 // SAFETY: Type is allowed to take any value, including all zeros.
1392 {<T>} MaybeUninit<T>,
1393
1394 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1395 Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1396 Option<NonZeroU128>, Option<NonZeroUsize>,
1397 Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1398 Option<NonZeroI128>, Option<NonZeroIsize>,
1399
1400 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1401 //
1402 // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
1403 {<T: ?Sized>} Option<NonNull<T>>,
1404 {<T: ?Sized>} Option<Box<T>>,
1405
1406 // SAFETY: `null` pointer is valid.
1407 //
1408 // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1409 // null.
1410 //
1411 // When `Pointee` gets stabilized, we could use
1412 // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1413 {<T>} *mut T, {<T>} *const T,
1414
1415 // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1416 // zero.
1417 {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1418
1419 // SAFETY: `T` is `Zeroable`.
1420 {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1421}
1422
1423macro_rules! impl_tuple_zeroable {
1424 ($(,)?) => {};
1425 ($first:ident, $($t:ident),* $(,)?) => {
1426 // SAFETY: All elements are zeroable and padding can be zero.
1427 unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1428 impl_tuple_zeroable!($($t),* ,);
1429 }
1430}
1431
1432impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);