projects/linux·blob v6.4 · refs

1427 lines · 48.2 KiB

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4//!
5//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6//! overflow.
7//!
8//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10//!
11//! # Overview
12//!
13//! To initialize a `struct` with an in-place constructor you will need two things:
14//! - an in-place constructor,
15//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
16//! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]).
17//!
18//! To get an in-place constructor there are generally three options:
19//! - directly creating an in-place constructor using the [`pin_init!`] macro,
20//! - a custom function/macro returning an in-place constructor provided by someone else,
21//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
22//!
23//! Aside from pinned initialization, this API also supports in-place construction without pinning,
24//! the macros/types/functions are generally named like the pinned variants without the `pin`
25//! prefix.
26//!
27//! # Examples
28//!
29//! ## Using the [`pin_init!`] macro
30//!
31//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
32//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
33//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
34//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
35//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
36//!
37//! ```rust
38//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
39//! use kernel::{prelude::*, sync::Mutex, new_mutex};
40//! # use core::pin::Pin;
41//! #[pin_data]
42//! struct Foo {
43//! #[pin]
44//! a: Mutex<usize>,
45//! b: u32,
46//! }
47//!
48//! let foo = pin_init!(Foo {
49//! a <- new_mutex!(42, "Foo::a"),
50//! b: 24,
51//! });
52//! ```
53//!
54//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
55//! (or just the stack) to actually initialize a `Foo`:
56//!
57//! ```rust
58//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
59//! # use kernel::{prelude::*, sync::Mutex, new_mutex};
60//! # use core::pin::Pin;
61//! # #[pin_data]
62//! # struct Foo {
63//! # #[pin]
64//! # a: Mutex<usize>,
65//! # b: u32,
66//! # }
67//! # let foo = pin_init!(Foo {
68//! # a <- new_mutex!(42, "Foo::a"),
69//! # b: 24,
70//! # });
71//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo);
72//! ```
73//!
74//! For more information see the [`pin_init!`] macro.
75//!
76//! ## Using a custom function/macro that returns an initializer
77//!
78//! Many types from the kernel supply a function/macro that returns an initializer, because the
79//! above method only works for types where you can access the fields.
80//!
81//! ```rust
82//! # use kernel::{new_mutex, sync::{Arc, Mutex}};
83//! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx"));
84//! ```
85//!
86//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
87//!
88//! ```rust
89//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
90//! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init};
91//! #[pin_data]
92//! struct DriverData {
93//! #[pin]
94//! status: Mutex<i32>,
95//! buffer: Box<[u8; 1_000_000]>,
96//! }
97//!
98//! impl DriverData {
99//! fn new() -> impl PinInit<Self, Error> {
100//! try_pin_init!(Self {
101//! status <- new_mutex!(0, "DriverData::status"),
102//! buffer: Box::init(kernel::init::zeroed())?,
103//! })
104//! }
105//! }
106//! ```
107//!
108//! ## Manual creation of an initializer
109//!
110//! Often when working with primitives the previous approaches are not sufficient. That is where
111//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
112//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
113//! actually does the initialization in the correct way. Here are the things to look out for
114//! (we are calling the parameter to the closure `slot`):
115//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
116//! `slot` now contains a valid bit pattern for the type `T`,
117//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
118//! you need to take care to clean up anything if your initialization fails mid-way,
119//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
120//! `slot` gets called.
121//!
122//! ```rust
123//! use kernel::{prelude::*, init};
124//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
125//! # mod bindings {
126//! # pub struct foo;
127//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
128//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
129//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
130//! # }
131//! /// # Invariants
132//! ///
133//! /// `foo` is always initialized
134//! #[pin_data(PinnedDrop)]
135//! pub struct RawFoo {
136//! #[pin]
137//! foo: Opaque<bindings::foo>,
138//! #[pin]
139//! _p: PhantomPinned,
140//! }
141//!
142//! impl RawFoo {
143//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
144//! // SAFETY:
145//! // - when the closure returns `Ok(())`, then it has successfully initialized and
146//! // enabled `foo`,
147//! // - when it returns `Err(e)`, then it has cleaned up before
148//! unsafe {
149//! init::pin_init_from_closure(move |slot: *mut Self| {
150//! // `slot` contains uninit memory, avoid creating a reference.
151//! let foo = addr_of_mut!((*slot).foo);
152//!
153//! // Initialize the `foo`
154//! bindings::init_foo(Opaque::raw_get(foo));
155//!
156//! // Try to enable it.
157//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
158//! if err != 0 {
159//! // Enabling has failed, first clean up the foo and then return the error.
160//! bindings::destroy_foo(Opaque::raw_get(foo));
161//! return Err(Error::from_kernel_errno(err));
162//! }
163//!
164//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
165//! Ok(())
166//! })
167//! }
168//! }
169//! }
170//!
171//! #[pinned_drop]
172//! impl PinnedDrop for RawFoo {
173//! fn drop(self: Pin<&mut Self>) {
174//! // SAFETY: Since `foo` is initialized, destroying is safe.
175//! unsafe { bindings::destroy_foo(self.foo.get()) };
176//! }
177//! }
178//! ```
179//!
180//! For the special case where initializing a field is a single FFI-function call that cannot fail,
181//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
182//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
183//! with [`pin_init!`].
184//!
185//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
186//! the `kernel` crate. The [`sync`] module is a good starting point.
187//!
188//! [`sync`]: kernel::sync
189//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
190//! [structurally pinned fields]:
191//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
192//! [stack]: crate::stack_pin_init
193//! [`Arc<T>`]: crate::sync::Arc
194//! [`impl PinInit<Foo>`]: PinInit
195//! [`impl PinInit<T, E>`]: PinInit
196//! [`impl Init<T, E>`]: Init
197//! [`Opaque`]: kernel::types::Opaque
198//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
199//! [`pin_data`]: ::macros::pin_data
200
201use crate::{
202 error::{self, Error},
203 sync::UniqueArc,
204};
205use alloc::boxed::Box;
206use core::{
207 alloc::AllocError,
208 cell::Cell,
209 convert::Infallible,
210 marker::PhantomData,
211 mem::MaybeUninit,
212 num::*,
213 pin::Pin,
214 ptr::{self, NonNull},
215};
216
217#[doc(hidden)]
218pub mod __internal;
219#[doc(hidden)]
220pub mod macros;
221
222/// Initialize and pin a type directly on the stack.
223///
224/// # Examples
225///
226/// ```rust
227/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
228/// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
229/// # use macros::pin_data;
230/// # use core::pin::Pin;
231/// #[pin_data]
232/// struct Foo {
233/// #[pin]
234/// a: Mutex<usize>,
235/// b: Bar,
236/// }
237///
238/// #[pin_data]
239/// struct Bar {
240/// x: u32,
241/// }
242///
243/// stack_pin_init!(let foo = pin_init!(Foo {
244/// a <- new_mutex!(42),
245/// b: Bar {
246/// x: 64,
247/// },
248/// }));
249/// let foo: Pin<&mut Foo> = foo;
250/// pr_info!("a: {}", &*foo.a.lock());
251/// ```
252///
253/// # Syntax
254///
255/// A normal `let` binding with optional type annotation. The expression is expected to implement
256/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
257/// type, then use [`stack_try_pin_init!`].
258#[macro_export]
259macro_rules! stack_pin_init {
260 (let $var:ident $(: $t:ty)? = $val:expr) => {
261 let val = $val;
262 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
263 let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
264 Ok(res) => res,
265 Err(x) => {
266 let x: ::core::convert::Infallible = x;
267 match x {}
268 }
269 };
270 };
271}
272
273/// Initialize and pin a type directly on the stack.
274///
275/// # Examples
276///
277/// ```rust
278/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
279/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
280/// # use macros::pin_data;
281/// # use core::{alloc::AllocError, pin::Pin};
282/// #[pin_data]
283/// struct Foo {
284/// #[pin]
285/// a: Mutex<usize>,
286/// b: Box<Bar>,
287/// }
288///
289/// struct Bar {
290/// x: u32,
291/// }
292///
293/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
294/// a <- new_mutex!(42),
295/// b: Box::try_new(Bar {
296/// x: 64,
297/// })?,
298/// }));
299/// let foo = foo.unwrap();
300/// pr_info!("a: {}", &*foo.a.lock());
301/// ```
302///
303/// ```rust
304/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
305/// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
306/// # use macros::pin_data;
307/// # use core::{alloc::AllocError, pin::Pin};
308/// #[pin_data]
309/// struct Foo {
310/// #[pin]
311/// a: Mutex<usize>,
312/// b: Box<Bar>,
313/// }
314///
315/// struct Bar {
316/// x: u32,
317/// }
318///
319/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
320/// a <- new_mutex!(42),
321/// b: Box::try_new(Bar {
322/// x: 64,
323/// })?,
324/// }));
325/// pr_info!("a: {}", &*foo.a.lock());
326/// # Ok::<_, AllocError>(())
327/// ```
328///
329/// # Syntax
330///
331/// A normal `let` binding with optional type annotation. The expression is expected to implement
332/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
333/// `=` will propagate this error.
334#[macro_export]
335macro_rules! stack_try_pin_init {
336 (let $var:ident $(: $t:ty)? = $val:expr) => {
337 let val = $val;
338 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
339 let mut $var = $crate::init::__internal::StackInit::init($var, val);
340 };
341 (let $var:ident $(: $t:ty)? =? $val:expr) => {
342 let val = $val;
343 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
344 let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
345 };
346}
347
348/// Construct an in-place, pinned initializer for `struct`s.
349///
350/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
351/// [`try_pin_init!`].
352///
353/// The syntax is almost identical to that of a normal `struct` initializer:
354///
355/// ```rust
356/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
357/// # use kernel::{init, pin_init, macros::pin_data, init::*};
358/// # use core::pin::Pin;
359/// #[pin_data]
360/// struct Foo {
361/// a: usize,
362/// b: Bar,
363/// }
364///
365/// #[pin_data]
366/// struct Bar {
367/// x: u32,
368/// }
369///
370/// # fn demo() -> impl PinInit<Foo> {
371/// let a = 42;
372///
373/// let initializer = pin_init!(Foo {
374/// a,
375/// b: Bar {
376/// x: 64,
377/// },
378/// });
379/// # initializer }
380/// # Box::pin_init(demo()).unwrap();
381/// ```
382///
383/// Arbitrary Rust expressions can be used to set the value of a variable.
384///
385/// The fields are initialized in the order that they appear in the initializer. So it is possible
386/// to read already initialized fields using raw pointers.
387///
388/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
389/// initializer.
390///
391/// # Init-functions
392///
393/// When working with this API it is often desired to let others construct your types without
394/// giving access to all fields. This is where you would normally write a plain function `new`
395/// that would return a new instance of your type. With this API that is also possible.
396/// However, there are a few extra things to keep in mind.
397///
398/// To create an initializer function, simply declare it like this:
399///
400/// ```rust
401/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
402/// # use kernel::{init, pin_init, prelude::*, init::*};
403/// # use core::pin::Pin;
404/// # #[pin_data]
405/// # struct Foo {
406/// # a: usize,
407/// # b: Bar,
408/// # }
409/// # #[pin_data]
410/// # struct Bar {
411/// # x: u32,
412/// # }
413/// impl Foo {
414/// fn new() -> impl PinInit<Self> {
415/// pin_init!(Self {
416/// a: 42,
417/// b: Bar {
418/// x: 64,
419/// },
420/// })
421/// }
422/// }
423/// ```
424///
425/// Users of `Foo` can now create it like this:
426///
427/// ```rust
428/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
429/// # use kernel::{init, pin_init, macros::pin_data, init::*};
430/// # use core::pin::Pin;
431/// # #[pin_data]
432/// # struct Foo {
433/// # a: usize,
434/// # b: Bar,
435/// # }
436/// # #[pin_data]
437/// # struct Bar {
438/// # x: u32,
439/// # }
440/// # impl Foo {
441/// # fn new() -> impl PinInit<Self> {
442/// # pin_init!(Self {
443/// # a: 42,
444/// # b: Bar {
445/// # x: 64,
446/// # },
447/// # })
448/// # }
449/// # }
450/// let foo = Box::pin_init(Foo::new());
451/// ```
452///
453/// They can also easily embed it into their own `struct`s:
454///
455/// ```rust
456/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
457/// # use kernel::{init, pin_init, macros::pin_data, init::*};
458/// # use core::pin::Pin;
459/// # #[pin_data]
460/// # struct Foo {
461/// # a: usize,
462/// # b: Bar,
463/// # }
464/// # #[pin_data]
465/// # struct Bar {
466/// # x: u32,
467/// # }
468/// # impl Foo {
469/// # fn new() -> impl PinInit<Self> {
470/// # pin_init!(Self {
471/// # a: 42,
472/// # b: Bar {
473/// # x: 64,
474/// # },
475/// # })
476/// # }
477/// # }
478/// #[pin_data]
479/// struct FooContainer {
480/// #[pin]
481/// foo1: Foo,
482/// #[pin]
483/// foo2: Foo,
484/// other: u32,
485/// }
486///
487/// impl FooContainer {
488/// fn new(other: u32) -> impl PinInit<Self> {
489/// pin_init!(Self {
490/// foo1 <- Foo::new(),
491/// foo2 <- Foo::new(),
492/// other,
493/// })
494/// }
495/// }
496/// ```
497///
498/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
499/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
500/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
501///
502/// # Syntax
503///
504/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
505/// the following modifications is expected:
506/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
507/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
508/// pointer named `this` inside of the initializer.
509///
510/// For instance:
511///
512/// ```rust
513/// # use kernel::pin_init;
514/// # use macros::pin_data;
515/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
516/// #[pin_data]
517/// struct Buf {
518/// // `ptr` points into `buf`.
519/// ptr: *mut u8,
520/// buf: [u8; 64],
521/// #[pin]
522/// pin: PhantomPinned,
523/// }
524/// pin_init!(&this in Buf {
525/// buf: [0; 64],
526/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
527/// pin: PhantomPinned,
528/// });
529/// ```
530///
531/// [`try_pin_init!`]: kernel::try_pin_init
532/// [`NonNull<Self>`]: core::ptr::NonNull
533// For a detailed example of how this macro works, see the module documentation of the hidden
534// module `__internal` inside of `init/__internal.rs`.
535#[macro_export]
536macro_rules! pin_init {
537 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
538 $($fields:tt)*
539 }) => {
540 $crate::try_pin_init!(
541 @this($($this)?),
542 @typ($t $(::<$($generics),*>)?),
543 @fields($($fields)*),
544 @error(::core::convert::Infallible),
545 )
546 };
547}
548
549/// Construct an in-place, fallible pinned initializer for `struct`s.
550///
551/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
552///
553/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
554/// initialization and return the error.
555///
556/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
557/// initialization fails, the memory can be safely deallocated without any further modifications.
558///
559/// This macro defaults the error to [`Error`].
560///
561/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
562/// after the `struct` initializer to specify the error type you want to use.
563///
564/// # Examples
565///
566/// ```rust
567/// # #![feature(new_uninit)]
568/// use kernel::{init::{self, PinInit}, error::Error};
569/// #[pin_data]
570/// struct BigBuf {
571/// big: Box<[u8; 1024 * 1024 * 1024]>,
572/// small: [u8; 1024 * 1024],
573/// ptr: *mut u8,
574/// }
575///
576/// impl BigBuf {
577/// fn new() -> impl PinInit<Self, Error> {
578/// try_pin_init!(Self {
579/// big: Box::init(init::zeroed())?,
580/// small: [0; 1024 * 1024],
581/// ptr: core::ptr::null_mut(),
582/// }? Error)
583/// }
584/// }
585/// ```
586// For a detailed example of how this macro works, see the module documentation of the hidden
587// module `__internal` inside of `init/__internal.rs`.
588#[macro_export]
589macro_rules! try_pin_init {
590 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
591 $($fields:tt)*
592 }) => {
593 $crate::try_pin_init!(
594 @this($($this)?),
595 @typ($t $(::<$($generics),*>)? ),
596 @fields($($fields)*),
597 @error($crate::error::Error),
598 )
599 };
600 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
601 $($fields:tt)*
602 }? $err:ty) => {
603 $crate::try_pin_init!(
604 @this($($this)?),
605 @typ($t $(::<$($generics),*>)? ),
606 @fields($($fields)*),
607 @error($err),
608 )
609 };
610 (
611 @this($($this:ident)?),
612 @typ($t:ident $(::<$($generics:ty),*>)?),
613 @fields($($fields:tt)*),
614 @error($err:ty),
615 ) => {{
616 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
617 // type and shadow it later when we insert the arbitrary user code. That way there will be
618 // no possibility of returning without `unsafe`.
619 struct __InitOk;
620 // Get the pin data from the supplied type.
621 let data = unsafe {
622 use $crate::init::__internal::HasPinData;
623 $t$(::<$($generics),*>)?::__pin_data()
624 };
625 // Ensure that `data` really is of type `PinData` and help with type inference:
626 let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>(
627 data,
628 move |slot| {
629 {
630 // Shadow the structure so it cannot be used to return early.
631 struct __InitOk;
632 // Create the `this` so it can be referenced by the user inside of the
633 // expressions creating the individual fields.
634 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
635 // Initialize every field.
636 $crate::try_pin_init!(init_slot:
637 @data(data),
638 @slot(slot),
639 @munch_fields($($fields)*,),
640 );
641 // We use unreachable code to ensure that all fields have been mentioned exactly
642 // once, this struct initializer will still be type-checked and complain with a
643 // very natural error message if a field is forgotten/mentioned more than once.
644 #[allow(unreachable_code, clippy::diverging_sub_expression)]
645 if false {
646 $crate::try_pin_init!(make_initializer:
647 @slot(slot),
648 @type_name($t),
649 @munch_fields($($fields)*,),
650 @acc(),
651 );
652 }
653 // Forget all guards, since initialization was a success.
654 $crate::try_pin_init!(forget_guards:
655 @munch_fields($($fields)*,),
656 );
657 }
658 Ok(__InitOk)
659 }
660 );
661 let init = move |slot| -> ::core::result::Result<(), $err> {
662 init(slot).map(|__InitOk| ())
663 };
664 let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) };
665 init
666 }};
667 (init_slot:
668 @data($data:ident),
669 @slot($slot:ident),
670 @munch_fields($(,)?),
671 ) => {
672 // Endpoint of munching, no fields are left.
673 };
674 (init_slot:
675 @data($data:ident),
676 @slot($slot:ident),
677 // In-place initialization syntax.
678 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
679 ) => {
680 let $field = $val;
681 // Call the initializer.
682 //
683 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
684 // return when an error/panic occurs.
685 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
686 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? };
687 // Create the drop guard.
688 //
689 // We only give access to `&DropGuard`, so it cannot be forgotten via safe code.
690 //
691 // SAFETY: We forget the guard later when initialization has succeeded.
692 let $field = &unsafe {
693 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
694 };
695
696 $crate::try_pin_init!(init_slot:
697 @data($data),
698 @slot($slot),
699 @munch_fields($($rest)*),
700 );
701 };
702 (init_slot:
703 @data($data:ident),
704 @slot($slot:ident),
705 // Direct value init, this is safe for every field.
706 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
707 ) => {
708 $(let $field = $val;)?
709 // Initialize the field.
710 //
711 // SAFETY: The memory at `slot` is uninitialized.
712 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
713 // Create the drop guard:
714 //
715 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
716 //
717 // SAFETY: We forget the guard later when initialization has succeeded.
718 let $field = &unsafe {
719 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
720 };
721
722 $crate::try_pin_init!(init_slot:
723 @data($data),
724 @slot($slot),
725 @munch_fields($($rest)*),
726 );
727 };
728 (make_initializer:
729 @slot($slot:ident),
730 @type_name($t:ident),
731 @munch_fields($(,)?),
732 @acc($($acc:tt)*),
733 ) => {
734 // Endpoint, nothing more to munch, create the initializer.
735 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
736 // get the correct type inference here:
737 unsafe {
738 ::core::ptr::write($slot, $t {
739 $($acc)*
740 });
741 }
742 };
743 (make_initializer:
744 @slot($slot:ident),
745 @type_name($t:ident),
746 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
747 @acc($($acc:tt)*),
748 ) => {
749 $crate::try_pin_init!(make_initializer:
750 @slot($slot),
751 @type_name($t),
752 @munch_fields($($rest)*),
753 @acc($($acc)* $field: ::core::panic!(),),
754 );
755 };
756 (make_initializer:
757 @slot($slot:ident),
758 @type_name($t:ident),
759 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
760 @acc($($acc:tt)*),
761 ) => {
762 $crate::try_pin_init!(make_initializer:
763 @slot($slot),
764 @type_name($t),
765 @munch_fields($($rest)*),
766 @acc($($acc)* $field: ::core::panic!(),),
767 );
768 };
769 (forget_guards:
770 @munch_fields($(,)?),
771 ) => {
772 // Munching finished.
773 };
774 (forget_guards:
775 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
776 ) => {
777 unsafe { $crate::init::__internal::DropGuard::forget($field) };
778
779 $crate::try_pin_init!(forget_guards:
780 @munch_fields($($rest)*),
781 );
782 };
783 (forget_guards:
784 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
785 ) => {
786 unsafe { $crate::init::__internal::DropGuard::forget($field) };
787
788 $crate::try_pin_init!(forget_guards:
789 @munch_fields($($rest)*),
790 );
791 };
792}
793
794/// Construct an in-place initializer for `struct`s.
795///
796/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
797/// [`try_init!`].
798///
799/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
800/// - `unsafe` code must guarantee either full initialization or return an error and allow
801/// deallocation of the memory.
802/// - the fields are initialized in the order given in the initializer.
803/// - no references to fields are allowed to be created inside of the initializer.
804///
805/// This initializer is for initializing data in-place that might later be moved. If you want to
806/// pin-initialize, use [`pin_init!`].
807// For a detailed example of how this macro works, see the module documentation of the hidden
808// module `__internal` inside of `init/__internal.rs`.
809#[macro_export]
810macro_rules! init {
811 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
812 $($fields:tt)*
813 }) => {
814 $crate::try_init!(
815 @this($($this)?),
816 @typ($t $(::<$($generics),*>)?),
817 @fields($($fields)*),
818 @error(::core::convert::Infallible),
819 )
820 }
821}
822
823/// Construct an in-place fallible initializer for `struct`s.
824///
825/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
826/// [`init!`].
827///
828/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
829/// append `? $type` after the `struct` initializer.
830/// The safety caveats from [`try_pin_init!`] also apply:
831/// - `unsafe` code must guarantee either full initialization or return an error and allow
832/// deallocation of the memory.
833/// - the fields are initialized in the order given in the initializer.
834/// - no references to fields are allowed to be created inside of the initializer.
835///
836/// # Examples
837///
838/// ```rust
839/// use kernel::{init::PinInit, error::Error, InPlaceInit};
840/// struct BigBuf {
841/// big: Box<[u8; 1024 * 1024 * 1024]>,
842/// small: [u8; 1024 * 1024],
843/// }
844///
845/// impl BigBuf {
846/// fn new() -> impl Init<Self, Error> {
847/// try_init!(Self {
848/// big: Box::init(zeroed())?,
849/// small: [0; 1024 * 1024],
850/// }? Error)
851/// }
852/// }
853/// ```
854// For a detailed example of how this macro works, see the module documentation of the hidden
855// module `__internal` inside of `init/__internal.rs`.
856#[macro_export]
857macro_rules! try_init {
858 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
859 $($fields:tt)*
860 }) => {
861 $crate::try_init!(
862 @this($($this)?),
863 @typ($t $(::<$($generics),*>)?),
864 @fields($($fields)*),
865 @error($crate::error::Error),
866 )
867 };
868 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
869 $($fields:tt)*
870 }? $err:ty) => {
871 $crate::try_init!(
872 @this($($this)?),
873 @typ($t $(::<$($generics),*>)?),
874 @fields($($fields)*),
875 @error($err),
876 )
877 };
878 (
879 @this($($this:ident)?),
880 @typ($t:ident $(::<$($generics:ty),*>)?),
881 @fields($($fields:tt)*),
882 @error($err:ty),
883 ) => {{
884 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
885 // type and shadow it later when we insert the arbitrary user code. That way there will be
886 // no possibility of returning without `unsafe`.
887 struct __InitOk;
888 // Get the init data from the supplied type.
889 let data = unsafe {
890 use $crate::init::__internal::HasInitData;
891 $t$(::<$($generics),*>)?::__init_data()
892 };
893 // Ensure that `data` really is of type `InitData` and help with type inference:
894 let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>(
895 data,
896 move |slot| {
897 {
898 // Shadow the structure so it cannot be used to return early.
899 struct __InitOk;
900 // Create the `this` so it can be referenced by the user inside of the
901 // expressions creating the individual fields.
902 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
903 // Initialize every field.
904 $crate::try_init!(init_slot:
905 @slot(slot),
906 @munch_fields($($fields)*,),
907 );
908 // We use unreachable code to ensure that all fields have been mentioned exactly
909 // once, this struct initializer will still be type-checked and complain with a
910 // very natural error message if a field is forgotten/mentioned more than once.
911 #[allow(unreachable_code, clippy::diverging_sub_expression)]
912 if false {
913 $crate::try_init!(make_initializer:
914 @slot(slot),
915 @type_name($t),
916 @munch_fields($($fields)*,),
917 @acc(),
918 );
919 }
920 // Forget all guards, since initialization was a success.
921 $crate::try_init!(forget_guards:
922 @munch_fields($($fields)*,),
923 );
924 }
925 Ok(__InitOk)
926 }
927 );
928 let init = move |slot| -> ::core::result::Result<(), $err> {
929 init(slot).map(|__InitOk| ())
930 };
931 let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) };
932 init
933 }};
934 (init_slot:
935 @slot($slot:ident),
936 @munch_fields( $(,)?),
937 ) => {
938 // Endpoint of munching, no fields are left.
939 };
940 (init_slot:
941 @slot($slot:ident),
942 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
943 ) => {
944 let $field = $val;
945 // Call the initializer.
946 //
947 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
948 // return when an error/panic occurs.
949 unsafe {
950 $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?;
951 }
952 // Create the drop guard.
953 //
954 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
955 //
956 // SAFETY: We forget the guard later when initialization has succeeded.
957 let $field = &unsafe {
958 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
959 };
960
961 $crate::try_init!(init_slot:
962 @slot($slot),
963 @munch_fields($($rest)*),
964 );
965 };
966 (init_slot:
967 @slot($slot:ident),
968 // Direct value init.
969 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
970 ) => {
971 $(let $field = $val;)?
972 // Call the initializer.
973 //
974 // SAFETY: The memory at `slot` is uninitialized.
975 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
976 // Create the drop guard.
977 //
978 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
979 //
980 // SAFETY: We forget the guard later when initialization has succeeded.
981 let $field = &unsafe {
982 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
983 };
984
985 $crate::try_init!(init_slot:
986 @slot($slot),
987 @munch_fields($($rest)*),
988 );
989 };
990 (make_initializer:
991 @slot($slot:ident),
992 @type_name($t:ident),
993 @munch_fields( $(,)?),
994 @acc($($acc:tt)*),
995 ) => {
996 // Endpoint, nothing more to munch, create the initializer.
997 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
998 // get the correct type inference here:
999 unsafe {
1000 ::core::ptr::write($slot, $t {
1001 $($acc)*
1002 });
1003 }
1004 };
1005 (make_initializer:
1006 @slot($slot:ident),
1007 @type_name($t:ident),
1008 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1009 @acc($($acc:tt)*),
1010 ) => {
1011 $crate::try_init!(make_initializer:
1012 @slot($slot),
1013 @type_name($t),
1014 @munch_fields($($rest)*),
1015 @acc($($acc)*$field: ::core::panic!(),),
1016 );
1017 };
1018 (make_initializer:
1019 @slot($slot:ident),
1020 @type_name($t:ident),
1021 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1022 @acc($($acc:tt)*),
1023 ) => {
1024 $crate::try_init!(make_initializer:
1025 @slot($slot),
1026 @type_name($t),
1027 @munch_fields($($rest)*),
1028 @acc($($acc)*$field: ::core::panic!(),),
1029 );
1030 };
1031 (forget_guards:
1032 @munch_fields($(,)?),
1033 ) => {
1034 // Munching finished.
1035 };
1036 (forget_guards:
1037 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1038 ) => {
1039 unsafe { $crate::init::__internal::DropGuard::forget($field) };
1040
1041 $crate::try_init!(forget_guards:
1042 @munch_fields($($rest)*),
1043 );
1044 };
1045 (forget_guards:
1046 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1047 ) => {
1048 unsafe { $crate::init::__internal::DropGuard::forget($field) };
1049
1050 $crate::try_init!(forget_guards:
1051 @munch_fields($($rest)*),
1052 );
1053 };
1054}
1055
1056/// A pin-initializer for the type `T`.
1057///
1058/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
1059/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
1060/// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
1061///
1062/// Also see the [module description](self).
1063///
1064/// # Safety
1065///
1066/// When implementing this type you will need to take great care. Also there are probably very few
1067/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
1068///
1069/// The [`PinInit::__pinned_init`] function
1070/// - returns `Ok(())` if it initialized every field of `slot`,
1071/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1072/// - `slot` can be deallocated without UB occurring,
1073/// - `slot` does not need to be dropped,
1074/// - `slot` is not partially initialized.
1075/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1076///
1077/// [`Arc<T>`]: crate::sync::Arc
1078/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
1079#[must_use = "An initializer must be used in order to create its value."]
1080pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
1081 /// Initializes `slot`.
1082 ///
1083 /// # Safety
1084 ///
1085 /// - `slot` is a valid pointer to uninitialized memory.
1086 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
1087 /// deallocate.
1088 /// - `slot` will not move until it is dropped, i.e. it will be pinned.
1089 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
1090}
1091
1092/// An initializer for `T`.
1093///
1094/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
1095/// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
1096/// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
1097/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
1098///
1099/// Also see the [module description](self).
1100///
1101/// # Safety
1102///
1103/// When implementing this type you will need to take great care. Also there are probably very few
1104/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
1105///
1106/// The [`Init::__init`] function
1107/// - returns `Ok(())` if it initialized every field of `slot`,
1108/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1109/// - `slot` can be deallocated without UB occurring,
1110/// - `slot` does not need to be dropped,
1111/// - `slot` is not partially initialized.
1112/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1113///
1114/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
1115/// code as `__init`.
1116///
1117/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
1118/// move the pointee after initialization.
1119///
1120/// [`Arc<T>`]: crate::sync::Arc
1121#[must_use = "An initializer must be used in order to create its value."]
1122pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized {
1123 /// Initializes `slot`.
1124 ///
1125 /// # Safety
1126 ///
1127 /// - `slot` is a valid pointer to uninitialized memory.
1128 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
1129 /// deallocate.
1130 unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
1131}
1132
1133// SAFETY: Every in-place initializer can also be used as a pin-initializer.
1134unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I
1135where
1136 I: Init<T, E>,
1137{
1138 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1139 // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not
1140 // require `slot` to not move after init.
1141 unsafe { self.__init(slot) }
1142 }
1143}
1144
1145/// Creates a new [`PinInit<T, E>`] from the given closure.
1146///
1147/// # Safety
1148///
1149/// The closure:
1150/// - returns `Ok(())` if it initialized every field of `slot`,
1151/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1152/// - `slot` can be deallocated without UB occurring,
1153/// - `slot` does not need to be dropped,
1154/// - `slot` is not partially initialized.
1155/// - may assume that the `slot` does not move if `T: !Unpin`,
1156/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1157#[inline]
1158pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1159 f: impl FnOnce(*mut T) -> Result<(), E>,
1160) -> impl PinInit<T, E> {
1161 __internal::InitClosure(f, PhantomData)
1162}
1163
1164/// Creates a new [`Init<T, E>`] from the given closure.
1165///
1166/// # Safety
1167///
1168/// The closure:
1169/// - returns `Ok(())` if it initialized every field of `slot`,
1170/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1171/// - `slot` can be deallocated without UB occurring,
1172/// - `slot` does not need to be dropped,
1173/// - `slot` is not partially initialized.
1174/// - the `slot` may move after initialization.
1175/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1176#[inline]
1177pub const unsafe fn init_from_closure<T: ?Sized, E>(
1178 f: impl FnOnce(*mut T) -> Result<(), E>,
1179) -> impl Init<T, E> {
1180 __internal::InitClosure(f, PhantomData)
1181}
1182
1183/// An initializer that leaves the memory uninitialized.
1184///
1185/// The initializer is a no-op. The `slot` memory is not changed.
1186#[inline]
1187pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1188 // SAFETY: The memory is allowed to be uninitialized.
1189 unsafe { init_from_closure(|_| Ok(())) }
1190}
1191
1192// SAFETY: Every type can be initialized by-value.
1193unsafe impl<T, E> Init<T, E> for T {
1194 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1195 unsafe { slot.write(self) };
1196 Ok(())
1197 }
1198}
1199
1200/// Smart pointer that can initialize memory in-place.
1201pub trait InPlaceInit<T>: Sized {
1202 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1203 /// type.
1204 ///
1205 /// If `T: !Unpin` it will not be able to move afterwards.
1206 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1207 where
1208 E: From<AllocError>;
1209
1210 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1211 /// type.
1212 ///
1213 /// If `T: !Unpin` it will not be able to move afterwards.
1214 fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>>
1215 where
1216 Error: From<E>,
1217 {
1218 // SAFETY: We delegate to `init` and only change the error type.
1219 let init = unsafe {
1220 pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1221 };
1222 Self::try_pin_init(init)
1223 }
1224
1225 /// Use the given initializer to in-place initialize a `T`.
1226 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1227 where
1228 E: From<AllocError>;
1229
1230 /// Use the given initializer to in-place initialize a `T`.
1231 fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
1232 where
1233 Error: From<E>,
1234 {
1235 // SAFETY: We delegate to `init` and only change the error type.
1236 let init = unsafe {
1237 init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1238 };
1239 Self::try_init(init)
1240 }
1241}
1242
1243impl<T> InPlaceInit<T> for Box<T> {
1244 #[inline]
1245 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1246 where
1247 E: From<AllocError>,
1248 {
1249 let mut this = Box::try_new_uninit()?;
1250 let slot = this.as_mut_ptr();
1251 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1252 // slot is valid and will not be moved, because we pin it later.
1253 unsafe { init.__pinned_init(slot)? };
1254 // SAFETY: All fields have been initialized.
1255 Ok(unsafe { this.assume_init() }.into())
1256 }
1257
1258 #[inline]
1259 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1260 where
1261 E: From<AllocError>,
1262 {
1263 let mut this = Box::try_new_uninit()?;
1264 let slot = this.as_mut_ptr();
1265 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1266 // slot is valid.
1267 unsafe { init.__init(slot)? };
1268 // SAFETY: All fields have been initialized.
1269 Ok(unsafe { this.assume_init() })
1270 }
1271}
1272
1273impl<T> InPlaceInit<T> for UniqueArc<T> {
1274 #[inline]
1275 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1276 where
1277 E: From<AllocError>,
1278 {
1279 let mut this = UniqueArc::try_new_uninit()?;
1280 let slot = this.as_mut_ptr();
1281 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1282 // slot is valid and will not be moved, because we pin it later.
1283 unsafe { init.__pinned_init(slot)? };
1284 // SAFETY: All fields have been initialized.
1285 Ok(unsafe { this.assume_init() }.into())
1286 }
1287
1288 #[inline]
1289 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1290 where
1291 E: From<AllocError>,
1292 {
1293 let mut this = UniqueArc::try_new_uninit()?;
1294 let slot = this.as_mut_ptr();
1295 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1296 // slot is valid.
1297 unsafe { init.__init(slot)? };
1298 // SAFETY: All fields have been initialized.
1299 Ok(unsafe { this.assume_init() })
1300 }
1301}
1302
1303/// Trait facilitating pinned destruction.
1304///
1305/// Use [`pinned_drop`] to implement this trait safely:
1306///
1307/// ```rust
1308/// # use kernel::sync::Mutex;
1309/// use kernel::macros::pinned_drop;
1310/// use core::pin::Pin;
1311/// #[pin_data(PinnedDrop)]
1312/// struct Foo {
1313/// #[pin]
1314/// mtx: Mutex<usize>,
1315/// }
1316///
1317/// #[pinned_drop]
1318/// impl PinnedDrop for Foo {
1319/// fn drop(self: Pin<&mut Self>) {
1320/// pr_info!("Foo is being dropped!");
1321/// }
1322/// }
1323/// ```
1324///
1325/// # Safety
1326///
1327/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1328///
1329/// [`pinned_drop`]: kernel::macros::pinned_drop
1330pub unsafe trait PinnedDrop: __internal::HasPinData {
1331 /// Executes the pinned destructor of this type.
1332 ///
1333 /// While this function is marked safe, it is actually unsafe to call it manually. For this
1334 /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1335 /// and thus prevents this function from being called where it should not.
1336 ///
1337 /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1338 /// automatically.
1339 fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1340}
1341
1342/// Marker trait for types that can be initialized by writing just zeroes.
1343///
1344/// # Safety
1345///
1346/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1347/// this is not UB:
1348///
1349/// ```rust,ignore
1350/// let val: Self = unsafe { core::mem::zeroed() };
1351/// ```
1352pub unsafe trait Zeroable {}
1353
1354/// Create a new zeroed T.
1355///
1356/// The returned initializer will write `0x00` to every byte of the given `slot`.
1357#[inline]
1358pub fn zeroed<T: Zeroable>() -> impl Init<T> {
1359 // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1360 // and because we write all zeroes, the memory is initialized.
1361 unsafe {
1362 init_from_closure(|slot: *mut T| {
1363 slot.write_bytes(0, 1);
1364 Ok(())
1365 })
1366 }
1367}
1368
1369macro_rules! impl_zeroable {
1370 ($($({$($generics:tt)*})? $t:ty, )*) => {
1371 $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1372 };
1373}
1374
1375impl_zeroable! {
1376 // SAFETY: All primitives that are allowed to be zero.
1377 bool,
1378 char,
1379 u8, u16, u32, u64, u128, usize,
1380 i8, i16, i32, i64, i128, isize,
1381 f32, f64,
1382
1383 // SAFETY: These are ZSTs, there is nothing to zero.
1384 {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (),
1385
1386 // SAFETY: Type is allowed to take any value, including all zeros.
1387 {<T>} MaybeUninit<T>,
1388
1389 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1390 Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1391 Option<NonZeroU128>, Option<NonZeroUsize>,
1392 Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1393 Option<NonZeroI128>, Option<NonZeroIsize>,
1394
1395 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1396 //
1397 // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
1398 {<T: ?Sized>} Option<NonNull<T>>,
1399 {<T: ?Sized>} Option<Box<T>>,
1400
1401 // SAFETY: `null` pointer is valid.
1402 //
1403 // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1404 // null.
1405 //
1406 // When `Pointee` gets stabilized, we could use
1407 // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1408 {<T>} *mut T, {<T>} *const T,
1409
1410 // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1411 // zero.
1412 {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1413
1414 // SAFETY: `T` is `Zeroable`.
1415 {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1416}
1417
1418macro_rules! impl_tuple_zeroable {
1419 ($(,)?) => {};
1420 ($first:ident, $($t:ident),* $(,)?) => {
1421 // SAFETY: All elements are zeroable and padding can be zero.
1422 unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1423 impl_tuple_zeroable!($($t),* ,);
1424 }
1425}
1426
1427impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);