projects/linux·blob master · refs

409 lines · 12.0 KiB

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3//! This module contains library internal items.
4//!
5//! These items must not be used outside of this crate and the pin-init-internal crate located at
6//! `../internal`.
7
8use super::*;
9
10/// Zero-sized type used to mark a type as invariant.
11///
12/// This is a polyfill for the [unstable type] in the standard library of the same name.
13///
14/// See the [nomicon] for what subtyping is. See also [this table].
15///
16/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariant.html
17/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
18/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
19#[repr(transparent)]
20pub struct PhantomInvariant<T: ?Sized>(PhantomData<fn(T) -> T>);
21
22impl<T: ?Sized> Clone for PhantomInvariant<T> {
23 #[inline(always)]
24 fn clone(&self) -> Self {
25 *self
26 }
27}
28
29impl<T: ?Sized> Copy for PhantomInvariant<T> {}
30
31impl<T: ?Sized> Default for PhantomInvariant<T> {
32 #[inline(always)]
33 fn default() -> Self {
34 Self::new()
35 }
36}
37
38impl<T: ?Sized> PhantomInvariant<T> {
39 #[inline(always)]
40 pub const fn new() -> Self {
41 Self(PhantomData)
42 }
43}
44
45/// Zero-sized type used to mark a lifetime as invariant.
46///
47/// This is a polyfill for the [unstable type] in the standard library of the same name.
48///
49/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html
50#[repr(transparent)]
51#[derive(Clone, Copy, Default)]
52pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>);
53
54impl PhantomInvariantLifetime<'_> {
55 #[inline(always)]
56 pub const fn new() -> Self {
57 Self(PhantomInvariant::new())
58 }
59}
60
61/// Token type to signify successful initialization.
62///
63/// Can only be constructed via the unsafe [`Self::new`] function. The initializer macros use this
64/// token type to prevent returning `Ok` from an initializer without initializing all fields.
65pub struct InitOk(());
66
67impl InitOk {
68 /// Creates a new token.
69 ///
70 /// # Safety
71 ///
72 /// This function may only be called from the `init!` macro in `../internal/src/init.rs`.
73 #[inline(always)]
74 pub unsafe fn new() -> Self {
75 Self(())
76 }
77}
78
79/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
80/// the pin projections within the initializers.
81///
82/// # Safety
83///
84/// Only the `init` module is allowed to use this trait.
85pub unsafe trait HasPinData {
86 type PinData;
87
88 #[expect(clippy::missing_safety_doc)]
89 unsafe fn __pin_data() -> Self::PinData;
90}
91
92/// This trait is automatically implemented for every type. It aims to provide the same type
93/// inference help as `HasPinData`.
94///
95/// # Safety
96///
97/// Only the `init` module is allowed to use this trait.
98pub unsafe trait HasInitData {
99 type InitData;
100
101 #[expect(clippy::missing_safety_doc)]
102 unsafe fn __init_data() -> Self::InitData;
103}
104
105pub struct AllData<T: ?Sized>(PhantomInvariant<T>);
106
107impl<T: ?Sized> Clone for AllData<T> {
108 #[inline]
109 fn clone(&self) -> Self {
110 *self
111 }
112}
113
114impl<T: ?Sized> Copy for AllData<T> {}
115
116impl<T: ?Sized> AllData<T> {
117 /// Type inference helper function.
118 #[inline(always)]
119 pub fn __make_closure<F, E>(self, f: F) -> F
120 where
121 F: FnOnce(*mut T) -> Result<InitOk, E>,
122 {
123 f
124 }
125}
126
127// SAFETY: TODO.
128unsafe impl<T: ?Sized> HasInitData for T {
129 type InitData = AllData<T>;
130
131 #[inline]
132 unsafe fn __init_data() -> Self::InitData {
133 AllData(PhantomInvariant::new())
134 }
135}
136
137/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
138///
139/// # Invariants
140///
141/// If `self.is_init` is true, then `self.value` is initialized.
142///
143/// [`stack_pin_init`]: crate::stack_pin_init
144pub struct StackInit<T> {
145 value: MaybeUninit<T>,
146 is_init: bool,
147}
148
149impl<T> Drop for StackInit<T> {
150 #[inline]
151 fn drop(&mut self) {
152 if self.is_init {
153 // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is
154 // true, `self.value` is initialized.
155 unsafe { self.value.assume_init_drop() };
156 }
157 }
158}
159
160impl<T> StackInit<T> {
161 /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this
162 /// primitive.
163 ///
164 /// [`stack_pin_init`]: crate::stack_pin_init
165 #[inline]
166 pub fn uninit() -> Self {
167 Self {
168 value: MaybeUninit::uninit(),
169 is_init: false,
170 }
171 }
172
173 /// Initializes the contents and returns the result.
174 #[inline]
175 pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> {
176 // SAFETY: We never move out of `this`.
177 let this = unsafe { Pin::into_inner_unchecked(self) };
178 // The value is currently initialized, so it needs to be dropped before we can reuse
179 // the memory (this is a safety guarantee of `Pin`).
180 if this.is_init {
181 this.is_init = false;
182 // SAFETY: `this.is_init` was true and therefore `this.value` is initialized.
183 unsafe { this.value.assume_init_drop() };
184 }
185 // SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
186 unsafe { init.__init(this.value.as_mut_ptr())? };
187 // INVARIANT: `this.value` is initialized above.
188 this.is_init = true;
189 // SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
190 Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) })
191 }
192}
193
194#[test]
195#[cfg(feature = "std")]
196fn stack_init_reuse() {
197 use ::std::{borrow::ToOwned, println, string::String};
198 use core::pin::pin;
199
200 #[derive(Debug)]
201 struct Foo {
202 a: usize,
203 b: String,
204 }
205 let mut slot: Pin<&mut StackInit<Foo>> = pin!(StackInit::uninit());
206 let value: Result<Pin<&mut Foo>, core::convert::Infallible> =
207 slot.as_mut().init(crate::init!(Foo {
208 a: 42,
209 b: "Hello".to_owned(),
210 }));
211 let value = value.unwrap();
212 println!("{value:?}");
213 let value: Result<Pin<&mut Foo>, core::convert::Infallible> =
214 slot.as_mut().init(crate::init!(Foo {
215 a: 24,
216 b: "world!".to_owned(),
217 }));
218 let value = value.unwrap();
219 println!("{value:?}");
220}
221
222// Marker types that determines type of `DropGuard`'s let bindings.
223pub struct Pinned;
224pub struct Unpinned;
225
226/// Represent an uninitialized field.
227///
228/// # Invariants
229///
230/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory.
231/// - If `P` is `Pinned`, then `ptr` is structurally pinned.
232pub struct Slot<P, T: ?Sized> {
233 ptr: *mut T,
234 _phantom: PhantomData<P>,
235}
236
237impl<P, T: ?Sized> Slot<P, T> {
238 /// # Safety
239 ///
240 /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed
241 /// memory.
242 /// - If `P` is `Pinned`, then `ptr` is structurally pinned.
243 #[inline(always)]
244 pub unsafe fn new(ptr: *mut T) -> Self {
245 // INVARIANT: Per safety requirement.
246 Self {
247 ptr,
248 _phantom: PhantomData,
249 }
250 }
251
252 /// Initialize the field by value.
253 #[inline(always)]
254 pub fn write(self, value: T) -> DropGuard<P, T>
255 where
256 T: Sized,
257 {
258 // SAFETY: `self.ptr` is a valid and aligned pointer for write.
259 unsafe { self.ptr.write(value) }
260 // SAFETY:
261 // - `self.ptr` is valid and properly aligned per type invariant.
262 // - `*self.ptr` is initialized above and the ownership is transferred to the guard.
263 // - If `P` is `Pinned`, `self.ptr` is pinned.
264 unsafe { DropGuard::new(self.ptr) }
265 }
266}
267
268impl<T: ?Sized> Slot<Unpinned, T> {
269 /// Initialize the field.
270 #[inline(always)]
271 pub fn init<E>(self, init: impl Init<T, E>) -> Result<DropGuard<Unpinned, T>, E> {
272 // SAFETY:
273 // - `self.ptr` is valid and properly aligned.
274 // - when `Err` is returned, we also propagate the error without touching `slot`;
275 // also `self` is consumed so it cannot be touched further.
276 unsafe { init.__init(self.ptr)? };
277
278 // SAFETY:
279 // - `self.ptr` is valid and properly aligned per type invariant.
280 // - `*self.ptr` is initialized above and the ownership is transferred to the guard.
281 Ok(unsafe { DropGuard::new(self.ptr) })
282 }
283}
284
285impl<T: ?Sized> Slot<Pinned, T> {
286 /// Initialize the field.
287 #[inline(always)]
288 pub fn init<E>(self, init: impl PinInit<T, E>) -> Result<DropGuard<Pinned, T>, E> {
289 // SAFETY:
290 // - `self.ptr` is valid and properly aligned.
291 // - when `Err` is returned, we also propagate the error without touching `ptr`;
292 // also `self` is consumed so it cannot be touched further.
293 // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`).
294 unsafe { init.__init(self.ptr)? };
295
296 // SAFETY:
297 // - `self.ptr` is valid, properly aligned and pinned per type invariant.
298 // - `*self.ptr` is initialized above and the ownership is transferred to the guard.
299 Ok(unsafe { DropGuard::new(self.ptr) })
300 }
301}
302
303/// When a value of this type is dropped, it drops a `T`.
304///
305/// Can be forgotten to prevent the drop.
306///
307/// # Invariants
308///
309/// - `ptr` is valid and properly aligned.
310/// - `*ptr` is initialized and owned by this guard.
311/// - if `P` is `Pinned`, `ptr` is pinned.
312pub struct DropGuard<P, T: ?Sized> {
313 ptr: *mut T,
314 phantom: PhantomData<P>,
315}
316
317impl<P, T: ?Sized> DropGuard<P, T> {
318 /// Creates a drop guard and transfer the ownership of the pointer content.
319 ///
320 /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`].
321 ///
322 /// # Safety
323 ///
324 /// - `ptr` is valid and properly aligned.
325 /// - `*ptr` is initialized, and the ownership is transferred to this guard.
326 /// - if `P` is `Pinned`, `ptr` is pinned.
327 #[inline]
328 pub unsafe fn new(ptr: *mut T) -> Self {
329 // INVARIANT: By safety requirement.
330 Self {
331 ptr,
332 phantom: PhantomData,
333 }
334 }
335}
336
337impl<T: ?Sized> DropGuard<Unpinned, T> {
338 /// Create a let binding for accessor use.
339 #[inline]
340 pub fn let_binding(&mut self) -> &mut T {
341 // SAFETY: Per type invariant.
342 unsafe { &mut *self.ptr }
343 }
344}
345
346impl<T: ?Sized> DropGuard<Pinned, T> {
347 /// Create a let binding for accessor use.
348 #[inline]
349 pub fn let_binding(&mut self) -> Pin<&mut T> {
350 // SAFETY: `self.ptr` is valid, properly aligned, initialized, exclusively accessible and
351 // pinned per type invariant.
352 unsafe { Pin::new_unchecked(&mut *self.ptr) }
353 }
354}
355
356impl<P, T: ?Sized> Drop for DropGuard<P, T> {
357 #[inline]
358 fn drop(&mut self) {
359 // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard.
360 unsafe { ptr::drop_in_place(self.ptr) }
361 }
362}
363
364/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely
365/// created struct. This is needed, because the `drop` function is safe, but should not be called
366/// manually.
367pub struct OnlyCallFromDrop(());
368
369impl OnlyCallFromDrop {
370 /// # Safety
371 ///
372 /// This function should only be called from the [`Drop::drop`] function and only be used to
373 /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type.
374 pub unsafe fn new() -> Self {
375 Self(())
376 }
377}
378
379/// Initializer that always fails.
380///
381/// Used by [`assert_pinned!`].
382///
383/// [`assert_pinned!`]: crate::assert_pinned
384pub struct AlwaysFail<T: ?Sized> {
385 _t: PhantomData<T>,
386}
387
388impl<T: ?Sized> AlwaysFail<T> {
389 /// Creates a new initializer that always fails.
390 #[inline]
391 pub fn new() -> Self {
392 Self { _t: PhantomData }
393 }
394}
395
396impl<T: ?Sized> Default for AlwaysFail<T> {
397 #[inline]
398 fn default() -> Self {
399 Self::new()
400 }
401}
402
403// SAFETY: `__init` always fails, which is always okay.
404unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
405 #[inline]
406 unsafe fn __init(self, _slot: *mut T) -> Result<(), ()> {
407 Err(())
408 }
409}