projects/linux·blob master · refs

500 lines · 19.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3// Copyright (C) 2024 Google LLC.
4
5//! A wrapper around `Arc` for linked lists.
6
7use crate::alloc::{AllocError, Flags};
8use crate::prelude::*;
9use crate::sync::atomic::{ordering, AtomicFlag};
10use crate::sync::{Arc, ArcBorrow, UniqueArc};
11use core::marker::PhantomPinned;
12use core::ops::Deref;
13use core::pin::Pin;
14
15/// Declares that this type has some way to ensure that there is exactly one `ListArc` instance for
16/// this id.
17///
18/// Types that implement this trait should include some kind of logic for keeping track of whether
19/// a [`ListArc`] exists or not. We refer to this logic as "the tracking inside `T`".
20///
21/// We allow the case where the tracking inside `T` thinks that a [`ListArc`] exists, but actually,
22/// there isn't a [`ListArc`]. However, we do not allow the opposite situation where a [`ListArc`]
23/// exists, but the tracking thinks it doesn't. This is because the former can at most result in us
24/// failing to create a [`ListArc`] when the operation could succeed, whereas the latter can result
25/// in the creation of two [`ListArc`] references. Only the latter situation can lead to memory
26/// safety issues.
27///
28/// A consequence of the above is that you may implement the tracking inside `T` by not actually
29/// keeping track of anything. To do this, you always claim that a [`ListArc`] exists, even if
30/// there isn't one. This implementation is allowed by the above rule, but it means that
31/// [`ListArc`] references can only be created if you have ownership of *all* references to the
32/// refcounted object, as you otherwise have no way of knowing whether a [`ListArc`] exists.
33pub trait ListArcSafe<const ID: u64 = 0> {
34 /// Informs the tracking inside this type that it now has a [`ListArc`] reference.
35 ///
36 /// This method may be called even if the tracking inside this type thinks that a `ListArc`
37 /// reference exists. (But only if that's not actually the case.)
38 ///
39 /// # Safety
40 ///
41 /// Must not be called if a [`ListArc`] already exist for this value.
42 unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>);
43
44 /// Informs the tracking inside this type that there is no [`ListArc`] reference anymore.
45 ///
46 /// # Safety
47 ///
48 /// Must only be called if there is no [`ListArc`] reference, but the tracking thinks there is.
49 unsafe fn on_drop_list_arc(&self);
50}
51
52/// Declares that this type is able to safely attempt to create `ListArc`s at any time.
53///
54/// # Safety
55///
56/// The guarantees of `try_new_list_arc` must be upheld.
57pub unsafe trait TryNewListArc<const ID: u64 = 0>: ListArcSafe<ID> {
58 /// Attempts to convert an `Arc<Self>` into an `ListArc<Self>`. Returns `true` if the
59 /// conversion was successful.
60 ///
61 /// This method should not be called directly. Use [`ListArc::try_from_arc`] instead.
62 ///
63 /// # Guarantees
64 ///
65 /// If this call returns `true`, then there is no [`ListArc`] pointing to this value.
66 /// Additionally, this call will have transitioned the tracking inside `Self` from not thinking
67 /// that a [`ListArc`] exists, to thinking that a [`ListArc`] exists.
68 fn try_new_list_arc(&self) -> bool;
69}
70
71/// Declares that this type supports [`ListArc`].
72///
73/// This macro supports a few different strategies for implementing the tracking inside the type:
74///
75/// * The `untracked` strategy does not actually keep track of whether a [`ListArc`] exists. When
76/// using this strategy, the only way to create a [`ListArc`] is using a [`UniqueArc`].
77/// * The `tracked_by` strategy defers the tracking to a field of the struct. The user must specify
78/// which field to defer the tracking to. The field must implement [`ListArcSafe`]. If the field
79/// implements [`TryNewListArc`], then the type will also implement [`TryNewListArc`].
80///
81/// The `tracked_by` strategy is usually used by deferring to a field of type
82/// [`AtomicTracker`]. However, it is also possible to defer the tracking to another struct
83/// using also using this macro.
84#[macro_export]
85#[doc(hidden)]
86macro_rules! impl_list_arc_safe {
87 (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty { untracked; } $($rest:tt)*) => {
88 impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t {
89 unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) {}
90 unsafe fn on_drop_list_arc(&self) {}
91 }
92 $crate::list::impl_list_arc_safe! { $($rest)* }
93 };
94
95 (impl$({$($generics:tt)*})? ListArcSafe<$num:tt> for $t:ty {
96 tracked_by $field:ident : $fty:ty;
97 } $($rest:tt)*) => {
98 impl$(<$($generics)*>)? $crate::list::ListArcSafe<$num> for $t {
99 unsafe fn on_create_list_arc_from_unique(self: ::core::pin::Pin<&mut Self>) {
100 ::pin_init::assert_pinned!($t, $field, $fty, inline);
101
102 // SAFETY: This field is structurally pinned as per the above assertion.
103 let field = unsafe {
104 ::core::pin::Pin::map_unchecked_mut(self, |me| &mut me.$field)
105 };
106 // SAFETY: The caller promises that there is no `ListArc`.
107 unsafe {
108 <$fty as $crate::list::ListArcSafe<$num>>::on_create_list_arc_from_unique(field)
109 };
110 }
111 unsafe fn on_drop_list_arc(&self) {
112 // SAFETY: The caller promises that there is no `ListArc` reference, and also
113 // promises that the tracking thinks there is a `ListArc` reference.
114 unsafe { <$fty as $crate::list::ListArcSafe<$num>>::on_drop_list_arc(&self.$field) };
115 }
116 }
117 unsafe impl$(<$($generics)*>)? $crate::list::TryNewListArc<$num> for $t
118 where
119 $fty: TryNewListArc<$num>,
120 {
121 fn try_new_list_arc(&self) -> bool {
122 <$fty as $crate::list::TryNewListArc<$num>>::try_new_list_arc(&self.$field)
123 }
124 }
125 $crate::list::impl_list_arc_safe! { $($rest)* }
126 };
127
128 () => {};
129}
130pub use impl_list_arc_safe;
131
132/// A wrapper around [`Arc`] that's guaranteed unique for the given id.
133///
134/// The `ListArc` type can be thought of as a special reference to a refcounted object that owns the
135/// permission to manipulate the `next`/`prev` pointers stored in the refcounted object. By ensuring
136/// that each object has only one `ListArc` reference, the owner of that reference is assured
137/// exclusive access to the `next`/`prev` pointers. When a `ListArc` is inserted into a [`List`],
138/// the [`List`] takes ownership of the `ListArc` reference.
139///
140/// There are various strategies to ensuring that a value has only one `ListArc` reference. The
141/// simplest is to convert a [`UniqueArc`] into a `ListArc`. However, the refcounted object could
142/// also keep track of whether a `ListArc` exists using a boolean, which could allow for the
143/// creation of new `ListArc` references from an [`Arc`] reference. Whatever strategy is used, the
144/// relevant tracking is referred to as "the tracking inside `T`", and the [`ListArcSafe`] trait
145/// (and its subtraits) are used to update the tracking when a `ListArc` is created or destroyed.
146///
147/// Note that we allow the case where the tracking inside `T` thinks that a `ListArc` exists, but
148/// actually, there isn't a `ListArc`. However, we do not allow the opposite situation where a
149/// `ListArc` exists, but the tracking thinks it doesn't. This is because the former can at most
150/// result in us failing to create a `ListArc` when the operation could succeed, whereas the latter
151/// can result in the creation of two `ListArc` references.
152///
153/// While this `ListArc` is unique for the given id, there still might exist normal `Arc`
154/// references to the object.
155///
156/// # Invariants
157///
158/// * Each reference counted object has at most one `ListArc` for each value of `ID`.
159/// * The tracking inside `T` is aware that a `ListArc` reference exists.
160///
161/// [`List`]: crate::list::List
162#[repr(transparent)]
163#[derive(core::marker::CoercePointee)]
164pub struct ListArc<T, const ID: u64 = 0>
165where
166 T: ListArcSafe<ID> + ?Sized,
167{
168 arc: Arc<T>,
169}
170
171impl<T: ListArcSafe<ID>, const ID: u64> ListArc<T, ID> {
172 /// Constructs a new reference counted instance of `T`.
173 #[inline]
174 pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
175 Ok(Self::from(UniqueArc::new(contents, flags)?))
176 }
177
178 /// Use the given initializer to in-place initialize a `T`.
179 ///
180 /// If `T: !Unpin` it will not be able to move afterwards.
181 // We don't implement `InPlaceInit` because `ListArc` is implicitly pinned. This is similar to
182 // what we do for `Arc`.
183 #[inline]
184 pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self, E>
185 where
186 E: From<AllocError>,
187 {
188 Ok(Self::from(UniqueArc::try_pin_init(init, flags)?))
189 }
190
191 /// Use the given initializer to in-place initialize a `T`.
192 ///
193 /// This is equivalent to [`ListArc<T>::pin_init`], since a [`ListArc`] is always pinned.
194 #[inline]
195 pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
196 where
197 E: From<AllocError>,
198 {
199 Ok(Self::from(UniqueArc::try_init(init, flags)?))
200 }
201}
202
203impl<T, const ID: u64> From<UniqueArc<T>> for ListArc<T, ID>
204where
205 T: ListArcSafe<ID> + ?Sized,
206{
207 /// Convert a [`UniqueArc`] into a [`ListArc`].
208 #[inline]
209 fn from(unique: UniqueArc<T>) -> Self {
210 Self::from(Pin::from(unique))
211 }
212}
213
214impl<T, const ID: u64> From<Pin<UniqueArc<T>>> for ListArc<T, ID>
215where
216 T: ListArcSafe<ID> + ?Sized,
217{
218 /// Convert a pinned [`UniqueArc`] into a [`ListArc`].
219 #[inline]
220 fn from(mut unique: Pin<UniqueArc<T>>) -> Self {
221 // SAFETY: We have a `UniqueArc`, so there is no `ListArc`.
222 unsafe { T::on_create_list_arc_from_unique(unique.as_mut()) };
223 let arc = Arc::from(unique);
224 // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc`,
225 // so we can create a `ListArc`.
226 unsafe { Self::transmute_from_arc(arc) }
227 }
228}
229
230impl<T, const ID: u64> ListArc<T, ID>
231where
232 T: ListArcSafe<ID> + ?Sized,
233{
234 /// Creates two `ListArc`s from a [`UniqueArc`].
235 ///
236 /// The two ids must be different.
237 #[inline]
238 pub fn pair_from_unique<const ID2: u64>(unique: UniqueArc<T>) -> (Self, ListArc<T, ID2>)
239 where
240 T: ListArcSafe<ID2>,
241 {
242 Self::pair_from_pin_unique(Pin::from(unique))
243 }
244
245 /// Creates two `ListArc`s from a pinned [`UniqueArc`].
246 ///
247 /// The two ids must be different.
248 #[inline]
249 pub fn pair_from_pin_unique<const ID2: u64>(
250 mut unique: Pin<UniqueArc<T>>,
251 ) -> (Self, ListArc<T, ID2>)
252 where
253 T: ListArcSafe<ID2>,
254 {
255 build_assert!(ID != ID2);
256
257 // SAFETY: We have a `UniqueArc`, so there is no `ListArc`.
258 unsafe { <T as ListArcSafe<ID>>::on_create_list_arc_from_unique(unique.as_mut()) };
259 // SAFETY: We have a `UniqueArc`, so there is no `ListArc`.
260 unsafe { <T as ListArcSafe<ID2>>::on_create_list_arc_from_unique(unique.as_mut()) };
261
262 let arc1 = Arc::from(unique);
263 let arc2 = Arc::clone(&arc1);
264
265 // SAFETY: We just called `on_create_list_arc_from_unique` on an arc without a `ListArc`
266 // for both IDs (which are different), so we can create two `ListArc`s.
267 unsafe {
268 (
269 Self::transmute_from_arc(arc1),
270 ListArc::transmute_from_arc(arc2),
271 )
272 }
273 }
274
275 /// Try to create a new `ListArc`.
276 ///
277 /// This fails if this value already has a `ListArc`.
278 pub fn try_from_arc(arc: Arc<T>) -> Result<Self, Arc<T>>
279 where
280 T: TryNewListArc<ID>,
281 {
282 if arc.try_new_list_arc() {
283 // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think
284 // that a `ListArc` exists. This lets us create a `ListArc`.
285 Ok(unsafe { Self::transmute_from_arc(arc) })
286 } else {
287 Err(arc)
288 }
289 }
290
291 /// Try to create a new `ListArc`.
292 ///
293 /// This fails if this value already has a `ListArc`.
294 pub fn try_from_arc_borrow(arc: ArcBorrow<'_, T>) -> Option<Self>
295 where
296 T: TryNewListArc<ID>,
297 {
298 if arc.try_new_list_arc() {
299 // SAFETY: The `try_new_list_arc` method returned true, so we made the tracking think
300 // that a `ListArc` exists. This lets us create a `ListArc`.
301 Some(unsafe { Self::transmute_from_arc(Arc::from(arc)) })
302 } else {
303 None
304 }
305 }
306
307 /// Try to create a new `ListArc`.
308 ///
309 /// If it's not possible to create a new `ListArc`, then the `Arc` is dropped. This will never
310 /// run the destructor of the value.
311 pub fn try_from_arc_or_drop(arc: Arc<T>) -> Option<Self>
312 where
313 T: TryNewListArc<ID>,
314 {
315 match Self::try_from_arc(arc) {
316 Ok(list_arc) => Some(list_arc),
317 Err(arc) => Arc::into_unique_or_drop(arc).map(Self::from),
318 }
319 }
320
321 /// Transmutes an [`Arc`] into a `ListArc` without updating the tracking inside `T`.
322 ///
323 /// # Safety
324 ///
325 /// * The value must not already have a `ListArc` reference.
326 /// * The tracking inside `T` must think that there is a `ListArc` reference.
327 #[inline]
328 unsafe fn transmute_from_arc(arc: Arc<T>) -> Self {
329 // INVARIANT: By the safety requirements, the invariants on `ListArc` are satisfied.
330 Self { arc }
331 }
332
333 /// Transmutes a `ListArc` into an [`Arc`] without updating the tracking inside `T`.
334 ///
335 /// After this call, the tracking inside `T` will still think that there is a `ListArc`
336 /// reference.
337 #[inline]
338 fn transmute_to_arc(self) -> Arc<T> {
339 // Use a transmute to skip destructor.
340 //
341 // SAFETY: ListArc is repr(transparent).
342 unsafe { core::mem::transmute(self) }
343 }
344
345 /// Convert ownership of this `ListArc` into a raw pointer.
346 ///
347 /// The returned pointer is indistinguishable from pointers returned by [`Arc::into_raw`]. The
348 /// tracking inside `T` will still think that a `ListArc` exists after this call.
349 #[inline]
350 pub fn into_raw(self) -> *const T {
351 Arc::into_raw(Self::transmute_to_arc(self))
352 }
353
354 /// Take ownership of the `ListArc` from a raw pointer.
355 ///
356 /// # Safety
357 ///
358 /// * `ptr` must satisfy the safety requirements of [`Arc::from_raw`].
359 /// * The value must not already have a `ListArc` reference.
360 /// * The tracking inside `T` must think that there is a `ListArc` reference.
361 #[inline]
362 pub unsafe fn from_raw(ptr: *const T) -> Self {
363 // SAFETY: The pointer satisfies the safety requirements for `Arc::from_raw`.
364 let arc = unsafe { Arc::from_raw(ptr) };
365 // SAFETY: The value doesn't already have a `ListArc` reference, but the tracking thinks it
366 // does.
367 unsafe { Self::transmute_from_arc(arc) }
368 }
369
370 /// Converts the `ListArc` into an [`Arc`].
371 #[inline]
372 pub fn into_arc(self) -> Arc<T> {
373 let arc = Self::transmute_to_arc(self);
374 // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is.
375 unsafe { T::on_drop_list_arc(&arc) };
376 arc
377 }
378
379 /// Clone a `ListArc` into an [`Arc`].
380 #[inline]
381 pub fn clone_arc(&self) -> Arc<T> {
382 self.arc.clone()
383 }
384
385 /// Returns a reference to an [`Arc`] from the given [`ListArc`].
386 ///
387 /// This is useful when the argument of a function call is an [`&Arc`] (e.g., in a method
388 /// receiver), but we have a [`ListArc`] instead.
389 ///
390 /// [`&Arc`]: Arc
391 #[inline]
392 pub fn as_arc(&self) -> &Arc<T> {
393 &self.arc
394 }
395
396 /// Returns an [`ArcBorrow`] from the given [`ListArc`].
397 ///
398 /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
399 /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
400 #[inline]
401 pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
402 self.arc.as_arc_borrow()
403 }
404
405 /// Compare whether two [`ListArc`] pointers reference the same underlying object.
406 #[inline]
407 pub fn ptr_eq(this: &Self, other: &Self) -> bool {
408 Arc::ptr_eq(&this.arc, &other.arc)
409 }
410}
411
412impl<T, const ID: u64> Deref for ListArc<T, ID>
413where
414 T: ListArcSafe<ID> + ?Sized,
415{
416 type Target = T;
417
418 #[inline]
419 fn deref(&self) -> &Self::Target {
420 self.arc.deref()
421 }
422}
423
424impl<T, const ID: u64> Drop for ListArc<T, ID>
425where
426 T: ListArcSafe<ID> + ?Sized,
427{
428 #[inline]
429 fn drop(&mut self) {
430 // SAFETY: There is no longer a `ListArc`, but the tracking thinks there is by the type
431 // invariants on `Self`.
432 unsafe { T::on_drop_list_arc(&self.arc) };
433 }
434}
435
436impl<T, const ID: u64> AsRef<Arc<T>> for ListArc<T, ID>
437where
438 T: ListArcSafe<ID> + ?Sized,
439{
440 #[inline]
441 fn as_ref(&self) -> &Arc<T> {
442 self.as_arc()
443 }
444}
445
446/// A utility for tracking whether a [`ListArc`] exists using an atomic.
447///
448/// # Invariants
449///
450/// If the boolean is `false`, then there is no [`ListArc`] for this value.
451#[repr(transparent)]
452pub struct AtomicTracker<const ID: u64 = 0> {
453 inner: AtomicFlag,
454 // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`.
455 _pin: PhantomPinned,
456}
457
458impl<const ID: u64> AtomicTracker<ID> {
459 /// Creates a new initializer for this type.
460 pub fn new() -> impl PinInit<Self> {
461 // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will
462 // not be constructed in an `Arc` that already has a `ListArc`.
463 Self {
464 inner: AtomicFlag::new(false),
465 _pin: PhantomPinned,
466 }
467 }
468
469 fn project_inner(self: Pin<&mut Self>) -> &mut AtomicFlag {
470 // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable
471 // reference to it even if we only have a pinned reference to `self`.
472 unsafe { &mut Pin::into_inner_unchecked(self).inner }
473 }
474}
475
476impl<const ID: u64> ListArcSafe<ID> for AtomicTracker<ID> {
477 unsafe fn on_create_list_arc_from_unique(self: Pin<&mut Self>) {
478 // INVARIANT: We just created a ListArc, so the boolean should be true.
479 *self.project_inner().get_mut() = true;
480 }
481
482 unsafe fn on_drop_list_arc(&self) {
483 // INVARIANT: We just dropped a ListArc, so the boolean should be false.
484 self.inner.store(false, ordering::Release);
485 }
486}
487
488// SAFETY: If this method returns `true`, then by the type invariant there is no `ListArc` before
489// this call, so it is okay to create a new `ListArc`.
490//
491// The acquire ordering will synchronize with the release store from the destruction of any
492// previous `ListArc`, so if there was a previous `ListArc`, then the destruction of the previous
493// `ListArc` happens-before the creation of the new `ListArc`.
494unsafe impl<const ID: u64> TryNewListArc<ID> for AtomicTracker<ID> {
495 fn try_new_list_arc(&self) -> bool {
496 // INVARIANT: If this method returns true, then the boolean used to be false, and is no
497 // longer false, so it is okay for the caller to create a new [`ListArc`].
498 self.inner.cmpxchg(false, true, ordering::Acquire).is_ok()
499 }
500}