projects/linux·blob 1e098dec · refs

524 lines · 17.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! A reference-counted pointer.
4//!
5//! This module implements a way for users to create reference-counted objects and pointers to
6//! them. Such a pointer automatically increments and decrements the count, and drops the
7//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8//! threads.
9//!
10//! It is different from the standard library's [`Arc`] in a few ways:
11//! 1. It is backed by the kernel's `refcount_t` type.
12//! 2. It does not support weak references, which allows it to be half the size.
13//! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14//! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15//!
16//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
17
18use crate::{
19 bindings,
20 error::Result,
21 types::{ForeignOwnable, Opaque},
22};
23use alloc::boxed::Box;
24use core::{
25 marker::{PhantomData, Unsize},
26 mem::{ManuallyDrop, MaybeUninit},
27 ops::{Deref, DerefMut},
28 pin::Pin,
29 ptr::NonNull,
30};
31
32/// A reference-counted pointer to an instance of `T`.
33///
34/// The reference count is incremented when new instances of [`Arc`] are created, and decremented
35/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
36///
37/// # Invariants
38///
39/// The reference count on an instance of [`Arc`] is always non-zero.
40/// The object pointed to by [`Arc`] is always pinned.
41///
42/// # Examples
43///
44/// ```
45/// use kernel::sync::Arc;
46///
47/// struct Example {
48/// a: u32,
49/// b: u32,
50/// }
51///
52/// // Create a ref-counted instance of `Example`.
53/// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
54///
55/// // Get a new pointer to `obj` and increment the refcount.
56/// let cloned = obj.clone();
57///
58/// // Assert that both `obj` and `cloned` point to the same underlying object.
59/// assert!(core::ptr::eq(&*obj, &*cloned));
60///
61/// // Destroy `obj` and decrement its refcount.
62/// drop(obj);
63///
64/// // Check that the values are still accessible through `cloned`.
65/// assert_eq!(cloned.a, 10);
66/// assert_eq!(cloned.b, 20);
67///
68/// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
69/// ```
70///
71/// Using `Arc<T>` as the type of `self`:
72///
73/// ```
74/// use kernel::sync::Arc;
75///
76/// struct Example {
77/// a: u32,
78/// b: u32,
79/// }
80///
81/// impl Example {
82/// fn take_over(self: Arc<Self>) {
83/// // ...
84/// }
85///
86/// fn use_reference(self: &Arc<Self>) {
87/// // ...
88/// }
89/// }
90///
91/// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
92/// obj.use_reference();
93/// obj.take_over();
94/// ```
95///
96/// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
97///
98/// ```
99/// use kernel::sync::{Arc, ArcBorrow};
100///
101/// trait MyTrait {
102/// // Trait has a function whose `self` type is `Arc<Self>`.
103/// fn example1(self: Arc<Self>) {}
104///
105/// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
106/// fn example2(self: ArcBorrow<'_, Self>) {}
107/// }
108///
109/// struct Example;
110/// impl MyTrait for Example {}
111///
112/// // `obj` has type `Arc<Example>`.
113/// let obj: Arc<Example> = Arc::try_new(Example)?;
114///
115/// // `coerced` has type `Arc<dyn MyTrait>`.
116/// let coerced: Arc<dyn MyTrait> = obj;
117/// ```
118pub struct Arc<T: ?Sized> {
119 ptr: NonNull<ArcInner<T>>,
120 _p: PhantomData<ArcInner<T>>,
121}
122
123#[repr(C)]
124struct ArcInner<T: ?Sized> {
125 refcount: Opaque<bindings::refcount_t>,
126 data: T,
127}
128
129// This is to allow [`Arc`] (and variants) to be used as the type of `self`.
130impl<T: ?Sized> core::ops::Receiver for Arc<T> {}
131
132// This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
133// dynamically-sized type (DST) `U`.
134impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
135
136// This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
137impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
138
139// SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
140// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
141// `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` directly, for
142// example, when the reference count reaches zero and `T` is dropped.
143unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
144
145// SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync` for the
146// same reason as above. `T` needs to be `Send` as well because a thread can clone an `&Arc<T>`
147// into an `Arc<T>`, which may lead to `T` being accessed by the same reasoning as above.
148unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
149
150impl<T> Arc<T> {
151 /// Constructs a new reference counted instance of `T`.
152 pub fn try_new(contents: T) -> Result<Self> {
153 // INVARIANT: The refcount is initialised to a non-zero value.
154 let value = ArcInner {
155 // SAFETY: There are no safety requirements for this FFI call.
156 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
157 data: contents,
158 };
159
160 let inner = Box::try_new(value)?;
161
162 // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
163 // `Arc` object.
164 Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })
165 }
166}
167
168impl<T: ?Sized> Arc<T> {
169 /// Constructs a new [`Arc`] from an existing [`ArcInner`].
170 ///
171 /// # Safety
172 ///
173 /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
174 /// count, one of which will be owned by the new [`Arc`] instance.
175 unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
176 // INVARIANT: By the safety requirements, the invariants hold.
177 Arc {
178 ptr: inner,
179 _p: PhantomData,
180 }
181 }
182
183 /// Returns an [`ArcBorrow`] from the given [`Arc`].
184 ///
185 /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
186 /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
187 #[inline]
188 pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
189 // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
190 // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
191 // reference can be created.
192 unsafe { ArcBorrow::new(self.ptr) }
193 }
194}
195
196impl<T: 'static> ForeignOwnable for Arc<T> {
197 type Borrowed<'a> = ArcBorrow<'a, T>;
198
199 fn into_foreign(self) -> *const core::ffi::c_void {
200 ManuallyDrop::new(self).ptr.as_ptr() as _
201 }
202
203 unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {
204 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
205 // a previous call to `Arc::into_foreign`.
206 let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();
207
208 // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
209 // for the lifetime of the returned value. Additionally, the safety requirements of
210 // `ForeignOwnable::borrow_mut` ensure that no new mutable references are created.
211 unsafe { ArcBorrow::new(inner) }
212 }
213
214 unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
215 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
216 // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
217 // holds a reference count increment that is transferrable to us.
218 unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }
219 }
220}
221
222impl<T: ?Sized> Deref for Arc<T> {
223 type Target = T;
224
225 fn deref(&self) -> &Self::Target {
226 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
227 // safe to dereference it.
228 unsafe { &self.ptr.as_ref().data }
229 }
230}
231
232impl<T: ?Sized> Clone for Arc<T> {
233 fn clone(&self) -> Self {
234 // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
235 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
236 // safe to increment the refcount.
237 unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };
238
239 // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
240 unsafe { Self::from_inner(self.ptr) }
241 }
242}
243
244impl<T: ?Sized> Drop for Arc<T> {
245 fn drop(&mut self) {
246 // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
247 // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
248 // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
249 // freed/invalid memory as long as it is never dereferenced.
250 let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
251
252 // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
253 // this instance is being dropped, so the broken invariant is not observable.
254 // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
255 let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
256 if is_zero {
257 // The count reached zero, we must free the memory.
258 //
259 // SAFETY: The pointer was initialised from the result of `Box::leak`.
260 unsafe { Box::from_raw(self.ptr.as_ptr()) };
261 }
262 }
263}
264
265impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
266 fn from(item: UniqueArc<T>) -> Self {
267 item.inner
268 }
269}
270
271impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
272 fn from(item: Pin<UniqueArc<T>>) -> Self {
273 // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
274 unsafe { Pin::into_inner_unchecked(item).inner }
275 }
276}
277
278/// A borrowed reference to an [`Arc`] instance.
279///
280/// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
281/// to use just `&T`, which we can trivially get from an `Arc<T>` instance.
282///
283/// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
284/// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
285/// to a pointer (`Arc<T>`) to the object (`T`). An [`ArcBorrow`] eliminates this double
286/// indirection while still allowing one to increment the refcount and getting an `Arc<T>` when/if
287/// needed.
288///
289/// # Invariants
290///
291/// There are no mutable references to the underlying [`Arc`], and it remains valid for the
292/// lifetime of the [`ArcBorrow`] instance.
293///
294/// # Example
295///
296/// ```
297/// use crate::sync::{Arc, ArcBorrow};
298///
299/// struct Example;
300///
301/// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
302/// e.into()
303/// }
304///
305/// let obj = Arc::try_new(Example)?;
306/// let cloned = do_something(obj.as_arc_borrow());
307///
308/// // Assert that both `obj` and `cloned` point to the same underlying object.
309/// assert!(core::ptr::eq(&*obj, &*cloned));
310/// ```
311///
312/// Using `ArcBorrow<T>` as the type of `self`:
313///
314/// ```
315/// use crate::sync::{Arc, ArcBorrow};
316///
317/// struct Example {
318/// a: u32,
319/// b: u32,
320/// }
321///
322/// impl Example {
323/// fn use_reference(self: ArcBorrow<'_, Self>) {
324/// // ...
325/// }
326/// }
327///
328/// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
329/// obj.as_arc_borrow().use_reference();
330/// ```
331pub struct ArcBorrow<'a, T: ?Sized + 'a> {
332 inner: NonNull<ArcInner<T>>,
333 _p: PhantomData<&'a ()>,
334}
335
336// This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.
337impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}
338
339// This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
340// `ArcBorrow<U>`.
341impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
342 for ArcBorrow<'_, T>
343{
344}
345
346impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
347 fn clone(&self) -> Self {
348 *self
349 }
350}
351
352impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
353
354impl<T: ?Sized> ArcBorrow<'_, T> {
355 /// Creates a new [`ArcBorrow`] instance.
356 ///
357 /// # Safety
358 ///
359 /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
360 /// 1. That `inner` remains valid;
361 /// 2. That no mutable references to `inner` are created.
362 unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
363 // INVARIANT: The safety requirements guarantee the invariants.
364 Self {
365 inner,
366 _p: PhantomData,
367 }
368 }
369}
370
371impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
372 fn from(b: ArcBorrow<'_, T>) -> Self {
373 // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
374 // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
375 // increment.
376 ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
377 .deref()
378 .clone()
379 }
380}
381
382impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
383 type Target = T;
384
385 fn deref(&self) -> &Self::Target {
386 // SAFETY: By the type invariant, the underlying object is still alive with no mutable
387 // references to it, so it is safe to create a shared reference.
388 unsafe { &self.inner.as_ref().data }
389 }
390}
391
392/// A refcounted object that is known to have a refcount of 1.
393///
394/// It is mutable and can be converted to an [`Arc`] so that it can be shared.
395///
396/// # Invariants
397///
398/// `inner` always has a reference count of 1.
399///
400/// # Examples
401///
402/// In the following example, we make changes to the inner object before turning it into an
403/// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
404/// cannot fail.
405///
406/// ```
407/// use kernel::sync::{Arc, UniqueArc};
408///
409/// struct Example {
410/// a: u32,
411/// b: u32,
412/// }
413///
414/// fn test() -> Result<Arc<Example>> {
415/// let mut x = UniqueArc::try_new(Example { a: 10, b: 20 })?;
416/// x.a += 1;
417/// x.b += 1;
418/// Ok(x.into())
419/// }
420///
421/// # test().unwrap();
422/// ```
423///
424/// In the following example we first allocate memory for a ref-counted `Example` but we don't
425/// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
426/// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
427/// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
428///
429/// ```
430/// use kernel::sync::{Arc, UniqueArc};
431///
432/// struct Example {
433/// a: u32,
434/// b: u32,
435/// }
436///
437/// fn test() -> Result<Arc<Example>> {
438/// let x = UniqueArc::try_new_uninit()?;
439/// Ok(x.write(Example { a: 10, b: 20 }).into())
440/// }
441///
442/// # test().unwrap();
443/// ```
444///
445/// In the last example below, the caller gets a pinned instance of `Example` while converting to
446/// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
447/// initialisation, for example, when initialising fields that are wrapped in locks.
448///
449/// ```
450/// use kernel::sync::{Arc, UniqueArc};
451///
452/// struct Example {
453/// a: u32,
454/// b: u32,
455/// }
456///
457/// fn test() -> Result<Arc<Example>> {
458/// let mut pinned = Pin::from(UniqueArc::try_new(Example { a: 10, b: 20 })?);
459/// // We can modify `pinned` because it is `Unpin`.
460/// pinned.as_mut().a += 1;
461/// Ok(pinned.into())
462/// }
463///
464/// # test().unwrap();
465/// ```
466pub struct UniqueArc<T: ?Sized> {
467 inner: Arc<T>,
468}
469
470impl<T> UniqueArc<T> {
471 /// Tries to allocate a new [`UniqueArc`] instance.
472 pub fn try_new(value: T) -> Result<Self> {
473 Ok(Self {
474 // INVARIANT: The newly-created object has a ref-count of 1.
475 inner: Arc::try_new(value)?,
476 })
477 }
478
479 /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
480 pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>> {
481 Ok(UniqueArc::<MaybeUninit<T>> {
482 // INVARIANT: The newly-created object has a ref-count of 1.
483 inner: Arc::try_new(MaybeUninit::uninit())?,
484 })
485 }
486}
487
488impl<T> UniqueArc<MaybeUninit<T>> {
489 /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
490 pub fn write(mut self, value: T) -> UniqueArc<T> {
491 self.deref_mut().write(value);
492 let inner = ManuallyDrop::new(self).inner.ptr;
493 UniqueArc {
494 // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
495 // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
496 inner: unsafe { Arc::from_inner(inner.cast()) },
497 }
498 }
499}
500
501impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
502 fn from(obj: UniqueArc<T>) -> Self {
503 // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
504 // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
505 unsafe { Pin::new_unchecked(obj) }
506 }
507}
508
509impl<T: ?Sized> Deref for UniqueArc<T> {
510 type Target = T;
511
512 fn deref(&self) -> &Self::Target {
513 self.inner.deref()
514 }
515}
516
517impl<T: ?Sized> DerefMut for UniqueArc<T> {
518 fn deref_mut(&mut self) -> &mut Self::Target {
519 // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
520 // it is safe to dereference it. Additionally, we know there is only one reference when
521 // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
522 unsafe { &mut self.inner.ptr.as_mut().data }
523 }
524}