projects/linux·blob master · refs

352 lines · 11.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Atomic internal implementations.
4//!
5//! Provides 1:1 mapping to the C atomic operations.
6
7use crate::{
8 bindings,
9 build_assert::static_assert,
10 macros::paste, //
11};
12use core::cell::UnsafeCell;
13use ffi::c_void;
14
15mod private {
16 /// Sealed trait marker to disable customized impls on atomic implementation traits.
17 pub trait Sealed {}
18}
19
20// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`),
21// while the Rust side also provides atomic support for `i8`, `i16` and `*const c_void` on top of
22// lower-level C primitives.
23impl private::Sealed for i8 {}
24impl private::Sealed for i16 {}
25impl private::Sealed for *const c_void {}
26impl private::Sealed for i32 {}
27impl private::Sealed for i64 {}
28
29/// A marker trait for types that implement atomic operations with C side primitives.
30///
31/// This trait is sealed, and only types that map directly to the C side atomics
32/// or can be implemented with lower-level C primitives are allowed to implement this:
33///
34/// - `i8`, `i16` and `*const c_void` are implemented with lower-level C primitives.
35/// - `i32` map to `atomic_t`
36/// - `i64` map to `atomic64_t`
37pub trait AtomicImpl: Sized + Copy + private::Sealed {
38 /// The type of the delta in arithmetic or logical operations.
39 ///
40 /// For example, in `atomic_add(ptr, v)`, it's the type of `v`. Usually it's the same type of
41 /// [`Self`], but it may be different for the atomic pointer type.
42 type Delta;
43}
44
45// The current helpers of load/store of atomic `i8`, `i16` and pointers use `{WRITE,READ}_ONCE()`
46// hence the atomicity is only guaranteed against read-modify-write operations if the architecture
47// supports native atomic RmW.
48//
49// In the future when a CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=n architecture plans to support Rust, the
50// load/store helpers that guarantee atomicity against RmW operations (usually via a lock) need to
51// be added.
52static_assert!(
53 cfg!(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW),
54 "The current implementation of atomic i8/i16/ptr relies on the architecure being \
55 ARCH_SUPPORTS_ATOMIC_RMW"
56);
57
58impl AtomicImpl for i8 {
59 type Delta = Self;
60}
61
62impl AtomicImpl for i16 {
63 type Delta = Self;
64}
65
66impl AtomicImpl for *const c_void {
67 type Delta = isize;
68}
69
70// `atomic_t` implements atomic operations on `i32`.
71impl AtomicImpl for i32 {
72 type Delta = Self;
73}
74
75// `atomic64_t` implements atomic operations on `i64`.
76impl AtomicImpl for i64 {
77 type Delta = Self;
78}
79
80/// Atomic representation.
81#[repr(transparent)]
82pub struct AtomicRepr<T: AtomicImpl>(UnsafeCell<T>);
83
84impl<T: AtomicImpl> AtomicRepr<T> {
85 /// Creates a new atomic representation `T`.
86 pub const fn new(v: T) -> Self {
87 Self(UnsafeCell::new(v))
88 }
89
90 /// Returns a pointer to the underlying `T`.
91 ///
92 /// # Guarantees
93 ///
94 /// The returned pointer is valid and properly aligned (i.e. aligned to [`align_of::<T>()`]).
95 pub const fn as_ptr(&self) -> *mut T {
96 // GUARANTEE: `self.0` is an `UnsafeCell<T>`, therefore the pointer returned by `.get()`
97 // must be valid and properly aligned.
98 self.0.get()
99 }
100}
101
102// This macro generates the function signature with given argument list and return type.
103macro_rules! declare_atomic_method {
104 (
105 $(#[doc=$doc:expr])*
106 $func:ident($($arg:ident : $arg_type:ty),*) $(-> $ret:ty)?
107 ) => {
108 paste!(
109 $(#[doc = $doc])*
110 fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)?;
111 );
112 };
113 (
114 $(#[doc=$doc:expr])*
115 $func:ident [$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)?
116 ) => {
117 paste!(
118 declare_atomic_method!(
119 $(#[doc = $doc])*
120 [< $func _ $variant >]($($arg_sig)*) $(-> $ret)?
121 );
122 );
123
124 declare_atomic_method!(
125 $(#[doc = $doc])*
126 $func [$($rest)*]($($arg_sig)*) $(-> $ret)?
127 );
128 };
129 (
130 $(#[doc=$doc:expr])*
131 $func:ident []($($arg_sig:tt)*) $(-> $ret:ty)?
132 ) => {
133 declare_atomic_method!(
134 $(#[doc = $doc])*
135 $func($($arg_sig)*) $(-> $ret)?
136 );
137 }
138}
139
140// This macro generates the function implementation with given argument list and return type, and it
141// will replace "call(...)" expression with "$ctype _ $func" to call the real C function.
142macro_rules! impl_atomic_method {
143 (
144 ($ctype:ident) $func:ident($($arg:ident: $arg_type:ty),*) $(-> $ret:ty)? {
145 $unsafe:tt { call($($c_arg:expr),*) }
146 }
147 ) => {
148 paste!(
149 #[inline(always)]
150 fn [< atomic_ $func >]($($arg: $arg_type,)*) $(-> $ret)? {
151 // TODO: Ideally we want to use the SAFETY comments written at the macro invocation
152 // (e.g. in `declare_and_impl_atomic_methods!()`, however, since SAFETY comments
153 // are just comments, and they are not passed to macros as tokens, therefore we
154 // cannot use them here. One potential improvement is that if we support using
155 // attributes as an alternative for SAFETY comments, then we can use that for macro
156 // generating code.
157 //
158 // SAFETY: specified on macro invocation.
159 $unsafe { bindings::[< $ctype _ $func >]($($c_arg,)*) }
160 }
161 );
162 };
163 (
164 ($ctype:ident) $func:ident[$variant:ident $($rest:ident)*]($($arg_sig:tt)*) $(-> $ret:ty)? {
165 $unsafe:tt { call($($arg:tt)*) }
166 }
167 ) => {
168 paste!(
169 impl_atomic_method!(
170 ($ctype) [< $func _ $variant >]($($arg_sig)*) $( -> $ret)? {
171 $unsafe { call($($arg)*) }
172 }
173 );
174 );
175 impl_atomic_method!(
176 ($ctype) $func [$($rest)*]($($arg_sig)*) $( -> $ret)? {
177 $unsafe { call($($arg)*) }
178 }
179 );
180 };
181 (
182 ($ctype:ident) $func:ident[]($($arg_sig:tt)*) $( -> $ret:ty)? {
183 $unsafe:tt { call($($arg:tt)*) }
184 }
185 ) => {
186 impl_atomic_method!(
187 ($ctype) $func($($arg_sig)*) $(-> $ret)? {
188 $unsafe { call($($arg)*) }
189 }
190 );
191 }
192}
193
194macro_rules! declare_atomic_ops_trait {
195 (
196 $(#[$attr:meta])* $pub:vis trait $ops:ident {
197 $(
198 $(#[doc=$doc:expr])*
199 fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? {
200 $unsafe:tt { bindings::#call($($arg:tt)*) }
201 }
202 )*
203 }
204 ) => {
205 $(#[$attr])*
206 $pub trait $ops: AtomicImpl {
207 $(
208 declare_atomic_method!(
209 $(#[doc=$doc])*
210 $func[$($variant)*]($($arg_sig)*) $(-> $ret)?
211 );
212 )*
213 }
214 }
215}
216
217macro_rules! impl_atomic_ops_for_one {
218 (
219 $ty:ty => $ctype:ident,
220 $(#[$attr:meta])* $pub:vis trait $ops:ident {
221 $(
222 $(#[doc=$doc:expr])*
223 fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? {
224 $unsafe:tt { bindings::#call($($arg:tt)*) }
225 }
226 )*
227 }
228 ) => {
229 impl $ops for $ty {
230 $(
231 impl_atomic_method!(
232 ($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? {
233 $unsafe { call($($arg)*) }
234 }
235 );
236 )*
237 }
238 }
239}
240
241// Declares $ops trait with methods and implements the trait.
242macro_rules! declare_and_impl_atomic_methods {
243 (
244 [ $($map:tt)* ]
245 $(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* }
246 ) => {
247 declare_and_impl_atomic_methods!(
248 @with_ops_def
249 [ $($map)* ]
250 ( $(#[$attr])* $pub trait $ops { $($body)* } )
251 );
252 };
253
254 (@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => {
255 declare_atomic_ops_trait!( $($ops_def)* );
256
257 declare_and_impl_atomic_methods!(
258 @munch
259 [ $($map)* ]
260 ( $($ops_def)* )
261 );
262 };
263
264 (@munch [] ( $($ops_def:tt)* )) => {};
265
266 (@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => {
267 impl_atomic_ops_for_one!(
268 $ty => $ctype,
269 $($ops_def)*
270 );
271
272 declare_and_impl_atomic_methods!(
273 @munch
274 [ $($($rest)*)? ]
275 ( $($ops_def)* )
276 );
277 };
278}
279
280declare_and_impl_atomic_methods!(
281 [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
282 /// Basic atomic operations
283 pub trait AtomicBasicOps {
284 /// Atomic read (load).
285 fn read[acquire](a: &AtomicRepr<Self>) -> Self {
286 // SAFETY: `a.as_ptr()` is valid and properly aligned.
287 unsafe { bindings::#call(a.as_ptr().cast()) }
288 }
289
290 /// Atomic set (store).
291 fn set[release](a: &AtomicRepr<Self>, v: Self) {
292 // SAFETY: `a.as_ptr()` is valid and properly aligned.
293 unsafe { bindings::#call(a.as_ptr().cast(), v) }
294 }
295 }
296);
297
298declare_and_impl_atomic_methods!(
299 [ i8 => atomic_i8, i16 => atomic_i16, *const c_void => atomic_ptr, i32 => atomic, i64 => atomic64 ]
300 /// Exchange and compare-and-exchange atomic operations
301 pub trait AtomicExchangeOps {
302 /// Atomic exchange.
303 ///
304 /// Atomically updates `*a` to `v` and returns the old value.
305 fn xchg[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self) -> Self {
306 // SAFETY: `a.as_ptr()` is valid and properly aligned.
307 unsafe { bindings::#call(a.as_ptr().cast(), v) }
308 }
309
310 /// Atomic compare and exchange.
311 ///
312 /// If `*a` == `*old`, atomically updates `*a` to `new`. Otherwise, `*a` is not
313 /// modified, `*old` is updated to the current value of `*a`.
314 ///
315 /// Return `true` if the update of `*a` occurred, `false` otherwise.
316 fn try_cmpxchg[acquire, release, relaxed](
317 a: &AtomicRepr<Self>, old: &mut Self, new: Self
318 ) -> bool {
319 // SAFETY: `a.as_ptr()` is valid and properly aligned. `core::ptr::from_mut(old)`
320 // is valid and properly aligned.
321 unsafe { bindings::#call(a.as_ptr().cast(), core::ptr::from_mut(old), new) }
322 }
323 }
324);
325
326declare_and_impl_atomic_methods!(
327 [ i32 => atomic, i64 => atomic64 ]
328 /// Atomic arithmetic operations
329 pub trait AtomicArithmeticOps {
330 /// Atomic add (wrapping).
331 ///
332 /// Atomically updates `*a` to `(*a).wrapping_add(v)`.
333 fn add[](a: &AtomicRepr<Self>, v: Self::Delta) {
334 // SAFETY: `a.as_ptr()` is valid and properly aligned.
335 unsafe { bindings::#call(v, a.as_ptr().cast()) }
336 }
337
338 /// Atomic fetch and add (wrapping).
339 ///
340 /// Atomically updates `*a` to `(*a).wrapping_add(v)`, and returns the value of `*a`
341 /// before the update.
342 fn fetch_add[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
343 // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
344 unsafe { bindings::#call(v, a.as_ptr().cast()) }
345 }
346
347 fn fetch_sub[acquire, release, relaxed](a: &AtomicRepr<Self>, v: Self::Delta) -> Self {
348 // SAFETY: `a.as_ptr()` guarantees the returned pointer is valid and properly aligned.
349 unsafe { bindings::#call(v, a.as_ptr().cast()) }
350 }
351 }
352);