projects/linux·blob master · refs

465 lines · 15.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! A kernel spinlock.
4//!
5//! This module allows Rust code to use the kernel's `spinlock_t`.
6use super::*;
7use crate::{
8 interrupt::LocalInterruptDisabled,
9 prelude::*, //
10};
11
12/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
13///
14/// It uses the name if one is given, otherwise it generates one based on the file name and line
15/// number.
16#[macro_export]
17macro_rules! new_spinlock {
18 ($inner:expr $(, $name:literal)? $(,)?) => {
19 $crate::sync::SpinLock::new(
20 $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
21 };
22}
23pub use new_spinlock;
24
25/// A spinlock.
26///
27/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only
28/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is
29/// unlocked, at which point another CPU will be allowed to make progress.
30///
31/// Instances of [`SpinLock`] need a lock class and to be pinned. The recommended way to create such
32/// instances is with the [`pin_init`](pin_init::pin_init) and [`new_spinlock`] macros.
33///
34/// # Examples
35///
36/// The following example shows how to declare, allocate and initialise a struct (`Example`) that
37/// contains an inner struct (`Inner`) that is protected by a spinlock.
38///
39/// ```
40/// use kernel::sync::{new_spinlock, SpinLock};
41///
42/// struct Inner {
43/// a: u32,
44/// b: u32,
45/// }
46///
47/// #[pin_data]
48/// struct Example {
49/// c: u32,
50/// #[pin]
51/// d: SpinLock<Inner>,
52/// }
53///
54/// impl Example {
55/// fn new() -> impl PinInit<Self> {
56/// pin_init!(Self {
57/// c: 10,
58/// d <- new_spinlock!(Inner { a: 20, b: 30 }),
59/// })
60/// }
61/// }
62///
63/// // Allocate a boxed `Example`.
64/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
65/// assert_eq!(e.c, 10);
66/// assert_eq!(e.d.lock().a, 20);
67/// assert_eq!(e.d.lock().b, 30);
68/// # Ok::<(), Error>(())
69/// ```
70///
71/// The following example shows how to use interior mutability to modify the contents of a struct
72/// protected by a spinlock despite only having a shared reference:
73///
74/// ```
75/// use kernel::sync::SpinLock;
76///
77/// struct Example {
78/// a: u32,
79/// b: u32,
80/// }
81///
82/// fn example(m: &SpinLock<Example>) {
83/// let mut guard = m.lock();
84/// guard.a += 10;
85/// guard.b += 20;
86/// }
87/// ```
88///
89/// [`spinlock_t`]: srctree/include/linux/spinlock.h
90pub type SpinLock<T> = Lock<T, SpinLockBackend>;
91
92/// A kernel `spinlock_t` lock backend.
93pub struct SpinLockBackend;
94
95/// A [`Guard`] acquired from locking a [`SpinLock`].
96///
97/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock
98/// the [`SpinLock`] upon being dropped.
99pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>;
100
101// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
102// default implementation that always calls the same locking method.
103unsafe impl Backend for SpinLockBackend {
104 type State = bindings::spinlock_t;
105 type GuardState = ();
106
107 #[inline]
108 unsafe fn init(
109 ptr: *mut Self::State,
110 name: *const crate::ffi::c_char,
111 key: *mut bindings::lock_class_key,
112 ) {
113 // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
114 // `key` are valid for read indefinitely.
115 unsafe { bindings::__spin_lock_init(ptr, name, key) }
116 }
117
118 #[inline]
119 unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
120 // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
121 // memory, and that it has been initialised before.
122 unsafe { bindings::spin_lock(ptr) }
123 }
124
125 #[inline]
126 unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
127 // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
128 // caller is the owner of the spinlock.
129 unsafe { bindings::spin_unlock(ptr) }
130 }
131
132 #[inline]
133 unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
134 // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
135 let result = unsafe { bindings::spin_trylock(ptr) };
136
137 if result != 0 {
138 Some(())
139 } else {
140 None
141 }
142 }
143
144 #[inline]
145 unsafe fn assert_is_held(ptr: *mut Self::State) {
146 // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
147 unsafe { bindings::spin_assert_is_held(ptr) }
148 }
149}
150
151/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class.
152///
153/// It uses the name if one is given, otherwise it generates one based on the file name and line
154/// number.
155#[macro_export]
156macro_rules! new_spinlock_irq {
157 ($inner:expr $(, $name:literal)? $(,)?) => {
158 $crate::sync::SpinLockIrq::new(
159 $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
160 };
161}
162pub use new_spinlock_irq;
163
164/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section.
165///
166/// This lock can be acquired in two ways:
167///
168/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the
169/// interrupt state to ensure that local processor interrupts remain disabled for at least as
170/// long as the [`SpinLockIrqGuard`] exists.
171/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and
172/// local processor interrupts are already known to be disabled, in which case the local
173/// interrupt state will not be touched. This method should be preferred if a
174/// [`LocalInterruptDisabled`] token is present in the scope.
175///
176/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts,
177/// [see the interrupt module](kernel::interrupt).
178///
179/// # Examples
180///
181/// The following example shows how to declare, allocate initialise and access a struct (`Example`)
182/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local
183/// processor interrupts to be disabled.
184///
185/// ```
186/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
187///
188/// struct Inner {
189/// a: u32,
190/// b: u32,
191/// }
192///
193/// #[pin_data]
194/// struct Example {
195/// #[pin]
196/// c: SpinLockIrq<Inner>,
197/// #[pin]
198/// d: SpinLockIrq<Inner>,
199/// }
200///
201/// impl Example {
202/// fn new() -> impl PinInit<Self> {
203/// pin_init!(Self {
204/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }),
205/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }),
206/// })
207/// }
208/// }
209///
210/// // Allocate a boxed `Example`
211/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
212///
213/// // Accessing an `Example` from a context where interrupts may not be disabled already.
214/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount
215/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount
216///
217/// assert_eq!(c_guard.a, 0);
218/// assert_eq!(c_guard.b, 10);
219/// assert_eq!(d_guard.a, 20);
220/// assert_eq!(d_guard.b, 30);
221///
222/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is
223/// // still in scope.
224/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled
225/// // now
226/// # Ok::<(), Error>(())
227/// ```
228///
229/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function
230/// which can only be called when local processor interrupts are already disabled.
231///
232/// ```
233/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
234/// use kernel::interrupt::*;
235///
236/// struct Inner {
237/// a: u32,
238/// }
239///
240/// #[pin_data]
241/// struct Example {
242/// #[pin]
243/// inner: SpinLockIrq<Inner>,
244/// }
245///
246/// impl Example {
247/// fn new() -> impl PinInit<Self> {
248/// pin_init!(Self {
249/// inner <- new_spinlock_irq!(Inner { a: 20 }),
250/// })
251/// }
252/// }
253///
254/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts.
255/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) {
256/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling
257/// // interrupt state using lock_with() and the provided token
258/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20);
259/// }
260///
261/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
262/// # let interrupt_guard = local_interrupt_disable();
263/// # noirq_work(&e, &interrupt_guard);
264/// #
265/// # Ok::<(), Error>(())
266/// ```
267///
268/// [`lock()`]: SpinLockIrq::lock
269/// [`lock_with()`]: SpinLockIrq::lock_with
270pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>;
271
272/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
273pub struct SpinLockIrqBackend;
274
275/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`].
276///
277/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using
278/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt
279/// disablement refcount upon being dropped.
280///
281/// [`lock()`]: SpinLockIrq::lock
282pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>;
283
284// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
285// default implementation that always calls the same locking method.
286unsafe impl Backend for SpinLockIrqBackend {
287 type State = bindings::spinlock_t;
288 type GuardState = ();
289
290 #[inline]
291 unsafe fn init(
292 ptr: *mut Self::State,
293 name: *const crate::ffi::c_char,
294 key: *mut bindings::lock_class_key,
295 ) {
296 // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
297 // `key` are valid for read indefinitely.
298 unsafe { bindings::__spin_lock_init(ptr, name, key) }
299 }
300
301 #[inline]
302 unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
303 // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
304 // memory, and that it has been initialised before.
305 unsafe { bindings::spin_lock_irq_disable(ptr) }
306 }
307
308 #[inline]
309 unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
310 // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
311 // caller is the owner of the spinlock.
312 unsafe { bindings::spin_unlock_irq_enable(ptr) }
313 }
314
315 #[inline]
316 unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
317 // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
318 let result = unsafe { bindings::spin_trylock_irq_disable(ptr) };
319
320 if result != 0 {
321 Some(())
322 } else {
323 None
324 }
325 }
326
327 #[inline]
328 unsafe fn assert_is_held(ptr: *mut Self::State) {
329 // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
330 unsafe { bindings::spin_assert_is_held(ptr) }
331 }
332}
333
334impl<T: ?Sized> Lock<T, SpinLockIrqBackend> {
335 /// Casts the lock as a `Lock<T, SpinLockBackend>`.
336 #[inline]
337 fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> {
338 // SAFETY:
339 // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data
340 // layouts.
341 // - As long as local interrupts are disabled (which is proven to be true by _context), it
342 // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock.
343 unsafe { core::mem::transmute(self) }
344 }
345
346 /// Acquires the lock without modifying local interrupt state.
347 ///
348 /// This function should be used in place of the more expensive [`Lock::lock()`] function when
349 /// possible for [`SpinLockIrq`] locks.
350 #[inline]
351 pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> {
352 self.as_lock_in_interrupt(context).lock()
353 }
354
355 /// Tries to acquire the lock without modifying local interrupt state.
356 ///
357 /// This function should be used in place of the more expensive [`Lock::try_lock()`] function
358 /// when possible for [`SpinLockIrq`] locks.
359 ///
360 /// Returns a guard that can be used to access the data protected by the lock if successful.
361 #[must_use = "if unused, the lock will be immediately unlocked"]
362 #[inline]
363 pub fn try_lock_with<'a>(
364 &'a self,
365 context: &'a LocalInterruptDisabled,
366 ) -> Option<SpinLockGuard<'a, T>> {
367 self.as_lock_in_interrupt(context).try_lock()
368 }
369}
370
371#[kunit_tests(rust_spinlock_irq_condvar)]
372mod tests {
373 use super::*;
374 use crate::{
375 sync::*,
376 workqueue::{
377 self,
378 impl_has_work,
379 new_work,
380 Work,
381 WorkItem, //
382 },
383 };
384
385 struct TestState {
386 value: u32,
387 waiter_ready: bool,
388 }
389
390 #[pin_data]
391 struct Test {
392 #[pin]
393 state: SpinLockIrq<TestState>,
394
395 #[pin]
396 state_changed: CondVar,
397
398 #[pin]
399 waiter_state_changed: CondVar,
400
401 #[pin]
402 wait_work: Work<Self>,
403 }
404
405 impl_has_work! {
406 impl HasWork<Self> for Test { self.wait_work }
407 }
408
409 impl Test {
410 pub(crate) fn new() -> Result<Arc<Self>> {
411 Arc::try_pin_init(
412 try_pin_init!(
413 Self {
414 state <- new_spinlock_irq!(TestState {
415 value: 1,
416 waiter_ready: false
417 }),
418 state_changed <- new_condvar!(),
419 waiter_state_changed <- new_condvar!(),
420 wait_work <- new_work!("IrqCondvarTest::wait_work")
421 }
422 ),
423 GFP_KERNEL,
424 )
425 }
426 }
427
428 impl WorkItem for Test {
429 type Pointer = Arc<Self>;
430
431 fn run(this: Arc<Self>) {
432 // Wait for the test to be ready to wait for us
433 let mut state = this.state.lock();
434
435 // Make sure the interrupts actually turned off
436 // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`
437 unsafe { bindings::lockdep_assert_irqs_disabled() };
438
439 while !state.waiter_ready {
440 this.waiter_state_changed.wait(&mut state);
441 }
442
443 // Deliver the exciting value update our test has been waiting for
444 state.value += 1;
445 this.state_changed.notify_sync();
446 }
447 }
448
449 #[test]
450 fn spinlock_irq_condvar() -> Result {
451 let testdata = Test::new()?;
452
453 let _ = workqueue::system().enqueue(testdata.clone());
454
455 // Let the updater know when we're ready to wait
456 let mut state = testdata.state.lock();
457 state.waiter_ready = true;
458 testdata.waiter_state_changed.notify_sync();
459
460 // Wait for the exciting value update
461 testdata.state_changed.wait(&mut state);
462 assert_eq!(state.value, 2);
463 Ok(())
464 }
465}