projects/linux·blob master · refs

132 lines · 3.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3//! Memory barriers.
4//!
5//! These primitives have the same semantics as their C counterparts: and the precise definitions
6//! of semantics can be found at [`LKMM`].
7//!
8//! [`LKMM`]: srctree/tools/memory-model/
9
10#![expect(private_bounds, reason = "sealed implementation")]
11
12/// Memory barrier orderings.
13///
14/// The semantics of these orderings follows the [`LKMM`] definitions and rules.
15///
16/// - [`Read`] provides ordering between preceding load operations and succeeding load operations.
17/// - [`Write`] provides ordering between preceding store operations and succeeding store
18/// operations.
19/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory
20/// accesses.
21///
22/// [`LKMM`]: srctree/tools/memory-model/
23pub mod ordering {
24 pub use crate::sync::atomic::ordering::Full;
25
26 /// The annotation type for read-read barrier ordering.
27 pub struct Read;
28
29 /// The annotation type for write-write barrier ordering.
30 pub struct Write;
31}
32
33pub use ordering::{
34 Full,
35 Read,
36 Write, //
37};
38
39struct Smp;
40struct Dma;
41
42/// A compiler barrier.
43///
44/// A barrier that prevents compiler from reordering memory accesses across the barrier.
45#[inline(always)]
46pub(crate) fn barrier() {
47 // By default, Rust inline asms are treated as being able to access any memory or flags, hence
48 // it suffices as a compiler barrier.
49 //
50 // SAFETY: An empty asm block.
51 unsafe { core::arch::asm!("") };
52}
53
54trait MemoryBarrier<Flavour = ()> {
55 fn run();
56}
57
58macro_rules! define_barrier {
59 ($([$flavour:ident])? $ordering:ident, $binding:ident) => {
60 impl MemoryBarrier$(<$flavour>)? for $ordering {
61 #[inline]
62 fn run() {
63 // SAFETY: barrier methods are safe to call.
64 unsafe { bindings::$binding() };
65 }
66 }
67 };
68}
69
70define_barrier!(Full, mb);
71define_barrier!(Read, rmb);
72define_barrier!(Write, wmb);
73define_barrier!([Dma] Full, dma_mb);
74define_barrier!([Dma] Read, dma_rmb);
75define_barrier!([Dma] Write, dma_wmb);
76define_barrier!([Smp] Full, smp_mb);
77define_barrier!([Smp] Read, smp_rmb);
78define_barrier!([Smp] Write, smp_wmb);
79
80/// Memory barrier.
81///
82/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
83///
84/// The specific forms of reordering can be specified using the parameter.
85/// - `mb(Read)` provides a read-read barrier.
86/// - `mb(Write)` provides a write-write barrier.
87/// - `mb(Full)` provides a full barrier.
88///
89/// # Examples
90///
91/// ```
92/// # use kernel::sync::barrier::*;
93/// mb(Read);
94/// mb(Write);
95/// mb(Full);
96/// ```
97#[inline]
98#[doc(alias = "rmb")]
99#[doc(alias = "wmb")]
100pub fn mb<T: MemoryBarrier>(_: T) {
101 T::run()
102}
103
104/// Memory barrier between CPUs.
105///
106/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
107/// Does not prevent re-ordering with respect to other bus-mastering devices.
108///
109/// See [`mb`] for usage.
110#[inline]
111#[doc(alias = "smp_rmb")]
112#[doc(alias = "smp_wmb")]
113pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) {
114 if cfg!(CONFIG_SMP) {
115 T::run()
116 } else {
117 barrier()
118 }
119}
120
121/// Memory barrier between local CPU and bus-mastering devices.
122///
123/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
124/// Does not prevent re-ordering with respect to other CPUs.
125///
126/// See [`mb`] for usage.
127#[inline]
128#[doc(alias = "dma_rmb")]
129#[doc(alias = "dma_wmb")]
130pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) {
131 T::run()
132}