mem_barrier/lib.rs
1//! Cross-architecture, no-std memory barriers.
2//!
3//! When compiling with optimizations, the compiler may try to improve performance by reordering independent memory accesses and instructions.
4//! Modern CPUs use similar techniques for improving performance, such as out-of-order execution.
5//! Memory barriers affect both the compiler and the CPU by restricting reordering of certain memory operations across these barriers respective to other CPUs or devices, allowing proper communication with them.
6//!
7//! To insert a memory barrier, use the [`mem_barrier`] function.
8//!
9//! The memory barriers provided by this crate are similar to the [Linux kernel memory barriers].
10//! For more details on _that_ API, also see the [_Linux Kernel Memory Consistency Model_ (LKMM)].
11//!
12//! [Linux kernel memory barriers]: https://www.kernel.org/doc/html/latest/core-api/wrappers/memory-barriers.html
13//! [_Linux Kernel Memory Consistency Model_ (LKMM)]: https://www.kernel.org/doc/html/latest/dev-tools/lkmm/index.html
14//!
15//! # Examples
16//!
17//! ```
18//! use mem_barrier::{BarrierKind, BarrierType, mem_barrier};
19//!
20//! mem_barrier(BarrierKind::Mmio, BarrierType::General);
21//! ```
22//!
23//! # Supported architectures
24//!
25//! | Architecture | `target_arch` | Supported |
26//! | ------------ | ------------- | --------- |
27//! | AArch64 | `aarch64` | ✅ |
28//! | RISC-V RV32 | `riscv32` | ✅ |
29//! | RISC-V RV64 | `riscv64` | ✅ |
30//! | x86 | `x86` | ✅ |
31//! | x86-64 | `x86_64` | ✅ |
32//!
33//! # Cargo features
34//!
35//! This crate has the following Cargo features:
36//! - `nightly`—Disabled by default, this feature enables memory barrier implementations based on unstable, nightly-only Rust features.
37//! - `stdarch`—Enabled by default, this feature enables memory barrier implementations based on [`core::arch`] intrinsics.
38//! If available, these intrinsics replace the fallback implementations based on inline assembly.
39//!
40//! # Related crates
41//!
42//! Several crates provide alternative approaches to memory barriers:
43//!
44//! - [membarrier](https://docs.rs/membarrier) provides OS-based process-wide memory barriers.
45//! - [mbarrier](https://docs.rs/mbarrier) closely resembles Linux kernel memory barriers by providing `mb`, `rmb`, and `wmb`.
46//! It also provides `smp_mb`, `smp_rmb`, and `smp_wmb`, which either are the same as the non-SMP functions or fall back to compiler fences, depending on a crate feature.
47//! It does not, however, distinguish between MMIO-suitable, SMP-suitable, and DMA-suitable memory barriers.
48
49#![no_std]
50#![cfg_attr(
51 all(target_arch = "aarch64", feature = "stdarch", feature = "nightly"),
52 feature(stdarch_arm_barrier)
53)]
54
55mod arch;
56
57/// The kind of a memory barrier.
58///
59/// This enum determines the strength or flavor of the memory barrier.
60///
61/// # Current implementation
62///
63/// On x86, this does not affect instruction generation.
64#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)]
65#[non_exhaustive]
66pub enum BarrierKind {
67 /// MMIO.
68 ///
69 /// This is the strongest kind of barrier.
70 /// It enforces ordering on memory accesses as well as on MMIO-based device I/O.
71 ///
72 /// # Corresponding functions
73 ///
74 /// This kind of barrier corresponds to the _mandatory_ `mb`, `rmb`, and `wmb` Linux functions.
75 ///
76 /// # Current implementation
77 ///
78 /// On Arm, this runs a [DSB] instruction; see _[Data Synchronization Barrier]_.
79 ///
80 /// [DSB]: https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/DSB--Data-synchronization-barrier-
81 /// [Data Synchronization Barrier]: https://developer.arm.com/documentation/102336/0100/Data-Synchronization-Barrier
82 #[doc(alias = "mb")]
83 #[doc(alias = "rmb")]
84 #[doc(alias = "wmb")]
85 #[default]
86 Mmio,
87
88 /// SMP.
89 ///
90 /// This kind of barrier enforces ordering on memory across an SMP system.
91 /// It is also suitable to enforce ordering on memory shared from a single-CPU VM guest with an SMP host.
92 ///
93 /// # Corresponding functions
94 ///
95 /// This kind of barrier corresponds to the _VM-guest_-flavoured `virt_mb`, `virt_rmb`, and `virt_wmb` Linux functions.
96 /// Those functions are equivalent to the _SMP_-flavoured `smp_mb`, `smp_rmb`, and `smp_wmb` Linux functions when SMP support is turned on.
97 ///
98 /// Note that this kind of barrier does not change its behavior based on the build configuration.
99 ///
100 /// # Current implementation
101 ///
102 /// On Arm, this runs a [DMB] instruction; see _[Data Memory Barrier]_.
103 ///
104 /// [DMB]: https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/DMB--Data-memory-barrier-
105 /// [Data Memory Barrier]: https://developer.arm.com/documentation/102336/0100/Data-Memory-Barrier
106 #[doc(alias = "smp_mb")]
107 #[doc(alias = "smp_rmb")]
108 #[doc(alias = "smp_wmb")]
109 #[doc(alias = "virt_mb")]
110 #[doc(alias = "virt_rmb")]
111 #[doc(alias = "virt_wmb")]
112 Smp,
113
114 /// DMA.
115 ///
116 /// This kind of barrier enforces ordering on memory accessed by the CPU and DMA-capable devices.
117 ///
118 /// # Corresponding functions
119 ///
120 /// This kind of barrier corresponds to the _DMA_-flavoured `dma_mb`, `dma_rmb`, and `dma_wmb` Linux functions.
121 ///
122 /// # Current implementation
123 ///
124 /// On Arm, this runs a [DMB] instruction; see _[Data Memory Barrier]_.
125 ///
126 /// [DMB]: https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/DMB--Data-memory-barrier-
127 /// [Data Memory Barrier]: https://developer.arm.com/documentation/102336/0100/Data-Memory-Barrier
128 ///
129 /// # Examples
130 ///
131 /// This example is inspired by the [Linux `dma_rmb` and `dma_wmb` example].
132 ///
133 /// [Linux `dma_rmb` and `dma_wmb` example]: https://www.kernel.org/doc/html/latest/core-api/wrappers/memory-barriers.html
134 ///
135 /// ```
136 /// # struct Desc {
137 /// # device_owns_memory: bool,
138 /// # data: *mut [u8],
139 /// # }
140 /// #
141 /// # impl Desc {
142 /// # fn device_owns_memory(&self) -> bool {
143 /// # self.device_owns_memory
144 /// # }
145 /// #
146 /// # fn set_device_owns_memory(&mut self, device_owns_memory: bool) {
147 /// # self.device_owns_memory = device_owns_memory;
148 /// # }
149 /// #
150 /// # fn data(&self) -> *mut [u8] {
151 /// # self.data
152 /// # }
153 /// #
154 /// # fn set_data(&mut self, data: *mut [u8]) {
155 /// # self.data = data;
156 /// # }
157 /// # }
158 /// #
159 /// # struct Device;
160 /// #
161 /// # impl Device {
162 /// # fn notify(&mut self) {}
163 /// # }
164 /// #
165 /// # let data = core::ptr::slice_from_raw_parts_mut(core::ptr::null_mut(), 0);
166 /// # let mut desc = Desc { device_owns_memory: false, data };
167 /// # let mut device = Device;
168 /// # let mut read_data = data;
169 /// # let mut write_data = data;
170 /// #
171 /// use mem_barrier::{BarrierKind, BarrierType, mem_barrier};
172 ///
173 /// if !desc.device_owns_memory() {
174 /// // Don't read until we own the descriptor.
175 /// mem_barrier(BarrierKind::Dma, BarrierType::Read);
176 ///
177 /// // Read/modify data
178 /// read_data = desc.data();
179 /// desc.set_data(write_data);
180 ///
181 /// // Flush modifications.
182 /// mem_barrier(BarrierKind::Dma, BarrierType::Write);
183 ///
184 /// // Give the descriptor ownership back to the device.
185 /// desc.set_device_owns_memory(true);
186 ///
187 /// // Notify the device.
188 /// device.notify();
189 /// }
190 /// ```
191 #[doc(alias = "dma_mb")]
192 #[doc(alias = "dma_rmb")]
193 #[doc(alias = "dma_wmb")]
194 Dma,
195
196 /// Compiler.
197 ///
198 /// This kind of barrier does not run any CPU instructions.
199 /// Instead, it only prevents the compiler from moving memory accesses through the barrier.
200 ///
201 /// # Corresponding functions
202 ///
203 /// This kind of barrier corresponds to the `barrier` Linux function.
204 Compiler,
205}
206
207/// The type of a memory barrier.
208///
209/// This enum determines which type of memory accesses are ordered: read, write, or both (general).
210#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)]
211#[non_exhaustive]
212pub enum BarrierType {
213 /// General.
214 ///
215 /// This type of barrier orders both read and write memory accesses.
216 ///
217 /// # Corresponding functions
218 ///
219 /// This type of barrier corresponds to the `*_mb` family of Linux functions.
220 ///
221 /// # Current implementation
222 ///
223 /// On x86, this runs an [MFENCE] instruction.
224 ///
225 /// [MFENCE]: https://www.felixcloutier.com/x86/mfence
226 #[doc(alias = "mb")]
227 #[doc(alias = "smp_mb")]
228 #[doc(alias = "virt_mb")]
229 #[doc(alias = "dma_mb")]
230 #[default]
231 General,
232
233 /// Read.
234 ///
235 /// This type of barrier orders read memory accesses.
236 ///
237 /// # Corresponding functions
238 ///
239 /// This type of barrier corresponds to the `*_rmb` family of Linux functions.
240 ///
241 /// # Current implementation
242 ///
243 /// On x86, this runs an [LFENCE] instruction.
244 ///
245 /// [LFENCE]: https://www.felixcloutier.com/x86/lfence
246 #[doc(alias = "rmb")]
247 #[doc(alias = "smp_rmb")]
248 #[doc(alias = "virt_rmb")]
249 #[doc(alias = "dma_rmb")]
250 Read,
251
252 /// Write.
253 ///
254 /// This type of barrier orders write memory accesses and corresponds to the `*_wmb` family of Linux functions.
255 ///
256 /// # Current implementation
257 ///
258 /// On x86, this runs an [SFENCE] instruction.
259 ///
260 /// [SFENCE]: https://www.felixcloutier.com/x86/sfence
261 #[doc(alias = "wmb")]
262 #[doc(alias = "smp_wmb")]
263 #[doc(alias = "virt_wmb")]
264 #[doc(alias = "dma_wmb")]
265 Write,
266}
267
268/// A memory barrier.
269///
270/// This function runs the appropriate CPU instructions for enforcing memory ordering according to the provided [`BarrierKind`] and [`BarrierType`].
271///
272/// # Current implementation
273///
274/// On RISC-V, this runs a [FENCE] instruction.
275///
276/// [FENCE]: https://docs.riscv.org/reference/isa/unpriv/rv32.html#fence
277#[inline]
278pub fn mem_barrier(kind: BarrierKind, ty: BarrierType) {
279 let cpu_barrier_kind = match kind {
280 BarrierKind::Mmio => arch::CpuBarrierKind::Mmio,
281 BarrierKind::Smp => arch::CpuBarrierKind::Smp,
282 BarrierKind::Dma => arch::CpuBarrierKind::Dma,
283 BarrierKind::Compiler => {
284 compiler_barrier();
285 return;
286 }
287 };
288
289 arch::mem_barrier(cpu_barrier_kind, ty);
290}
291
292#[inline]
293fn compiler_barrier() {
294 // SAFETY: This asm invocation is empty.
295 unsafe {
296 core::arch::asm!("", options(preserves_flags, nostack));
297 }
298}
299
300#[cfg(test)]
301mod tests {
302 use super::*;
303
304 #[test]
305 fn test_memory_barrier() {
306 for kind in [
307 BarrierKind::Mmio,
308 BarrierKind::Smp,
309 BarrierKind::Dma,
310 BarrierKind::Compiler,
311 ] {
312 for ty in [BarrierType::General, BarrierType::Read, BarrierType::Write] {
313 mem_barrier(kind, ty);
314 }
315 }
316 }
317}