Skip to main content

libc_rs/
sync.rs

1//! Pure-Rust futex synchronization primitives (`FutexMutex`) with zero `pthreads` dependencies.
2
3use core::cell::UnsafeCell;
4use core::ops::{Deref, DerefMut};
5use core::sync::atomic::{AtomicU32, Ordering};
6
7#[cfg(all(target_arch = "aarch64", target_os = "none"))]
8use crate::raw::{nr, syscall4, syscall6};
9use crate::time::Timespec;
10
11/// Private futex wait operation (`FUTEX_WAIT | FUTEX_PRIVATE_FLAG`).
12pub const FUTEX_WAIT_PRIVATE: i32 = 128;
13/// Private futex wake operation (`FUTEX_WAKE | FUTEX_PRIVATE_FLAG`).
14pub const FUTEX_WAKE_PRIVATE: i32 = 129;
15
16/// Waits on futex `word` while its value equals `expected`, with an optional relative `timeout`.
17pub fn futex_wait(word: &AtomicU32, expected: u32, timeout: Option<&Timespec>) {
18    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
19    {
20        let ts_ptr = timeout.map_or(0, |t| core::ptr::from_ref(t) as usize);
21        // SAFETY: `word` is a valid 4-byte aligned `AtomicU32` and `ts_ptr` is null or valid.
22        let _ = unsafe {
23            syscall6(
24                nr::FUTEX,
25                core::ptr::from_ref(word) as usize,
26                FUTEX_WAIT_PRIVATE as usize,
27                expected as usize,
28                ts_ptr,
29                0,
30                0,
31            )
32        };
33    }
34    #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
35    {
36        let _ = (word, expected, timeout);
37        core::hint::spin_loop();
38    }
39}
40
41/// Wakes at most one waiter blocked on futex `word`.
42pub fn futex_wake_one(word: &AtomicU32) -> usize {
43    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
44    {
45        // SAFETY: `word` is a valid 4-byte aligned `AtomicU32`.
46        unsafe {
47            syscall4(
48                nr::FUTEX,
49                core::ptr::from_ref(word) as usize,
50                FUTEX_WAKE_PRIVATE as usize,
51                1,
52                0,
53            )
54            .unwrap_or(0)
55        }
56    }
57    #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
58    {
59        let _ = word;
60        0
61    }
62}
63
64/// Wakes all waiters blocked on futex `word`.
65pub fn futex_wake_all(word: &AtomicU32) -> usize {
66    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
67    {
68        // SAFETY: `word` is a valid 4-byte aligned `AtomicU32`.
69        unsafe {
70            syscall4(
71                nr::FUTEX,
72                core::ptr::from_ref(word) as usize,
73                FUTEX_WAKE_PRIVATE as usize,
74                i32::MAX as usize,
75                0,
76            )
77            .unwrap_or(0)
78        }
79    }
80    #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
81    {
82        let _ = word;
83        0
84    }
85}
86
87const UNLOCKED: u32 = 0;
88const LOCKED_UNCONTENDED: u32 = 1;
89const LOCKED_CONTENDED: u32 = 2;
90
91/// Move-safe, zero-allocation 3-state futex mutex (`0` = unlocked, `1` = locked, `2` = contended).
92pub struct FutexMutex<T: ?Sized> {
93    state: AtomicU32,
94    value: UnsafeCell<T>,
95}
96
97// SAFETY: `FutexMutex` serializes access to `T` across threads via acquire/release operations.
98unsafe impl<T: ?Sized + Send> Sync for FutexMutex<T> {}
99// SAFETY: `FutexMutex` can be sent across threads when its payload `T` is `Send`.
100unsafe impl<T: ?Sized + Send> Send for FutexMutex<T> {}
101
102impl<T> FutexMutex<T> {
103    /// Creates a new unlocked [`FutexMutex`] wrapping `value`.
104    #[must_use]
105    pub const fn new(value: T) -> Self {
106        Self {
107            state: AtomicU32::new(UNLOCKED),
108            value: UnsafeCell::new(value),
109        }
110    }
111
112    /// Consumes the mutex and returns its inner value.
113    #[must_use]
114    pub fn into_inner(self) -> T {
115        self.value.into_inner()
116    }
117}
118
119impl<T: ?Sized> FutexMutex<T> {
120    /// Attempts to acquire the mutex without blocking.
121    #[must_use]
122    pub fn try_lock(&self) -> Option<FutexMutexGuard<'_, T>> {
123        if self
124            .state
125            .compare_exchange(
126                UNLOCKED,
127                LOCKED_UNCONTENDED,
128                Ordering::Acquire,
129                Ordering::Relaxed,
130            )
131            .is_ok()
132        {
133            Some(FutexMutexGuard { mutex: self })
134        } else {
135            None
136        }
137    }
138
139    /// Acquires the mutex, spinning briefly before blocking via futex wait under contention.
140    #[must_use]
141    pub fn lock(&self) -> FutexMutexGuard<'_, T> {
142        if self
143            .state
144            .compare_exchange(
145                UNLOCKED,
146                LOCKED_UNCONTENDED,
147                Ordering::Acquire,
148                Ordering::Relaxed,
149            )
150            .is_err()
151        {
152            self.lock_contended();
153        }
154        FutexMutexGuard { mutex: self }
155    }
156
157    #[cfg(target_arch = "aarch64")]
158    #[inline]
159    fn wait_monitor_pause(&self) {
160        let ptr = self.state.as_ptr();
161        // SAFETY: `ptr` points to the valid, 4-byte aligned `AtomicU32` futex word.
162        // Uses `sevl` + bounded 4-round `ldaxr`/`wfe` exclusive monitor wait and `clrex` on exit.
163        unsafe {
164            core::arch::asm!(
165                "sevl",
166                "mov {cnt:w}, #4",
167                "1:",
168                "ldaxr {val:w}, [{ptr}]",
169                "cmp {val:w}, #1",
170                "b.ne 2f",
171                "wfe",
172                "subs {cnt:w}, {cnt:w}, #1",
173                "b.ne 1b",
174                "2:",
175                "clrex",
176                ptr = in(reg) ptr,
177                cnt = out(reg) _,
178                val = out(reg) _,
179                options(nostack),
180            );
181        }
182    }
183
184    #[cold]
185    fn lock_contended(&self) {
186        let mut c = self.state.load(Ordering::Relaxed);
187        #[cfg(target_arch = "aarch64")]
188        {
189            if c == LOCKED_UNCONTENDED {
190                self.wait_monitor_pause();
191                c = self.state.load(Ordering::Relaxed);
192            }
193            if c == UNLOCKED {
194                match self.state.compare_exchange(
195                    UNLOCKED,
196                    LOCKED_UNCONTENDED,
197                    Ordering::Acquire,
198                    Ordering::Relaxed,
199                ) {
200                    Ok(_) => return,
201                    Err(prev) => c = prev,
202                }
203            }
204        }
205        #[cfg(not(target_arch = "aarch64"))]
206        for _ in 0..32 {
207            if c == UNLOCKED {
208                match self.state.compare_exchange(
209                    UNLOCKED,
210                    LOCKED_UNCONTENDED,
211                    Ordering::Acquire,
212                    Ordering::Relaxed,
213                ) {
214                    Ok(_) => return,
215                    Err(prev) => c = prev,
216                }
217            }
218            if c != LOCKED_UNCONTENDED {
219                break;
220            }
221            core::hint::spin_loop();
222            c = self.state.load(Ordering::Relaxed);
223        }
224        loop {
225            if c == LOCKED_CONTENDED
226                || self
227                    .state
228                    .compare_exchange(
229                        LOCKED_UNCONTENDED,
230                        LOCKED_CONTENDED,
231                        Ordering::Relaxed,
232                        Ordering::Relaxed,
233                    )
234                    .is_err()
235            {
236                futex_wait(&self.state, LOCKED_CONTENDED, None);
237            }
238            if self.state.swap(LOCKED_CONTENDED, Ordering::Acquire) == UNLOCKED {
239                return;
240            }
241            c = LOCKED_CONTENDED;
242        }
243    }
244
245    /// Releases the mutex without a guard (used by [`FutexMutexGuard`] drop).
246    ///
247    /// # Safety
248    ///
249    /// The caller must currently hold the lock on `self` and must not access the protected
250    /// value after calling `raw_unlock` without re-acquiring the lock.
251    pub unsafe fn raw_unlock(&self) {
252        if self.state.swap(UNLOCKED, Ordering::Release) == LOCKED_CONTENDED {
253            let _ = futex_wake_one(&self.state);
254        }
255    }
256
257    /// Returns a mutable reference to the underlying data when holding exclusive `&mut self`.
258    #[must_use]
259    pub fn get_mut(&mut self) -> &mut T {
260        self.value.get_mut()
261    }
262}
263
264/// RAII guard for [`FutexMutex`].
265pub struct FutexMutexGuard<'a, T: ?Sized> {
266    mutex: &'a FutexMutex<T>,
267}
268
269impl<T: ?Sized> Deref for FutexMutexGuard<'_, T> {
270    type Target = T;
271
272    fn deref(&self) -> &Self::Target {
273        // SAFETY: The guard holds exclusive access to the mutex value.
274        unsafe { &*self.mutex.value.get() }
275    }
276}
277
278impl<T: ?Sized> DerefMut for FutexMutexGuard<'_, T> {
279    fn deref_mut(&mut self) -> &mut Self::Target {
280        // SAFETY: The guard holds exclusive mutable access to the mutex value.
281        unsafe { &mut *self.mutex.value.get() }
282    }
283}
284
285impl<T: ?Sized> Drop for FutexMutexGuard<'_, T> {
286    fn drop(&mut self) {
287        // SAFETY: `self` owns the lock and is being dropped.
288        unsafe {
289            self.mutex.raw_unlock();
290        }
291    }
292}
293
294#[cfg(test)]
295mod tests {
296    use super::*;
297
298    #[test]
299    fn futex_mutex_lock_try_lock_and_move_by_value() {
300        let m = FutexMutex::new(10u64);
301        {
302            let mut g = m.lock();
303            assert!(m.try_lock().is_none());
304            *g += 5;
305        }
306        let mut moved = m;
307        assert_eq!(*moved.get_mut(), 15);
308        *moved.try_lock().unwrap() += 27;
309        assert_eq!(moved.into_inner(), 42);
310    }
311}