1use core::cell::UnsafeCell;
4use core::ops::{Deref, DerefMut};
5use core::sync::atomic::{AtomicU32, Ordering};
6
7#[cfg(all(target_arch = "aarch64", target_os = "none"))]
8use crate::raw::{nr, syscall4, syscall6};
9use crate::time::Timespec;
10
11pub const FUTEX_WAIT_PRIVATE: i32 = 128;
13pub const FUTEX_WAKE_PRIVATE: i32 = 129;
15
16pub fn futex_wait(word: &AtomicU32, expected: u32, timeout: Option<&Timespec>) {
18 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
19 {
20 let ts_ptr = timeout.map_or(0, |t| core::ptr::from_ref(t) as usize);
21 let _ = unsafe {
23 syscall6(
24 nr::FUTEX,
25 core::ptr::from_ref(word) as usize,
26 FUTEX_WAIT_PRIVATE as usize,
27 expected as usize,
28 ts_ptr,
29 0,
30 0,
31 )
32 };
33 }
34 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
35 {
36 let _ = (word, expected, timeout);
37 core::hint::spin_loop();
38 }
39}
40
41pub fn futex_wake_one(word: &AtomicU32) -> usize {
43 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
44 {
45 unsafe {
47 syscall4(
48 nr::FUTEX,
49 core::ptr::from_ref(word) as usize,
50 FUTEX_WAKE_PRIVATE as usize,
51 1,
52 0,
53 )
54 .unwrap_or(0)
55 }
56 }
57 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
58 {
59 let _ = word;
60 0
61 }
62}
63
64pub fn futex_wake_all(word: &AtomicU32) -> usize {
66 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
67 {
68 unsafe {
70 syscall4(
71 nr::FUTEX,
72 core::ptr::from_ref(word) as usize,
73 FUTEX_WAKE_PRIVATE as usize,
74 i32::MAX as usize,
75 0,
76 )
77 .unwrap_or(0)
78 }
79 }
80 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
81 {
82 let _ = word;
83 0
84 }
85}
86
87const UNLOCKED: u32 = 0;
88const LOCKED_UNCONTENDED: u32 = 1;
89const LOCKED_CONTENDED: u32 = 2;
90
91pub struct FutexMutex<T: ?Sized> {
93 state: AtomicU32,
94 value: UnsafeCell<T>,
95}
96
97unsafe impl<T: ?Sized + Send> Sync for FutexMutex<T> {}
99unsafe impl<T: ?Sized + Send> Send for FutexMutex<T> {}
101
102impl<T> FutexMutex<T> {
103 #[must_use]
105 pub const fn new(value: T) -> Self {
106 Self {
107 state: AtomicU32::new(UNLOCKED),
108 value: UnsafeCell::new(value),
109 }
110 }
111
112 #[must_use]
114 pub fn into_inner(self) -> T {
115 self.value.into_inner()
116 }
117}
118
119impl<T: ?Sized> FutexMutex<T> {
120 #[must_use]
122 pub fn try_lock(&self) -> Option<FutexMutexGuard<'_, T>> {
123 if self
124 .state
125 .compare_exchange(
126 UNLOCKED,
127 LOCKED_UNCONTENDED,
128 Ordering::Acquire,
129 Ordering::Relaxed,
130 )
131 .is_ok()
132 {
133 Some(FutexMutexGuard { mutex: self })
134 } else {
135 None
136 }
137 }
138
139 #[must_use]
141 pub fn lock(&self) -> FutexMutexGuard<'_, T> {
142 if self
143 .state
144 .compare_exchange(
145 UNLOCKED,
146 LOCKED_UNCONTENDED,
147 Ordering::Acquire,
148 Ordering::Relaxed,
149 )
150 .is_err()
151 {
152 self.lock_contended();
153 }
154 FutexMutexGuard { mutex: self }
155 }
156
157 #[cfg(target_arch = "aarch64")]
158 #[inline]
159 fn wait_monitor_pause(&self) {
160 let ptr = self.state.as_ptr();
161 unsafe {
164 core::arch::asm!(
165 "sevl",
166 "mov {cnt:w}, #4",
167 "1:",
168 "ldaxr {val:w}, [{ptr}]",
169 "cmp {val:w}, #1",
170 "b.ne 2f",
171 "wfe",
172 "subs {cnt:w}, {cnt:w}, #1",
173 "b.ne 1b",
174 "2:",
175 "clrex",
176 ptr = in(reg) ptr,
177 cnt = out(reg) _,
178 val = out(reg) _,
179 options(nostack),
180 );
181 }
182 }
183
184 #[cold]
185 fn lock_contended(&self) {
186 let mut c = self.state.load(Ordering::Relaxed);
187 #[cfg(target_arch = "aarch64")]
188 {
189 if c == LOCKED_UNCONTENDED {
190 self.wait_monitor_pause();
191 c = self.state.load(Ordering::Relaxed);
192 }
193 if c == UNLOCKED {
194 match self.state.compare_exchange(
195 UNLOCKED,
196 LOCKED_UNCONTENDED,
197 Ordering::Acquire,
198 Ordering::Relaxed,
199 ) {
200 Ok(_) => return,
201 Err(prev) => c = prev,
202 }
203 }
204 }
205 #[cfg(not(target_arch = "aarch64"))]
206 for _ in 0..32 {
207 if c == UNLOCKED {
208 match self.state.compare_exchange(
209 UNLOCKED,
210 LOCKED_UNCONTENDED,
211 Ordering::Acquire,
212 Ordering::Relaxed,
213 ) {
214 Ok(_) => return,
215 Err(prev) => c = prev,
216 }
217 }
218 if c != LOCKED_UNCONTENDED {
219 break;
220 }
221 core::hint::spin_loop();
222 c = self.state.load(Ordering::Relaxed);
223 }
224 loop {
225 if c == LOCKED_CONTENDED
226 || self
227 .state
228 .compare_exchange(
229 LOCKED_UNCONTENDED,
230 LOCKED_CONTENDED,
231 Ordering::Relaxed,
232 Ordering::Relaxed,
233 )
234 .is_err()
235 {
236 futex_wait(&self.state, LOCKED_CONTENDED, None);
237 }
238 if self.state.swap(LOCKED_CONTENDED, Ordering::Acquire) == UNLOCKED {
239 return;
240 }
241 c = LOCKED_CONTENDED;
242 }
243 }
244
245 pub unsafe fn raw_unlock(&self) {
252 if self.state.swap(UNLOCKED, Ordering::Release) == LOCKED_CONTENDED {
253 let _ = futex_wake_one(&self.state);
254 }
255 }
256
257 #[must_use]
259 pub fn get_mut(&mut self) -> &mut T {
260 self.value.get_mut()
261 }
262}
263
264pub struct FutexMutexGuard<'a, T: ?Sized> {
266 mutex: &'a FutexMutex<T>,
267}
268
269impl<T: ?Sized> Deref for FutexMutexGuard<'_, T> {
270 type Target = T;
271
272 fn deref(&self) -> &Self::Target {
273 unsafe { &*self.mutex.value.get() }
275 }
276}
277
278impl<T: ?Sized> DerefMut for FutexMutexGuard<'_, T> {
279 fn deref_mut(&mut self) -> &mut Self::Target {
280 unsafe { &mut *self.mutex.value.get() }
282 }
283}
284
285impl<T: ?Sized> Drop for FutexMutexGuard<'_, T> {
286 fn drop(&mut self) {
287 unsafe {
289 self.mutex.raw_unlock();
290 }
291 }
292}
293
294#[cfg(test)]
295mod tests {
296 use super::*;
297
298 #[test]
299 fn futex_mutex_lock_try_lock_and_move_by_value() {
300 let m = FutexMutex::new(10u64);
301 {
302 let mut g = m.lock();
303 assert!(m.try_lock().is_none());
304 *g += 5;
305 }
306 let mut moved = m;
307 assert_eq!(*moved.get_mut(), 15);
308 *moved.try_lock().unwrap() += 27;
309 assert_eq!(moved.into_inner(), 42);
310 }
311}