linux_futex/
lib.rs

1//! Futex: A Linux-specific fast user-space locking primitive.
2//!
3//! This crate provides easy-to-use wrappers around the not-so-easy-to-use `SYS_futex` Linux syscall.
4//!
5//! The documentation of Linux's futexes can be found in the
6//! [relevant man page](http://man7.org/linux/man-pages/man2/futex.2.html).
7//! The most important details are also explained in the documentation of this crate.
8//!
9//! The two main types of this crate are [`Futex`] and [`PiFutex`], which are
10//! simply wrappers containing an [`AtomicU32`] exposing all the futex
11//! operations Linux can apply to them.
12//!
13//! Existing [`AtomicU32`]s can be used as futexes through [`AsFutex`]
14//! without changing their type.
15
16mod errors;
17mod scope;
18mod sys;
19mod timeout;
20
21pub mod op;
22
23use op::OpAndCmp;
24use std::marker::PhantomData;
25use std::sync::atomic::AtomicU32;
26use std::time::Duration;
27use sys::{Error, FutexCall};
28use timeout::as_timespec;
29
30pub use errors::*;
31pub use scope::{Private, Scope, Shared};
32pub use timeout::Timeout;
33
34/// A Linux-specific fast user-space locking primitive.
35///
36/// `Futex<Private>` may only be used from the same address space (the same
37/// process) and is faster than a `Futex<Shared>`, which may be used accross
38/// address spaces (processes).
39#[repr(transparent)]
40pub struct Futex<Scope> {
41	pub value: AtomicU32,
42	phantom: PhantomData<Scope>,
43}
44
45/// A Linux-specific priority inheriting fast user-space locking primitive.
46///
47/// Unlike with a regular [`Futex`], the value of a [`PiFutex`] has meaning
48/// to the Linux kernel, taking away some flexibility. User-space must follow
49/// the assumed protocol to allow the kernel to properly implement priority
50/// inheritance.
51///
52/// See the *Priority-inheritance futexes* section of [the Linux futex man
53/// page](http://man7.org/linux/man-pages/man2/futex.2.html) for details.
54///
55/// `PiFutex<Private>` may only be used from the same address space (the same
56/// process) and is faster than a `PiFutex<Shared>`, which may be used accross
57/// address spaces (processes).
58#[repr(transparent)]
59pub struct PiFutex<Scope> {
60	pub value: AtomicU32,
61	phantom: PhantomData<Scope>,
62}
63
64/// Use any [`AtomicU32`] as [`Futex`] or [`PiFutex`].
65///
66/// This also allows you to convert between a [`Futex`] and a [`PiFutex`] or
67/// between [`Private`] and [`Shared`] futexes if you ever need that, as they
68/// expose their internal [`AtomicU32`] through `.value`.
69pub trait AsFutex<S> {
70	fn as_futex(&self) -> &Futex<S>;
71	fn as_pi_futex(&self) -> &PiFutex<S>;
72}
73
74impl<S> AsFutex<S> for AtomicU32 {
75	#[must_use]
76	#[inline]
77	fn as_futex(&self) -> &Futex<S> {
78		unsafe { std::mem::transmute(self) }
79	}
80	#[inline]
81	#[must_use]
82	fn as_pi_futex(&self) -> &PiFutex<S> {
83		unsafe { std::mem::transmute(self) }
84	}
85}
86
87impl<S> Futex<S> {
88	/// Create a new [`Futex`] with an initial value.
89	#[inline]
90	pub const fn new(value: u32) -> Self {
91		Self {
92			value: AtomicU32::new(value),
93			phantom: PhantomData,
94		}
95	}
96}
97
98impl<S> PiFutex<S> {
99	/// Create a new [`PiFutex`] with an initial value.
100	#[inline]
101	pub const fn new(value: u32) -> Self {
102		Self {
103			value: AtomicU32::new(value),
104			phantom: PhantomData,
105		}
106	}
107
108	/// The `FUTEX_WAITERS` bit that indicates there are threads waiting.
109	pub const WAITERS: u32 = 0x8000_0000;
110
111	/// The `FUTEX_OWNER_DIED` bit that indicates the owning thread died.
112	pub const OWNER_DIED: u32 = 0x4000_0000;
113
114	/// The bits that are used for storing the thread id (`FUTEX_TID_MASK`).
115	pub const TID_MASK: u32 = 0x3fffffff;
116}
117
118impl<S> Default for Futex<S> {
119	fn default() -> Self {
120		Self::new(0)
121	}
122}
123
124impl<S> Default for PiFutex<S> {
125	fn default() -> Self {
126		Self::new(0)
127	}
128}
129
130impl<S: Scope> Futex<S> {
131	/// Wait until this futex is awoken by a `wake` call.
132	///
133	/// The thread will only be sent to sleep if the futex's value matches the
134	/// expected value. Otherwise, it returns directly with [`WaitError::WrongValue`].
135	#[inline]
136	pub fn wait(&self, expected_value: u32) -> Result<(), WaitError> {
137		let r = unsafe {
138			FutexCall::new()
139				.futex_op(libc::FUTEX_WAIT + S::futex_flag())
140				.uaddr(&self.value)
141				.val(expected_value)
142				.call()
143		};
144		match r {
145			Err(Error(libc::EAGAIN)) => Err(WaitError::WrongValue),
146			Err(Error(libc::EINTR)) => Err(WaitError::Interrupted),
147			Err(e) => e.panic("FUTEX_WAIT"),
148			Ok(_) => Ok(()),
149		}
150	}
151
152	/// Wait until this futex is awoken by a `wake` call, or until the timeout expires.
153	///
154	/// The thread will only be sent to sleep if the futex's value matches the
155	/// expected value. Otherwise, it returns directly with [`TimedWaitError::WrongValue`].
156	///
157	/// If you want an absolute point in time as timeout, use
158	/// [`wait_bitset_until`][Futex::wait_bitset_until] instead, using a bitset of `!0`.
159	#[inline]
160	pub fn wait_for(&self, expected_value: u32, timeout: Duration) -> Result<(), TimedWaitError> {
161		let timeout = as_timespec(timeout);
162		let r = unsafe {
163			FutexCall::new()
164				.futex_op(libc::FUTEX_WAIT + S::futex_flag())
165				.uaddr(&self.value)
166				.val(expected_value)
167				.timeout(&timeout)
168				.call()
169		};
170		match r {
171			Err(Error(libc::EAGAIN)) => Err(TimedWaitError::WrongValue),
172			Err(Error(libc::EINTR)) => Err(TimedWaitError::Interrupted),
173			Err(Error(libc::ETIMEDOUT)) => Err(TimedWaitError::TimedOut),
174			Err(e) => e.panic("FUTEX_WAIT"),
175			Ok(_) => Ok(()),
176		}
177	}
178
179	/// Wake up `n` waiters.
180	///
181	/// Returns the number of waiters that were woken up.
182	#[inline]
183	pub fn wake(&self, n: i32) -> i32 {
184		let r = unsafe {
185			FutexCall::new()
186				.futex_op(libc::FUTEX_WAKE + S::futex_flag())
187				.uaddr(&self.value)
188				.val(n as u32)
189				.call()
190		};
191		match r {
192			Err(e) => e.panic("FUTEX_WAKE"),
193			Ok(v) => v,
194		}
195	}
196
197	/// Wake up `n_wake` waiters, and requeue up to `n_requeue` waiters to another futex.
198	///
199	/// Returns the number of waiters that were woken up.
200	#[inline]
201	pub fn requeue(&self, n_wake: i32, to: &Futex<S>, n_requeue: i32) -> i32 {
202		let r = unsafe {
203			FutexCall::new()
204				.futex_op(libc::FUTEX_REQUEUE + S::futex_flag())
205				.uaddr(&self.value)
206				.uaddr2(&to.value)
207				.val(n_wake as u32)
208				.val2(n_requeue as u32)
209				.call()
210		};
211		match r {
212			Err(e) => e.panic("FUTEX_REQUEUE"),
213			Ok(v) => v,
214		}
215	}
216
217	/// Wake up `n_wake` waiters, and requeue up to `n_requeue` waiters to another futex.
218	///
219	/// The operation will only execute if the futex's value matches the
220	/// expected value. Otherwise, it returns directly with a [`WrongValueError`].
221	///
222	/// Returns the total number of waiters that were woken up or requeued to the other futex.
223	#[inline]
224	pub fn cmp_requeue(
225		&self,
226		expected_value: u32,
227		n_wake: i32,
228		to: &Futex<S>,
229		n_requeue: i32,
230	) -> Result<i32, WrongValueError> {
231		let r = unsafe {
232			FutexCall::new()
233				.futex_op(libc::FUTEX_CMP_REQUEUE + S::futex_flag())
234				.uaddr(&self.value)
235				.uaddr2(&to.value)
236				.val(n_wake as u32)
237				.val2(n_requeue as u32)
238				.val3(expected_value)
239				.call()
240		};
241		match r {
242			Err(Error(libc::EAGAIN)) => Err(WrongValueError::WrongValue),
243			Err(e) => e.panic("FUTEX_CMP_REQUEUE"),
244			Ok(v) => Ok(v),
245		}
246	}
247
248	/// Wait until this futex is awoken by a `wake` call matching a bitset.
249	///
250	/// - Calls to [`wake`][Futex::wake] will match any bitset.
251	/// - Calls to [`wake_bitset`][Futex::wake_bitset] will match if at least one 1-bit matches.
252	///
253	/// The thread will only be sent to sleep if the futex's value matches the
254	/// expected value. Otherwise, it returns directly with [`WaitError::WrongValue`].
255	#[inline]
256	pub fn wait_bitset(&self, expected_value: u32, bitset: u32) -> Result<(), WaitError> {
257		let r = unsafe {
258			FutexCall::new()
259				.uaddr(&self.value)
260				.futex_op(libc::FUTEX_WAIT_BITSET + S::futex_flag())
261				.val(expected_value)
262				.val3(bitset)
263				.call()
264		};
265		match r {
266			Err(Error(libc::EAGAIN)) => Err(WaitError::WrongValue),
267			Err(Error(libc::EINTR)) => Err(WaitError::Interrupted),
268			Err(e) => e.panic("FUTEX_WAIT_BITSET"),
269			Ok(_) => Ok(()),
270		}
271	}
272
273	/// Wait until this futex is awoken by a `wake` call matching a bitset, or until the timeout expires.
274	///
275	/// - Calls to [`wake`][Futex::wake] will match any bitset.
276	/// - Calls to [`wake_bitset`][Futex::wake_bitset] will match if at least one 1-bit matches.
277	///
278	/// The thread will only be sent to sleep if the futex's value matches the
279	/// expected value. Otherwise, it returns directly with [`TimedWaitError::WrongValue`].
280	#[inline]
281	pub fn wait_bitset_until(
282		&self,
283		expected_value: u32,
284		bitset: u32,
285		timeout: impl Timeout,
286	) -> Result<(), TimedWaitError> {
287		let timeout = timeout.as_timespec();
288		let r = unsafe {
289			FutexCall::new()
290				.uaddr(&self.value)
291				.futex_op(libc::FUTEX_WAIT_BITSET + timeout.0 + S::futex_flag())
292				.val(expected_value)
293				.val3(bitset)
294				.timeout(&timeout.1)
295				.call()
296		};
297		match r {
298			Err(Error(libc::EAGAIN)) => Err(TimedWaitError::WrongValue),
299			Err(Error(libc::EINTR)) => Err(TimedWaitError::Interrupted),
300			Err(Error(libc::ETIMEDOUT)) => Err(TimedWaitError::TimedOut),
301			Err(e) => e.panic("FUTEX_WAIT_BITSET"),
302			Ok(_) => Ok(()),
303		}
304	}
305
306	/// Wake up `n` waiters matching a bitset.
307	///
308	/// - Waiters waiting using [`wait`][Futex::wait] are always woken up,
309	///   regardless of the bitset.
310	/// - Waiters waiting using [`wait_bitset`][Futex::wait_bitset] are woken up
311	///   if they match at least one 1-bit.
312	///
313	/// Returns the number of waiters that were woken up.
314	#[inline]
315	pub fn wake_bitset(&self, n: i32, bitset: u32) -> i32 {
316		let r = unsafe {
317			FutexCall::new()
318				.futex_op(libc::FUTEX_WAKE_BITSET + S::futex_flag())
319				.uaddr(&self.value)
320				.val(n as u32)
321				.val3(bitset)
322				.call()
323		};
324		match r {
325			Err(e) => e.panic("FUTEX_WAKE_BITSET"),
326			Ok(v) => v,
327		}
328	}
329
330	/// Wake up `n` waiters, and conditionally `n2` waiters on another futex after modifying it.
331	///
332	/// This operation first applies an [operation][`op::Op`] to the second futex while remembering its old value,
333	/// then wakes up `n` waiters on the first futex, and finally wakes `n2` waiters on the second futex if
334	/// its old value matches [a condition][`op::Cmp`]. This all happens atomically.
335	///
336	/// Returns the total number of waiters that were woken up on either futex.
337	#[inline]
338	pub fn wake_op(&self, n: i32, second: &Futex<S>, op: OpAndCmp, n2: i32) -> i32 {
339		let r = unsafe {
340			FutexCall::new()
341				.futex_op(libc::FUTEX_WAKE_OP + S::futex_flag())
342				.uaddr(&self.value)
343				.uaddr2(&second.value)
344				.val(n as u32)
345				.val2(n2 as u32)
346				.val3(op.raw_bits())
347				.call()
348		};
349		match r {
350			Err(e) => e.panic("FUTEX_WAKE_OP"),
351			Ok(v) => v,
352		}
353	}
354
355	/// Wake up one waiter, and requeue up to `n_requeue` to a [`PiFutex`].
356	///
357	/// Only requeues waiters that are blocked by [`wait_requeue_pi`][Futex::wait_requeue_pi]
358	/// or [`wait_requeue_pi_until`][Futex::wait_requeue_pi_until].
359	/// The [`PiFutex`] must be the same as the one the waiters are waiting to be requeued to.
360	///
361	/// The number of waiters to wake cannot be chosen and is always 1.
362	///
363	/// Returns the total number of waiters that were woken up or requeued to the other futex.
364	#[inline]
365	pub fn cmp_requeue_pi(
366		&self,
367		expected_value: i32,
368		to: &PiFutex<S>,
369		n_requeue: i32,
370	) -> Result<i32, TryAgainError> {
371		let r = unsafe {
372			FutexCall::new()
373				.futex_op(libc::FUTEX_CMP_REQUEUE_PI + S::futex_flag())
374				.uaddr(&self.value)
375				.uaddr2(&to.value)
376				.val(1)
377				.val2(n_requeue as u32)
378				.val3(expected_value as u32)
379				.call()
380		};
381		match r {
382			Err(Error(libc::EAGAIN)) => Err(TryAgainError::TryAgain),
383			Err(e) => e.panic("FUTEX_CMP_REQUEUE_PI"),
384			Ok(v) => Ok(v),
385		}
386	}
387
388	/// Wait until this futex is awoken after potentially being requeued to a [`PiFutex`].
389	///
390	/// A call to [`cmp_requeue_pi`][Futex::cmp_requeue_pi] will requeue this waiter to
391	/// the [`PiFutex`]. The call must refer to the same [`PiFutex`].
392	///
393	/// A call to [`wake`][Futex::wake] (or [`wake_bitset`][Futex::wake_bitset]) will
394	/// wake this thread without requeueing. This results in an [`RequeuePiError::TryAgain`].
395	#[inline]
396	pub fn wait_requeue_pi(
397		&self,
398		expected_value: u32,
399		second: &PiFutex<S>,
400	) -> Result<(), RequeuePiError> {
401		let r = unsafe {
402			FutexCall::new()
403				.futex_op(libc::FUTEX_WAIT_REQUEUE_PI + S::futex_flag())
404				.uaddr(&self.value)
405				.uaddr2(&second.value)
406				.val(expected_value)
407				.call()
408		};
409		match r {
410			Err(Error(libc::EAGAIN)) => Err(RequeuePiError::TryAgain),
411			Err(e) => e.panic("FUTEX_WAIT_REQUEUE_PI"),
412			Ok(_) => Ok(()),
413		}
414	}
415
416	/// Wait until this futex is awoken after potentially being requeued to a [`PiFutex`], or until the timeout expires.
417	///
418	/// A call to [`cmp_requeue_pi`][Futex::cmp_requeue_pi] will requeue this waiter to
419	/// the [`PiFutex`]. The call must refer to the same [`PiFutex`].
420	///
421	/// A call to [`wake`][Futex::wake] (or [`wake_bitset`][Futex::wake_bitset]) will
422	/// wake this thread without requeueing. This results in an [`TimedRequeuePiError::TryAgain`].
423	#[inline]
424	pub fn wait_requeue_pi_until(
425		&self,
426		expected_value: u32,
427		second: &PiFutex<S>,
428		timeout: impl Timeout,
429	) -> Result<(), TimedRequeuePiError> {
430		let timeout = timeout.as_timespec();
431		let r = unsafe {
432			FutexCall::new()
433				.futex_op(libc::FUTEX_WAIT_REQUEUE_PI + timeout.0 + S::futex_flag())
434				.uaddr(&self.value)
435				.uaddr2(&second.value)
436				.val(expected_value)
437				.timeout(&timeout.1)
438				.call()
439		};
440		match r {
441			Err(Error(libc::EAGAIN)) => Err(TimedRequeuePiError::TryAgain),
442			Err(Error(libc::ETIMEDOUT)) => Err(TimedRequeuePiError::TimedOut),
443			Err(e) => e.panic("FUTEX_WAIT_REQUEUE_PI"),
444			Ok(_) => Ok(()),
445		}
446	}
447}
448
449impl<S: Scope> PiFutex<S> {
450	/// See `FUTEX_LOCK_PI` in the [Linux futex man page](http://man7.org/linux/man-pages/man2/futex.2.html).
451	#[inline]
452	pub fn lock_pi(&self) -> Result<(), TryAgainError> {
453		let r = unsafe {
454			FutexCall::new()
455				.futex_op(libc::FUTEX_LOCK_PI + S::futex_flag())
456				.uaddr(&self.value)
457				.call()
458		};
459		match r {
460			Err(Error(libc::EAGAIN)) => Err(TryAgainError::TryAgain),
461			Err(e) => e.panic("FUTEX_LOCK_PI"),
462			Ok(_) => Ok(()),
463		}
464	}
465
466	/// See `FUTEX_LOCK_PI` in the [Linux futex man page](http://man7.org/linux/man-pages/man2/futex.2.html).
467	#[inline]
468	pub fn lock_pi_until(&self, timeout: impl Timeout) -> Result<(), TimedLockError> {
469		let (clock, timespec) = timeout.as_timespec();
470		let op = if clock == libc::FUTEX_CLOCK_REALTIME {
471			libc::FUTEX_LOCK_PI
472		} else {
473			// Only available since Linux 5.14.
474			libc::FUTEX_LOCK_PI2
475		};
476		let r = unsafe {
477			FutexCall::new()
478				.futex_op(op + S::futex_flag())
479				.uaddr(&self.value)
480				.timeout(&timespec)
481				.call()
482		};
483		match r {
484			Err(Error(libc::EAGAIN)) => Err(TimedLockError::TryAgain),
485			Err(Error(libc::ETIMEDOUT)) => Err(TimedLockError::TimedOut),
486			Err(e) if op == libc::FUTEX_LOCK_PI2 => e.panic("FUTEX_LOCK_PI2"),
487			Err(e) => e.panic("FUTEX_LOCK_PI"),
488			Ok(_) => Ok(()),
489		}
490	}
491
492	/// See `FUTEX_TRYLOCK_PI` in the [Linux futex man page](http://man7.org/linux/man-pages/man2/futex.2.html).
493	#[inline]
494	pub fn trylock_pi(&self) -> Result<(), TryAgainError> {
495		let r = unsafe {
496			FutexCall::new()
497				.futex_op(libc::FUTEX_TRYLOCK_PI + S::futex_flag())
498				.uaddr(&self.value)
499				.call()
500		};
501		match r {
502			Err(Error(libc::EAGAIN)) => Err(TryAgainError::TryAgain),
503			Err(e) => e.panic("FUTEX_LOCK_PI"),
504			Ok(_) => Ok(()),
505		}
506	}
507
508	/// See `FUTEX_UNLOCK_PI` in the [Linux futex man page](http://man7.org/linux/man-pages/man2/futex.2.html).
509	#[inline]
510	pub fn unlock_pi(&self) {
511		let r = unsafe {
512			FutexCall::new()
513				.futex_op(libc::FUTEX_UNLOCK_PI + S::futex_flag())
514				.uaddr(&self.value)
515				.call()
516		};
517		if let Err(e) = r {
518			e.panic("FUTEX_UNLOCK_PI");
519		}
520	}
521}
522
523impl<S> std::fmt::Debug for Futex<S> {
524	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
525		f.debug_struct("Futex")
526			.field("scope", &std::any::type_name::<S>())
527			.field("value", &self.value)
528			.finish()
529	}
530}
531
532impl<S> std::fmt::Debug for PiFutex<S> {
533	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
534		f.debug_struct("PiFutex")
535			.field("scope", &std::any::type_name::<S>())
536			.field("value", &self.value)
537			.finish()
538	}
539}
linux_futex/lib.rs

linux_futex/
lib.rs
