use core::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::thread::{self, Thread};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WaitStrategy {
StdPark,
Waitpkg,
}
impl WaitStrategy {
pub fn pick() -> Self {
if crate::cpu_info::has_waitpkg() {
Self::Waitpkg
} else {
Self::StdPark
}
}
}
#[derive(Debug)]
pub struct Parker {
thread: Thread,
shutdown: AtomicBool,
spin_rounds: u32,
wake_counter: AtomicU64,
wait_strategy: WaitStrategy,
}
impl Parker {
pub fn new(spin_rounds: u32) -> Self {
Self::with_strategy(spin_rounds, WaitStrategy::pick())
}
pub fn with_strategy(spin_rounds: u32, wait_strategy: WaitStrategy) -> Self {
Self {
thread: thread::current(),
shutdown: AtomicBool::new(false),
spin_rounds,
wake_counter: AtomicU64::new(0),
wait_strategy,
}
}
pub fn wait_strategy(&self) -> WaitStrategy {
self.wait_strategy
}
pub fn park_until<F: FnMut() -> bool>(&self, mut is_ready: F) -> bool {
let initial_wake = self.wake_counter.load(Ordering::Acquire);
for _ in 0..self.spin_rounds {
if self.shutdown.load(Ordering::Acquire) {
return false;
}
if is_ready() {
return true;
}
thread::yield_now();
}
if self.shutdown.load(Ordering::Acquire) {
return false;
}
if is_ready() {
return true;
}
match self.wait_strategy {
WaitStrategy::StdPark => {
thread::park();
}
WaitStrategy::Waitpkg => {
self.wait_via_waitpkg(initial_wake);
}
}
if self.shutdown.load(Ordering::Acquire) {
return false;
}
true
}
pub fn unpark(&self) {
self.wake_counter.fetch_add(1, Ordering::Release);
self.thread.unpark();
}
pub fn shutdown(&self) {
self.shutdown.store(true, Ordering::Release);
self.unpark();
}
#[cfg(target_arch = "x86_64")]
fn wait_via_waitpkg(&self, initial_wake: u64) {
const WAIT_DEADLINE_NS: u64 = 10_000_000;
const TSC_HZ_ESTIMATE: u64 = 2_500_000_000;
let cycles =
WAIT_DEADLINE_NS.saturating_mul(TSC_HZ_ESTIMATE / 1_000_000_000);
let now = unsafe { core::arch::x86_64::_rdtsc() };
let deadline = now.wrapping_add(cycles);
let lo = deadline as u32;
let hi = (deadline >> 32) as u32;
let addr = (&raw const self.wake_counter).cast::<u8>();
unsafe {
core::arch::asm!(
"umonitor rax",
in("rax") addr,
options(nostack, preserves_flags),
);
}
if self.wake_counter.load(Ordering::Acquire) != initial_wake {
return;
}
unsafe {
core::arch::asm!(
"umwait {hint:e}",
hint = in(reg) 1u32,
in("eax") lo,
in("edx") hi,
options(nostack),
);
}
}
#[cfg(not(target_arch = "x86_64"))]
fn wait_via_waitpkg(&self, _initial_wake: u64) {
thread::park();
}
pub fn is_shutdown(&self) -> bool {
self.shutdown.load(Ordering::Acquire)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::AtomicU32;
use std::thread;
use std::time::{Duration, Instant};
#[test]
fn park_until_returns_immediately_when_ready() {
let p = Parker::new(8);
let t0 = Instant::now();
let ok = p.park_until(|| true);
let elapsed = t0.elapsed();
assert!(ok);
assert!(elapsed < Duration::from_millis(10),
"park_until with ready=true must be fast; took {elapsed:?}");
}
#[test]
fn park_until_returns_false_on_shutdown() {
let p = Arc::new(Parker::new(8));
let p_signal = Arc::clone(&p);
let signal = thread::spawn(move || {
thread::sleep(Duration::from_millis(50));
p_signal.shutdown();
});
let ok = p.park_until(|| false);
signal.join().unwrap();
assert!(!ok, "park_until must return false after shutdown");
}
#[test]
fn park_until_wakes_on_unpark_from_other_thread() {
let ready = Arc::new(AtomicU32::new(0));
let ready_clone = Arc::clone(&ready);
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::new(8));
tx.send(Arc::clone(&p)).unwrap();
let t0 = Instant::now();
let ok = p.park_until(|| ready.load(Ordering::Acquire) == 1);
(ok, t0.elapsed())
});
let p_owner = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(50));
ready_clone.store(1, Ordering::Release);
p_owner.unpark();
let (ok, elapsed) = owner.join().unwrap();
assert!(ok, "park_until must return true after ready becomes true");
assert!(elapsed < Duration::from_millis(500),
"park_until took too long: {elapsed:?}");
}
#[test]
fn park_until_spin_floor_eight_rounds_no_park() {
let p = Parker::new(8);
let mut polls = 0u32;
let ok = p.park_until(|| {
polls += 1;
polls >= 3
});
assert!(ok);
assert_eq!(polls, 3);
}
#[test]
fn park_until_zero_spin_floor_goes_straight_to_park() {
let p = Parker::new(0);
let ok = p.park_until(|| true);
assert!(ok);
}
#[test]
fn unpark_before_park_is_observable_via_permit() {
let p = Arc::new(Parker::new(0)); let ready = Arc::new(AtomicU32::new(0));
let p_clone = Arc::clone(&p);
let ready_clone = Arc::clone(&ready);
let signal = thread::spawn(move || {
ready_clone.store(1, Ordering::Release);
p_clone.unpark();
});
signal.join().unwrap();
let ok = p.park_until(|| ready.load(Ordering::Acquire) == 1);
assert!(ok);
}
#[test]
fn is_shutdown_reflects_shutdown_call() {
let p = Parker::new(8);
assert!(!p.is_shutdown());
p.shutdown();
assert!(p.is_shutdown());
}
#[test]
fn wait_strategy_pick_matches_cpuid() {
let want = if crate::cpu_info::has_waitpkg() {
WaitStrategy::Waitpkg
} else {
WaitStrategy::StdPark
};
assert_eq!(WaitStrategy::pick(), want);
}
#[test]
fn with_strategy_stdpark_round_trips_like_default() {
let p = Parker::with_strategy(8, WaitStrategy::StdPark);
assert_eq!(p.wait_strategy(), WaitStrategy::StdPark);
let ok = p.park_until(|| true);
assert!(ok, "StdPark park_until must return true on ready=true");
}
#[test]
fn unpark_increments_wake_counter() {
let p = Parker::new(8);
let before = p.wake_counter.load(Ordering::Acquire);
p.unpark();
let after = p.wake_counter.load(Ordering::Acquire);
assert_eq!(after, before + 1, "unpark must increment wake_counter");
}
#[test]
fn shutdown_increments_wake_counter() {
let p = Parker::new(8);
let before = p.wake_counter.load(Ordering::Acquire);
p.shutdown();
let after = p.wake_counter.load(Ordering::Acquire);
assert_eq!(after, before + 1, "shutdown must increment wake_counter via unpark");
assert!(p.is_shutdown());
}
#[test]
fn waitpkg_strategy_wakes_on_unpark_when_available() {
if !crate::cpu_info::has_waitpkg() {
eprintln!(
"skip waitpkg_strategy_wakes_on_unpark_when_available: \
host has no WAITPKG (cpuid leaf 7 ECX bit 5 = 0)"
);
return;
}
let ready = Arc::new(AtomicU32::new(0));
let ready_clone = Arc::clone(&ready);
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::with_strategy(8, WaitStrategy::Waitpkg));
tx.send(Arc::clone(&p)).unwrap();
let t0 = Instant::now();
let ok = p.park_until(|| ready.load(Ordering::Acquire) == 1);
(ok, t0.elapsed())
});
let p_owner = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(20));
ready_clone.store(1, Ordering::Release);
p_owner.unpark();
let (ok, elapsed) = owner.join().unwrap();
assert!(ok, "Waitpkg park_until must return true on unpark");
assert!(elapsed < Duration::from_millis(100),
"Waitpkg park_until took {elapsed:?}, expected < 100ms");
}
#[test]
fn shutdown_unparks_so_blocked_thread_exits() {
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::new(8));
tx.send(Arc::clone(&p)).unwrap();
p.park_until(|| false)
});
let p_owner = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(50));
p_owner.shutdown();
let ok = owner.join().unwrap();
assert!(!ok, "shutdown must surface as park_until -> false");
}
}