use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering};
use std::thread::{self, Thread};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WaitStrategy {
StdPark,
Waitpkg,
Monitorx,
}
static MONITOR_HOLDS: AtomicBool = AtomicBool::new(true);
static MONITOR_DOUBTS: AtomicU32 = AtomicU32::new(0);
const DOUBTS_BEFORE_GIVING_UP: u32 = 8;
fn monitor_holds() -> bool {
MONITOR_HOLDS.load(Ordering::Relaxed)
}
static LAST_DOUBT_CYCLES: AtomicU64 = AtomicU64::new(0);
fn note_monitor_does_not_hold(span_cycles: u64) {
LAST_DOUBT_CYCLES.store(span_cycles, Ordering::Relaxed);
if MONITOR_DOUBTS.fetch_add(1, Ordering::Relaxed) + 1 >= DOUBTS_BEFORE_GIVING_UP {
MONITOR_HOLDS.store(false, Ordering::Relaxed);
}
}
pub fn monitor_doubts() -> (u32, u64) {
(
MONITOR_DOUBTS.load(Ordering::Relaxed),
LAST_DOUBT_CYCLES.load(Ordering::Relaxed),
)
}
pub fn monitor_wait_held() -> bool {
monitor_holds()
}
pub(crate) fn monitor_wait_available() -> bool {
(crate::cpu_info::has_waitpkg() || crate::cpu_info::has_monitorx()) && monitor_holds()
}
#[inline]
pub(crate) fn cycles_now() -> u64 {
tsc_now()
}
pub fn total_latch_monitor_waits() -> u64 {
LATCH_MONITOR_WAITS.load(Ordering::Relaxed)
}
static LATCH_MONITOR_WAITS: AtomicU64 = AtomicU64::new(0);
pub fn total_latch_monitor_arms() -> u64 {
LATCH_MONITOR_ARMS.load(Ordering::Relaxed)
}
static LATCH_MONITOR_ARMS: AtomicU64 = AtomicU64::new(0);
#[inline]
pub(crate) fn note_latch_monitor_arm() {
LATCH_MONITOR_ARMS.fetch_add(1, Ordering::Relaxed);
}
pub fn total_backend_monitor_waits() -> u64 {
BACKEND_MONITOR_WAITS.load(Ordering::Relaxed)
}
static BACKEND_MONITOR_WAITS: AtomicU64 = AtomicU64::new(0);
pub fn total_backend_monitor_arms() -> u64 {
BACKEND_MONITOR_ARMS.load(Ordering::Relaxed)
}
static BACKEND_MONITOR_ARMS: AtomicU64 = AtomicU64::new(0);
#[cfg(feature = "shared-memory-worker-reference")]
#[inline]
pub(crate) fn note_backend_monitor_wait() {
BACKEND_MONITOR_WAITS.fetch_add(1, Ordering::Relaxed);
}
#[cfg(feature = "shared-memory-worker-reference")]
#[inline]
pub(crate) fn note_backend_monitor_arm() {
BACKEND_MONITOR_ARMS.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub(crate) fn note_latch_monitor_wait(slot: bool) {
LATCH_MONITOR_WAITS.fetch_add(1, Ordering::Relaxed);
if crate::sched::trace::is_enabled() {
let strategy = if crate::cpu_info::has_waitpkg() { 1 } else { 2 };
crate::sched::trace::emit(
crate::sched::trace::TraceEvent::LatchMonitor,
strategy | (u32::from(slot) * 32),
);
}
}
#[cfg(target_arch = "x86_64")]
pub(crate) unsafe fn monitor_wait_once(
line: *const u8,
budget_cycles: u64,
armed: impl FnOnce() -> bool,
) -> bool {
if !monitor_holds() {
return false;
}
let budget = budget_cycles.max(1);
if crate::cpu_info::has_waitpkg() {
unsafe {
core::arch::asm!(
"umonitor rax",
in("rax") line,
options(nostack, preserves_flags),
);
}
if armed() {
return true;
}
let deadline = unsafe { core::arch::x86_64::_rdtsc() }.wrapping_add(budget);
unsafe {
core::arch::asm!(
"umwait {hint:e}",
hint = in(reg) 1u32,
in("eax") deadline as u32,
in("edx") (deadline >> 32) as u32,
options(nostack),
);
}
return true;
}
if crate::cpu_info::has_monitorx() {
unsafe {
core::arch::asm!(
".byte 0x0f, 0x01, 0xfa",
in("rax") line,
in("ecx") 0u32,
in("edx") 0u32,
options(nostack, preserves_flags),
);
}
if armed() {
return true;
}
let ask = u32::try_from(budget).unwrap_or(u32::MAX);
unsafe {
core::arch::asm!(
"push rbx",
"mov ebx, {ask:e}",
".byte 0x0f, 0x01, 0xfb",
"pop rbx",
ask = in(reg) ask,
inout("eax") 0xF0u32 => _,
inout("ecx") 2u32 => _,
);
}
return true;
}
false
}
#[cfg(not(target_arch = "x86_64"))]
pub(crate) unsafe fn monitor_wait_once(
_line: *const u8,
_budget_cycles: u64,
_armed: impl FnOnce() -> bool,
) -> bool {
false
}
impl WaitStrategy {
pub fn baseline() -> Self {
if crate::cpu_info::has_waitpkg() {
Self::Waitpkg
} else {
Self::StdPark
}
}
pub fn pick() -> Self {
wait_controller().choose()
}
fn trace_code(self) -> u32 {
match self {
Self::StdPark => 0,
Self::Waitpkg => 1,
Self::Monitorx => 2,
}
}
}
pub struct WaitController {
mean_ns: [AtomicU64; 2],
samples: [AtomicU32; 2],
parks: AtomicU64,
current: AtomicU32,
switches: AtomicU64,
probe_every: AtomicU32,
score: AtomicI32,
}
const SCORE_CAP: i32 = 64;
const SWITCH_AT: i32 = 24;
const PROBE_EVERY_MIN: u32 = 64;
const PROBE_EVERY_MAX: u32 = 4096;
#[derive(Clone, Copy)]
struct ProbeCursor {
since: u32,
every: u32,
pending: u32,
held_slot: u32,
held_ns: u64,
}
const BOTH_SLOTS: u32 = (1 << SLOT_BASELINE) | (1 << SLOT_CHALLENGER);
const NO_SLOT: u32 = u32::MAX;
thread_local! {
static PROBE: core::cell::Cell<ProbeCursor> = const {
core::cell::Cell::new(ProbeCursor {
since: 0,
every: PROBE_EVERY_MIN,
pending: 0,
held_slot: NO_SLOT,
held_ns: 0,
})
};
}
const SAMPLES_BEFORE_EWMA: u32 = 32;
const MARGIN_PER_CENT: u64 = 20;
static WAIT_CONTROLLER: WaitController = WaitController {
mean_ns: [AtomicU64::new(0), AtomicU64::new(0)],
samples: [AtomicU32::new(0), AtomicU32::new(0)],
parks: AtomicU64::new(0),
current: AtomicU32::new(SLOT_BASELINE),
switches: AtomicU64::new(0),
probe_every: AtomicU32::new(PROBE_EVERY_MIN),
score: AtomicI32::new(0),
};
const SLOT_BASELINE: u32 = 0;
const SLOT_CHALLENGER: u32 = 1;
pub fn wait_controller() -> &'static WaitController {
&WAIT_CONTROLLER
}
impl WaitController {
fn choose(&self) -> WaitStrategy {
if self.current.load(Ordering::Relaxed) == SLOT_CHALLENGER
&& let Some(c) = challenger()
{
return c;
}
WaitStrategy::baseline()
}
fn sample_plan(&self) -> Option<(WaitStrategy, u32)> {
let challenger = challenger()?;
let slot = PROBE.with(|p| {
let mut c = p.get();
if c.pending == 0 {
c.since += 1;
if c.since < c.every {
p.set(c);
return None;
}
c.pending = BOTH_SLOTS;
c.every = self.probe_every.load(Ordering::Relaxed);
}
let slot = if c.pending == BOTH_SLOTS {
(self.parks.fetch_add(1, Ordering::Relaxed) % 2) as u32
} else {
u32::from(c.pending == (1 << SLOT_CHALLENGER))
};
p.set(c);
Some(slot)
})?;
Some((
if slot == SLOT_CHALLENGER {
challenger
} else {
WaitStrategy::baseline()
},
slot,
))
}
#[cfg(test)]
fn open_pair() {
PROBE.with(|p| {
let mut c = p.get();
c.pending = BOTH_SLOTS;
c.held_slot = NO_SLOT;
p.set(c);
});
}
fn probe_served(slot: u32, wake_ns: u64) -> Option<u64> {
PROBE.with(|p| {
let mut c = p.get();
c.pending &= !(1u32 << slot);
if c.pending != 0 {
c.held_slot = slot;
c.held_ns = wake_ns;
p.set(c);
return None;
}
c.since = 0;
let partner = (c.held_slot != NO_SLOT && c.held_slot != slot).then_some(c.held_ns);
c.held_slot = NO_SLOT;
c.held_ns = 0;
p.set(c);
partner
})
}
fn record(&self, slot: u32, wake_ns: u64) {
if let Some(partner_ns) = Self::probe_served(slot, wake_ns) {
let (base, chal) = if slot == SLOT_CHALLENGER {
(partner_ns, wake_ns)
} else {
(wake_ns, partner_ns)
};
self.score_pair(base, chal);
}
let i = slot as usize;
let n = self.samples[i].fetch_add(1, Ordering::Relaxed) + 1;
let prev = self.mean_ns[i].load(Ordering::Relaxed);
let next = if prev == 0 {
wake_ns
} else if n <= SAMPLES_BEFORE_EWMA {
let prev_i = prev as i64;
(prev_i + (wake_ns as i64 - prev_i) / i64::from(n)).max(0) as u64
} else {
(prev * 7 + wake_ns) / 8
};
self.mean_ns[i].store(next, Ordering::Relaxed);
self.decide();
}
fn score_pair(&self, base_ns: u64, chal_ns: u64) {
let step = if chal_ns * 100 < base_ns * (100 - MARGIN_PER_CENT) {
1
} else if base_ns * 100 < chal_ns * (100 - MARGIN_PER_CENT) {
-1
} else {
return;
};
self.score
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |s| {
Some((s + step).clamp(-SCORE_CAP, SCORE_CAP))
})
.expect("the update closure returns Some on every call");
}
fn decide(&self) {
let score = self.score.load(Ordering::Relaxed);
if score >= SWITCH_AT || score <= -SWITCH_AT {
let want = if score > 0 {
SLOT_CHALLENGER
} else {
SLOT_BASELINE
};
if self.current.swap(want, Ordering::Relaxed) != want {
self.switches.fetch_add(1, Ordering::Relaxed);
crate::sched::trace::emit(
crate::sched::trace::TraceEvent::WaitSwitch,
self.choose().trace_code(),
);
}
}
if score.abs() >= SCORE_CAP {
let e = self.probe_every.load(Ordering::Relaxed);
if e < PROBE_EVERY_MAX {
self.probe_every
.store((e * 2).min(PROBE_EVERY_MAX), Ordering::Relaxed);
}
} else if score.abs() < SWITCH_AT {
self.probe_every.store(PROBE_EVERY_MIN, Ordering::Relaxed);
}
}
pub fn report(&self) -> WaitControllerReport {
WaitControllerReport {
baseline_ns: self.mean_ns[SLOT_BASELINE as usize].load(Ordering::Relaxed),
challenger_ns: self.mean_ns[SLOT_CHALLENGER as usize].load(Ordering::Relaxed),
baseline_samples: self.samples[SLOT_BASELINE as usize].load(Ordering::Relaxed),
challenger_samples: self.samples[SLOT_CHALLENGER as usize].load(Ordering::Relaxed),
in_use: self.choose(),
switches: self.switches.load(Ordering::Relaxed),
probe_every: self.probe_every.load(Ordering::Relaxed),
score: self.score.load(Ordering::Relaxed),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WaitControllerReport {
pub baseline_ns: u64,
pub challenger_ns: u64,
pub baseline_samples: u32,
pub challenger_samples: u32,
pub in_use: WaitStrategy,
pub switches: u64,
pub probe_every: u32,
pub score: i32,
}
#[cfg(target_arch = "x86_64")]
#[inline(always)]
fn tsc_now() -> u64 {
unsafe { core::arch::x86_64::_rdtsc() }
}
#[cfg(not(target_arch = "x86_64"))]
#[inline(always)]
fn tsc_now() -> u64 {
0
}
fn cycles_to_ns(cycles: u64) -> u64 {
let per_ns_16 = crate::sched::par_iter::tsc_per_ns_16();
if per_ns_16 == 0 {
return cycles;
}
cycles.saturating_mul(16) / per_ns_16
}
fn challenger() -> Option<WaitStrategy> {
if !crate::cpu_info::has_monitorx() || crate::cpu_info::has_waitpkg() {
return None;
}
if !monitor_holds() {
return None;
}
Some(WaitStrategy::Monitorx)
}
#[derive(Debug)]
#[repr(align(64))]
pub struct Parker {
thread: Thread,
shutdown: AtomicBool,
spin_rounds: u32,
wake_counter: AtomicU64,
pinned: Option<WaitStrategy>,
wake_stamp: AtomicU64,
}
const SAMPLE_ARMED: u64 = 1;
impl Parker {
pub fn new(spin_rounds: u32) -> Self {
Self {
thread: thread::current(),
shutdown: AtomicBool::new(false),
spin_rounds,
wake_counter: AtomicU64::new(0),
pinned: None,
wake_stamp: AtomicU64::new(0),
}
}
pub fn with_strategy(spin_rounds: u32, wait_strategy: WaitStrategy) -> Self {
Self {
thread: thread::current(),
shutdown: AtomicBool::new(false),
spin_rounds,
wake_counter: AtomicU64::new(0),
pinned: Some(wait_strategy),
wake_stamp: AtomicU64::new(0),
}
}
pub fn wait_strategy(&self) -> WaitStrategy {
self.pinned.unwrap_or_else(WaitStrategy::pick)
}
pub fn park_until<F: FnMut() -> bool>(&self, mut is_ready: F) -> bool {
let initial_wake = self.wake_counter.load(Ordering::Acquire);
for _ in 0..self.spin_rounds {
if self.shutdown.load(Ordering::Acquire) {
return false;
}
if is_ready() {
return true;
}
thread::yield_now();
}
if self.shutdown.load(Ordering::Acquire) {
return false;
}
if is_ready() {
return true;
}
let (strategy, sampling) = match self.pinned {
Some(p) => (p, None),
None => match wait_controller().sample_plan() {
Some((s, slot)) => {
self.wake_stamp.store(SAMPLE_ARMED, Ordering::Relaxed);
(s, Some(slot))
}
None => (WaitStrategy::pick(), None),
},
};
if crate::sched::trace::is_enabled() {
crate::sched::trace::emit(
crate::sched::trace::TraceEvent::ParkEnter,
strategy.trace_code() | if sampling.is_some() { 16 } else { 0 },
);
}
match strategy {
WaitStrategy::StdPark => {
thread::park();
}
WaitStrategy::Waitpkg | WaitStrategy::Monitorx => loop {
match strategy {
WaitStrategy::Monitorx => self.wait_via_monitorx(initial_wake),
_ => self.wait_via_waitpkg(initial_wake),
}
if self.wake_counter.load(Ordering::Acquire) != initial_wake {
break;
}
if self.shutdown.load(Ordering::Acquire) {
break;
}
if is_ready() {
break;
}
},
}
if let Some(slot) = sampling {
let stamp = self.wake_stamp.swap(0, Ordering::Relaxed);
if stamp > SAMPLE_ARMED {
let now = tsc_now();
let wake_ns = cycles_to_ns(now.wrapping_sub(stamp));
crate::sched::trace::emit(
crate::sched::trace::TraceEvent::WorkerWake,
u32::try_from(wake_ns).unwrap_or(u32::MAX),
);
wait_controller().record(slot, wake_ns);
}
}
if self.shutdown.load(Ordering::Acquire) {
return false;
}
true
}
pub fn unpark(&self) {
if self.wake_stamp.load(Ordering::Relaxed) == SAMPLE_ARMED {
self.wake_stamp.store(tsc_now(), Ordering::Relaxed);
}
self.wake_counter.fetch_add(1, Ordering::Release);
self.thread.unpark();
}
pub fn shutdown(&self) {
self.shutdown.store(true, Ordering::Release);
self.unpark();
}
#[cfg(target_arch = "x86_64")]
fn wait_via_waitpkg(&self, initial_wake: u64) {
const WAIT_DEADLINE_NS: u64 = 10_000_000;
const TSC_HZ_ESTIMATE: u64 = 2_500_000_000;
let cycles =
WAIT_DEADLINE_NS.saturating_mul(TSC_HZ_ESTIMATE / 1_000_000_000);
let now = unsafe { core::arch::x86_64::_rdtsc() };
let deadline = now.wrapping_add(cycles);
let lo = deadline as u32;
let hi = (deadline >> 32) as u32;
let addr = (&raw const self.wake_counter).cast::<u8>();
unsafe {
core::arch::asm!(
"umonitor rax",
in("rax") addr,
options(nostack, preserves_flags),
);
}
if self.wake_counter.load(Ordering::Acquire) != initial_wake {
return;
}
unsafe {
core::arch::asm!(
"umwait {hint:e}",
hint = in(reg) 1u32,
in("eax") lo,
in("edx") hi,
options(nostack),
);
}
}
#[cfg(not(target_arch = "x86_64"))]
fn wait_via_waitpkg(&self, _initial_wake: u64) {
thread::park();
}
#[cfg(target_arch = "x86_64")]
fn wait_via_monitorx(&self, initial_wake: u64) {
const WAIT_DEADLINE_NS: u64 = 10_000_000;
const TSC_HZ_ESTIMATE: u64 = 2_500_000_000;
const MAX_ARMS: u32 = 256;
const ARMS_BEFORE_JUDGING: u32 = 4;
const ARMS_TOO_FAST_CYCLES: u64 = 50_000;
if !monitor_holds() {
thread::park();
return;
}
let budget = WAIT_DEADLINE_NS.saturating_mul(TSC_HZ_ESTIMATE / 1_000_000_000);
let start = unsafe { core::arch::x86_64::_rdtsc() };
let addr = (&raw const self.wake_counter).cast::<u8>();
let mut arms = 0u32;
for _ in 0..MAX_ARMS {
arms += 1;
unsafe {
core::arch::asm!(
".byte 0x0f, 0x01, 0xfa",
in("rax") addr,
in("ecx") 0u32,
in("edx") 0u32,
options(nostack, preserves_flags),
);
}
if self.wake_counter.load(Ordering::Acquire) != initial_wake {
return;
}
let spent = unsafe { core::arch::x86_64::_rdtsc() }.wrapping_sub(start);
let left = match budget.checked_sub(spent) {
Some(left) if left > 0 => left,
_ => return,
};
let ask = u32::try_from(left).unwrap_or(u32::MAX);
unsafe {
core::arch::asm!(
"push rbx",
"mov ebx, {ask:e}",
".byte 0x0f, 0x01, 0xfb",
"pop rbx",
ask = in(reg) ask,
inout("eax") 0u32 => _,
inout("ecx") 2u32 => _,
);
}
if self.wake_counter.load(Ordering::Acquire) != initial_wake {
return;
}
if self.shutdown.load(Ordering::Acquire) {
return;
}
if arms >= ARMS_BEFORE_JUDGING {
let span = unsafe { core::arch::x86_64::_rdtsc() }.wrapping_sub(start);
if span < ARMS_TOO_FAST_CYCLES {
note_monitor_does_not_hold(span);
thread::park();
return;
}
}
}
}
#[cfg(not(target_arch = "x86_64"))]
fn wait_via_monitorx(&self, _initial_wake: u64) {
thread::park();
}
pub fn is_shutdown(&self) -> bool {
self.shutdown.load(Ordering::Acquire)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::AtomicU32;
use std::thread;
use std::time::{Duration, Instant};
const PARK_DEADLINE: Duration = Duration::from_secs(10);
fn parked_result<T: Send + 'static>(
rx: &std::sync::mpsc::Receiver<T>,
owner: thread::JoinHandle<T>,
what: &str,
) -> T {
match rx.recv_timeout(PARK_DEADLINE) {
Ok(value) => {
owner.join().expect("the thread reported before this join");
value
}
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => panic!(
"{what} did not reach the parked thread within {PARK_DEADLINE:?}, \
which is the lost wakeup this test exists to catch"
),
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {
owner.join().expect("the parking thread panicked without reporting");
unreachable!("a join over a dropped sender re-raises the panic that dropped it")
}
}
}
#[test]
fn park_until_returns_immediately_when_ready() {
let p = Parker::new(8);
let t0 = Instant::now();
let ok = p.park_until(|| true);
let elapsed = t0.elapsed();
assert!(ok);
assert!(elapsed < Duration::from_millis(10),
"park_until with ready=true must be fast; took {elapsed:?}");
}
#[test]
fn park_until_returns_false_on_shutdown() {
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let (done_tx, done_rx) = std::sync::mpsc::channel::<bool>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::new(8));
tx.send(Arc::clone(&p)).expect("send parker");
let ok = p.park_until(|| false);
if done_tx.send(ok).is_err() {
eprintln!("park_until returned after the test stopped waiting for it");
}
ok
});
let p = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(50));
p.shutdown();
let ok = parked_result(&done_rx, owner, "the shutdown");
assert!(!ok, "park_until must return false after shutdown");
}
#[test]
fn park_until_wakes_on_unpark_from_other_thread() {
let ready = Arc::new(AtomicU32::new(0));
let ready_clone = Arc::clone(&ready);
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let (done_tx, done_rx) = std::sync::mpsc::channel::<(bool, Duration)>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::new(8));
tx.send(Arc::clone(&p)).unwrap();
let t0 = Instant::now();
let ok = p.park_until(|| ready.load(Ordering::Acquire) == 1);
let result = (ok, t0.elapsed());
if done_tx.send(result).is_err() {
eprintln!("park_until returned after the test stopped waiting for it");
}
result
});
let p_owner = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(50));
ready_clone.store(1, Ordering::Release);
p_owner.unpark();
let (ok, elapsed) = parked_result(&done_rx, owner, "the unpark");
assert!(ok, "park_until must return true after ready becomes true");
assert!(elapsed < Duration::from_millis(500),
"park_until took too long: {elapsed:?}");
}
#[test]
fn park_until_spin_floor_eight_rounds_no_park() {
let p = Parker::new(8);
let mut polls = 0u32;
let ok = p.park_until(|| {
polls += 1;
polls >= 3
});
assert!(ok);
assert_eq!(polls, 3);
}
#[test]
fn park_until_zero_spin_floor_goes_straight_to_park() {
let p = Parker::new(0);
let ok = p.park_until(|| true);
assert!(ok);
}
#[test]
fn unpark_before_park_is_observable_via_permit() {
let p = Arc::new(Parker::new(0)); let ready = Arc::new(AtomicU32::new(0));
let p_clone = Arc::clone(&p);
let ready_clone = Arc::clone(&ready);
let signal = thread::spawn(move || {
ready_clone.store(1, Ordering::Release);
p_clone.unpark();
});
signal.join().unwrap();
let ok = p.park_until(|| ready.load(Ordering::Acquire) == 1);
assert!(ok);
}
#[test]
fn is_shutdown_reflects_shutdown_call() {
let p = Parker::new(8);
assert!(!p.is_shutdown());
p.shutdown();
assert!(p.is_shutdown());
}
#[test]
fn the_baseline_is_what_the_silicon_supports_and_nothing_learned() {
let want = if crate::cpu_info::has_waitpkg() {
WaitStrategy::Waitpkg
} else {
WaitStrategy::StdPark
};
assert_eq!(WaitStrategy::baseline(), want);
}
fn fresh_controller() -> WaitController {
WaitController {
mean_ns: [AtomicU64::new(0), AtomicU64::new(0)],
samples: [AtomicU32::new(0), AtomicU32::new(0)],
parks: AtomicU64::new(0),
current: AtomicU32::new(SLOT_BASELINE),
switches: AtomicU64::new(0),
probe_every: AtomicU32::new(PROBE_EVERY_MIN),
score: AtomicI32::new(0),
}
}
#[test]
fn evidence_short_of_the_gate_decides_nothing() {
let c = fresh_controller();
for _ in 0..3 {
WaitController::open_pair();
c.record(SLOT_CHALLENGER, 100);
c.record(SLOT_BASELINE, 100_000);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_BASELINE);
assert_eq!(c.switches.load(Ordering::Relaxed), 0);
}
#[test]
fn a_clear_win_moves_the_verdict_and_a_clear_loss_moves_it_back() {
let c = fresh_controller();
for _ in 0..SWITCH_AT {
WaitController::open_pair();
c.record(SLOT_BASELINE, 6_000);
c.record(SLOT_CHALLENGER, 1_000);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_CHALLENGER);
for _ in 0..SCORE_CAP + SWITCH_AT {
WaitController::open_pair();
c.record(SLOT_CHALLENGER, 300_000);
c.record(SLOT_BASELINE, 6_000);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_BASELINE);
assert!(c.switches.load(Ordering::Relaxed) >= 2);
}
#[test]
fn two_arms_within_the_margin_leave_the_verdict_alone() {
let c = fresh_controller();
for _ in 0..SCORE_CAP * 2 {
WaitController::open_pair();
c.record(SLOT_BASELINE, 5_000);
c.record(SLOT_CHALLENGER, 4_600);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_BASELINE);
assert_eq!(c.switches.load(Ordering::Relaxed), 0);
assert_eq!(c.score.load(Ordering::Relaxed), 0);
}
#[test]
fn a_mean_follows_its_samples_down_as_well_as_up() {
let c = fresh_controller();
c.record(SLOT_CHALLENGER, 300_000);
assert_eq!(
c.mean_ns[SLOT_CHALLENGER as usize].load(Ordering::Relaxed),
300_000
);
for _ in 0..SAMPLES_BEFORE_EWMA - 1 {
c.record(SLOT_CHALLENGER, 1_000);
}
let m = c.mean_ns[SLOT_CHALLENGER as usize].load(Ordering::Relaxed);
assert!(m < 20_000, "mean stayed high at {m}");
assert!(m > 1_000, "mean dropped the slow draw at {m}");
}
#[test]
fn a_win_survives_the_spread_a_real_wake_has() {
let c = fresh_controller();
let base = [5_000u64, 6_500, 5_200, 40_000, 5_800, 6_100];
let chal = [1_100u64, 1_400, 1_200, 30_000, 1_300, 1_250];
for i in 0..(SWITCH_AT as usize) {
WaitController::open_pair();
c.record(SLOT_BASELINE, base[i % base.len()]);
c.record(SLOT_CHALLENGER, chal[i % chal.len()]);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_CHALLENGER);
}
#[test]
fn a_rare_enormous_draw_does_not_decide_against_an_arm() {
let c = fresh_controller();
for i in 0..SWITCH_AT * 2 {
WaitController::open_pair();
c.record(SLOT_BASELINE, 6_000);
let chal = if i % 20 == 19 { 340_000 } else { 900 };
c.record(SLOT_CHALLENGER, chal);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_CHALLENGER);
assert!(
c.mean_ns[SLOT_CHALLENGER as usize].load(Ordering::Relaxed)
> c.mean_ns[SLOT_BASELINE as usize].load(Ordering::Relaxed),
"the tail should still show in the reported mean"
);
}
#[test]
fn a_settled_verdict_probes_less_often_and_a_closing_gap_probes_more() {
let c = fresh_controller();
for _ in 0..SCORE_CAP * 2 {
WaitController::open_pair();
c.record(SLOT_BASELINE, 6_000);
c.record(SLOT_CHALLENGER, 1_000);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_CHALLENGER);
assert_eq!(c.score.load(Ordering::Relaxed), SCORE_CAP);
assert_eq!(c.probe_every.load(Ordering::Relaxed), PROBE_EVERY_MAX);
for _ in 0..(SCORE_CAP - SWITCH_AT) + 1 {
WaitController::open_pair();
c.record(SLOT_CHALLENGER, 6_000);
c.record(SLOT_BASELINE, 4_000);
}
assert_eq!(c.current.load(Ordering::Relaxed), SLOT_CHALLENGER);
assert_eq!(c.probe_every.load(Ordering::Relaxed), PROBE_EVERY_MIN);
}
#[test]
fn a_cold_controller_starts_on_the_baseline() {
let r = wait_controller().report();
if r.switches == 0 {
assert_eq!(r.in_use, WaitStrategy::baseline());
}
}
#[test]
fn the_controller_never_offers_a_wait_this_host_cannot_execute() {
match WaitStrategy::pick() {
WaitStrategy::Waitpkg => assert!(crate::cpu_info::has_waitpkg()),
WaitStrategy::Monitorx => assert!(crate::cpu_info::has_monitorx()),
WaitStrategy::StdPark => {}
}
}
#[test]
fn a_challenger_is_only_offered_where_it_could_win() {
match challenger() {
Some(c) => {
assert_eq!(c, WaitStrategy::Monitorx);
assert!(crate::cpu_info::has_monitorx());
assert!(monitor_wait_held());
}
None => assert!(
!crate::cpu_info::has_monitorx()
|| crate::cpu_info::has_waitpkg()
|| !monitor_wait_held()
),
}
}
#[test]
fn a_pinned_parker_ignores_the_controller() {
let p = Parker::with_strategy(0, WaitStrategy::StdPark);
assert_eq!(p.wait_strategy(), WaitStrategy::StdPark);
if crate::cpu_info::has_monitorx() && !crate::cpu_info::has_waitpkg() {
let m = Parker::with_strategy(0, WaitStrategy::Monitorx);
assert_eq!(m.wait_strategy(), WaitStrategy::Monitorx);
}
}
#[test]
fn no_two_parkers_can_share_a_cache_line() {
assert_eq!(std::mem::align_of::<Parker>(), 64);
assert_eq!(std::mem::size_of::<Parker>() % 64, 0);
}
#[test]
fn the_parker_never_picks_a_wait_its_host_cannot_execute() {
match Parker::new(0).wait_strategy() {
WaitStrategy::Waitpkg => assert!(crate::cpu_info::has_waitpkg()),
WaitStrategy::Monitorx => assert!(crate::cpu_info::has_monitorx()),
WaitStrategy::StdPark => assert!(!crate::cpu_info::has_waitpkg()),
}
}
#[test]
fn the_hosts_own_strategy_wakes_on_unpark() {
let ready = Arc::new(AtomicU32::new(0));
let woke = Arc::new(AtomicU32::new(0));
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let ready_thread = Arc::clone(&ready);
let woke_thread = Arc::clone(&woke);
let (done_tx, done_rx) = std::sync::mpsc::channel::<bool>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::new(0));
tx.send(Arc::clone(&p)).expect("send parker");
let ok = p.park_until(|| ready_thread.load(Ordering::Acquire) == 1);
woke_thread.store(1, Ordering::Release);
if done_tx.send(ok).is_err() {
eprintln!("park_until returned after the test stopped waiting for it");
}
ok
});
let p = rx.recv().expect("recv parker");
thread::sleep(Duration::from_millis(20));
ready.store(1, Ordering::Release);
p.unpark();
let ok = parked_result(&done_rx, owner, "the unpark");
assert!(ok, "park_until must report a wake rather than a shutdown");
assert_eq!(woke.load(Ordering::Acquire), 1);
}
#[test]
fn with_strategy_stdpark_round_trips_like_default() {
let p = Parker::with_strategy(8, WaitStrategy::StdPark);
assert_eq!(p.wait_strategy(), WaitStrategy::StdPark);
let ok = p.park_until(|| true);
assert!(ok, "StdPark park_until must return true on ready=true");
}
#[test]
fn unpark_increments_wake_counter() {
let p = Parker::new(8);
let before = p.wake_counter.load(Ordering::Acquire);
p.unpark();
let after = p.wake_counter.load(Ordering::Acquire);
assert_eq!(after, before + 1, "unpark must increment wake_counter");
}
#[test]
fn shutdown_increments_wake_counter() {
let p = Parker::new(8);
let before = p.wake_counter.load(Ordering::Acquire);
p.shutdown();
let after = p.wake_counter.load(Ordering::Acquire);
assert_eq!(after, before + 1, "shutdown must increment wake_counter via unpark");
assert!(p.is_shutdown());
}
#[test]
fn waitpkg_strategy_wakes_on_unpark_when_available() {
if !crate::cpu_info::has_waitpkg() {
eprintln!(
"skip waitpkg_strategy_wakes_on_unpark_when_available: \
host has no WAITPKG (cpuid leaf 7 ECX bit 5 = 0)"
);
return;
}
let ready = Arc::new(AtomicU32::new(0));
let ready_clone = Arc::clone(&ready);
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let (done_tx, done_rx) = std::sync::mpsc::channel::<(bool, Duration)>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::with_strategy(8, WaitStrategy::Waitpkg));
tx.send(Arc::clone(&p)).unwrap();
let t0 = Instant::now();
let ok = p.park_until(|| ready.load(Ordering::Acquire) == 1);
let result = (ok, t0.elapsed());
if done_tx.send(result).is_err() {
eprintln!("park_until returned after the test stopped waiting for it");
}
result
});
let p_owner = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(20));
ready_clone.store(1, Ordering::Release);
p_owner.unpark();
let (ok, elapsed) = parked_result(&done_rx, owner, "the unpark under WAITPKG");
assert!(ok, "Waitpkg park_until must return true on unpark");
assert!(elapsed < Duration::from_millis(100),
"Waitpkg park_until took {elapsed:?}, expected < 100ms");
}
#[test]
fn shutdown_unparks_so_blocked_thread_exits() {
let (tx, rx) = std::sync::mpsc::channel::<Arc<Parker>>();
let (done_tx, done_rx) = std::sync::mpsc::channel::<bool>();
let owner = thread::spawn(move || {
let p = Arc::new(Parker::new(8));
tx.send(Arc::clone(&p)).unwrap();
let woke = p.park_until(|| false);
if done_tx.send(woke).is_err() {
eprintln!("park_until returned after the test stopped waiting for it");
}
woke
});
let p_owner = rx.recv().expect("owner must send its parker");
thread::sleep(Duration::from_millis(50));
p_owner.shutdown();
let ok = parked_result(&done_rx, owner, "the shutdown");
assert!(!ok, "shutdown must surface as park_until -> false");
}
}