#![allow(clippy::missing_errors_doc)]
use std::sync::OnceLock;
use std::time::Instant;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum KGating {
CounterOnly,
PerSlot,
Auto,
}
impl KGating {
#[inline]
pub fn resolved(self) -> KGating {
match self {
KGating::Auto => *CALIBRATED_GATING.get_or_init(calibrate_k_gating),
other => other,
}
}
}
static CALIBRATED_GATING: OnceLock<KGating> = OnceLock::new();
#[derive(Debug, Clone, Copy)]
pub struct CalibrationResult {
pub per_slot_ns: u64,
pub counter_only_ns: u64,
pub winner: KGating,
}
pub fn calibrate_k_gating() -> KGating {
run_calibration().winner
}
pub fn calibrate_k_gating_verbose() -> CalibrationResult {
run_calibration()
}
fn run_calibration() -> CalibrationResult {
use crate::sched::chase_lev_local::{Steal, new_chase_lev};
const N: usize = 64;
const ROUNDS: usize = 4096;
let counter_only_ns = {
let (w, _s) = new_chase_lev::<u64>(N.next_power_of_two().max(2));
let t0 = Instant::now();
for _ in 0..ROUNDS {
for i in 0..N as u64 {
w.push(i).expect("push");
}
for _ in 0..N {
match w.pop() {
Steal::Success(_) | Steal::Empty | Steal::Retry => {}
}
}
}
t0.elapsed().as_nanos() as u64 / ROUNDS as u64
};
let per_slot_ns = run_vyukov_ring_calibration(N, ROUNDS);
let winner = if per_slot_ns < counter_only_ns {
KGating::PerSlot
} else {
KGating::CounterOnly
};
CalibrationResult {
per_slot_ns,
counter_only_ns,
winner,
}
}
fn run_vyukov_ring_calibration(n: usize, rounds: usize) -> u64 {
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
let cap = n.next_power_of_two().max(2);
let mask = (cap - 1) as i64;
let bottom = AtomicI64::new(0);
let head = AtomicI64::new(0);
let slots: Vec<(AtomicU64, std::cell::UnsafeCell<u64>)> = (0..cap)
.map(|i| (AtomicU64::new(i as u64), std::cell::UnsafeCell::new(0)))
.collect();
let t0 = Instant::now();
for _ in 0..rounds {
for i in 0..n as u64 {
let b = bottom.load(Ordering::Relaxed);
let slot = &slots[(b & mask) as usize];
while slot.0.load(Ordering::Acquire) != b as u64 {
std::hint::spin_loop();
}
unsafe { *slot.1.get() = i; }
slot.0.store((b as u64) + 1, Ordering::Release);
bottom.store(b + 1, Ordering::Relaxed);
}
for _ in 0..n {
let t = head.load(Ordering::Acquire);
let b = bottom.load(Ordering::Acquire);
if t >= b { break; }
if head
.compare_exchange(t, t + 1, Ordering::Acquire, Ordering::Relaxed)
.is_err()
{
continue;
}
let slot = &slots[(t & mask) as usize];
while slot.0.load(Ordering::Acquire) != (t as u64) + 1 {
std::hint::spin_loop();
}
let v = unsafe { *slot.1.get() };
std::hint::black_box(v);
slot.0.store(
(t as u64) + (cap as u64),
Ordering::Release,
);
}
}
let elapsed = t0.elapsed().as_nanos() as u64;
elapsed / rounds as u64
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn gating_variants_are_distinct() {
assert_ne!(KGating::CounterOnly, KGating::PerSlot);
assert_ne!(KGating::CounterOnly, KGating::Auto);
assert_ne!(KGating::PerSlot, KGating::Auto);
}
#[test]
fn resolved_keeps_explicit_choices() {
assert_eq!(KGating::CounterOnly.resolved(), KGating::CounterOnly);
assert_eq!(KGating::PerSlot.resolved(), KGating::PerSlot);
let auto_resolved = KGating::Auto.resolved();
assert!(matches!(auto_resolved, KGating::CounterOnly | KGating::PerSlot),
"Auto must resolve to a concrete variant, got {auto_resolved:?}");
}
#[test]
fn calibration_returns_finite_timings_and_picks_winner() {
let r = run_calibration();
assert!(r.per_slot_ns > 0, "per_slot timing must be positive: {r:?}");
assert!(r.counter_only_ns > 0, "counter_only timing must be positive: {r:?}");
let expected = if r.per_slot_ns < r.counter_only_ns {
KGating::PerSlot
} else {
KGating::CounterOnly
};
assert_eq!(r.winner, expected,
"winner must match the smaller timing: {r:?}");
}
#[test]
fn calibration_is_cached_for_auto() {
let r1 = KGating::Auto.resolved();
let r2 = KGating::Auto.resolved();
assert_eq!(r1, r2, "auto.resolved() must return the cached value");
}
}