use std::fs::{File, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering, fence};
use memmap2::{MmapMut, MmapOptions};
pub const CALIBRATION_MAGIC: u64 = 0x464C_4342_0000_0001;
pub const LAYOUT_VERSION: u32 = 9;
pub const MAX_ACCEL: usize = 8;
pub const LEASE_GRACE_EPOCHS: u64 = 2;
pub const NO_WRITER: u32 = 0;
pub const PROVISIONAL_SPREAD_PER_MILLE: u32 = 250;
pub fn fnv1a_64(bytes: &[u8]) -> u64 {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in bytes {
h ^= *b as u64;
h = h.wrapping_mul(0x0000_0100_0000_01b3);
}
h
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct HostStamp {
pub vendor: String,
pub cpuid_signature: u32,
pub arch: &'static str,
pub os: &'static str,
pub primary_workers: u32,
pub total_workers: u32,
pub layout_version: u32,
}
impl HostStamp {
pub fn detect() -> Self {
let arena = crate::sched::arena::global_local_arena();
let (vendor, cpuid_signature) = cpuid_identity();
Self {
vendor,
cpuid_signature,
arch: std::env::consts::ARCH,
os: std::env::consts::OS,
primary_workers: arena.primary_workers() as u32,
total_workers: arena.total_workers() as u32,
layout_version: LAYOUT_VERSION,
}
}
pub fn canonical(&self) -> String {
format!(
"vendor={};sig={:#010x};arch={};os={};primary={};total={};v={}",
self.vendor,
self.cpuid_signature,
self.arch,
self.os,
self.primary_workers,
self.total_workers,
self.layout_version,
)
}
pub fn hash(&self) -> u64 {
fnv1a_64(self.canonical().as_bytes())
}
}
#[cfg(target_arch = "x86_64")]
fn cpuid_identity() -> (String, u32) {
use std::arch::x86_64::__cpuid_count;
let (leaf0, leaf1) = (__cpuid_count(0, 0), __cpuid_count(1, 0));
let mut bytes = [0u8; 12];
bytes[0..4].copy_from_slice(&leaf0.ebx.to_le_bytes());
bytes[4..8].copy_from_slice(&leaf0.edx.to_le_bytes());
bytes[8..12].copy_from_slice(&leaf0.ecx.to_le_bytes());
let vendor = String::from_utf8_lossy(&bytes)
.trim_end_matches('\0')
.to_string();
(vendor, leaf1.eax)
}
#[cfg(not(target_arch = "x86_64"))]
fn cpuid_identity() -> (String, u32) {
("unknown".to_string(), 0)
}
#[repr(C, align(64))]
#[derive(Copy, Clone, Debug, PartialEq, Eq, Default)]
pub struct CpuCalibration {
pub dispatch_cost_ns: u64,
pub collapse_threshold_ns: u64,
pub jec_wake_threshold_ns: u64,
pub measured_unix_s: u64,
pub spread_per_mille: u32,
pub samples: u32,
pub occupancy_per_mille: u32,
pub confirmations: u32,
_pad: [u8; 16],
}
pub const OCCUPANCY_UNRECORDED: u32 = u32::MAX;
impl CpuCalibration {
pub fn new(
dispatch_cost_ns: u64,
collapse_threshold_ns: u64,
jec_wake_threshold_ns: u64,
spread_per_mille: u32,
samples: u32,
occupancy_per_mille: Option<u32>,
) -> Self {
Self {
dispatch_cost_ns,
collapse_threshold_ns,
jec_wake_threshold_ns,
measured_unix_s: now_unix_s(),
spread_per_mille,
samples,
occupancy_per_mille: occupancy_per_mille.unwrap_or(OCCUPANCY_UNRECORDED),
confirmations: 0,
_pad: [0; 16],
}
}
pub fn occupancy(&self) -> Option<u32> {
match self.occupancy_per_mille {
OCCUPANCY_UNRECORDED => None,
share => Some(share),
}
}
pub fn is_trustworthy(&self) -> bool {
self.samples > 0 && self.confirmations > 0
}
}
#[repr(u32)]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum AccelKind {
None = 0,
Cuda = 1,
GpuPeer = 2,
}
pub const ACCEL_DOORBELL_OK: u32 = 1 << 0;
pub const ACCEL_TIMED_LOCK_OK: u32 = 1 << 1;
pub const ACCEL_SYS_ATOMICS_OK: u32 = 1 << 2;
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct WaveCostRecord {
pub width: u32,
pub barrier_ns: u64,
pub fixed_ns: u64,
pub segment_ps: u64,
pub rebalance_fixed_ns: u64,
pub copy_ps_per_id: u64,
pub skew_ns: u32,
pub generation_ns: u64,
}
#[repr(C, align(64))]
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct AccelCalibration {
pub kind: u32,
pub ordinal: u32,
pub capability: u32,
pub multiprocessors: u32,
pub clock_khz: u32,
pub memory_mib: u32,
pub flags: u32,
pub spread_per_mille: u32,
pub rtt_min_ns: u64,
pub rtt_median_ns: u64,
pub rtt_p99_ns: u64,
pub one_way_ns: u64,
pub clock_err_ns: u64,
pub delta_ns: u64,
pub launch_ns: u64,
pub wave_width: u32,
pub wave_skew_ns: u32,
pub wave_barrier_ns: u64,
pub wave_fixed_ns: u64,
pub wave_segment_ps: u64,
pub wave_copy_ps_per_id: u64,
pub wave_generation_ns: u64,
pub wave_rebalance_fixed_ns: u64,
_pad: [u8; 48],
}
impl Default for AccelCalibration {
fn default() -> Self {
Self {
kind: AccelKind::None as u32,
ordinal: 0,
capability: 0,
multiprocessors: 0,
clock_khz: 0,
memory_mib: 0,
flags: 0,
spread_per_mille: 0,
rtt_min_ns: 0,
rtt_median_ns: 0,
rtt_p99_ns: 0,
one_way_ns: 0,
clock_err_ns: 0,
delta_ns: 0,
launch_ns: 0,
wave_width: 0,
wave_skew_ns: 0,
wave_barrier_ns: 0,
wave_fixed_ns: 0,
wave_segment_ps: 0,
wave_copy_ps_per_id: 0,
wave_generation_ns: 0,
wave_rebalance_fixed_ns: 0,
_pad: [0; 48],
}
}
}
impl AccelCalibration {
#[allow(clippy::too_many_arguments)]
pub fn new(
kind: AccelKind,
ordinal: u32,
capability: u32,
multiprocessors: u32,
clock_khz: u32,
memory_mib: u32,
flags: u32,
rtt_min_ns: u64,
rtt_median_ns: u64,
rtt_p99_ns: u64,
one_way_ns: u64,
clock_err_ns: u64,
delta_ns: u64,
launch_ns: u64,
) -> Self {
let spread_per_mille = crate::sched::par_iter::sample_spread_per_mille(&[
rtt_min_ns,
rtt_median_ns,
rtt_p99_ns,
]);
Self {
kind: kind as u32,
ordinal,
capability,
multiprocessors,
clock_khz,
memory_mib,
flags,
spread_per_mille,
rtt_min_ns,
rtt_median_ns,
rtt_p99_ns,
one_way_ns,
clock_err_ns,
delta_ns,
launch_ns,
wave_width: 0,
wave_skew_ns: 0,
wave_barrier_ns: 0,
wave_fixed_ns: 0,
wave_segment_ps: 0,
wave_copy_ps_per_id: 0,
wave_generation_ns: 0,
wave_rebalance_fixed_ns: 0,
_pad: [0; 48],
}
}
pub fn is_present(&self) -> bool {
self.kind != AccelKind::None as u32
}
pub fn is_trustworthy(&self) -> bool {
self.is_present() && self.spread_per_mille <= PROVISIONAL_SPREAD_PER_MILLE
}
pub fn has(&self, flag: u32) -> bool {
self.flags & flag != 0
}
pub fn with_wave(mut self, wave: WaveCostRecord) -> Self {
self.wave_width = wave.width;
self.wave_barrier_ns = wave.barrier_ns;
self.wave_fixed_ns = wave.fixed_ns;
self.wave_segment_ps = wave.segment_ps;
self.wave_copy_ps_per_id = wave.copy_ps_per_id;
self.wave_skew_ns = wave.skew_ns;
self.wave_rebalance_fixed_ns = wave.rebalance_fixed_ns;
self.wave_generation_ns = wave.generation_ns;
self
}
pub fn wave(&self) -> Option<WaveCostRecord> {
if self.wave_width == 0 {
return None;
}
Some(WaveCostRecord {
width: self.wave_width,
barrier_ns: self.wave_barrier_ns,
fixed_ns: self.wave_fixed_ns,
segment_ps: self.wave_segment_ps,
copy_ps_per_id: self.wave_copy_ps_per_id,
skew_ns: self.wave_skew_ns,
rebalance_fixed_ns: self.wave_rebalance_fixed_ns,
generation_ns: self.wave_generation_ns,
})
}
}
#[repr(C, align(64))]
struct Header {
magic: u64,
layout_version: u32,
record_bytes: u32,
seq_version: AtomicU32,
writer_pid: AtomicU32,
heartbeat_epoch: AtomicU64,
stamp_hash: u64,
accel_count: AtomicU32,
_pad: [u8; 20],
}
const FILE_SIZE: usize = size_of::<Header>()
+ size_of::<CpuCalibration>()
+ MAX_ACCEL * size_of::<AccelCalibration>();
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StoreError {
LayoutMismatch,
StampMismatch,
WriterActive,
Io(io::ErrorKind),
}
impl From<io::Error> for StoreError {
fn from(e: io::Error) -> Self {
Self::Io(e.kind())
}
}
pub fn calibration_dir() -> Option<PathBuf> {
if let Some(dir) = std::env::var_os("FLYNNEL_CALIBRATION_DIR") {
return Some(PathBuf::from(dir));
}
#[cfg(windows)]
{
std::env::var_os("LOCALAPPDATA")
.map(|d| PathBuf::from(d).join("flynnel").join("calibration"))
}
#[cfg(not(windows))]
{
if let Some(x) = std::env::var_os("XDG_CACHE_HOME") {
return Some(PathBuf::from(x).join("flynnel").join("calibration"));
}
std::env::var_os("HOME")
.map(|h| PathBuf::from(h).join(".cache").join("flynnel").join("calibration"))
}
}
pub fn table_path(dir: &Path, stamp: &HostStamp) -> PathBuf {
dir.join(format!("flynnel-{:016x}.cal", stamp.hash()))
}
pub struct CalibrationStore {
_file: File,
mmap: MmapMut,
stamp_hash: u64,
}
unsafe impl Send for CalibrationStore {}
unsafe impl Sync for CalibrationStore {}
impl CalibrationStore {
pub fn open_or_create(dir: &Path, stamp: &HostStamp) -> Result<Self, StoreError> {
std::fs::create_dir_all(dir)?;
let path = table_path(dir, stamp);
let stamp_hash = stamp.hash();
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)?;
let fresh = file.metadata()?.len() < FILE_SIZE as u64;
if fresh {
file.set_len(FILE_SIZE as u64)?;
}
let mut mmap = unsafe { MmapOptions::new().len(FILE_SIZE).map_mut(&file)? };
if fresh {
unsafe { lay_out(mmap.as_mut_ptr(), stamp_hash) };
} else {
let hdr = unsafe { &*(mmap.as_ptr() as *const Header) };
if hdr.magic != CALIBRATION_MAGIC
|| hdr.layout_version != LAYOUT_VERSION
|| hdr.record_bytes as usize != size_of::<CpuCalibration>()
{
return Err(StoreError::LayoutMismatch);
}
if hdr.stamp_hash != stamp_hash {
return Err(StoreError::StampMismatch);
}
}
Ok(Self { _file: file, mmap, stamp_hash })
}
pub fn stamp_hash(&self) -> u64 {
self.stamp_hash
}
pub fn clear(&self) -> Result<(), StoreError> {
let guard = self.try_acquire_writer()?;
guard.publish(&CpuCalibration::default(), &[]);
Ok(())
}
fn header(&self) -> &Header {
unsafe { &*(self.mmap.as_ptr() as *const Header) }
}
fn cpu_ptr(&self) -> *const CpuCalibration {
unsafe { self.mmap.as_ptr().add(size_of::<Header>()) as *const CpuCalibration }
}
fn accel_ptr(&self, i: usize) -> *const AccelCalibration {
let off = size_of::<Header>()
+ size_of::<CpuCalibration>()
+ i * size_of::<AccelCalibration>();
unsafe { self.mmap.as_ptr().add(off) as *const AccelCalibration }
}
pub fn read(&self) -> Option<(CpuCalibration, Vec<AccelCalibration>)> {
let hdr = self.header();
for _ in 0..1024 {
let before = hdr.seq_version.load(Ordering::Acquire);
if before & 1 != 0 {
std::hint::spin_loop();
continue;
}
let n = (hdr.accel_count.load(Ordering::Acquire) as usize).min(MAX_ACCEL);
let cpu = unsafe { std::ptr::read_volatile(self.cpu_ptr()) };
let mut accel = Vec::with_capacity(n);
for i in 0..n {
accel.push(unsafe { std::ptr::read_volatile(self.accel_ptr(i)) });
}
fence(Ordering::Acquire);
if hdr.seq_version.load(Ordering::Acquire) == before {
return Some((cpu, accel));
}
}
None
}
pub fn try_acquire_writer(&self) -> Result<WriterGuard<'_>, StoreError> {
let hdr = self.header();
let me = std::process::id();
loop {
let holder = hdr.writer_pid.load(Ordering::Acquire);
if holder == me {
return Ok(WriterGuard { store: self });
}
if holder != NO_WRITER {
let first = hdr.heartbeat_epoch.load(Ordering::Acquire);
std::thread::yield_now();
let second = hdr.heartbeat_epoch.load(Ordering::Acquire);
if second.wrapping_sub(first) > 0 || second == 0 {
return Err(StoreError::WriterActive);
}
if second.wrapping_sub(first) < LEASE_GRACE_EPOCHS
&& hdr.writer_pid.load(Ordering::Acquire) == holder
{
if hdr
.writer_pid
.compare_exchange(holder, me, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Ok(WriterGuard { store: self });
}
continue;
}
return Err(StoreError::WriterActive);
}
if hdr
.writer_pid
.compare_exchange(NO_WRITER, me, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
{
return Ok(WriterGuard { store: self });
}
}
}
}
pub struct WriterGuard<'a> {
store: &'a CalibrationStore,
}
fn two_draw_disagreement_per_mille(a: &CpuCalibration, b: &CpuCalibration) -> [u64; 3] {
fn gap(x: u64, y: u64) -> u64 {
let sum = x.saturating_add(y);
if sum == 0 {
return 0;
}
x.abs_diff(y).saturating_mul(2_000) / sum
}
[
gap(a.dispatch_cost_ns, b.dispatch_cost_ns),
gap(a.collapse_threshold_ns, b.collapse_threshold_ns),
gap(a.jec_wake_threshold_ns, b.jec_wake_threshold_ns),
]
}
fn confirmations_after(incumbent: &CpuCalibration, offered: &CpuCalibration) -> u32 {
if incumbent.samples == 0 {
return 0;
}
let agreed = two_draw_disagreement_per_mille(incumbent, offered)
.iter()
.all(|gap| *gap <= PROVISIONAL_SPREAD_PER_MILLE as u64);
if agreed {
incumbent.confirmations.saturating_add(1)
} else if incumbent.dispatch_cost_ns < offered.dispatch_cost_ns {
incumbent.confirmations
} else {
0
}
}
fn prefers_incumbent(
incumbent: &CpuCalibration,
offered: &CpuCalibration,
) -> Option<PublishOutcome> {
if incumbent.samples == 0 {
return None;
}
let (held, new) = (incumbent.dispatch_cost_ns, offered.dispatch_cost_ns);
(held < new).then_some(PublishOutcome::KeptIncumbent {
incumbent: held,
offered: new,
confirmations: confirmations_after(incumbent, offered),
})
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum PublishOutcome {
Published {
confirmations: u32,
},
KeptIncumbent {
incumbent: u64,
offered: u64,
confirmations: u32,
},
}
impl WriterGuard<'_> {
pub fn beat(&self) {
self.store
.header()
.heartbeat_epoch
.fetch_add(1, Ordering::Release);
}
pub fn publish_if_better(
&self,
cpu: &CpuCalibration,
accel: &[AccelCalibration],
) -> PublishOutcome {
if std::env::var_os("FLYNNEL_OCCUPANCY").is_some()
&& let Some((incumbent, _)) = self.store.read()
&& incumbent.samples > 0
{
let [dispatch, collapse, wake] = two_draw_disagreement_per_mille(&incumbent, cpu);
let floor = incumbent.dispatch_cost_ns.max(1);
let times_floor = cpu.dispatch_cost_ns as f64 / floor as f64;
notice!(
"flynnel: this draw costs {times_floor:.1}x the cheapest seen for this stamp \
({} against {} ns); two draws disagree by {dispatch},{collapse},{wake} per \
mille on dispatch,collapse,wake; incumbent spread {} occupancy {:?}, \
offered spread {} occupancy {:?}",
cpu.dispatch_cost_ns,
incumbent.dispatch_cost_ns,
incumbent.spread_per_mille,
incumbent.occupancy(),
cpu.spread_per_mille,
cpu.occupancy(),
);
}
if crate::sched::levers::calibration_refusal()
&& let Some((held, held_accel)) = self.store.read()
&& let Some(kept) = prefers_incumbent(&held, cpu)
{
if let PublishOutcome::KeptIncumbent { confirmations, .. } = kept
&& confirmations != held.confirmations
{
let mut raised = held;
raised.confirmations = confirmations;
self.publish(&raised, &held_accel);
}
return kept;
}
let mut record = *cpu;
record.confirmations = match self.store.read() {
Some((held, _)) => confirmations_after(&held, cpu),
None => 0,
};
self.publish(&record, accel);
PublishOutcome::Published { confirmations: record.confirmations }
}
pub fn publish(&self, cpu: &CpuCalibration, accel: &[AccelCalibration]) {
let hdr = self.store.header();
let n = accel.len().min(MAX_ACCEL);
hdr.seq_version.fetch_add(1, Ordering::AcqRel);
fence(Ordering::Release);
unsafe {
let cpu_dst = self.store.cpu_ptr() as *mut CpuCalibration;
std::ptr::write_volatile(cpu_dst, *cpu);
for (i, a) in accel.iter().take(n).enumerate() {
let dst = self.store.accel_ptr(i) as *mut AccelCalibration;
std::ptr::write_volatile(dst, *a);
}
}
hdr.accel_count.store(n as u32, Ordering::Release);
fence(Ordering::Release);
hdr.seq_version.fetch_add(1, Ordering::AcqRel);
}
}
impl Drop for WriterGuard<'_> {
fn drop(&mut self) {
let hdr = self.store.header();
let me = std::process::id();
if let Err(holder) =
hdr.writer_pid
.compare_exchange(me, NO_WRITER, Ordering::AcqRel, Ordering::Acquire)
{
notice!(
"flynnel: the calibration lease held by pid {me} was taken by pid {holder} \
mid-measurement; leaving it with the new holder"
);
}
}
}
unsafe fn lay_out(ptr: *mut u8, stamp_hash: u64) {
unsafe {
let hdr = ptr as *mut Header;
std::ptr::write(
hdr,
Header {
magic: 0,
layout_version: LAYOUT_VERSION,
record_bytes: size_of::<CpuCalibration>() as u32,
seq_version: AtomicU32::new(0),
writer_pid: AtomicU32::new(NO_WRITER),
heartbeat_epoch: AtomicU64::new(0),
stamp_hash,
accel_count: AtomicU32::new(0),
_pad: [0; 20],
},
);
let cpu = ptr.add(size_of::<Header>()) as *mut CpuCalibration;
std::ptr::write(cpu, CpuCalibration::default());
for i in 0..MAX_ACCEL {
let off = size_of::<Header>()
+ size_of::<CpuCalibration>()
+ i * size_of::<AccelCalibration>();
std::ptr::write(ptr.add(off) as *mut AccelCalibration, AccelCalibration::default());
}
fence(Ordering::Release);
std::ptr::write_volatile(std::ptr::addr_of_mut!((*hdr).magic), CALIBRATION_MAGIC);
}
}
pub fn now_unix_s() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
fn cleanup(dir: &Path) {
match std::fs::remove_dir_all(dir) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::NotFound => {}
Err(e) => eprintln!("calibration test left {} behind: {e}", dir.display()),
}
}
fn temp_dir(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("flynnel-cal-test-{name}-{}", std::process::id()));
cleanup(&d);
d
}
fn stamp(primary: u32, total: u32) -> HostStamp {
HostStamp {
vendor: "TestVendor".to_string(),
cpuid_signature: 0x000A_0671,
arch: "test-arch",
os: "test-os",
primary_workers: primary,
total_workers: total,
layout_version: LAYOUT_VERSION,
}
}
fn sample_cpu() -> CpuCalibration {
CpuCalibration {
dispatch_cost_ns: 1_000,
collapse_threshold_ns: 70_000,
jec_wake_threshold_ns: 40_000,
measured_unix_s: now_unix_s(),
spread_per_mille: 41,
samples: 9,
occupancy_per_mille: 970,
confirmations: 1,
_pad: [0; 16],
}
}
fn confirmed(mut cpu: CpuCalibration) -> CpuCalibration {
cpu.confirmations = 1;
cpu
}
#[test]
fn a_record_from_a_platform_with_no_thread_clock_reports_no_occupancy() {
let unmeasured =
CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, None);
assert_eq!(unmeasured.occupancy(), None);
assert_eq!(unmeasured.occupancy_per_mille, OCCUPANCY_UNRECORDED);
let starved = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(0));
assert_eq!(starved.occupancy(), Some(0));
}
#[test]
fn occupancy_is_provenance_and_gates_nothing() {
let quiet = confirmed(CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990)));
let loaded = confirmed(CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(210)));
assert!(quiet.is_trustworthy());
assert!(loaded.is_trustworthy());
assert!(quiet.occupancy() > loaded.occupancy());
}
fn offer_after(
tag: &str,
first: CpuCalibration,
second: CpuCalibration,
) -> (PublishOutcome, CpuCalibration, PathBuf) {
let dir = temp_dir(tag);
let s = stamp(12, 24);
let store = CalibrationStore::open_or_create(&dir, &s).expect("create");
{
let w = store.try_acquire_writer().expect("no other writer");
w.publish(&first, &[]);
}
let outcome = {
let w = store.try_acquire_writer().expect("still the only writer");
w.publish_if_better(&second, &[])
};
let (held, _) = store.read().expect("a record is present either way");
(outcome, held, dir)
}
#[test]
fn a_cheaper_draw_displaces_a_more_expensive_one() {
let loaded = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(210));
let quiet = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
assert_eq!(prefers_incumbent(&loaded, &quiet), None, "the cheaper draw wins");
}
#[test]
fn a_more_expensive_draw_does_not_displace_a_cheaper_one() {
let quiet = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
let loaded = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(210));
assert_eq!(
prefers_incumbent(&quiet, &loaded),
Some(PublishOutcome::KeptIncumbent {
incumbent: 1_000,
offered: 9_000,
confirmations: 0,
}),
"the refusal names both costs so a caller can say why"
);
}
#[test]
fn a_scattered_incumbent_stands_while_it_is_the_cheapest_seen() {
let scattered = CpuCalibration::new(
1_000,
70_000,
40_000,
PROVISIONAL_SPREAD_PER_MILLE + 1,
9,
Some(990),
);
assert!(!scattered.is_trustworthy(), "nothing has agreed with it");
let dearer = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(210));
assert_eq!(
prefers_incumbent(&scattered, &dearer),
Some(PublishOutcome::KeptIncumbent {
incumbent: 1_000,
offered: 9_000,
confirmations: 0,
}),
"cost orders them, and the incumbent's spread does not enter it"
);
}
#[test]
fn a_record_with_no_occupancy_is_ordered_like_any_other() {
let dear = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(990));
let cheap_unmeasured = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, None);
assert_eq!(
prefers_incumbent(&cheap_unmeasured, &dear),
Some(PublishOutcome::KeptIncumbent {
incumbent: 1_000,
offered: 9_000,
confirmations: 0,
}),
"a cheaper incumbent stands without an occupancy of its own"
);
assert_eq!(
prefers_incumbent(&dear, &cheap_unmeasured),
None,
"and a cheaper offer displaces without one either"
);
}
#[test]
fn an_equal_cost_publishes_so_a_fresher_measurement_wins_a_tie() {
let first = CpuCalibration::new(4_000, 70_000, 40_000, 41, 9, Some(700));
let second = CpuCalibration::new(4_000, 12_000, 9_000, 41, 9, Some(700));
assert_eq!(prefers_incumbent(&first, &second), None);
}
#[test]
fn a_table_nobody_published_never_wins_on_its_zero() {
let empty = CpuCalibration::new(0, 0, 0, 0, 0, None);
let measured = CpuCalibration::new(4_000, 70_000, 40_000, 41, 9, Some(990));
assert_eq!(
prefers_incumbent(&empty, &measured),
None,
"an unpublished table has no draw to defend"
);
}
#[test]
fn the_only_incumbent_the_caller_delivers_is_one_the_refusal_declines() {
let passes = confirmed(CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990)));
assert!(
passes.is_trustworthy(),
"a record like this is returned by the caller and never offered"
);
}
#[test]
fn two_draws_disagree_by_one_figure_whichever_is_called_the_incumbent() {
let quiet = CpuCalibration::new(1_000, 4_000, 4_200, 41, 9, Some(990));
let loaded = CpuCalibration::new(2_000, 20_000, 6_500, 41, 9, Some(660));
let forward = two_draw_disagreement_per_mille(&quiet, &loaded);
let backward = two_draw_disagreement_per_mille(&loaded, &quiet);
assert_eq!(forward, backward, "the figure must not depend on the argument order");
assert_eq!(forward, [666, 1333, 429]);
let same = two_draw_disagreement_per_mille(&quiet, &quiet);
assert_eq!(same, [0, 0, 0], "a draw against itself disagrees by nothing");
let empty = CpuCalibration::new(0, 0, 0, 41, 9, Some(990));
assert_eq!(two_draw_disagreement_per_mille(&empty, &empty), [0, 0, 0]);
}
#[test]
fn an_aged_incumbent_is_ordered_rather_than_dismissed() {
let aged = confirmed(CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990)));
assert!(aged.is_trustworthy(), "an aged record is drawn again, not distrusted");
let offered = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(210));
assert_eq!(
prefers_incumbent(&aged, &offered),
Some(PublishOutcome::KeptIncumbent {
incumbent: 1_000,
offered: 9_000,
confirmations: 1,
}),
"the cheaper of the two stands, keeping its agreement against a dearer offer"
);
}
#[test]
fn the_write_path_publishes_while_the_switch_is_off() {
let quiet = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
let loaded = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(210));
let (outcome, held, dir) = offer_after("switch-off", quiet, loaded);
if crate::sched::levers::calibration_refusal() {
assert_eq!(
outcome,
PublishOutcome::KeptIncumbent {
incumbent: 1_000,
offered: 9_000,
confirmations: 0,
},
"the switch is on in this process, so the refusal must fire"
);
assert_eq!(held.dispatch_cost_ns, 1_000);
} else {
assert_eq!(
outcome,
PublishOutcome::Published { confirmations: 0 },
"off publishes unconditionally, and the two costs did not agree"
);
assert_eq!(held.dispatch_cost_ns, 9_000, "the offer is what stands");
}
cleanup(&dir);
}
#[test]
fn a_fresh_table_offers_a_record_nothing_will_use() {
let dir = temp_dir("fresh");
let s = stamp(12, 24);
let store = CalibrationStore::open_or_create(&dir, &s).expect("create");
let (cpu, accel) = store.read().expect("a fresh table reads back");
assert_eq!(cpu.samples, 0, "nothing has been measured into it");
assert!(
!cpu.is_trustworthy(),
"so a caller must measure rather than take this"
);
assert!(accel.is_empty(), "and it describes no devices");
cleanup(&dir);
}
#[test]
fn a_published_record_reads_back_whole() {
let dir = temp_dir("roundtrip");
let s = stamp(12, 24);
let store = CalibrationStore::open_or_create(&dir, &s).expect("create");
let cpu = sample_cpu();
let accel = [AccelCalibration::new(
AccelKind::GpuPeer,
0,
89,
46,
2_505_000,
12_282,
ACCEL_DOORBELL_OK | ACCEL_TIMED_LOCK_OK,
3_000,
3_400,
3_700,
1_970,
120,
19_700,
41_000,
)];
{
let w = store.try_acquire_writer().expect("no other writer");
w.beat();
w.publish(&cpu, &accel);
}
let (got_cpu, got_accel) = store.read().expect("a published record reads back");
assert_eq!(got_cpu, cpu, "the CPU record survives the round trip");
assert_eq!(got_accel.len(), 1, "one device was published");
assert_eq!(got_accel[0], accel[0], "the device record survives too");
assert!(got_cpu.is_trustworthy(), "41 per mille is a quiet host");
assert!(
got_accel[0].is_trustworthy(),
"a round trip of 3000/3400/3700 is a device nothing else is resident on"
);
assert!(
got_accel[0].has(ACCEL_DOORBELL_OK) && got_accel[0].has(ACCEL_TIMED_LOCK_OK),
"the capability bits survive the round trip"
);
assert!(
!got_accel[0].has(ACCEL_SYS_ATOMICS_OK),
"a bit that was not set does not read as set"
);
cleanup(&dir);
}
#[test]
fn a_device_timed_under_load_is_not_trustworthy() {
let quiet = AccelCalibration::new(
AccelKind::GpuPeer, 0, 89, 46, 2_505_000, 12_282,
ACCEL_DOORBELL_OK, 3_000, 3_400, 3_700, 1_970, 120, 19_700, 41_000,
);
let loaded = AccelCalibration::new(
AccelKind::GpuPeer, 0, 89, 46, 2_505_000, 12_282,
ACCEL_DOORBELL_OK, 3_000, 3_400, 9_400, 4_820, 120, 48_200, 41_000,
);
assert!(quiet.is_trustworthy(), "a tight distribution describes the device");
assert!(
!loaded.is_trustworthy(),
"a p99 nearly three times the minimum describes what else was running"
);
assert!(
!AccelCalibration::default().is_trustworthy(),
"an empty slot describes no device at all"
);
}
#[test]
fn wave_costs_ride_the_device_record_through_a_round_trip() {
let dir = temp_dir("wave");
let s = stamp(12, 24);
let store = CalibrationStore::open_or_create(&dir, &s).expect("create");
let wave = WaveCostRecord {
width: 32,
barrier_ns: 2_100,
fixed_ns: 180_000,
segment_ps: 450,
copy_ps_per_id: 1_900,
skew_ns: 490_000,
rebalance_fixed_ns: 5_200,
generation_ns: 1_800,
};
let device = AccelCalibration::new(
AccelKind::GpuPeer, 0, 120, 48, 2_505_000, 12_227,
ACCEL_DOORBELL_OK, 3_000, 3_400, 3_700, 1_970, 120, 19_700, 41_000,
);
assert_eq!(device.wave(), None, "a device record starts with no wave costs");
let carried = device.with_wave(wave);
{
let w = store.try_acquire_writer().expect("no other writer");
w.publish(&sample_cpu(), &[carried]);
}
let (_cpu, accel) = store.read().expect("a published record reads back");
assert_eq!(accel[0].wave(), Some(wave), "the wave costs survive the round trip");
assert_eq!(accel[0].rtt_median_ns, 3_400, "and leave the device timings as they were");
cleanup(&dir);
}
#[test]
fn a_second_process_reads_what_the_first_published() {
let dir = temp_dir("reattach");
let s = stamp(12, 24);
let cpu = sample_cpu();
{
let store = CalibrationStore::open_or_create(&dir, &s).expect("create");
let w = store.try_acquire_writer().expect("no other writer");
w.publish(&cpu, &[]);
}
let reopened = CalibrationStore::open_or_create(&dir, &s).expect("attach");
let (got, accel) = reopened.read().expect("the record persists across handles");
assert_eq!(got, cpu, "a later attach reads the published record");
assert!(accel.is_empty(), "no devices were published");
cleanup(&dir);
}
#[test]
fn a_different_stamp_is_a_different_table() {
let dir = temp_dir("stamp");
let a = stamp(12, 24);
let b = stamp(8, 16);
assert_ne!(a.hash(), b.hash(), "core counts change the stamp");
assert_ne!(
table_path(&dir, &a),
table_path(&dir, &b),
"a different stamp names a different file, so neither reads the other's numbers"
);
cleanup(&dir);
}
#[test]
fn the_lease_releases_on_drop_and_is_retakeable() {
let dir = temp_dir("lease");
let s = stamp(12, 24);
let store = CalibrationStore::open_or_create(&dir, &s).expect("create");
{
let _w = store.try_acquire_writer().expect("free lease");
}
let _again = store
.try_acquire_writer()
.expect("a dropped guard leaves the lease free");
cleanup(&dir);
}
#[test]
fn the_provisional_threshold_separates_a_quiet_host_from_a_loaded_one() {
use crate::sched::par_iter::sample_spread_per_mille;
assert!(
sample_spread_per_mille(&[68_200, 69_600, 71_000]) < PROVISIONAL_SPREAD_PER_MILLE,
"a four percent spread is a host whose numbers describe the machine"
);
assert!(
sample_spread_per_mille(&[42_400, 55_000, 68_600]) > PROVISIONAL_SPREAD_PER_MILLE,
"a forty-eight percent spread is a host describing its load"
);
}
#[test]
fn trust_follows_agreement_between_draws_not_the_spread_within_one() {
let mut scattered = sample_cpu();
scattered.spread_per_mille = PROVISIONAL_SPREAD_PER_MILLE * 20;
assert!(
scattered.is_trustworthy(),
"a confirmed record stands however far its own nine samples scattered"
);
let mut tight = sample_cpu();
tight.spread_per_mille = 0;
tight.confirmations = 0;
assert!(
!tight.is_trustworthy(),
"and one nothing has agreed with does not, however tightly it sampled"
);
}
#[test]
fn a_draw_confirms_the_stored_record_when_their_medians_agree() {
let held = CpuCalibration::new(1_300, 70_000, 40_000, 41, 9, Some(990));
let near = CpuCalibration::new(1_400, 70_000, 40_000, 41, 9, Some(990));
assert_eq!(confirmations_after(&held, &near), 1);
let far = CpuCalibration::new(9_000, 70_000, 40_000, 41, 9, Some(210));
assert_eq!(
confirmations_after(&held, &far),
0,
"a dearer draw that disagrees leaves the count where it was"
);
let same_cost_far_collapse =
CpuCalibration::new(1_400, 29_000, 40_000, 41, 9, Some(990));
assert_eq!(
confirmations_after(&held, &same_cost_far_collapse),
0,
"dispatch within the bound does not carry a collapse outside it"
);
let cheaper_and_apart = CpuCalibration::new(200, 70_000, 40_000, 41, 9, Some(990));
assert_eq!(
confirmations_after(&confirmed(held), &cheaper_and_apart),
0,
"a cheaper draw that disagrees replaces and starts over"
);
assert_eq!(
confirmations_after(&confirmed(held), &far),
1,
"while a dearer one keeps the agreement already earned"
);
}
}