use std::collections::BTreeMap;
use std::fs;
use std::os::unix::process::ExitStatusExt;
use std::path::{Path, PathBuf};
use std::process::Command;
use crate::ExatokError;
pub const BYTES_PER_MB: f64 = 1_000_000.0;
pub const BYTES_PER_GB: f64 = 1_000_000_000.0;
pub const PLAUSIBLE_FLOOR_MB_S_PER_CORE: f64 = 5.0;
pub const MIN_TRUSTWORTHY_TIMED_SECS: f64 = 0.25;
pub const MIN_PARALLEL_EFFICIENCY: f64 = 0.5;
pub const MIN_SINGLE_THREAD_EFFICIENCY: f64 = 0.9;
pub const MAX_FOREIGN_LOAD_RATIO: f64 = 0.05;
pub const KNOWN_SLOW_BASELINES: &[(&str, &str)] = &[(
"hf-tokenizers",
"single-sourced, from an interactive run rather than this harness, pending \
re-derivation: 8.1 MB/s/core at 60 KB falling to 4.6 MB/s/core at 1 MB \
with encode_batch_fast, which returns ids only. The library is genuinely \
in this range on this vocabulary and degrades with document length; the \
offset-building encode() path is a further 1.4x to 2.0x slower and is not \
used for timing.",
)];
#[must_use]
pub fn known_slow_reason(tokenizer: &str) -> Option<&'static str> {
KNOWN_SLOW_BASELINES.iter().find(|(n, _)| *n == tokenizer).map(|(_, r)| *r)
}
pub const AMORTIZED_DOC_BYTES: usize = 4096;
pub const SIGNIFICANT_CLOCK_DRIFT: f64 = 0.05;
pub const NOISY_REL_STDDEV: f64 = 0.05;
pub const MAX_PUBLISHABLE_REL_STDDEV: f64 = 0.20;
pub const MIN_DEVICE_UTILIZATION: f64 = 0.25;
#[must_use]
pub fn corpus_digest(bytes: &[u8]) -> u64 {
let lo = crate::fnv1a32(crate::FNV1A32_OFFSET_BASIS, bytes);
let hi = crate::fnv1a32(crate::FNV1A32_PRIME, bytes);
(u64::from(hi) << 32) | u64::from(lo)
}
fn artifact_err(artifact: &str, location: &str, detail: &str, fix: &str) -> ExatokError {
ExatokError::Artifact {
artifact: artifact.to_owned(),
location: location.to_owned(),
detail: detail.to_owned(),
fix: fix.to_owned(),
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GpuInfo {
pub name: String,
pub memory_mib: String,
pub driver_version: String,
pub compute_capability: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Machine {
pub cpu_model: String,
pub physical_cores: usize,
pub logical_cores: usize,
pub physical_core_cpus: Vec<usize>,
pub governor: Option<String>,
pub total_memory_kib: u64,
pub gpu: Option<GpuInfo>,
}
impl Machine {
pub fn probe() -> Result<Self, ExatokError> {
let cpuinfo = fs::read_to_string("/proc/cpuinfo").map_err(|e| {
artifact_err(
"/proc/cpuinfo",
"open",
&e.to_string(),
"run the sweep on Linux, or supply a Machine literal",
)
})?;
Self::from_probe_inputs(&cpuinfo, Self::probe_sibling_lists(), Self::probe_governor(), Self::probe_meminfo(), Self::probe_gpu())
}
pub fn from_probe_inputs(
cpuinfo: &str,
sibling_lists: Vec<String>,
governor: Option<String>,
total_memory_kib: u64,
gpu: Option<GpuInfo>,
) -> Result<Self, ExatokError> {
let cpu_model = cpuinfo
.lines()
.find_map(|l| l.strip_prefix("model name").and_then(|r| r.split_once(':')))
.map_or_else(|| "unknown".to_owned(), |(_, v)| v.trim().to_owned());
let logical_cores = cpuinfo
.lines()
.filter(|l| l.starts_with("processor") && l.contains(':'))
.count();
if logical_cores == 0 {
return Err(artifact_err(
"/proc/cpuinfo",
"processor lines",
"no logical CPU found",
"a throughput figure without a core count is not comparable; fix the probe",
));
}
let (physical_cores, physical_core_cpus) =
Self::physical_from_siblings(&sibling_lists, logical_cores);
Ok(Self {
cpu_model,
physical_cores,
logical_cores,
physical_core_cpus,
governor,
total_memory_kib,
gpu,
})
}
#[must_use]
pub fn physical_from_siblings(
sibling_lists: &[String],
logical_cores: usize,
) -> (usize, Vec<usize>) {
if sibling_lists.len() != logical_cores {
return (logical_cores, (0..logical_cores).collect());
}
let mut groups: BTreeMap<String, usize> = BTreeMap::new();
for (cpu, list) in sibling_lists.iter().enumerate() {
let key = list.trim().to_owned();
groups.entry(key).and_modify(|rep| *rep = (*rep).min(cpu)).or_insert(cpu);
}
let mut reps: Vec<usize> = groups.values().copied().collect();
reps.sort_unstable();
(reps.len(), reps)
}
#[must_use]
pub fn smt_siblings_per_core(&self) -> usize {
if self.physical_cores == 0 { 1 } else { self.logical_cores / self.physical_cores }
}
fn probe_sibling_lists() -> Vec<String> {
let mut out = Vec::new();
for cpu in 0.. {
let p = format!("/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list");
match fs::read_to_string(&p) {
Ok(s) => out.push(s.trim().to_owned()),
Err(_) => break,
}
}
out
}
fn probe_governor() -> Option<String> {
fs::read_to_string("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor")
.ok()
.map(|s| s.trim().to_owned())
}
fn probe_meminfo() -> u64 {
fs::read_to_string("/proc/meminfo")
.ok()
.and_then(|s| {
s.lines()
.find_map(|l| l.strip_prefix("MemTotal:"))
.and_then(|r| r.split_whitespace().next().map(str::to_owned))
})
.and_then(|n| n.parse().ok())
.unwrap_or(0)
}
fn probe_gpu() -> Option<GpuInfo> {
let out = Command::new("nvidia-smi")
.args([
"--query-gpu=name,memory.total,driver_version,compute_cap",
"--format=csv,noheader",
])
.output()
.ok()?;
if !out.status.success() {
return None;
}
let text = String::from_utf8_lossy(&out.stdout);
let line = text.lines().next()?;
let mut f = line.split(',').map(str::trim);
Some(GpuInfo {
name: f.next()?.to_owned(),
memory_mib: f.next()?.to_owned(),
driver_version: f.next()?.to_owned(),
compute_capability: f.next()?.to_owned(),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct ClockSample {
pub mean_mhz: f64,
pub min_mhz: f64,
pub max_mhz: f64,
}
impl ClockSample {
#[must_use]
pub fn parse(cpuinfo: &str) -> Option<Self> {
let mhz: Vec<f64> = cpuinfo
.lines()
.filter_map(|l| l.strip_prefix("cpu MHz"))
.filter_map(|r| r.split_once(':'))
.filter_map(|(_, v)| v.trim().parse::<f64>().ok())
.filter(|v| v.is_finite() && *v > 0.0)
.collect();
if mhz.is_empty() {
return None;
}
let sum: f64 = mhz.iter().sum();
let mut min = f64::INFINITY;
let mut max = f64::NEG_INFINITY;
for v in &mhz {
min = min.min(*v);
max = max.max(*v);
}
#[allow(clippy::cast_precision_loss)]
let n = mhz.len() as f64;
Some(Self { mean_mhz: sum / n, min_mhz: min, max_mhz: max })
}
#[must_use]
pub fn probe() -> Option<Self> {
Self::parse(&fs::read_to_string("/proc/cpuinfo").ok()?)
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct ClockDrift {
pub first: ClockSample,
pub last: ClockSample,
}
impl ClockDrift {
#[must_use]
pub fn drift_ratio(&self) -> f64 {
if self.first.mean_mhz <= 0.0 {
return 0.0;
}
(self.last.mean_mhz - self.first.mean_mhz) / self.first.mean_mhz
}
#[must_use]
pub fn is_significant(&self) -> bool {
self.drift_ratio().abs() >= SIGNIFICANT_CLOCK_DRIFT
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct RegionClock {
pub start_mhz: f64,
pub end_mhz: f64,
pub busy_cpus: u32,
}
impl RegionClock {
#[must_use]
pub fn drift_ratio(&self) -> f64 {
if self.start_mhz <= 0.0 {
return 0.0;
}
(self.end_mhz - self.start_mhz) / self.start_mhz
}
#[must_use]
pub fn is_significant(&self) -> bool {
self.drift_ratio().abs() >= SIGNIFICANT_CLOCK_DRIFT
}
}
const CORPUS_SKIP_DIRS: &[&str] =
&["target", ".git", "node_modules", ".venv", "bench-venv", "_archive", ".cargo-target"];
const CORPUS_EXTENSIONS: &[&str] = &["md", "rs", "toml", "txt", "py"];
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CorpusFacts {
pub origin: String,
pub bytes: usize,
pub files: usize,
pub digest: u64,
}
#[derive(Debug, Clone)]
pub struct Corpus {
pub text: String,
pub facts: CorpusFacts,
}
impl Corpus {
pub fn from_tree(root: &Path, target_bytes: usize) -> Result<Self, ExatokError> {
let mut text = String::with_capacity(target_bytes.min(1 << 26));
let mut files = 0usize;
Self::walk(root, target_bytes, &mut text, &mut files);
if text.is_empty() {
return Err(artifact_err(
"benchmark corpus",
&root.display().to_string(),
"no admissible text files found under the corpus root",
"pass an explicit corpus file, or point the root at a tree containing .md or .rs files",
));
}
let bytes = text.len();
let digest = corpus_digest(text.as_bytes());
Ok(Self {
text,
facts: CorpusFacts {
origin: format!("deterministic sorted walk of {}", root.display()),
bytes,
files,
digest,
},
})
}
pub fn from_file(path: &Path, target_bytes: usize) -> Result<Self, ExatokError> {
let raw = fs::read_to_string(path).map_err(|e| {
artifact_err(
"benchmark corpus",
&path.display().to_string(),
&e.to_string(),
"supply a readable UTF-8 text file",
)
})?;
let text = truncate_at_char_boundary(&raw, target_bytes).to_owned();
if text.is_empty() {
return Err(artifact_err(
"benchmark corpus",
&path.display().to_string(),
"corpus file is empty after truncation",
"supply a non-empty corpus, or raise the byte budget",
));
}
let bytes = text.len();
let digest = corpus_digest(text.as_bytes());
Ok(Self {
text,
facts: CorpusFacts {
origin: format!("explicit file {}", path.display()),
bytes,
files: 1,
digest,
},
})
}
fn walk(dir: &Path, target: usize, out: &mut String, files: &mut usize) {
if out.len() >= target {
return;
}
let Ok(entries) = fs::read_dir(dir) else { return };
let mut dirs: Vec<PathBuf> = Vec::new();
let mut texts: Vec<PathBuf> = Vec::new();
for e in entries.flatten() {
let path = e.path();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else { continue };
if e.file_type().is_ok_and(|t| t.is_dir()) {
if !CORPUS_SKIP_DIRS.contains(&name) && !name.starts_with('.') {
dirs.push(path);
}
} else if path
.extension()
.and_then(|x| x.to_str())
.is_some_and(|x| CORPUS_EXTENSIONS.contains(&x))
{
texts.push(path);
}
}
texts.sort();
dirs.sort();
for f in texts {
if out.len() >= target {
return;
}
if let Ok(s) = fs::read_to_string(&f) {
let room = target - out.len();
out.push_str(truncate_at_char_boundary(&s, room));
*files += 1;
}
}
for d in dirs {
Self::walk(&d, target, out, files);
}
}
}
#[must_use]
pub fn truncate_at_char_boundary(s: &str, max: usize) -> &str {
if s.len() <= max {
return s;
}
let mut end = max;
while end > 0 && !s.is_char_boundary(end) {
end -= 1;
}
&s[..end]
}
pub const DEFAULT_TARGET_TOKENS: &[usize] =
&[256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576, 4_194_304];
#[must_use]
pub const fn batch_docs(doc_bytes: usize, budget_bytes: usize) -> usize {
if doc_bytes == 0 {
return 1;
}
let n = budget_bytes.div_ceil(doc_bytes);
if n == 0 { 1 } else { n }
}
#[must_use]
pub const fn batch_admits(doc_bytes: usize, budget_bytes: usize, threads: usize) -> bool {
batch_docs(doc_bytes, budget_bytes) >= threads
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Regime {
SingleDocument,
Batch,
}
impl Regime {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::SingleDocument => "single",
Self::Batch => "batch",
}
}
#[must_use]
pub fn from_str_opt(s: &str) -> Option<Self> {
match s {
"single" => Some(Self::SingleDocument),
"batch" => Some(Self::Batch),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ThreadPlan {
pub threads: usize,
pub cpus: Vec<usize>,
pub pinned_to_physical: bool,
pub include_single_doc: bool,
}
impl ThreadPlan {
#[must_use]
pub fn standard(machine: &Machine) -> Vec<Self> {
let phys = &machine.physical_core_cpus;
let mut plans = vec![Self {
threads: 1,
cpus: phys.first().copied().map(|c| vec![c]).unwrap_or_default(),
pinned_to_physical: !phys.is_empty(),
include_single_doc: true,
}];
if machine.physical_cores > 1 {
plans.push(Self {
threads: machine.physical_cores,
cpus: phys.clone(),
pinned_to_physical: true,
include_single_doc: false,
});
}
if machine.logical_cores > machine.physical_cores {
plans.push(Self {
threads: machine.logical_cores,
cpus: Vec::new(),
pinned_to_physical: false,
include_single_doc: false,
});
}
plans
}
#[must_use]
pub fn cpu_list(&self) -> Option<String> {
if self.cpus.is_empty() {
return None;
}
let mut s = String::new();
for (i, c) in self.cpus.iter().enumerate() {
if i > 0 {
s.push(',');
}
s.push_str(&c.to_string());
}
Some(s)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct RunStats {
samples: Vec<f64>,
}
impl RunStats {
pub fn new(samples: Vec<f64>) -> Result<Self, ExatokError> {
if samples.is_empty() {
return Err(artifact_err(
"measurement samples",
"count",
"no wall-clock samples",
"raise --reps to at least 1",
));
}
if let Some(bad) = samples.iter().find(|s| !s.is_finite() || **s <= 0.0) {
return Err(artifact_err(
"measurement samples",
"value",
&format!("non-positive or non-finite wall time {bad}"),
"a zero wall time divides into infinite throughput; use a monotonic clock and a larger input",
));
}
Ok(Self { samples })
}
#[must_use]
pub fn reps(&self) -> usize {
self.samples.len()
}
#[must_use]
pub fn best(&self) -> f64 {
self.samples.iter().copied().fold(f64::INFINITY, f64::min)
}
#[must_use]
pub fn mean(&self) -> f64 {
#[allow(clippy::cast_precision_loss)]
let n = self.samples.len() as f64;
self.samples.iter().sum::<f64>() / n
}
#[must_use]
pub fn median(&self) -> f64 {
let mut v = self.samples.clone();
v.sort_by(f64::total_cmp);
v[v.len() / 2]
}
#[must_use]
pub fn rel_stddev(&self) -> f64 {
if self.samples.len() < 2 {
return 0.0;
}
let mean = self.mean();
if mean <= 0.0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let n1 = (self.samples.len() - 1) as f64;
let var = self.samples.iter().map(|s| (s - mean) * (s - mean)).sum::<f64>() / n1;
var.sqrt() / mean
}
#[must_use]
pub fn samples(&self) -> &[f64] {
&self.samples
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MachineContext {
pub physical_cores: usize,
pub logical_cores: usize,
pub pinned_to_physical: bool,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct ActiveTime {
pub cpu_secs: f64,
pub device_secs: Option<f64>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Measurement {
pub tokenizer: String,
pub version: String,
pub vocab: String,
pub regime: Regime,
pub target_tokens: usize,
pub doc_bytes: usize,
pub docs: usize,
pub input_bytes: u64,
pub tokens: u64,
pub stats: RunStats,
pub warmups: u32,
pub threads: usize,
pub machine: MachineContext,
pub mean_mhz: f64,
pub region_clock: Option<RegionClock>,
pub matches_reference: bool,
pub active: ActiveTime,
pub foreign_cpu_secs: f64,
pub inner_reps: u32,
}
impl Measurement {
#[must_use]
pub fn timed_bytes(&self) -> u64 {
self.input_bytes.saturating_mul(u64::from(self.inner_reps))
}
#[must_use]
pub fn timed_tokens(&self) -> u64 {
self.tokens.saturating_mul(u64::from(self.inner_reps))
}
#[must_use]
pub fn bytes_per_second(&self) -> f64 {
let t = self.stats.best();
if t <= 0.0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let b = self.timed_bytes() as f64;
b / t
}
#[must_use]
pub fn mb_per_second(&self) -> f64 {
self.bytes_per_second() / BYTES_PER_MB
}
#[must_use]
pub fn gb_per_second(&self) -> f64 {
self.bytes_per_second() / BYTES_PER_GB
}
#[must_use]
pub fn mb_per_second_per_thread(&self) -> f64 {
if self.threads == 0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let t = self.threads as f64;
self.mb_per_second() / t
}
#[must_use]
pub fn mb_per_second_per_core(&self) -> f64 {
let cores = self.threads.min(self.machine.physical_cores);
if cores == 0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let c = cores as f64;
self.mb_per_second() / c
}
#[must_use]
pub fn tokens_per_second(&self) -> f64 {
let t = self.stats.best();
if t <= 0.0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let n = self.timed_tokens() as f64;
n / t
}
#[must_use]
pub fn ns_per_token(&self) -> f64 {
let tps = self.tokens_per_second();
if tps <= 0.0 {
return 0.0;
}
1e9 / tps
}
#[must_use]
pub fn bytes_per_token(&self) -> f64 {
if self.tokens == 0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let (b, t) = (self.input_bytes as f64, self.tokens as f64);
b / t
}
#[must_use]
pub fn active_secs(&self) -> f64 {
self.active.device_secs.unwrap_or(self.active.cpu_secs)
}
#[must_use]
pub fn achieved_parallelism(&self) -> f64 {
let wall = self.stats.best();
if wall <= 0.0 {
return 0.0;
}
self.active.cpu_secs / wall
}
#[must_use]
pub fn foreign_load_ratio(&self) -> f64 {
let wall = self.stats.best();
if wall <= 0.0 {
return 0.0;
}
self.foreign_cpu_secs / wall
}
#[must_use]
pub fn device_utilization(&self) -> Option<f64> {
let wall = self.stats.best();
if wall <= 0.0 {
return None;
}
self.active.device_secs.map(|d| d / wall)
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum Availability {
Measured(Measurement),
Unavailable {
tokenizer: String,
reason: String,
},
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Plausibility {
Plausible,
PerCallBound,
ShortTimedRegion {
timed_secs: f64,
minimum_secs: f64,
},
ThreadsNotHonoured {
requested: usize,
achieved_parallelism: f64,
},
Contended {
achieved_parallelism: f64,
minimum_efficiency: f64,
},
ForeignLoad {
foreign_load_ratio: f64,
maximum_ratio: f64,
},
EstimatorDependent {
rel_stddev: f64,
maximum_rel_stddev: f64,
},
DeviceStarved {
device_utilization: f64,
minimum_utilization: f64,
},
KnownSlowLibrary {
observed_mb_s_per_core: f64,
reason: &'static str,
},
SuspectedMeasurementDefect {
observed_mb_s_per_core: f64,
floor_mb_s_per_core: f64,
},
}
impl Plausibility {
#[must_use]
pub const fn is_harness_defect(self) -> bool {
matches!(
self,
Self::ShortTimedRegion { .. }
| Self::ThreadsNotHonoured { .. }
| Self::Contended { .. }
| Self::ForeignLoad { .. }
| Self::EstimatorDependent { .. }
| Self::DeviceStarved { .. }
)
}
#[must_use]
pub const fn is_publishable(self) -> bool {
!self.is_harness_defect()
}
#[must_use]
pub const fn table_flag(self) -> &'static str {
match self {
Self::ShortTimedRegion { .. } => " <-- HARNESS: TIMED REGION TOO SHORT",
Self::ThreadsNotHonoured { .. } => " <-- HARNESS: THREADS NOT DELIVERED",
Self::Contended { .. } => " <-- HARNESS: CORE NOT EXCLUSIVE",
Self::ForeignLoad { .. } => " <-- HARNESS: MACHINE NOT EXCLUSIVE",
Self::EstimatorDependent { .. } => " <-- HARNESS: REPETITIONS DISAGREE",
Self::DeviceStarved { .. } => " <-- HARNESS: DEVICE NOT EXCLUSIVE",
Self::SuspectedMeasurementDefect { .. } => " <-- UNEXPLAINED, UNDER FLOOR",
Self::KnownSlowLibrary { .. } => " <-- known slow, see note",
Self::PerCallBound | Self::Plausible => "",
}
}
#[must_use]
pub fn diagnosis(self) -> Option<String> {
match self {
Self::ShortTimedRegion { timed_secs, minimum_secs } => Some(format!(
"timed region {timed_secs:.4} s is under the {minimum_secs:.2} s minimum; \
raise inner repetitions or the byte budget"
)),
Self::ThreadsNotHonoured { requested, achieved_parallelism } => Some(format!(
"requested {requested} threads but achieved {achieved_parallelism:.2}x \
parallelism; the thread request was not delivered"
)),
Self::Contended { achieved_parallelism, minimum_efficiency } => Some(format!(
"pinned single-threaded cell achieved only {achieved_parallelism:.3}x of one \
core against a {minimum_efficiency:.2} minimum; another process was on that \
CPU, so this figure understates the baseline"
)),
Self::ForeignLoad { foreign_load_ratio, maximum_ratio } => Some(format!(
"{foreign_load_ratio:.3}x of one core was consumed elsewhere on the machine \
against a {maximum_ratio:.2} maximum; this cell measured a contended machine \
and understates the baseline"
)),
Self::EstimatorDependent { rel_stddev, maximum_rel_stddev } => Some(format!(
"repetitions spread {:.1}% against a {:.0}% maximum; the headline figure is \
whichever repetition was picked rather than a property of the tokenizer",
rel_stddev * 100.0,
maximum_rel_stddev * 100.0
)),
Self::DeviceStarved { device_utilization, minimum_utilization } => Some(format!(
"the device was busy for only {:.1}% of the wall window against a {:.0}% \
minimum; this cell measured queueing or host stalls rather than device \
throughput",
device_utilization * 100.0,
minimum_utilization * 100.0
)),
Self::SuspectedMeasurementDefect { observed_mb_s_per_core, floor_mb_s_per_core } => {
Some(format!(
"{observed_mb_s_per_core:.3} MB/s/core is under the \
{floor_mb_s_per_core:.1} MB/s/core floor with no recorded explanation"
))
}
Self::KnownSlowLibrary { observed_mb_s_per_core, reason } => Some(format!(
"{observed_mb_s_per_core:.3} MB/s/core, legitimately: {reason}"
)),
Self::PerCallBound | Self::Plausible => None,
}
}
}
#[must_use]
pub fn assess_plausibility(m: &Measurement) -> Plausibility {
let timed = m.stats.best();
if timed < MIN_TRUSTWORTHY_TIMED_SECS {
return Plausibility::ShortTimedRegion {
timed_secs: timed,
minimum_secs: MIN_TRUSTWORTHY_TIMED_SECS,
};
}
let foreign = m.foreign_load_ratio();
if foreign > MAX_FOREIGN_LOAD_RATIO {
return Plausibility::ForeignLoad {
foreign_load_ratio: foreign,
maximum_ratio: MAX_FOREIGN_LOAD_RATIO,
};
}
if let Some(utilization) = m.device_utilization() {
if utilization < MIN_DEVICE_UTILIZATION {
return Plausibility::DeviceStarved {
device_utilization: utilization,
minimum_utilization: MIN_DEVICE_UTILIZATION,
};
}
}
if m.threads > 1 && m.threads <= m.machine.physical_cores {
let achieved = m.achieved_parallelism();
#[allow(clippy::cast_precision_loss)]
let floor = m.threads as f64 * MIN_PARALLEL_EFFICIENCY;
if achieved < floor {
return Plausibility::ThreadsNotHonoured {
requested: m.threads,
achieved_parallelism: achieved,
};
}
}
if m.threads == 1 {
let achieved = m.achieved_parallelism();
if achieved < MIN_SINGLE_THREAD_EFFICIENCY {
return Plausibility::Contended {
achieved_parallelism: achieved,
minimum_efficiency: MIN_SINGLE_THREAD_EFFICIENCY,
};
}
}
let spread = m.stats.rel_stddev();
if spread > MAX_PUBLISHABLE_REL_STDDEV {
return Plausibility::EstimatorDependent {
rel_stddev: spread,
maximum_rel_stddev: MAX_PUBLISHABLE_REL_STDDEV,
};
}
if m.doc_bytes < AMORTIZED_DOC_BYTES {
return Plausibility::PerCallBound;
}
let per_core = m.mb_per_second_per_core();
if per_core < PLAUSIBLE_FLOOR_MB_S_PER_CORE {
return match known_slow_reason(&m.tokenizer) {
Some(reason) => {
Plausibility::KnownSlowLibrary { observed_mb_s_per_core: per_core, reason }
}
None => Plausibility::SuspectedMeasurementDefect {
observed_mb_s_per_core: per_core,
floor_mb_s_per_core: PLAUSIBLE_FLOOR_MB_S_PER_CORE,
},
};
}
Plausibility::Plausible
}
#[derive(Debug, Clone, PartialEq)]
pub enum DriverEvent {
Record(Measurement),
Unavailable {
tokenizer: String,
reason: String,
},
Calibration {
vocab: String,
bytes_per_token: f64,
},
}
fn field<'a>(
fields: &BTreeMap<&'a str, &'a str>,
key: &str,
location: &str,
) -> Result<&'a str, ExatokError> {
fields.get(key).copied().ok_or_else(|| {
artifact_err(
"driver record stream",
location,
&format!("missing field `{key}`"),
"the driver and bench_support disagree on the record protocol; update both together",
)
})
}
fn num<T: std::str::FromStr>(
fields: &BTreeMap<&str, &str>,
key: &str,
location: &str,
) -> Result<T, ExatokError> {
let raw = field(fields, key, location)?;
raw.parse().map_err(|_| {
artifact_err(
"driver record stream",
location,
&format!("field `{key}` is not a number: {raw:?}"),
"emit plain decimal values with no thousands separators or units",
)
})
}
pub fn parse_driver_line(
line: &str,
machine: MachineContext,
line_no: usize,
) -> Result<Option<DriverEvent>, ExatokError> {
let line = line.trim_end_matches(['\r', '\n']);
let mut parts = line.split('\t');
let Some(kind) = parts.next() else { return Ok(None) };
if !matches!(kind, "record" | "unavailable" | "calibration") {
return Ok(None);
}
let location = format!("line {line_no}");
let mut fields: BTreeMap<&str, &str> = BTreeMap::new();
for p in parts {
if let Some((k, v)) = p.split_once('=') {
fields.insert(k, v);
}
}
match kind {
"unavailable" => Ok(Some(DriverEvent::Unavailable {
tokenizer: field(&fields, "tokenizer", &location)?.to_owned(),
reason: field(&fields, "reason", &location)?.to_owned(),
})),
"calibration" => Ok(Some(DriverEvent::Calibration {
vocab: field(&fields, "vocab", &location)?.to_owned(),
bytes_per_token: num(&fields, "bytes_per_token", &location)?,
})),
_ => {
let regime_raw = field(&fields, "regime", &location)?;
let regime = Regime::from_str_opt(regime_raw).ok_or_else(|| {
artifact_err(
"driver record stream",
&location,
&format!("unknown regime {regime_raw:?}"),
"regime must be `single` or `batch`",
)
})?;
let samples_raw = field(&fields, "samples", &location)?;
let mut samples = Vec::new();
for s in samples_raw.split(',').filter(|s| !s.is_empty()) {
samples.push(s.parse::<f64>().map_err(|_| {
artifact_err(
"driver record stream",
&location,
&format!("sample {s:?} is not a number"),
"emit comma-separated decimal seconds",
)
})?);
}
Ok(Some(DriverEvent::Record(Measurement {
tokenizer: field(&fields, "tokenizer", &location)?.to_owned(),
version: field(&fields, "version", &location)?.to_owned(),
vocab: field(&fields, "vocab", &location)?.to_owned(),
regime,
target_tokens: num(&fields, "target_tokens", &location)?,
doc_bytes: num(&fields, "doc_bytes", &location)?,
docs: num(&fields, "docs", &location)?,
input_bytes: num(&fields, "input_bytes", &location)?,
tokens: num(&fields, "tokens", &location)?,
stats: RunStats::new(samples)?,
warmups: num(&fields, "warmups", &location)?,
threads: num(&fields, "threads", &location)?,
machine,
mean_mhz: num(&fields, "mean_mhz", &location)?,
region_clock: match (
fields.get("busy_mhz_start"),
fields.get("busy_mhz_end"),
fields.get("busy_cpus"),
) {
(Some(_), Some(_), Some(_)) => Some(RegionClock {
start_mhz: num(&fields, "busy_mhz_start", &location)?,
end_mhz: num(&fields, "busy_mhz_end", &location)?,
busy_cpus: num(&fields, "busy_cpus", &location)?,
}),
_ => None,
},
matches_reference: field(&fields, "matches_ref", &location)? != "0",
active: ActiveTime {
cpu_secs: num(&fields, "cpu_secs", &location)?,
device_secs: match fields.get("device_secs") {
Some(_) => Some(num(&fields, "device_secs", &location)?),
None => None,
},
},
foreign_cpu_secs: num(&fields, "foreign_cpu_secs", &location)?,
inner_reps: num(&fields, "inner_reps", &location)?,
})))
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Provenance {
Probed,
Derived,
Fixture,
}
impl Provenance {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Probed => "probed",
Self::Derived => "derived",
Self::Fixture => "fixture",
}
}
#[must_use]
pub const fn is_quotable_as_measured(self) -> bool {
matches!(self, Self::Probed)
}
}
#[must_use]
pub fn compiled_features() -> Vec<&'static str> {
let mut out = Vec::new();
if cfg!(feature = "confluence") {
out.push("confluence");
}
if cfg!(feature = "cuda") {
out.push("cuda");
}
if cfg!(feature = "gpu-pretokenize") {
out.push("gpu-pretokenize");
}
if cfg!(feature = "multi-merge") {
out.push("multi-merge");
}
out
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TestCount {
pub passed: u32,
pub ignored: u32,
pub features: Vec<String>,
}
impl TestCount {
#[must_use]
pub fn here(passed: u32, ignored: u32) -> Self {
Self {
passed,
ignored,
features: compiled_features().into_iter().map(str::to_owned).collect(),
}
}
#[must_use]
pub fn render(&self) -> String {
let features =
if self.features.is_empty() { "default".to_owned() } else { self.features.join(",") };
format!("{} passed / {} ignored (features: {features})", self.passed, self.ignored, )
}
#[must_use]
pub fn comparable_to(&self, other: &Self) -> bool {
self.features == other.features
}
}
#[derive(Debug, Clone)]
pub struct Report {
pub machine: Machine,
pub corpus: CorpusFacts,
pub vocab: String,
pub rows: Vec<Availability>,
pub clock: Option<ClockDrift>,
pub bytes_per_token: Option<f64>,
pub report_path: Option<PathBuf>,
}
impl Report {
#[must_use]
pub fn measured(&self) -> impl Iterator<Item = &Measurement> {
self.rows.iter().filter_map(|r| match r {
Availability::Measured(m) => Some(m),
Availability::Unavailable { .. } => None,
})
}
#[must_use]
pub fn suspected_defects(&self) -> Vec<&Measurement> {
self.measured()
.filter(|m| {
matches!(assess_plausibility(m), Plausibility::SuspectedMeasurementDefect { .. })
})
.collect()
}
#[must_use]
pub fn harness_defects(&self) -> Vec<&Measurement> {
self.measured().filter(|m| assess_plausibility(m).is_harness_defect()).collect()
}
#[must_use]
pub fn known_slow(&self) -> Vec<&Measurement> {
self.measured()
.filter(|m| matches!(assess_plausibility(m), Plausibility::KnownSlowLibrary { .. }))
.collect()
}
#[must_use]
pub fn disagreeing(&self) -> Vec<&Measurement> {
self.measured().filter(|m| !m.matches_reference).collect()
}
#[must_use]
pub fn peak_for(&self, tokenizer: &str) -> Option<&Measurement> {
self.measured()
.filter(|m| m.tokenizer == tokenizer)
.max_by(|a, b| a.mb_per_second().total_cmp(&b.mb_per_second()))
}
#[must_use]
pub fn render_table(&self) -> String {
let mut out = String::new();
out.push_str(&self.render_header());
out.push_str(
"\n== absolute throughput (no ratios; divide two rows yourself if you want one) ==\n\n",
);
out.push_str(&format!(
"{:<13} {:<7} {:>4} {:>10} {:>7} {:>9} {:>7} {:>10} {:>6} {:>11} {:>8} {:>10} {:>12} {:>8} {:>7} {:>5} {:>6}\n",
"tokenizer",
"regime",
"thr",
"doc_bytes",
"docs",
"unit_MB",
"xreps",
"tokens",
"B/tok",
"MB/s",
"GB/s",
"MB/s/core",
"MB/s/thread",
"ns/tok",
"relsd%",
"exact",
"par",
));
out.push_str(&"-".repeat(162));
out.push('\n');
let mut rows: Vec<&Measurement> = self.measured().collect();
rows.sort_by(|a, b| {
a.tokenizer
.cmp(&b.tokenizer)
.then(a.regime.as_str().cmp(b.regime.as_str()))
.then(a.threads.cmp(&b.threads))
.then(a.target_tokens.cmp(&b.target_tokens))
});
for m in rows {
#[allow(clippy::cast_precision_loss)]
let unit_mb = m.input_bytes as f64 / BYTES_PER_MB;
let flag = assess_plausibility(m).table_flag();
out.push_str(&format!(
"{:<13} {:<7} {:>4} {:>10} {:>7} {:>9.4} {:>7} {:>10} {:>6.3} {:>11.2} {:>8.3} {:>10.2} {:>12.2} {:>8.1} {:>7.2} {:>5} {:>6.2}{}\n",
m.tokenizer,
m.regime.as_str(),
m.threads,
m.doc_bytes,
m.docs,
unit_mb,
m.inner_reps,
m.tokens,
m.bytes_per_token(),
m.mb_per_second(),
m.gb_per_second(),
m.mb_per_second_per_core(),
m.mb_per_second_per_thread(),
m.ns_per_token(),
m.stats.rel_stddev() * 100.0,
if m.matches_reference { "yes" } else { "NO" },
m.achieved_parallelism(),
flag,
));
}
out.push_str(&self.render_versions());
out.push_str(&self.render_unavailable());
out.push_str(&self.render_notes());
out
}
fn render_header(&self) -> String {
let mut s = String::from("== machine ==\n");
let features = compiled_features();
s.push_str(&format!(
"features : {}\n",
if features.is_empty() { "default".to_owned() } else { features.join(",") }
));
s.push_str(&format!("cpu : {}\n", self.machine.cpu_model));
s.push_str(&format!(
"cores : {} physical / {} logical ({} SMT siblings per core)\n",
self.machine.physical_cores,
self.machine.logical_cores,
self.machine.smt_siblings_per_core(),
));
s.push_str(&format!(
"governor : {}\n",
self.machine.governor.as_deref().unwrap_or("unknown (no cpufreq)"),
));
s.push_str(&format!(
"memory : {} KiB\n",
self.machine.total_memory_kib
));
match &self.machine.gpu {
Some(g) => s.push_str(&format!(
"gpu : {} ({} MiB, driver {}, compute {})\n",
g.name, g.memory_mib, g.driver_version, g.compute_capability,
)),
None => s.push_str("gpu : none detected (nvidia-smi absent or failed)\n"),
}
s.push_str(&format!("vocabulary : {}\n", self.vocab));
s.push_str(&format!(
"corpus : {} bytes from {} files, digest {:#018x}\n",
self.corpus.bytes, self.corpus.files, self.corpus.digest,
));
s.push_str(&format!("corpus origin : {}\n", self.corpus.origin));
match self.bytes_per_token {
Some(b) => s.push_str(&format!(
"corpus bytes/token: {b:.4} (MB/s scales with this; rows at different \
bytes/token are not comparable)\n"
)),
None => s.push_str("corpus bytes/token: not calibrated\n"),
}
if let Some(c) = self.clock {
s.push_str(&format!(
"clock : {:.0} MHz mean at start, {:.0} MHz at end, drift {:+.2}%{}\n",
c.first.mean_mhz,
c.last.mean_mhz,
c.drift_ratio() * 100.0,
if c.is_significant() {
" <-- boundary samples differ; occupancy-confounded, not proof of bias"
} else {
""
},
));
} else {
s.push_str("clock : kernel reported no per-CPU MHz; drift unknown\n");
}
s
}
fn render_versions(&self) -> String {
let mut seen: BTreeMap<&str, (&str, &str)> = BTreeMap::new();
for m in self.measured() {
seen.insert(&m.tokenizer, (&m.version, &m.vocab));
}
let mut s = String::from("\n== versions under test ==\n\n");
for (name, (ver, vocab)) in seen {
s.push_str(&format!("{name:<12} {ver:<16} vocab {vocab}\n"));
}
s
}
fn render_unavailable(&self) -> String {
let missing: Vec<(&str, &str)> = self
.rows
.iter()
.filter_map(|r| match r {
Availability::Unavailable { tokenizer, reason } => {
Some((tokenizer.as_str(), reason.as_str()))
}
Availability::Measured(_) => None,
})
.collect();
if missing.is_empty() {
return String::from("\n== unavailable baselines ==\n\nnone; every baseline ran\n");
}
let mut s = String::from(
"\n== unavailable baselines (recorded, never estimated) ==\n\n",
);
let mut seen: BTreeMap<&str, &str> = BTreeMap::new();
for (t, r) in missing {
seen.entry(t).or_insert(r);
}
for (t, r) in seen {
s.push_str(&format!("{t:<12} {r}\n"));
}
s
}
fn render_notes(&self) -> String {
let harness = self.harness_defects();
let defects = self.suspected_defects();
let slow = self.known_slow();
let mut s = String::new();
if harness.is_empty() {
s.push_str(&format!(
"\nharness: every cell timed at least {MIN_TRUSTWORTHY_TIMED_SECS:.2} s, \
delivered at least {:.0}% of its requested threads, and ran with under \
{:.0}% of one core consumed elsewhere on the machine\n",
MIN_PARALLEL_EFFICIENCY * 100.0,
MAX_FOREIGN_LOAD_RATIO * 100.0,
));
} else {
s.push_str(&format!(
"\nHARNESS DEFECT: {} cell(s) did not measure what the table claims. \
These are bugs in this harness, not results.\n",
harness.len(),
));
for d in harness {
let why = assess_plausibility(d)
.diagnosis()
.unwrap_or_else(|| "unclassified".to_owned());
s.push_str(&format!(
" {} {} thr={} doc_bytes={} -> {why}\n",
d.tokenizer,
d.regime.as_str(),
d.threads,
d.doc_bytes,
));
}
}
if defects.is_empty() {
s.push_str(&format!(
"plausibility: every large-input cell clears the \
{PLAUSIBLE_FLOOR_MB_S_PER_CORE:.1} MB/s per-core floor, or is recorded below as \
legitimately slower\n"
));
} else {
s.push_str(&format!(
"plausibility: {} cell(s) fall under the {:.1} MB/s per-core floor with no \
recorded explanation. Investigate before publishing: either the harness \
mismeasured or the library needs an evidenced KNOWN_SLOW_BASELINES entry.\n",
defects.len(),
PLAUSIBLE_FLOOR_MB_S_PER_CORE,
));
for d in defects {
s.push_str(&format!(
" {} {} thr={} doc_bytes={} -> {:.3} MB/s/core\n",
d.tokenizer,
d.regime.as_str(),
d.threads,
d.doc_bytes,
d.mb_per_second_per_core(),
));
}
}
if !slow.is_empty() {
let mut names: Vec<&str> = slow.iter().map(|m| m.tokenizer.as_str()).collect();
names.sort_unstable();
names.dedup();
s.push_str(&format!(
"known-slow: {} cell(s) across {} librar(y/ies) are below the floor for recorded \
reasons; the number stands, the reason is printed with it\n",
slow.len(),
names.len(),
));
for n in names {
if let Some(reason) = known_slow_reason(n) {
s.push_str(&format!(" {n}: {reason}\n"));
}
}
}
let noisy = self
.measured()
.filter(|m| m.stats.rel_stddev() >= NOISY_REL_STDDEV)
.count();
s.push_str(&format!(
"variance: {} of {} cells exceeded {:.0}% relative standard deviation across reps\n",
noisy,
self.measured().count(),
NOISY_REL_STDDEV * 100.0,
));
let disagreeing = self.disagreeing();
if disagreeing.is_empty() {
s.push_str(
"exactness: every measured library produced ids identical to the reference on the \
agreement sample, so the rows are vocabulary-comparable\n",
);
} else {
let mut names: Vec<&str> = disagreeing.iter().map(|m| m.tokenizer.as_str()).collect();
names.sort_unstable();
names.dedup();
s.push_str(&format!(
"exactness: {} produced ids DIFFERENT from the reference; those rows measure a \
different computation and are not vocabulary-comparable\n",
names.join(", "),
));
}
s
}
#[must_use]
pub fn to_json(&self) -> String {
let mut s = String::from("{\n");
let features: Vec<String> =
compiled_features().iter().map(|f| json_str(f)).collect();
s.push_str(&format!(" \"feature_set\": [{}],\n", features.join(", ")));
s.push_str(&format!(
" \"provenance\": {},\n",
json_str(Provenance::Probed.as_str())
));
s.push_str(&format!(" \"cpu_model\": {},\n", json_str(&self.machine.cpu_model)));
s.push_str(&format!(" \"physical_cores\": {},\n", self.machine.physical_cores));
s.push_str(&format!(" \"logical_cores\": {},\n", self.machine.logical_cores));
s.push_str(&format!(
" \"governor\": {},\n",
self.machine.governor.as_deref().map_or_else(|| "null".to_owned(), json_str)
));
s.push_str(&format!(" \"total_memory_kib\": {},\n", self.machine.total_memory_kib));
match &self.machine.gpu {
Some(g) => s.push_str(&format!(
" \"gpu\": {{\"name\": {}, \"memory_mib\": {}, \"driver_version\": {}, \"compute_capability\": {}}},\n",
json_str(&g.name),
json_str(&g.memory_mib),
json_str(&g.driver_version),
json_str(&g.compute_capability),
)),
None => s.push_str(" \"gpu\": null,\n"),
}
s.push_str(&format!(" \"vocab\": {},\n", json_str(&self.vocab)));
s.push_str(&format!(
" \"corpus\": {{\"origin\": {}, \"bytes\": {}, \"files\": {}, \"digest\": \"{:#018x}\"}},\n",
json_str(&self.corpus.origin),
self.corpus.bytes,
self.corpus.files,
self.corpus.digest,
));
s.push_str(&format!(
" \"bytes_per_token\": {},\n",
self.bytes_per_token.map_or_else(|| "null".to_owned(), |b| format!("{b:.6}"))
));
match self.clock {
Some(c) => s.push_str(&format!(
" \"clock\": {{\"start_mean_mhz\": {:.1}, \"end_mean_mhz\": {:.1}, \"drift_ratio\": {:.6}, \"significant\": {}}},\n",
c.first.mean_mhz,
c.last.mean_mhz,
c.drift_ratio(),
c.is_significant(),
)),
None => s.push_str(" \"clock\": null,\n"),
}
s.push_str(" \"rows\": [\n");
for (i, row) in self.rows.iter().enumerate() {
if i > 0 {
s.push_str(",\n");
}
match row {
Availability::Unavailable { tokenizer, reason } => s.push_str(&format!(
" {{\"status\": \"unavailable\", \"tokenizer\": {}, \"reason\": {}}}",
json_str(tokenizer),
json_str(reason)
)),
Availability::Measured(m) => {
let samples: Vec<String> =
m.stats.samples().iter().map(|v| format!("{v:.9}")).collect();
let region = m.region_clock.map_or_else(
|| "null".to_owned(),
|c| {
format!(
"{{\"start_mhz\": {:.1}, \"end_mhz\": {:.1}, \
\"drift_ratio\": {:.6}, \"busy_cpus\": {}, \"significant\": {}}}",
c.start_mhz,
c.end_mhz,
c.drift_ratio(),
c.busy_cpus,
c.is_significant(),
)
},
);
s.push_str(&format!(
" {{\"status\": \"measured\", \"tokenizer\": {}, \"version\": {}, \"vocab\": {}, \
\"regime\": {}, \"target_tokens\": {}, \"doc_bytes\": {}, \"docs\": {}, \
\"input_bytes\": {}, \"tokens\": {}, \"threads\": {}, \"physical_cores\": {}, \
\"logical_cores\": {}, \"pinned_to_physical\": {}, \"warmups\": {}, \
\"matches_reference\": {}, \"cpu_secs\": {:.9}, \"device_secs\": {}, \
\"foreign_cpu_secs\": {:.9}, \"foreign_load_ratio\": {:.4}, \
\"achieved_parallelism\": {:.4}, \"inner_reps\": {}, \
\"timed_bytes\": {}, \"timed_tokens\": {}, \
\"mean_mhz\": {:.1}, \"region_clock\": {}, \
\"best_secs\": {:.9}, \"rel_stddev\": {:.6}, \
\"mb_per_s\": {:.4}, \"gb_per_s\": {:.6}, \"mb_per_s_per_thread\": {:.4}, \
\"tokens_per_s\": {:.1}, \"ns_per_token\": {:.3}, \"bytes_per_token\": {:.4}, \
\"samples_secs\": [{}]}}",
json_str(&m.tokenizer),
json_str(&m.version),
json_str(&m.vocab),
json_str(m.regime.as_str()),
m.target_tokens,
m.doc_bytes,
m.docs,
m.input_bytes,
m.tokens,
m.threads,
m.machine.physical_cores,
m.machine.logical_cores,
m.machine.pinned_to_physical,
m.warmups,
m.matches_reference,
m.active.cpu_secs,
m.active
.device_secs
.map_or_else(|| "null".to_owned(), |d| format!("{d:.9}")),
m.foreign_cpu_secs,
m.foreign_load_ratio(),
m.achieved_parallelism(),
m.inner_reps,
m.timed_bytes(),
m.timed_tokens(),
m.mean_mhz,
region,
m.stats.best(),
m.stats.rel_stddev(),
m.mb_per_second(),
m.gb_per_second(),
m.mb_per_second_per_thread(),
m.tokens_per_second(),
m.ns_per_token(),
m.bytes_per_token(),
samples.join(", "),
));
}
}
}
s.push_str("\n ]\n}\n");
s
}
pub fn persist(&self, dir: &Path) -> Result<PathBuf, ExatokError> {
let fix = "point SweepConfig::report_dir somewhere writable, or set it to None";
fs::create_dir_all(dir).map_err(|e| {
artifact_err("report directory", &dir.display().to_string(), &e.to_string(), fix)
})?;
let path =
dir.join(format!("host-baseline-{}.json", utc_stamp(std::time::SystemTime::now())));
fs::write(&path, self.to_json()).map_err(|e| {
artifact_err("report file", &path.display().to_string(), &e.to_string(), fix)
})?;
Ok(path)
}
}
#[must_use]
pub fn utc_stamp(at: std::time::SystemTime) -> String {
let secs = at.duration_since(std::time::SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs());
let (days, rem) = (secs / 86_400, secs % 86_400);
let (hh, mm, ss) = (rem / 3600, (rem % 3600) / 60, rem % 60);
let z = days + 719_468;
let era = z / 146_097;
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let day = doy - (153 * mp + 2) / 5 + 1;
let month = if mp < 10 { mp + 3 } else { mp - 9 };
let year = yoe + era * 400 + u64::from(month <= 2);
format!("{year:04}{month:02}{day:02}T{hh:02}{mm:02}{ss:02}Z")
}
#[must_use]
pub fn json_str(s: &str) -> String {
let mut out = String::with_capacity(s.len() + 2);
out.push('"');
for c in s.chars() {
match c {
'"' => out.push_str("\\\""),
'\\' => out.push_str("\\\\"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
'\t' => out.push_str("\\t"),
c if (c as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", c as u32)),
c => out.push(c),
}
}
out.push('"');
out
}
pub const BENCH_VENV_REL: &str = ".cache/exatok-bench-venv";
pub const BENCH_REPORT_REL: &str = ".cache/exatok-bench/reports";
pub const BASELINE_MODULES: &[&str] = &["tiktoken", "tokenizers", "gigatoken", "fastokens"];
#[must_use]
pub fn home_dir() -> PathBuf {
std::env::var_os("HOME").map_or_else(|| PathBuf::from("/root"), PathBuf::from)
}
#[derive(Debug, Clone)]
pub struct SweepConfig {
pub python: PathBuf,
pub driver: PathBuf,
pub vocab: String,
pub tiktoken_ranks: PathBuf,
pub hf_repo: String,
pub corpus_root: PathBuf,
pub corpus_file: Option<PathBuf>,
pub corpus_scratch: PathBuf,
pub batch_budget_bytes: usize,
pub target_tokens: Vec<usize>,
pub reps: u32,
pub warmups: u32,
pub tiktoken_cache_dir: Option<PathBuf>,
pub report_dir: Option<PathBuf>,
pub require_baselines: bool,
}
impl SweepConfig {
#[must_use]
pub fn defaults(crate_dir: &Path, repo_root: &Path) -> Self {
let scratch = scratch_root();
let home = home_dir();
Self {
python: home.join(BENCH_VENV_REL).join("bin/python"),
driver: crate_dir.join("benches/baselines/host_baselines.py"),
vocab: "cl100k_base".to_owned(),
tiktoken_ranks: home.join(".cache/exatok-vocab/cl100k_base.tiktoken"),
hf_repo: "Xenova/gpt-4".to_owned(),
corpus_root: repo_root.to_path_buf(),
corpus_file: None,
corpus_scratch: scratch.join("bench-corpus.txt"),
batch_budget_bytes: 32 * 1024 * 1024,
target_tokens: DEFAULT_TARGET_TOKENS.to_vec(),
reps: 3,
warmups: 1,
tiktoken_cache_dir: Some(home.join(".cache/tiktoken")),
report_dir: Some(home.join(BENCH_REPORT_REL)),
require_baselines: true,
}
}
}
pub fn preflight_baselines(cfg: &SweepConfig) -> Result<(), ExatokError> {
let venv = home_dir().join(BENCH_VENV_REL);
let fix = format!(
"python3 -m venv --system-site-packages {} && {}/bin/pip install {}",
venv.display(),
venv.display(),
BASELINE_MODULES.join(" ")
);
if !cfg.python.is_file() {
return Err(artifact_err(
"baseline interpreter",
&cfg.python.display().to_string(),
"no such file",
&fix,
));
}
let script = format!(
"import importlib\nbad=[]\nfor m in {BASELINE_MODULES:?}:\n \
try:\n importlib.import_module(m)\n \
except Exception as e:\n bad.append(f'{{m}}: {{type(e).__name__}}')\n\
print(';'.join(bad))\n"
);
let out = Command::new(&cfg.python).arg("-c").arg(&script).output().map_err(|e| {
artifact_err("baseline interpreter", &cfg.python.display().to_string(), &e.to_string(), &fix)
})?;
let missing = String::from_utf8_lossy(&out.stdout).trim().to_owned();
if !out.status.success() {
return Err(artifact_err(
"baseline import probe",
&cfg.python.display().to_string(),
&describe_stderr(&out.stderr),
&fix,
));
}
if missing.is_empty() {
Ok(())
} else {
Err(artifact_err("baseline libraries", &missing, "not importable", &fix))
}
}
pub fn run_sweep(cfg: &SweepConfig, machine: &Machine) -> Result<Report, ExatokError> {
if cfg.require_baselines {
preflight_baselines(cfg)?;
}
let corpus_bytes = cfg.batch_budget_bytes.saturating_add(
cfg.target_tokens.iter().copied().max().unwrap_or(0).saturating_mul(8),
);
let corpus = match &cfg.corpus_file {
Some(p) => Corpus::from_file(p, corpus_bytes)?,
None => Corpus::from_tree(&cfg.corpus_root, corpus_bytes)?,
};
if let Some(parent) = cfg.corpus_scratch.parent() {
fs::create_dir_all(parent).map_err(|e| {
artifact_err(
"corpus scratch directory",
&parent.display().to_string(),
&e.to_string(),
"ensure the crate target directory is writable",
)
})?;
}
fs::write(&cfg.corpus_scratch, corpus.text.as_bytes()).map_err(|e| {
artifact_err(
"corpus scratch file",
&cfg.corpus_scratch.display().to_string(),
&e.to_string(),
"ensure the crate target directory is writable",
)
})?;
let clock_first = ClockSample::probe();
let mut rows = Vec::new();
let mut bytes_per_token = None;
for plan in ThreadPlan::standard(machine) {
let ctx = MachineContext {
physical_cores: machine.physical_cores,
logical_cores: machine.logical_cores,
pinned_to_physical: plan.pinned_to_physical,
};
let stdout = spawn_driver(cfg, &plan)?;
for (i, line) in stdout.lines().enumerate() {
match parse_driver_line(line, ctx, i + 1)? {
Some(DriverEvent::Record(m)) => rows.push(Availability::Measured(m)),
Some(DriverEvent::Unavailable { tokenizer, reason }) => {
rows.push(Availability::Unavailable { tokenizer, reason });
}
Some(DriverEvent::Calibration { bytes_per_token: b, .. }) => {
bytes_per_token = Some(b);
}
None => {}
}
}
}
refuse_noncomparable_cells(&rows, &corpus.facts)?;
let clock_last = ClockSample::probe();
let clock = match (clock_first, clock_last) {
(Some(first), Some(last)) => Some(ClockDrift { first, last }),
_ => None,
};
let mut report = Report {
machine: machine.clone(),
corpus: corpus.facts,
vocab: cfg.vocab.clone(),
rows,
clock,
bytes_per_token,
report_path: None,
};
if let Some(dir) = &cfg.report_dir {
report.report_path = Some(report.persist(dir)?);
}
Ok(report)
}
fn refuse_noncomparable_cells(
rows: &[Availability],
corpus: &CorpusFacts,
) -> Result<(), ExatokError> {
let mut cells: BTreeMap<(&str, &str, usize), Vec<(&str, u64, u64)>> = BTreeMap::new();
for m in rows.iter().filter_map(|r| match r {
Availability::Measured(m) => Some(m),
Availability::Unavailable { .. } => None,
}) {
cells
.entry((m.vocab.as_str(), m.regime.as_str(), m.target_tokens))
.or_default()
.push((m.tokenizer.as_str(), m.input_bytes, m.tokens));
}
for ((vocab, regime, target), entries) in &cells {
let &(first_tok, first_bytes, first_tokens) = &entries[0];
for &(tok, bytes, tokens) in &entries[1..] {
if bytes != first_bytes || tokens != first_tokens {
return Err(artifact_err(
"baseline sweep corpus",
&format!("{vocab} {regime} target_tokens={target}"),
&format!(
"{tok} emitted {tokens} tokens from {bytes} bytes but {first_tok} emitted \
{first_tokens} from {first_bytes} bytes of the same corpus ({} bytes, \
digest {})",
corpus.bytes, corpus.digest
),
"the baselines do not tokenize this corpus identically, so no ratio between \
them is falsifiable; remove the diverging text from the corpus tree or point \
corpus_file at text every baseline encodes to the same ids",
));
}
}
}
Ok(())
}
fn spawn_driver(cfg: &SweepConfig, plan: &ThreadPlan) -> Result<String, ExatokError> {
let cpu_list = plan.cpu_list();
let mut cmd = match &cpu_list {
Some(list) => {
let mut c = Command::new("taskset");
c.arg("-c").arg(list).arg(&cfg.python);
c
}
None => Command::new(&cfg.python),
};
cmd.arg(&cfg.driver)
.arg("--corpus")
.arg(&cfg.corpus_scratch)
.arg("--vocab")
.arg(&cfg.vocab)
.arg("--ranks")
.arg(&cfg.tiktoken_ranks)
.arg("--hf-repo")
.arg(&cfg.hf_repo)
.arg("--threads")
.arg(plan.threads.to_string())
.arg("--batch-budget")
.arg(cfg.batch_budget_bytes.to_string())
.arg("--reps")
.arg(cfg.reps.to_string())
.arg("--warmups")
.arg(cfg.warmups.to_string())
.arg("--target-tokens")
.arg(
cfg.target_tokens
.iter()
.map(usize::to_string)
.collect::<Vec<_>>()
.join(","),
);
if plan.include_single_doc {
cmd.arg("--include-single-doc");
}
cmd.env("RAYON_NUM_THREADS", plan.threads.to_string())
.env("TOKENIZERS_PARALLELISM", if plan.threads > 1 { "true" } else { "false" })
.env("OMP_NUM_THREADS", "1")
.env("MKL_NUM_THREADS", "1")
.env("PYTHONHASHSEED", "0");
if let Some(dir) = &cfg.tiktoken_cache_dir {
cmd.env("TIKTOKEN_CACHE_DIR", dir);
}
let out = cmd.output().map_err(|e| {
ExatokError::Backend(format!(
"cannot spawn baseline driver {} via {}: {e}. Fix: create the bench venv \
off-tree with `python3 -m venv --system-site-packages {}` and install \
{} into it",
cfg.driver.display(),
cfg.python.display(),
home_dir().join(BENCH_VENV_REL).display(),
BASELINE_MODULES.join(" "),
))
})?;
if !out.status.success() {
return Err(ExatokError::Backend(format!(
"baseline driver failed with {} for threads={}. stderr: {}{}",
describe_termination(&out.status),
plan.threads,
describe_stderr(&out.stderr),
kill_hint(&out.status),
)));
}
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
}
fn kill_hint(status: &std::process::ExitStatus) -> &'static str {
match status.signal() {
Some(9) => {
" A sweep is long and CPU-bound, so signal 9 is usually an external \
reaper rather than a fault in the driver. Check /tmp/cpu-watchdog.log \
and /tmp/watchdog.log for a kill naming this pid, and check dmesg for \
an OOM kill. If a watchdog reaped it, allowlist the driver rather than \
shortening the measurement."
}
Some(7) => " Signal 7 means the corpus changed under a mapped read; keep it off-tree.",
_ => "",
}
}
fn describe_termination(status: &std::process::ExitStatus) -> String {
if let Some(code) = status.code() {
return format!("exit code {code}");
}
match status.signal() {
Some(signal) => format!("no exit code, killed by signal {signal}"),
None => "neither an exit code nor a signal".to_owned(),
}
}
fn describe_stderr(stderr: &[u8]) -> String {
let text = String::from_utf8_lossy(stderr);
let trimmed = text.trim();
if trimmed.is_empty() {
"<empty, which means the driver reported nothing before it ended>".to_owned()
} else {
trimmed.to_owned()
}
}
#[must_use]
pub fn scratch_root() -> PathBuf {
if let Ok(exe) = std::env::current_exe() {
for dir in exe.ancestors() {
let profile = dir.file_name().and_then(|n| n.to_str());
if matches!(profile, Some("debug" | "release")) {
if let Some(target) = dir.parent() {
return target.to_path_buf();
}
}
}
}
std::env::temp_dir()
}
#[must_use]
pub fn find_repo_root(start: &Path) -> Option<PathBuf> {
let mut outer = None;
let mut cur = Some(start);
while let Some(dir) = cur {
if dir.join(".git").exists() {
outer = Some(dir.to_path_buf());
}
cur = dir.parent();
}
outer
}
pub const CALIBRATION_TARGET_SECS: f64 = 0.5;
const CALIBRATION_ATTEMPTS: u32 = 12;
const CALIBRATION_HEADROOM: f64 = 1.2;
const USER_HZ: f64 = 100.0;
const BUSY_CPU_MIN_OCCUPANCY: f64 = 0.10;
pub const MIN_PAIRED_ROUNDS: u32 = 8;
pub const DEVICE_RESIDENCY_TOLERANCE_MIB: u64 = 64;
pub const MAX_PAIRED_HOST_LOAD_DRIFT: f64 = 0.25;
pub const MAX_VOIDED_ROUND_RETRIES: u32 = 3;
pub const DEVICE_LOCK_TIMEOUT_SECS: u64 = 900;
pub const BENCH_LOCK_REL: &str = ".cache/exatok-bench/device.lock";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WorkUnit {
pub bytes: u64,
pub tokens: u64,
}
pub struct Arm<'a> {
pub label: &'a str,
pub work: &'a mut dyn FnMut() -> Result<WorkUnit, ExatokError>,
}
fn machine_busy_secs() -> Option<f64> {
let stat = fs::read_to_string("/proc/stat").ok()?;
let aggregate = stat.lines().next()?;
let ticks: Vec<f64> =
aggregate.split_whitespace().skip(1).filter_map(|t| t.parse::<f64>().ok()).collect();
if ticks.len() < 4 {
return None;
}
let idle = ticks[3] + ticks.get(4).copied().unwrap_or(0.0);
Some((ticks.iter().sum::<f64>() - idle) / USER_HZ)
}
fn process_cpu_secs() -> Option<f64> {
let stat = fs::read_to_string("/proc/self/stat").ok()?;
let after_comm = stat.rsplit_once(')')?.1;
let fields: Vec<&str> = after_comm.split_whitespace().collect();
let utime = fields.get(11)?.parse::<f64>().ok()?;
let stime = fields.get(12)?.parse::<f64>().ok()?;
Some((utime + stime) / USER_HZ)
}
fn per_cpu_busy_secs() -> Vec<f64> {
let Ok(stat) = fs::read_to_string("/proc/stat") else {
return Vec::new();
};
stat.lines()
.filter_map(|line| {
let (head, rest) = line.split_once(char::is_whitespace)?;
let index = head.strip_prefix("cpu")?;
if index.is_empty() || !index.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let ticks: Vec<f64> =
rest.split_whitespace().filter_map(|t| t.parse::<f64>().ok()).collect();
if ticks.len() < 4 {
return None;
}
let idle = ticks[3] + ticks.get(4).copied().unwrap_or(0.0);
Some((ticks.iter().sum::<f64>() - idle) / USER_HZ)
})
.collect()
}
fn per_cpu_mhz() -> Vec<f64> {
let Ok(info) = fs::read_to_string("/proc/cpuinfo") else {
return Vec::new();
};
info.lines()
.filter_map(|l| l.strip_prefix("cpu MHz"))
.filter_map(|r| r.split_once(':'))
.filter_map(|(_, v)| v.trim().parse::<f64>().ok())
.collect()
}
#[must_use]
pub fn busy_weighted_mhz(mhz: &[f64], weights: &[f64]) -> f64 {
let n = mhz.len().min(weights.len());
if n == 0 {
return 0.0;
}
let total: f64 = weights[..n].iter().sum();
if total <= 0.0 {
return 0.0;
}
mhz[..n].iter().zip(&weights[..n]).map(|(m, w)| m * w).sum::<f64>() / total
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DeviceLoad {
pub foreign_resident_mib: u64,
pub foreign_sm_percent: u32,
pub foreign_processes: u32,
pub own_resident_mib: u64,
}
#[must_use]
pub fn sample_device_load() -> Option<DeviceLoad> {
let out = Command::new("nvidia-smi").args(["pmon", "-c", "1", "-s", "um"]).output().ok()?;
if !out.status.success() {
return None;
}
parse_pmon(&String::from_utf8_lossy(&out.stdout), std::process::id())
}
fn parse_pmon(text: &str, own_pid: u32) -> Option<DeviceLoad> {
let mut load = DeviceLoad {
foreign_resident_mib: 0,
foreign_sm_percent: 0,
foreign_processes: 0,
own_resident_mib: 0,
};
let mut saw_row = false;
for line in text.lines() {
if line.starts_with('#') {
continue;
}
let f: Vec<&str> = line.split_whitespace().collect();
if f.len() < 11 {
continue;
}
let Ok(pid) = f[1].parse::<u32>() else {
continue;
};
saw_row = true;
if f[2] != "C" {
continue;
}
let resident = f[9].parse::<u64>().unwrap_or(0);
if pid == own_pid {
load.own_resident_mib += resident;
continue;
}
load.foreign_processes += 1;
load.foreign_resident_mib += resident;
load.foreign_sm_percent += f[3].parse::<u32>().unwrap_or(0);
}
saw_row.then_some(load)
}
#[derive(Debug)]
pub struct DeviceLock {
path: PathBuf,
}
impl DeviceLock {
pub fn acquire(timeout: std::time::Duration) -> Result<Self, ExatokError> {
Self::acquire_at(home_dir().join(BENCH_LOCK_REL), timeout)
}
pub fn acquire_at(path: PathBuf, timeout: std::time::Duration) -> Result<Self, ExatokError> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).map_err(|e| {
artifact_err(
"device lock directory",
&parent.display().to_string(),
&e.to_string(),
"ensure $HOME/.cache is writable",
)
})?;
}
let deadline = std::time::Instant::now() + timeout;
loop {
match fs::OpenOptions::new().write(true).create_new(true).open(&path) {
Ok(mut file) => {
use std::io::Write;
let _ = write!(file, "{}", std::process::id());
return Ok(Self { path });
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
if Self::clear_if_stale(&path) {
continue;
}
if std::time::Instant::now() >= deadline {
return Err(artifact_err(
"device measurement lock",
&path.display().to_string(),
&format!(
"still held by {} after {} s",
Self::describe_holder(&path),
timeout.as_secs()
),
"wait for the running measurement to finish, or delete the lock \
file if its holder is gone",
));
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
Err(e) => {
return Err(artifact_err(
"device measurement lock",
&path.display().to_string(),
&e.to_string(),
"ensure $HOME/.cache is writable",
));
}
}
}
}
fn clear_if_stale(path: &Path) -> bool {
let Ok(text) = fs::read_to_string(path) else {
return false;
};
let Ok(pid) = text.trim().parse::<u32>() else {
return false;
};
if Path::new(&format!("/proc/{pid}")).exists() {
return false;
}
fs::remove_file(path).is_ok()
}
fn describe_holder(path: &Path) -> String {
let Ok(text) = fs::read_to_string(path) else {
return "an unreadable lock file".to_owned();
};
let pid = text.trim();
match fs::read_to_string(format!("/proc/{pid}/comm")) {
Ok(comm) => format!("pid {pid} ({})", comm.trim()),
Err(_) => format!("pid {pid}"),
}
}
}
impl Drop for DeviceLock {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
}
}
struct RegionSample {
secs: f64,
unit: WorkUnit,
cpu_secs: f64,
foreign_cpu_secs: f64,
mean_mhz: f64,
region_clock: Option<RegionClock>,
foreign_resident_mib: u64,
foreign_sm_percent: u32,
own_resident_mib: u64,
}
fn calibrate(
work: &mut dyn FnMut() -> Result<WorkUnit, ExatokError>,
target_secs: f64,
) -> Result<(u32, WorkUnit), ExatokError> {
let start = std::time::Instant::now();
let unit = work()?;
let cold = start.elapsed().as_secs_f64();
work()?;
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let mut inner = ((target_secs / cold.max(1e-9)).ceil() as u32).max(1);
for _ in 0..CALIBRATION_ATTEMPTS {
let probe = std::time::Instant::now();
for _ in 0..inner {
work()?;
}
let observed = probe.elapsed().as_secs_f64();
if observed >= target_secs {
break;
}
let grow = target_secs / observed.max(1e-9) * CALIBRATION_HEADROOM;
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let grown = (f64::from(inner) * grow).ceil() as u32;
inner = grown.max(inner.saturating_add(1));
}
Ok((inner, unit))
}
fn time_region(
work: &mut dyn FnMut() -> Result<WorkUnit, ExatokError>,
inner: u32,
) -> Result<RegionSample, ExatokError> {
let lock = DeviceLock::acquire(std::time::Duration::from_secs(DEVICE_LOCK_TIMEOUT_SECS))?;
let device_before = sample_device_load();
let per_cpu_before = per_cpu_busy_secs();
let mhz_before = per_cpu_mhz();
let busy_before = machine_busy_secs();
let cpu_before = process_cpu_secs();
let start = std::time::Instant::now();
let mut unit = WorkUnit { bytes: 0, tokens: 0 };
for _ in 0..inner {
unit = work()?;
}
let secs = start.elapsed().as_secs_f64();
let mhz_after = per_cpu_mhz();
let per_cpu_after = per_cpu_busy_secs();
let cpu_after = process_cpu_secs();
let busy_after = machine_busy_secs();
let device_after = sample_device_load();
drop(lock);
let cpu_secs = match (cpu_before, cpu_after) {
(Some(a), Some(b)) => (b - a).max(0.0),
_ => 0.0,
};
let machine_busy = match (busy_before, busy_after) {
(Some(a), Some(b)) => (b - a).max(0.0),
_ => 0.0,
};
let foreign_cpu_secs = (machine_busy - cpu_secs).max(0.0);
let width = per_cpu_before.len().min(per_cpu_after.len());
let weights: Vec<f64> =
(0..width).map(|i| (per_cpu_after[i] - per_cpu_before[i]).max(0.0)).collect();
let region_clock = if weights.is_empty() {
None
} else {
let floor = secs * BUSY_CPU_MIN_OCCUPANCY;
#[allow(clippy::cast_possible_truncation)]
let busy_cpus = weights.iter().filter(|w| **w >= floor).count() as u32;
Some(RegionClock {
start_mhz: busy_weighted_mhz(&mhz_before, &weights),
end_mhz: busy_weighted_mhz(&mhz_after, &weights),
busy_cpus,
})
};
let mean_mhz = if mhz_after.is_empty() {
0.0
} else {
#[allow(clippy::cast_precision_loss)]
let n = mhz_after.len() as f64;
mhz_after.iter().sum::<f64>() / n
};
let foreign_resident_mib = device_before
.map_or(0, |d| d.foreign_resident_mib)
.max(device_after.map_or(0, |d| d.foreign_resident_mib));
let foreign_sm_percent = device_before
.map_or(0, |d| d.foreign_sm_percent)
.max(device_after.map_or(0, |d| d.foreign_sm_percent));
Ok(RegionSample {
secs,
unit,
cpu_secs,
foreign_cpu_secs,
mean_mhz,
region_clock,
foreign_resident_mib,
own_resident_mib: device_before
.map_or(0, |d| d.own_resident_mib)
.max(device_after.map_or(0, |d| d.own_resident_mib)),
foreign_sm_percent,
})
}
pub fn measure_cell(
label: &str,
threads: usize,
work: &mut dyn FnMut() -> Result<WorkUnit, ExatokError>,
) -> Result<Measurement, ExatokError> {
let machine = Machine::probe()?;
let (inner, _) = calibrate(work, CALIBRATION_TARGET_SECS)?;
let baseline = sample_device_load().map_or(0, |d| d.foreign_resident_mib);
let mut samples = Vec::new();
let mut last: Option<RegionSample> = None;
for _ in 0..MIN_PAIRED_ROUNDS {
let region = time_region(work, inner)?;
if region.own_resident_mib > 0
&& region.foreign_resident_mib > baseline + DEVICE_RESIDENCY_TOLERANCE_MIB
{
return Err(artifact_err(
"device exclusivity during measurement",
&format!("{} MiB foreign resident", region.foreign_resident_mib),
&format!(
"rose above the {baseline} MiB baseline by more than \
{DEVICE_RESIDENCY_TOLERANCE_MIB} MiB, so another process landed on the \
card mid-measurement"
),
"wait for sibling device work to finish and re-run; absolute device throughput \
is not certifiable while another process holds the card",
));
}
samples.push(region.secs);
last = Some(region);
}
let Some(region) = last else {
return Err(artifact_err(
"measured cell",
label,
"no timed region completed",
"check that MIN_PAIRED_ROUNDS is above zero",
));
};
let regime = if threads > 1 { Regime::Batch } else { Regime::SingleDocument };
#[allow(clippy::cast_possible_truncation)]
let doc_bytes = region.unit.bytes as usize;
Ok(Measurement {
tokenizer: label.to_owned(),
version: env!("CARGO_PKG_VERSION").to_owned(),
vocab: "in-process".to_owned(),
regime,
target_tokens: 0,
doc_bytes,
docs: 1,
input_bytes: region.unit.bytes.saturating_mul(u64::from(inner)),
tokens: region.unit.tokens.saturating_mul(u64::from(inner)),
stats: RunStats::new(samples)?,
warmups: 1,
threads,
machine: MachineContext {
physical_cores: machine.physical_cores,
logical_cores: machine.logical_cores,
pinned_to_physical: false,
},
mean_mhz: region.mean_mhz,
region_clock: region.region_clock,
matches_reference: true,
active: ActiveTime { cpu_secs: region.cpu_secs, device_secs: None },
foreign_cpu_secs: region.foreign_cpu_secs,
inner_reps: inner,
})
}
#[derive(Debug, Clone, PartialEq)]
pub struct Paired {
pub label_a: String,
pub label_b: String,
pub a_unit_secs: Vec<f64>,
pub b_unit_secs: Vec<f64>,
pub speedups: Vec<f64>,
pub voided_rounds: u32,
pub foreign_resident_baseline_mib: u64,
pub foreign_sm_percent_max: u32,
pub host_foreign_load: Vec<f64>,
pub host_foreign_load_arms: Vec<(f64, f64)>,
pub unit_work: Option<(WorkUnit, WorkUnit)>,
}
impl Paired {
#[must_use]
pub fn median_speedup(&self) -> f64 {
if self.speedups.is_empty() {
return 0.0;
}
let mut sorted = self.speedups.clone();
sorted.sort_by(f64::total_cmp);
let mid = sorted.len() / 2;
if sorted.len() % 2 == 0 { (sorted[mid - 1] + sorted[mid]) / 2.0 } else { sorted[mid] }
}
#[must_use]
pub fn speedup_range(&self) -> (f64, f64) {
let mut lo = f64::INFINITY;
let mut hi = f64::NEG_INFINITY;
for s in &self.speedups {
lo = lo.min(*s);
hi = hi.max(*s);
}
if self.speedups.is_empty() { (0.0, 0.0) } else { (lo, hi) }
}
#[must_use]
pub fn speedup_rel_stddev(&self) -> f64 {
if self.speedups.len() < 2 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let n = self.speedups.len() as f64;
let mean = self.speedups.iter().sum::<f64>() / n;
if mean <= 0.0 {
return 0.0;
}
let variance = self.speedups.iter().map(|s| (s - mean).powi(2)).sum::<f64>() / (n - 1.0);
variance.sqrt() / mean
}
#[must_use]
pub fn median_host_load(&self) -> Option<f64> {
if self.host_foreign_load.is_empty() {
return None;
}
let mut sorted = self.host_foreign_load.clone();
sorted.sort_by(f64::total_cmp);
let mid = sorted.len() / 2;
Some(if sorted.len() % 2 == 0 {
(sorted[mid - 1] + sorted[mid]) / 2.0
} else {
sorted[mid]
})
}
#[must_use]
pub fn host_load_rel_stddev(&self) -> Option<f64> {
if self.host_foreign_load.len() < 2 {
return None;
}
#[allow(clippy::cast_precision_loss)]
let n = self.host_foreign_load.len() as f64;
let mean = self.host_foreign_load.iter().sum::<f64>() / n;
if mean <= 1e-6 {
return Some(0.0);
}
let variance =
self.host_foreign_load.iter().map(|s| (s - mean).powi(2)).sum::<f64>() / (n - 1.0);
Some(variance.sqrt() / mean)
}
#[must_use]
pub fn is_publishable(&self) -> bool {
#[allow(clippy::cast_possible_truncation)]
let rounds = self.speedups.len() as u32;
let stationary = self
.host_load_rel_stddev()
.is_none_or(|drift| drift <= MAX_PAIRED_HOST_LOAD_DRIFT);
rounds >= MIN_PAIRED_ROUNDS
&& self.speedup_rel_stddev() <= MAX_PUBLISHABLE_REL_STDDEV
&& stationary
}
pub fn from_unit_secs(
label_a: &str,
label_b: &str,
a_unit_secs: &[f64],
b_unit_secs: &[f64],
foreign_resident_baseline_mib: u64,
foreign_sm_percent_max: u32,
) -> Result<Self, ExatokError> {
if a_unit_secs.len() != b_unit_secs.len() {
return Err(artifact_err(
"paired rounds",
&format!("{} A rounds against {} B rounds", a_unit_secs.len(), b_unit_secs.len()),
"the arms must alternate, so round i of each is the same round",
"emit one timing per arm per round and pass them in alternation order",
));
}
#[allow(clippy::cast_possible_truncation)]
let rounds = a_unit_secs.len() as u32;
if rounds < MIN_PAIRED_ROUNDS {
return Err(artifact_err(
"paired comparison rounds",
&format!("{rounds} rounds"),
"too few rounds to establish a delta spread against a bursty contaminant",
&format!("run at least {MIN_PAIRED_ROUNDS} alternating rounds per arm"),
));
}
for (i, secs) in a_unit_secs.iter().chain(b_unit_secs).enumerate() {
if !secs.is_finite() || *secs <= 0.0 {
return Err(artifact_err(
"paired round timing",
&format!("value {i} is {secs}"),
"a per-unit time must be finite and positive to form a ratio",
"check that the emitting build divided its region by the inner \
iteration count and reported a real unit count",
));
}
}
let speedups = a_unit_secs
.iter()
.zip(b_unit_secs)
.map(|(a, b)| a / b)
.collect();
Ok(Self {
label_a: label_a.to_owned(),
label_b: label_b.to_owned(),
a_unit_secs: a_unit_secs.to_vec(),
b_unit_secs: b_unit_secs.to_vec(),
speedups,
voided_rounds: 0,
foreign_resident_baseline_mib,
foreign_sm_percent_max,
host_foreign_load: Vec::new(),
host_foreign_load_arms: Vec::new(),
unit_work: None,
})
}
#[must_use]
pub fn render(&self) -> String {
let (lo, hi) = self.speedup_range();
let mut s = String::new();
s.push_str(&format!("paired: {} against {}\n", self.label_a, self.label_b));
s.push_str(&format!(
"rounds : {} interleaved, {} voided for device contention\n",
self.speedups.len(),
self.voided_rounds
));
let a_best = self.a_unit_secs.iter().copied().fold(f64::INFINITY, f64::min);
let b_best = self.b_unit_secs.iter().copied().fold(f64::INFINITY, f64::min);
s.push_str(&format!(
"best per unit : {a_best:.9} s ({}), {b_best:.9} s ({})\n",
self.label_a, self.label_b
));
match self.unit_work {
Some((ua, ub)) => s.push_str(&format!(
"work per unit : {} B, {} tokens ({}); {} B, {} tokens ({})\n",
ua.bytes, ua.tokens, self.label_a, ub.bytes, ub.tokens, self.label_b
)),
None => s.push_str(
"work per unit : NOT RECORDED (a cross-process pairing ran no work here)\n",
),
}
s.push_str(&format!(
"median speedup : {:.4}x (range {lo:.4}x to {hi:.4}x, rel stddev {:.4})\n",
self.median_speedup(),
self.speedup_rel_stddev()
));
let ratio = self.median_speedup();
let (faster, factor) = if ratio >= 1.0 {
(&self.label_b, ratio)
} else {
(&self.label_a, 1.0 / ratio)
};
s.push_str(&format!(
"faster arm : {faster} by {factor:.4}x (ratio is {}_secs/{}_secs)\n",
self.label_a, self.label_b
));
s.push_str(&format!(
"foreign device : {} MiB resident baseline, peak {}% SM (evidence, not gated)\n",
self.foreign_resident_baseline_mib, self.foreign_sm_percent_max
));
match (self.median_host_load(), self.host_load_rel_stddev()) {
(Some(level), Some(drift)) => s.push_str(&format!(
"host load : {level:.2} cores foreign (median), drift {drift:.4} \
(gated at {MAX_PAIRED_HOST_LOAD_DRIFT:.2})\n"
)),
_ => s.push_str(
"host load : NOT RECORDED (cross-process pairing; absence is not a \
quiet machine)\n",
),
}
if self.is_publishable() {
s.push_str("verdict : publishable\n");
} else {
let drift = self
.host_load_rel_stddev()
.map_or_else(|| "n/a".to_owned(), |d| format!("{d:.4}"));
s.push_str(&format!(
"verdict : NOT publishable; needs {MIN_PAIRED_ROUNDS} rounds, delta \
spread at or under {MAX_PUBLISHABLE_REL_STDDEV:.2} (got {:.4}) and host-load \
drift at or under {MAX_PAIRED_HOST_LOAD_DRIFT:.2} (got {drift})\n",
self.speedup_rel_stddev()
));
}
s
}
}
pub fn compare_cells(a: Arm<'_>, b: Arm<'_>, rounds: u32) -> Result<Paired, ExatokError> {
compare_cells_at(a, b, rounds, CALIBRATION_TARGET_SECS)
}
fn compare_cells_at(
a: Arm<'_>,
b: Arm<'_>,
rounds: u32,
target_secs: f64,
) -> Result<Paired, ExatokError> {
if rounds < MIN_PAIRED_ROUNDS {
return Err(artifact_err(
"paired comparison rounds",
&rounds.to_string(),
&format!("below the {MIN_PAIRED_ROUNDS} needed for the spread to mean anything"),
"raise rounds to at least MIN_PAIRED_ROUNDS",
));
}
let label_a = a.label.to_owned();
let label_b = b.label.to_owned();
let work_a = a.work;
let work_b = b.work;
let (inner_a, _) = calibrate(work_a, target_secs)?;
let (inner_b, _) = calibrate(work_b, target_secs)?;
let baseline = sample_device_load().map_or(0, |d| d.foreign_resident_mib);
let ceiling = baseline + DEVICE_RESIDENCY_TOLERANCE_MIB;
let mut out = Paired {
label_a,
label_b,
a_unit_secs: Vec::new(),
b_unit_secs: Vec::new(),
speedups: Vec::new(),
voided_rounds: 0,
foreign_resident_baseline_mib: baseline,
foreign_sm_percent_max: 0,
host_foreign_load: Vec::new(),
host_foreign_load_arms: Vec::new(),
unit_work: None,
};
for round in 0..rounds {
let mut attempt = 0;
loop {
let ra = time_region(work_a, inner_a)?;
let rb = time_region(work_b, inner_b)?;
let on_device = ra.own_resident_mib > 0 || rb.own_resident_mib > 0;
let worst = ra.foreign_resident_mib.max(rb.foreign_resident_mib);
out.foreign_sm_percent_max =
out.foreign_sm_percent_max.max(ra.foreign_sm_percent).max(rb.foreign_sm_percent);
if !on_device || worst <= ceiling {
let a_unit = ra.secs / f64::from(inner_a);
let b_unit = rb.secs / f64::from(inner_b);
out.a_unit_secs.push(a_unit);
out.b_unit_secs.push(b_unit);
let load_a = ra.foreign_cpu_secs / ra.secs.max(1e-9);
let load_b = rb.foreign_cpu_secs / rb.secs.max(1e-9);
let host_load = load_a.max(load_b);
out.host_foreign_load.push(host_load);
out.host_foreign_load_arms.push((load_a, load_b));
match out.unit_work {
None => {
if ra.unit.bytes == 0 || rb.unit.bytes == 0 {
return Err(artifact_err(
"paired unit work",
&format!("{} B against {} B", ra.unit.bytes, rb.unit.bytes),
"an arm consumed no input, so its duration measures nothing",
"give both arms real input before comparing them",
));
}
out.unit_work = Some((ra.unit, rb.unit));
}
Some((pin_a, pin_b)) => {
if pin_a != ra.unit || pin_b != rb.unit {
return Err(artifact_err(
"paired unit work",
&format!(
"round {round} did {} B/{} tok against {} B/{} tok, pinned at \
{} B/{} tok against {} B/{} tok",
ra.unit.bytes,
ra.unit.tokens,
rb.unit.bytes,
rb.unit.tokens,
pin_a.bytes,
pin_a.tokens,
pin_b.bytes,
pin_b.tokens
),
"the work changed between rounds, so the ratio compares two \
different quantities",
"make both arms process the same input every round",
));
}
}
}
out.speedups.push(if b_unit > 0.0 { a_unit / b_unit } else { 0.0 });
break;
}
out.voided_rounds += 1;
attempt += 1;
if attempt > MAX_VOIDED_ROUND_RETRIES {
return Err(artifact_err(
"clean paired round",
&format!("round {} of {rounds}", round + 1),
&format!(
"foreign device residency reached {worst} MiB against a {baseline} MiB \
baseline across {} attempts, so every attempt was contaminated",
MAX_VOIDED_ROUND_RETRIES + 1
),
"wait for sibling device work to finish; a paired delta measured through \
another process's residency is not publishable",
));
}
}
}
Ok(out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
const TEST_REGION_SECS: f64 = 0.1;
fn measurement(mb_s: f64, threads: usize, doc_bytes: usize) -> Measurement {
let secs = 0.5_f64;
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let input_bytes = (mb_s * BYTES_PER_MB * secs) as u64;
Measurement {
tokenizer: "probe".to_owned(),
version: "0".to_owned(),
vocab: "cl100k_base".to_owned(),
regime: Regime::Batch,
target_tokens: 1024,
doc_bytes,
docs: 1,
input_bytes,
tokens: input_bytes / 4,
stats: RunStats::new(vec![secs]).unwrap(),
warmups: 1,
threads,
machine: MachineContext {
physical_cores: 16,
logical_cores: 32,
pinned_to_physical: true,
},
mean_mhz: 5000.0,
region_clock: Some(RegionClock { start_mhz: 5000.0, end_mhz: 5000.0, busy_cpus: 16 }),
matches_reference: true,
#[allow(clippy::cast_precision_loss)]
active: ActiveTime { cpu_secs: secs * threads as f64, device_secs: None },
foreign_cpu_secs: 0.0,
inner_reps: 1,
}
}
#[test]
fn published_094_mb_s_baseline_is_flagged_as_a_defect() {
let m = measurement(0.94, 1, 65_536);
match assess_plausibility(&m) {
Plausibility::SuspectedMeasurementDefect {
observed_mb_s_per_core,
floor_mb_s_per_core,
} => {
assert!((observed_mb_s_per_core - 0.94).abs() < 1e-6);
assert!((floor_mb_s_per_core - 5.0).abs() < 1e-12);
}
other => panic!("0.94 MB/s on a 64 KiB document must be a defect, got {other:?}"),
}
}
#[test]
fn real_single_core_tiktoken_throughput_is_plausible() {
assert_eq!(assess_plausibility(&measurement(36.9, 1, 65_536)), Plausibility::Plausible);
}
#[test]
fn small_documents_are_per_call_bound_not_defects() {
assert_eq!(assess_plausibility(&measurement(4.4, 1, 2)), Plausibility::PerCallBound);
assert_eq!(
assess_plausibility(&measurement(4.4, 1, AMORTIZED_DOC_BYTES - 1)),
Plausibility::PerCallBound
);
}
#[test]
fn floor_is_applied_per_core_not_absolute() {
assert!(matches!(
assess_plausibility(&measurement(60.0, 16, 65_536)),
Plausibility::SuspectedMeasurementDefect { .. }
));
assert_eq!(
assess_plausibility(&measurement(2339.0, 16, 65_536)),
Plausibility::Plausible
);
}
#[test]
fn smt_oversubscription_divides_by_cores_not_threads() {
let m = measurement(120.0, 32, 65_536);
assert_eq!(m.machine.physical_cores, 16);
assert!((m.mb_per_second_per_core() - 7.5).abs() < 1e-9);
assert!((m.mb_per_second_per_thread() - 3.75).abs() < 1e-9);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
}
#[test]
fn a_timed_region_too_short_to_trust_is_a_harness_defect() {
let mut m = measurement(800.0, 16, 65_536);
m.stats = RunStats::new(vec![0.019]).unwrap();
match assess_plausibility(&m) {
Plausibility::ShortTimedRegion { timed_secs, minimum_secs } => {
assert!((timed_secs - 0.019).abs() < 1e-12);
assert!((minimum_secs - MIN_TRUSTWORTHY_TIMED_SECS).abs() < 1e-12);
}
other => panic!("a 19 ms timed region must be a harness defect, got {other:?}"),
}
assert!(assess_plausibility(&m).is_harness_defect());
assert!(!assess_plausibility(&m).is_publishable());
m.stats = RunStats::new(vec![MIN_TRUSTWORTHY_TIMED_SECS]).unwrap();
assert!(!assess_plausibility(&m).is_harness_defect());
}
#[test]
fn a_thread_request_that_was_not_delivered_is_a_harness_defect() {
let mut m = measurement(400.0, 16, 65_536);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: None };
match assess_plausibility(&m) {
Plausibility::ThreadsNotHonoured { requested, achieved_parallelism } => {
assert_eq!(requested, 16);
assert!((achieved_parallelism - 1.0).abs() < 1e-9);
}
other => panic!("16 threads at 1x parallelism must be a defect, got {other:?}"),
}
m.active = ActiveTime { cpu_secs: 0.5 * 8.0, device_secs: None };
assert!(!assess_plausibility(&m).is_harness_defect());
m.active = ActiveTime { cpu_secs: 0.5 * 7.9, device_secs: None };
assert!(assess_plausibility(&m).is_harness_defect());
}
#[test]
fn oversubscribed_requests_are_exempt_from_the_parallelism_check() {
let mut m = measurement(120.0, 32, 65_536);
m.active = ActiveTime { cpu_secs: 0.5 * 17.4, device_secs: None };
assert!(!assess_plausibility(&m).is_harness_defect());
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
}
#[test]
fn a_contended_single_thread_cell_is_a_harness_defect_not_a_slow_library() {
let mut m = measurement(19.0, 1, 65_536);
m.active = ActiveTime { cpu_secs: 0.35, device_secs: None };
match assess_plausibility(&m) {
Plausibility::Contended { achieved_parallelism, minimum_efficiency } => {
assert!((achieved_parallelism - 0.7).abs() < 1e-9);
assert!((minimum_efficiency - MIN_SINGLE_THREAD_EFFICIENCY).abs() < 1e-12);
}
other => panic!("0.7x of one pinned core must be contention, got {other:?}"),
}
assert!(assess_plausibility(&m).is_harness_defect());
m.active = ActiveTime { cpu_secs: 0.5 * MIN_SINGLE_THREAD_EFFICIENCY, device_secs: None };
assert!(!assess_plausibility(&m).is_harness_defect());
m.active = ActiveTime { cpu_secs: 0.5 * 0.89, device_secs: None };
assert!(assess_plausibility(&m).is_harness_defect());
m.active = ActiveTime { cpu_secs: 0.5, device_secs: None };
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
}
#[test]
fn foreign_cpu_load_voids_a_cell_that_kept_its_own_core() {
let mut m = measurement(36.9, 1, 65_536);
m.foreign_cpu_secs = 0.5 * 0.20;
assert!((m.achieved_parallelism() - 1.0).abs() < 1e-12);
assert!((m.foreign_load_ratio() - 0.20).abs() < 1e-12);
match assess_plausibility(&m) {
Plausibility::ForeignLoad { foreign_load_ratio, maximum_ratio } => {
assert!((foreign_load_ratio - 0.20).abs() < 1e-12);
assert!((maximum_ratio - MAX_FOREIGN_LOAD_RATIO).abs() < 1e-12);
}
other => panic!("0.2x of a core stolen elsewhere must void the cell, got {other:?}"),
}
assert!(assess_plausibility(&m).is_harness_defect());
assert!(!assess_plausibility(&m).is_publishable());
}
#[test]
fn an_idle_machine_publishes_the_same_cell_the_foreign_gate_voids() {
let mut m = measurement(36.9, 1, 65_536);
m.foreign_cpu_secs = 0.0;
assert!((m.foreign_load_ratio() - 0.0).abs() < 1e-12);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
m.foreign_cpu_secs = 0.5 * 0.04;
assert!((m.foreign_load_ratio() - 0.04).abs() < 1e-12);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
}
#[test]
fn the_foreign_load_gate_boundary_is_exact_and_inclusive() {
let mut m = measurement(36.9, 1, 65_536);
m.foreign_cpu_secs = 0.5 * MAX_FOREIGN_LOAD_RATIO;
assert!((m.foreign_load_ratio() - MAX_FOREIGN_LOAD_RATIO).abs() < 1e-15);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
m.foreign_cpu_secs = 0.5 * 0.06;
match assess_plausibility(&m) {
Plausibility::ForeignLoad { foreign_load_ratio, maximum_ratio } => {
assert!((foreign_load_ratio - 0.06).abs() < 1e-12);
assert!((maximum_ratio - MAX_FOREIGN_LOAD_RATIO).abs() < 1e-12);
}
other => panic!("0.06x is over the {MAX_FOREIGN_LOAD_RATIO} limit, got {other:?}"),
}
}
#[test]
fn a_cell_voided_for_foreign_load_cannot_reach_a_published_table() {
let mut m = measurement(36.9, 1, 65_536);
m.foreign_cpu_secs = 0.5 * 0.30;
let r = report_with(vec![Availability::Measured(m)]);
assert_eq!(r.harness_defects().len(), 1);
let text = r.render_table();
assert!(text.contains("MACHINE NOT EXCLUSIVE"), "{text}");
assert!(text.contains("HARNESS DEFECT"), "{text}");
assert!(text.contains("0.300x of one core"), "{text}");
assert!(!text.contains("harness: every cell timed at least"), "{text}");
}
#[test]
fn foreign_load_outranks_the_thread_gate_on_a_starved_parallel_cell() {
let mut m = measurement(60.0, 16, 65_536);
m.active = ActiveTime { cpu_secs: 0.5 * 4.0, device_secs: None };
assert!((m.achieved_parallelism() - 4.0).abs() < 1e-12);
match assess_plausibility(&m) {
Plausibility::ThreadsNotHonoured { requested, achieved_parallelism } => {
assert_eq!(requested, 16);
assert!((achieved_parallelism - 4.0).abs() < 1e-12);
}
other => panic!("an idle machine delivering 4 of 16 threads is a thread bug: {other:?}"),
}
m.foreign_cpu_secs = 0.5 * 3.0;
match assess_plausibility(&m) {
Plausibility::ForeignLoad { foreign_load_ratio, maximum_ratio } => {
assert!((foreign_load_ratio - 3.0).abs() < 1e-12);
assert!((maximum_ratio - MAX_FOREIGN_LOAD_RATIO).abs() < 1e-12);
}
other => panic!("a starved parallel cell must not be filed as a thread bug: {other:?}"),
}
}
#[test]
fn every_flagged_verdict_explains_itself_and_clean_ones_stay_silent() {
let flagged = [
Plausibility::ShortTimedRegion { timed_secs: 0.019, minimum_secs: 0.25 },
Plausibility::ThreadsNotHonoured { requested: 16, achieved_parallelism: 1.0 },
Plausibility::Contended { achieved_parallelism: 0.7, minimum_efficiency: 0.9 },
Plausibility::ForeignLoad { foreign_load_ratio: 0.3, maximum_ratio: 0.05 },
Plausibility::SuspectedMeasurementDefect {
observed_mb_s_per_core: 3.183,
floor_mb_s_per_core: 5.0,
},
Plausibility::KnownSlowLibrary { observed_mb_s_per_core: 4.6, reason: "measured" },
Plausibility::EstimatorDependent { rel_stddev: 0.31, maximum_rel_stddev: 0.20 },
Plausibility::DeviceStarved { device_utilization: 0.08, minimum_utilization: 0.25 },
];
for v in flagged {
let d = v.diagnosis().unwrap_or_else(|| panic!("{v:?} must explain itself"));
assert!(!d.is_empty(), "{v:?}");
assert!(!d.contains("unclassified"), "{v:?} fell through to a wildcard: {d}");
assert!(!v.table_flag().is_empty(), "{v:?} must be marked in the table");
}
for v in [Plausibility::Plausible, Plausibility::PerCallBound] {
assert_eq!(v.diagnosis(), None, "{v:?}");
assert_eq!(v.table_flag(), "", "{v:?}");
}
}
#[test]
fn table_flags_separate_harness_bugs_from_library_results() {
for v in [
Plausibility::ShortTimedRegion { timed_secs: 0.019, minimum_secs: 0.25 },
Plausibility::ThreadsNotHonoured { requested: 16, achieved_parallelism: 1.0 },
Plausibility::Contended { achieved_parallelism: 0.7, minimum_efficiency: 0.9 },
Plausibility::ForeignLoad { foreign_load_ratio: 0.3, maximum_ratio: 0.05 },
Plausibility::EstimatorDependent { rel_stddev: 0.31, maximum_rel_stddev: 0.20 },
Plausibility::DeviceStarved { device_utilization: 0.08, minimum_utilization: 0.25 },
] {
assert!(v.table_flag().contains("HARNESS"), "{v:?} -> {}", v.table_flag());
assert!(v.is_harness_defect(), "{v:?}");
assert!(!v.is_publishable(), "{v:?}");
}
for v in [
Plausibility::SuspectedMeasurementDefect {
observed_mb_s_per_core: 3.0,
floor_mb_s_per_core: 5.0,
},
Plausibility::KnownSlowLibrary { observed_mb_s_per_core: 4.6, reason: "measured" },
] {
assert!(!v.table_flag().contains("HARNESS"), "{v:?} -> {}", v.table_flag());
assert!(!v.is_harness_defect(), "{v:?}");
assert!(v.is_publishable(), "{v:?}");
}
}
#[test]
fn a_contended_device_cell_cannot_publish_though_every_cpu_gate_is_clean() {
let mut m = measurement(36.9, 1, 65_536);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: Some(0.0625) };
m.foreign_cpu_secs = 0.0;
assert!((m.achieved_parallelism() - 1.0).abs() < 1e-15);
assert!((m.foreign_load_ratio() - 0.0).abs() < 1e-15);
assert!((m.device_utilization().unwrap_or(0.0) - 0.125).abs() < 1e-15);
match assess_plausibility(&m) {
Plausibility::DeviceStarved { device_utilization, minimum_utilization } => {
assert!((device_utilization - 0.125).abs() < 1e-15);
assert!((minimum_utilization - MIN_DEVICE_UTILIZATION).abs() < 1e-15);
}
other => panic!("a device idle for 87.5% of the window must void the cell: {other:?}"),
}
}
#[test]
fn an_exclusive_device_cell_publishes_the_shape_the_starved_gate_voids() {
let mut m = measurement(36.9, 1, 65_536);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: Some(0.4375) };
assert!((m.device_utilization().unwrap_or(0.0) - 0.875).abs() < 1e-15);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
}
#[test]
fn the_device_utilization_gate_boundary_is_exact_and_inclusive() {
let mut m = measurement(36.9, 1, 65_536);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: Some(0.125) };
assert!((m.device_utilization().unwrap_or(0.0) - MIN_DEVICE_UTILIZATION).abs() < 1e-15);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: Some(0.09375) };
match assess_plausibility(&m) {
Plausibility::DeviceStarved { device_utilization, .. } => {
assert!((device_utilization - 0.1875).abs() < 1e-15);
}
other => panic!("0.1875 is under the {MIN_DEVICE_UTILIZATION} limit: {other:?}"),
}
}
#[test]
fn a_host_baseline_without_a_device_timeline_is_never_device_starved() {
let mut m = measurement(36.9, 1, 65_536);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: None };
assert_eq!(m.device_utilization(), None);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
}
#[test]
fn repetitions_that_disagree_cannot_publish_a_best_of_n_figure() {
let mut m = measurement(36.9, 1, 65_536);
m.stats = RunStats::new(vec![0.375, 0.5, 0.625]).unwrap();
m.active = ActiveTime { cpu_secs: m.stats.best(), device_secs: None };
assert!((m.stats.rel_stddev() - 0.25).abs() < 1e-15);
match assess_plausibility(&m) {
Plausibility::EstimatorDependent { rel_stddev, maximum_rel_stddev } => {
assert!((rel_stddev - 0.25).abs() < 1e-15);
assert!((maximum_rel_stddev - MAX_PUBLISHABLE_REL_STDDEV).abs() < 1e-15);
}
other => panic!("a 25% spread across reps must void the cell: {other:?}"),
}
}
#[test]
fn the_variance_interlock_is_looser_than_the_noisy_advisory() {
let mut m = measurement(36.9, 1, 65_536);
m.stats = RunStats::new(vec![0.40625, 0.5, 0.59375]).unwrap();
m.active = ActiveTime { cpu_secs: m.stats.best(), device_secs: None };
assert!((m.stats.rel_stddev() - 0.1875).abs() < 1e-15);
assert!(m.stats.rel_stddev() > NOISY_REL_STDDEV);
assert_eq!(assess_plausibility(&m), Plausibility::Plausible);
m.stats = RunStats::new(vec![0.390625, 0.5, 0.609375]).unwrap();
m.active = ActiveTime { cpu_secs: m.stats.best(), device_secs: None };
assert!((m.stats.rel_stddev() - 0.21875).abs() < 1e-15);
assert!(assess_plausibility(&m).is_harness_defect());
}
#[test]
fn a_per_call_bound_cell_is_still_held_to_repeatability() {
let mut m = measurement(4.4, 1, AMORTIZED_DOC_BYTES - 1);
m.stats = RunStats::new(vec![0.375, 0.5, 0.625]).unwrap();
m.active = ActiveTime { cpu_secs: m.stats.best(), device_secs: None };
assert!(m.doc_bytes < AMORTIZED_DOC_BYTES);
match assess_plausibility(&m) {
Plausibility::EstimatorDependent { rel_stddev, .. } => {
assert!((rel_stddev - 0.25).abs() < 1e-15);
}
other => panic!("an unrepeatable small-input cell is not merely per-call: {other:?}"),
}
}
#[test]
fn device_starvation_outranks_the_per_core_floor_as_a_cause() {
let mut m = measurement(1.0, 1, 65_536);
m.active = ActiveTime { cpu_secs: 0.5, device_secs: Some(0.0625) };
assert!(m.mb_per_second_per_core() < PLAUSIBLE_FLOOR_MB_S_PER_CORE);
match assess_plausibility(&m) {
Plausibility::DeviceStarved { device_utilization, .. } => {
assert!((device_utilization - 0.125).abs() < 1e-15);
}
other => panic!("a contended device cell must not be filed sub-floor: {other:?}"),
}
}
#[test]
fn a_known_slow_library_is_recorded_with_its_reason_not_suppressed() {
let mut m = measurement(4.6, 1, 65_536);
m.tokenizer = "hf-tokenizers".to_owned();
match assess_plausibility(&m) {
Plausibility::KnownSlowLibrary { observed_mb_s_per_core, reason } => {
assert!((observed_mb_s_per_core - 4.6).abs() < 1e-6);
assert!(reason.contains("encode_batch_fast"));
}
other => panic!("hf-tokenizers under the floor must be known-slow, got {other:?}"),
}
assert!(!assess_plausibility(&m).is_harness_defect());
m.tokenizer = "tiktoken".to_owned();
assert!(matches!(
assess_plausibility(&m),
Plausibility::SuspectedMeasurementDefect { .. }
));
assert_eq!(known_slow_reason("tiktoken"), None);
assert_eq!(known_slow_reason("gigatoken"), None);
}
#[test]
fn plausibility_boundary_is_inclusive_at_the_floor() {
assert_eq!(
assess_plausibility(&measurement(PLAUSIBLE_FLOOR_MB_S_PER_CORE, 1, AMORTIZED_DOC_BYTES)),
Plausibility::Plausible
);
assert!(matches!(
assess_plausibility(&measurement(4.999, 1, AMORTIZED_DOC_BYTES)),
Plausibility::SuspectedMeasurementDefect { .. }
));
}
#[test]
fn zero_and_negative_wall_times_are_rejected() {
assert!(RunStats::new(vec![]).is_err());
assert!(RunStats::new(vec![0.0]).is_err());
assert!(RunStats::new(vec![-1e-9]).is_err());
assert!(RunStats::new(vec![f64::NAN]).is_err());
assert!(RunStats::new(vec![f64::INFINITY]).is_err());
assert!(RunStats::new(vec![1e-9]).is_ok());
}
#[test]
fn run_stats_computes_exact_min_mean_median_and_spread() {
let s = RunStats::new(vec![0.20, 0.10, 0.30]).unwrap();
assert_eq!(s.reps(), 3);
assert!((s.best() - 0.10).abs() < 1e-12);
assert!((s.mean() - 0.20).abs() < 1e-12);
assert!((s.median() - 0.20).abs() < 1e-12);
assert!((s.rel_stddev() - 0.5).abs() < 1e-12);
let one = RunStats::new(vec![0.25]).unwrap();
assert!((one.rel_stddev() - 0.0).abs() < 1e-12);
assert!((one.best() - 0.25).abs() < 1e-12);
}
#[test]
fn throughput_derivations_are_exact_and_si_based() {
let mut m = measurement(16.0, 16, 65_536);
m.input_bytes = 8_000_000;
m.tokens = 2_000_000;
m.stats = RunStats::new(vec![0.5]).unwrap();
assert!((m.bytes_per_second() - 16_000_000.0).abs() < 1e-6);
assert!((m.mb_per_second() - 16.0).abs() < 1e-12);
assert!((m.gb_per_second() - 0.016).abs() < 1e-12);
assert!((m.mb_per_second_per_thread() - 1.0).abs() < 1e-12);
assert!((m.tokens_per_second() - 4_000_000.0).abs() < 1e-6);
assert!((m.ns_per_token() - 250.0).abs() < 1e-9);
assert!((m.bytes_per_token() - 4.0).abs() < 1e-12);
}
#[test]
fn derived_rates_never_produce_nan_or_infinity() {
let mut m = measurement(16.0, 0, 65_536);
m.tokens = 0;
assert!((m.mb_per_second_per_thread() - 0.0).abs() < 1e-12);
assert!((m.tokens_per_second() - 0.0).abs() < 1e-12);
assert!((m.ns_per_token() - 0.0).abs() < 1e-12);
assert!((m.bytes_per_token() - 0.0).abs() < 1e-12);
}
#[test]
fn smt_siblings_reduce_to_one_cpu_per_physical_core() {
let lists: Vec<String> = (0..32).map(|c| format!("{},{}", c % 16, (c % 16) + 16)).collect();
let (cores, reps) = Machine::physical_from_siblings(&lists, 32);
assert_eq!(cores, 16);
assert_eq!(reps, (0..16).collect::<Vec<usize>>());
}
#[test]
fn absent_topology_assumes_no_smt_rather_than_guessing() {
let (cores, reps) = Machine::physical_from_siblings(&[], 8);
assert_eq!(cores, 8);
assert_eq!(reps, (0..8).collect::<Vec<usize>>());
let partial = vec!["0,4".to_owned(), "1,5".to_owned()];
let (cores, reps) = Machine::physical_from_siblings(&partial, 8);
assert_eq!(cores, 8);
assert_eq!(reps.len(), 8);
}
#[test]
fn cpuinfo_without_processors_is_an_error() {
let err = Machine::from_probe_inputs("model name : Nothing\n", vec![], None, 0, None);
assert!(err.is_err());
let ok = Machine::from_probe_inputs(
"processor : 0\nmodel name : AMD Ryzen 9 9950X 16-Core Processor\nprocessor : 1\n",
vec![],
Some("performance".to_owned()),
96_399_728,
None,
)
.unwrap();
assert_eq!(ok.logical_cores, 2);
assert_eq!(ok.cpu_model, "AMD Ryzen 9 9950X 16-Core Processor");
assert_eq!(ok.governor.as_deref(), Some("performance"));
assert_eq!(ok.total_memory_kib, 96_399_728);
assert_eq!(ok.smt_siblings_per_core(), 1);
}
#[test]
fn clock_drift_reports_signed_ratio_and_flags_significance() {
let d = ClockDrift {
first: ClockSample { mean_mhz: 5000.0, min_mhz: 4900.0, max_mhz: 5100.0 },
last: ClockSample { mean_mhz: 4600.0, min_mhz: 4500.0, max_mhz: 4700.0 },
};
assert!((d.drift_ratio() - -0.08).abs() < 1e-12);
assert!(d.is_significant());
let steady = ClockDrift {
first: ClockSample { mean_mhz: 5000.0, min_mhz: 5000.0, max_mhz: 5000.0 },
last: ClockSample { mean_mhz: 4950.0, min_mhz: 4950.0, max_mhz: 4950.0 },
};
assert!((steady.drift_ratio() - -0.01).abs() < 1e-12);
assert!(!steady.is_significant());
let edge = ClockDrift {
first: ClockSample { mean_mhz: 100.0, min_mhz: 100.0, max_mhz: 100.0 },
last: ClockSample { mean_mhz: 95.0, min_mhz: 95.0, max_mhz: 95.0 },
};
assert!(edge.is_significant());
}
#[test]
fn clock_sample_parses_mhz_and_returns_none_when_absent() {
let s = ClockSample::parse("cpu MHz : 4000.0\ncpu MHz : 5000.0\ncpu MHz : 4500.0\n")
.expect("three MHz lines parse");
assert!((s.mean_mhz - 4500.0).abs() < 1e-12);
assert!((s.min_mhz - 4000.0).abs() < 1e-12);
assert!((s.max_mhz - 5000.0).abs() < 1e-12);
assert!(ClockSample::parse("processor : 0\nmodel name : x\n").is_none());
assert!(ClockSample::parse("cpu MHz : unknown\n").is_none());
}
#[test]
fn corpus_truncation_respects_utf8_boundaries() {
let s = "héllo";
assert_eq!(s.len(), 6);
assert_eq!(truncate_at_char_boundary(s, 6), "héllo");
assert_eq!(truncate_at_char_boundary(s, 2), "h");
assert_eq!(truncate_at_char_boundary(s, 3), "hé");
assert_eq!(truncate_at_char_boundary(s, 0), "");
assert_eq!(truncate_at_char_boundary(s, 99), "héllo");
let emoji = "a\u{1F600}b";
assert_eq!(emoji.len(), 6);
assert_eq!(truncate_at_char_boundary(emoji, 4), "a");
assert_eq!(truncate_at_char_boundary(emoji, 5), "a\u{1F600}");
}
#[test]
fn corpus_digest_detects_single_byte_change() {
let a = corpus_digest(b"the quick brown fox");
let b = corpus_digest(b"the quick brown fox");
let c = corpus_digest(b"the quick brown fbx");
assert_eq!(a, b);
assert_ne!(a, c);
assert_ne!(corpus_digest(b"ab"), corpus_digest(b"ba"));
assert_ne!(corpus_digest(b""), corpus_digest(b"\0"));
}
#[test]
fn standard_thread_plans_pin_one_thread_per_physical_core() {
let machine = Machine {
cpu_model: "AMD Ryzen 9 9950X 16-Core Processor".to_owned(),
physical_cores: 16,
logical_cores: 32,
physical_core_cpus: (0..16).collect(),
governor: Some("performance".to_owned()),
total_memory_kib: 96_399_728,
gpu: None,
};
let plans = ThreadPlan::standard(&machine);
assert_eq!(plans.len(), 3);
assert_eq!(plans[0].threads, 1);
assert_eq!(plans[0].cpus, vec![0]);
assert!(plans[0].pinned_to_physical);
assert!(plans[0].include_single_doc);
assert_eq!(plans[0].cpu_list().as_deref(), Some("0"));
assert_eq!(plans[1].threads, 16);
assert_eq!(plans[1].cpus, (0..16).collect::<Vec<usize>>());
assert!(plans[1].pinned_to_physical);
assert!(!plans[1].include_single_doc);
assert_eq!(
plans[1].cpu_list().as_deref(),
Some("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15")
);
assert_eq!(plans[2].threads, 32);
assert!(plans[2].cpus.is_empty());
assert!(!plans[2].pinned_to_physical);
assert_eq!(plans[2].cpu_list(), None);
}
#[test]
fn thread_plans_omit_redundant_invocations() {
let no_smt = Machine {
cpu_model: "x".to_owned(),
physical_cores: 4,
logical_cores: 4,
physical_core_cpus: (0..4).collect(),
governor: None,
total_memory_kib: 0,
gpu: None,
};
let plans = ThreadPlan::standard(&no_smt);
assert_eq!(plans.len(), 2);
assert_eq!(plans[1].threads, 4);
let single = Machine { physical_cores: 1, logical_cores: 1, physical_core_cpus: vec![0], ..no_smt };
let plans = ThreadPlan::standard(&single);
assert_eq!(plans.len(), 1);
assert_eq!(plans[0].threads, 1);
}
#[test]
fn driver_record_parses_every_field_exactly() {
let ctx = MachineContext { physical_cores: 16, logical_cores: 32, pinned_to_physical: true };
let line = "record\ttokenizer=tiktoken\tversion=0.12.0\tvocab=cl100k_base\tregime=batch\t\
target_tokens=65536\tdoc_bytes=262144\tdocs=32\tinput_bytes=8388608\t\
tokens=2097152\tsamples=0.25,0.26,0.30\twarmups=1\tthreads=16\t\
mean_mhz=5100.5\tmatches_ref=1\tcpu_secs=3.90\tinner_reps=4\t\
foreign_cpu_secs=0.012";
let ev = parse_driver_line(line, ctx, 7).unwrap().expect("a record");
let DriverEvent::Record(m) = ev else { panic!("expected a record event") };
assert_eq!(m.tokenizer, "tiktoken");
assert_eq!(m.version, "0.12.0");
assert_eq!(m.vocab, "cl100k_base");
assert_eq!(m.regime, Regime::Batch);
assert_eq!(m.target_tokens, 65_536);
assert_eq!(m.doc_bytes, 262_144);
assert_eq!(m.docs, 32);
assert_eq!(m.input_bytes, 8_388_608);
assert_eq!(m.tokens, 2_097_152);
assert_eq!(m.threads, 16);
assert_eq!(m.warmups, 1);
assert_eq!(m.machine, ctx);
assert!((m.mean_mhz - 5100.5).abs() < 1e-9);
assert_eq!(m.stats.reps(), 3);
assert!((m.stats.best() - 0.25).abs() < 1e-12);
assert_eq!(m.inner_reps, 4);
assert_eq!(m.active.device_secs, None);
assert_eq!(m.timed_bytes(), 33_554_432);
assert_eq!(m.timed_tokens(), 8_388_608);
assert!((m.mb_per_second() - 134.217_728).abs() < 1e-9);
assert!((m.mb_per_second_per_thread() - 8.388_608).abs() < 1e-9);
assert!((m.achieved_parallelism() - 15.6).abs() < 1e-9);
}
#[test]
fn malformed_records_are_errors_not_silent_skips() {
let ctx = MachineContext { physical_cores: 1, logical_cores: 1, pinned_to_physical: false };
let missing = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=batch\ttarget_tokens=1\t\
doc_bytes=1\tdocs=1\tinput_bytes=1\tsamples=0.1\twarmups=0\tthreads=1\t\
mean_mhz=1\tmatches_ref=1\tcpu_secs=0.1\tinner_reps=1\t\
foreign_cpu_secs=0.0";
assert!(parse_driver_line(missing, ctx, 1).is_err());
let bad_regime = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=streaming\t\
target_tokens=1\tdoc_bytes=1\tdocs=1\tinput_bytes=1\ttokens=1\t\
samples=0.1\twarmups=0\tthreads=1\tmean_mhz=1\tmatches_ref=1\t\
cpu_secs=0.1\tinner_reps=1\tforeign_cpu_secs=0.0";
assert!(parse_driver_line(bad_regime, ctx, 2).is_err());
let bad_sample = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=batch\ttarget_tokens=1\t\
doc_bytes=1\tdocs=1\tinput_bytes=1\ttokens=1\tsamples=fast\twarmups=0\t\
threads=1\tmean_mhz=1\tmatches_ref=1\tcpu_secs=0.1\tinner_reps=1\t\
foreign_cpu_secs=0.0";
assert!(parse_driver_line(bad_sample, ctx, 3).is_err());
let zero = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=batch\ttarget_tokens=1\t\
doc_bytes=1\tdocs=1\tinput_bytes=1\ttokens=1\tsamples=0.0\twarmups=0\t\
threads=1\tmean_mhz=1\tmatches_ref=1\tcpu_secs=0.1\tinner_reps=1\t\
foreign_cpu_secs=0.0";
assert!(parse_driver_line(zero, ctx, 4).is_err());
let no_inner = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=batch\ttarget_tokens=1\t\
doc_bytes=1\tdocs=1\tinput_bytes=1\ttokens=1\tsamples=0.1\twarmups=0\t\
threads=1\tmean_mhz=1\tmatches_ref=1\tcpu_secs=0.1\t\
foreign_cpu_secs=0.0";
assert!(parse_driver_line(no_inner, ctx, 5).is_err());
}
#[test]
fn id_agreement_flag_round_trips_and_is_required() {
let ctx = MachineContext { physical_cores: 1, logical_cores: 1, pinned_to_physical: false };
let base = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=batch\ttarget_tokens=1\t\
doc_bytes=1\tdocs=1\tinput_bytes=1\ttokens=1\tsamples=0.1\twarmups=0\t\
threads=1\tmean_mhz=1\tcpu_secs=0.1\tinner_reps=1\tforeign_cpu_secs=0.0";
assert!(
parse_driver_line(base, ctx, 1).is_err(),
"a record without matches_ref must be rejected, not assumed equivalent"
);
let yes = parse_driver_line(&format!("{base}\tmatches_ref=1"), ctx, 2).unwrap();
let Some(DriverEvent::Record(m)) = yes else { panic!("expected a record") };
assert!(m.matches_reference);
let no = parse_driver_line(&format!("{base}\tmatches_ref=0"), ctx, 3).unwrap();
let Some(DriverEvent::Record(m)) = no else { panic!("expected a record") };
assert!(!m.matches_reference);
}
#[test]
fn active_time_is_recorded_separately_and_device_time_is_optional() {
let ctx = MachineContext { physical_cores: 16, logical_cores: 32, pinned_to_physical: true };
let base = "record\ttokenizer=t\tversion=1\tvocab=v\tregime=batch\ttarget_tokens=1\t\
doc_bytes=8192\tdocs=1\tinput_bytes=33554432\ttokens=8388608\tsamples=0.25\t\
warmups=1\tthreads=16\tmean_mhz=5000\tmatches_ref=1\tinner_reps=1\t\
foreign_cpu_secs=0.0";
assert!(parse_driver_line(base, ctx, 1).is_err());
let host = parse_driver_line(&format!("{base}\tcpu_secs=3.75"), ctx, 2).unwrap();
let Some(DriverEvent::Record(m)) = host else { panic!("expected a record") };
assert!((m.active.cpu_secs - 3.75).abs() < 1e-12);
assert_eq!(m.active.device_secs, None);
assert_eq!(m.device_utilization(), None);
assert!((m.achieved_parallelism() - 15.0).abs() < 1e-12);
assert!((m.active_secs() - 3.75).abs() < 1e-12);
let dev =
parse_driver_line(&format!("{base}\tcpu_secs=0.30\tdevice_secs=0.05"), ctx, 3).unwrap();
let Some(DriverEvent::Record(m)) = dev else { panic!("expected a record") };
assert_eq!(m.active.device_secs, Some(0.05));
assert!((m.active_secs() - 0.05).abs() < 1e-12);
assert!((m.device_utilization().unwrap_or(0.0) - 0.2).abs() < 1e-12);
}
#[test]
fn achieved_parallelism_exposes_a_thread_count_that_never_ran() {
let mut honest = measurement(4115.0, 16, 65_536);
honest.active = ActiveTime { cpu_secs: honest.stats.best() * 15.6, device_secs: None };
assert!((honest.achieved_parallelism() - 15.6).abs() < 1e-9);
let mut lying = measurement(60.0, 16, 65_536);
lying.active = ActiveTime { cpu_secs: lying.stats.best(), device_secs: None };
assert!((lying.achieved_parallelism() - 1.0).abs() < 1e-12);
let text = report_with(vec![Availability::Measured(lying)]).render_table();
assert!(text.contains("par"), "{text}");
assert!(text.contains(" 1.00"), "{text}");
}
#[test]
fn disagreeing_libraries_are_named_and_flagged_in_the_table() {
let mut good = measurement(36.9, 1, 65_536);
good.tokenizer = "tiktoken".to_owned();
let mut bad = measurement(90.0, 1, 65_536);
bad.tokenizer = "hf-tokenizers".to_owned();
bad.matches_reference = false;
let r = report_with(vec![Availability::Measured(good), Availability::Measured(bad)]);
assert_eq!(r.disagreeing().len(), 1);
assert_eq!(r.disagreeing()[0].tokenizer, "hf-tokenizers");
let text = r.render_table();
assert!(text.contains("ids DIFFERENT from the reference"), "{text}");
assert!(text.contains("hf-tokenizers"), "{text}");
assert!(text.contains(" NO"), "{text}");
let clean = report_with(vec![Availability::Measured(measurement(36.9, 1, 65_536))]);
assert!(clean.disagreeing().is_empty());
assert!(clean.render_table().contains("ids identical to the reference"));
}
#[test]
fn the_sweep_refuses_a_corpus_the_baselines_tokenize_differently() {
let corpus = CorpusFacts {
origin: "deterministic sorted walk of /crate".to_owned(),
bytes: 1_048_576,
files: 24,
digest: 6_380_138_651_704_881_638,
};
let mut tiktoken = measurement(36.9, 1, 65_536);
tiktoken.tokenizer = "tiktoken".to_owned();
tiktoken.input_bytes = 1_048_576;
tiktoken.tokens = 254_143;
let mut hf = tiktoken.clone();
hf.tokenizer = "hf-tokenizers".to_owned();
hf.tokens = 254_124;
let rows = vec![Availability::Measured(tiktoken), Availability::Measured(hf)];
let err = refuse_noncomparable_cells(&rows, &corpus)
.expect_err("a 19-token disagreement on identical bytes must refuse the sweep");
let text = err.to_string();
assert!(text.contains("cl100k_base batch target_tokens=1024"), "{text}");
assert!(text.contains("hf-tokenizers"), "{text}");
assert!(text.contains("tiktoken"), "{text}");
assert!(text.contains("254124"), "{text}");
assert!(text.contains("254143"), "{text}");
assert!(text.contains("1048576"), "{text}");
assert!(text.contains("6380138651704881638"), "{text}");
}
#[test]
fn the_sweep_publishes_a_corpus_the_baselines_tokenize_identically() {
let corpus = CorpusFacts {
origin: "test".to_owned(),
bytes: 1_048_576,
files: 24,
digest: 7,
};
let mut tiktoken = measurement(36.9, 1, 65_536);
tiktoken.tokenizer = "tiktoken".to_owned();
tiktoken.input_bytes = 1_048_576;
tiktoken.tokens = 254_143;
let mut hf = tiktoken.clone();
hf.tokenizer = "hf-tokenizers".to_owned();
let mut other_cell = tiktoken.clone();
other_cell.tokenizer = "hf-tokenizers".to_owned();
other_cell.regime = Regime::SingleDocument;
other_cell.input_bytes = 262_144;
other_cell.tokens = 65_536;
let rows = vec![
Availability::Measured(tiktoken),
Availability::Measured(hf),
Availability::Measured(other_cell),
Availability::Unavailable {
tokenizer: "gigatoken".to_owned(),
reason: "not installed".to_owned(),
},
];
refuse_noncomparable_cells(&rows, &corpus)
.expect("agreeing cells, distinct cells and unavailable rows all pass");
let single = vec![Availability::Measured(measurement(36.9, 1, 65_536))];
refuse_noncomparable_cells(&single, &corpus).expect("nothing to disagree with");
}
#[test]
fn the_sweep_refuses_a_cell_whose_libraries_read_different_bytes() {
let corpus = CorpusFacts {
origin: "test".to_owned(),
bytes: 1_048_576,
files: 24,
digest: 7,
};
let mut a = measurement(36.9, 1, 65_536);
a.tokenizer = "tiktoken".to_owned();
a.input_bytes = 1_048_576;
a.tokens = 254_143;
let mut b = a.clone();
b.tokenizer = "hf-tokenizers".to_owned();
b.input_bytes = 1_048_320;
let rows = vec![Availability::Measured(a), Availability::Measured(b)];
let err = refuse_noncomparable_cells(&rows, &corpus)
.expect_err("different inputs void the cell before tokenization is reached");
let text = err.to_string();
assert!(text.contains("1048320"), "{text}");
assert!(text.contains("1048576"), "{text}");
}
#[test]
fn batch_doc_count_covers_the_budget_exactly() {
assert_eq!(batch_docs(1024, 16 * 1024 * 1024), 16_384);
assert_eq!(batch_docs(1000, 16 * 1024 * 1024), 16_778);
assert_eq!(batch_docs(3, 10), 4);
assert_eq!(batch_docs(10, 10), 1);
assert_eq!(batch_docs(1 << 24, 1 << 20), 1);
assert_eq!(batch_docs(0, 1 << 20), 1);
assert_eq!(batch_docs(1024, 0), 1);
}
#[test]
fn batch_regime_refuses_sizes_that_starve_threads() {
let budget = 16 * 1024 * 1024;
assert_eq!(batch_docs(512 * 1024, budget), 32);
assert!(batch_admits(512 * 1024, budget, 32));
assert!(batch_admits(512 * 1024, budget, 16));
assert_eq!(batch_docs(1024 * 1024, budget), 16);
assert!(batch_admits(1024 * 1024, budget, 16));
assert!(!batch_admits(1024 * 1024, budget, 32));
assert!(batch_admits(16 * 1024 * 1024, budget, 1));
assert!(!batch_admits(16 * 1024 * 1024, budget, 2));
}
#[test]
fn notes_are_ignored_and_unavailable_reasons_survive_verbatim() {
let ctx = MachineContext { physical_cores: 1, logical_cores: 1, pinned_to_physical: false };
assert_eq!(parse_driver_line("", ctx, 1).unwrap(), None);
assert_eq!(parse_driver_line("loading vocabulary...", ctx, 2).unwrap(), None);
assert_eq!(parse_driver_line("# a comment", ctx, 3).unwrap(), None);
let ev = parse_driver_line(
"unavailable\ttokenizer=fastokens\treason=import failed: no module named x=y",
ctx,
4,
)
.unwrap()
.expect("an unavailable event");
assert_eq!(
ev,
DriverEvent::Unavailable {
tokenizer: "fastokens".to_owned(),
reason: "import failed: no module named x=y".to_owned(),
}
);
}
#[test]
fn calibration_event_carries_bytes_per_token() {
let ctx = MachineContext { physical_cores: 1, logical_cores: 1, pinned_to_physical: false };
let ev = parse_driver_line(
"calibration\tvocab=cl100k_base\tbytes_per_token=3.9041",
ctx,
1,
)
.unwrap()
.expect("a calibration event");
match ev {
DriverEvent::Calibration { vocab, bytes_per_token } => {
assert_eq!(vocab, "cl100k_base");
assert!((bytes_per_token - 3.9041).abs() < 1e-9);
}
other => panic!("expected calibration, got {other:?}"),
}
}
#[test]
fn json_strings_escape_quotes_backslashes_and_controls() {
assert_eq!(json_str("plain"), "\"plain\"");
assert_eq!(json_str("a\"b"), "\"a\\\"b\"");
assert_eq!(json_str("C:\\path"), "\"C:\\\\path\"");
assert_eq!(json_str("a\nb\tc\rd"), "\"a\\nb\\tc\\rd\"");
assert_eq!(json_str("\u{1}"), "\"\\u0001\"");
}
fn report_with(rows: Vec<Availability>) -> Report {
Report {
machine: Machine {
cpu_model: "AMD Ryzen 9 9950X 16-Core Processor".to_owned(),
physical_cores: 16,
logical_cores: 32,
physical_core_cpus: (0..16).collect(),
governor: Some("performance".to_owned()),
total_memory_kib: 96_399_728,
gpu: Some(GpuInfo {
name: "NVIDIA GeForce RTX 5090".to_owned(),
memory_mib: "32607 MiB".to_owned(),
driver_version: "570.211.01".to_owned(),
compute_capability: "12.0".to_owned(),
}),
},
corpus: CorpusFacts {
origin: "test".to_owned(),
bytes: 1024,
files: 2,
digest: 0x1234_5678_9abc_def0,
},
vocab: "cl100k_base".to_owned(),
rows,
clock: None,
bytes_per_token: Some(3.9041),
report_path: None,
}
}
#[test]
fn table_states_core_counts_and_per_thread_throughput_inline() {
let mut m = measurement(4115.0, 16, 1_048_576);
m.tokenizer = "gigatoken".to_owned();
let text = report_with(vec![Availability::Measured(m)]).render_table();
assert!(text.contains("16 physical / 32 logical"), "{text}");
assert!(text.contains("2 SMT siblings per core"), "{text}");
assert!(text.contains("MB/s/core"), "{text}");
assert!(text.contains("MB/s/thread"), "{text}");
assert!(text.contains("NVIDIA GeForce RTX 5090"), "{text}");
assert!(text.contains("cl100k_base"), "{text}");
assert!(text.contains("3.9041"), "{text}");
assert!(text.contains("257.19"), "{text}");
assert!(text.contains("gigatoken"), "{text}");
}
#[test]
fn unavailable_baselines_are_listed_with_reasons_and_no_number() {
let text = report_with(vec![Availability::Unavailable {
tokenizer: "fastokens".to_owned(),
reason: "wheel has no cp312 build for this platform".to_owned(),
}])
.render_table();
assert!(text.contains("fastokens"), "{text}");
assert!(text.contains("wheel has no cp312 build for this platform"), "{text}");
assert!(text.contains("never estimated"), "{text}");
let r = report_with(vec![Availability::Unavailable {
tokenizer: "x".to_owned(),
reason: "y".to_owned(),
}]);
assert_eq!(r.measured().count(), 0);
assert_eq!(r.peak_for("x"), None);
}
#[test]
fn rendered_table_flags_suspected_defect_rows() {
let mut bad = measurement(50.92, 16, 60_477);
bad.tokenizer = "tiktoken".to_owned();
assert!((bad.mb_per_second_per_core() - 3.1825).abs() < 1e-4);
assert!(bad.mb_per_second_per_core() < PLAUSIBLE_FLOOR_MB_S_PER_CORE);
let r = report_with(vec![Availability::Measured(bad)]);
assert_eq!(r.suspected_defects().len(), 1);
assert_eq!(r.harness_defects().len(), 0);
assert_eq!(r.known_slow().len(), 0);
let text = r.render_table();
assert!(text.contains("UNEXPLAINED, UNDER FLOOR"), "{text}");
assert!(text.contains("5.0 MB/s per-core floor with no"), "{text}");
assert!(text.contains("3.183 MB/s/core"), "{text}");
assert!(text.contains("50.92"), "{text}");
}
#[test]
fn clean_sweep_states_the_floor_was_cleared() {
let r = report_with(vec![Availability::Measured(measurement(36.9, 1, 65_536))]);
assert!(r.suspected_defects().is_empty());
assert!(r.harness_defects().is_empty());
let text = r.render_table();
assert!(text.contains("clears the 5.0 MB/s per-core floor"), "{text}");
assert!(text.contains("timed at least 0.25 s"), "{text}");
assert!(text.contains("delivered at least 50% of its requested threads"), "{text}");
assert!(!text.contains("UNEXPLAINED"), "{text}");
assert!(!text.contains("HARNESS DEFECT"), "{text}");
}
#[test]
fn peak_selects_the_fastest_row_for_that_tokenizer_only() {
let mut slow = measurement(40.0, 1, 65_536);
slow.tokenizer = "tiktoken".to_owned();
let mut fast = measurement(130.0, 16, 65_536);
fast.tokenizer = "tiktoken".to_owned();
let mut other = measurement(4115.0, 16, 65_536);
other.tokenizer = "gigatoken".to_owned();
let r = report_with(vec![
Availability::Measured(slow),
Availability::Measured(fast),
Availability::Measured(other),
]);
let peak = r.peak_for("tiktoken").expect("tiktoken rows exist");
assert!((peak.mb_per_second() - 130.0).abs() < 1e-9);
assert_eq!(peak.threads, 16);
let g = r.peak_for("gigatoken").expect("gigatoken row exists");
assert!((g.mb_per_second() - 4115.0).abs() < 1e-9);
assert_eq!(r.peak_for("absent"), None);
}
#[test]
fn json_carries_absolute_figures_and_core_counts() {
let mut m = measurement(36.9, 1, 65_536);
m.tokenizer = "tiktoken".to_owned();
let json = report_with(vec![
Availability::Measured(m),
Availability::Unavailable {
tokenizer: "fastokens".to_owned(),
reason: "not installed".to_owned(),
},
])
.to_json();
for needle in [
"\"physical_cores\": 16",
"\"logical_cores\": 32",
"\"threads\": 1",
"\"pinned_to_physical\": true",
"\"mb_per_s\": 36.9",
"\"mb_per_s_per_thread\": 36.9",
"\"status\": \"unavailable\"",
"\"reason\": \"not installed\"",
"\"digest\": \"0x123456789abcdef0\"",
"\"bytes_per_token\": 3.904100",
] {
assert!(json.contains(needle), "missing {needle} in {json}");
}
assert_eq!(json.matches('{').count(), json.matches('}').count());
assert_eq!(json.matches('[').count(), json.matches(']').count());
}
#[test]
fn regime_round_trips_and_rejects_unknown_spellings() {
assert_eq!(Regime::SingleDocument.as_str(), "single");
assert_eq!(Regime::Batch.as_str(), "batch");
assert_eq!(Regime::from_str_opt("single"), Some(Regime::SingleDocument));
assert_eq!(Regime::from_str_opt("batch"), Some(Regime::Batch));
assert_eq!(Regime::from_str_opt("Batch"), None);
assert_eq!(Regime::from_str_opt(""), None);
assert_eq!(Regime::from_str_opt("stream"), None);
}
#[test]
fn default_sweep_straddles_the_crossover_regime() {
assert_eq!(DEFAULT_TARGET_TOKENS.first(), Some(&256));
assert!(
DEFAULT_TARGET_TOKENS.last().copied().unwrap_or(0) >= 131_072,
"sweep must reach 128K+ tokens"
);
for w in DEFAULT_TARGET_TOKENS.windows(2) {
assert!(w[1] > w[0], "sweep must be ascending: {w:?}");
}
for t in DEFAULT_TARGET_TOKENS {
assert!(t.is_power_of_two(), "{t} is not a power of two");
}
}
#[test]
fn corpus_from_tree_is_deterministic_and_respects_the_budget() {
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
let a = Corpus::from_tree(root, 4096).expect("crate tree has .rs and .toml text");
let b = Corpus::from_tree(root, 4096).expect("second build");
assert_eq!(a.text, b.text);
assert_eq!(a.facts.digest, b.facts.digest);
assert_eq!(a.facts.bytes, a.text.len());
assert!(a.facts.bytes <= 4096, "budget exceeded: {}", a.facts.bytes);
assert!(a.facts.bytes > 0);
assert!(a.facts.files >= 1);
let big = Corpus::from_tree(root, 65_536).expect("larger budget");
assert!(big.facts.bytes > a.facts.bytes);
assert!(big.text.starts_with(&a.text));
}
#[test]
fn corpus_rejects_empty_sources() {
assert!(Corpus::from_tree(Path::new("/nonexistent-exatok-corpus-root"), 4096).is_err());
assert!(Corpus::from_file(Path::new("/nonexistent-exatok-corpus-file"), 4096).is_err());
let root = Path::new(env!("CARGO_MANIFEST_DIR"));
assert!(Corpus::from_tree(root, 0).is_err(), "a zero-byte budget is not a corpus");
}
#[test]
fn repo_root_is_found_by_walking_up_to_the_git_directory() {
let here = Path::new(env!("CARGO_MANIFEST_DIR"));
let root = find_repo_root(here).expect("crate lives inside a git checkout");
assert!(root.join(".git").exists());
assert!(here.starts_with(&root));
assert_eq!(
find_repo_root(&root).as_deref(),
Some(root.as_path()),
"the walk must be outermost-idempotent: re-querying from {} moved the root",
root.display()
);
assert_eq!(find_repo_root(Path::new("/")), None);
}
#[test]
fn sweep_config_defaults_point_at_driver_and_durable_vocab() {
let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let root = find_repo_root(crate_dir).expect("git checkout");
let cfg = SweepConfig::defaults(crate_dir, &root);
assert!(cfg.driver.ends_with("benches/baselines/host_baselines.py"));
assert_eq!(cfg.vocab, "cl100k_base");
assert!(cfg.tiktoken_ranks.ends_with(".cache/exatok-vocab/cl100k_base.tiktoken"));
assert_eq!(cfg.hf_repo, "Xenova/gpt-4");
assert_eq!(cfg.batch_budget_bytes, 32 * 1024 * 1024);
assert_eq!(cfg.target_tokens, DEFAULT_TARGET_TOKENS.to_vec());
assert!(cfg.reps >= 3, "fewer than three reps cannot yield a spread");
assert!(cfg.warmups >= 1, "an unwarmed first call measures page faults");
assert_eq!(
cfg.corpus_scratch.file_name().and_then(|n| n.to_str()),
Some("bench-corpus.txt")
);
assert!(
!cfg.corpus_scratch.starts_with(crate_dir),
"the corpus must not land inside the source tree, but it is at {}",
cfg.corpus_scratch.display()
);
assert_eq!(
cfg.python,
home_dir().join(BENCH_VENV_REL).join("bin/python"),
"the interpreter must be the durable off-tree venv, with no system fallback"
);
assert!(
!cfg.python.starts_with(scratch_root()),
"a baseline venv under the cargo target directory gets cleaned away, but it is at {}",
cfg.python.display()
);
assert_eq!(
cfg.report_dir,
Some(home_dir().join(BENCH_REPORT_REL)),
"every sweep must persist a report by default"
);
assert!(cfg.require_baselines, "a wiped venv must fail loudly, not yield empty rows");
}
#[test]
fn the_scratch_corpus_never_lands_inside_the_source_tree() {
let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let root = scratch_root();
assert!(
!root.starts_with(crate_dir),
"scratch_root resolved into the crate at {}",
root.display()
);
assert!(root.is_absolute(), "scratch root {} is not absolute", root.display());
}
#[test]
fn scratch_root_resolves_to_the_cargo_target_directory() {
let exe = std::env::current_exe().expect("test binary path");
let profile = exe
.ancestors()
.find(|d| matches!(d.file_name().and_then(|n| n.to_str()), Some("debug" | "release")))
.expect("test binary must sit under a profile directory");
assert_eq!(scratch_root(), profile.parent().expect("target dir"));
}
#[test]
fn a_driver_killed_by_a_signal_is_reported_by_its_signal_number() {
for signal in [9_i32, 7, 11, 15] {
let status = std::process::ExitStatus::from_raw(signal);
let text = describe_termination(&status);
assert_eq!(
text,
format!("no exit code, killed by signal {signal}"),
"signal {signal} was not named"
);
assert!(!text.contains("None"), "signal {signal} rendered as None: {text}");
}
}
#[test]
fn a_driver_that_exits_non_zero_is_reported_by_its_exit_code() {
for code in [1_i32, 2, 101] {
let status = std::process::ExitStatus::from_raw(code << 8);
assert_eq!(describe_termination(&status), format!("exit code {code}"));
}
}
#[test]
fn empty_stderr_is_labelled_rather_than_printed_as_nothing() {
let described = describe_stderr(b"");
assert!(described.contains("<empty"), "empty stderr rendered as {described:?}");
assert!(described.contains("reported nothing"));
assert_eq!(describe_stderr(b" \n \t "), described, "whitespace is not a report");
assert_eq!(describe_stderr(b" boom: bad rank "), "boom: bad rank");
}
#[test]
fn a_sigkilled_driver_points_at_the_watchdog_logs() {
let hint = kill_hint(&std::process::ExitStatus::from_raw(9));
assert!(hint.contains("/tmp/cpu-watchdog.log"), "hint lost the log path: {hint}");
assert!(hint.contains("external reaper"), "hint does not name a reaper: {hint}");
assert!(hint.contains("allowlist"), "hint does not say how to fix it: {hint}");
}
#[test]
fn only_the_relevant_signal_earns_advice() {
let bus = kill_hint(&std::process::ExitStatus::from_raw(7));
assert!(bus.contains("off-tree"), "SIGBUS hint lost the corpus advice: {bus}");
assert!(!bus.contains("cpu-watchdog"), "SIGBUS must not blame the reaper: {bus}");
for code in [1_i32, 101] {
let status = std::process::ExitStatus::from_raw(code << 8);
assert_eq!(kill_hint(&status), "", "exit code {code} must earn no signal advice");
}
assert_eq!(kill_hint(&std::process::ExitStatus::from_raw(11)), "");
}
#[test]
fn utc_stamp_matches_the_calendar_at_epoch_and_leap_boundaries() {
let at = |secs: u64| {
utc_stamp(std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(secs))
};
assert_eq!(at(0), "19700101T000000Z");
assert_eq!(at(1), "19700101T000001Z");
assert_eq!(at(59), "19700101T000059Z");
assert_eq!(at(60), "19700101T000100Z");
assert_eq!(at(86_399), "19700101T235959Z");
assert_eq!(at(86_400), "19700102T000000Z");
assert_eq!(at(951_782_400), "20000229T000000Z");
assert_eq!(at(1_709_164_800), "20240229T000000Z");
assert_eq!(at(1_751_328_000), "20250701T000000Z");
assert_eq!(at(4_107_456_000), "21000228T000000Z");
assert_eq!(at(4_107_542_400), "21000301T000000Z");
assert_eq!(at(253_402_300_799), "99991231T235959Z");
}
#[test]
fn utc_stamp_is_fixed_width_and_sorts_as_text_in_time_order() {
let at = |secs: u64| {
utc_stamp(std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(secs))
};
let early = at(951_782_400);
let late = at(1_751_328_000);
assert_eq!(early.len(), 16, "stamp {early} is not the fixed 16 characters");
assert_eq!(late.len(), 16);
assert!(early < late, "{early} must sort before {late}");
assert_eq!(at(0), "19700101T000000Z");
}
#[test]
fn region_clock_drift_is_signed_and_reads_zero_as_no_claim() {
let fell = RegionClock { start_mhz: 5000.0, end_mhz: 4600.0, busy_cpus: 16 };
assert!((fell.drift_ratio() - -0.08).abs() < 1e-12);
assert!(fell.is_significant());
let rose = RegionClock { start_mhz: 4600.0, end_mhz: 5000.0, busy_cpus: 16 };
assert!((rose.drift_ratio() - 0.086_956_521_739_130_43).abs() < 1e-12);
assert!(rose.is_significant(), "a boost mid-region biases throughput too");
let steady = RegionClock { start_mhz: 5000.0, end_mhz: 4990.0, busy_cpus: 32 };
assert!((steady.drift_ratio() - -0.002).abs() < 1e-12);
assert!(!steady.is_significant());
let absent = RegionClock { start_mhz: 0.0, end_mhz: 0.0, busy_cpus: 0 };
assert!((absent.drift_ratio() - 0.0).abs() < f64::EPSILON);
assert!(!absent.is_significant());
}
#[test]
fn a_record_without_busy_clock_fields_parses_as_no_reading() {
let ctx =
MachineContext { physical_cores: 16, logical_cores: 32, pinned_to_physical: true };
let line = "record\ttokenizer=tiktoken\tversion=0.12.0\tvocab=cl100k_base\t\
regime=single\ttarget_tokens=1024\tdoc_bytes=4096\tdocs=1\t\
input_bytes=4096\ttokens=1024\tsamples=0.25\twarmups=1\tthreads=1\t\
mean_mhz=5100.5\tmatches_ref=1\tcpu_secs=0.25\tinner_reps=1\t\
foreign_cpu_secs=0.0";
let ev = parse_driver_line(line, ctx, 1).unwrap().expect("a record");
let DriverEvent::Record(m) = ev else { panic!("expected a record event") };
assert_eq!(m.region_clock, None, "an absent probe must read as None, not as 0 MHz");
assert!((m.mean_mhz - 5100.5).abs() < 1e-9);
assert_eq!(m.tokens, 1024);
}
#[test]
fn a_record_with_busy_clock_fields_parses_the_exact_values() {
let ctx =
MachineContext { physical_cores: 16, logical_cores: 32, pinned_to_physical: true };
let line = "record\ttokenizer=tiktoken\tversion=0.12.0\tvocab=cl100k_base\t\
regime=single\ttarget_tokens=1024\tdoc_bytes=4096\tdocs=1\t\
input_bytes=4096\ttokens=1024\tsamples=0.25\twarmups=1\tthreads=1\t\
mean_mhz=3274.4\tbusy_mhz_start=4618.6\tbusy_mhz_end=4568.0\tbusy_cpus=22\t\
matches_ref=1\tcpu_secs=0.25\tinner_reps=1\tforeign_cpu_secs=0.0";
let ev = parse_driver_line(line, ctx, 1).unwrap().expect("a record");
let DriverEvent::Record(m) = ev else { panic!("expected a record event") };
let clock = m.region_clock.expect("the record carried all three fields");
assert!((clock.start_mhz - 4618.6).abs() < 1e-9, "start was {}", clock.start_mhz);
assert!((clock.end_mhz - 4568.0).abs() < 1e-9, "end was {}", clock.end_mhz);
assert_eq!(clock.busy_cpus, 22);
assert!(clock.drift_ratio() < 0.0, "a fall must read negative");
assert!((clock.drift_ratio() - -0.010_955_700_861_733_07).abs() < 1e-12);
assert!(!clock.is_significant());
assert!((m.mean_mhz - 3274.4).abs() < 1e-9);
}
#[test]
fn a_record_with_a_malformed_busy_clock_is_rejected() {
let ctx =
MachineContext { physical_cores: 16, logical_cores: 32, pinned_to_physical: true };
let base = "record\ttokenizer=tiktoken\tversion=0.12.0\tvocab=cl100k_base\t\
regime=single\ttarget_tokens=1024\tdoc_bytes=4096\tdocs=1\t\
input_bytes=4096\ttokens=1024\tsamples=0.25\twarmups=1\tthreads=1\t\
mean_mhz=5000\tmatches_ref=1\tcpu_secs=0.25\tinner_reps=1\t\
foreign_cpu_secs=0.0";
for bad in [
"\tbusy_mhz_start=fast\tbusy_mhz_end=4568.0\tbusy_cpus=22",
"\tbusy_mhz_start=4618.6\tbusy_mhz_end=slow\tbusy_cpus=22",
"\tbusy_mhz_start=4618.6\tbusy_mhz_end=4568.0\tbusy_cpus=lots",
] {
let line = format!("{base}{bad}");
assert!(
parse_driver_line(&line, ctx, 1).is_err(),
"a malformed busy clock must be rejected, but this parsed: {bad}"
);
}
}
#[test]
fn preflight_names_the_expected_venv_and_the_command_that_rebuilds_it() {
let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let root = find_repo_root(crate_dir).expect("git checkout");
let mut cfg = SweepConfig::defaults(crate_dir, &root);
cfg.python = PathBuf::from("/nonexistent/exatok-bench-venv/bin/python");
let err = preflight_baselines(&cfg).expect_err("a missing interpreter must refuse");
let text = err.to_string();
assert!(
text.contains("/nonexistent/exatok-bench-venv/bin/python"),
"the refusal must name the interpreter it looked for, but said: {text}"
);
assert!(
text.contains(BENCH_VENV_REL),
"the refusal must name the durable venv location, but said: {text}"
);
assert!(
text.contains("python3 -m venv"),
"the refusal must carry the command that rebuilds it, but said: {text}"
);
for module in BASELINE_MODULES {
assert!(
text.contains(module),
"the rebuild command must install {module}, but said: {text}"
);
}
}
#[test]
fn the_durable_bench_paths_are_not_under_build_output() {
for rel in [BENCH_VENV_REL, BENCH_REPORT_REL] {
let path = home_dir().join(rel);
assert!(
!path.starts_with(scratch_root()),
"{} is under the cargo target directory and will be cleaned away",
path.display()
);
assert!(!rel.contains("target"), "{rel} names a build-output directory");
assert!(path.is_absolute(), "{} must be absolute", path.display());
}
assert_ne!(BENCH_VENV_REL, BENCH_REPORT_REL);
}
#[test]
fn persist_writes_a_timestamped_report_carrying_the_diagnostic_fields() {
let report = report_with(vec![Availability::Measured(measurement(20.0, 1, 4096))]);
let dir = std::env::temp_dir().join(format!(
"exatok-persist-{}-{}",
std::process::id(),
utc_stamp(std::time::SystemTime::now())
));
let path = report.persist(&dir).expect("a writable temp directory");
let name = path.file_name().and_then(|n| n.to_str()).expect("a filename");
assert!(name.starts_with("host-baseline-"), "unexpected report name {name}");
assert!(name.ends_with("Z.json"), "report name {name} is not UTC-stamped");
assert_eq!(name.len(), "host-baseline-".len() + 16 + ".json".len());
let text = std::fs::read_to_string(&path).expect("the report we just wrote");
for key in [
"\"foreign_load_ratio\"",
"\"achieved_parallelism\"",
"\"best_secs\"",
"\"region_clock\"",
] {
assert!(text.contains(key), "the report must carry {key}, but it did not");
}
assert!(text.contains("\"status\": \"measured\""));
assert!(text.trim_end().ends_with('}'), "the report must be closed JSON");
std::fs::remove_file(&path).expect("cleanup");
std::fs::remove_dir(&dir).expect("cleanup");
}
#[test]
fn an_absent_region_clock_serializes_as_null_not_as_zero() {
let mut m = measurement(20.0, 1, 4096);
m.region_clock = None;
let json = report_with(vec![Availability::Measured(m)]).to_json();
assert!(
json.contains("\"region_clock\": null"),
"an unmeasured clock must serialize as null, but the report said: {json}"
);
let present =
report_with(vec![Availability::Measured(measurement(20.0, 1, 4096))]).to_json();
assert!(
present.contains("\"busy_cpus\": 16"),
"a measured clock must serialize its busy CPU count, but said: {present}"
);
assert!(present.contains("\"significant\": false"));
}
fn paired_with(speedups: Vec<f64>) -> Paired {
Paired {
label_a: "old".to_owned(),
label_b: "new".to_owned(),
a_unit_secs: speedups.iter().map(|_| 1.0).collect(),
b_unit_secs: speedups.iter().map(|s| 1.0 / s).collect(),
speedups,
voided_rounds: 0,
foreign_resident_baseline_mib: 24_204,
foreign_sm_percent_max: 41,
host_foreign_load: Vec::new(),
host_foreign_load_arms: Vec::new(),
unit_work: None,
}
}
#[test]
fn a_paired_comparison_refuses_host_load_that_moved_while_it_ran() {
let mut drifting = paired_with(vec![1.20; MIN_PAIRED_ROUNDS as usize]);
assert!((drifting.speedup_rel_stddev() - 0.0).abs() < 1e-12);
drifting.host_foreign_load = vec![0.1, 0.2, 0.4, 1.0, 2.0, 4.0, 6.0, 8.0];
let drift = drifting.host_load_rel_stddev().expect("eight readings have a spread");
assert!(
drift > MAX_PAIRED_HOST_LOAD_DRIFT,
"a 0.1 to 8.0 core ramp must exceed the {MAX_PAIRED_HOST_LOAD_DRIFT} drift gate, \
got {drift}"
);
assert!(
!drifting.is_publishable(),
"zero delta spread must NOT publish when the contaminant moved: that is the \
precise combination that published a wrong cjk sign"
);
let text = drifting.render();
assert!(text.contains("NOT publishable"), "{text}");
assert!(text.contains("host-load"), "the refusal must name host load as the cause: {text}");
let mut loaded = paired_with(vec![1.20; MIN_PAIRED_ROUNDS as usize]);
loaded.host_foreign_load = vec![9.8; MIN_PAIRED_ROUNDS as usize];
assert!(
(loaded.host_load_rel_stddev().expect("eight readings") - 0.0).abs() < 1e-12,
"identical readings are perfectly stationary"
);
assert!(
(loaded.median_host_load().expect("eight readings") - 9.8).abs() < 1e-12,
"the level must be reported even though it is not gated"
);
assert!(
loaded.is_publishable(),
"9.8 cores of EVEN foreign load still yields a valid ratio; gating on level \
would refuse the work this estimator exists to do"
);
assert!(loaded.render().contains("9.80 cores foreign"), "{}", loaded.render());
}
#[test]
fn an_unrecorded_host_load_renders_as_absent_not_as_idle() {
let eight = vec![1.0; MIN_PAIRED_ROUNDS as usize];
let half = vec![0.5; MIN_PAIRED_ROUNDS as usize];
let cross = Paired::from_unit_secs("old", "new", &eight, &half, 0, 0)
.expect("eight clean rounds per arm");
assert!(cross.host_foreign_load.is_empty(), "a cross-process pairing probes no region");
assert_eq!(cross.median_host_load(), None, "absent must not read as a level");
assert_eq!(cross.host_load_rel_stddev(), None, "absent must not read as stationary");
let text = cross.render();
assert!(text.contains("NOT RECORDED"), "absence must be disclosed: {text}");
assert!(
!text.contains("0.00 cores foreign"),
"an unmeasured machine must never render as an idle one: {text}"
);
assert!(cross.is_publishable(), "a valid cross-process delta stays publishable");
let mut idle = paired_with(vec![1.20; MIN_PAIRED_ROUNDS as usize]);
idle.host_foreign_load = vec![0.0; MIN_PAIRED_ROUNDS as usize];
assert_eq!(idle.median_host_load(), Some(0.0), "idle is measured, not absent");
assert_eq!(
idle.host_load_rel_stddev(),
Some(0.0),
"a near-zero mean makes relative spread undefined, not infinite, and idle is \
maximally stationary"
);
assert!(idle.render().contains("0.00 cores foreign"), "{}", idle.render());
}
#[test]
fn the_two_host_load_series_are_absent_together() {
let eight = vec![1.0; MIN_PAIRED_ROUNDS as usize];
let half = vec![0.5; MIN_PAIRED_ROUNDS as usize];
let cross = Paired::from_unit_secs("old", "new", &eight, &half, 0, 0)
.expect("eight clean rounds per arm");
assert!(cross.host_foreign_load.is_empty(), "no region probe, so no gated reading");
assert_eq!(
cross.host_foreign_load_arms.len(),
cross.host_foreign_load.len(),
"the per-arm series must be absent exactly when the gated one is, got {} against {}",
cross.host_foreign_load_arms.len(),
cross.host_foreign_load.len()
);
assert_eq!(cross.median_host_load(), None, "absent must not read as a level");
}
#[test]
fn the_paired_median_is_the_middle_round_not_the_luckiest() {
let odd = paired_with(vec![1.10, 0.95, 3.00]);
assert!((odd.median_speedup() - 1.10).abs() < 1e-12);
assert!(
odd.median_speedup() < 3.00,
"the median must not be the best round, which was 3.00x"
);
let even = paired_with(vec![1.0, 2.0, 3.0, 4.0]);
assert!((even.median_speedup() - 2.5).abs() < 1e-12);
let (lo, hi) = even.speedup_range();
assert!((lo - 1.0).abs() < 1e-12);
assert!((hi - 4.0).abs() < 1e-12);
assert!((paired_with(Vec::new()).median_speedup() - 0.0).abs() < f64::EPSILON);
assert_eq!(paired_with(Vec::new()).speedup_range(), (0.0, 0.0));
}
#[test]
fn the_paired_spread_gates_publication_like_an_absolute_cell() {
let tight = paired_with(vec![1.20; 8]);
assert!((tight.speedup_rel_stddev() - 0.0).abs() < 1e-12);
assert!(tight.is_publishable());
let short = paired_with(vec![1.20; 7]);
assert_eq!(short.speedups.len(), MIN_PAIRED_ROUNDS as usize - 1);
assert!(!short.is_publishable(), "seven rounds cannot establish a spread");
let mut wide = vec![1.0; 7];
wide.push(3.0);
let wide = paired_with(wide);
assert!((wide.speedup_rel_stddev() - 0.565_685_424_949_238).abs() < 1e-12);
assert!(wide.speedup_rel_stddev() > MAX_PUBLISHABLE_REL_STDDEV);
assert!(!wide.is_publishable(), "a 0.57 delta spread is uncontrolled, not a small win");
assert!((paired_with(vec![1.2]).speedup_rel_stddev() - 0.0).abs() < f64::EPSILON);
assert!(!paired_with(vec![1.2]).is_publishable());
}
#[test]
fn the_paired_render_carries_absolutes_and_the_contaminant() {
let text = paired_with(vec![1.20; 8]).render();
assert!(text.contains("best per unit"), "no absolute figures in: {text}");
assert!(text.contains("rel stddev"), "no spread in: {text}");
assert!(text.contains("24204 MiB"), "no foreign residency in: {text}");
assert!(text.contains("41% SM"), "no foreign SM evidence in: {text}");
assert!(text.contains("evidence, not gated"), "sm% must be labelled ungated: {text}");
assert!(text.contains("publishable"));
let bad = paired_with(vec![1.0, 3.0]).render();
assert!(bad.contains("NOT publishable"), "a two-round comparison must refuse: {bad}");
}
#[test]
fn the_paired_render_names_the_faster_arm_and_never_inverts_it() {
let quick_b = Paired::from_unit_secs("host", "device", &[1.0; 8], &[0.5; 8], 0, 0)
.expect("eight clean rounds");
assert!((quick_b.median_speedup() - 2.0).abs() < 1e-12);
let text = quick_b.render();
assert!(
text.contains("faster arm : device by 2.0000x"),
"arm B is quicker at 0.5 s against 1.0 s, so it must be named: {text}"
);
assert!(
text.contains("(ratio is host_secs/device_secs)"),
"the ratio's orientation must be stated so it cannot be read backwards: {text}"
);
assert!(!text.contains("faster arm : host"), "naming the slow arm is the bug: {text}");
let quick_a = Paired::from_unit_secs("host", "device", &[0.25; 8], &[1.0; 8], 0, 0)
.expect("eight clean rounds");
assert!((quick_a.median_speedup() - 0.25).abs() < 1e-12);
let mirrored = quick_a.render();
assert!(
mirrored.contains("faster arm : host by 4.0000x"),
"0.25x means the HOST is 4x quicker, stated as 4x not 0.25x: {mirrored}"
);
assert!(!mirrored.contains("faster arm : device"), "inverted: {mirrored}");
let short = Paired::from_unit_secs(
"host",
"device",
&[0.003_239_845; 8],
&[0.008_483_396; 8],
24_204,
0,
)
.expect("eight clean rounds");
assert!(
short.render().contains("faster arm : host by 2.6185x"),
"the observed short_pretokens cell is a host win: {}",
short.render()
);
let tie = Paired::from_unit_secs("host", "device", &[1.0; 8], &[1.0; 8], 0, 0)
.expect("eight clean rounds");
assert!(
tie.render().contains("faster arm : device by 1.0000x"),
"a 1.0000x tie must report unit factor, never a reciprocal blowup: {}",
tie.render()
);
}
#[test]
fn the_device_lock_excludes_a_second_holder_and_releases_on_drop() {
let path = std::env::temp_dir().join(format!("exatok-lock-{}.lock", std::process::id()));
let _ = fs::remove_file(&path);
let held = DeviceLock::acquire_at(path.clone(), std::time::Duration::from_secs(5))
.expect("an uncontended lock");
assert!(path.is_file(), "the lock file must exist while held");
let refused = DeviceLock::acquire_at(path.clone(), std::time::Duration::from_millis(0))
.expect_err("a second holder must be refused");
let text = refused.to_string();
assert!(
text.contains(&std::process::id().to_string()),
"the refusal must name the holding pid, but said: {text}"
);
drop(held);
assert!(!path.exists(), "the lock must be released on drop, not leaked");
let again = DeviceLock::acquire_at(path.clone(), std::time::Duration::from_secs(5))
.expect("the lock is free again");
drop(again);
assert!(!path.exists());
}
#[test]
fn a_dead_holders_lock_is_reclaimed_rather_than_wedging_the_card() {
let path = std::env::temp_dir().join(format!("exatok-stale-{}.lock", std::process::id()));
fs::write(&path, "4294967000").expect("write a stale lock");
let taken = DeviceLock::acquire_at(path.clone(), std::time::Duration::from_millis(0))
.expect("a stale lock must be reclaimed immediately, not waited out");
assert_eq!(
fs::read_to_string(&path).expect("the reclaimed lock").trim(),
std::process::id().to_string(),
"the reclaiming process must record itself as the new holder"
);
drop(taken);
fs::write(&path, std::process::id().to_string()).expect("write a live lock");
assert!(
DeviceLock::acquire_at(path.clone(), std::time::Duration::from_millis(0)).is_err(),
"a live holder's lock must not be reclaimed"
);
fs::write(&path, "not-a-pid").expect("write a corrupt lock");
assert!(
DeviceLock::acquire_at(path.clone(), std::time::Duration::from_millis(0)).is_err(),
"an unreadable holder must not be assumed dead"
);
let _ = fs::remove_file(&path);
}
#[test]
fn the_proc_probes_report_this_machine_not_zero() {
let machine = Machine::probe().expect("a Linux machine");
let mut acc = 0u64;
for i in 0..2_000_000u64 {
acc = acc.wrapping_add(i).wrapping_mul(31);
}
assert_ne!(acc, 0);
let own = process_cpu_secs().expect("/proc/self/stat is readable");
assert!(own > 0.0, "this process has burned CPU, but the probe read {own}");
let busy = machine_busy_secs().expect("/proc/stat is readable");
assert!(
busy >= own,
"machine-wide busy {busy} cannot be below this process's own {own}"
);
let per_cpu = per_cpu_busy_secs();
assert_eq!(
per_cpu.len(),
machine.logical_cores,
"one /proc/stat row per logical CPU is required for the weighted clock"
);
let mhz = per_cpu_mhz();
assert_eq!(mhz.len(), machine.logical_cores, "one MHz reading per logical CPU");
assert!(mhz.iter().all(|m| *m > 0.0), "a zero clock reading is a parse failure");
assert!(per_cpu.iter().sum::<f64>() > 0.0);
}
#[test]
fn the_weighted_clock_ignores_cores_that_did_no_work() {
let mhz = [5000.0, 1000.0, 1000.0, 1000.0];
let weights = [1.0, 0.0, 0.0, 0.0];
assert!((busy_weighted_mhz(&mhz, &weights) - 5000.0).abs() < 1e-12);
assert!((mhz.iter().sum::<f64>() / 4.0 - 2000.0).abs() < 1e-12);
let two = busy_weighted_mhz(&[3000.0, 6000.0], &[2.0, 1.0]);
assert!((two - 4000.0).abs() < 1e-12, "expected 4000 MHz, got {two}");
assert!((busy_weighted_mhz(&mhz, &[0.0; 4]) - 0.0).abs() < f64::EPSILON);
assert!((busy_weighted_mhz(&[], &[]) - 0.0).abs() < f64::EPSILON);
assert!((busy_weighted_mhz(&[5000.0, 1000.0], &[1.0]) - 5000.0).abs() < 1e-12);
}
#[test]
fn compare_cells_refuses_fewer_rounds_than_the_spread_needs() {
let mut a = || Ok(WorkUnit { bytes: 1, tokens: 1 });
let mut b = || Ok(WorkUnit { bytes: 1, tokens: 1 });
let err = compare_cells(
Arm { label: "a", work: &mut a },
Arm { label: "b", work: &mut b },
MIN_PAIRED_ROUNDS - 1,
)
.expect_err("seven rounds must be refused before any work runs");
let text = err.to_string();
assert!(text.contains(&(MIN_PAIRED_ROUNDS - 1).to_string()), "{text}");
assert!(text.contains(&MIN_PAIRED_ROUNDS.to_string()), "{text}");
}
#[test]
fn compare_cells_refuses_an_arm_that_consumed_no_input() {
fn burn(rounds: u64) -> u64 {
let mut acc = 1u64;
for i in 0..rounds {
acc = acc.wrapping_mul(31).wrapping_add(i);
}
acc
}
let mut real = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let mut vacuous = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 0, tokens: v % 7 + 1 })
};
let err = compare_cells_at(
Arm { label: "real", work: &mut real },
Arm { label: "vacuous", work: &mut vacuous },
MIN_PAIRED_ROUNDS,
TEST_REGION_SECS,
)
.expect_err("an arm that consumed no input must be refused, not timed");
let text = err.to_string();
assert!(text.contains("consumed no input"), "{text}");
assert!(text.contains("200000 B against 0 B"), "{text}");
}
#[test]
fn compare_cells_refuses_work_that_changes_between_rounds() {
fn burn(rounds: u64) -> u64 {
let mut acc = 1u64;
for i in 0..rounds {
acc = acc.wrapping_mul(31).wrapping_add(i);
}
acc
}
let mut calls = 0u64;
let mut drifting = move || {
calls += 1;
std::hint::black_box(burn(200_000));
Ok(WorkUnit { bytes: 200_000, tokens: calls })
};
let mut steady = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let err = compare_cells_at(
Arm { label: "drifting", work: &mut drifting },
Arm { label: "steady", work: &mut steady },
MIN_PAIRED_ROUNDS,
TEST_REGION_SECS,
)
.expect_err("work that changes between rounds must be refused");
let text = err.to_string();
assert!(text.contains("the work changed between rounds"), "{text}");
assert!(text.contains("pinned at"), "{text}");
}
#[test]
fn compare_cells_reads_identical_work_as_no_change() {
fn burn(rounds: u64) -> u64 {
let mut acc = 1u64;
for i in 0..rounds {
acc = acc.wrapping_mul(31).wrapping_add(i);
}
acc
}
let mut a = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let mut b = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let paired = compare_cells_at(
Arm { label: "self-a", work: &mut a },
Arm { label: "self-b", work: &mut b },
MIN_PAIRED_ROUNDS,
TEST_REGION_SECS,
)
.expect("a paired comparison of identical work");
assert_eq!(paired.speedups.len(), MIN_PAIRED_ROUNDS as usize);
assert_eq!(paired.a_unit_secs.len(), MIN_PAIRED_ROUNDS as usize);
assert_eq!(paired.b_unit_secs.len(), MIN_PAIRED_ROUNDS as usize);
assert!(paired.a_unit_secs.iter().all(|s| *s > 0.0 && s.is_finite()));
let median = paired.median_speedup();
assert!(
(median - 1.0).abs() < 0.25,
"identical work must read near 1.0x, got {median}x; calibration or per-unit \
normalization is wrong"
);
}
#[test]
fn compare_cells_detects_a_real_two_fold_difference() {
fn burn(rounds: u64) -> u64 {
let mut acc = 1u64;
for i in 0..rounds {
acc = acc.wrapping_mul(31).wrapping_add(i);
}
acc
}
let mut slow = || {
let v = burn(400_000);
Ok(WorkUnit { bytes: 400_000, tokens: v % 7 + 1 })
};
let mut fast = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let paired = compare_cells_at(
Arm { label: "slow", work: &mut slow },
Arm { label: "fast", work: &mut fast },
MIN_PAIRED_ROUNDS,
TEST_REGION_SECS,
)
.expect("a paired comparison of unequal work");
let median = paired.median_speedup();
assert!(
(median - 2.0).abs() < 0.4,
"half the work must read near 2.0x, got {median}x; the harness is not \
distinguishing its arms"
);
let a_best = paired.a_unit_secs.iter().copied().fold(f64::INFINITY, f64::min);
let b_best = paired.b_unit_secs.iter().copied().fold(f64::INFINITY, f64::min);
assert!(a_best > b_best, "arm A does twice the work, so {a_best} must exceed {b_best}");
println!("{}", paired.render());
}
#[test]
fn the_paired_loop_keeps_both_arms_behind_the_gated_max() {
fn burn(rounds: u64) -> u64 {
let mut acc = 1u64;
for i in 0..rounds {
acc = acc.wrapping_mul(31).wrapping_add(i);
}
acc
}
let mut a = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let mut b = || {
let v = burn(200_000);
Ok(WorkUnit { bytes: 200_000, tokens: v % 7 + 1 })
};
let paired = compare_cells_at(
Arm { label: "arm-a", work: &mut a },
Arm { label: "arm-b", work: &mut b },
MIN_PAIRED_ROUNDS,
TEST_REGION_SECS,
)
.expect("a paired comparison of equal work");
assert_eq!(
paired.host_foreign_load.len(),
MIN_PAIRED_ROUNDS as usize,
"one gated reading per round"
);
assert_eq!(
paired.host_foreign_load_arms.len(),
paired.host_foreign_load.len(),
"the per-arm series must not fall behind the series it explains"
);
for (round, ((load_a, load_b), gated)) in
paired.host_foreign_load_arms.iter().zip(&paired.host_foreign_load).enumerate()
{
assert!(
(load_a.max(*load_b) - gated).abs() < 1e-12,
"round {round}: gated {gated} must be the max of {load_a} and {load_b}"
);
assert!(
*load_a >= 0.0 && *load_b >= 0.0,
"round {round}: a foreign load is never negative, got {load_a} and {load_b}"
);
}
}
}
#[cfg(test)]
mod provenance_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::{
MIN_PAIRED_ROUNDS, Paired, Provenance, TestCount, compiled_features,
};
#[test]
fn a_test_count_is_not_comparable_across_feature_sets() {
let default = TestCount { passed: 255, ignored: 2, features: Vec::new() };
let confluence =
TestCount { passed: 286, ignored: 2, features: vec!["confluence".to_owned()] };
assert!(
!default.comparable_to(&confluence),
"255 default and 286 confluence describe different builds and must never compare"
);
assert!(default.comparable_to(&TestCount {
passed: 261,
ignored: 2,
features: Vec::new()
}));
assert_eq!(default.render(), "255 passed / 2 ignored (features: default)");
assert_eq!(confluence.render(), "286 passed / 2 ignored (features: confluence)");
let both = TestCount {
passed: 1,
ignored: 0,
features: vec!["confluence".to_owned(), "cuda".to_owned()],
};
assert_eq!(both.render(), "1 passed / 0 ignored (features: confluence,cuda)");
assert!(
!both.comparable_to(&confluence),
"a superset of features is still a different build"
);
}
#[test]
fn only_a_probed_figure_may_be_quoted_as_measured() {
assert!(Provenance::Probed.is_quotable_as_measured());
assert!(!Provenance::Derived.is_quotable_as_measured());
assert!(!Provenance::Fixture.is_quotable_as_measured());
assert_eq!(Provenance::Probed.as_str(), "probed");
assert_eq!(Provenance::Derived.as_str(), "derived");
assert_eq!(Provenance::Fixture.as_str(), "fixture");
}
#[test]
fn the_feature_set_is_probed_from_this_binary() {
let features = compiled_features();
assert_eq!(features.contains(&"confluence"), cfg!(feature = "confluence"));
assert_eq!(features.contains(&"cuda"), cfg!(feature = "cuda"));
assert_eq!(features.contains(&"gpu-pretokenize"), cfg!(feature = "gpu-pretokenize"));
assert_eq!(features.contains(&"multi-merge"), cfg!(feature = "multi-merge"));
let mut sorted = features.clone();
sorted.sort_unstable();
sorted.dedup();
assert_eq!(sorted, features, "the feature list must be stable and duplicate-free");
assert_eq!(TestCount::here(1, 0).features.len(), features.len());
}
#[test]
fn cross_process_pairing_enforces_the_same_gates_as_the_in_process_path() {
let eight = vec![1.0; MIN_PAIRED_ROUNDS as usize];
let half = vec![0.5; MIN_PAIRED_ROUNDS as usize];
let ok = Paired::from_unit_secs("old", "new", &eight, &half, 24_204, 41)
.expect("eight clean rounds per arm");
assert!((ok.median_speedup() - 2.0).abs() < 1e-12);
assert!(ok.is_publishable());
assert_eq!(ok.foreign_resident_baseline_mib, 24_204);
assert_eq!(ok.foreign_sm_percent_max, 41);
assert!(ok.render().contains("24204 MiB"));
let short = vec![1.0; MIN_PAIRED_ROUNDS as usize - 1];
let err = Paired::from_unit_secs("old", "new", &short, &short, 0, 0)
.expect_err("seven rounds must be refused");
assert!(err.to_string().contains(&MIN_PAIRED_ROUNDS.to_string()));
assert!(
Paired::from_unit_secs("old", "new", &eight, &short, 0, 0).is_err(),
"8 A rounds against 7 B rounds is not a pairing"
);
let mut zeroed = eight.clone();
zeroed[3] = 0.0;
assert!(Paired::from_unit_secs("old", "new", &eight, &zeroed, 0, 0).is_err());
let mut nan = eight.clone();
nan[0] = f64::NAN;
assert!(Paired::from_unit_secs("old", "new", &nan, &eight, 0, 0).is_err());
let mut negative = eight.clone();
negative[7] = -1.0;
assert!(Paired::from_unit_secs("old", "new", &negative, &eight, 0, 0).is_err());
let mut inf = eight.clone();
inf[2] = f64::INFINITY;
assert!(Paired::from_unit_secs("old", "new", &inf, &eight, 0, 0).is_err());
let mut wide = vec![1.0; MIN_PAIRED_ROUNDS as usize];
wide[0] = 3.0;
let loose = Paired::from_unit_secs("old", "new", &wide, &eight, 0, 0)
.expect("valid timings, wide spread");
assert!(!loose.is_publishable(), "a wide cross-build spread is uncontrolled too");
}
}
#[cfg(test)]
mod device_scope_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::{parse_pmon, sample_device_load};
const CAPTURE: &str = "\
# gpu pid type sm mem enc dec jpg ofa fb ccpm command
# Idx # C/G % % % % % % MB MB name
0 999001 C 0 0 - - - - 12 0 exatok-test
0 123456 C 41 12 - - - - 24204 0 llama-server
0 555000 G 3 1 - - - - 512 0 Xorg
0 777000 C 7 2 - - - - 670 0 device_determ
";
#[test]
fn own_and_foreign_device_residency_are_never_swapped() {
let load = parse_pmon(CAPTURE, 999_001).expect("four parseable rows");
assert_eq!(load.own_resident_mib, 12, "our own row is 12 MiB, not the server's 24204");
assert_eq!(
load.foreign_resident_mib,
24_204 + 670,
"foreign residency is the sum of the OTHER compute rows"
);
assert_eq!(load.foreign_processes, 2, "two foreign compute processes, excluding ourselves");
assert_eq!(load.foreign_sm_percent, 41 + 7, "foreign SM percent excludes our own row");
let other = parse_pmon(CAPTURE, 123_456).expect("same capture, different self");
assert_eq!(other.own_resident_mib, 24_204);
assert_eq!(other.foreign_resident_mib, 12 + 670);
assert_eq!(other.foreign_processes, 2);
let stranger = parse_pmon(CAPTURE, 1).expect("same capture, uninvolved self");
assert_eq!(stranger.own_resident_mib, 0);
assert_eq!(stranger.foreign_resident_mib, 12 + 24_204 + 670);
assert_eq!(stranger.foreign_processes, 3);
}
#[test]
fn graphics_contexts_are_excluded_and_an_unreadable_capture_is_not_an_idle_card() {
let load = parse_pmon(CAPTURE, 999_001).expect("parseable");
assert_eq!(load.own_resident_mib + load.foreign_resident_mib, 12 + 24_204 + 670);
assert!(
!format!("{load:?}").contains("512"),
"the desktop's 512 MiB must not appear in any total"
);
let headers = CAPTURE.lines().take(2).collect::<Vec<_>>().join("\n");
assert!(parse_pmon(&headers, 1).is_none(), "headers alone are not an observation");
assert!(parse_pmon("", 1).is_none(), "empty output is not an idle card");
assert!(parse_pmon("garbage\nmore garbage\n", 1).is_none());
let idle = " 0 555000 G 3 1 - - - - 512 0 Xorg\n";
let idle = parse_pmon(idle, 1).expect("a parsed graphics row is an observation");
assert_eq!(idle.foreign_processes, 0);
assert_eq!(idle.foreign_resident_mib, 0);
assert_eq!(idle.own_resident_mib, 0);
}
#[test]
fn unsampled_and_short_rows_do_not_corrupt_a_residency_total() {
let ragged = "\
0 123456 C - - - - - - - 0 llama-server
0 123457 C
0 notapid C 10 5 - - - - 100 0 weird
0 123458 C 10 5 - - - - 256 0 real
";
let load = parse_pmon(ragged, 999_001).expect("at least one parseable row");
assert_eq!(load.foreign_resident_mib, 256, "only the one sampled row has bytes");
assert_eq!(load.foreign_sm_percent, 10, "the unsampled `-` contributes no SM percent");
assert_eq!(load.foreign_processes, 2, "the dash row and the real row, not the malformed");
assert_eq!(load.own_resident_mib, 0);
}
#[test]
fn a_live_capture_never_attributes_gigabytes_to_this_test_process() {
let Some(load) = sample_device_load() else {
return;
};
assert!(
load.own_resident_mib < 4096,
"this test process holds {} MiB of device memory; a test binary holding gigabytes \
means a foreign process was attributed to self, which voids host-only cells",
load.own_resident_mib
);
if load.foreign_processes > 0 {
assert!(
load.foreign_resident_mib > 0,
"{} foreign compute processes were counted, so their residency cannot be 0",
load.foreign_processes
);
}
}
}