use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc};
use std::time::{Duration, Instant};
use super::budget::Budget;
use super::fingerprint::{GpuId, HardwareFingerprint};
use super::gpu::{self, GpuOutcome};
use super::report::{sign, BenchReport, SignedReport};
use super::schedule::LastRun;
use super::submit::SubmitError;
use super::{cpu, disk, net, score, SUITE};
use crate::broker::node_identity::NodeKey;
const CPU_SHARE: f64 = 0.45;
const GPU_SHARE: f64 = 0.35;
const DISK_SHARE: f64 = 0.10;
const NET_SHARE: f64 = 0.10;
const WATCHDOG_FLOOR: Duration = Duration::from_secs(if cfg!(test) { 120 } else { 5 });
const GPU_GRACE: Duration = Duration::from_secs(10);
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum SubmitState {
Pending,
Done,
Rejected(u16),
Declined,
}
pub struct RunOpts {
pub save_as: SubmitState,
pub budget: Duration,
pub no_gpu: bool,
pub state_dir: PathBuf,
pub hub_url: Option<String>,
pub peers: Vec<String>,
}
pub type SubmitHook<'a> = &'a mut dyn FnMut(&SignedReport) -> Result<(), SubmitError>;
pub struct RunCtx<'a> {
pub key: &'a NodeKey,
pub cancel: Arc<AtomicBool>,
pub progress: &'a dyn Fn(&str),
pub submit: Option<SubmitHook<'a>>,
}
#[derive(Debug)]
pub enum RunOutcome {
Cancelled,
AlreadyRunning,
Completed {
signed: SignedReport,
report: Box<BenchReport>,
submit_error: Option<SubmitError>,
},
}
fn bench_dir(state_dir: &Path) -> PathBuf {
state_dir.join("bench")
}
fn last_path(state_dir: &Path) -> PathBuf {
bench_dir(state_dir).join("last.json")
}
pub fn save_last(state_dir: &Path, s: &SignedReport, state: SubmitState) -> std::io::Result<()> {
std::fs::create_dir_all(bench_dir(state_dir))?;
let body = serde_json::json!({
"report_json": s.report_json,
"signature": s.signature,
"submitted": state == SubmitState::Done,
"declined": state == SubmitState::Declined,
"rejected": match state { SubmitState::Rejected(c) => Some(c), _ => None },
"rejected_at": match state { SubmitState::Rejected(_) => Some(crate::broker::node_identity::now_secs()), _ => None },
});
let tmp = bench_dir(state_dir).join("last.json.tmp");
std::fs::write(&tmp, body.to_string())?;
std::fs::rename(tmp, last_path(state_dir))
}
pub fn load_last(state_dir: &Path) -> Option<(SignedReport, SubmitState)> {
let v: serde_json::Value =
serde_json::from_slice(&std::fs::read(last_path(state_dir)).ok()?).ok()?;
Some((
SignedReport {
report_json: v.get("report_json")?.as_str()?.to_string(),
signature: v.get("signature")?.as_str()?.to_string(),
},
match (
v.get("submitted").and_then(|b| b.as_bool()) == Some(true),
v.get("declined").and_then(|b| b.as_bool()) == Some(true),
v.get("rejected").and_then(|c| c.as_u64()),
) {
(true, _, _) => SubmitState::Done,
(_, true, _) => SubmitState::Declined,
(_, _, Some(c)) => SubmitState::Rejected(c as u16),
_ => SubmitState::Pending,
},
))
}
pub fn pending_submission(state_dir: &Path, rejected: Option<&str>) -> Option<SignedReport> {
match load_last(state_dir)? {
(s, SubmitState::Pending) if rejected != Some(s.signature.as_str()) => Some(s),
_ => None,
}
}
pub fn last_run(state_dir: &Path) -> Option<LastRun> {
let (s, state) = load_last(state_dir)?;
let rejected = match state {
SubmitState::Rejected(code) => {
let at = std::fs::read(last_path(state_dir))
.ok()
.and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok())
.and_then(|v| v.get("rejected_at")?.as_u64())
.unwrap_or(0);
let since = crate::broker::node_identity::now_secs().saturating_sub(at);
Some((code, Duration::from_secs(since)))
}
_ => None,
};
let r: serde_json::Value = serde_json::from_str(&s.report_json).ok()?;
let age = chrono::DateTime::parse_from_rfc3339(r.get("measured_at")?.as_str()?)
.ok()
.and_then(|t| {
(chrono::Utc::now() - t.with_timezone(&chrono::Utc))
.to_std()
.ok()
})
.unwrap_or(Duration::ZERO);
Some(LastRun {
age,
suite: r.get("suite")?.as_str()?.to_string(),
fingerprint_sha256: r.get("fingerprint_sha256")?.as_str()?.to_string(),
submitted: state == SubmitState::Done,
partial: r.get("partial").and_then(|p| p.as_bool()) == Some(true),
rejected,
busy_streak: load_busy_streak(state_dir),
})
}
fn busy_streak_path(state_dir: &Path) -> PathBuf {
bench_dir(state_dir).join("busy_streak")
}
pub fn load_busy_streak(state_dir: &Path) -> u32 {
std::fs::read_to_string(busy_streak_path(state_dir))
.ok()
.and_then(|s| s.trim().parse().ok())
.unwrap_or(0)
}
fn next_busy_streak(prev: u32, partial: bool, skipped: &[String]) -> u32 {
if partial && skipped.iter().any(|s| gpu::cuda::is_busy_note(s)) {
prev.saturating_add(1)
} else {
0
}
}
fn stage(total: Duration, share: f64, overall: &Budget, cancel: &Arc<AtomicBool>) -> Budget {
Budget::new(
total.mul_f64(share).min(overall.remaining()),
cancel.clone(),
)
}
fn gpu_with_timeout(b: &Budget, cancel: &AtomicBool) -> GpuOutcome {
let (tx, rx) = mpsc::channel();
let bb = b.clone();
std::thread::spawn(move || {
let _ = tx.send(gpu::probe(&bb));
});
let give_up = Instant::now() + b.remaining() + GPU_GRACE;
loop {
match rx.recv_timeout(Duration::from_millis(100)) {
Ok(o) => return o,
Err(mpsc::RecvTimeoutError::Disconnected) => {
return GpuOutcome {
gpus: vec![],
skipped: Some("gpu: probe thread died".into()),
}
}
Err(mpsc::RecvTimeoutError::Timeout) => {
if cancel.load(Ordering::SeqCst) || Instant::now() >= give_up {
return GpuOutcome {
gpus: vec![],
skipped: Some("gpu: probe timed out".into()),
};
}
}
}
}
}
pub(super) struct Prober {
busy: Arc<AtomicBool>,
cache: Arc<std::sync::Mutex<Option<Vec<GpuId>>>>,
}
impl Prober {
pub(super) fn new() -> Self {
Prober {
busy: Arc::new(AtomicBool::new(false)),
cache: Arc::new(std::sync::Mutex::new(None)),
}
}
fn cached(&self) -> Option<Vec<GpuId>> {
self.cache.lock().ok().and_then(|c| c.clone())
}
pub(super) fn enumerate(
&self,
probe: impl FnOnce() -> Vec<GpuId> + Send + 'static,
cancel: &AtomicBool,
grace: Duration,
) -> Option<Vec<GpuId>> {
if self.busy.swap(true, Ordering::SeqCst) {
return self.cached();
}
let (tx, rx) = mpsc::channel();
let (busy, cache) = (self.busy.clone(), self.cache.clone());
std::thread::spawn(move || {
struct Clear(Arc<AtomicBool>);
impl Drop for Clear {
fn drop(&mut self) {
self.0.store(false, Ordering::SeqCst);
}
}
let clear = Clear(busy);
let ids = probe();
if let Ok(mut c) = cache.lock() {
*c = Some(ids.clone());
}
drop(clear);
let _ = tx.send(ids);
});
let give_up = Instant::now() + grace;
loop {
match rx.recv_timeout(Duration::from_millis(100).min(grace)) {
Ok(ids) => return Some(ids),
Err(mpsc::RecvTimeoutError::Timeout)
if !cancel.load(Ordering::SeqCst) && Instant::now() < give_up => {}
Err(_) => return None,
}
}
}
}
pub(crate) fn enumerate_with_timeout(cancel: &AtomicBool) -> Option<Vec<GpuId>> {
static PROBER: std::sync::OnceLock<Prober> = std::sync::OnceLock::new();
PROBER
.get_or_init(Prober::new)
.enumerate(gpu::enumerate, cancel, GPU_GRACE)
}
fn fingerprint_gpus(enumerated: Option<Vec<GpuId>>, previous: Option<Vec<GpuId>>) -> Vec<GpuId> {
enumerated.or(previous).unwrap_or_default()
}
#[cfg(any(not(unix), test))]
struct PidLock {
path: PathBuf,
}
#[cfg(any(not(unix), test))]
const UNREADABLE_LOCK_STALE_AFTER: Duration = Duration::from_secs(60);
#[cfg(any(not(unix), test))]
fn lock_is_stale(contents: &str, age: Duration, alive: impl Fn(u32) -> bool) -> bool {
match contents.trim().parse::<u32>() {
Ok(pid) => !alive(pid),
Err(_) => age >= UNREADABLE_LOCK_STALE_AFTER,
}
}
#[cfg(any(not(unix), test))]
impl PidLock {
fn acquire(path: &Path, alive: impl Fn(u32) -> bool) -> Option<PidLock> {
use std::io::Write;
for _ in 0..2 {
match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
{
Ok(mut f) => {
let _ = write!(f, "{}", std::process::id());
return Some(PidLock { path: path.into() });
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
let contents = std::fs::read_to_string(path).unwrap_or_default();
let age = std::fs::metadata(path)
.and_then(|m| m.modified())
.ok()
.and_then(|t| t.elapsed().ok())
.unwrap_or_default();
if !lock_is_stale(&contents, age, &alive) {
return None;
}
let grave = path.with_extension(format!("stale-{}", std::process::id()));
if std::fs::rename(path, &grave).is_err() {
continue;
}
let _ = std::fs::remove_file(&grave);
}
Err(_) => return None,
}
}
None
}
}
#[cfg(any(not(unix), test))]
impl Drop for PidLock {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
#[cfg(any(not(unix), test))]
fn pid_alive(pid: u32) -> bool {
#[cfg(unix)]
{
let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
rc == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
}
#[cfg(windows)]
{
std::process::Command::new("tasklist")
.args(["/FI", &format!("PID eq {pid}"), "/NH"])
.output()
.map(|o| String::from_utf8_lossy(&o.stdout).contains(&pid.to_string()))
.unwrap_or(true)
}
#[cfg(not(any(unix, windows)))]
{
let _ = pid;
true
}
}
pub struct RunLock {
#[cfg(unix)]
_file: std::fs::File,
#[cfg(not(unix))]
_lock: PidLock,
}
#[cfg(unix)]
impl Drop for RunLock {
fn drop(&mut self) {
use std::os::unix::io::AsRawFd;
unsafe { libc::flock(self._file.as_raw_fd(), libc::LOCK_UN) };
}
}
impl RunLock {
pub fn acquire(dir: &Path) -> Option<RunLock> {
std::fs::create_dir_all(dir).ok()?;
let path = dir.join("run.lock");
#[cfg(unix)]
{
use std::os::unix::io::AsRawFd;
let file = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(&path)
.ok()?;
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
(rc == 0).then_some(RunLock { _file: file })
}
#[cfg(not(unix))]
{
PidLock::acquire(&path, pid_alive).map(|_lock| RunLock { _lock })
}
}
}
fn gpu_partial(
no_gpu: bool,
enumerated: Option<&[GpuId]>,
measured: &[super::report::GpuMetrics],
) -> bool {
if no_gpu {
return false;
}
let Some(ids) = enumerated else { return true };
ids.iter()
.any(|id| !measured.iter().any(|m| m.uuid == id.uuid))
|| measured
.iter()
.any(|m| !(m.tflops_f32 > 0.0 && m.tflops_f16 > 0.0 && m.mem_bw_gbps > 0.0))
}
pub fn run(opts: &RunOpts, ctx: &mut RunCtx) -> RunOutcome {
let Some(_lock) = RunLock::acquire(&bench_dir(&opts.state_dir)) else {
return RunOutcome::AlreadyRunning;
};
disk::sweep_stale_scratch(
&bench_dir(&opts.state_dir),
std::time::SystemTime::now(),
disk::STALE_SCRATCH_AFTER,
);
let started = Instant::now();
let cancel = ctx.cancel.clone();
let overall = Budget::new(opts.budget, cancel.clone());
let done = Arc::new(AtomicBool::new(false));
{
let (done, cancel) = (done.clone(), cancel.clone());
let limit = Instant::now() + (opts.budget * 2).max(WATCHDOG_FLOOR);
std::thread::spawn(move || {
while !done.load(Ordering::SeqCst) {
if Instant::now() >= limit {
cancel.store(true, Ordering::SeqCst);
return;
}
std::thread::sleep(Duration::from_millis(50));
}
});
}
let outcome = measure_and_send(opts, ctx, &overall, started);
done.store(true, Ordering::SeqCst);
outcome
}
fn measure_and_send(
opts: &RunOpts,
ctx: &mut RunCtx,
overall: &Budget,
started: Instant,
) -> RunOutcome {
let cancel = &ctx.cancel;
let mut skipped: Vec<String> = vec![];
let mut partial = false;
(ctx.progress)("cpu + memory");
let cpu = match cpu::probe(&stage(opts.budget, CPU_SHARE, overall, cancel)) {
Ok(p) => {
partial |= !p.notes.is_empty();
skipped.extend(p.notes);
Some(p.metrics)
}
Err(e) => {
partial = true;
skipped.push(format!("cpu: {e}"));
None
}
};
if cancel.load(Ordering::SeqCst) {
return RunOutcome::Cancelled;
}
(ctx.progress)("gpu");
let gpu_out = if opts.no_gpu {
GpuOutcome {
gpus: vec![],
skipped: Some("gpu: disabled by --no-gpu".into()),
}
} else {
gpu_with_timeout(&stage(opts.budget, GPU_SHARE, overall, cancel), cancel)
};
skipped.extend(gpu_out.skipped.clone());
if cancel.load(Ordering::SeqCst) {
return RunOutcome::Cancelled;
}
(ctx.progress)("disk");
let disk = match std::fs::create_dir_all(bench_dir(&opts.state_dir))
.map_err(|e| e.to_string())
.and_then(|_| {
disk::probe(
&bench_dir(&opts.state_dir),
&stage(opts.budget, DISK_SHARE, overall, cancel),
)
}) {
Ok(m) => Some(m),
Err(e) => {
partial = true;
skipped.push(format!("disk: {e}"));
None
}
};
if cancel.load(Ordering::SeqCst) {
return RunOutcome::Cancelled;
}
(ctx.progress)("network");
let net = net::probe(
opts.hub_url.as_deref(),
&opts.peers,
&stage(opts.budget, NET_SHARE, overall, cancel),
);
if net.hub_rtt_ms.is_none() && opts.hub_url.is_some() {
skipped.push("net: hub unreachable".into());
}
if cancel.load(Ordering::SeqCst) {
return RunOutcome::Cancelled;
}
let enumerated = enumerate_with_timeout(cancel);
partial |= gpu_partial(opts.no_gpu, enumerated.as_deref(), &gpu_out.gpus);
let fingerprint = HardwareFingerprint::detect(fingerprint_gpus(
enumerated,
load_last(&opts.state_dir)
.and_then(|(s, _)| serde_json::from_str::<BenchReport>(&s.report_json).ok())
.map(|r| r.fingerprint.gpus),
));
skipped.extend(super::fingerprint::busy_host_note(
super::fingerprint::load_average_1m(),
fingerprint.logical_cores,
));
let cpu_units = cpu.as_ref().map(score::cpu_units).unwrap_or(0.0);
let gpu_units = score::gpu_units(&gpu_out.gpus);
let build = crate::build_id::current();
let report = BenchReport {
suite: SUITE.into(),
zc_version: build.version.clone(),
zc_git: build.git.clone(),
node_pubkey: ctx.key.public_b64(),
fingerprint_sha256: fingerprint.sha256(),
fingerprint,
measured_at: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
duration_ms: started.elapsed().as_millis() as u64,
partial: partial || overall.expired(),
cpu,
disk,
net: Some(net),
gpus: gpu_out.gpus,
skipped,
cpu_units,
gpu_units,
numa_mem: super::fingerprint::numa_memory(),
};
let signed = sign(&report, ctx.key);
if opts.save_as == SubmitState::Pending {
let streak = next_busy_streak(
load_busy_streak(&opts.state_dir),
report.partial,
&report.skipped,
);
if let Err(e) = std::fs::write(busy_streak_path(&opts.state_dir), streak.to_string()) {
eprintln!(" [BENCH] could not save busy_streak: {e}");
}
}
if let Err(e) = save_last(&opts.state_dir, &signed, opts.save_as) {
eprintln!(" [BENCH] could not save last.json: {e}");
}
let submit_error = match ctx.submit.as_mut().map(|submit| submit(&signed)) {
Some(Ok(())) => {
let _ = save_last(&opts.state_dir, &signed, SubmitState::Done);
None
}
Some(Err(e)) => Some(e),
None => None,
};
RunOutcome::Completed {
signed,
report: Box::new(report),
submit_error,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn opts(dir: &Path, budget: Duration) -> RunOpts {
RunOpts {
save_as: SubmitState::Pending,
budget,
no_gpu: true,
state_dir: dir.to_path_buf(),
hub_url: None,
peers: vec![],
}
}
fn tmp(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("zc-bench-run-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
#[test]
fn cancelled_run_is_not_submitted() {
let dir = tmp("cancel");
let key = NodeKey::generate();
let cancel = Arc::new(AtomicBool::new(false));
let c2 = cancel.clone();
std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(200));
c2.store(true, Ordering::SeqCst);
});
let mut calls = 0;
let mut submit = |_: &SignedReport| {
calls += 1;
Ok(())
};
let progress = |_: &str| std::thread::sleep(Duration::from_millis(400));
let out = run(
&opts(&dir, Duration::from_secs(60)),
&mut RunCtx {
key: &key,
cancel,
progress: &progress,
submit: Some(&mut submit),
},
);
assert!(matches!(out, RunOutcome::Cancelled));
assert_eq!(calls, 0);
assert!(!dir.join("bench/last.json").exists());
}
#[test]
fn a_completed_run_saves_last_json_before_submitting() {
let dir = tmp("save");
let key = NodeKey::generate();
let mut seen_file = None;
let d2 = dir.clone();
let mut submit = |s: &SignedReport| {
let (saved, state) = load_last(&d2).expect("last.json exists at submit time");
seen_file = Some((saved == *s, state));
Ok(())
};
let out = run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: Some(&mut submit),
},
);
assert!(matches!(
out,
RunOutcome::Completed {
submit_error: None,
..
}
));
assert_eq!(seen_file, Some((true, SubmitState::Pending)));
assert_eq!(load_last(&dir).unwrap().1, SubmitState::Done);
assert_eq!(last_run(&dir).unwrap().suite, SUITE);
}
#[test]
fn a_failed_submit_leaves_last_json_unsubmitted() {
let dir = tmp("retry");
let key = NodeKey::generate();
let mut submit = |_: &SignedReport| Err(SubmitError::Retryable("down".into()));
let out = run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: Some(&mut submit),
},
);
assert!(matches!(
out,
RunOutcome::Completed {
submit_error: Some(SubmitError::Retryable(_)),
..
}
));
assert_eq!(load_last(&dir).unwrap().1, SubmitState::Pending);
}
#[test]
fn report_fills_skipped_reasons_and_partial() {
let dir = tmp("partial");
let key = NodeKey::generate();
let mut submit = |_: &SignedReport| Ok(());
let out = run(
&opts(&dir, Duration::ZERO),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: Some(&mut submit),
},
);
let RunOutcome::Completed { report, .. } = out else {
panic!("expected a completed run")
};
assert!(report.partial);
assert!(report.skipped.iter().any(|s| s.starts_with("gpu:")));
}
#[test]
fn a_run_with_the_gpu_probe_off_is_rerun_only_if_it_was_partial() {
use crate::bench_node::schedule::{decide, Decision, IDLE_BEFORE_RUN};
let dir = tmp("norerun");
let key = NodeKey::generate();
run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: None,
},
);
let last = last_run(&dir).unwrap();
let current = HardwareFingerprint::detect(gpu::enumerate()).sha256();
assert_eq!(last.fingerprint_sha256, current);
for tick in 1..=5u64 {
let idle = IDLE_BEFORE_RUN * tick as u32 * 2;
let d = decide(Some(&last), ¤t, SUITE, idle, 0, true);
if last.partial {
assert!(matches!(d, Decision::Wait(_)), "tick {tick}: {d:?}");
} else {
assert_eq!(d, Decision::Never, "tick {tick}");
}
}
}
#[test]
fn declined_and_delivered_reports_are_never_pending() {
let dir = tmp("declined");
let s = SignedReport {
report_json: "{}".into(),
signature: "sig".into(),
};
save_last(&dir, &s, SubmitState::Declined).unwrap();
assert_eq!(pending_submission(&dir, None), None);
assert_eq!(load_last(&dir).unwrap().1, SubmitState::Declined);
save_last(&dir, &s, SubmitState::Done).unwrap();
assert_eq!(pending_submission(&dir, None), None);
save_last(&dir, &s, SubmitState::Pending).unwrap();
assert_eq!(pending_submission(&dir, None), Some(s.clone()));
assert_eq!(
pending_submission(&dir, Some("sig")),
None,
"refused by the hub"
);
}
#[test]
fn an_enumerate_timeout_keeps_the_previous_runs_gpus() {
let gpu = |u: &str| GpuId {
vendor: "nvidia".into(),
model: "m".into(),
uuid: u.into(),
vram_bytes: 1,
};
let prev = Some(vec![gpu("a")]);
assert_eq!(fingerprint_gpus(None, prev.clone()), vec![gpu("a")]);
assert_eq!(fingerprint_gpus(Some(vec![gpu("b")]), prev), vec![gpu("b")]);
assert_eq!(fingerprint_gpus(Some(vec![]), Some(vec![gpu("a")])), vec![]);
assert!(fingerprint_gpus(None, None).is_empty());
let fp = |g| HardwareFingerprint::detect(g).sha256();
assert_eq!(
fp(fingerprint_gpus(None, Some(vec![gpu("a")]))),
fp(vec![gpu("a")])
);
}
fn gpu_id(u: &str) -> GpuId {
GpuId {
vendor: "nvidia".into(),
model: "m".into(),
uuid: u.into(),
vram_bytes: 1,
}
}
fn gpu_m(u: &str, f32: f64, f16: f64, bw: f64) -> super::super::report::GpuMetrics {
super::super::report::GpuMetrics {
vendor: "nvidia".into(),
model: "m".into(),
uuid: u.into(),
tflops_f32: f32,
tflops_f16: f16,
mem_bw_gbps: bw,
vram_bytes: 1,
}
}
#[test]
fn a_gpu_that_was_not_fully_measured_makes_the_report_partial() {
let full = [gpu_m("a", 1.0, 2.0, 3.0)];
let ids = [gpu_id("a")];
assert!(!gpu_partial(false, Some(&ids), &full));
assert!(gpu_partial(false, Some(&ids), &[]));
assert!(gpu_partial(false, Some(&[gpu_id("a"), gpu_id("b")]), &full));
assert!(gpu_partial(false, None, &[]));
assert!(gpu_partial(false, Some(&ids), &[gpu_m("a", 1.0, 0.0, 3.0)]));
assert!(gpu_partial(false, Some(&ids), &[gpu_m("a", 1.0, 2.0, 0.0)]));
assert!(!gpu_partial(true, Some(&ids), &[]));
assert!(!gpu_partial(false, Some(&[]), &[]));
}
#[test]
fn a_partial_last_run_is_read_back_as_partial() {
let dir = tmp("partial-last");
let key = NodeKey::generate();
let mut o = opts(&dir, Duration::ZERO);
o.save_as = SubmitState::Declined;
run(
&o,
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: None,
},
);
assert!(last_run(&dir).unwrap().partial, "zero budget is partial");
}
#[test]
fn a_refused_report_is_remembered_with_its_status_and_time() {
let dir = tmp("rejected");
let key = NodeKey::generate();
let mut submit = |_: &SignedReport| Err(SubmitError::Rejected(400, "stale".into()));
let RunOutcome::Completed { signed, .. } = run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: Some(&mut submit),
},
) else {
panic!("expected a completed run")
};
save_last(&dir, &signed, SubmitState::Rejected(400)).unwrap();
assert_eq!(load_last(&dir).unwrap().1, SubmitState::Rejected(400));
assert_eq!(pending_submission(&dir, None), None, "not re-sent");
let (code, since) = last_run(&dir).unwrap().rejected.unwrap();
assert_eq!(code, 400);
assert!(since < Duration::from_secs(5));
save_last(&dir, &signed, SubmitState::Done).unwrap();
assert!(last_run(&dir).unwrap().rejected.is_none());
}
#[test]
fn a_404_from_the_hub_leaves_the_report_pending_and_is_not_remeasured() {
use crate::bench_node::schedule::{decide, Decision, IDLE_BEFORE_RUN};
let dir = tmp("hub-404");
let key = NodeKey::generate();
let url = crate::bench_node::submit::tests::stub(vec!["404 Not Found"]);
let mut submit =
|s: &SignedReport| crate::bench_node::submit::post(&url, "k", s).map(|_| ());
let RunOutcome::Completed {
signed,
submit_error,
..
} = run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: Some(&mut submit),
},
)
else {
panic!("expected a completed run")
};
assert!(matches!(submit_error, Some(SubmitError::Retryable(_))));
assert_eq!(pending_submission(&dir, None), Some(signed));
let last = last_run(&dir).unwrap();
assert!(last.rejected.is_none());
assert_ne!(
decide(
Some(&last),
&last.fingerprint_sha256,
&last.suite,
IDLE_BEFORE_RUN * 10,
0,
true
),
Decision::RunNow
);
}
#[test]
fn a_second_run_in_the_same_state_dir_is_refused_not_interleaved() {
let dir = tmp("lock");
let first = RunLock::acquire(&bench_dir(&dir)).expect("first lock");
assert!(RunLock::acquire(&bench_dir(&dir)).is_none(), "held");
let key = NodeKey::generate();
let out = run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: None,
},
);
assert!(matches!(out, RunOutcome::AlreadyRunning));
assert!(!dir.join("bench/last.json").exists());
drop(first);
let deadline = Instant::now() + Duration::from_secs(5);
let mut again = RunLock::acquire(&bench_dir(&dir));
while again.is_none() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(20));
again = RunLock::acquire(&bench_dir(&dir));
}
assert!(again.is_some(), "released on drop");
}
#[test]
fn a_lock_is_stale_when_its_pid_is_dead_or_it_is_junk_and_old() {
let old = Duration::from_secs(120);
let fresh = Duration::from_secs(1);
assert!(lock_is_stale("4242", fresh, |_| false));
assert!(!lock_is_stale("4242\n", fresh, |p| p == 4242));
assert!(!lock_is_stale("", fresh, |_| false));
assert!(lock_is_stale("", old, |_| false));
assert!(lock_is_stale("xyz", old, |_| true));
}
#[test]
fn the_pid_lock_records_its_pid_and_takes_over_a_dead_owners_file() {
let dir = tmp("pidlock");
let path = dir.join("run.lock");
let mine = std::process::id().to_string();
let first = PidLock::acquire(&path, |_| true).expect("first");
assert_eq!(std::fs::read_to_string(&path).unwrap(), mine);
assert!(PidLock::acquire(&path, |_| true).is_none());
std::fs::write(&path, "999999999").unwrap();
let second = PidLock::acquire(&path, |p| p != 999_999_999).expect("takeover");
assert_eq!(std::fs::read_to_string(&path).unwrap(), mine);
drop(second);
assert!(!path.exists(), "released on drop");
std::mem::forget(first); }
#[test]
fn the_real_pid_check_sees_this_process_and_not_a_dead_one() {
assert!(pid_alive(std::process::id()));
assert!(!pid_alive(i32::MAX as u32));
}
fn id(uuid: &str) -> GpuId {
GpuId {
vendor: "nvidia".into(),
model: "m".into(),
uuid: uuid.into(),
vram_bytes: 1,
}
}
#[test]
fn a_hung_prober_is_never_joined_by_a_second_thread() {
use std::sync::atomic::AtomicUsize;
let p = Prober::new();
let spawned = Arc::new(AtomicUsize::new(0));
let cancel = AtomicBool::new(false);
let grace = Duration::from_millis(150);
let s = spawned.clone();
let ids = p.enumerate(
move || {
s.fetch_add(1, Ordering::SeqCst);
vec![id("GPU-1")]
},
&cancel,
grace,
);
assert_eq!(ids, Some(vec![id("GPU-1")]));
let (release, hold) = mpsc::channel::<()>();
let s = spawned.clone();
let t = Instant::now();
let hung = p.enumerate(
move || {
s.fetch_add(1, Ordering::SeqCst);
let _ = hold.recv();
vec![id("GPU-2")]
},
&cancel,
grace,
);
assert_eq!(hung, None);
assert!(t.elapsed() >= grace);
let s = spawned.clone();
let t = Instant::now();
let again = p.enumerate(
move || {
s.fetch_add(1, Ordering::SeqCst);
vec![id("GPU-3")]
},
&cancel,
Duration::from_secs(5),
);
assert_eq!(again, Some(vec![id("GPU-1")]));
assert!(t.elapsed() < Duration::from_secs(1));
assert_eq!(spawned.load(Ordering::SeqCst), 2, "no third thread");
release.send(()).unwrap();
let deadline = Instant::now() + Duration::from_secs(5);
while p.busy.load(Ordering::SeqCst) && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
let ids = p.enumerate(|| vec![id("GPU-4")], &cancel, Duration::from_secs(5));
assert_eq!(ids, Some(vec![id("GPU-4")]));
}
#[test]
fn a_panicking_probe_does_not_kill_later_enumeration() {
let p = Prober::new();
let cancel = AtomicBool::new(false);
let grace = Duration::from_secs(5);
assert_eq!(
p.enumerate(|| panic!("driver exploded"), &cancel, grace),
None
);
let deadline = Instant::now() + Duration::from_secs(5);
while p.busy.load(Ordering::SeqCst) && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
assert_eq!(
p.enumerate(|| vec![id("GPU-9")], &cancel, grace),
Some(vec![id("GPU-9")])
);
}
#[test]
fn the_busy_streak_grows_while_busy_and_resets_on_a_complete_run() {
let busy = vec!["gpu: RTX busy (20.0 GiB used by other processes)".to_string()];
let other = vec!["gpu: probe timed out".to_string()];
assert_eq!(next_busy_streak(0, true, &busy), 1);
assert_eq!(next_busy_streak(1, true, &busy), 2);
assert_eq!(next_busy_streak(2, true, &other), 0);
assert_eq!(next_busy_streak(5, false, &[]), 0);
assert_eq!(next_busy_streak(5, false, &busy), 0);
let dir = tmp("streak");
assert_eq!(load_busy_streak(&dir), 0);
std::fs::create_dir_all(dir.join("bench")).unwrap();
std::fs::write(dir.join("bench/busy_streak"), "3").unwrap();
assert_eq!(load_busy_streak(&dir), 3);
}
#[test]
fn a_manual_run_does_not_touch_the_busy_streak() {
let dir = tmp("manualstreak");
std::fs::create_dir_all(dir.join("bench")).unwrap();
std::fs::write(dir.join("bench/busy_streak"), "2").unwrap();
let key = NodeKey::generate();
let go = |save_as| {
let mut o = opts(&dir, Duration::from_secs(5));
o.save_as = save_as;
let _ = run(
&o,
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: None,
},
);
};
go(SubmitState::Declined);
assert_eq!(load_busy_streak(&dir), 2, "manual run changed the streak");
go(SubmitState::Pending);
assert_eq!(load_busy_streak(&dir), 0);
}
#[test]
fn a_run_clears_stale_scratch_files_at_start() {
let dir = tmp("sweepstart");
let bench = dir.join("bench");
std::fs::create_dir_all(&bench).unwrap();
let stale = bench.join("bench-424242.tmp");
std::fs::write(&stale, b"x").unwrap();
let f = std::fs::File::options().write(true).open(&stale).unwrap();
f.set_modified(std::time::SystemTime::now() - Duration::from_secs(2 * 3600))
.unwrap();
drop(f);
let key = NodeKey::generate();
let _ = run(
&opts(&dir, Duration::from_secs(5)),
&mut RunCtx {
key: &key,
cancel: Arc::new(AtomicBool::new(false)),
progress: &|_| {},
submit: None,
},
);
assert!(!stale.exists(), "stale scratch file survived the run");
}
}