use std::time::Duration;
#[derive(Clone, Debug, PartialEq)]
pub struct LastRun {
pub suite: String,
pub fingerprint_sha256: String,
pub submitted: bool,
pub age: Duration,
pub partial: bool,
pub rejected: Option<(u16, Duration)>,
pub busy_streak: u32,
}
#[derive(Debug, PartialEq)]
pub enum Decision {
RunNow,
Wait(Duration),
Never,
}
pub const IDLE_BEFORE_RUN: Duration = Duration::from_secs(30);
pub fn backoff(failures: u32) -> Duration {
Duration::from_secs(match failures {
0 | 1 => 60,
2 => 300,
3 => 1800,
_ => 86400,
})
}
pub fn decide(
last: Option<&LastRun>,
current_fp: &str,
suite: &str,
idle_for: Duration,
failures: u32,
auto_enabled: bool,
) -> Decision {
if !auto_enabled {
return Decision::Never;
}
if let Some(l) = last {
if l.suite == suite {
match l.rejected {
Some((code, since)) if code != 400 && since < REJECTED_HOLD_OFF => {
return Decision::Wait(REJECTED_HOLD_OFF - since);
}
Some(_) => {}
None if l.fingerprint_sha256 == current_fp => {
if !l.partial {
return Decision::Never;
}
let hold = partial_rerun_after(l.busy_streak);
if l.age < hold {
return Decision::Wait(hold - l.age);
}
}
None => {}
}
if l.fingerprint_sha256 != current_fp
&& l.rejected.is_none()
&& l.age < COMPLETED_HOLD_OFF
{
return Decision::Wait(COMPLETED_HOLD_OFF - l.age);
}
}
}
if failures > 0 {
return Decision::Wait(backoff(failures));
}
if idle_for < IDLE_BEFORE_RUN {
return Decision::Wait(IDLE_BEFORE_RUN - idle_for);
}
Decision::RunNow
}
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Instant;
use super::fingerprint::HardwareFingerprint;
use super::report::SignedReport;
use super::run::{self, RunCtx, RunOpts, RunOutcome};
use super::submit::{self, SubmitError};
use super::SUITE;
use crate::broker::BrokerState;
const LOOP_EVERY: Duration = Duration::from_secs(10);
const RESUBMIT_EVERY: Duration = Duration::from_secs(60);
pub const COMPLETED_HOLD_OFF: Duration = Duration::from_secs(24 * 3600);
pub const PARTIAL_RERUN_AFTER: Duration = Duration::from_secs(6 * 3600);
pub const PARTIAL_RERUN_MAX: Duration = Duration::from_secs(24 * 3600);
pub fn partial_rerun_after(busy_streak: u32) -> Duration {
let doublings = busy_streak.saturating_sub(1).min(2);
(PARTIAL_RERUN_AFTER * (1 << doublings)).min(PARTIAL_RERUN_MAX)
}
pub const REJECTED_HOLD_OFF: Duration = Duration::from_secs(24 * 3600);
const REJECTED_PAUSE: Duration = Duration::from_secs(3600);
const LOCK_BUSY_PAUSE: Duration = Duration::from_secs(300);
const RUN_BUDGET: Duration = Duration::from_secs(60);
const START_JITTER_SECS: u64 = 120;
fn start_jitter() -> Duration {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.subsec_nanos() as u64)
.unwrap_or(0);
jitter_from(nanos ^ (std::process::id() as u64).wrapping_mul(0x9E37_79B9))
}
fn jitter_from(seed: u64) -> Duration {
Duration::from_secs(seed % START_JITTER_SECS)
}
pub fn spawn_once(state: Arc<BrokerState>) {
static STARTED: AtomicBool = AtomicBool::new(false);
if STARTED.swap(true, Ordering::SeqCst) {
return;
}
let _ = std::thread::Builder::new()
.name("bench-sched".into())
.spawn(move || scheduler_loop(&state));
}
fn scheduler_loop(state: &BrokerState) {
use crate::broker::node_identity::now_secs;
let (Some(api_url), Some(api_key)) = (&state.config.api_url, &state.config.api_key) else {
return;
};
let Some(dir) = crate::credentials::dir() else {
return;
};
let auto_enabled = std::env::var("ZAKURO_BENCH_AUTO").as_deref() != Ok("0");
let mut fingerprint: Option<String> = None;
let mut failures = 0u32;
let mut not_before = Instant::now();
let mut next_submit = Instant::now();
loop {
std::thread::sleep(LOOP_EVERY);
if !auto_enabled {
continue;
}
if let Some(signed) = run::pending_submission(&dir, None) {
if Instant::now() >= next_submit {
match submit::post(api_url, api_key, &signed) {
Ok(_) => {
let _ = run::save_last(&dir, &signed, run::SubmitState::Done);
}
Err(SubmitError::Retryable(e)) => {
eprintln!(" [BENCH] submit failed, will retry: {e}");
next_submit = Instant::now() + RESUBMIT_EVERY;
}
Err(e @ SubmitError::Rejected(code, _)) => {
eprintln!(" [BENCH] {e}");
let _ = run::save_last(&dir, &signed, run::SubmitState::Rejected(code));
not_before = Instant::now() + REJECTED_PAUSE;
}
}
}
}
if Instant::now() < not_before {
continue;
}
if fingerprint.is_none() {
let Some(ids) = run::enumerate_with_timeout(&AtomicBool::new(false)) else {
eprintln!(" [BENCH] GPU enumeration timed out; trying again later");
not_before = Instant::now() + backoff(1);
continue;
};
fingerprint = Some(HardwareFingerprint::detect(ids).sha256());
}
let fp = fingerprint.clone().unwrap_or_default();
let space_busy = crate::spaces::space_containers();
let spaces_running = || space_busy.load(Ordering::SeqCst) > 0;
let idle_secs = || {
if state.inflight.load(Ordering::SeqCst) > 0 || spaces_running() {
Duration::ZERO
} else {
Duration::from_secs(
now_secs().saturating_sub(state.last_busy.load(Ordering::SeqCst)),
)
}
};
let last = run::last_run(&dir);
if decide(last.as_ref(), &fp, SUITE, idle_secs(), 0, auto_enabled) != Decision::RunNow {
continue;
}
std::thread::sleep(start_jitter());
if idle_secs() < IDLE_BEFORE_RUN {
continue;
}
state.bench_cancel.store(false, Ordering::SeqCst);
if state.inflight.load(Ordering::SeqCst) > 0 || spaces_running() {
state.bench_cancel.store(true, Ordering::SeqCst);
continue;
}
println!(" [BENCH] idle: measuring this node");
let measuring = crate::spaces::bench_running();
measuring.store(true, Ordering::SeqCst);
let opts = RunOpts {
save_as: run::SubmitState::Pending,
budget: RUN_BUDGET,
no_gpu: false,
state_dir: dir.clone(),
hub_url: Some(api_url.clone()),
peers: state.peer_manager.peer_urls(),
};
let mut submit_hook = |s: &SignedReport| submit::post(api_url, api_key, s).map(|_| ());
let outcome = run::run(
&opts,
&mut RunCtx {
key: &state.node_key,
cancel: state.bench_cancel.clone(),
progress: &|_| {},
submit: Some(&mut submit_hook),
},
);
measuring.store(false, Ordering::SeqCst);
match outcome {
RunOutcome::AlreadyRunning => {
println!(
" [BENCH] a benchmark is already running; next try in {LOCK_BUSY_PAUSE:?}"
);
not_before = Instant::now() + LOCK_BUSY_PAUSE;
}
RunOutcome::Cancelled => {
failures += 1;
not_before = Instant::now() + backoff(failures);
println!(
" [BENCH] cancelled by traffic; next try in {:?}",
backoff(failures)
);
}
RunOutcome::Completed {
signed,
submit_error,
..
} => {
failures = 0;
match submit_error {
None => println!(" [BENCH] measured and submitted"),
Some(SubmitError::Retryable(e)) => {
eprintln!(" [BENCH] measured; submit failed, will retry: {e}");
next_submit = Instant::now() + RESUBMIT_EVERY;
}
Some(e @ SubmitError::Rejected(code, _)) => {
eprintln!(" [BENCH] measured; {e}");
let _ = run::save_last(&dir, &signed, run::SubmitState::Rejected(code));
not_before = Instant::now() + REJECTED_PAUSE;
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bench_node::SUITE;
#[test]
fn first_run_waits_for_thirty_idle_seconds() {
assert!(matches!(
decide(None, "fp", SUITE, Duration::from_secs(10), 0, true),
Decision::Wait(_)
));
assert!(matches!(
decide(None, "fp", SUITE, Duration::from_secs(30), 0, true),
Decision::RunNow
));
}
#[test]
fn same_fingerprint_and_suite_never_reruns() {
let l = LastRun {
suite: SUITE.into(),
fingerprint_sha256: "fp".into(),
submitted: true,
age: Duration::ZERO,
partial: false,
rejected: None,
busy_streak: 0,
};
assert!(matches!(
decide(Some(&l), "fp", SUITE, Duration::from_secs(999), 0, true),
Decision::Never
));
}
#[test]
fn a_fingerprint_change_right_after_a_run_waits_but_a_suite_change_does_not() {
let l = LastRun {
suite: SUITE.into(),
fingerprint_sha256: "old".into(),
submitted: true,
age: Duration::from_secs(3600),
partial: false,
rejected: None,
busy_streak: 0,
};
assert_eq!(
decide(Some(&l), "new", SUITE, Duration::from_secs(60), 0, true),
Decision::Wait(COMPLETED_HOLD_OFF - Duration::from_secs(3600))
);
let old_enough = LastRun {
age: COMPLETED_HOLD_OFF,
partial: false,
rejected: None,
busy_streak: 0,
..l.clone()
};
assert_eq!(
decide(
Some(&old_enough),
"new",
SUITE,
Duration::from_secs(60),
0,
true
),
Decision::RunNow
);
let other_suite = LastRun {
suite: "zc-bench-node/0".into(),
..l
};
assert_eq!(
decide(
Some(&other_suite),
"new",
SUITE,
Duration::from_secs(60),
0,
true
),
Decision::RunNow
);
}
#[test]
fn hardware_or_suite_change_reruns() {
let l = LastRun {
suite: SUITE.into(),
fingerprint_sha256: "old".into(),
submitted: true,
age: COMPLETED_HOLD_OFF,
partial: false,
rejected: None,
busy_streak: 0,
};
assert!(matches!(
decide(Some(&l), "new", SUITE, Duration::from_secs(60), 0, true),
Decision::RunNow
));
let l2 = LastRun {
suite: "zc-bench-node/0".into(),
fingerprint_sha256: "fp".into(),
submitted: true,
age: Duration::ZERO,
partial: false,
rejected: None,
busy_streak: 0,
};
assert!(matches!(
decide(Some(&l2), "fp", SUITE, Duration::from_secs(60), 0, true),
Decision::RunNow
));
}
#[test]
fn backoff_after_cancellations() {
let waits: Vec<_> = (1..=5)
.map(
|f| match decide(None, "fp", SUITE, Duration::from_secs(60), f, true) {
Decision::Wait(d) => d.as_secs(),
_ => 0,
},
)
.collect();
assert_eq!(waits, vec![60, 300, 1800, 86400, 86400]);
}
#[test]
fn auto_disabled_never_runs() {
assert!(matches!(
decide(None, "fp", SUITE, Duration::from_secs(60), 0, false),
Decision::Never
));
}
fn last(partial: bool, rejected: Option<(u16, Duration)>, age: Duration) -> LastRun {
LastRun {
suite: SUITE.into(),
fingerprint_sha256: "fp".into(),
submitted: true,
age,
partial,
rejected,
busy_streak: 0,
}
}
const IDLE: Duration = Duration::from_secs(60);
#[test]
fn a_partial_run_is_rerun_after_six_hours_not_never() {
let h = |n: u64| Duration::from_secs(n * 3600);
assert_eq!(
decide(Some(&last(true, None, h(1))), "fp", SUITE, IDLE, 0, true),
Decision::Wait(PARTIAL_RERUN_AFTER - h(1))
);
assert_eq!(
decide(Some(&last(true, None, h(6))), "fp", SUITE, IDLE, 0, true),
Decision::RunNow
);
assert_eq!(
decide(Some(&last(false, None, h(100))), "fp", SUITE, IDLE, 0, true),
Decision::Never
);
}
#[test]
fn a_report_from_the_previous_suite_is_remeasured_at_once() {
let old = LastRun {
suite: "zc-bench-node/1".into(),
..last(false, None, Duration::from_secs(100 * 3600))
};
assert_ne!(old.suite, SUITE);
assert_eq!(
decide(Some(&old), "fp", SUITE, IDLE, 0, true),
Decision::RunNow
);
}
#[test]
fn a_permanently_busy_gpu_backs_off_six_twelve_then_twenty_four_hours() {
let h = |n: u64| Duration::from_secs(n * 3600);
assert_eq!(partial_rerun_after(0), h(6));
assert_eq!(partial_rerun_after(1), h(6));
assert_eq!(partial_rerun_after(2), h(12));
assert_eq!(partial_rerun_after(3), h(24));
assert_eq!(partial_rerun_after(50), h(24));
let streak = |n| LastRun {
busy_streak: n,
..last(true, None, h(7))
};
assert_eq!(
decide(Some(&streak(1)), "fp", SUITE, IDLE, 0, true),
Decision::RunNow
);
assert_eq!(
decide(Some(&streak(2)), "fp", SUITE, IDLE, 0, true),
Decision::Wait(h(5))
);
assert_eq!(
decide(Some(&streak(3)), "fp", SUITE, IDLE, 0, true),
Decision::Wait(h(17))
);
}
#[test]
fn a_refused_report_is_remeasured_a_400_at_once_and_other_4xx_after_a_day() {
let rej = |c, since| Some((c, since));
assert_eq!(
decide(
Some(&last(false, rej(400, Duration::ZERO), Duration::ZERO)),
"fp",
SUITE,
IDLE,
0,
true
),
Decision::RunNow
);
assert_eq!(
decide(
Some(&last(
false,
rej(403, Duration::from_secs(3600)),
Duration::ZERO
)),
"fp",
SUITE,
IDLE,
0,
true
),
Decision::Wait(REJECTED_HOLD_OFF - Duration::from_secs(3600))
);
assert_eq!(
decide(
Some(&last(false, rej(403, REJECTED_HOLD_OFF), Duration::ZERO)),
"fp",
SUITE,
IDLE,
0,
true
),
Decision::RunNow
);
}
#[test]
fn start_jitter_stays_within_two_minutes() {
for seed in [0u64, 1, 119, 120, 121, u64::MAX] {
assert!(jitter_from(seed) < Duration::from_secs(120));
}
assert!(start_jitter() < Duration::from_secs(120));
}
}