use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use super::client::{Hub, Lease};
use super::runner::{is_expired, node_label_filter, RunnerDeps, Runtime, SpaceRunner};
use super::{read_settings, ActiveSpace, Stats};
use crate::bench_node::fingerprint::GpuId;
use crate::broker::node_identity::NodeKey;
pub const CLAIM_EVERY: Duration = Duration::from_secs(15);
pub const IDLE_STOP: Duration = Duration::from_secs(600);
pub const BACKOFF_BASE: Duration = Duration::from_secs(60);
pub const BACKOFF_CAP: Duration = Duration::from_secs(30 * 60);
pub const GPU_REFRESH: Duration = Duration::from_secs(600);
pub struct Config {
pub state_dir: PathBuf,
pub max_leases: usize,
pub max_inflight: u64,
pub idle_timeout: Duration,
pub network: String,
pub backoff_base: Duration,
pub backoff_cap: Duration,
pub complete_retry_base: Duration,
}
impl Config {
pub fn from_env(state_dir: PathBuf) -> Config {
let num = |k: &str, d: u64| {
std::env::var(k)
.ok()
.and_then(|v| v.trim().parse().ok())
.unwrap_or(d)
};
Config {
state_dir,
max_leases: num("ZAKURO_SPACES_MAX_LEASES", 1).max(1) as usize,
max_inflight: num("ZAKURO_SPACES_MAX_INFLIGHT", 1),
idle_timeout: IDLE_STOP,
backoff_base: BACKOFF_BASE,
backoff_cap: BACKOFF_CAP,
complete_retry_base: Duration::from_secs(1),
network: {
let (net, warn) = spaces_network(
std::env::var("ZAKURO_SPACES_NETWORK").ok().as_deref(),
&crate::broker::deploy::runtime::deploy_network(),
);
if let Some(w) = warn {
eprintln!(" [SPACES] {w}");
}
net
},
}
}
}
pub fn spaces_network(requested: Option<&str>, deploy_network: &str) -> (String, Option<String>) {
let req = requested.map(str::trim).filter(|s| !s.is_empty());
match req {
None => ("bridge".to_string(), None),
Some(n)
if n != "bridge"
&& (n == "host" || n.starts_with("container:") || n == deploy_network) =>
{
(
"bridge".to_string(),
Some(format!(
"ZAKURO_SPACES_NETWORK={n} refused (mesh/host access); using bridge"
)),
)
}
Some(n) => (n.to_string(), None),
}
}
pub fn bench_units(dir: &std::path::Path) -> Option<Value> {
let last: Value =
serde_json::from_slice(&std::fs::read(dir.join("bench").join("last.json")).ok()?).ok()?;
let report: Value = serde_json::from_str(last.get("report_json")?.as_str()?).ok()?;
let cpu = report.get("cpu_units")?.as_f64()?;
let gpu = report.get("gpu_units").and_then(|g| g.as_f64());
Some(json!({ "cpu": cpu, "gpu": gpu }))
}
pub struct Scheduler {
hub: Hub,
key: Arc<NodeKey>,
runtime: Runtime,
cfg: Config,
inflight: Arc<dyn Fn() -> u64 + Send + Sync>,
stats: Arc<Stats>,
runners: HashMap<String, SpaceRunner>,
node_fp: String,
gpu_probe: GpuProbe,
gpus: Vec<GpuId>,
gpus_at: Option<Instant>,
backoff: HashMap<String, Backoff>,
bench_running: Arc<AtomicBool>,
containers: Arc<AtomicUsize>,
}
#[derive(Debug, PartialEq)]
pub enum Tick {
Disabled,
NoCapacity,
BackedOff,
BenchRunning,
NoRuntime,
ClaimFailed(String),
Claimed(usize),
}
fn skip_reason(t: &Tick) -> Option<&'static str> {
match t {
Tick::Disabled => Some("disabled (`zc spaces enable` to opt in)"),
Tick::NoCapacity => Some("at capacity (all Space slots or in-flight requests busy)"),
Tick::BackedOff => Some("paused after a failed Space (backoff)"),
Tick::BenchRunning => Some("the node benchmark is measuring"),
Tick::NoRuntime => Some(
"no container runtime: `docker` is missing or $DOCKER_HOST is unreachable/forbidden",
),
Tick::ClaimFailed(_) | Tick::Claimed(_) => None,
}
}
fn state_change(prev: Option<&Tick>, cur: &Tick) -> Option<String> {
let (was, now) = (prev.map(skip_reason), skip_reason(cur));
match (was, now) {
(Some(a), b) if a == b => None,
(None, Some(_)) if matches!(cur, Tick::Disabled) => None,
(_, Some(why)) => Some(format!(" [SPACES] not claiming: {why}")),
(Some(Some(_)), None) => Some(" [SPACES] claiming".to_string()),
_ => None,
}
}
fn memory_gb() -> f64 {
#[cfg(target_os = "linux")]
{
if let Ok(t) = std::fs::read_to_string("/proc/meminfo") {
for l in t.lines() {
if let Some(r) = l.strip_prefix("MemTotal:") {
let kb: f64 = r
.trim()
.trim_end_matches("kB")
.trim()
.parse()
.unwrap_or(0.0);
return kb / 1024.0 / 1024.0;
}
}
}
}
#[cfg(target_os = "macos")]
{
if let Ok(o) = std::process::Command::new("sysctl")
.args(["-n", "hw.memsize"])
.output()
{
if let Ok(b) = String::from_utf8_lossy(&o.stdout).trim().parse::<f64>() {
return b / 1024.0 / 1024.0 / 1024.0;
}
}
}
0.0
}
pub fn capabilities(gpus: &[GpuId]) -> Value {
let cpus = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
let cuda: Vec<&GpuId> = gpus.iter().filter(|g| g.vendor == "nvidia").collect();
let vram_gb = cuda
.iter()
.map(|g| g.vram_bytes)
.min()
.map(|b| ((b as f64 / (1u64 << 30) as f64) * 10.0).floor() / 10.0)
.unwrap_or(0.0);
json!({
"cpus": cpus,
"memory_gb": (memory_gb() * 10.0).round() / 10.0,
"gpus": cuda.len(),
"vram_gb": vram_gb,
})
}
pub type GpuProbe = Arc<dyn Fn() -> Option<Vec<GpuId>> + Send + Sync>;
pub fn real_gpu_probe() -> GpuProbe {
Arc::new(|| crate::bench_node::run::enumerate_with_timeout(&AtomicBool::new(false)))
}
#[derive(Debug, Clone, Copy)]
struct Backoff {
until: Instant,
delay: Duration,
}
impl Scheduler {
pub fn new(
hub: Hub,
key: Arc<NodeKey>,
runtime: Runtime,
cfg: Config,
inflight: Arc<dyn Fn() -> u64 + Send + Sync>,
) -> Scheduler {
let stats = Arc::new(Stats::new(&cfg.state_dir));
let node_fp = key.fingerprint();
for id in runtime.list_by_label_key(&node_label_filter(&node_fp)) {
eprintln!(" [SPACES] removing orphaned Space container {id}");
let _ = runtime.stop(&id);
let _ = runtime.rm(&id);
}
Scheduler {
hub,
key,
runtime,
cfg,
inflight,
stats,
runners: HashMap::new(),
node_fp,
gpu_probe: real_gpu_probe(),
gpus: Vec::new(),
gpus_at: None,
backoff: HashMap::new(),
bench_running: super::bench_running(),
containers: super::space_containers(),
}
}
fn gpus(&mut self) -> &[GpuId] {
if self.gpus_at.is_none_or(|t| t.elapsed() >= GPU_REFRESH) {
if let Some(g) = (self.gpu_probe)() {
self.gpus = g;
}
self.gpus_at = Some(Instant::now());
}
&self.gpus
}
fn record_failure(&mut self, space_id: &str) {
let delay = match self.backoff.get(space_id) {
Some(b) => (b.delay * 2).min(self.cfg.backoff_cap),
None => self.cfg.backoff_base.min(self.cfg.backoff_cap),
};
eprintln!(" [SPACES] space {space_id} failed; pausing claims for {delay:?}");
self.backoff.insert(
space_id.to_string(),
Backoff {
until: Instant::now() + delay,
delay,
},
);
}
fn in_backoff(&self, space_id: &str) -> bool {
self.backoff
.get(space_id)
.is_some_and(|b| b.until > Instant::now())
}
fn reap_runners(&mut self) {
let mut failed = Vec::new();
let mut healthy = Vec::new();
for (id, r) in &self.runners {
if r.session.completions() > 0 && !r.session.failed() {
healthy.push(id.clone());
}
if r.finished() && r.session.failed() {
failed.push(id.clone());
}
}
for id in healthy {
self.backoff.remove(&id);
}
for id in failed {
self.record_failure(&id);
}
self.runners.retain(|_, r| !r.finished());
self.containers.store(self.runners.len(), Ordering::SeqCst);
}
fn outstanding(&self) -> usize {
self.runners.values().map(|r| r.session.outstanding()).sum()
}
fn deps(&self) -> RunnerDeps {
RunnerDeps {
hub: self.hub.clone(),
runtime: self.runtime.clone(),
node_key: self.key.clone(),
state_dir: self.cfg.state_dir.clone(),
stats: self.stats.clone(),
network: self.cfg.network.clone(),
idle_timeout: self.cfg.idle_timeout,
node_fp: self.node_fp.clone(),
complete_retry_base: self.cfg.complete_retry_base,
}
}
pub fn runner_count(&self) -> usize {
self.runners.len()
}
fn publish_status(&self) {
let active = self
.runners
.iter()
.map(|(id, r)| {
let (queued, running) = r.session.counts();
ActiveSpace {
space_id: id.clone(),
queued,
running,
}
})
.collect();
self.stats.set_active(active);
}
pub fn tick(&mut self) -> Tick {
self.reap_runners();
let enabled = read_settings(&self.cfg.state_dir).enabled;
if !enabled {
self.publish_status();
return Tick::Disabled;
}
let spare = self.cfg.max_leases.saturating_sub(self.outstanding());
if spare == 0 || (self.inflight)() >= self.cfg.max_inflight {
self.publish_status();
return Tick::NoCapacity;
}
if self.bench_running.load(Ordering::SeqCst) {
self.publish_status();
return Tick::BenchRunning;
}
if self.backoff.values().any(|b| b.until > Instant::now()) {
self.publish_status();
return Tick::BackedOff;
}
if !self.runtime.available() {
return Tick::NoRuntime;
}
let caps = {
let gpus = self.gpus().to_vec();
capabilities(&gpus)
};
let leases = match self.hub.claim(
&self.key.public_b64(),
&caps,
bench_units(&self.cfg.state_dir),
spare as u32,
) {
Ok(l) => l,
Err(e) => return Tick::ClaimFailed(e),
};
let n = leases.len();
for lease in leases {
self.accept(lease);
}
self.containers.store(self.runners.len(), Ordering::SeqCst);
self.publish_status();
Tick::Claimed(n)
}
fn accept(&mut self, lease: Lease) {
if is_expired(&lease) {
eprintln!(
" [SPACES] lease {} already expired; abandoned",
lease.lease_id
);
return;
}
if let Some(r) = self.runners.get(&lease.space_id) {
if !r.finished() {
r.session.enqueue(lease);
return;
}
}
if self.in_backoff(&lease.space_id) {
eprintln!(
" [SPACES] space {} is backing off; lease {} left to expire",
lease.space_id, lease.lease_id
);
return;
}
match SpaceRunner::start(&self.deps(), &lease) {
Ok(r) => {
r.session.enqueue(lease.clone());
self.runners.insert(lease.space_id.clone(), r);
}
Err(e) => {
eprintln!(
" [SPACES] could not start space {}: {e} (lease {} left to expire)",
lease.space_id, lease.lease_id
);
self.record_failure(&lease.space_id);
}
}
}
pub fn shutdown(&mut self) {
for r in self.runners.values() {
r.shutdown();
}
self.runners.clear();
self.containers.store(0, Ordering::SeqCst);
}
}
impl Drop for Scheduler {
fn drop(&mut self) {
self.shutdown();
}
}
pub fn spawn_once(state: Arc<crate::broker::BrokerState>) {
static STARTED: AtomicBool = AtomicBool::new(false);
let (Some(api_url), Some(api_key)) =
(state.config.api_url.clone(), state.config.api_key.clone())
else {
return;
};
let Some(dir) = crate::credentials::dir() else {
return;
};
if STARTED.swap(true, Ordering::SeqCst) {
return;
}
let key = state.node_key.clone();
let st = state.clone();
let _ = std::thread::Builder::new()
.name("spaces-sched".into())
.spawn(move || {
let mut s = Scheduler::new(
Hub::new(&api_url, &api_key, &key.public_b64()),
key,
Arc::new(crate::broker::deploy::DockerCli),
Config::from_env(dir),
Arc::new(move || st.inflight.load(Ordering::SeqCst)),
);
let mut last: Option<Tick> = None;
loop {
let t = s.tick();
if let Some(line) = state_change(last.as_ref(), &t) {
println!("{line}");
}
match &t {
Tick::ClaimFailed(e) => eprintln!(" [SPACES] claim failed: {e}"),
Tick::Claimed(n) if *n > 0 => println!(" [SPACES] claimed {n} lease(s)"),
_ => {}
}
last = Some(t);
std::thread::sleep(CLAIM_EVERY);
}
});
}
#[cfg(test)]
mod tests {
#[test]
fn skip_reason_is_logged_once_per_change_not_every_tick() {
let first = state_change(None, &Tick::NoRuntime).expect("first skip is logged");
assert!(
first.contains("not claiming") && first.contains("docker"),
"{first}"
);
assert_eq!(state_change(Some(&Tick::NoRuntime), &Tick::NoRuntime), None);
let other = state_change(Some(&Tick::NoRuntime), &Tick::BackedOff).unwrap();
assert!(other.contains("backoff"), "{other}");
let back = state_change(Some(&Tick::BackedOff), &Tick::Claimed(0)).unwrap();
assert!(back.contains("claiming"), "{back}");
assert_eq!(
state_change(Some(&Tick::Claimed(0)), &Tick::Claimed(2)),
None
);
assert_eq!(state_change(None, &Tick::Disabled), None);
assert!(state_change(Some(&Tick::Claimed(0)), &Tick::Disabled)
.unwrap()
.contains("disabled"));
}
use super::*;
use crate::broker::deploy::{ContainerRuntime, RunSpec};
use crate::broker::node_identity::verify_sig;
use crate::spaces::socket::request;
use crate::spaces::testutil::{stub_hub, StubHub};
use crate::spaces::{write_settings, Settings};
use std::path::Path;
use std::sync::Mutex;
const DIGEST: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
const SPACE: &str = "22222222-2222-4222-8222-222222222222";
type RunRec = (
String,
Vec<(String, String)>,
HashMap<String, String>,
Option<f64>,
);
type SpecRec = (String, Vec<(String, String)>, bool);
struct FakeRt {
digest: Mutex<String>,
logins: Mutex<Vec<String>>,
pulls: Mutex<Vec<String>>,
runs: Mutex<Vec<RunRec>>,
stopped: Mutex<Vec<String>>,
removed: Mutex<Vec<String>>,
orphans: Mutex<Vec<String>>,
cfg_dirs: Mutex<Vec<PathBuf>>,
specs: Mutex<Vec<SpecRec>>,
list_keys: Mutex<Vec<String>>,
fail_run: AtomicBool,
}
impl FakeRt {
fn new(digest: &str) -> Arc<FakeRt> {
Arc::new(FakeRt {
digest: Mutex::new(digest.to_string()),
logins: Mutex::new(vec![]),
pulls: Mutex::new(vec![]),
runs: Mutex::new(vec![]),
stopped: Mutex::new(vec![]),
removed: Mutex::new(vec![]),
orphans: Mutex::new(vec![]),
cfg_dirs: Mutex::new(vec![]),
specs: Mutex::new(vec![]),
list_keys: Mutex::new(vec![]),
fail_run: AtomicBool::new(false),
})
}
}
impl ContainerRuntime for FakeRt {
fn available(&self) -> bool {
true
}
fn pull(&self, image: &str) -> Result<(), String> {
self.pulls.lock().unwrap().push(image.to_string());
Ok(())
}
fn run(&self, spec: &RunSpec) -> Result<String, String> {
if self.fail_run.load(Ordering::SeqCst) {
return Err("docker run failed".to_string());
}
self.specs.lock().unwrap().push((
spec.name.clone(),
spec.labels.clone(),
spec.hardened,
));
self.runs.lock().unwrap().push((
spec.image.to_string(),
spec.mounts.clone(),
spec.env.clone(),
spec.cpus,
));
Ok("cont-1".to_string())
}
fn stop(&self, id: &str) -> Result<(), String> {
self.stopped.lock().unwrap().push(id.to_string());
Ok(())
}
fn rm(&self, id: &str) -> Result<(), String> {
self.removed.lock().unwrap().push(id.to_string());
Ok(())
}
fn list_by_label_key(&self, k: &str) -> Vec<String> {
self.list_keys.lock().unwrap().push(k.to_string());
self.orphans.lock().unwrap().clone()
}
fn login_in(&self, dir: &Path, r: &str, u: &str, pw: &str) -> Result<(), String> {
self.cfg_dirs.lock().unwrap().push(dir.to_path_buf());
assert!(dir.exists(), "config dir exists during login");
self.login(r, u, pw)
}
fn restart(&self, _id: &str) -> Result<(), String> {
Ok(())
}
fn is_running(&self, _id: &str) -> bool {
true
}
fn container_ip(&self, _id: &str) -> Result<String, String> {
Ok("1.2.3.4".into())
}
fn health_check(&self, _id: &str, _hc: &crate::broker::deploy::Healthcheck) -> bool {
true
}
fn logs_tail(&self, _id: &str, _n: u32) -> String {
String::new()
}
fn list_by_label(&self, _id: &str) -> Vec<String> {
vec![]
}
fn login(&self, registry: &str, user: &str, _pw: &str) -> Result<(), String> {
self.logins
.lock()
.unwrap()
.push(format!("{registry}/{user}"));
Ok(())
}
fn image_digest(&self, _id: &str) -> Result<String, String> {
Ok(self.digest.lock().unwrap().clone())
}
}
static N: AtomicUsize = AtomicUsize::new(0);
fn temp_state() -> PathBuf {
let d = PathBuf::from(format!(
"/tmp/zksp{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::SeqCst)
));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
write_settings(
&d,
&Settings {
enabled: true,
min_price_per_hour: None,
},
)
.unwrap();
d
}
fn in_secs(s: i64) -> String {
(chrono::Utc::now() + chrono::Duration::seconds(s))
.format("%Y-%m-%dT%H:%M:%SZ")
.to_string()
}
fn lease_json(deadline: &str, hb: u64) -> String {
format!(
r#"{{"leases":[{{"lease_id":"11111111-1111-4111-8111-111111111111","batch_id":123,"space_id":"{SPACE}","seq":7,
"image":{{"mirror_ref":"registry:5000/spaces/{SPACE}@{DIGEST}","digest":"{DIGEST}"}},
"pull":{{"registry":"registry:5000","username":"zk-lease","token":"tok"}},
"inputs":[{{"name":"a.fastq","url":"https://example.com/a?sig=1"}}],
"requirements":{{"cpus":4,"memory_gb":8,"gpus":0,"min_vram_gb":0,"max_runtime_s":600,"est_runtime_s":120}},
"deadline":"{deadline}","heartbeat_every_s":{hb}}}]}}"#
)
}
fn hub_with(lease: String) -> StubHub {
let first = Arc::new(std::sync::atomic::AtomicBool::new(true));
stub_hub(move |_m, p, _b| {
if p.ends_with("/claim") {
if first.swap(false, Ordering::SeqCst) {
(200, lease.clone())
} else {
(200, r#"{"leases":[]}"#.to_string())
}
} else if p.ends_with("/heartbeat") {
(200, "{}".to_string())
} else if p.ends_with("outputs:presign") {
(
200,
r#"{"urls":{"out.txt":"https://put.example/out.txt"}}"#.to_string(),
)
} else if p.ends_with("/complete") {
(200, r#"{"status":"succeeded","paid":0.9}"#.to_string())
} else {
(404, "{}".to_string())
}
})
}
fn sched(
hub: &StubHub,
rt: &Arc<FakeRt>,
dir: &Path,
key: &Arc<NodeKey>,
inflight: u64,
) -> Scheduler {
let mut cfg = Config::from_env(dir.to_path_buf());
cfg.max_leases = 1;
cfg.max_inflight = 1;
cfg.idle_timeout = Duration::from_secs(60);
cfg.complete_retry_base = Duration::from_millis(20);
let mut s = Scheduler::new(
Hub::new(&hub.url, "zk_test", &key.public_b64()),
key.clone(),
rt.clone() as Runtime,
cfg,
Arc::new(move || inflight),
);
s.bench_running = Arc::default();
s.containers = Arc::default();
s.gpu_probe = Arc::new(|| Some(vec![]));
s
}
fn sock(dir: &Path) -> PathBuf {
crate::spaces::runner::socket_path(dir, SPACE)
}
#[test]
fn claim_pull_next_outputs_done_complete() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(
rt.pulls.lock().unwrap().as_slice(),
[format!("registry:5000/spaces/{SPACE}@{DIGEST}")]
);
assert_eq!(
rt.logins.lock().unwrap().as_slice(),
["registry:5000/zk-lease"]
);
{
let runs = rt.runs.lock().unwrap();
let (_, mounts, env, cpus) = &runs[0];
assert_eq!(mounts[0].1, "/run/zakuro/space.sock");
assert_eq!(env["ZAKURO_SPACE_ID"], SPACE);
assert_eq!(env["ZAKURO_SPACE_SOCKET"], "/run/zakuro/space.sock");
assert_eq!(*cpus, Some(4.0));
}
let (code, body) = request(&sock(&dir), "GET", "/next", "");
assert_eq!(code, 200, "{body}");
let v: Value = serde_json::from_str(&body).unwrap();
assert_eq!(v["batch_id"], 123);
assert_eq!(v["inputs"][0]["name"], "a.fastq");
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 204);
let (code, body) = request(
&sock(&dir),
"POST",
"/outputs",
r#"{"batch_id":123,"names":["out.txt"]}"#,
);
assert_eq!(code, 200);
assert!(body.contains("https://put.example/out.txt"));
assert_eq!(hub.count("outputs:presign"), 1);
hub.assert_pubkey("outputs:presign", &key.public_b64());
hub.assert_pubkey("/claim", &key.public_b64());
let hash = "ab".repeat(32);
let (code, body) = request(
&sock(&dir),
"POST",
"/done",
&format!(
r#"{{"batch_id":123,"output_hashes":{{"out.txt":"{hash}"}},"exit_code":0,"stderr_tail":""}}"#
),
);
assert_eq!(code, 200, "{body}");
assert!(body.contains("\"accepted\":true"));
let reqs = hub.requests.lock().unwrap();
let (_, path, cbody) = reqs
.iter()
.find(|(_, p, _)| p.ends_with("/complete"))
.unwrap();
assert!(path.contains("11111111-1111-4111-8111-111111111111"));
let c: Value = serde_json::from_str(cbody).unwrap();
let rj = c["receipt_json"].as_str().unwrap();
assert!(verify_sig(
&key.public_b64(),
rj.as_bytes(),
c["signature"].as_str().unwrap()
));
let r: Value = serde_json::from_str(rj).unwrap();
assert_eq!(r["image_digest"], DIGEST);
assert_eq!(r["output_hashes"]["out.txt"], hash);
assert_eq!(r["exit_code"], 0);
drop(reqs);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_never_reports_and_kills() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
*rt.digest.lock().unwrap() = "sha256:bbbb".to_string();
let (code, body) = request(
&sock(&dir),
"POST",
"/done",
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#,
);
assert_eq!(code, 409);
assert!(body.contains("digest mismatch"));
assert_eq!(hub.count("/complete"), 0);
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_at_start_kills_and_takes_no_work() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new("sha256:cccc");
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 0);
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(hub.count("/complete"), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn heartbeats_are_sent_every_period_while_a_batch_runs() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 1));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
std::thread::sleep(Duration::from_millis(2600));
assert!(
hub.count("/heartbeat") >= 2,
"got {}",
hub.count("/heartbeat")
);
hub.assert_pubkey("/heartbeat", &key.public_b64());
let before = hub.count("/heartbeat");
let (code, _) = request(
&sock(&dir),
"POST",
"/done",
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#,
);
assert_eq!(code, 200);
std::thread::sleep(Duration::from_millis(1500));
assert_eq!(hub.count("/heartbeat"), before, "no heartbeats once idle");
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn expired_lease_is_abandoned_and_reports_nothing() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(2), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
std::thread::sleep(Duration::from_millis(2600));
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(hub.count("/complete"), 0);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn lease_already_expired_at_claim_starts_nothing() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(-30), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert!(rt.runs.lock().unwrap().is_empty());
assert_eq!(s.runner_count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn outputs_over_64_names_is_rejected() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let names: Vec<String> = (0..65).map(|i| format!("\"f{i}\"")).collect();
let (code, _) = request(
&sock(&dir),
"POST",
"/outputs",
&format!(r#"{{"batch_id":123,"names":[{}]}}"#, names.join(",")),
);
assert_eq!(code, 400);
assert_eq!(hub.count("outputs:presign"), 0);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn no_claim_when_disabled_or_busy() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut busy = sched(&hub, &rt, &dir, &key, 1);
assert_eq!(busy.tick(), Tick::NoCapacity);
write_settings(
&dir,
&Settings {
enabled: false,
min_price_per_hour: None,
},
)
.unwrap();
let mut off = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(off.tick(), Tick::Disabled);
assert_eq!(hub.count("/claim"), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn idle_space_container_is_stopped() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.cfg.idle_timeout = Duration::from_millis(400);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let _ = request(
&sock(&dir),
"POST",
"/done",
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#,
);
std::thread::sleep(Duration::from_millis(1200));
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn spaces_network_is_pure_and_never_the_deploy_network() {
assert_eq!(spaces_network(None, "hubnet").0, "bridge");
assert_eq!(spaces_network(Some(" "), "hubnet").0, "bridge");
let (n, w) = spaces_network(Some("hubnet"), "hubnet");
assert_eq!(n, "bridge");
assert!(w.is_some());
assert_eq!(spaces_network(Some("host"), "hubnet").0, "bridge");
assert_eq!(
spaces_network(Some("container:broker"), "hubnet").0,
"bridge"
);
assert!(spaces_network(Some("container:abc"), "hubnet").1.is_some());
assert_eq!(
spaces_network(Some("none"), "hubnet"),
("none".into(), None)
);
assert_eq!(
spaces_network(Some("egress"), "hubnet"),
("egress".into(), None)
);
assert_eq!(
spaces_network(Some("bridge"), "bridge"),
("bridge".into(), None)
);
}
#[test]
fn space_container_runs_on_bridge_with_private_docker_config() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
let cfg = rt.cfg_dirs.lock().unwrap()[0].clone();
assert_eq!(cfg, crate::spaces::runner::docker_config_dir(&dir, SPACE));
assert!(cfg.exists());
s.shutdown();
assert!(!cfg.exists(), "private docker config removed on stop");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn stale_space_containers_are_removed_on_start() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
rt.orphans
.lock()
.unwrap()
.extend(["old-1".to_string(), "old-2".to_string()]);
let key = Arc::new(NodeKey::generate());
let _s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["old-1", "old-2"]);
assert_eq!(rt.removed.lock().unwrap().as_slice(), ["old-1", "old-2"]);
assert_eq!(
rt.list_keys.lock().unwrap().as_slice(),
[format!("zakuro.space.node={}", key.fingerprint())]
);
let _ = std::fs::remove_dir_all(&dir);
}
fn done_body_ok() -> &'static str {
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#
}
#[test]
fn polling_worker_does_not_keep_container_alive() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.cfg.idle_timeout = Duration::from_millis(400);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let _ = request(&sock(&dir), "POST", "/done", done_body_ok());
let t = std::time::Instant::now();
while t.elapsed() < Duration::from_millis(1500) && rt.stopped.lock().unwrap().is_empty() {
let _ = request(&sock(&dir), "GET", "/next", "");
std::thread::sleep(Duration::from_millis(50));
}
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn failed_start_backs_off_claims_with_doubling_delay() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
rt.fail_run.store(true, Ordering::SeqCst);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.cfg.backoff_base = Duration::from_millis(300);
s.cfg.backoff_cap = Duration::from_millis(500);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 0);
assert_eq!(s.backoff[SPACE].delay, Duration::from_millis(300));
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(hub.count("/claim"), 1);
std::thread::sleep(Duration::from_millis(350));
s.record_failure(SPACE);
assert_eq!(s.backoff[SPACE].delay, Duration::from_millis(500));
s.record_failure(SPACE);
assert_eq!(s.backoff[SPACE].delay, Duration::from_millis(500), "cap");
std::thread::sleep(Duration::from_millis(550));
assert_eq!(s.tick(), Tick::Claimed(0));
assert_eq!(hub.count("/claim"), 2);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_at_start_backs_off() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new("sha256:cccc");
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(hub.count("/claim"), 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_at_done_backs_off() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
*rt.digest.lock().unwrap() = "sha256:bbbb".to_string();
assert_eq!(request(&sock(&dir), "POST", "/done", done_body_ok()).0, 409);
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(hub.count("/claim"), 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn expired_active_lease_kills_the_container() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(2), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
assert!(rt.stopped.lock().unwrap().is_empty());
let t = std::time::Instant::now();
while rt.stopped.lock().unwrap().is_empty() && t.elapsed() < Duration::from_secs(5) {
std::thread::sleep(Duration::from_millis(50));
}
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(rt.removed.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(hub.count("/complete"), 0);
assert_eq!(s.tick(), Tick::BackedOff);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn next_hands_the_worker_the_lease_max_runtime() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
let (_, body) = request(&sock(&dir), "GET", "/next", "");
let v: Value = serde_json::from_str(&body).unwrap();
assert_eq!(v["max_runtime_s"], 600);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn container_is_hardened_and_named_and_labelled_per_node() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
let fp = key.fingerprint();
{
let specs = rt.specs.lock().unwrap();
let (name, labels, hardened) = &specs[0];
assert_eq!(name, &format!("zk-space-{fp}-{SPACE}"));
assert!(labels.contains(&("zakuro.space.node".to_string(), fp.clone())));
assert!(labels.contains(&("zakuro.space".to_string(), SPACE.to_string())));
assert!(*hardened);
}
let other = NodeKey::generate();
assert_ne!(other.fingerprint(), fp);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn capabilities_report_cuda_vram_and_no_metal_gpus() {
let gib = 1u64 << 30;
let cuda = |v: u64| GpuId {
vendor: "nvidia".into(),
model: "x".into(),
uuid: "u".into(),
vram_bytes: v * gib,
};
let c = capabilities(&[cuda(24), cuda(16)]);
assert_eq!(c["gpus"], 2);
assert_eq!(c["vram_gb"], 16.0, "per-GPU minimum");
let metal = GpuId {
vendor: "apple".into(),
model: "m".into(),
uuid: "u".into(),
vram_bytes: 48 * gib,
};
let c = capabilities(&[metal]);
assert_eq!(c["gpus"], 0);
assert_eq!(c["vram_gb"], 0.0);
assert_eq!(capabilities(&[])["gpus"], 0);
}
#[test]
fn gpu_enumeration_is_cached_across_ticks() {
let dir = temp_state();
let hub = hub_with(r#"{"leases":[]}"#.to_string());
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
let calls = Arc::new(AtomicUsize::new(0));
let c2 = calls.clone();
s.gpu_probe = Arc::new(move || {
c2.fetch_add(1, Ordering::SeqCst);
Some(vec![GpuId {
vendor: "nvidia".into(),
model: "x".into(),
uuid: "u".into(),
vram_bytes: 24 << 30,
}])
});
for _ in 0..3 {
assert_eq!(s.tick(), Tick::Claimed(0));
}
assert_eq!(calls.load(Ordering::SeqCst), 1);
let reqs = hub.requests.lock().unwrap();
let (_, _, b) = reqs.iter().find(|(_, p, _)| p.ends_with("/claim")).unwrap();
let v: Value = serde_json::from_str(b).unwrap();
assert_eq!(v["capabilities"]["gpus"], 1);
assert_eq!(v["capabilities"]["vram_gb"], 24.0);
drop(reqs);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn no_claim_while_benchmark_runs_and_container_count_is_published() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.bench_running.store(true, Ordering::SeqCst);
assert_eq!(s.tick(), Tick::BenchRunning);
assert_eq!(hub.count("/claim"), 0);
s.bench_running.store(false, Ordering::SeqCst);
assert_eq!(s.containers.load(Ordering::SeqCst), 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(
s.containers.load(Ordering::SeqCst),
1,
"bench sees a busy node"
);
s.shutdown();
assert_eq!(s.containers.load(Ordering::SeqCst), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn transient_complete_failures_are_retried_until_accepted() {
let dir = temp_state();
let lease = lease_json(&in_secs(300), 60);
let first = Arc::new(AtomicBool::new(true));
let completes = Arc::new(AtomicUsize::new(0));
let c2 = completes.clone();
let hub = stub_hub(move |_m, p, _b| {
if p.ends_with("/claim") {
if first.swap(false, Ordering::SeqCst) {
(200, lease.clone())
} else {
(200, r#"{"leases":[]}"#.to_string())
}
} else if p.ends_with("/complete") {
if c2.fetch_add(1, Ordering::SeqCst) < 2 {
(503, "{}".to_string())
} else {
(200, r#"{"status":"succeeded","paid":0.9}"#.to_string())
}
} else {
(404, "{}".to_string())
}
});
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let (code, _) = request(&sock(&dir), "POST", "/done", done_body_ok());
assert_eq!(code, 202, "503 is not final: the broker keeps the receipt");
let t = std::time::Instant::now();
while completes.load(Ordering::SeqCst) < 3 && t.elapsed() < Duration::from_secs(5) {
std::thread::sleep(Duration::from_millis(20));
}
assert_eq!(completes.load(Ordering::SeqCst), 3);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn final_complete_answers_are_not_retried() {
let dir = temp_state();
let lease = lease_json(&in_secs(300), 60);
let first = Arc::new(AtomicBool::new(true));
let hub = stub_hub(move |_m, p, _b| {
if p.ends_with("/claim") {
if first.swap(false, Ordering::SeqCst) {
(200, lease.clone())
} else {
(200, r#"{"leases":[]}"#.to_string())
}
} else if p.ends_with("/complete") {
(409, "{}".to_string())
} else {
(404, "{}".to_string())
}
});
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let (code, _) = request(&sock(&dir), "POST", "/done", done_body_ok());
assert_eq!(code, 409);
std::thread::sleep(Duration::from_millis(200));
assert_eq!(hub.count("/complete"), 1);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn bench_units_reads_saved_report() {
let dir = temp_state();
assert_eq!(bench_units(&dir), None);
std::fs::create_dir_all(dir.join("bench")).unwrap();
let report = r#"{"cpu_units":8.5,"gpu_units":null}"#;
std::fs::write(
dir.join("bench").join("last.json"),
json!({"report_json": report, "signature": "x"}).to_string(),
)
.unwrap();
assert_eq!(bench_units(&dir), Some(json!({"cpu": 8.5, "gpu": null})));
let report = r#"{"cpu_units":2.0,"gpu_units":30.5}"#;
std::fs::write(
dir.join("bench").join("last.json"),
json!({"report_json": report}).to_string(),
)
.unwrap();
assert_eq!(bench_units(&dir), Some(json!({"cpu": 2.0, "gpu": 30.5})));
let _ = std::fs::remove_dir_all(&dir);
}
}