use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use super::client::{Hub, Lease};
use super::preempt;
use super::runner::{is_expired, node_label_filter, RunnerDeps, Runtime, SpaceRunner};
use super::{read_settings, ActiveSpace, Stats};
use crate::bench_node::fingerprint::GpuId;
use crate::broker::node_identity::NodeKey;
use crate::broker::revenue::{lease_slots, Need, HUB_MAX_LEASES};
use crate::policy::admit::{Request, Source, Workload};
pub const CLAIM_EVERY: Duration = Duration::from_secs(15);
pub const IDLE_STOP: Duration = Duration::from_secs(600);
pub const BACKOFF_BASE: Duration = Duration::from_secs(60);
pub const BACKOFF_CAP: Duration = Duration::from_secs(30 * 60);
pub const GPU_REFRESH: Duration = Duration::from_secs(600);
pub struct Config {
pub state_dir: PathBuf,
pub max_leases: usize,
pub max_leases_explicit: bool,
pub max_inflight: u64,
pub max_inflight_explicit: bool,
pub idle_timeout: Duration,
pub network: String,
pub backoff_base: Duration,
pub backoff_cap: Duration,
pub complete_retry_base: Duration,
}
impl Config {
pub fn from_env(state_dir: PathBuf) -> Config {
let num = |k: &str, d: u64| {
std::env::var(k)
.ok()
.and_then(|v| v.trim().parse().ok())
.unwrap_or(d)
};
Config {
state_dir,
max_leases: num("ZAKURO_SPACES_MAX_LEASES", 1).max(1) as usize,
max_leases_explicit: std::env::var("ZAKURO_SPACES_MAX_LEASES").is_ok(),
max_inflight: num("ZAKURO_SPACES_MAX_INFLIGHT", 1),
max_inflight_explicit: std::env::var("ZAKURO_SPACES_MAX_INFLIGHT").is_ok(),
idle_timeout: IDLE_STOP,
backoff_base: BACKOFF_BASE,
backoff_cap: BACKOFF_CAP,
complete_retry_base: Duration::from_secs(1),
network: {
let (net, warn) = spaces_network(
std::env::var("ZAKURO_SPACES_NETWORK").ok().as_deref(),
&crate::broker::deploy::runtime::deploy_network(),
);
if let Some(w) = warn {
eprintln!(" [SPACES] {w}");
}
net
},
}
}
}
pub fn spaces_network(requested: Option<&str>, deploy_network: &str) -> (String, Option<String>) {
let req = requested.map(str::trim).filter(|s| !s.is_empty());
match req {
None => ("bridge".to_string(), None),
Some(n)
if n != "bridge"
&& (n == "host" || n.starts_with("container:") || n == deploy_network) =>
{
(
"bridge".to_string(),
Some(format!(
"ZAKURO_SPACES_NETWORK={n} refused (mesh/host access); using bridge"
)),
)
}
Some(n) => (n.to_string(), None),
}
}
pub fn bench_units(dir: &std::path::Path) -> Option<Value> {
let last: Value =
serde_json::from_slice(&std::fs::read(dir.join("bench").join("last.json")).ok()?).ok()?;
let report: Value = serde_json::from_str(last.get("report_json")?.as_str()?).ok()?;
let cpu = report.get("cpu_units")?.as_f64()?;
let gpu = report.get("gpu_units").and_then(|g| g.as_f64());
Some(json!({ "cpu": cpu, "gpu": gpu }))
}
pub struct Scheduler {
hub: Hub,
key: Arc<NodeKey>,
runtime: Runtime,
cfg: Config,
inflight: Arc<dyn Fn() -> u64 + Send + Sync>,
stats: Arc<Stats>,
runners: HashMap<String, SpaceRunner>,
node_fp: String,
gpu_probe: GpuProbe,
gpus: Vec<GpuId>,
gpus_at: Option<Instant>,
backoff: HashMap<String, Backoff>,
bench_running: Arc<AtomicBool>,
containers: Arc<AtomicUsize>,
gate: Option<crate::policy::Admission>,
revenue: Option<crate::broker::revenue::Revenue>,
typical: Need,
preempt_grace: Duration,
}
#[derive(Debug, PartialEq)]
pub enum Tick {
Disabled,
Refused(&'static str),
NoCapacity,
BackedOff,
BenchRunning,
NoRuntime,
ClaimFailed(String),
Claimed(usize),
}
fn skip_reason(t: &Tick) -> Option<&'static str> {
match t {
Tick::Disabled => Some("disabled (workloads.spaces in policy.toml)"),
Tick::Refused(_) => Some("refused by the node policy (`zc policy show`)"),
Tick::NoCapacity => Some("at capacity (all Space slots or in-flight requests busy)"),
Tick::BackedOff => Some("paused after a failed Space (backoff)"),
Tick::BenchRunning => Some("the node benchmark is measuring"),
Tick::NoRuntime => Some(
"no container runtime: `docker` is missing or $DOCKER_HOST is unreachable/forbidden",
),
Tick::ClaimFailed(_) | Tick::Claimed(_) => None,
}
}
fn state_change(prev: Option<&Tick>, cur: &Tick) -> Option<String> {
let (was, now) = (prev.map(skip_reason), skip_reason(cur));
match (was, now) {
(Some(a), b) if a == b => None,
(None, Some(_)) if matches!(cur, Tick::Disabled) => None,
(_, Some(why)) => Some(format!(" [SPACES] not claiming: {why}")),
(Some(Some(_)), None) => Some(" [SPACES] claiming".to_string()),
_ => None,
}
}
pub(crate) fn memory_gb() -> f64 {
#[cfg(target_os = "linux")]
{
if let Ok(t) = std::fs::read_to_string("/proc/meminfo") {
for l in t.lines() {
if let Some(r) = l.strip_prefix("MemTotal:") {
let kb: f64 = r
.trim()
.trim_end_matches("kB")
.trim()
.parse()
.unwrap_or(0.0);
return kb / 1024.0 / 1024.0;
}
}
}
}
#[cfg(target_os = "macos")]
{
if let Ok(o) = std::process::Command::new("sysctl")
.args(["-n", "hw.memsize"])
.output()
{
if let Ok(b) = String::from_utf8_lossy(&o.stdout).trim().parse::<f64>() {
return b / 1024.0 / 1024.0 / 1024.0;
}
}
}
0.0
}
pub fn capabilities(gpus: &[GpuId]) -> Value {
let cpus = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
let cuda: Vec<&GpuId> = gpus.iter().filter(|g| g.vendor == "nvidia").collect();
let vram_gb = cuda
.iter()
.map(|g| g.vram_bytes)
.min()
.map(|b| ((b as f64 / (1u64 << 30) as f64) * 10.0).floor() / 10.0)
.unwrap_or(0.0);
json!({
"cpus": cpus,
"memory_gb": (memory_gb() * 10.0).round() / 10.0,
"gpus": cuda.len(),
"vram_gb": vram_gb,
})
}
pub type GpuProbe = Arc<dyn Fn() -> Option<Vec<GpuId>> + Send + Sync>;
pub fn real_gpu_probe() -> GpuProbe {
Arc::new(|| crate::bench_node::run::enumerate_with_timeout(&AtomicBool::new(false)))
}
#[derive(Debug, Clone, Copy)]
struct Backoff {
until: Instant,
delay: Duration,
}
impl Scheduler {
pub fn new(
hub: Hub,
key: Arc<NodeKey>,
runtime: Runtime,
cfg: Config,
inflight: Arc<dyn Fn() -> u64 + Send + Sync>,
) -> Scheduler {
let stats = Arc::new(Stats::new(&cfg.state_dir));
let node_fp = key.fingerprint();
for id in runtime.list_by_label_key(&node_label_filter(&node_fp)) {
eprintln!(" [SPACES] removing orphaned Space container {id}");
let _ = runtime.stop(&id);
let _ = runtime.rm(&id);
}
Scheduler {
hub,
key,
runtime,
cfg,
inflight,
stats,
runners: HashMap::new(),
node_fp,
gpu_probe: real_gpu_probe(),
gpus: Vec::new(),
gpus_at: None,
backoff: HashMap::new(),
bench_running: super::bench_running(),
containers: super::space_containers(),
gate: None,
revenue: None,
typical: Need {
cpus: 0.0,
memory_bytes: 0,
gpus: 0,
},
preempt_grace: preempt::grace(),
}
}
pub fn with_revenue(mut self, revenue: crate::broker::revenue::Revenue) -> Self {
self.revenue = Some(revenue);
self
}
pub fn with_policy(mut self, gate: crate::policy::Admission) -> Self {
self.gate = Some(gate);
self
}
fn gpus(&mut self) -> &[GpuId] {
if self.gpus_at.is_none_or(|t| t.elapsed() >= GPU_REFRESH) {
if let Some(g) = (self.gpu_probe)() {
self.gpus = g;
}
self.gpus_at = Some(Instant::now());
}
&self.gpus
}
fn record_failure(&mut self, space_id: &str) {
let delay = match self.backoff.get(space_id) {
Some(b) => (b.delay * 2).min(self.cfg.backoff_cap),
None => self.cfg.backoff_base.min(self.cfg.backoff_cap),
};
eprintln!(" [SPACES] space {space_id} failed; pausing claims for {delay:?}");
self.backoff.insert(
space_id.to_string(),
Backoff {
until: Instant::now() + delay,
delay,
},
);
}
fn in_backoff(&self, space_id: &str) -> bool {
self.backoff
.get(space_id)
.is_some_and(|b| b.until > Instant::now())
}
fn reap_runners(&mut self) {
let mut failed = Vec::new();
let mut healthy = Vec::new();
for (id, r) in &self.runners {
if r.session.completions() > 0 && !r.session.failed() {
healthy.push(id.clone());
}
if r.finished() && r.session.failed() {
failed.push(id.clone());
}
}
for id in healthy {
self.backoff.remove(&id);
}
for id in failed {
self.record_failure(&id);
}
self.runners.retain(|_, r| !r.finished());
self.containers.store(self.runners.len(), Ordering::SeqCst);
}
fn outstanding(&self) -> usize {
self.runners.values().map(|r| r.session.outstanding()).sum()
}
fn left_budget(&self) -> Option<(u64, u64)> {
let budget = self.gate.as_ref()?.budget_limits()?;
let running = self.runtime.allocations().unwrap_or_default();
let (mut cpu, mut mem): (u64, u64) = (
running.iter().map(|a| a.cpu_millis as u64).sum(),
running.iter().map(|a| a.memory_bytes).sum(),
);
if let Some(r) = &self.revenue {
let h = r.held();
cpu += (h.cpus * 1000.0).round() as u64;
mem += h.memory_bytes;
}
Some((
(budget.cpu_millis as u64).saturating_sub(cpu),
budget.memory_bytes.saturating_sub(mem),
))
}
fn preempt_pass(&mut self) {
let Some(gate) = self.gate.clone() else {
return;
};
if self.runners.is_empty() {
return;
}
let now = chrono::Utc::now().timestamp();
let cands: Vec<preempt::Candidate> = self
.runners
.iter()
.filter(|(_, r)| !r.finished())
.map(|(id, r)| preempt::candidate(id, &r.session.lease_infos(), now))
.collect();
let grace = self.preempt_grace;
let (why, picks) = if let Some(reason) = gate.owner_yield() {
eprintln!(" [SPACES] the owner is back ({reason}): stopping running Spaces");
("owner_active", preempt::pick_owner_active(&cands, grace))
} else if let Some(want) = self.revenue.as_ref().and_then(|r| r.take_room_wanted()) {
let Some(left) = self.left_budget() else {
return;
};
(
"better_offer",
preempt::pick_better_offer(&cands, grace, &want.need, want.price_per_hour, left),
)
} else {
return;
};
for id in picks {
let Some(r) = self.runners.remove(&id) else {
continue;
};
let leases = r.session.outstanding() as u64;
r.shutdown();
eprintln!(
" [SPACES] preempted space {id} ({why}): {leases} lease(s) released; the hub re-leases them after expiry"
);
if let Some(rev) = &self.revenue {
rev.record_preempted(why, leases);
}
}
self.containers.store(self.runners.len(), Ordering::SeqCst);
}
fn deps(&self) -> RunnerDeps {
RunnerDeps {
hub: self.hub.clone(),
runtime: self.runtime.clone(),
node_key: self.key.clone(),
state_dir: self.cfg.state_dir.clone(),
stats: self.stats.clone(),
network: self.cfg.network.clone(),
idle_timeout: self.cfg.idle_timeout,
node_fp: self.node_fp.clone(),
complete_retry_base: self.cfg.complete_retry_base,
}
}
pub fn runner_count(&self) -> usize {
self.runners.len()
}
fn publish_status(&self) {
let active = self
.runners
.iter()
.map(|(id, r)| {
let (queued, running) = r.session.counts();
ActiveSpace {
space_id: id.clone(),
queued,
running,
}
})
.collect();
self.stats.set_active(active);
}
pub fn tick(&mut self) -> Tick {
self.reap_runners();
self.preempt_pass();
let enabled = match &self.gate {
Some(g) => g.workload_enabled(&Workload::Space),
None => read_settings(&self.cfg.state_dir).enabled,
};
if !enabled {
self.publish_status();
return Tick::Disabled;
}
if let Some(g) = &self.gate {
if let Err(r) = g.check(&Request::new(Source::SpacesLease, Workload::Space)) {
self.publish_status();
return Tick::Refused(r.reason.as_str());
}
}
let (spare, budgeted) = self.spare();
let inflight_gate = !budgeted || self.cfg.max_inflight_explicit;
if spare == 0 || (inflight_gate && (self.inflight)() >= self.cfg.max_inflight) {
self.publish_status();
return Tick::NoCapacity;
}
if self.bench_running.load(Ordering::SeqCst) {
self.publish_status();
return Tick::BenchRunning;
}
if self.backoff.values().any(|b| b.until > Instant::now()) {
self.publish_status();
return Tick::BackedOff;
}
if !self.runtime.available() {
return Tick::NoRuntime;
}
let caps = {
let gpus = self.gpus().to_vec();
capabilities(&gpus)
};
let leases = match self.hub.claim(
&self.key.public_b64(),
&caps,
bench_units(&self.cfg.state_dir),
spare as u32,
) {
Ok(l) => l,
Err(e) => return Tick::ClaimFailed(e),
};
let n = leases.len();
if n == 0 && budgeted {
if let Some(r) = &self.revenue {
r.eligible_idle(Source::SpacesLease, CLAIM_EVERY);
}
}
for lease in leases {
self.accept(lease);
}
self.containers.store(self.runners.len(), Ordering::SeqCst);
self.publish_status();
Tick::Claimed(n)
}
fn spare(&self) -> (usize, bool) {
let outstanding = self.outstanding();
let counter = self.cfg.max_leases.saturating_sub(outstanding);
let Some(budget) = self.gate.as_ref().and_then(|g| g.budget_limits()) else {
return (counter, false);
};
let ceiling = if self.cfg.max_leases_explicit {
self.cfg.max_leases
} else {
HUB_MAX_LEASES
};
let running = self.runtime.allocations().unwrap_or_default();
let (mut cpu, mut mem): (u64, u64) = (
running.iter().map(|a| a.cpu_millis as u64).sum(),
running.iter().map(|a| a.memory_bytes).sum(),
);
if let Some(r) = &self.revenue {
let h = r.held();
cpu += (h.cpus * 1000.0).round() as u64;
mem += h.memory_bytes;
}
let unit_cpu = ((self.typical.cpus * 1000.0).round() as u64).max(1000);
let unit_mem = self.typical.memory_bytes.max(1 << 30);
cpu = cpu.max(outstanding as u64 * unit_cpu);
mem = mem.max(outstanding as u64 * unit_mem);
let slots = lease_slots(
&budget,
cpu,
mem,
&self.typical,
ceiling.saturating_sub(outstanding),
);
(slots, true)
}
fn accept(&mut self, lease: Lease) {
let req = &lease.requirements;
self.typical = Need {
cpus: req.cpus,
memory_bytes: (req.memory_gb * (1u64 << 30) as f64) as u64,
gpus: req.gpus,
};
if is_expired(&lease) {
eprintln!(
" [SPACES] lease {} already expired; abandoned",
lease.lease_id
);
return;
}
if let Some(r) = self.runners.get(&lease.space_id) {
if !r.finished() {
r.session.enqueue(lease);
return;
}
}
if let Some(g) = &self.gate {
let req = &lease.requirements;
let mut r = Request::new(Source::SpacesLease, Workload::Space);
r.cpus = req.cpus;
r.memory_bytes = (req.memory_gb * (1u64 << 30) as f64) as u64;
r.gpus = req.gpus;
if let Err(f) = g.check(&r) {
eprintln!(
" [SPACES] lease {} refused by the node policy ({}): {}; left to expire",
lease.lease_id,
f.reason.as_str(),
f.detail
);
return;
}
}
if self.in_backoff(&lease.space_id) {
eprintln!(
" [SPACES] space {} is backing off; lease {} left to expire",
lease.space_id, lease.lease_id
);
return;
}
match SpaceRunner::start(&self.deps(), &lease) {
Ok(r) => {
r.session.enqueue(lease.clone());
self.runners.insert(lease.space_id.clone(), r);
}
Err(e) => {
eprintln!(
" [SPACES] could not start space {}: {e} (lease {} left to expire)",
lease.space_id, lease.lease_id
);
self.record_failure(&lease.space_id);
}
}
}
pub fn shutdown(&mut self) {
for r in self.runners.values() {
r.shutdown();
}
self.runners.clear();
self.containers.store(0, Ordering::SeqCst);
}
}
impl Drop for Scheduler {
fn drop(&mut self) {
self.shutdown();
}
}
pub fn spawn_once(state: Arc<crate::broker::BrokerState>) {
static STARTED: AtomicBool = AtomicBool::new(false);
let (Some(api_url), Some(api_key)) =
(state.config.api_url.clone(), state.config.api_key.clone())
else {
return;
};
let Some(dir) = crate::credentials::dir() else {
return;
};
if STARTED.swap(true, Ordering::SeqCst) {
return;
}
let key = state.node_key.clone();
let st = state.clone();
let _ = std::thread::Builder::new()
.name("spaces-sched".into())
.spawn(move || {
let gate = st.admission.clone();
let rev = st.revenue.clone();
let mut s = Scheduler::new(
Hub::new(&api_url, &api_key, &key.public_b64()),
key,
Arc::new(crate::broker::deploy::DockerCli),
Config::from_env(dir),
Arc::new(move || st.inflight.load(Ordering::SeqCst)),
)
.with_policy(gate)
.with_revenue(rev);
let mut last: Option<Tick> = None;
loop {
let t = s.tick();
if let Some(line) = state_change(last.as_ref(), &t) {
println!("{line}");
}
match &t {
Tick::ClaimFailed(e) => eprintln!(" [SPACES] claim failed: {e}"),
Tick::Claimed(n) if *n > 0 => println!(" [SPACES] claimed {n} lease(s)"),
_ => {}
}
last = Some(t);
std::thread::sleep(CLAIM_EVERY);
}
});
}
#[cfg(test)]
mod tests {
#[test]
fn skip_reason_is_logged_once_per_change_not_every_tick() {
let first = state_change(None, &Tick::NoRuntime).expect("first skip is logged");
assert!(
first.contains("not claiming") && first.contains("docker"),
"{first}"
);
assert_eq!(state_change(Some(&Tick::NoRuntime), &Tick::NoRuntime), None);
let other = state_change(Some(&Tick::NoRuntime), &Tick::BackedOff).unwrap();
assert!(other.contains("backoff"), "{other}");
let back = state_change(Some(&Tick::BackedOff), &Tick::Claimed(0)).unwrap();
assert!(back.contains("claiming"), "{back}");
assert_eq!(
state_change(Some(&Tick::Claimed(0)), &Tick::Claimed(2)),
None
);
assert_eq!(state_change(None, &Tick::Disabled), None);
assert!(state_change(Some(&Tick::Claimed(0)), &Tick::Disabled)
.unwrap()
.contains("disabled"));
}
use super::*;
use crate::broker::deploy::{ContainerRuntime, RunSpec};
use crate::broker::node_identity::verify_sig;
use crate::spaces::socket::request;
use crate::spaces::testutil::{stub_hub, StubHub};
use crate::spaces::{write_settings, Settings};
use std::path::Path;
use std::sync::Mutex;
const DIGEST: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
const SPACE: &str = "22222222-2222-4222-8222-222222222222";
type RunRec = (String, Vec<(String, String)>, HashMap<String, String>, f64);
type SpecRec = (String, Vec<(String, String)>, bool);
struct FakeRt {
digest: Mutex<String>,
logins: Mutex<Vec<String>>,
pulls: Mutex<Vec<String>>,
runs: Mutex<Vec<RunRec>>,
stopped: Mutex<Vec<String>>,
removed: Mutex<Vec<String>>,
orphans: Mutex<Vec<String>>,
cfg_dirs: Mutex<Vec<PathBuf>>,
specs: Mutex<Vec<SpecRec>>,
list_keys: Mutex<Vec<String>>,
fail_run: AtomicBool,
budget: Mutex<Result<crate::sandbox::Limits, String>>,
running: Mutex<Vec<crate::sandbox::Allocation>>,
}
impl FakeRt {
fn new(digest: &str) -> Arc<FakeRt> {
Arc::new(FakeRt {
digest: Mutex::new(digest.to_string()),
logins: Mutex::new(vec![]),
pulls: Mutex::new(vec![]),
runs: Mutex::new(vec![]),
stopped: Mutex::new(vec![]),
removed: Mutex::new(vec![]),
orphans: Mutex::new(vec![]),
cfg_dirs: Mutex::new(vec![]),
specs: Mutex::new(vec![]),
list_keys: Mutex::new(vec![]),
fail_run: AtomicBool::new(false),
budget: Mutex::new(Ok(crate::sandbox::Limits::new(
8.0,
16.0,
crate::sandbox::Gpus::None,
))),
running: Mutex::new(vec![]),
})
}
}
impl ContainerRuntime for FakeRt {
fn available(&self) -> bool {
true
}
fn pull(&self, image: &str) -> Result<(), String> {
self.pulls.lock().unwrap().push(image.to_string());
Ok(())
}
fn budget(&self) -> Result<crate::sandbox::Limits, String> {
self.budget.lock().unwrap().clone()
}
fn allocations(&self) -> Result<Vec<crate::sandbox::Allocation>, String> {
Ok(self.running.lock().unwrap().clone())
}
fn run(&self, spec: &RunSpec) -> Result<String, String> {
if self.fail_run.load(Ordering::SeqCst) {
return Err("docker run failed".to_string());
}
self.specs.lock().unwrap().push((
spec.name.clone(),
spec.labels.clone(),
spec.hardened,
));
self.runs.lock().unwrap().push((
spec.image.to_string(),
spec.mounts.clone(),
spec.env.clone(),
spec.limits.cpus(),
));
Ok("cont-1".to_string())
}
fn stop(&self, id: &str) -> Result<(), String> {
self.stopped.lock().unwrap().push(id.to_string());
Ok(())
}
fn rm(&self, id: &str) -> Result<(), String> {
self.removed.lock().unwrap().push(id.to_string());
Ok(())
}
fn list_by_label_key(&self, k: &str) -> Vec<String> {
self.list_keys.lock().unwrap().push(k.to_string());
self.orphans.lock().unwrap().clone()
}
fn login_in(&self, dir: &Path, r: &str, u: &str, pw: &str) -> Result<(), String> {
self.cfg_dirs.lock().unwrap().push(dir.to_path_buf());
assert!(dir.exists(), "config dir exists during login");
self.login(r, u, pw)
}
fn restart(&self, _id: &str) -> Result<(), String> {
Ok(())
}
fn is_running(&self, _id: &str) -> bool {
true
}
fn container_ip(&self, _id: &str) -> Result<String, String> {
Ok("1.2.3.4".into())
}
fn health_check(&self, _id: &str, _hc: &crate::broker::deploy::Healthcheck) -> bool {
true
}
fn logs_tail(&self, _id: &str, _n: u32) -> String {
String::new()
}
fn list_by_label(&self, _id: &str) -> Vec<String> {
vec![]
}
fn login(&self, registry: &str, user: &str, _pw: &str) -> Result<(), String> {
self.logins
.lock()
.unwrap()
.push(format!("{registry}/{user}"));
Ok(())
}
fn image_digest(&self, _id: &str) -> Result<String, String> {
Ok(self.digest.lock().unwrap().clone())
}
}
static N: AtomicUsize = AtomicUsize::new(0);
fn temp_state() -> PathBuf {
let d = PathBuf::from(format!(
"/tmp/zksp{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::SeqCst)
));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
write_settings(
&d,
&Settings {
enabled: true,
min_price_per_hour: None,
},
)
.unwrap();
d
}
fn in_secs(s: i64) -> String {
(chrono::Utc::now() + chrono::Duration::seconds(s))
.format("%Y-%m-%dT%H:%M:%SZ")
.to_string()
}
fn lease_json(deadline: &str, hb: u64) -> String {
format!(
r#"{{"leases":[{{"lease_id":"11111111-1111-4111-8111-111111111111","batch_id":123,"space_id":"{SPACE}","seq":7,
"image":{{"mirror_ref":"registry:5000/spaces/{SPACE}@{DIGEST}","digest":"{DIGEST}"}},
"pull":{{"registry":"registry:5000","username":"zk-lease","token":"tok"}},
"inputs":[{{"name":"a.fastq","url":"https://example.com/a?sig=1"}}],
"requirements":{{"cpus":4,"memory_gb":8,"gpus":0,"min_vram_gb":0,"max_runtime_s":600,"est_runtime_s":120}},
"deadline":"{deadline}","heartbeat_every_s":{hb}}}]}}"#
)
}
fn hub_with(lease: String) -> StubHub {
let first = Arc::new(std::sync::atomic::AtomicBool::new(true));
stub_hub(move |_m, p, _b| {
if p.ends_with("/claim") {
if first.swap(false, Ordering::SeqCst) {
(200, lease.clone())
} else {
(200, r#"{"leases":[]}"#.to_string())
}
} else if p.ends_with("/heartbeat") {
(200, "{}".to_string())
} else if p.ends_with("outputs:presign") {
(
200,
r#"{"urls":{"out.txt":"https://put.example/out.txt"}}"#.to_string(),
)
} else if p.ends_with("/complete") {
(200, r#"{"status":"succeeded","paid":0.9}"#.to_string())
} else {
(404, "{}".to_string())
}
})
}
fn sched(
hub: &StubHub,
rt: &Arc<FakeRt>,
dir: &Path,
key: &Arc<NodeKey>,
inflight: u64,
) -> Scheduler {
let mut cfg = Config::from_env(dir.to_path_buf());
cfg.max_leases = 1;
cfg.max_inflight = 1;
cfg.idle_timeout = Duration::from_secs(60);
cfg.complete_retry_base = Duration::from_millis(20);
let mut s = Scheduler::new(
Hub::new(&hub.url, "zk_test", &key.public_b64()),
key.clone(),
rt.clone() as Runtime,
cfg,
Arc::new(move || inflight),
);
s.bench_running = Arc::default();
s.containers = Arc::default();
s.gpu_probe = Arc::new(|| Some(vec![]));
s
}
fn sock(dir: &Path) -> PathBuf {
crate::spaces::runner::socket_path(dir, SPACE)
}
#[test]
fn claim_pull_next_outputs_done_complete() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(
rt.pulls.lock().unwrap().as_slice(),
[format!("registry:5000/spaces/{SPACE}@{DIGEST}")]
);
assert_eq!(
rt.logins.lock().unwrap().as_slice(),
["registry:5000/zk-lease"]
);
{
let runs = rt.runs.lock().unwrap();
let (_, mounts, env, cpus) = &runs[0];
assert_eq!(mounts[0].1, "/run/zakuro/space.sock");
assert_eq!(env["ZAKURO_SPACE_ID"], SPACE);
assert_eq!(env["ZAKURO_SPACE_SOCKET"], "/run/zakuro/space.sock");
assert_eq!(*cpus, 4.0);
}
let (code, body) = request(&sock(&dir), "GET", "/next", "");
assert_eq!(code, 200, "{body}");
let v: Value = serde_json::from_str(&body).unwrap();
assert_eq!(v["batch_id"], 123);
assert_eq!(v["inputs"][0]["name"], "a.fastq");
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 204);
let (code, body) = request(
&sock(&dir),
"POST",
"/outputs",
r#"{"batch_id":123,"names":["out.txt"]}"#,
);
assert_eq!(code, 200);
assert!(body.contains("https://put.example/out.txt"));
assert_eq!(hub.count("outputs:presign"), 1);
hub.assert_pubkey("outputs:presign", &key.public_b64());
hub.assert_pubkey("/claim", &key.public_b64());
let hash = "ab".repeat(32);
let (code, body) = request(
&sock(&dir),
"POST",
"/done",
&format!(
r#"{{"batch_id":123,"output_hashes":{{"out.txt":"{hash}"}},"exit_code":0,"stderr_tail":""}}"#
),
);
assert_eq!(code, 200, "{body}");
assert!(body.contains("\"accepted\":true"));
let reqs = hub.requests.lock().unwrap();
let (_, path, cbody) = reqs
.iter()
.find(|(_, p, _)| p.ends_with("/complete"))
.unwrap();
assert!(path.contains("11111111-1111-4111-8111-111111111111"));
let c: Value = serde_json::from_str(cbody).unwrap();
let rj = c["receipt_json"].as_str().unwrap();
assert!(verify_sig(
&key.public_b64(),
rj.as_bytes(),
c["signature"].as_str().unwrap()
));
let r: Value = serde_json::from_str(rj).unwrap();
assert_eq!(r["image_digest"], DIGEST);
assert_eq!(r["output_hashes"]["out.txt"], hash);
assert_eq!(r["exit_code"], 0);
drop(reqs);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_never_reports_and_kills() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
*rt.digest.lock().unwrap() = "sha256:bbbb".to_string();
let (code, body) = request(
&sock(&dir),
"POST",
"/done",
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#,
);
assert_eq!(code, 409);
assert!(body.contains("digest mismatch"));
assert_eq!(hub.count("/complete"), 0);
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_at_start_kills_and_takes_no_work() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new("sha256:cccc");
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 0);
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(hub.count("/complete"), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn heartbeats_are_sent_every_period_while_a_batch_runs() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 1));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
std::thread::sleep(Duration::from_millis(2600));
assert!(
hub.count("/heartbeat") >= 2,
"got {}",
hub.count("/heartbeat")
);
hub.assert_pubkey("/heartbeat", &key.public_b64());
let before = hub.count("/heartbeat");
let (code, _) = request(
&sock(&dir),
"POST",
"/done",
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#,
);
assert_eq!(code, 200);
std::thread::sleep(Duration::from_millis(1500));
assert_eq!(hub.count("/heartbeat"), before, "no heartbeats once idle");
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn expired_lease_is_abandoned_and_reports_nothing() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(2), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
std::thread::sleep(Duration::from_millis(2600));
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(hub.count("/complete"), 0);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn lease_already_expired_at_claim_starts_nothing() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(-30), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert!(rt.runs.lock().unwrap().is_empty());
assert_eq!(s.runner_count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn outputs_over_64_names_is_rejected() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let names: Vec<String> = (0..65).map(|i| format!("\"f{i}\"")).collect();
let (code, _) = request(
&sock(&dir),
"POST",
"/outputs",
&format!(r#"{{"batch_id":123,"names":[{}]}}"#, names.join(",")),
);
assert_eq!(code, 400);
assert_eq!(hub.count("outputs:presign"), 0);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn no_claim_when_disabled_or_busy() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut busy = sched(&hub, &rt, &dir, &key, 1);
assert_eq!(busy.tick(), Tick::NoCapacity);
write_settings(
&dir,
&Settings {
enabled: false,
min_price_per_hour: None,
},
)
.unwrap();
let mut off = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(off.tick(), Tick::Disabled);
assert_eq!(hub.count("/claim"), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn idle_space_container_is_stopped() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.cfg.idle_timeout = Duration::from_millis(400);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let _ = request(
&sock(&dir),
"POST",
"/done",
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#,
);
std::thread::sleep(Duration::from_millis(1200));
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn spaces_network_is_pure_and_never_the_deploy_network() {
assert_eq!(spaces_network(None, "hubnet").0, "bridge");
assert_eq!(spaces_network(Some(" "), "hubnet").0, "bridge");
let (n, w) = spaces_network(Some("hubnet"), "hubnet");
assert_eq!(n, "bridge");
assert!(w.is_some());
assert_eq!(spaces_network(Some("host"), "hubnet").0, "bridge");
assert_eq!(
spaces_network(Some("container:broker"), "hubnet").0,
"bridge"
);
assert!(spaces_network(Some("container:abc"), "hubnet").1.is_some());
assert_eq!(
spaces_network(Some("none"), "hubnet"),
("none".into(), None)
);
assert_eq!(
spaces_network(Some("egress"), "hubnet"),
("egress".into(), None)
);
assert_eq!(
spaces_network(Some("bridge"), "bridge"),
("bridge".into(), None)
);
}
#[test]
fn space_container_runs_on_bridge_with_private_docker_config() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
let cfg = rt.cfg_dirs.lock().unwrap()[0].clone();
assert_eq!(cfg, crate::spaces::runner::docker_config_dir(&dir, SPACE));
assert!(cfg.exists());
s.shutdown();
assert!(!cfg.exists(), "private docker config removed on stop");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn stale_space_containers_are_removed_on_start() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
rt.orphans
.lock()
.unwrap()
.extend(["old-1".to_string(), "old-2".to_string()]);
let key = Arc::new(NodeKey::generate());
let _s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["old-1", "old-2"]);
assert_eq!(rt.removed.lock().unwrap().as_slice(), ["old-1", "old-2"]);
assert_eq!(
rt.list_keys.lock().unwrap().as_slice(),
[format!("zakuro.space.node={}", key.fingerprint())]
);
let _ = std::fs::remove_dir_all(&dir);
}
fn done_body_ok() -> &'static str {
r#"{"batch_id":123,"output_hashes":{},"exit_code":0}"#
}
#[test]
fn polling_worker_does_not_keep_container_alive() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.cfg.idle_timeout = Duration::from_millis(400);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let _ = request(&sock(&dir), "POST", "/done", done_body_ok());
let t = std::time::Instant::now();
while t.elapsed() < Duration::from_millis(1500) && rt.stopped.lock().unwrap().is_empty() {
let _ = crate::spaces::socket::try_request(&sock(&dir), "GET", "/next", "");
std::thread::sleep(Duration::from_millis(50));
}
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn failed_start_backs_off_claims_with_doubling_delay() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
rt.fail_run.store(true, Ordering::SeqCst);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.cfg.backoff_base = Duration::from_millis(300);
s.cfg.backoff_cap = Duration::from_millis(500);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 0);
assert_eq!(s.backoff[SPACE].delay, Duration::from_millis(300));
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(hub.count("/claim"), 1);
std::thread::sleep(Duration::from_millis(350));
s.record_failure(SPACE);
assert_eq!(s.backoff[SPACE].delay, Duration::from_millis(500));
s.record_failure(SPACE);
assert_eq!(s.backoff[SPACE].delay, Duration::from_millis(500), "cap");
std::thread::sleep(Duration::from_millis(550));
assert_eq!(s.tick(), Tick::Claimed(0));
assert_eq!(hub.count("/claim"), 2);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_lease_asking_past_the_budget_is_clamped_to_it() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
*rt.budget.lock().unwrap() = Ok(crate::sandbox::Limits::new(
2.0,
4.0,
crate::sandbox::Gpus::None,
));
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(rt.runs.lock().unwrap()[0].3, 2.0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_lease_past_what_is_left_of_the_budget_is_refused() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let held = crate::sandbox::Limits::new(5.0, 4.0, crate::sandbox::Gpus::None);
*rt.running.lock().unwrap() =
vec![
crate::sandbox::Allocation::from_label("zk-dep-a-v1", &held.budget_label())
.unwrap(),
];
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 0);
assert!(rt.runs.lock().unwrap().is_empty(), "no docker run");
assert!(s.backoff.contains_key(SPACE), "the refused start backs off");
assert!(!sock(&dir).exists(), "the socket is removed again");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_invalid_budget_refuses_the_lease_without_running_it() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
*rt.budget.lock().unwrap() =
Err("ZAKURO_BUDGET_CPU=\"lots\" is not a cpu count or a percentage".into());
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 0);
assert!(rt.runs.lock().unwrap().is_empty(), "no docker run");
assert!(s.backoff.contains_key(SPACE), "the failed start backs off");
assert!(!sock(&dir).exists(), "the socket is removed again");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_at_start_backs_off() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new("sha256:cccc");
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(hub.count("/claim"), 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn digest_mismatch_at_done_backs_off() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
*rt.digest.lock().unwrap() = "sha256:bbbb".to_string();
assert_eq!(request(&sock(&dir), "POST", "/done", done_body_ok()).0, 409);
assert_eq!(s.tick(), Tick::BackedOff);
assert_eq!(hub.count("/claim"), 1);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn expired_active_lease_kills_the_container() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(2), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
assert!(rt.stopped.lock().unwrap().is_empty());
let t = std::time::Instant::now();
while rt.stopped.lock().unwrap().is_empty() && t.elapsed() < Duration::from_secs(5) {
std::thread::sleep(Duration::from_millis(50));
}
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(rt.removed.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(hub.count("/complete"), 0);
assert_eq!(s.tick(), Tick::BackedOff);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn next_hands_the_worker_the_lease_max_runtime() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
let (_, body) = request(&sock(&dir), "GET", "/next", "");
let v: Value = serde_json::from_str(&body).unwrap();
assert_eq!(v["max_runtime_s"], 600);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn container_is_hardened_and_named_and_labelled_per_node() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
let fp = key.fingerprint();
{
let specs = rt.specs.lock().unwrap();
let (name, labels, hardened) = &specs[0];
assert_eq!(name, &format!("zk-space-{fp}-{SPACE}"));
assert!(labels.contains(&("zakuro.space.node".to_string(), fp.clone())));
assert!(labels.contains(&("zakuro.space".to_string(), SPACE.to_string())));
assert!(*hardened);
}
let other = NodeKey::generate();
assert_ne!(other.fingerprint(), fp);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn capabilities_report_cuda_vram_and_no_metal_gpus() {
let gib = 1u64 << 30;
let cuda = |v: u64| GpuId {
vendor: "nvidia".into(),
model: "x".into(),
uuid: "u".into(),
vram_bytes: v * gib,
};
let c = capabilities(&[cuda(24), cuda(16)]);
assert_eq!(c["gpus"], 2);
assert_eq!(c["vram_gb"], 16.0, "per-GPU minimum");
let metal = GpuId {
vendor: "apple".into(),
model: "m".into(),
uuid: "u".into(),
vram_bytes: 48 * gib,
};
let c = capabilities(&[metal]);
assert_eq!(c["gpus"], 0);
assert_eq!(c["vram_gb"], 0.0);
assert_eq!(capabilities(&[])["gpus"], 0);
}
#[test]
fn gpu_enumeration_is_cached_across_ticks() {
let dir = temp_state();
let hub = hub_with(r#"{"leases":[]}"#.to_string());
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
let calls = Arc::new(AtomicUsize::new(0));
let c2 = calls.clone();
s.gpu_probe = Arc::new(move || {
c2.fetch_add(1, Ordering::SeqCst);
Some(vec![GpuId {
vendor: "nvidia".into(),
model: "x".into(),
uuid: "u".into(),
vram_bytes: 24 << 30,
}])
});
for _ in 0..3 {
assert_eq!(s.tick(), Tick::Claimed(0));
}
assert_eq!(calls.load(Ordering::SeqCst), 1);
let reqs = hub.requests.lock().unwrap();
let (_, _, b) = reqs.iter().find(|(_, p, _)| p.ends_with("/claim")).unwrap();
let v: Value = serde_json::from_str(b).unwrap();
assert_eq!(v["capabilities"]["gpus"], 1);
assert_eq!(v["capabilities"]["vram_gb"], 24.0);
drop(reqs);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn no_claim_while_benchmark_runs_and_container_count_is_published() {
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
s.bench_running.store(true, Ordering::SeqCst);
assert_eq!(s.tick(), Tick::BenchRunning);
assert_eq!(hub.count("/claim"), 0);
s.bench_running.store(false, Ordering::SeqCst);
assert_eq!(s.containers.load(Ordering::SeqCst), 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(
s.containers.load(Ordering::SeqCst),
1,
"bench sees a busy node"
);
s.shutdown();
assert_eq!(s.containers.load(Ordering::SeqCst), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn transient_complete_failures_are_retried_until_accepted() {
let dir = temp_state();
let lease = lease_json(&in_secs(300), 60);
let first = Arc::new(AtomicBool::new(true));
let completes = Arc::new(AtomicUsize::new(0));
let c2 = completes.clone();
let hub = stub_hub(move |_m, p, _b| {
if p.ends_with("/claim") {
if first.swap(false, Ordering::SeqCst) {
(200, lease.clone())
} else {
(200, r#"{"leases":[]}"#.to_string())
}
} else if p.ends_with("/complete") {
if c2.fetch_add(1, Ordering::SeqCst) < 2 {
(503, "{}".to_string())
} else {
(200, r#"{"status":"succeeded","paid":0.9}"#.to_string())
}
} else {
(404, "{}".to_string())
}
});
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let (code, _) = request(&sock(&dir), "POST", "/done", done_body_ok());
assert_eq!(code, 202, "503 is not final: the broker keeps the receipt");
let t = std::time::Instant::now();
while completes.load(Ordering::SeqCst) < 3 && t.elapsed() < Duration::from_secs(5) {
std::thread::sleep(Duration::from_millis(20));
}
assert_eq!(completes.load(Ordering::SeqCst), 3);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn final_complete_answers_are_not_retried() {
let dir = temp_state();
let lease = lease_json(&in_secs(300), 60);
let first = Arc::new(AtomicBool::new(true));
let hub = stub_hub(move |_m, p, _b| {
if p.ends_with("/claim") {
if first.swap(false, Ordering::SeqCst) {
(200, lease.clone())
} else {
(200, r#"{"leases":[]}"#.to_string())
}
} else if p.ends_with("/complete") {
(409, "{}".to_string())
} else {
(404, "{}".to_string())
}
});
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut s = sched(&hub, &rt, &dir, &key, 0);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(request(&sock(&dir), "GET", "/next", "").0, 200);
let (code, _) = request(&sock(&dir), "POST", "/done", done_body_ok());
assert_eq!(code, 409);
std::thread::sleep(Duration::from_millis(200));
assert_eq!(hub.count("/complete"), 1);
s.shutdown();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn bench_units_reads_saved_report() {
let dir = temp_state();
assert_eq!(bench_units(&dir), None);
std::fs::create_dir_all(dir.join("bench")).unwrap();
let report = r#"{"cpu_units":8.5,"gpu_units":null}"#;
std::fs::write(
dir.join("bench").join("last.json"),
json!({"report_json": report, "signature": "x"}).to_string(),
)
.unwrap();
assert_eq!(bench_units(&dir), Some(json!({"cpu": 8.5, "gpu": null})));
let report = r#"{"cpu_units":2.0,"gpu_units":30.5}"#;
std::fs::write(
dir.join("bench").join("last.json"),
json!({"report_json": report}).to_string(),
)
.unwrap();
assert_eq!(bench_units(&dir), Some(json!({"cpu": 2.0, "gpu": 30.5})));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_node_policy_decides_whether_spaces_claim_at_all() {
use crate::policy::{Admission, Policy};
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let mut p = Policy::default();
p.workloads.spaces = false;
let mut s = sched(&hub, &rt, &dir, &key, 0).with_policy(Admission::pinned(p, None));
assert_eq!(s.tick(), Tick::Disabled);
assert_eq!(hub.count("/claim"), 0);
let mut p = Policy::default();
p.sharing.squads = Vec::new();
let gate = Admission::pinned(p, None);
let mut s = sched(&hub, &rt, &dir, &key, 0).with_policy(gate.clone());
assert_eq!(s.tick(), Tick::Refused("squad"));
assert_eq!(hub.count("/claim"), 0);
assert_eq!(gate.snapshot()["refused"]["squad"], 1);
}
#[test]
fn a_lease_larger_than_the_policy_budget_is_left_to_expire() {
use crate::policy::{Admission, Policy};
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let gate = Admission::pinned(
Policy::default(),
Some(crate::sandbox::Limits::new(
2.0,
16.0,
crate::sandbox::Gpus::None,
)),
);
let mut s = sched(&hub, &rt, &dir, &key, 0).with_policy(gate.clone());
assert_eq!(s.tick(), Tick::Claimed(1));
assert!(rt.runs.lock().unwrap().is_empty(), "nothing started");
assert_eq!(gate.snapshot()["refused"]["budget"], 1);
}
fn lease_with_price(price: f64) -> String {
lease_json(&in_secs(300), 60).replace(
"\"deadline\"",
&format!("\"price_per_hour\":{price},\"deadline\""),
)
}
#[test]
fn the_owner_coming_back_stops_the_running_space_and_counts_it() {
use crate::policy::owner::Owner;
use crate::policy::{Admission, Policy};
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let gate = Admission::pinned(
Policy::default(),
Some(crate::sandbox::Limits::new(
16.0,
64.0,
crate::sandbox::Gpus::None,
)),
);
let rev = crate::broker::revenue::Revenue::new();
let mut s = sched(&hub, &rt, &dir, &key, 0)
.with_policy(gate.clone())
.with_revenue(rev.clone());
s.preempt_grace = Duration::from_secs(10);
assert_eq!(s.tick(), Tick::Claimed(1));
assert_eq!(s.runner_count(), 1);
assert!(
rt.stopped.lock().unwrap().is_empty(),
"owner away: left alone"
);
gate.pin_owner(Owner::Active("owner-active"));
assert_eq!(s.tick(), Tick::Refused("idle"));
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(s.runner_count(), 0);
assert_eq!(rev.preempted()["owner_active"], 1);
assert_eq!(rev.preempted()["better_offer"], 0);
assert_eq!(gate.snapshot()["refused"]["idle"], 1);
}
#[test]
fn a_space_about_to_finish_outlives_the_owners_return() {
use crate::policy::owner::Owner;
use crate::policy::{Admission, Policy};
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let gate = Admission::pinned(
Policy::default(),
Some(crate::sandbox::Limits::new(
16.0,
64.0,
crate::sandbox::Gpus::None,
)),
);
let rev = crate::broker::revenue::Revenue::new();
let mut s = sched(&hub, &rt, &dir, &key, 0)
.with_policy(gate.clone())
.with_revenue(rev.clone());
s.preempt_grace = Duration::from_secs(600);
assert_eq!(s.tick(), Tick::Claimed(1));
gate.pin_owner(Owner::Active("owner-active"));
assert_eq!(s.tick(), Tick::Refused("idle"));
assert_eq!(s.runner_count(), 1);
assert!(rt.stopped.lock().unwrap().is_empty());
assert_eq!(rev.preempted()["owner_active"], 0);
}
#[test]
fn an_offer_that_pays_more_takes_the_room_of_a_cheaper_lease() {
use crate::broker::revenue::Need;
use crate::policy::{Admission, Policy};
let dir = temp_state();
let hub = hub_with(lease_with_price(5.0));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let limits = crate::sandbox::Limits::new(4.0, 8.0, crate::sandbox::Gpus::None);
let gate = Admission::pinned(Policy::default(), Some(limits));
let rev = crate::broker::revenue::Revenue::new();
let mut s = sched(&hub, &rt, &dir, &key, 0)
.with_policy(gate.clone())
.with_revenue(rev.clone());
s.preempt_grace = Duration::from_secs(10);
assert_eq!(s.tick(), Tick::Claimed(1));
*rt.running.lock().unwrap() =
vec![crate::sandbox::Allocation::from_label("zk-space", "4000m,8589934592").unwrap()];
let want = Need {
cpus: 4.0,
memory_bytes: 8 << 30,
gpus: 0,
};
rev.want_room(want, 3.0);
s.tick();
assert_eq!(s.runner_count(), 1);
rev.want_room(want, 50.0);
s.tick();
assert_eq!(rt.stopped.lock().unwrap().as_slice(), ["cont-1"]);
assert_eq!(s.runner_count(), 0);
assert_eq!(rev.preempted()["better_offer"], 1);
}
#[test]
fn a_lease_of_unknown_value_is_not_stopped_for_an_offer() {
use crate::broker::revenue::Need;
use crate::policy::{Admission, Policy};
let dir = temp_state();
let hub = hub_with(lease_json(&in_secs(300), 60));
let rt = FakeRt::new(DIGEST);
let key = Arc::new(NodeKey::generate());
let limits = crate::sandbox::Limits::new(4.0, 8.0, crate::sandbox::Gpus::None);
let gate = Admission::pinned(Policy::default(), Some(limits));
let rev = crate::broker::revenue::Revenue::new();
let mut s = sched(&hub, &rt, &dir, &key, 0)
.with_policy(gate)
.with_revenue(rev.clone());
s.preempt_grace = Duration::from_secs(10);
assert_eq!(s.tick(), Tick::Claimed(1));
rev.want_room(
Need {
cpus: 4.0,
memory_bytes: 8 << 30,
gpus: 0,
},
1e6,
);
s.tick();
assert_eq!(s.runner_count(), 1);
assert_eq!(rev.preempted()["better_offer"], 0);
}
}