use std::collections::{BTreeMap, HashMap, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use super::client::{Completion, Hub, Lease};
use super::receipt;
use super::socket;
use super::Stats;
use crate::broker::deploy::{ContainerRuntime, RunSpec};
use crate::broker::node_identity::NodeKey;
pub const CONTAINER_SOCKET: &str = "/run/zakuro/space.sock";
pub const MAX_OUTPUTS: usize = 64;
pub type Runtime = Arc<dyn ContainerRuntime + Send + Sync>;
fn now_epoch() -> i64 {
chrono::Utc::now().timestamp()
}
fn rfc3339_now() -> String {
chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string()
}
fn parse_epoch(s: &str) -> Option<i64> {
chrono::DateTime::parse_from_rfc3339(s)
.ok()
.map(|d| d.timestamp())
}
pub fn is_expired(lease: &Lease) -> bool {
parse_epoch(&lease.deadline).is_none_or(|d| d <= now_epoch())
}
pub fn pull_ref(lease: &Lease) -> String {
if lease.image.mirror_ref.contains('@') {
lease.image.mirror_ref.clone()
} else {
format!("{}@{}", lease.image.mirror_ref, lease.image.digest)
}
}
pub const NODE_LABEL: &str = "zakuro.space.node";
pub fn container_name(node_fp: &str, space_id: &str) -> String {
format!("zk-space-{node_fp}-{space_id}")
}
pub fn node_label_filter(node_fp: &str) -> String {
format!("{NODE_LABEL}={node_fp}")
}
pub fn docker_config_dir(state_dir: &Path, space_id: &str) -> PathBuf {
state_dir.join("spaces").join(space_id).join("docker")
}
pub fn socket_path(state_dir: &Path, space_id: &str) -> PathBuf {
state_dir.join("spaces").join(format!("{space_id}.sock"))
}
struct Active {
lease: Lease,
started: String,
deadline: i64,
last_hb: Instant,
}
struct Inner {
queue: VecDeque<Lease>,
active: HashMap<i64, Active>,
last_activity: Instant,
}
pub struct Session {
pub space_id: String,
hub: Hub,
runtime: Runtime,
node_key: Arc<NodeKey>,
container_id: Mutex<String>,
expected_digest: String,
docker_cfg: PathBuf,
inner: Mutex<Inner>,
stats: Arc<Stats>,
dead: AtomicBool,
failed: AtomicBool,
completions: AtomicU64,
complete_retry_base: Duration,
}
const COMPLETE_RETRY_CAP: Duration = Duration::from_secs(30);
const COMPLETE_GRACE_S: i64 = 60;
fn complete_with_retry(
hub: &Hub,
lease_id: &str,
signed: &receipt::SignedReceipt,
give_up_at: i64,
base: Duration,
) -> Result<Completion, String> {
let mut delay = base;
loop {
std::thread::sleep(delay);
match hub.complete(lease_id, signed) {
Ok(c) => return Ok(c),
Err(e) => {
if now_epoch() >= give_up_at {
return Err(e);
}
}
}
delay = (delay * 2).min(COMPLETE_RETRY_CAP.max(base));
}
}
impl Session {
fn touch(inner: &mut Inner) {
inner.last_activity = Instant::now();
}
pub fn is_dead(&self) -> bool {
self.dead.load(Ordering::SeqCst)
}
pub fn failed(&self) -> bool {
self.failed.load(Ordering::SeqCst)
}
pub fn completions(&self) -> u64 {
self.completions.load(Ordering::SeqCst)
}
fn kill_failed(&self) {
self.failed.store(true, Ordering::SeqCst);
self.kill();
}
pub fn lease_infos(&self) -> Vec<super::preempt::LeaseInfo> {
let g = self.inner.lock().unwrap();
let info = |l: &Lease, started: Option<&str>| super::preempt::LeaseInfo {
started_epoch: started.and_then(parse_epoch),
est_runtime_s: l.requirements.est_runtime_s,
max_runtime_s: l.requirements.max_runtime_s,
price_per_hour: l.price_per_hour,
need: crate::broker::revenue::Need {
cpus: l.requirements.cpus,
memory_bytes: (l.requirements.memory_gb * (1u64 << 30) as f64) as u64,
gpus: l.requirements.gpus,
},
};
g.active
.values()
.map(|a| info(&a.lease, Some(&a.started)))
.chain(g.queue.iter().map(|l| info(l, None)))
.collect()
}
pub fn outstanding(&self) -> usize {
let g = self.inner.lock().unwrap();
g.queue.len() + g.active.len()
}
pub fn counts(&self) -> (usize, usize) {
let g = self.inner.lock().unwrap();
(g.queue.len(), g.active.len())
}
pub fn enqueue(&self, lease: Lease) {
let mut g = self.inner.lock().unwrap();
g.queue.push_back(lease);
Self::touch(&mut g);
}
pub fn kill(&self) {
if self.dead.swap(true, Ordering::SeqCst) {
return;
}
let cid = self.container_id.lock().unwrap().clone();
let _ = self.runtime.stop(&cid);
let _ = self.runtime.rm(&cid);
let _ = std::fs::remove_dir_all(&self.docker_cfg);
let mut g = self.inner.lock().unwrap();
g.queue.clear();
g.active.clear();
}
fn digest_ok(&self) -> bool {
let cid = self.container_id.lock().unwrap().clone();
match self.runtime.image_digest(&cid) {
Ok(d) => d.split(',').any(|x| x.trim() == self.expected_digest),
Err(_) => false,
}
}
pub fn next(&self) -> (u16, String) {
if self.is_dead() {
return (204, String::new());
}
let mut g = self.inner.lock().unwrap();
while let Some(lease) = g.queue.pop_front() {
let Some(deadline) = parse_epoch(&lease.deadline).filter(|d| *d > now_epoch()) else {
eprintln!(
" [SPACES] lease {} expired before start; abandoned",
lease.lease_id
);
continue;
};
let body = json!({
"batch_id": lease.batch_id,
"seq": lease.seq,
"inputs": lease.inputs,
"output_presign_endpoint": "/outputs",
"max_runtime_s": lease.requirements.max_runtime_s,
})
.to_string();
g.active.insert(
lease.batch_id,
Active {
started: rfc3339_now(),
deadline,
last_hb: Instant::now(),
lease,
},
);
Self::touch(&mut g);
return (200, body);
}
(204, String::new())
}
fn live_lease(&self, batch_id: i64) -> Result<String, (u16, String)> {
let mut g = self.inner.lock().unwrap();
Self::touch(&mut g);
let Some(a) = g.active.get(&batch_id) else {
return Err((
409,
json!({"error": "unknown or abandoned batch"}).to_string(),
));
};
if a.deadline <= now_epoch() {
g.active.remove(&batch_id);
return Err((409, json!({"error": "lease expired"}).to_string()));
}
Ok(a.lease.lease_id.clone())
}
pub fn outputs(&self, body: &[u8]) -> (u16, String) {
let v: Value = match serde_json::from_slice(body) {
Ok(v) => v,
Err(_) => return (400, json!({"error": "bad json"}).to_string()),
};
let Some(batch_id) = v.get("batch_id").and_then(|b| b.as_i64()) else {
return (400, json!({"error": "batch_id required"}).to_string());
};
let names: Vec<String> = v
.get("names")
.and_then(|n| n.as_array())
.map(|a| {
a.iter()
.filter_map(|s| s.as_str().map(String::from))
.collect()
})
.unwrap_or_default();
if names.len() > MAX_OUTPUTS {
return (
400,
json!({"error": "too many outputs (max 64)"}).to_string(),
);
}
let lease_id = match self.live_lease(batch_id) {
Ok(l) => l,
Err(e) => return e,
};
match self.hub.presign(&lease_id, &names) {
Ok((code, text)) => (code, text),
Err(e) => (502, json!({"error": e}).to_string()),
}
}
pub fn done(&self, body: &[u8]) -> (u16, String) {
let v: Value = match serde_json::from_slice(body) {
Ok(v) => v,
Err(_) => return (400, json!({"error": "bad json"}).to_string()),
};
let (Some(batch_id), Some(exit_code)) = (
v.get("batch_id").and_then(|b| b.as_i64()),
v.get("exit_code").and_then(|b| b.as_i64()),
) else {
return (
400,
json!({"error": "batch_id and exit_code required"}).to_string(),
);
};
let mut hashes = BTreeMap::new();
if let Some(m) = v.get("output_hashes").and_then(|m| m.as_object()) {
for (k, h) in m {
if let Some(h) = h.as_str() {
hashes.insert(k.clone(), h.to_string());
}
}
}
if hashes.len() > MAX_OUTPUTS {
return (
400,
json!({"error": "too many outputs (max 64)"}).to_string(),
);
}
if let Err(e) = self.live_lease(batch_id) {
return e;
}
if !self.digest_ok() {
eprintln!(
" [SPACES] image digest mismatch for space {}; killing container, no receipt",
self.space_id
);
self.kill_failed();
return (
409,
json!({"accepted": false, "reason": "image digest mismatch"}).to_string(),
);
}
let Some(active) = self.inner.lock().unwrap().active.remove(&batch_id) else {
return (
409,
json!({"accepted": false, "reason": "abandoned"}).to_string(),
);
};
let signed = receipt::build(
&active.lease,
&hashes,
exit_code,
&active.started,
&rfc3339_now(),
&self.node_key,
);
{
let mut g = self.inner.lock().unwrap();
Self::touch(&mut g);
}
match self.hub.complete(&active.lease.lease_id, &signed) {
Ok(c) => self.finish_completion(c),
Err(first_err) => {
eprintln!(
" [SPACES] complete for batch {batch_id} failed ({first_err}); retrying in background"
);
let hub = self.hub.clone();
let stats = self.stats.clone();
let base = self.complete_retry_base;
let give_up_at = active.deadline + COMPLETE_GRACE_S;
let lease_id = active.lease.lease_id.clone();
let spawned = std::thread::Builder::new()
.name("spaces-complete".into())
.spawn(move || {
match complete_with_retry(&hub, &lease_id, &signed, give_up_at, base) {
Ok(Completion::Accepted { paid }) => stats.record_paid(paid),
Ok(Completion::Rejected { status }) => eprintln!(
" [SPACES] hub rejected receipt for batch {batch_id} ({status})"
),
Err(e) => {
eprintln!(" [SPACES] gave up completing batch {batch_id}: {e}")
}
}
});
match spawned {
Ok(_) => (
202,
json!({"accepted": "pending", "reason": first_err}).to_string(),
),
Err(e) => (
502,
json!({"accepted": false, "reason": e.to_string()}).to_string(),
),
}
}
}
}
fn finish_completion(&self, c: Completion) -> (u16, String) {
match c {
Completion::Accepted { paid } => {
self.completions.fetch_add(1, Ordering::SeqCst);
self.stats.record_paid(paid);
(200, json!({"accepted": true}).to_string())
}
Completion::Rejected { status } => (
409,
json!({"accepted": false, "reason": format!("hub rejected ({status})")})
.to_string(),
),
}
}
pub fn maintain(&self) {
let now = now_epoch();
let mut due: Vec<(i64, String, u64)> = Vec::new();
let mut lost = false;
{
let mut g = self.inner.lock().unwrap();
let expired: Vec<i64> = g
.active
.iter()
.filter(|(_, a)| a.deadline <= now)
.map(|(b, _)| *b)
.collect();
for b in expired {
eprintln!(" [SPACES] lease for batch {b} expired; abandoned, killing container");
g.active.remove(&b);
lost = true;
}
for (b, a) in g.active.iter() {
let every = a.lease.heartbeat_every_s.max(1);
if a.last_hb.elapsed() >= Duration::from_secs(every) {
due.push((*b, a.lease.lease_id.clone(), every));
}
}
}
for (batch, lease_id, _) in due {
let res = self.hub.heartbeat(&lease_id);
let mut g = self.inner.lock().unwrap();
match res {
Ok(until) => {
if let Some(a) = g.active.get_mut(&batch) {
a.last_hb = Instant::now();
if let Some(d) = until.as_deref().and_then(parse_epoch) {
a.deadline = d;
}
}
}
Err(e) => {
eprintln!(" [SPACES] heartbeat for batch {batch} failed: {e}");
if e.contains("HTTP 409") || e.contains("HTTP 404") {
g.active.remove(&batch);
lost = true;
} else if let Some(a) = g.active.get_mut(&batch) {
a.last_hb = Instant::now();
}
}
}
}
if lost {
self.kill_failed();
}
}
fn idle_for(&self) -> Duration {
let g = self.inner.lock().unwrap();
if g.queue.is_empty() && g.active.is_empty() {
g.last_activity.elapsed()
} else {
Duration::ZERO
}
}
}
pub struct RunnerDeps {
pub hub: Hub,
pub runtime: Runtime,
pub node_key: Arc<NodeKey>,
pub state_dir: PathBuf,
pub stats: Arc<Stats>,
pub network: String,
pub idle_timeout: Duration,
pub node_fp: String,
pub complete_retry_base: Duration,
}
pub struct SpaceRunner {
pub session: Arc<Session>,
stop: Arc<AtomicBool>,
sock: PathBuf,
}
impl SpaceRunner {
pub fn start(deps: &RunnerDeps, lease: &Lease) -> Result<SpaceRunner, String> {
let rt = &deps.runtime;
let docker_cfg = docker_config_dir(&deps.state_dir, &lease.space_id);
std::fs::create_dir_all(&docker_cfg).map_err(|e| format!("docker config dir: {e}"))?;
let cleanup = |e: String| {
let _ = std::fs::remove_dir_all(&docker_cfg);
e
};
rt.login_in(
&docker_cfg,
&lease.pull.registry,
&lease.pull.username,
&lease.pull.token,
)
.map_err(|e| cleanup(format!("registry login failed: {e}")))?;
let image = pull_ref(lease);
rt.pull_in(&docker_cfg, &image)
.map_err(|e| cleanup(format!("pull {image} failed: {e}")))?;
let sock = socket_path(&deps.state_dir, &lease.space_id);
let session = Arc::new(Session {
space_id: lease.space_id.clone(),
hub: deps.hub.clone(),
runtime: rt.clone(),
node_key: deps.node_key.clone(),
container_id: Mutex::new(String::new()),
expected_digest: lease.image.digest.clone(),
docker_cfg: docker_cfg.clone(),
inner: Mutex::new(Inner {
queue: VecDeque::new(),
active: HashMap::new(),
last_activity: Instant::now(),
}),
stats: deps.stats.clone(),
dead: AtomicBool::new(false),
failed: AtomicBool::new(false),
completions: AtomicU64::new(0),
complete_retry_base: deps.complete_retry_base,
});
let stop = Arc::new(AtomicBool::new(false));
socket::serve(&sock, session.clone(), stop.clone())?;
let mut env = HashMap::new();
env.insert("ZAKURO_SPACE_ID".to_string(), lease.space_id.clone());
env.insert(
"ZAKURO_SPACE_SOCKET".to_string(),
CONTAINER_SOCKET.to_string(),
);
let req = &lease.requirements;
let admission = crate::sandbox::admission_lock();
let admitted = rt.budget().and_then(|budget| {
let running = rt
.allocations()
.map_err(|e| format!("cannot read the node's allocations: {e}"))?;
let mut limits = crate::sandbox::lease_limits(
&budget,
&running,
&crate::sandbox::Request {
cpus: req.cpus,
memory_gib: req.memory_gb,
gpus: Some(req.gpus),
},
);
if req.gpus == 0 {
limits.gpus = crate::sandbox::Gpus::None;
}
crate::sandbox::admit(&budget, &running, &limits)?;
Ok(limits)
});
let limits = match admitted {
Ok(limits) => limits,
Err(e) => {
drop(admission);
stop.store(true, Ordering::SeqCst);
let _ = std::fs::remove_file(&sock);
return Err(cleanup(format!("budget: {e}")));
}
};
let spec = RunSpec {
name: container_name(&deps.node_fp, &lease.space_id),
image: &image,
cmd: None,
ports: &[],
env: &env,
labels: vec![
("zakuro.space".to_string(), lease.space_id.clone()),
(NODE_LABEL.to_string(), deps.node_fp.clone()),
],
mounts: vec![(sock.display().to_string(), CONTAINER_SOCKET.to_string())],
limits,
network: Some(deps.network.clone()),
hardened: true,
};
let ran = rt.run_leased(&spec, Some(&docker_cfg));
drop(admission);
let cid = match ran {
Ok(c) => c,
Err(e) => {
stop.store(true, Ordering::SeqCst);
let _ = std::fs::remove_file(&sock);
return Err(cleanup(format!("run failed: {e}")));
}
};
*session.container_id.lock().unwrap() = cid;
if !session.digest_ok() {
session.kill_failed();
stop.store(true, Ordering::SeqCst);
let _ = std::fs::remove_file(&sock);
return Err("image digest mismatch after start".to_string());
}
let runner = SpaceRunner {
session: session.clone(),
stop: stop.clone(),
sock,
};
let idle = deps.idle_timeout;
let s2 = session.clone();
let stop2 = stop.clone();
let sock2 = runner.sock.clone();
let _ = std::thread::Builder::new()
.name("spaces-maint".into())
.spawn(move || {
while !stop2.load(Ordering::SeqCst) && !s2.is_dead() {
std::thread::sleep(Duration::from_millis(100));
s2.maintain();
if s2.idle_for() >= idle {
eprintln!(" [SPACES] space {} idle; stopping container", s2.space_id);
break;
}
}
s2.kill();
stop2.store(true, Ordering::SeqCst);
let _ = std::fs::remove_file(&sock2);
});
Ok(runner)
}
pub fn finished(&self) -> bool {
self.session.is_dead() || self.stop.load(Ordering::SeqCst)
}
pub fn shutdown(&self) {
self.session.kill();
self.stop.store(true, Ordering::SeqCst);
let _ = std::fs::remove_file(&self.sock);
}
}