use crate::agent::host_runner::{plan_start, HostRunner, HostRunnerCfg, RunnerApi};
use crate::agent::hosting::{self, Hosting};
use crate::agent::runners::{catalogue, CatalogueEntry, Hardware, Isolation, RunnerKind};
use crate::agent::watchdog::{self, Action, MacSignals};
use std::collections::{HashMap, HashSet};
use std::net::IpAddr;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, PartialEq)]
pub struct Inputs {
pub enabled: bool,
pub installed: bool,
pub advertise: bool,
pub port_free: bool,
pub fits: bool,
pub owner_known: bool,
pub owner_changed: bool,
pub backoff: bool,
pub alive: bool,
pub running_jobs: Option<Vec<String>>,
pub draining: bool,
pub action: Action,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Do {
Start,
Stop,
Drain(bool),
WrapUp(Vec<String>),
}
pub fn step(i: &Inputs) -> (Vec<Do>, Option<&'static str>) {
let stand_down = |wrap: bool| match &i.running_jobs {
None => vec![Do::Drain(true)],
Some(j) if j.is_empty() => vec![Do::Drain(true), Do::Stop],
Some(j) if wrap => vec![Do::Drain(true), Do::WrapUp(j.clone())],
Some(_) => vec![Do::Drain(true)],
};
let yield_now = |reason, wrap| {
if i.alive {
(stand_down(wrap), Some(reason))
} else {
(vec![], Some(reason))
}
};
if !i.enabled {
return yield_now("disabled", true);
}
if !i.installed {
return yield_now("not-installed", true);
}
if !i.fits {
return if i.alive {
(vec![Do::Drain(true)], Some("budget"))
} else {
(vec![], Some("budget"))
};
}
if !i.alive {
return if !i.advertise {
(vec![], Some("unreachable"))
} else if !i.port_free {
(vec![], Some("port-busy"))
} else if !i.owner_known {
(vec![], Some("signed-out"))
} else if i.backoff {
(vec![], Some("restarting"))
} else {
(vec![Do::Start], None)
};
}
if i.owner_changed {
return (stand_down(false), Some("restarting"));
}
match &i.action {
Action::None if i.draining => (vec![Do::Drain(false)], None),
Action::None => (vec![], None),
Action::StopAccepting => (vec![Do::Drain(true)], Some("yielding")),
Action::WrapUp(j) => (
vec![Do::Drain(true), Do::WrapUp(j.clone())],
Some("yielding"),
),
}
}
#[derive(Debug, Default, Clone)]
pub struct Backoff {
delay: std::time::Duration,
}
impl Backoff {
pub const HEALTHY: std::time::Duration = std::time::Duration::from_secs(600);
const FIRST: std::time::Duration = std::time::Duration::from_secs(30);
const MAX: std::time::Duration = std::time::Duration::from_secs(600);
pub fn after_exit(&mut self, lived: std::time::Duration) -> std::time::Duration {
self.delay = if lived >= Self::HEALTHY {
std::time::Duration::ZERO
} else if self.delay.is_zero() {
Self::FIRST
} else {
(self.delay * 2).min(Self::MAX)
};
self.delay
}
}
pub fn advertise_ip(override_ip: Option<String>, mesh_ip: Option<String>) -> Option<IpAddr> {
match override_ip.filter(|v| !v.trim().is_empty()) {
Some(v) => v.trim().parse().ok(),
None => mesh_ip?.parse().ok(),
}
}
pub fn parse_env_file(s: &str) -> HashMap<String, String> {
s.lines()
.map(str::trim)
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.filter_map(|l| l.split_once('='))
.map(|(k, v)| {
let v = v.trim();
let v = v
.strip_prefix('\'')
.and_then(|v| v.strip_suffix('\''))
.or_else(|| v.strip_prefix('"').and_then(|v| v.strip_suffix('"')))
.unwrap_or(v);
(k.trim().to_string(), v.to_string())
})
.collect()
}
pub fn load_cfg(root: &Path, backend: &str) -> Option<HostRunnerCfg> {
let env = parse_env_file(&std::fs::read_to_string(root.join("runner.env")).ok()?);
let get = |k: &str| env.get(k).cloned().filter(|v| !v.is_empty());
Some(HostRunnerCfg {
backend: backend.to_string(),
port: get("FORGE_RUNNER_PORT")
.and_then(|p| p.parse().ok())
.unwrap_or(8102),
root: root.to_path_buf(),
broker: get("FORGE_ATELIER_ZC_BROKER")?,
key_file: PathBuf::from(get("FORGE_ATELIER_ZC_WORKER_KEY_FILE")?),
name: get("FORGE_ATELIER_ZC_NAME")?,
price_per_hour: get("FORGE_ATELIER_PRICE_PER_HOUR").unwrap_or_else(|| "0.9".into()),
gpu_model: get("FORGE_ATELIER_ZC_GPU_MODEL").unwrap_or_else(|| "Apple".into()),
gpu_mem_gb: get("FORGE_ATELIER_ZC_GPU_VRAM_GB")
.and_then(|v| v.parse().ok())
.unwrap_or(8),
owner: None,
token: get("FORGE_RUNNER_TOKEN").map(crate::agent::host_runner::Secret::new),
})
}
pub fn default_root() -> PathBuf {
std::env::var_os("ZAKURO_RUNNER_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join("atelier")
})
}
#[derive(Debug, Clone, Default)]
pub struct HostingStatus {
pub catalogue: Option<Vec<CatalogueEntry>>,
pub root: Option<PathBuf>,
pub reason: Option<&'static str>,
pub serving: bool,
pub runner_base: Option<String>,
pub runner_token: Option<crate::agent::host_runner::Secret>,
pub running_jobs: Option<Vec<String>>,
pub last_job_seen: Option<std::time::SystemTime>,
}
impl HostingStatus {
pub fn root(&self) -> PathBuf {
self.root.clone().unwrap_or_else(default_root)
}
pub fn catalogue(&mut self) -> Vec<CatalogueEntry> {
self.catalogue
.get_or_insert_with(|| catalogue(&Hardware::detect()))
.clone()
}
pub fn unified(&mut self) -> bool {
self.catalogue()
.iter()
.any(|e| e.supported && e.isolation == Isolation::Host)
}
pub fn image_store(&self, h: &Hosting) -> crate::agent::images::HostStore {
let base = self.runner_base.clone();
let token = self.runner_token.clone();
let last = self.running_jobs.clone();
crate::agent::images::HostStore {
root: self.root(),
enabled_backends: if h.budget.gpu {
h.workloads.atelier.clone()
} else {
vec![]
},
jobs: Box::new(move || match &base {
Some(b) => RunnerApi {
base: b.clone(),
token: token.clone(),
}
.running(),
None => last.clone(),
}),
last_job_seen: self.last_job_seen,
}
}
}
pub struct GcPlan {
pub store: crate::agent::images::HostStore,
pub ttl: std::time::Duration,
}
#[derive(Default)]
pub struct HostingRuntime {
pub runner: HostRunner,
pub draining: bool,
wrapped: HashSet<String>,
last_gc: Option<std::time::Instant>,
last_job_seen: Option<std::time::SystemTime>,
started: Option<(std::time::Instant, Option<String>)>,
backoff: Backoff,
next_start: Option<std::time::Instant>,
stopped_adopted: bool,
}
const MAX_LOG_BYTES: u64 = 50 << 20;
impl HostingRuntime {
pub fn tick(
&mut self,
state_dir: &Path,
owner: Option<String>,
status: &std::sync::Mutex<HostingStatus>,
) -> Option<GcPlan> {
let (cat, root) = {
let mut st = status.lock().unwrap_or_else(|p| p.into_inner());
(st.catalogue(), st.root())
};
let entry = cat
.iter()
.find(|e| e.kind == RunnerKind::AtelierPeft && e.isolation == Isolation::Host)
.cloned()?; let h = match hosting::load_or_default(state_dir, &cat) {
Ok(h) => h,
Err(e) => {
eprintln!(" [HOSTING] {e}");
return None;
}
};
let cfg = load_cfg(&root, "peft").map(|mut c| {
c.owner = owner.clone();
if crate::worker_key::is_loopback(&c.broker) {
if let Some(dir) = crate::credentials::dir() {
match crate::worker_key::file_for_local_client(
std::env::var(crate::worker_key::ENV).ok(),
&dir,
) {
Ok(p) => c.key_file = p,
Err(e) => eprintln!(" [HOSTING] worker key for the host runner: {e}"),
}
}
}
c
});
let addr = advertise_ip(
std::env::var("ZAKURO_RUNNER_ADVERTISE_IP").ok(),
crate::vpn::local_mesh().map(|m| m.ip),
);
let base = addr
.zip(cfg.as_ref())
.map(|(a, c)| crate::agent::host_runner::base_url(a, c.port));
let token = cfg.as_ref().and_then(|c| c.token.clone());
let api = base.clone().map(|b| RunnerApi {
base: b,
token: token.clone(),
});
let was_owned = self.runner.owned();
let child_alive = self.runner.alive();
if was_owned && !child_alive {
if let Some((at, _)) = self.started.take() {
let wait = self.backoff.after_exit(at.elapsed());
self.next_start = Some(std::time::Instant::now() + wait);
eprintln!(
" [HOSTING] host runner exited; next start in {}s",
wait.as_secs()
);
}
}
let st = api.as_ref().and_then(RunnerApi::status);
if let Some(s) = &st {
self.runner.adopt(s.pid);
}
let running_jobs = st.map(|s| s.jobs);
let alive = child_alive || running_jobs.is_some();
let jobs = running_jobs.clone().unwrap_or_default();
if !jobs.is_empty() {
self.last_job_seen = Some(std::time::SystemTime::now());
}
self.wrapped.retain(|j| jobs.contains(j));
let fits = h
.budget()
.and_then(|b| {
let total = b.total(&crate::sandbox::HostResources::detect())?;
let r = entry.reservation;
Ok(r.cpu_millis <= total.cpu_millis
&& r.memory_bytes + r.vram_bytes <= total.memory_bytes
&& r.disk_bytes <= b.disk_bytes)
})
.unwrap_or(false);
let signals = MacSignals {
runner_pid: self.runner.pid(),
};
let port_free = alive
|| addr.zip(cfg.as_ref()).is_none_or(|(a, c)| {
std::net::TcpStream::connect_timeout(
&std::net::SocketAddr::new(a, c.port),
std::time::Duration::from_millis(300),
)
.is_err()
});
let inputs = Inputs {
enabled: enabled(&h),
installed: cfg.as_ref().is_some_and(HostRunnerCfg::installed),
advertise: addr.is_some(),
port_free,
fits,
owner_known: owner.is_some(),
owner_changed: self.runner.owned()
&& self.started.as_ref().is_some_and(|(_, o)| *o != owner),
backoff: self
.next_start
.is_some_and(|t| std::time::Instant::now() < t),
alive,
running_jobs: running_jobs.clone(),
draining: self.draining,
action: watchdog::decide(&signals, &h, entry.reservation.memory_bytes, &jobs),
};
let jobs_unknown = running_jobs.is_none() && (alive || !port_free);
let (todo, reason) = step(&inputs);
for d in todo {
match d {
Do::Start => {
let (Some(cfg), Some(addr)) = (cfg.as_ref(), addr) else {
continue;
};
let Ok((argv, env)) = plan_start(Some(addr), cfg) else {
continue;
};
let _ =
crate::agent::supervisor::truncate_if_large(&cfg.log_path(), MAX_LOG_BYTES);
match self.runner.start(
&argv,
&env,
&cfg.root.join("zak-forge"),
&cfg.log_path(),
) {
Ok(()) => {
self.draining = false;
self.started = Some((std::time::Instant::now(), owner.clone()));
eprintln!(" [HOSTING] host runner started on {addr}:{}", cfg.port);
}
Err(e) => {
let wait = self.backoff.after_exit(std::time::Duration::ZERO);
self.next_start = Some(std::time::Instant::now() + wait);
eprintln!(" [HOSTING] host runner did not start: {e}");
}
}
}
Do::Stop => {
if self.runner.owned() || !self.stopped_adopted {
self.stopped_adopted |= !self.runner.owned();
self.runner.stop();
self.started = None;
self.draining = false;
}
}
Do::Drain(on) => {
if self.draining != on && api.as_ref().is_some_and(|a| a.drain(on)) {
self.draining = on;
}
}
Do::WrapUp(ids) => {
for id in ids {
if !self.wrapped.contains(&id)
&& api.as_ref().is_some_and(|a| a.wrap_up(&id))
{
eprintln!(" [HOSTING] wrap-up asked for job {id}");
self.wrapped.insert(id);
}
}
}
}
}
{
let mut st = status.lock().unwrap_or_else(|p| p.into_inner());
st.reason = reason;
st.serving = reason.is_none() && alive && !self.draining;
st.runner_base = if alive { base } else { None };
st.runner_token = token;
st.running_jobs = if jobs_unknown { None } else { Some(jobs) };
st.last_job_seen = self.last_job_seen;
}
let due = self
.last_gc
.is_none_or(|t| t.elapsed() >= crate::agent::images::GC_EVERY);
if !due || jobs_unknown {
return None;
}
self.last_gc = Some(std::time::Instant::now());
let ttl = match h.idle_ttl() {
Ok(t) => t,
Err(e) => {
eprintln!(" [HOSTING] cleanup skipped: {e}");
return None;
}
};
let store = status
.lock()
.unwrap_or_else(|p| p.into_inner())
.image_store(&h);
Some(GcPlan { store, ttl })
}
}
fn enabled(h: &Hosting) -> bool {
h.budget.gpu && h.workloads.atelier.iter().any(|a| a == "peft")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::agent::watchdog::Action;
use std::time::Duration;
fn base() -> Inputs {
Inputs {
enabled: true,
installed: true,
advertise: true,
port_free: true,
fits: true,
owner_known: true,
owner_changed: false,
backoff: false,
alive: false,
running_jobs: Some(vec![]),
draining: false,
action: Action::None,
}
}
fn j1() -> Option<Vec<String>> {
Some(vec!["j1".into()])
}
#[test]
fn starts_when_enabled_installed_and_reachable() {
assert_eq!(step(&base()), (vec![Do::Start], None));
}
#[test]
fn never_starts_without_an_address() {
let i = Inputs {
advertise: false,
..base()
};
assert_eq!(step(&i), (vec![], Some("unreachable")));
}
#[test]
fn busy_port_is_reported_not_retried() {
let i = Inputs {
port_free: false,
..base()
};
assert_eq!(step(&i), (vec![], Some("port-busy")));
}
#[test]
fn not_installed_is_reported() {
let i = Inputs {
installed: false,
..base()
};
assert_eq!(step(&i), (vec![], Some("not-installed")));
}
#[test]
fn disabled_idle_runner_is_drained_then_stopped() {
let i = Inputs {
enabled: false,
alive: true,
..base()
};
assert_eq!(
step(&i),
(vec![Do::Drain(true), Do::Stop], Some("disabled"))
);
}
#[test]
fn disabled_busy_runner_drains_and_wraps_up_never_killed() {
let i = Inputs {
enabled: false,
alive: true,
running_jobs: j1(),
..base()
};
assert_eq!(
step(&i),
(
vec![Do::Drain(true), Do::WrapUp(vec!["j1".into()])],
Some("disabled")
)
);
}
#[test]
fn unknown_jobs_are_never_stopped_only_drained() {
let i = Inputs {
enabled: false,
alive: true,
running_jobs: None,
..base()
};
assert_eq!(step(&i), (vec![Do::Drain(true)], Some("disabled")));
}
#[test]
fn over_budget_runner_drains_without_wrap_up_or_kill() {
let i = Inputs {
fits: false,
alive: true,
running_jobs: j1(),
..base()
};
assert_eq!(step(&i), (vec![Do::Drain(true)], Some("budget")));
}
#[test]
fn over_budget_runner_is_not_started() {
let i = Inputs {
fits: false,
..base()
};
assert_eq!(step(&i), (vec![], Some("budget")));
}
#[test]
fn no_start_before_the_owner_is_known() {
let i = Inputs {
owner_known: false,
..base()
};
assert_eq!(step(&i), (vec![], Some("signed-out")));
}
#[test]
fn owner_change_restarts_an_idle_runner_only() {
let i = Inputs {
alive: true,
owner_changed: true,
..base()
};
assert_eq!(
step(&i),
(vec![Do::Drain(true), Do::Stop], Some("restarting"))
);
let busy = Inputs {
alive: true,
owner_changed: true,
running_jobs: j1(),
..base()
};
assert_eq!(step(&busy), (vec![Do::Drain(true)], Some("restarting")));
}
#[test]
fn crashing_runner_waits_for_its_backoff() {
let i = Inputs {
backoff: true,
..base()
};
assert_eq!(step(&i), (vec![], Some("restarting")));
}
#[test]
fn watchdog_stop_accepting_drains_and_says_yielding() {
let i = Inputs {
alive: true,
action: Action::StopAccepting,
..base()
};
assert_eq!(step(&i), (vec![Do::Drain(true)], Some("yielding")));
}
#[test]
fn watchdog_wrap_up_drains_and_wraps() {
let i = Inputs {
alive: true,
action: Action::WrapUp(vec!["j1".into()]),
..base()
};
assert_eq!(
step(&i),
(
vec![Do::Drain(true), Do::WrapUp(vec!["j1".into()])],
Some("yielding")
)
);
}
#[test]
fn calm_again_undrains() {
let i = Inputs {
alive: true,
draining: true,
..base()
};
assert_eq!(step(&i), (vec![Do::Drain(false)], None));
}
#[test]
fn backoff_doubles_and_caps() {
let mut b = Backoff::default();
let quick = Duration::from_secs(5);
assert_eq!(b.after_exit(quick).as_secs(), 30);
assert_eq!(b.after_exit(quick).as_secs(), 60);
for _ in 0..10 {
b.after_exit(quick);
}
assert_eq!(b.after_exit(quick).as_secs(), 600);
assert_eq!(b.after_exit(Duration::from_secs(3600)).as_secs(), 0);
}
#[test]
fn advertise_ip_override_wins() {
let ip = advertise_ip(Some("10.5.5.3".into()), Some("10.13.13.7".into()));
assert_eq!(ip.unwrap().to_string(), "10.5.5.3");
let ip = advertise_ip(None, Some("10.13.13.7".into()));
assert_eq!(ip.unwrap().to_string(), "10.13.13.7");
assert!(advertise_ip(Some("nope".into()), None).is_none());
assert!(advertise_ip(None, None).is_none());
}
#[test]
fn runner_env_file_is_parsed() {
let s = "FORGE_ATELIER_ZC_NAME=atelier-m5\nFORGE_ATELIER_ZC_GPU_MODEL='Apple M5'\n# c\n\nX=\"y\"\n";
let m = parse_env_file(s);
assert_eq!(m.get("FORGE_ATELIER_ZC_NAME").unwrap(), "atelier-m5");
assert_eq!(m.get("FORGE_ATELIER_ZC_GPU_MODEL").unwrap(), "Apple M5");
assert_eq!(m.get("X").unwrap(), "y");
}
}