use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::process::Child;
use std::time::Duration;
#[derive(Debug, Clone, PartialEq)]
pub struct HostRunnerCfg {
pub backend: String,
pub port: u16,
pub root: PathBuf,
pub broker: String,
pub key_file: PathBuf,
pub name: String,
pub price_per_hour: String,
pub gpu_model: String,
pub gpu_mem_gb: u64,
pub owner: Option<String>,
}
pub const MIN_FREE_BYTES: u64 = 15 << 30;
const ENTRY: &str = "from forge.runner.service import main; main()";
impl HostRunnerCfg {
pub fn runner_python(&self) -> PathBuf {
self.root.join("zak-forge/.venv/bin/python")
}
pub fn backend_python(&self) -> PathBuf {
self.root
.join("envs")
.join(&self.backend)
.join("bin/python")
}
pub fn log_path(&self) -> PathBuf {
self.root.join("logs/runner.log")
}
pub fn installed(&self) -> bool {
self.runner_python().exists() && self.backend_python().exists()
}
}
pub fn base_url(addr: IpAddr, port: u16) -> String {
match addr {
IpAddr::V6(a) => format!("http://[{a}]:{port}"),
IpAddr::V4(a) => format!("http://{a}:{port}"),
}
}
pub type StartPlan = (Vec<String>, Vec<(String, String)>);
pub fn plan_start(
mesh_addr: Option<IpAddr>,
cfg: &HostRunnerCfg,
) -> Result<StartPlan, &'static str> {
let addr = mesh_addr.ok_or("unreachable")?;
let root = |p: &str| cfg.root.join(p).display().to_string();
let env: Vec<(String, String)> = [
("FORGE_RUNNER_HOST", addr.to_string()),
("FORGE_RUNNER_PORT", cfg.port.to_string()),
("FORGE_RUNNER_WORKDIR", root("work")),
("FORGE_RUNNER_MIN_FREE_BYTES", MIN_FREE_BYTES.to_string()),
(
"FORGE_ATELIER_PYTHONS",
format!(
r#"{{"{}": "{}"}}"#,
cfg.backend,
cfg.backend_python().display()
),
),
("FORGE_ATELIER_PUBLIC_URL", base_url(addr, cfg.port)),
("FORGE_ATELIER_ZC_BROKER", cfg.broker.clone()),
(
"FORGE_ATELIER_ZC_WORKER_KEY_FILE",
cfg.key_file.display().to_string(),
),
("FORGE_ATELIER_ZC_NAME", cfg.name.clone()),
(
"FORGE_ATELIER_ZC_TAGS",
format!(
"atelier-backend:{},atelier-export:mlx,atelier-scope:owner{}",
cfg.backend,
cfg.owner
.as_deref()
.map(|o| format!(",atelier-owner:{o}"))
.unwrap_or_default()
),
),
("FORGE_ATELIER_EXPORTS", "mlx,safetensors".into()),
("FORGE_ATELIER_ZC_GPU_MODEL", cfg.gpu_model.clone()),
("FORGE_ATELIER_ZC_GPU_VRAM_GB", cfg.gpu_mem_gb.to_string()),
("FORGE_ATELIER_PRICE_PER_HOUR", cfg.price_per_hour.clone()),
("HF_HOME", root("hf")),
(
"PYTHONPATH",
format!("{}:{}", root("zak-forge/src"), root("src/sakura-internal")),
),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v))
.collect();
let mut argv = Vec::new();
if cfg!(target_os = "macos") {
argv.extend(["taskpolicy".to_string(), "-b".to_string()]);
}
argv.extend([
cfg.runner_python().display().to_string(),
"-c".to_string(),
ENTRY.to_string(),
]);
Ok((argv, env))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunnerStatus {
pub jobs: Vec<String>,
pub pid: Option<u32>,
}
pub fn parse_status(v: &serde_json::Value) -> Option<RunnerStatus> {
Some(RunnerStatus {
jobs: v
.get("running")?
.as_array()?
.iter()
.filter_map(|j| j.as_str().map(String::from))
.collect(),
pid: v
.get("pid")
.and_then(serde_json::Value::as_u64)
.and_then(|p| u32::try_from(p).ok()),
})
}
pub struct RunnerApi {
pub base: String,
}
impl RunnerApi {
fn agent() -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(5)))
.http_status_as_error(false)
.build()
.into()
}
pub fn status(&self) -> Option<RunnerStatus> {
let mut resp = Self::agent()
.get(&format!("{}/status", self.base))
.call()
.ok()?;
if resp.status() != 200 {
return None;
}
let v: serde_json::Value = resp.body_mut().read_json().ok()?;
parse_status(&v)
}
pub fn running(&self) -> Option<Vec<String>> {
self.status().map(|s| s.jobs)
}
pub fn drain(&self, on: bool) -> bool {
Self::agent()
.post(&format!("{}/drain", self.base))
.send_json(serde_json::json!({ "on": on }))
.is_ok_and(|r| r.status() == 200)
}
pub fn wrap_up(&self, job_id: &str) -> bool {
Self::agent()
.post(&format!("{}/jobs/{job_id}/wrap-up", self.base))
.send_json(serde_json::json!({}))
.is_ok_and(|r| r.status().is_success())
}
}
#[derive(Default)]
pub struct HostRunner {
child: Option<Child>,
adopted: Option<u32>,
}
impl HostRunner {
pub fn pid(&self) -> Option<u32> {
self.child.as_ref().map(|c| c.id()).or(self.adopted)
}
pub fn owned(&self) -> bool {
self.child.is_some()
}
pub fn adopt(&mut self, pid: Option<u32>) {
if self.child.is_none() {
self.adopted = pid.filter(|p| *p > 1);
}
}
pub fn alive(&mut self) -> bool {
match self.child.as_mut().map(|c| c.try_wait()) {
Some(Ok(None)) => true,
Some(_) => {
self.child = None;
false
}
None => false,
}
}
pub fn start(
&mut self,
argv: &[String],
env: &[(String, String)],
cwd: &Path,
log: &Path,
) -> std::io::Result<()> {
if self.alive() {
return Ok(());
}
self.child = Some(crate::agent::supervisor::spawn_in_group(
argv,
Some(cwd),
env,
log,
)?);
Ok(())
}
pub fn stop(&mut self) {
if let Some(pid) = self.adopted.take() {
let alive = |p: u32| unsafe { libc::kill(p as libc::pid_t, 0) } == 0;
unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) };
for _ in 0..50 {
if !alive(pid) {
return;
}
std::thread::sleep(Duration::from_millis(100));
}
unsafe { libc::kill(pid as libc::pid_t, libc::SIGKILL) };
return;
}
if let Some(mut c) = self.child.take() {
crate::agent::supervisor::signal_group(c.id(), libc::SIGTERM);
for _ in 0..50 {
if matches!(c.try_wait(), Ok(Some(_))) {
return;
}
std::thread::sleep(Duration::from_millis(100));
}
crate::agent::supervisor::signal_group(c.id(), libc::SIGKILL);
let _ = c.wait();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr};
use std::path::PathBuf;
fn cfg() -> HostRunnerCfg {
HostRunnerCfg {
backend: "peft".into(),
port: 8102,
root: PathBuf::from("/Users/o/atelier"),
broker: "http://192.168.0.23:9150".into(),
key_file: PathBuf::from("/Users/o/atelier/worker.key"),
name: "atelier-m5".into(),
price_per_hour: "0.9".into(),
gpu_model: "Apple M5".into(),
gpu_mem_gb: 8,
owner: None,
}
}
fn mesh() -> Option<IpAddr> {
Some(IpAddr::V4(Ipv4Addr::new(10, 5, 5, 3)))
}
fn get<'a>(env: &'a [(String, String)], k: &str) -> &'a str {
env.iter()
.find(|(key, _)| key == k)
.map(|(_, v)| v.as_str())
.unwrap_or_else(|| panic!("{k} missing"))
}
#[test]
fn supervisor_refuses_without_mesh_address() {
assert_eq!(plan_start(None, &cfg()).unwrap_err(), "unreachable");
}
#[test]
fn env_matches_deploy_mac() {
let (_, env) = plan_start(mesh(), &cfg()).unwrap();
assert_eq!(get(&env, "FORGE_RUNNER_HOST"), "10.5.5.3");
assert_eq!(get(&env, "FORGE_RUNNER_PORT"), "8102");
assert_eq!(get(&env, "FORGE_RUNNER_WORKDIR"), "/Users/o/atelier/work");
assert_eq!(
get(&env, "FORGE_RUNNER_MIN_FREE_BYTES"),
(15u64 << 30).to_string()
);
assert_eq!(
get(&env, "FORGE_ATELIER_PYTHONS"),
r#"{"peft": "/Users/o/atelier/envs/peft/bin/python"}"#
);
assert_eq!(
get(&env, "FORGE_ATELIER_PUBLIC_URL"),
"http://10.5.5.3:8102"
);
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_BROKER"),
"http://192.168.0.23:9150"
);
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_WORKER_KEY_FILE"),
"/Users/o/atelier/worker.key"
);
assert_eq!(get(&env, "FORGE_ATELIER_ZC_NAME"), "atelier-m5");
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_TAGS"),
"atelier-backend:peft,atelier-export:mlx,atelier-scope:owner"
);
assert_eq!(get(&env, "FORGE_ATELIER_EXPORTS"), "mlx,safetensors");
assert_eq!(get(&env, "FORGE_ATELIER_ZC_GPU_MODEL"), "Apple M5");
assert_eq!(get(&env, "FORGE_ATELIER_ZC_GPU_VRAM_GB"), "8");
assert_eq!(get(&env, "FORGE_ATELIER_PRICE_PER_HOUR"), "0.9");
assert_eq!(get(&env, "HF_HOME"), "/Users/o/atelier/hf");
assert_eq!(
get(&env, "PYTHONPATH"),
"/Users/o/atelier/zak-forge/src:/Users/o/atelier/src/sakura-internal"
);
}
#[test]
fn owner_tag_names_the_owner_when_known() {
let mut c = cfg();
c.owner = Some("1234567890".into());
let (_, env) = plan_start(mesh(), &c).unwrap();
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_TAGS"),
"atelier-backend:peft,atelier-export:mlx,atelier-scope:owner,atelier-owner:1234567890"
);
}
#[test]
fn start_uses_background_qos() {
let (argv, _) = plan_start(mesh(), &cfg()).unwrap();
if cfg!(target_os = "macos") {
assert_eq!(&argv[..2], ["taskpolicy", "-b"]);
}
assert!(argv.contains(&"/Users/o/atelier/zak-forge/.venv/bin/python".to_string()));
assert_eq!(
argv.last().unwrap(),
"from forge.runner.service import main; main()"
);
}
#[test]
fn status_reads_jobs_and_pid() {
let v = serde_json::json!({"running": ["j1"], "draining": false, "pid": 4242});
assert_eq!(
parse_status(&v),
Some(RunnerStatus {
jobs: vec!["j1".into()],
pid: Some(4242)
})
);
let old = serde_json::json!({"running": []});
assert_eq!(
parse_status(&old),
Some(RunnerStatus {
jobs: vec![],
pid: None
})
);
assert_eq!(parse_status(&serde_json::json!({})), None);
}
#[test]
fn adopted_pid_is_used_and_never_pid_1() {
let mut r = HostRunner::default();
r.adopt(Some(4242));
assert_eq!(r.pid(), Some(4242));
assert!(!r.owned());
r.adopt(Some(1));
assert_eq!(r.pid(), None);
}
#[test]
fn base_url_is_the_advertised_address() {
assert_eq!(base_url(mesh().unwrap(), 8102), "http://10.5.5.3:8102");
}
}