use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::process::Child;
use std::time::Duration;
#[derive(Debug, Clone, PartialEq)]
pub struct HostRunnerCfg {
pub backend: String,
pub port: u16,
pub root: PathBuf,
pub broker: String,
pub key_file: PathBuf,
pub name: String,
pub price_per_hour: String,
pub gpu_model: String,
pub gpu_mem_gb: u64,
pub owner: Option<String>,
pub token: Option<Secret>,
}
#[derive(Clone, PartialEq, Eq)]
pub struct Secret(String);
impl Secret {
pub fn new(v: impl Into<String>) -> Self {
Self(v.into())
}
pub fn expose(&self) -> &str {
&self.0
}
}
impl std::fmt::Debug for Secret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Secret(redacted)")
}
}
pub const TASKPOLICY: &str = "/usr/sbin/taskpolicy";
pub const MIN_FREE_BYTES: u64 = 15 << 30;
const ENTRY: &str = "from forge.runner.service import main; main()";
impl HostRunnerCfg {
pub fn runner_python(&self) -> PathBuf {
self.root.join("zak-forge/.venv/bin/python")
}
pub fn backend_python(&self) -> PathBuf {
self.root
.join("envs")
.join(&self.backend)
.join("bin/python")
}
pub fn log_path(&self) -> PathBuf {
self.root.join("logs/runner.log")
}
pub fn installed(&self) -> bool {
self.runner_python().exists() && self.backend_python().exists()
}
}
pub fn base_url(addr: IpAddr, port: u16) -> String {
match addr {
IpAddr::V6(a) => format!("http://[{a}]:{port}"),
IpAddr::V4(a) => format!("http://{a}:{port}"),
}
}
pub type StartPlan = (Vec<String>, Vec<(String, String)>);
pub fn plan_start(
mesh_addr: Option<IpAddr>,
cfg: &HostRunnerCfg,
) -> Result<StartPlan, &'static str> {
let addr = mesh_addr.ok_or("unreachable")?;
let root = |p: &str| cfg.root.join(p).display().to_string();
let env: Vec<(String, String)> = [
("FORGE_RUNNER_HOST", addr.to_string()),
("FORGE_RUNNER_PORT", cfg.port.to_string()),
("FORGE_RUNNER_WORKDIR", root("work")),
("FORGE_RUNNER_MIN_FREE_BYTES", MIN_FREE_BYTES.to_string()),
(
"FORGE_ATELIER_PYTHONS",
format!(
r#"{{"{}": "{}"}}"#,
cfg.backend,
cfg.backend_python().display()
),
),
("FORGE_ATELIER_PUBLIC_URL", base_url(addr, cfg.port)),
("FORGE_ATELIER_ZC_BROKER", cfg.broker.clone()),
(
"FORGE_ATELIER_ZC_WORKER_KEY_FILE",
cfg.key_file.display().to_string(),
),
("FORGE_ATELIER_ZC_NAME", cfg.name.clone()),
(
"FORGE_ATELIER_ZC_TAGS",
format!(
"atelier-backend:{},atelier-export:mlx,atelier-scope:owner{}",
cfg.backend,
cfg.owner
.as_deref()
.map(|o| format!(",atelier-owner:{o}"))
.unwrap_or_default()
),
),
("FORGE_ATELIER_EXPORTS", "mlx,safetensors".into()),
("FORGE_ATELIER_ZC_GPU_MODEL", cfg.gpu_model.clone()),
("FORGE_ATELIER_ZC_GPU_VRAM_GB", cfg.gpu_mem_gb.to_string()),
("FORGE_ATELIER_PRICE_PER_HOUR", cfg.price_per_hour.clone()),
("HF_HOME", root("hf")),
(
"PYTHONPATH",
format!("{}:{}", root("zak-forge/src"), root("src/sakura-internal")),
),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v))
.chain(
cfg.token
.as_ref()
.map(|t| ("FORGE_RUNNER_TOKEN".to_string(), t.expose().to_string())),
)
.collect();
let mut argv = Vec::new();
if cfg!(target_os = "macos") {
argv.extend([TASKPOLICY.to_string(), "-b".to_string()]);
}
argv.extend([
cfg.runner_python().display().to_string(),
"-c".to_string(),
ENTRY.to_string(),
]);
Ok((argv, env))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunnerStatus {
pub jobs: Vec<String>,
pub pid: Option<u32>,
}
pub fn parse_status(v: &serde_json::Value) -> Option<RunnerStatus> {
Some(RunnerStatus {
jobs: v
.get("running")?
.as_array()?
.iter()
.filter_map(|j| j.as_str().map(String::from))
.collect(),
pid: v
.get("pid")
.and_then(serde_json::Value::as_u64)
.and_then(|p| u32::try_from(p).ok()),
})
}
pub struct RunnerApi {
pub base: String,
pub token: Option<Secret>,
}
impl RunnerApi {
fn agent() -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(5)))
.http_status_as_error(false)
.build()
.into()
}
fn auth<B>(&self, req: ureq::RequestBuilder<B>) -> ureq::RequestBuilder<B> {
match &self.token {
Some(t) => req.header("Authorization", &format!("Bearer {}", t.expose())),
None => req,
}
}
pub fn status(&self) -> Option<RunnerStatus> {
let mut resp = self
.auth(Self::agent().get(&format!("{}/status", self.base)))
.call()
.ok()?;
if resp.status() != 200 {
return None;
}
let v: serde_json::Value = resp.body_mut().read_json().ok()?;
parse_status(&v)
}
pub fn running(&self) -> Option<Vec<String>> {
self.status().map(|s| s.jobs)
}
pub fn drain(&self, on: bool) -> bool {
self.auth(Self::agent().post(&format!("{}/drain", self.base)))
.send_json(serde_json::json!({ "on": on }))
.is_ok_and(|r| r.status() == 200)
}
pub fn wrap_up(&self, job_id: &str) -> bool {
self.auth(Self::agent().post(&format!("{}/jobs/{job_id}/wrap-up", self.base)))
.send_json(serde_json::json!({}))
.is_ok_and(|r| r.status().is_success())
}
}
#[derive(Default)]
pub struct HostRunner {
child: Option<Child>,
adopted: Option<u32>,
}
impl HostRunner {
pub fn pid(&self) -> Option<u32> {
self.child.as_ref().map(|c| c.id()).or(self.adopted)
}
pub fn owned(&self) -> bool {
self.child.is_some()
}
pub fn adopt(&mut self, pid: Option<u32>) {
if self.child.is_none() {
self.adopted = pid.filter(|p| *p > 1);
}
}
pub fn alive(&mut self) -> bool {
match self.child.as_mut().map(|c| c.try_wait()) {
Some(Ok(None)) => true,
Some(_) => {
self.child = None;
false
}
None => false,
}
}
pub fn start(
&mut self,
argv: &[String],
env: &[(String, String)],
cwd: &Path,
log: &Path,
) -> std::io::Result<()> {
if self.alive() {
return Ok(());
}
self.child = Some(crate::agent::supervisor::spawn_in_group(
argv,
Some(cwd),
env,
log,
)?);
Ok(())
}
pub fn stop(&mut self) {
if let Some(pid) = self.adopted.take() {
let alive = |p: u32| unsafe { libc::kill(p as libc::pid_t, 0) } == 0;
unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) };
for _ in 0..50 {
if !alive(pid) {
return;
}
std::thread::sleep(Duration::from_millis(100));
}
unsafe { libc::kill(pid as libc::pid_t, libc::SIGKILL) };
return;
}
if let Some(mut c) = self.child.take() {
crate::agent::supervisor::signal_group(c.id(), libc::SIGTERM);
for _ in 0..50 {
if matches!(c.try_wait(), Ok(Some(_))) {
return;
}
std::thread::sleep(Duration::from_millis(100));
}
crate::agent::supervisor::signal_group(c.id(), libc::SIGKILL);
let _ = c.wait();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::net::{IpAddr, Ipv4Addr};
use std::path::PathBuf;
fn cfg() -> HostRunnerCfg {
HostRunnerCfg {
backend: "peft".into(),
port: 8102,
root: PathBuf::from("/Users/o/atelier"),
broker: "http://192.168.0.23:9150".into(),
key_file: PathBuf::from("/Users/o/atelier/worker.key"),
name: "atelier-m5".into(),
price_per_hour: "0.9".into(),
gpu_model: "Apple M5".into(),
gpu_mem_gb: 8,
owner: None,
token: None,
}
}
fn mesh() -> Option<IpAddr> {
Some(IpAddr::V4(Ipv4Addr::new(10, 5, 5, 3)))
}
fn get<'a>(env: &'a [(String, String)], k: &str) -> &'a str {
env.iter()
.find(|(key, _)| key == k)
.map(|(_, v)| v.as_str())
.unwrap_or_else(|| panic!("{k} missing"))
}
#[test]
fn supervisor_refuses_without_mesh_address() {
assert_eq!(plan_start(None, &cfg()).unwrap_err(), "unreachable");
}
#[test]
fn env_matches_deploy_mac() {
let (_, env) = plan_start(mesh(), &cfg()).unwrap();
assert_eq!(get(&env, "FORGE_RUNNER_HOST"), "10.5.5.3");
assert_eq!(get(&env, "FORGE_RUNNER_PORT"), "8102");
assert_eq!(get(&env, "FORGE_RUNNER_WORKDIR"), "/Users/o/atelier/work");
assert_eq!(
get(&env, "FORGE_RUNNER_MIN_FREE_BYTES"),
(15u64 << 30).to_string()
);
assert_eq!(
get(&env, "FORGE_ATELIER_PYTHONS"),
r#"{"peft": "/Users/o/atelier/envs/peft/bin/python"}"#
);
assert_eq!(
get(&env, "FORGE_ATELIER_PUBLIC_URL"),
"http://10.5.5.3:8102"
);
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_BROKER"),
"http://192.168.0.23:9150"
);
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_WORKER_KEY_FILE"),
"/Users/o/atelier/worker.key"
);
assert_eq!(get(&env, "FORGE_ATELIER_ZC_NAME"), "atelier-m5");
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_TAGS"),
"atelier-backend:peft,atelier-export:mlx,atelier-scope:owner"
);
assert_eq!(get(&env, "FORGE_ATELIER_EXPORTS"), "mlx,safetensors");
assert_eq!(get(&env, "FORGE_ATELIER_ZC_GPU_MODEL"), "Apple M5");
assert_eq!(get(&env, "FORGE_ATELIER_ZC_GPU_VRAM_GB"), "8");
assert_eq!(get(&env, "FORGE_ATELIER_PRICE_PER_HOUR"), "0.9");
assert_eq!(get(&env, "HF_HOME"), "/Users/o/atelier/hf");
assert_eq!(
get(&env, "PYTHONPATH"),
"/Users/o/atelier/zak-forge/src:/Users/o/atelier/src/sakura-internal"
);
}
#[test]
fn owner_tag_names_the_owner_when_known() {
let mut c = cfg();
c.owner = Some("1234567890".into());
let (_, env) = plan_start(mesh(), &c).unwrap();
assert_eq!(
get(&env, "FORGE_ATELIER_ZC_TAGS"),
"atelier-backend:peft,atelier-export:mlx,atelier-scope:owner,atelier-owner:1234567890"
);
}
#[test]
fn start_uses_background_qos() {
let (argv, _) = plan_start(mesh(), &cfg()).unwrap();
if cfg!(target_os = "macos") {
assert_eq!(&argv[..2], ["/usr/sbin/taskpolicy", "-b"]);
}
assert!(argv.contains(&"/Users/o/atelier/zak-forge/.venv/bin/python".to_string()));
assert_eq!(
argv.last().unwrap(),
"from forge.runner.service import main; main()"
);
}
#[test]
fn runner_token_reaches_the_runner_and_is_never_debug_printed() {
let mut c = cfg();
c.token = Some(Secret::new("s3cret-runner-token"));
let (_, env) = plan_start(mesh(), &c).unwrap();
assert_eq!(get(&env, "FORGE_RUNNER_TOKEN"), "s3cret-runner-token");
assert!(!format!("{c:?}").contains("s3cret"));
let (_, env) = plan_start(mesh(), &cfg()).unwrap();
assert!(env.iter().all(|(k, _)| k != "FORGE_RUNNER_TOKEN"));
}
#[test]
fn runner_api_sends_the_bearer() {
use std::io::{Read, Write};
let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = l.local_addr().unwrap().port();
let seen = std::thread::spawn(move || {
let (mut s, _) = l.accept().unwrap();
let mut buf = [0u8; 2048];
let n = s.read(&mut buf).unwrap();
let body = r#"{"running":[],"draining":false,"pid":7}"#;
let _ = s.write_all(
format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes(),
);
String::from_utf8_lossy(&buf[..n]).to_string()
});
let api = RunnerApi {
base: format!("http://127.0.0.1:{port}"),
token: Some(Secret::new("tok-1")),
};
assert_eq!(api.running(), Some(vec![]));
let req = seen.join().unwrap().to_ascii_lowercase();
assert!(req.contains("authorization: bearer tok-1"), "{req}");
}
#[test]
fn status_reads_jobs_and_pid() {
let v = serde_json::json!({"running": ["j1"], "draining": false, "pid": 4242});
assert_eq!(
parse_status(&v),
Some(RunnerStatus {
jobs: vec!["j1".into()],
pid: Some(4242)
})
);
let old = serde_json::json!({"running": []});
assert_eq!(
parse_status(&old),
Some(RunnerStatus {
jobs: vec![],
pid: None
})
);
assert_eq!(parse_status(&serde_json::json!({})), None);
}
#[test]
fn adopted_pid_is_used_and_never_pid_1() {
let mut r = HostRunner::default();
r.adopt(Some(4242));
assert_eq!(r.pid(), Some(4242));
assert!(!r.owned());
r.adopt(Some(1));
assert_eq!(r.pid(), None);
}
#[test]
fn base_url_is_the_advertised_address() {
assert_eq!(base_url(mesh().unwrap(), 8102), "http://10.5.5.3:8102");
}
}