use super::{NodeSpec, Registry, ID_ENV, NODE_LABEL};
use crate::broker::node_identity::NodeKey;
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
static STOP: AtomicBool = AtomicBool::new(false);
extern "C" fn on_signal(_: libc::c_int) {
STOP.store(true, Ordering::SeqCst);
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Status {
pub state: String,
#[serde(default)]
pub error: Option<String>,
#[serde(default)]
pub pid: u32,
#[serde(default)]
pub broker_port: u16,
#[serde(default)]
pub fingerprint: String,
#[serde(default)]
pub updated: String,
}
impl Status {
pub fn path(dir: &Path) -> PathBuf {
dir.join("status.json")
}
pub fn load(dir: &Path) -> Option<Status> {
serde_json::from_slice(&std::fs::read(Self::path(dir)).ok()?).ok()
}
fn save(&self, dir: &Path) {
let tmp = dir.join(format!(".status.{}", std::process::id()));
if let Ok(b) = serde_json::to_vec(self) {
if std::fs::write(&tmp, b).is_ok() {
let _ = std::fs::rename(&tmp, Self::path(dir));
}
}
}
}
fn now() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
pub(crate) fn engine(args: &[&str]) -> Result<String, String> {
let args: Vec<String> = args.iter().map(|a| a.to_string()).collect();
let out = crate::sandbox::engine::command(&args)?
.stdin(Stdio::null())
.output()
.map_err(|e| format!("container runtime not available: {e}"))?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
} else {
let e = String::from_utf8_lossy(&out.stderr).trim().to_string();
Err(if e.is_empty() {
format!(
"{} failed",
args.first().map(String::as_str).unwrap_or("docker")
)
} else {
e
})
}
}
pub fn prepare(home: &Path, spec: &NodeSpec) -> Result<NodeKey, String> {
let state = spec.state_dir(home);
std::fs::create_dir_all(&state).map_err(|e| format!("{}: {e}", state.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = std::fs::set_permissions(&state, std::fs::Permissions::from_mode(0o700));
}
let key = NodeKey::load_or_create_in(Some(state.clone()));
crate::policy::update(&state, None, None, |p| {
super::apply_policy(p, spec);
Ok(())
})
.map_err(|e| format!("node {} policy: {e}", spec.id))?;
Ok(key)
}
fn fp8(key: &NodeKey) -> String {
key.fingerprint().chars().take(8).collect()
}
fn ensure_workers(home: &Path, spec: &NodeSpec, key: &NodeKey) -> Result<(), String> {
let dir = spec.dir(home);
let file = dir.join("compose.yaml");
let image = crate::up::worker_container_image(&|k| std::env::var(k).ok());
let yaml = super::compose_yaml(spec, &fp8(key), &image);
if std::fs::read_to_string(&file).ok().as_deref() != Some(yaml.as_str()) {
std::fs::write(&file, &yaml).map_err(|e| format!("{}: {e}", file.display()))?;
}
let project = spec.project();
let file_s = file.display().to_string();
if spec.workers == 0 {
let _ = engine(&[
"compose",
"-p",
&project,
"down",
"--remove-orphans",
"-t",
"5",
]);
return Ok(());
}
engine(&[
"compose",
"-p",
&project,
"-f",
&file_s,
"up",
"-d",
"--remove-orphans",
])
.map_err(|e| format!("compose up: {e}"))?;
for w in super::compose_workers(spec, &fp8(key)) {
let alive = format!(
"kill -0 \"$(cat {} 2>/dev/null)\" 2>/dev/null",
crate::up::WORKER_CONTAINER_PID
);
if engine(&["exec", &w.container, "sh", "-c", &alive]).is_ok() {
continue;
}
let cmd = crate::up::worker_exec_cmd(w.port, &w.name, None);
let mut argv = vec!["exec", "-d", w.container.as_str()];
argv.extend(cmd.iter().map(String::as_str));
engine(&argv).map_err(|e| format!("worker {}: {e}", w.name))?;
}
Ok(())
}
pub fn teardown(spec_project: &str, id: &str) {
let _ = engine(&[
"compose",
"-p",
spec_project,
"down",
"--remove-orphans",
"-t",
"5",
]);
let filter = format!("label={NODE_LABEL}={id}");
if let Ok(ids) = engine(&["ps", "-aq", "--filter", &filter]) {
let ids: Vec<&str> = ids.split_whitespace().collect();
if !ids.is_empty() {
let mut argv = vec!["rm", "-f"];
argv.extend(ids);
let _ = engine(&argv);
}
}
}
fn node_name(id: &str) -> String {
let host = hostname::get()
.map(|h| h.to_string_lossy().into_owned())
.unwrap_or_default();
let host = host.trim().trim_end_matches(".local");
if host.is_empty() {
id.to_string()
} else {
format!("{host}-{id}")
}
}
pub(crate) fn broker_env(
home: &Path,
spec: &NodeSpec,
worker_key: Option<String>,
) -> Vec<(String, String)> {
let mut env = vec![
(
"ZAKURO_HOME".to_string(),
spec.state_dir(home).display().to_string(),
),
(
crate::credentials::MACHINE_HOME_ENV.to_string(),
home.display().to_string(),
),
(ID_ENV.to_string(), spec.id.clone()),
("ZAKURO_NODE_NAME".to_string(), node_name(&spec.id)),
("ZAKURO_SCAN_RANGE".to_string(), spec.scan_range()),
("ZAKURO_SCAN_INTERVAL".to_string(), "3".to_string()),
(
"ZAKURO_DEPLOY_BUDGET_SLICE".to_string(),
spec.deploy_slice().to_string(),
),
("ZC_NO_ENV_PROFILE".to_string(), "1".to_string()),
];
if let Some(k) = worker_key {
env.push((crate::worker_key::ENV.to_string(), k));
env.push(("ZAKURO_REQUIRE_WORKER_KEY".to_string(), "1".to_string()));
}
env
}
fn spawn_broker(home: &Path, spec: &NodeSpec) -> Result<Child, String> {
let exe = std::env::current_exe().map_err(|e| e.to_string())?;
let log_path = spec.dir(home).join("broker.log");
let log = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.map_err(|e| format!("{}: {e}", log_path.display()))?;
let err = log.try_clone().map_err(|e| e.to_string())?;
let key = crate::worker_key::ensure(None, &spec.state_dir(home)).ok();
let mut cmd = Command::new(exe);
cmd.arg("broker")
.arg(spec.broker_port().to_string())
.stdin(Stdio::null())
.stdout(log)
.stderr(err);
for (k, v) in broker_env(home, spec, key) {
cmd.env(k, v);
}
cmd.spawn()
.map_err(|e| format!("could not start the broker: {e}"))
}
fn stop_child(child: &mut Child, wait: Duration) {
crate::nodes::ask_to_stop(child);
let until = Instant::now() + wait;
while Instant::now() < until {
if matches!(child.try_wait(), Ok(Some(_))) {
return;
}
std::thread::sleep(Duration::from_millis(100));
}
let _ = child.kill();
let _ = child.wait();
}
fn pause(d: Duration) {
let until = Instant::now() + d;
while Instant::now() < until && !STOP.load(Ordering::SeqCst) {
std::thread::sleep(Duration::from_millis(200));
}
}
pub fn run(id: &str) -> i32 {
let Some(home) = crate::credentials::machine_dir() else {
eprintln!("zc node: no zakuro home (HOME is not set)");
return 1;
};
let Some(spec) = Registry::load(&home).get(id).cloned() else {
eprintln!("zc node: no node {id}. `zc node ls` lists them.");
return 1;
};
std::env::set_var(ID_ENV, &spec.id);
std::env::set_var(crate::credentials::MACHINE_HOME_ENV, &home);
unsafe {
libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t);
libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t);
}
let dir = spec.dir(&home);
let mut status = Status {
state: "starting".into(),
pid: std::process::id(),
broker_port: spec.broker_port(),
updated: now(),
..Default::default()
};
let key = match prepare(&home, &spec) {
Ok(k) => k,
Err(e) => {
eprintln!("zc node {id}: {e}");
let _ = std::fs::create_dir_all(&dir);
status.state = "error".into();
status.error = Some(e);
status.save(&dir);
return 1;
}
};
status.fingerprint = key.fingerprint();
status.save(&dir);
println!(
"[NODE {id}] {} · broker :{} · {} worker(s){}",
key.node_uri(),
spec.broker_port(),
spec.workers,
spec.environment
.as_deref()
.map(|e| format!(" · environment {e}"))
.unwrap_or_default()
);
let mut broker: Option<Child> = None;
let mut backoff = Duration::from_secs(2);
let mut next_broker = Instant::now();
let mut next_workers = Instant::now();
while !STOP.load(Ordering::SeqCst) {
let mut error = None;
if Instant::now() >= next_workers {
let live = Registry::load(&home).get(id).cloned();
match live {
None => break, Some(s) if s != spec => {
println!("[NODE {id}] its settings changed: restarting");
break;
}
Some(_) => {}
}
match ensure_workers(&home, &spec, &key) {
Ok(()) => next_workers = Instant::now() + Duration::from_secs(15),
Err(e) => {
eprintln!("[NODE {id}] {e}");
error = Some(e);
next_workers = Instant::now() + Duration::from_secs(10);
}
}
}
let dead = match broker.as_mut() {
None => true,
Some(c) => match c.try_wait() {
Ok(None) => false,
Ok(Some(st)) => {
eprintln!("[NODE {id}] the broker exited ({st})");
true
}
Err(_) => true,
},
};
if dead && Instant::now() >= next_broker {
match spawn_broker(&home, &spec) {
Ok(c) => {
broker = Some(c);
next_broker = Instant::now() + backoff;
backoff = (backoff * 2).min(Duration::from_secs(60));
}
Err(e) => {
eprintln!("[NODE {id}] {e}");
error = Some(e);
next_broker = Instant::now() + backoff;
}
}
} else if !dead && Instant::now() > next_broker + Duration::from_secs(60) {
backoff = Duration::from_secs(2); }
let state = if error.is_some() {
"error"
} else if dead {
"starting"
} else {
"running"
};
if status.state != state || status.error != error {
status.state = state.into();
status.error = error;
status.updated = now();
status.save(&dir);
}
pause(Duration::from_secs(2));
}
println!("[NODE {id}] stopping");
if let Some(mut c) = broker {
stop_child(&mut c, Duration::from_secs(10));
}
teardown(&spec.project(), &spec.id);
status.state = "stopped".into();
status.error = None;
status.updated = now();
status.save(&dir);
0
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_nodes_broker_gets_its_own_home_ports_and_the_machines_home() {
let spec = NodeSpec {
id: "pytorch".into(),
index: 2,
environment: Some("pytorch".into()),
environment_name: None,
workers: 0,
cpu_millis: 1000,
memory_bytes: 1 << 30,
disk_bytes: 10 << 30,
created: "t".into(),
};
let env = broker_env(Path::new("/h/.zakuro"), &spec, Some("k".into()));
let get = |k: &str| env.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str());
assert_eq!(get("ZAKURO_HOME"), Some("/h/.zakuro/nodes/pytorch/state"));
assert_eq!(get("ZAKURO_MACHINE_HOME"), Some("/h/.zakuro"));
assert_eq!(get("ZAKURO_NODE_ID"), Some("pytorch"));
assert_eq!(get("ZAKURO_SCAN_RANGE"), Some("4040-4051"));
assert_eq!(get("ZAKURO_DEPLOY_BUDGET_SLICE"), Some("100%"));
assert_eq!(get("ZAKURO_REQUIRE_WORKER_KEY"), Some("1"));
assert!(get("ZAKURO_NODE_NAME").unwrap().ends_with("pytorch"));
assert_eq!(spec.broker_port(), 9004);
}
#[test]
fn prepare_gives_the_node_its_own_key_and_share() {
let home = std::env::temp_dir().join(format!("zc-node-prep-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
let spec = NodeSpec {
id: "jobs".into(),
index: 1,
environment: None,
environment_name: None,
workers: 1,
cpu_millis: 1500,
memory_bytes: 2 << 30,
disk_bytes: 10 << 30,
created: "t".into(),
};
let a = prepare(&home, &spec).unwrap();
let b = prepare(&home, &spec).unwrap();
assert_eq!(a.fingerprint(), b.fingerprint(), "the key is kept");
assert!(home.join("nodes/jobs/state/node_key").is_file());
let p = crate::policy::load(&home.join("nodes/jobs/state")).unwrap();
assert_eq!(p.budget.cpu, "1.5");
assert!(!p.workloads.environments && p.workloads.spaces);
let _ = std::fs::remove_dir_all(&home);
}
}