use crate::nodes::{runner, NodeSpec, Registry};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
pub const SUBSCRIBED_MARKER: &str = "subscribed";
const RESTART_AFTER: Duration = Duration::from_secs(5);
const HUB_RETRY: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, PartialEq)]
pub enum HubWork {
Subscribe {
spec: NodeSpec,
node_pubkey: String,
},
Unsubscribe {
spec: NodeSpec,
node_pubkey: Option<String>,
},
}
#[derive(Default)]
pub struct Manager {
children: BTreeMap<String, Child>,
died: BTreeMap<String, Instant>,
hub_tried: BTreeMap<String, Instant>,
main_share: Option<crate::nodes::MachineLimit>,
}
pub fn shrank(before: &crate::nodes::MachineLimit, now: &crate::nodes::MachineLimit) -> bool {
now.cpu_millis < before.cpu_millis || now.memory_bytes < before.memory_bytes
}
fn node_pubkey(home: &Path, spec: &NodeSpec) -> Option<String> {
crate::broker::node_identity::NodeKey::load_in(Some(spec.state_dir(home)))
.map(|k| k.public_b64())
}
fn stop(child: &mut Child, wait: Duration) {
crate::nodes::ask_to_stop(child);
let until = Instant::now() + wait;
while Instant::now() < until {
if matches!(child.try_wait(), Ok(Some(_))) {
return;
}
std::thread::sleep(Duration::from_millis(100));
}
let _ = child.kill();
let _ = child.wait();
}
pub fn wanted(reg: &Registry, sharing: bool) -> Vec<String> {
if sharing {
reg.nodes.iter().map(|n| n.id.clone()).collect()
} else {
Vec::new()
}
}
impl Manager {
fn spawn(home: &Path, spec: &NodeSpec) -> Result<Child, String> {
let exe = std::env::current_exe().map_err(|e| e.to_string())?;
let dir = spec.dir(home);
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
let log = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(dir.join("node.log"))
.map_err(|e| e.to_string())?;
let err = log.try_clone().map_err(|e| e.to_string())?;
Command::new(exe)
.args(["node", "run", &spec.id])
.stdin(Stdio::null())
.stdout(log)
.stderr(err)
.spawn()
.map_err(|e| format!("could not start node {}: {e}", spec.id))
}
pub fn tick(&mut self, home: &Path, sharing: bool) -> Vec<HubWork> {
let reg = Registry::load(home);
if let Ok(limit) = crate::nodes::MachineLimit::load(home) {
let share = limit.main_share(®.reserved());
if self.main_share.as_ref().is_some_and(|b| shrank(b, &share)) {
let names = crate::nodes::usage::main_worker_containers();
if !names.is_empty() {
eprintln!(
" [NODES] the main node's share shrank: restarting its {} worker(s) at the new size",
names.len()
);
let mut argv = vec!["rm", "-f"];
argv.extend(names.iter().map(String::as_str));
let _ = runner::engine(&argv);
}
}
self.main_share = Some(share);
}
let want = wanted(®, sharing);
let extra: Vec<String> = self
.children
.keys()
.filter(|id| !want.contains(id))
.cloned()
.collect();
for id in extra {
if let Some(mut c) = self.children.remove(&id) {
eprintln!(" [NODES] stopping node {id}");
stop(&mut c, Duration::from_secs(25));
}
}
let exited: Vec<String> = self
.children
.iter_mut()
.filter_map(|(id, c)| matches!(c.try_wait(), Ok(Some(_)) | Err(_)).then(|| id.clone()))
.collect();
for id in exited {
self.children.remove(&id);
self.died.insert(id, Instant::now());
}
for spec in reg.nodes.iter().filter(|n| want.contains(&n.id)) {
if self.children.contains_key(&spec.id) {
continue;
}
if self
.died
.get(&spec.id)
.is_some_and(|t| t.elapsed() < RESTART_AFTER)
{
continue;
}
match Self::spawn(home, spec) {
Ok(c) => {
eprintln!(" [NODES] started node {} (pid {})", spec.id, c.id());
self.children.insert(spec.id.clone(), c);
}
Err(e) => {
eprintln!(" [NODES] {e}");
self.died.insert(spec.id.clone(), Instant::now());
}
}
}
let mut work = Vec::new();
let due = |id: &str, tried: &mut BTreeMap<String, Instant>| {
let ok = tried.get(id).is_none_or(|t| t.elapsed() >= HUB_RETRY);
if ok {
tried.insert(id.to_string(), Instant::now());
}
ok
};
for spec in ®.nodes {
let Some(template) = &spec.environment else {
continue;
};
let dir = spec.dir(home);
let marked = std::fs::read_to_string(dir.join(SUBSCRIBED_MARKER)).ok();
let running = runner::Status::load(&dir).is_some_and(|s| s.state == "running");
if marked.as_deref().map(str::trim) == Some(template) || !running {
continue;
}
if let Some(pk) = node_pubkey(home, spec) {
if due(&spec.id, &mut self.hub_tried) {
work.push(HubWork::Subscribe {
spec: spec.clone(),
node_pubkey: pk,
});
}
}
}
for spec in crate::nodes::removed(home) {
let dir = spec.dir(home);
if dir.join(SUBSCRIBED_MARKER).exists() {
if due(&format!("rm:{}", spec.id), &mut self.hub_tried) {
work.push(HubWork::Unsubscribe {
node_pubkey: node_pubkey(home, &spec),
spec,
});
}
} else {
finish_removal(home, &spec);
}
}
work
}
pub fn stop_all(&mut self) {
for c in self.children.values_mut() {
crate::nodes::ask_to_stop(c);
}
for (_, mut c) in std::mem::take(&mut self.children) {
stop(&mut c, Duration::from_secs(25));
}
}
}
pub fn mark_subscribed(home: &Path, spec: &NodeSpec) {
if let Some(t) = &spec.environment {
let _ = std::fs::write(spec.dir(home).join(SUBSCRIBED_MARKER), t);
}
}
pub fn mark_unsubscribed(home: &Path, spec: &NodeSpec) {
let _ = std::fs::remove_file(spec.dir(home).join(SUBSCRIBED_MARKER));
}
pub fn finish_removal(home: &Path, spec: &NodeSpec) {
let dir: PathBuf = spec.dir(home);
if !crate::nodes::valid_id(&spec.id)
|| !dir.starts_with(crate::nodes::nodes_dir(home))
|| !dir.join(crate::nodes::REMOVED_MARKER).is_file()
{
return;
}
runner::teardown(&spec.project(), &spec.id);
match std::fs::remove_dir_all(&dir) {
Ok(()) => eprintln!(" [NODES] removed node {}", spec.id),
Err(e) => eprintln!(" [NODES] could not delete {}: {e}", dir.display()),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn spec(id: &str) -> NodeSpec {
NodeSpec {
id: id.into(),
index: 1,
environment: None,
environment_name: None,
workers: 1,
cpu_millis: 1000,
memory_bytes: 1 << 30,
disk_bytes: 10 << 30,
created: "t".into(),
}
}
#[test]
fn only_a_smaller_share_restarts_the_main_nodes_workers() {
let m = |c, g: u64| crate::nodes::MachineLimit {
cpu_millis: c,
memory_bytes: g << 30,
disk_bytes: 0,
};
assert!(shrank(&m(5000, 8), &m(3000, 8)));
assert!(shrank(&m(5000, 8), &m(5000, 6)));
assert!(!shrank(&m(3000, 6), &m(5000, 8)));
assert!(!shrank(&m(5000, 8), &m(5000, 8)));
}
#[test]
fn nodes_run_only_while_sharing() {
let reg = Registry {
v: 1,
nodes: vec![spec("a"), spec("b")],
};
assert_eq!(wanted(®, true), ["a", "b"]);
assert!(wanted(®, false).is_empty());
}
#[test]
fn only_a_marked_directory_under_nodes_is_ever_deleted() {
let home = std::env::temp_dir().join(format!("zc-nm-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&home);
let s = spec("gone");
let dir = s.dir(&home);
std::fs::create_dir_all(dir.join("state")).unwrap();
finish_removal(&home, &s);
assert!(dir.is_dir());
let mut evil = spec("../../etc");
evil.id = "../x".into();
std::fs::create_dir_all(home.join("x")).unwrap();
std::fs::write(home.join("x").join(crate::nodes::REMOVED_MARKER), b"{}").unwrap();
finish_removal(&home, &evil);
assert!(home.join("x").is_dir());
let _ = std::fs::remove_dir_all(&home);
}
}