zc2 0.0.51

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! The agent's side of the machine's extra nodes (ADR ZC-0027).
//!
//! Every few seconds: one `zc node run <id>` child per registered node while
//! sharing is on (none while it is paused), a hub subscription for each node
//! that hosts an environment, and the clean-up of removed nodes (stop, take
//! the environment off the hub, delete the directory and its identity).
//! The main node is not here: it is the agent's own broker and workers.

use crate::nodes::{runner, NodeSpec, Registry};
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};

/// Written in a node's directory once the hub took its subscription: the
/// template id.
pub const SUBSCRIBED_MARKER: &str = "subscribed";
const RESTART_AFTER: Duration = Duration::from_secs(5);
const HUB_RETRY: Duration = Duration::from_secs(30);

/// What a tick asks of the hub, done by the caller (it needs the client).
#[derive(Debug, Clone, PartialEq)]
pub enum HubWork {
    Subscribe {
        spec: NodeSpec,
        node_pubkey: String,
    },
    /// `node_pubkey` is `None` when the node's key is already gone.
    Unsubscribe {
        spec: NodeSpec,
        node_pubkey: Option<String>,
    },
}

#[derive(Default)]
pub struct Manager {
    children: BTreeMap<String, Child>,
    died: BTreeMap<String, Instant>,
    hub_tried: BTreeMap<String, Instant>,
    /// The main node's share at the last tick, to see it shrink.
    main_share: Option<crate::nodes::MachineLimit>,
}

/// Whether the main node now has less than before: its running workers
/// were cut from the old, larger share and have to be started again.
pub fn shrank(before: &crate::nodes::MachineLimit, now: &crate::nodes::MachineLimit) -> bool {
    now.cpu_millis < before.cpu_millis || now.memory_bytes < before.memory_bytes
}

fn node_pubkey(home: &Path, spec: &NodeSpec) -> Option<String> {
    crate::broker::node_identity::NodeKey::load_in(Some(spec.state_dir(home)))
        .map(|k| k.public_b64())
}

fn stop(child: &mut Child, wait: Duration) {
    crate::nodes::ask_to_stop(child);
    let until = Instant::now() + wait;
    while Instant::now() < until {
        if matches!(child.try_wait(), Ok(Some(_))) {
            return;
        }
        std::thread::sleep(Duration::from_millis(100));
    }
    let _ = child.kill();
    let _ = child.wait();
}

/// Which registered nodes should have a runner: all of them while sharing,
/// none while paused.
pub fn wanted(reg: &Registry, sharing: bool) -> Vec<String> {
    if sharing {
        reg.nodes.iter().map(|n| n.id.clone()).collect()
    } else {
        Vec::new()
    }
}

impl Manager {
    fn spawn(home: &Path, spec: &NodeSpec) -> Result<Child, String> {
        let exe = std::env::current_exe().map_err(|e| e.to_string())?;
        let dir = spec.dir(home);
        std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
        let log = std::fs::OpenOptions::new()
            .create(true)
            .append(true)
            .open(dir.join("node.log"))
            .map_err(|e| e.to_string())?;
        let err = log.try_clone().map_err(|e| e.to_string())?;
        Command::new(exe)
            .args(["node", "run", &spec.id])
            .stdin(Stdio::null())
            .stdout(log)
            .stderr(err)
            .spawn()
            .map_err(|e| format!("could not start node {}: {e}", spec.id))
    }

    /// One pass over the registry. Blocking (it starts and stops processes).
    pub fn tick(&mut self, home: &Path, sharing: bool) -> Vec<HubWork> {
        let reg = Registry::load(home);
        // The main node's share shrank (a node was added, or the owner
        // lowered the limit): remove its workers, and the agent's supervisor
        // starts them again at their new, smaller size. A larger share
        // applies as workers restart on their own; nothing is interrupted
        // for it.
        if let Ok(limit) = crate::nodes::MachineLimit::load(home) {
            let share = limit.main_share(&reg.reserved());
            if self.main_share.as_ref().is_some_and(|b| shrank(b, &share)) {
                let names = crate::nodes::usage::main_worker_containers();
                if !names.is_empty() {
                    eprintln!(
                        "  [NODES] the main node's share shrank: restarting its {} worker(s) at the new size",
                        names.len()
                    );
                    let mut argv = vec!["rm", "-f"];
                    argv.extend(names.iter().map(String::as_str));
                    let _ = runner::engine(&argv);
                }
            }
            self.main_share = Some(share);
        }
        let want = wanted(&reg, sharing);
        // Stop what should not run: removed nodes, and everything when paused.
        let extra: Vec<String> = self
            .children
            .keys()
            .filter(|id| !want.contains(id))
            .cloned()
            .collect();
        for id in extra {
            if let Some(mut c) = self.children.remove(&id) {
                eprintln!("  [NODES] stopping node {id}");
                stop(&mut c, Duration::from_secs(25));
            }
        }
        // Reap the ones that exited by themselves (a changed spec makes the
        // runner leave so that it restarts with the new one).
        let exited: Vec<String> = self
            .children
            .iter_mut()
            .filter_map(|(id, c)| matches!(c.try_wait(), Ok(Some(_)) | Err(_)).then(|| id.clone()))
            .collect();
        for id in exited {
            self.children.remove(&id);
            self.died.insert(id, Instant::now());
        }
        for spec in reg.nodes.iter().filter(|n| want.contains(&n.id)) {
            if self.children.contains_key(&spec.id) {
                continue;
            }
            if self
                .died
                .get(&spec.id)
                .is_some_and(|t| t.elapsed() < RESTART_AFTER)
            {
                continue;
            }
            match Self::spawn(home, spec) {
                Ok(c) => {
                    eprintln!("  [NODES] started node {} (pid {})", spec.id, c.id());
                    self.children.insert(spec.id.clone(), c);
                }
                Err(e) => {
                    eprintln!("  [NODES] {e}");
                    self.died.insert(spec.id.clone(), Instant::now());
                }
            }
        }

        let mut work = Vec::new();
        let due = |id: &str, tried: &mut BTreeMap<String, Instant>| {
            let ok = tried.get(id).is_none_or(|t| t.elapsed() >= HUB_RETRY);
            if ok {
                tried.insert(id.to_string(), Instant::now());
            }
            ok
        };
        // A running environment node the hub does not know hosts it yet.
        for spec in &reg.nodes {
            let Some(template) = &spec.environment else {
                continue;
            };
            let dir = spec.dir(home);
            let marked = std::fs::read_to_string(dir.join(SUBSCRIBED_MARKER)).ok();
            let running = runner::Status::load(&dir).is_some_and(|s| s.state == "running");
            if marked.as_deref().map(str::trim) == Some(template) || !running {
                continue;
            }
            if let Some(pk) = node_pubkey(home, spec) {
                if due(&spec.id, &mut self.hub_tried) {
                    work.push(HubWork::Subscribe {
                        spec: spec.clone(),
                        node_pubkey: pk,
                    });
                }
            }
        }
        // Removed nodes: off the hub first (when they were on it), then gone.
        for spec in crate::nodes::removed(home) {
            let dir = spec.dir(home);
            if dir.join(SUBSCRIBED_MARKER).exists() {
                if due(&format!("rm:{}", spec.id), &mut self.hub_tried) {
                    work.push(HubWork::Unsubscribe {
                        node_pubkey: node_pubkey(home, &spec),
                        spec,
                    });
                }
            } else {
                finish_removal(home, &spec);
            }
        }
        work
    }

    /// Stop every node (the agent is going away).
    pub fn stop_all(&mut self) {
        for c in self.children.values_mut() {
            crate::nodes::ask_to_stop(c);
        }
        for (_, mut c) in std::mem::take(&mut self.children) {
            stop(&mut c, Duration::from_secs(25));
        }
    }
}

/// The hub took the subscription: remember it.
pub fn mark_subscribed(home: &Path, spec: &NodeSpec) {
    if let Some(t) = &spec.environment {
        let _ = std::fs::write(spec.dir(home).join(SUBSCRIBED_MARKER), t);
    }
}

/// The hub dropped it (or never had it): the removal can finish.
pub fn mark_unsubscribed(home: &Path, spec: &NodeSpec) {
    let _ = std::fs::remove_file(spec.dir(home).join(SUBSCRIBED_MARKER));
}

/// Delete a removed node for good: whatever it left in the engine, then its
/// directory (identity, policy, logs). Only ever a directory under
/// `<home>/nodes/` that carries the removal marker.
pub fn finish_removal(home: &Path, spec: &NodeSpec) {
    let dir: PathBuf = spec.dir(home);
    if !crate::nodes::valid_id(&spec.id)
        || !dir.starts_with(crate::nodes::nodes_dir(home))
        || !dir.join(crate::nodes::REMOVED_MARKER).is_file()
    {
        return;
    }
    runner::teardown(&spec.project(), &spec.id);
    match std::fs::remove_dir_all(&dir) {
        Ok(()) => eprintln!("  [NODES] removed node {}", spec.id),
        Err(e) => eprintln!("  [NODES] could not delete {}: {e}", dir.display()),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn spec(id: &str) -> NodeSpec {
        NodeSpec {
            id: id.into(),
            index: 1,
            environment: None,
            environment_name: None,
            workers: 1,
            cpu_millis: 1000,
            memory_bytes: 1 << 30,
            disk_bytes: 10 << 30,
            created: "t".into(),
        }
    }

    #[test]
    fn only_a_smaller_share_restarts_the_main_nodes_workers() {
        let m = |c, g: u64| crate::nodes::MachineLimit {
            cpu_millis: c,
            memory_bytes: g << 30,
            disk_bytes: 0,
        };
        assert!(shrank(&m(5000, 8), &m(3000, 8)));
        assert!(shrank(&m(5000, 8), &m(5000, 6)));
        assert!(!shrank(&m(3000, 6), &m(5000, 8)));
        assert!(!shrank(&m(5000, 8), &m(5000, 8)));
    }

    #[test]
    fn nodes_run_only_while_sharing() {
        let reg = Registry {
            v: 1,
            nodes: vec![spec("a"), spec("b")],
        };
        assert_eq!(wanted(&reg, true), ["a", "b"]);
        assert!(wanted(&reg, false).is_empty());
    }

    #[test]
    fn only_a_marked_directory_under_nodes_is_ever_deleted() {
        let home = std::env::temp_dir().join(format!("zc-nm-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&home);
        let s = spec("gone");
        let dir = s.dir(&home);
        std::fs::create_dir_all(dir.join("state")).unwrap();
        // No marker: a registered node's directory is never touched.
        finish_removal(&home, &s);
        assert!(dir.is_dir());
        // An id that climbs out of nodes/ is refused too.
        let mut evil = spec("../../etc");
        evil.id = "../x".into();
        std::fs::create_dir_all(home.join("x")).unwrap();
        std::fs::write(home.join("x").join(crate::nodes::REMOVED_MARKER), b"{}").unwrap();
        finish_removal(&home, &evil);
        assert!(home.join("x").is_dir());
        let _ = std::fs::remove_dir_all(&home);
    }
}