zc2 0.0.52

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! `GET /v2/local-nodes`: every node on this machine, the main one first,
//! with what it runs, its share of the owner's limit, and what its
//! containers use right now (ADR ZC-0027).

use crate::agent::summary::ThisMac;
use crate::nodes::usage::{node_total, ContainerUse};
use crate::nodes::{runner, MachineLimit, NodeSpec, Registry, MAIN};
use serde::Serialize;
use std::path::Path;
use std::sync::Mutex;
use std::time::{Duration, Instant};

#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct Share {
    pub cpu_cores: f64,
    pub memory_bytes: u64,
    pub disk_bytes: u64,
}

#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct Used {
    pub cpu_cores: Option<f64>,
    pub memory_bytes: Option<u64>,
}

#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct Workers {
    pub desired: u32,
    pub running: u32,
    pub max: u32,
}

#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct LocalNode {
    pub id: String,
    /// The name the hub lists it under.
    pub name: String,
    pub main: bool,
    /// `hub_jobs` or `environment`.
    pub runs: &'static str,
    pub environment: Option<String>,
    pub environment_name: Option<String>,
    /// `running`, `starting`, `paused`, `stopped` or `error`.
    pub state: String,
    pub error: Option<String>,
    /// `zc://node-<fingerprint>`, once the node has its key.
    pub address: Option<String>,
    pub broker_port: Option<u16>,
    pub workers: Workers,
    /// Its share of the owner's limit (`null`: could not be resolved).
    pub share: Option<Share>,
    pub used: Used,
    pub containers: Vec<ContainerUse>,
    /// Whether a terminal can open in it now (it has a running container).
    pub terminal: bool,
}

#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct View {
    pub v: u32,
    /// The owner's limit for the whole machine.
    pub limit: Option<Share>,
    /// What can still be given to a new node (the main node keeps its floor).
    pub available: Option<Share>,
    pub max_extra_nodes: u32,
    pub nodes: Vec<LocalNode>,
    /// Why the containers could not be read, when they could not.
    pub usage_error: Option<String>,
}

fn share(l: &MachineLimit) -> Share {
    Share {
        cpu_cores: f64::from(l.cpu_millis) / 1000.0,
        memory_bytes: l.memory_bytes,
        disk_bytes: l.disk_bytes,
    }
}

fn address(pubkey_b64: &str) -> Option<String> {
    crate::broker::node_identity::fingerprint_of_pubkey_b64(pubkey_b64)
        .map(|fp| format!("zc://node-{fp}"))
}

fn node_of(spec: &NodeSpec, home: &Path, sharing: bool, containers: &[ContainerUse]) -> LocalNode {
    let st = runner::Status::load(&spec.dir(home));
    let mine: Vec<ContainerUse> = containers
        .iter()
        .filter(|c| c.node == spec.id)
        .cloned()
        .collect();
    let (cpu, mem) = node_total(containers, &spec.id);
    let running_workers = mine
        .iter()
        .filter(|c| c.role == "worker" && c.running)
        .count() as u32;
    let state = match (&st, sharing) {
        (_, false) => "paused".to_string(),
        (Some(s), true) if s.state == "stopped" => "starting".to_string(),
        (Some(s), true) => s.state.clone(),
        (None, true) => "starting".to_string(),
    };
    LocalNode {
        id: spec.id.clone(),
        name: spec.id.clone(),
        main: false,
        runs: if spec.environment.is_some() {
            "environment"
        } else {
            "hub_jobs"
        },
        environment: spec.environment.clone(),
        environment_name: spec.environment_name.clone(),
        error: st
            .as_ref()
            .and_then(|s| s.error.clone())
            .filter(|_| sharing),
        address: st
            .as_ref()
            .map(|s| s.fingerprint.clone())
            .filter(|f| !f.is_empty())
            .map(|f| format!("zc://node-{f}")),
        broker_port: Some(spec.broker_port()),
        workers: Workers {
            desired: spec.workers,
            running: running_workers,
            max: crate::nodes::MAX_WORKERS,
        },
        share: Some(Share {
            cpu_cores: f64::from(spec.cpu_millis) / 1000.0,
            memory_bytes: spec.memory_bytes,
            disk_bytes: spec.disk_bytes,
        }),
        used: Used {
            cpu_cores: cpu,
            memory_bytes: mem,
        },
        terminal: mine.iter().any(|c| c.running),
        containers: mine,
        state,
    }
}

/// The view from its parts (no I/O beyond the nodes' status files).
pub fn build(
    home: &Path,
    this: &ThisMac,
    main_broker_port: u16,
    reg: &Registry,
    limit: Option<&MachineLimit>,
    containers: Result<Vec<ContainerUse>, String>,
) -> View {
    let (containers, usage_error) = match containers {
        Ok(c) => (c, None),
        Err(e) => (Vec::new(), Some(e)),
    };
    let reserved = reg.reserved();
    let main_share = limit.map(|l| l.main_share(&reserved));
    let mine: Vec<ContainerUse> = containers
        .iter()
        .filter(|c| c.node == MAIN)
        .cloned()
        .collect();
    let (cpu, mem) = node_total(&containers, MAIN);
    let main = LocalNode {
        id: MAIN.to_string(),
        name: this.name.clone(),
        main: true,
        runs: "hub_jobs",
        environment: None,
        environment_name: None,
        state: if !this.sharing {
            "paused".into()
        } else if this.workers.running > 0 {
            "running".into()
        } else {
            match this.broker.as_str() {
                "error" => "error".into(),
                _ => "starting".into(),
            }
        },
        error: None,
        address: address(&this.node_pubkey),
        broker_port: Some(main_broker_port),
        workers: Workers {
            desired: this.workers.desired,
            running: this.workers.running,
            max: this.workers.max,
        },
        share: main_share.as_ref().map(share),
        used: Used {
            cpu_cores: cpu,
            memory_bytes: mem,
        },
        terminal: mine.iter().any(|c| c.running),
        containers: mine,
    };
    let mut nodes = vec![main];
    nodes.extend(
        reg.nodes
            .iter()
            .map(|n| node_of(n, home, this.sharing, &containers)),
    );
    // What a new node may take: the main node's share less its floor.
    let available = main_share.map(|m| Share {
        cpu_cores: f64::from(
            m.cpu_millis
                .saturating_sub(crate::nodes::MAIN_MIN_CPU_MILLIS),
        ) / 1000.0,
        memory_bytes: m.memory_bytes.saturating_sub(crate::nodes::MAIN_MIN_MEMORY),
        disk_bytes: m.disk_bytes,
    });
    View {
        v: 1,
        limit: limit.map(share),
        available,
        max_extra_nodes: crate::nodes::MAX_EXTRA,
        nodes,
        usage_error,
    }
}

/// The containers' use, read at most every 4 s (`docker stats` samples for
/// a second or two). Blocking.
pub fn cached_usage() -> Result<Vec<ContainerUse>, String> {
    type Cache = Option<(Instant, Result<Vec<ContainerUse>, String>)>;
    static C: Mutex<Cache> = Mutex::new(None);
    let mut g = C.lock().unwrap_or_else(|p| p.into_inner());
    if let Some((at, r)) = g.as_ref() {
        if at.elapsed() < Duration::from_secs(4) {
            return r.clone();
        }
    }
    let r = crate::nodes::usage::read();
    *g = Some((Instant::now(), r.clone()));
    r
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::agent::summary::{Requests, WorkerCounts};

    const GIB: u64 = 1 << 30;

    fn this(sharing: bool, running: u32) -> ThisMac {
        ThisMac {
            node_pubkey: String::new(),
            name: "jeans-mbp".into(),
            sharing,
            broker: "running".into(),
            workers: WorkerCounts {
                desired: 2,
                running,
                busy: 0,
                draining: 0,
                max: 5,
            },
            requests: Requests::default(),
        }
    }

    fn c(name: &str, node: &str, role: &'static str, cpu: f64, mem: u64) -> ContainerUse {
        ContainerUse {
            name: name.into(),
            node: node.into(),
            role,
            what: None,
            running: true,
            limit_cpu_cores: None,
            limit_memory_bytes: None,
            cpu_cores: Some(cpu),
            memory_bytes: Some(mem),
        }
    }

    fn reg() -> Registry {
        Registry {
            v: 1,
            nodes: vec![NodeSpec {
                id: "pytorch".into(),
                index: 1,
                environment: Some("pytorch".into()),
                environment_name: Some("PyTorch".into()),
                workers: 0,
                cpu_millis: 2000,
                memory_bytes: 3 * GIB,
                disk_bytes: 10 * GIB,
                created: "t".into(),
            }],
        }
    }

    #[test]
    fn the_main_node_gets_what_the_others_leave_and_each_its_own_use() {
        let limit = MachineLimit {
            cpu_millis: 5000,
            memory_bytes: 8 * GIB,
            disk_bytes: 50 * GIB,
        };
        let home = std::env::temp_dir().join("zc-local-nodes-none");
        let containers = vec![
            c("zc-share-aa-w0", "main", "worker", 0.5, GIB),
            c("zk-dep-1-v1", "pytorch", "environment", 1.5, 2 * GIB),
        ];
        let v = build(
            &home,
            &this(true, 1),
            9000,
            &reg(),
            Some(&limit),
            Ok(containers),
        );
        assert_eq!(v.nodes.len(), 2);
        let (main, env) = (&v.nodes[0], &v.nodes[1]);
        assert!(main.main && main.runs == "hub_jobs" && main.state == "running");
        assert_eq!(main.share.as_ref().unwrap().cpu_cores, 3.0);
        assert_eq!(main.share.as_ref().unwrap().memory_bytes, 5 * GIB);
        assert_eq!(main.used.cpu_cores, Some(0.5));
        assert_eq!(
            (env.runs, env.environment_name.as_deref()),
            ("environment", Some("PyTorch"))
        );
        assert_eq!(env.used.memory_bytes, Some(2 * GIB));
        assert_eq!(env.broker_port, Some(9002));
        assert!(env.terminal && main.terminal);
        // No status file yet while sharing: starting, not an error.
        assert_eq!(env.state, "starting");
        // A new node may take the main node's share less its floor (1 core, 2 GB).
        let a = v.available.unwrap();
        assert_eq!((a.cpu_cores, a.memory_bytes), (2.0, 3 * GIB));
        assert_eq!(v.limit.unwrap().cpu_cores, 5.0);
    }

    #[test]
    fn paused_sharing_pauses_every_node_and_a_failed_read_is_said() {
        let home = std::env::temp_dir().join("zc-local-nodes-none");
        let v = build(
            &home,
            &this(false, 0),
            9000,
            &reg(),
            None,
            Err("colima is down".into()),
        );
        assert!(v.nodes.iter().all(|n| n.state == "paused"));
        assert_eq!(v.usage_error.as_deref(), Some("colima is down"));
        assert!(v.limit.is_none() && v.nodes[0].share.is_none());
        assert!(!v.nodes[1].terminal);
    }
}