zc2 0.0.53

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! What each node's containers use right now: `docker ps` for who they are
//! (the labels zc put on them) and `docker stats` for their CPU and memory,
//! both through the machine's engine.

use serde::Serialize;

/// One zakuro container and what it uses now.
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct ContainerUse {
    pub name: String,
    /// The node that started it ([`super::MAIN`] when unlabelled).
    pub node: String,
    /// `worker`, `environment`, `space` or `job`.
    pub role: &'static str,
    /// The worker, deployment or space it is.
    pub what: Option<String>,
    pub running: bool,
    /// Its own limits, from the budget label.
    pub limit_cpu_cores: Option<f64>,
    pub limit_memory_bytes: Option<u64>,
    /// `null` when not running or not measured.
    pub cpu_cores: Option<f64>,
    pub memory_bytes: Option<u64>,
}

/// `docker ps -a …` (without the leading `docker`): every container holding
/// a share of a budget, with the labels that say whose and what.
pub fn ps_args() -> Vec<String> {
    let label = |l: &str| format!("{{{{.Label \"{l}\"}}}}");
    vec![
        "ps".into(),
        "-a".into(),
        "--filter".into(),
        format!("label={}", crate::sandbox::BUDGET_LABEL),
        "--format".into(),
        [
            "{{.Names}}".to_string(),
            label(super::NODE_LABEL),
            label("zakuro.worker"),
            label("zakuro.deployment"),
            label("zakuro.space"),
            label(crate::sandbox::BUDGET_LABEL),
            "{{.State}}".to_string(),
        ]
        .join("|"),
    ]
}

/// `docker stats --no-stream …`: one `<name>|<cpu %>|<mem usage>` per line.
pub fn stats_args() -> Vec<String> {
    vec![
        "stats".into(),
        "--no-stream".into(),
        "--format".into(),
        "{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}".into(),
    ]
}

/// `"512MiB"`, `"1.5GiB"`, `"800kB"`, `"12B"` as bytes.
pub fn parse_size(v: &str) -> Option<u64> {
    let v = v.trim();
    let split = v.find(|c: char| !(c.is_ascii_digit() || c == '.'))?;
    let (num, unit) = v.split_at(split);
    let n: f64 = num.parse().ok()?;
    let mult: f64 = match unit.trim() {
        "B" => 1.0,
        "kB" | "KB" => 1e3,
        "MB" => 1e6,
        "GB" => 1e9,
        "TB" => 1e12,
        "KiB" => 1024.0,
        "MiB" => 1024.0 * 1024.0,
        "GiB" => 1024.0 * 1024.0 * 1024.0,
        "TiB" => 1024.0 * 1024.0 * 1024.0 * 1024.0,
        _ => return None,
    };
    Some((n * mult).round() as u64)
}

/// `<name>|12.34%|512MiB / 2GiB` lines as `(name, cores, bytes)`.
pub fn parse_stats(out: &str) -> Vec<(String, f64, u64)> {
    out.lines()
        .filter_map(|l| {
            let mut f = l.trim().split('|');
            let name = f.next()?.trim().to_string();
            let cpu: f64 = f.next()?.trim().trim_end_matches('%').parse().ok()?;
            let mem = parse_size(f.next()?.split('/').next()?)?;
            Some((name, cpu / 100.0, mem))
        })
        .collect()
}

/// [`ps_args`]' output joined with [`parse_stats`]' readings.
pub fn join(ps: &str, stats: &[(String, f64, u64)]) -> Vec<ContainerUse> {
    ps.lines()
        .filter_map(|l| {
            let f: Vec<&str> = l.trim().split('|').collect();
            let [name, node, worker, deployment, space, budget, state] = f[..] else {
                return None;
            };
            let some = |v: &str| Some(v.trim().to_string()).filter(|v| !v.is_empty());
            let (role, what) = if let Some(w) = some(worker) {
                ("worker", Some(w))
            } else if let Some(d) = some(deployment) {
                ("environment", Some(d))
            } else if let Some(s) = some(space) {
                ("space", Some(s))
            } else {
                ("job", None)
            };
            let limit = crate::sandbox::Allocation::from_label(name, budget);
            let running = state.trim() == "running";
            let used = stats.iter().find(|(n, _, _)| n == name.trim());
            Some(ContainerUse {
                name: name.trim().to_string(),
                node: some(node).unwrap_or_else(|| super::MAIN.to_string()),
                role,
                what,
                running,
                limit_cpu_cores: limit.as_ref().map(|a| f64::from(a.cpu_millis) / 1000.0),
                limit_memory_bytes: limit.as_ref().map(|a| a.memory_bytes),
                cpu_cores: used.filter(|_| running).map(|u| u.1),
                memory_bytes: used.filter(|_| running).map(|u| u.2),
            })
        })
        .collect()
}

/// Every zakuro container on this machine's engine with its use now.
/// `Ok(vec![])` when the sandbox does not exist yet. Takes a second or two
/// (`docker stats` samples): the agent caches it.
pub fn read() -> Result<Vec<ContainerUse>, String> {
    let run = |args: Vec<String>| -> Result<Option<String>, String> {
        let Some(mut cmd) = crate::sandbox::engine::command_if_present(&args)? else {
            return Ok(None);
        };
        let out = cmd
            .stdin(std::process::Stdio::null())
            .output()
            .map_err(|e| format!("container runtime not available: {e}"))?;
        if out.status.success() {
            Ok(Some(String::from_utf8_lossy(&out.stdout).into_owned()))
        } else {
            Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
        }
    };
    let Some(ps) = run(ps_args())? else {
        return Ok(Vec::new());
    };
    if ps.trim().is_empty() {
        return Ok(Vec::new());
    }
    // A failed stats read still lists the containers, unmeasured.
    let stats = run(stats_args())
        .ok()
        .flatten()
        .map(|s| parse_stats(&s))
        .unwrap_or_default();
    Ok(join(&ps, &stats))
}

/// The main node's running share-worker containers (no stats: quick).
pub fn main_worker_containers() -> Vec<String> {
    let Ok(Some(mut cmd)) = crate::sandbox::engine::command_if_present(&ps_args()) else {
        return Vec::new();
    };
    let Ok(out) = cmd.stdin(std::process::Stdio::null()).output() else {
        return Vec::new();
    };
    join(&String::from_utf8_lossy(&out.stdout), &[])
        .into_iter()
        .filter(|c| c.node == super::MAIN && c.role == "worker" && c.running)
        .map(|c| c.name)
        .collect()
}

/// The sum of `list`'s running containers that belong to `node`.
pub fn node_total(list: &[ContainerUse], node: &str) -> (Option<f64>, Option<u64>) {
    let mine: Vec<&ContainerUse> = list
        .iter()
        .filter(|c| c.node == node && c.running)
        .collect();
    let cpu: Vec<f64> = mine.iter().filter_map(|c| c.cpu_cores).collect();
    let mem: Vec<u64> = mine.iter().filter_map(|c| c.memory_bytes).collect();
    (
        (!cpu.is_empty() || mine.is_empty()).then(|| cpu.iter().sum()),
        (!mem.is_empty() || mine.is_empty()).then(|| mem.iter().sum()),
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn sizes_parse_in_both_unit_families() {
        assert_eq!(parse_size("512MiB"), Some(512 * 1024 * 1024));
        assert_eq!(parse_size("1.5GiB"), Some(1_610_612_736));
        assert_eq!(parse_size("800kB"), Some(800_000));
        assert_eq!(parse_size(" 12B "), Some(12));
        assert_eq!(parse_size("lots"), None);
        assert_eq!(parse_size("5 parsecs"), None);
    }

    #[test]
    fn stats_lines_become_cores_and_bytes() {
        let s =
            "zc-share-ab-w0|150.25%|512MiB / 2GiB\nzk-dep-1-v2|0.00%|12.5MiB / 1GiB\nbroken line\n";
        let v = parse_stats(s);
        assert_eq!(v.len(), 2);
        assert_eq!(v[0].0, "zc-share-ab-w0");
        assert!((v[0].1 - 1.5025).abs() < 1e-9);
        assert_eq!(v[0].2, 512 * 1024 * 1024);
        assert_eq!(v[1].2, 13_107_200);
    }

    #[test]
    fn containers_are_attributed_to_their_node_and_role() {
        let ps = "zc-share-ab-w0||ab-w0|||1000m,2147483648|running\n\
                  zc-share-cd-w0|pytorch|cd-w0|||500m,1073741824|running\n\
                  zk-dep-7-v1|pytorch||7||2000m,4294967296|running\n\
                  zk-space-x|||| s1|250m,536870912|exited\n\
                  not|enough|fields\n";
        let stats = parse_stats(
            "zc-share-ab-w0|50%|1GiB / 2GiB\nzc-share-cd-w0|25%|256MiB / 1GiB\nzk-dep-7-v1|100%|2GiB / 4GiB\n",
        );
        let v = join(ps, &stats);
        assert_eq!(v.len(), 4);
        assert_eq!((v[0].node.as_str(), v[0].role), ("main", "worker"));
        assert_eq!(
            (v[1].node.as_str(), v[1].what.as_deref()),
            ("pytorch", Some("cd-w0"))
        );
        assert_eq!(
            (v[2].role, v[2].limit_cpu_cores),
            ("environment", Some(2.0))
        );
        assert_eq!(
            (v[3].role, v[3].running, v[3].cpu_cores),
            ("space", false, None)
        );
        // Per node: the environment node's worker and its environment.
        let (cpu, mem) = node_total(&v, "pytorch");
        assert!((cpu.unwrap() - 1.25).abs() < 1e-9);
        assert_eq!(mem, Some(256 * 1024 * 1024 + 2 * 1024 * 1024 * 1024));
        // A node with nothing running uses nothing, not "unknown".
        assert_eq!(node_total(&v, "idle"), (Some(0.0), Some(0)));
    }
}