zc2 0.0.52

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! What is in use on the lent slice (zc#663): the `used` object of the
//! register body's `hub_allocation`, which feeds the hub's Resources charts
//! (zak-marketplace `GET /api/hub/resources` and `/history`).
//!
//! Each figure is what Zakuro occupies on this machine now, capped at the
//! slice lent for it:
//!
//! - `ram_gb`: Zakuro's memory in use, as `GET /v2/resources` reads it (the
//!   VM plus the agent's process tree). Unix only.
//! - `gpu_gb`: memory in use on the CUDA devices (`nvidia-smi`).
//! - `disk_gb`: bytes under the zc state dir, re-measured at most every
//!   [`DISK_EVERY`].
//!
//! A slice of 0 reports 0 without measuring. The hub reads a key left out as
//! 0, so a partial object would claim "nothing in use" for a reading that
//! failed. When a lent resource cannot be read, the whole `used` object is
//! left out instead, and the hub keeps its last reading.

use std::path::Path;
use std::sync::Mutex;
use std::time::{Duration, Instant};

use serde_json::{json, Value};

use super::HubAllocation;

const GIB: f64 = (1u64 << 30) as f64;
/// The state dir is walked at most this often; a register comes every ~60 s.
pub const DISK_EVERY: Duration = Duration::from_secs(600);
/// A state dir with more entries than this is not walked (fleet job dirs can
/// hold millions of files); the disk figure is then unknown.
pub const DISK_MAX_ENTRIES: usize = 200_000;

/// Raw readings in bytes; `None` is "could not be read".
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct Readings {
    pub ram_bytes: Option<u64>,
    pub gpu_bytes: Option<u64>,
    pub disk_bytes: Option<u64>,
}

/// One figure, in GB (GiB, like the lent sizes), capped at the slice.
/// `None` when the slice is lent and the reading is missing.
fn capped(lent_gb: u64, reading: Option<u64>) -> Option<f64> {
    if lent_gb == 0 {
        return Some(0.0);
    }
    let gb = reading? as f64 / GIB;
    Some((gb.min(lent_gb as f64) * 100.0).round() / 100.0)
}

/// The `used` object for `a`, or `None` (left out) while the allocation is
/// disabled or a lent resource has no reading.
pub fn used(a: &HubAllocation, r: &Readings) -> Option<Value> {
    if !a.enabled {
        return None;
    }
    Some(json!({
        "ram_gb": capped(a.ram_gb, r.ram_bytes)?,
        "gpu_gb": capped(a.vram_gb, r.gpu_bytes)?,
        "disk_gb": capped(a.disk_gb, r.disk_bytes)?,
    }))
}

/// `a.to_payload()` plus `used` when it can be given. Measures (blocking):
/// call it from the register thread, never from an async task.
pub fn payload_with_used(a: &HubAllocation) -> Value {
    let mut v = a.to_payload();
    if a.enabled {
        if let Some(u) = used(a, &measure(a)) {
            v["used"] = u;
        }
    }
    v
}

/// Read only what is lent: a slice of 0 needs no reading.
pub fn measure(a: &HubAllocation) -> Readings {
    Readings {
        ram_bytes: (a.ram_gb > 0).then(ram_bytes).flatten(),
        gpu_bytes: (a.vram_gb > 0).then(gpu_bytes).flatten(),
        disk_bytes: (a.disk_gb > 0).then(disk_bytes).flatten(),
    }
}

#[cfg(unix)]
fn ram_bytes() -> Option<u64> {
    crate::agent::resources::snapshot().zakuro.used.memory_bytes
}

#[cfg(not(unix))]
fn ram_bytes() -> Option<u64> {
    None
}

fn gpu_bytes() -> Option<u64> {
    use crate::broker::deploy::runtime::{run_bounded, CliOutcome, PROBE_TIMEOUT};
    let (outcome, out) = run_bounded(
        std::process::Command::new("nvidia-smi")
            .args(["--query-gpu=memory.used", "--format=csv,noheader,nounits"]),
        PROBE_TIMEOUT,
    );
    match outcome {
        CliOutcome::Exited { success: true, .. } => parse_nvidia_smi_used(&out),
        _ => None,
    }
}

/// Sum of `nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits`
/// (MiB, one GPU per line), in bytes. `None` when no line parses.
pub fn parse_nvidia_smi_used(out: &str) -> Option<u64> {
    let mib: Vec<u64> = out
        .lines()
        .filter_map(|l| l.trim().parse::<u64>().ok())
        .collect();
    (!mib.is_empty()).then(|| mib.iter().sum::<u64>() << 20)
}

fn disk_bytes() -> Option<u64> {
    static C: Mutex<Option<(Instant, Option<u64>)>> = Mutex::new(None);
    let mut g = C.lock().unwrap_or_else(|p| p.into_inner());
    if let Some((at, v)) = *g {
        if at.elapsed() < DISK_EVERY {
            return v;
        }
    }
    let v = crate::credentials::dir().and_then(|d| tree_bytes(&d, DISK_MAX_ENTRIES));
    *g = Some((Instant::now(), v));
    v
}

/// Bytes of the regular files under `root` (symlinks not followed). `None`
/// when `root` cannot be read or holds more than `max_entries` entries.
pub fn tree_bytes(root: &Path, max_entries: usize) -> Option<u64> {
    let mut stack = vec![std::fs::read_dir(root).ok()?];
    let (mut total, mut seen) = (0u64, 0usize);
    while let Some(rd) = stack.last_mut() {
        let Some(entry) = rd.next() else {
            stack.pop();
            continue;
        };
        let Ok(e) = entry else { continue };
        seen += 1;
        if seen > max_entries {
            return None;
        }
        match e.file_type() {
            Ok(t) if t.is_dir() => {
                if let Ok(sub) = std::fs::read_dir(e.path()) {
                    stack.push(sub);
                }
            }
            Ok(t) if t.is_file() => total += e.metadata().map(|m| m.len()).unwrap_or(0),
            _ => {}
        }
    }
    Some(total)
}

#[cfg(test)]
mod tests {
    use super::*;

    fn alloc(enabled: bool, vram: u64, ram: u64, disk: u64) -> HubAllocation {
        HubAllocation {
            enabled,
            vram_gb: vram,
            ram_gb: ram,
            disk_gb: disk,
            purpose: vec!["inference".into()],
        }
    }

    const G: u64 = 1 << 30;

    #[test]
    fn used_caps_each_figure_at_its_slice() {
        let r = Readings {
            ram_bytes: Some(3 * G + G / 2),
            gpu_bytes: Some(20 * G),
            disk_bytes: Some(G / 4),
        };
        assert_eq!(
            used(&alloc(true, 8, 16, 50), &r),
            Some(json!({"ram_gb": 3.5, "gpu_gb": 8.0, "disk_gb": 0.25}))
        );
    }

    #[test]
    fn nothing_lent_reports_zero_without_a_reading() {
        let r = Readings {
            ram_bytes: Some(2 * G),
            ..Default::default()
        };
        assert_eq!(
            used(&alloc(true, 0, 16, 0), &r),
            Some(json!({"ram_gb": 2.0, "gpu_gb": 0.0, "disk_gb": 0.0}))
        );
    }

    #[test]
    fn a_lent_resource_without_a_reading_leaves_used_out() {
        let r = Readings {
            ram_bytes: Some(2 * G),
            gpu_bytes: None,
            disk_bytes: Some(G),
        };
        assert_eq!(used(&alloc(true, 8, 16, 50), &r), None);
    }

    #[test]
    fn a_disabled_allocation_sends_no_used() {
        let r = Readings {
            ram_bytes: Some(G),
            gpu_bytes: Some(G),
            disk_bytes: Some(G),
        };
        assert_eq!(used(&alloc(false, 8, 16, 50), &r), None);
        let p = payload_with_used(&alloc(false, 8, 16, 50));
        assert!(p.get("used").is_none(), "{p}");
        assert_eq!(p["enabled"], false);
    }

    #[test]
    fn nvidia_smi_lines_sum_to_bytes() {
        assert_eq!(parse_nvidia_smi_used("1024\n 512 \n"), Some(1536 << 20));
        assert_eq!(parse_nvidia_smi_used("0\n"), Some(0));
        assert_eq!(parse_nvidia_smi_used(""), None);
        assert_eq!(
            parse_nvidia_smi_used("NVIDIA-SMI has failed because it couldn't communicate\n"),
            None
        );
    }

    #[test]
    fn tree_bytes_sums_files_and_gives_up_past_the_cap() {
        let d = std::env::temp_dir().join(format!("zc-usage-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&d);
        std::fs::create_dir_all(d.join("a/b")).unwrap();
        std::fs::write(d.join("x"), [0u8; 100]).unwrap();
        std::fs::write(d.join("a/b/y"), [0u8; 23]).unwrap();
        assert_eq!(tree_bytes(&d, 100), Some(123));
        // 4 entries: x, a, a/b, a/b/y.
        assert_eq!(tree_bytes(&d, 3), None);
        assert_eq!(tree_bytes(&d.join("missing"), 100), None);
        let _ = std::fs::remove_dir_all(&d);
    }
}