use std::path::Path;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use super::HubAllocation;
const GIB: f64 = (1u64 << 30) as f64;
pub const DISK_EVERY: Duration = Duration::from_secs(600);
pub const DISK_MAX_ENTRIES: usize = 200_000;
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct Readings {
pub ram_bytes: Option<u64>,
pub gpu_bytes: Option<u64>,
pub disk_bytes: Option<u64>,
}
fn capped(lent_gb: u64, reading: Option<u64>) -> Option<f64> {
if lent_gb == 0 {
return Some(0.0);
}
let gb = reading? as f64 / GIB;
Some((gb.min(lent_gb as f64) * 100.0).round() / 100.0)
}
pub fn used(a: &HubAllocation, r: &Readings) -> Option<Value> {
if !a.enabled {
return None;
}
Some(json!({
"ram_gb": capped(a.ram_gb, r.ram_bytes)?,
"gpu_gb": capped(a.vram_gb, r.gpu_bytes)?,
"disk_gb": capped(a.disk_gb, r.disk_bytes)?,
}))
}
pub fn payload_with_used(a: &HubAllocation) -> Value {
let mut v = a.to_payload();
if a.enabled {
if let Some(u) = used(a, &measure(a)) {
v["used"] = u;
}
}
v
}
pub fn measure(a: &HubAllocation) -> Readings {
Readings {
ram_bytes: (a.ram_gb > 0).then(ram_bytes).flatten(),
gpu_bytes: (a.vram_gb > 0).then(gpu_bytes).flatten(),
disk_bytes: (a.disk_gb > 0).then(disk_bytes).flatten(),
}
}
#[cfg(unix)]
fn ram_bytes() -> Option<u64> {
crate::agent::resources::snapshot().zakuro.used.memory_bytes
}
#[cfg(not(unix))]
fn ram_bytes() -> Option<u64> {
None
}
fn gpu_bytes() -> Option<u64> {
use crate::broker::deploy::runtime::{run_bounded, CliOutcome, PROBE_TIMEOUT};
let (outcome, out) = run_bounded(
std::process::Command::new("nvidia-smi")
.args(["--query-gpu=memory.used", "--format=csv,noheader,nounits"]),
PROBE_TIMEOUT,
);
match outcome {
CliOutcome::Exited { success: true, .. } => parse_nvidia_smi_used(&out),
_ => None,
}
}
pub fn parse_nvidia_smi_used(out: &str) -> Option<u64> {
let mib: Vec<u64> = out
.lines()
.filter_map(|l| l.trim().parse::<u64>().ok())
.collect();
(!mib.is_empty()).then(|| mib.iter().sum::<u64>() << 20)
}
fn disk_bytes() -> Option<u64> {
static C: Mutex<Option<(Instant, Option<u64>)>> = Mutex::new(None);
let mut g = C.lock().unwrap_or_else(|p| p.into_inner());
if let Some((at, v)) = *g {
if at.elapsed() < DISK_EVERY {
return v;
}
}
let v = crate::credentials::dir().and_then(|d| tree_bytes(&d, DISK_MAX_ENTRIES));
*g = Some((Instant::now(), v));
v
}
pub fn tree_bytes(root: &Path, max_entries: usize) -> Option<u64> {
let mut stack = vec![std::fs::read_dir(root).ok()?];
let (mut total, mut seen) = (0u64, 0usize);
while let Some(rd) = stack.last_mut() {
let Some(entry) = rd.next() else {
stack.pop();
continue;
};
let Ok(e) = entry else { continue };
seen += 1;
if seen > max_entries {
return None;
}
match e.file_type() {
Ok(t) if t.is_dir() => {
if let Ok(sub) = std::fs::read_dir(e.path()) {
stack.push(sub);
}
}
Ok(t) if t.is_file() => total += e.metadata().map(|m| m.len()).unwrap_or(0),
_ => {}
}
}
Some(total)
}
#[cfg(test)]
mod tests {
use super::*;
fn alloc(enabled: bool, vram: u64, ram: u64, disk: u64) -> HubAllocation {
HubAllocation {
enabled,
vram_gb: vram,
ram_gb: ram,
disk_gb: disk,
purpose: vec!["inference".into()],
}
}
const G: u64 = 1 << 30;
#[test]
fn used_caps_each_figure_at_its_slice() {
let r = Readings {
ram_bytes: Some(3 * G + G / 2),
gpu_bytes: Some(20 * G),
disk_bytes: Some(G / 4),
};
assert_eq!(
used(&alloc(true, 8, 16, 50), &r),
Some(json!({"ram_gb": 3.5, "gpu_gb": 8.0, "disk_gb": 0.25}))
);
}
#[test]
fn nothing_lent_reports_zero_without_a_reading() {
let r = Readings {
ram_bytes: Some(2 * G),
..Default::default()
};
assert_eq!(
used(&alloc(true, 0, 16, 0), &r),
Some(json!({"ram_gb": 2.0, "gpu_gb": 0.0, "disk_gb": 0.0}))
);
}
#[test]
fn a_lent_resource_without_a_reading_leaves_used_out() {
let r = Readings {
ram_bytes: Some(2 * G),
gpu_bytes: None,
disk_bytes: Some(G),
};
assert_eq!(used(&alloc(true, 8, 16, 50), &r), None);
}
#[test]
fn a_disabled_allocation_sends_no_used() {
let r = Readings {
ram_bytes: Some(G),
gpu_bytes: Some(G),
disk_bytes: Some(G),
};
assert_eq!(used(&alloc(false, 8, 16, 50), &r), None);
let p = payload_with_used(&alloc(false, 8, 16, 50));
assert!(p.get("used").is_none(), "{p}");
assert_eq!(p["enabled"], false);
}
#[test]
fn nvidia_smi_lines_sum_to_bytes() {
assert_eq!(parse_nvidia_smi_used("1024\n 512 \n"), Some(1536 << 20));
assert_eq!(parse_nvidia_smi_used("0\n"), Some(0));
assert_eq!(parse_nvidia_smi_used(""), None);
assert_eq!(
parse_nvidia_smi_used("NVIDIA-SMI has failed because it couldn't communicate\n"),
None
);
}
#[test]
fn tree_bytes_sums_files_and_gives_up_past_the_cap() {
let d = std::env::temp_dir().join(format!("zc-usage-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(d.join("a/b")).unwrap();
std::fs::write(d.join("x"), [0u8; 100]).unwrap();
std::fs::write(d.join("a/b/y"), [0u8; 23]).unwrap();
assert_eq!(tree_bytes(&d, 100), Some(123));
assert_eq!(tree_bytes(&d, 3), None);
assert_eq!(tree_bytes(&d.join("missing"), 100), None);
let _ = std::fs::remove_dir_all(&d);
}
}