use crate::agent::summary::ThisMac;
use crate::nodes::usage::{node_total, ContainerUse};
use crate::nodes::{runner, MachineLimit, NodeSpec, Registry, MAIN};
use serde::Serialize;
use std::path::Path;
use std::sync::Mutex;
use std::time::{Duration, Instant};
#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct Share {
pub cpu_cores: f64,
pub memory_bytes: u64,
pub disk_bytes: u64,
}
#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct Used {
pub cpu_cores: Option<f64>,
pub memory_bytes: Option<u64>,
}
#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct Workers {
pub desired: u32,
pub running: u32,
pub max: u32,
}
#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct LocalNode {
pub id: String,
pub name: String,
pub main: bool,
pub runs: &'static str,
pub environment: Option<String>,
pub environment_name: Option<String>,
pub state: String,
pub error: Option<String>,
pub address: Option<String>,
pub broker_port: Option<u16>,
pub workers: Workers,
pub share: Option<Share>,
pub used: Used,
pub containers: Vec<ContainerUse>,
pub terminal: bool,
}
#[derive(Serialize, Clone, Debug, PartialEq)]
pub struct View {
pub v: u32,
pub limit: Option<Share>,
pub available: Option<Share>,
pub max_extra_nodes: u32,
pub nodes: Vec<LocalNode>,
pub usage_error: Option<String>,
}
fn share(l: &MachineLimit) -> Share {
Share {
cpu_cores: f64::from(l.cpu_millis) / 1000.0,
memory_bytes: l.memory_bytes,
disk_bytes: l.disk_bytes,
}
}
fn address(pubkey_b64: &str) -> Option<String> {
crate::broker::node_identity::fingerprint_of_pubkey_b64(pubkey_b64)
.map(|fp| format!("zc://node-{fp}"))
}
fn node_of(spec: &NodeSpec, home: &Path, sharing: bool, containers: &[ContainerUse]) -> LocalNode {
let st = runner::Status::load(&spec.dir(home));
let mine: Vec<ContainerUse> = containers
.iter()
.filter(|c| c.node == spec.id)
.cloned()
.collect();
let (cpu, mem) = node_total(containers, &spec.id);
let running_workers = mine
.iter()
.filter(|c| c.role == "worker" && c.running)
.count() as u32;
let state = match (&st, sharing) {
(_, false) => "paused".to_string(),
(Some(s), true) if s.state == "stopped" => "starting".to_string(),
(Some(s), true) => s.state.clone(),
(None, true) => "starting".to_string(),
};
LocalNode {
id: spec.id.clone(),
name: spec.id.clone(),
main: false,
runs: if spec.environment.is_some() {
"environment"
} else {
"hub_jobs"
},
environment: spec.environment.clone(),
environment_name: spec.environment_name.clone(),
error: st
.as_ref()
.and_then(|s| s.error.clone())
.filter(|_| sharing),
address: st
.as_ref()
.map(|s| s.fingerprint.clone())
.filter(|f| !f.is_empty())
.map(|f| format!("zc://node-{f}")),
broker_port: Some(spec.broker_port()),
workers: Workers {
desired: spec.workers,
running: running_workers,
max: crate::nodes::MAX_WORKERS,
},
share: Some(Share {
cpu_cores: f64::from(spec.cpu_millis) / 1000.0,
memory_bytes: spec.memory_bytes,
disk_bytes: spec.disk_bytes,
}),
used: Used {
cpu_cores: cpu,
memory_bytes: mem,
},
terminal: mine.iter().any(|c| c.running),
containers: mine,
state,
}
}
pub fn build(
home: &Path,
this: &ThisMac,
main_broker_port: u16,
reg: &Registry,
limit: Option<&MachineLimit>,
containers: Result<Vec<ContainerUse>, String>,
) -> View {
let (containers, usage_error) = match containers {
Ok(c) => (c, None),
Err(e) => (Vec::new(), Some(e)),
};
let reserved = reg.reserved();
let main_share = limit.map(|l| l.main_share(&reserved));
let mine: Vec<ContainerUse> = containers
.iter()
.filter(|c| c.node == MAIN)
.cloned()
.collect();
let (cpu, mem) = node_total(&containers, MAIN);
let main = LocalNode {
id: MAIN.to_string(),
name: this.name.clone(),
main: true,
runs: "hub_jobs",
environment: None,
environment_name: None,
state: if !this.sharing {
"paused".into()
} else if this.workers.running > 0 {
"running".into()
} else {
match this.broker.as_str() {
"error" => "error".into(),
_ => "starting".into(),
}
},
error: None,
address: address(&this.node_pubkey),
broker_port: Some(main_broker_port),
workers: Workers {
desired: this.workers.desired,
running: this.workers.running,
max: this.workers.max,
},
share: main_share.as_ref().map(share),
used: Used {
cpu_cores: cpu,
memory_bytes: mem,
},
terminal: mine.iter().any(|c| c.running),
containers: mine,
};
let mut nodes = vec![main];
nodes.extend(
reg.nodes
.iter()
.map(|n| node_of(n, home, this.sharing, &containers)),
);
let available = main_share.map(|m| Share {
cpu_cores: f64::from(
m.cpu_millis
.saturating_sub(crate::nodes::MAIN_MIN_CPU_MILLIS),
) / 1000.0,
memory_bytes: m.memory_bytes.saturating_sub(crate::nodes::MAIN_MIN_MEMORY),
disk_bytes: m.disk_bytes,
});
View {
v: 1,
limit: limit.map(share),
available,
max_extra_nodes: crate::nodes::MAX_EXTRA,
nodes,
usage_error,
}
}
pub fn cached_usage() -> Result<Vec<ContainerUse>, String> {
type Cache = Option<(Instant, Result<Vec<ContainerUse>, String>)>;
static C: Mutex<Cache> = Mutex::new(None);
let mut g = C.lock().unwrap_or_else(|p| p.into_inner());
if let Some((at, r)) = g.as_ref() {
if at.elapsed() < Duration::from_secs(4) {
return r.clone();
}
}
let r = crate::nodes::usage::read();
*g = Some((Instant::now(), r.clone()));
r
}
#[cfg(test)]
mod tests {
use super::*;
use crate::agent::summary::{Requests, WorkerCounts};
const GIB: u64 = 1 << 30;
fn this(sharing: bool, running: u32) -> ThisMac {
ThisMac {
node_pubkey: String::new(),
name: "jeans-mbp".into(),
sharing,
broker: "running".into(),
workers: WorkerCounts {
desired: 2,
running,
busy: 0,
draining: 0,
max: 5,
},
requests: Requests::default(),
}
}
fn c(name: &str, node: &str, role: &'static str, cpu: f64, mem: u64) -> ContainerUse {
ContainerUse {
name: name.into(),
node: node.into(),
role,
what: None,
running: true,
limit_cpu_cores: None,
limit_memory_bytes: None,
cpu_cores: Some(cpu),
memory_bytes: Some(mem),
}
}
fn reg() -> Registry {
Registry {
v: 1,
nodes: vec![NodeSpec {
id: "pytorch".into(),
index: 1,
environment: Some("pytorch".into()),
environment_name: Some("PyTorch".into()),
workers: 0,
cpu_millis: 2000,
memory_bytes: 3 * GIB,
disk_bytes: 10 * GIB,
created: "t".into(),
}],
}
}
#[test]
fn the_main_node_gets_what_the_others_leave_and_each_its_own_use() {
let limit = MachineLimit {
cpu_millis: 5000,
memory_bytes: 8 * GIB,
disk_bytes: 50 * GIB,
};
let home = std::env::temp_dir().join("zc-local-nodes-none");
let containers = vec![
c("zc-share-aa-w0", "main", "worker", 0.5, GIB),
c("zk-dep-1-v1", "pytorch", "environment", 1.5, 2 * GIB),
];
let v = build(
&home,
&this(true, 1),
9000,
®(),
Some(&limit),
Ok(containers),
);
assert_eq!(v.nodes.len(), 2);
let (main, env) = (&v.nodes[0], &v.nodes[1]);
assert!(main.main && main.runs == "hub_jobs" && main.state == "running");
assert_eq!(main.share.as_ref().unwrap().cpu_cores, 3.0);
assert_eq!(main.share.as_ref().unwrap().memory_bytes, 5 * GIB);
assert_eq!(main.used.cpu_cores, Some(0.5));
assert_eq!(
(env.runs, env.environment_name.as_deref()),
("environment", Some("PyTorch"))
);
assert_eq!(env.used.memory_bytes, Some(2 * GIB));
assert_eq!(env.broker_port, Some(9002));
assert!(env.terminal && main.terminal);
assert_eq!(env.state, "starting");
let a = v.available.unwrap();
assert_eq!((a.cpu_cores, a.memory_bytes), (2.0, 3 * GIB));
assert_eq!(v.limit.unwrap().cpu_cores, 5.0);
}
#[test]
fn paused_sharing_pauses_every_node_and_a_failed_read_is_said() {
let home = std::env::temp_dir().join("zc-local-nodes-none");
let v = build(
&home,
&this(false, 0),
9000,
®(),
None,
Err("colima is down".into()),
);
assert!(v.nodes.iter().all(|n| n.state == "paused"));
assert_eq!(v.usage_error.as_deref(), Some("colima is down"));
assert!(v.limit.is_none() && v.nodes[0].share.is_none());
assert!(!v.nodes[1].terminal);
}
}