pub mod colima;
pub mod docker;
pub mod engine;
pub mod ephemeral;
pub mod job;
pub mod jobinst;
pub mod lima;
pub mod lock;
pub mod lxc;
pub use lock::{admission_lock, AdmissionGuard};
use std::fmt;
pub const MIN_MEMORY_BYTES: u64 = 6 * 1024 * 1024;
pub const MIN_CPU_MILLIS: u32 = 10;
const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum Gpus {
#[default]
None,
All,
Count(u32),
Spec(String),
}
impl Gpus {
pub fn flag_value(&self) -> Option<String> {
match self {
Gpus::None | Gpus::Count(0) => None,
Gpus::All => Some("all".into()),
Gpus::Count(n) => Some(n.to_string()),
Gpus::Spec(s) => Some(s.clone()),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Limits {
pub cpu_millis: u32,
pub memory_bytes: u64,
pub gpus: Gpus,
}
impl Limits {
pub fn new(cpus: f64, memory_gib: f64, gpus: Gpus) -> Self {
Self {
cpu_millis: ((cpus * 1000.0).round().max(0.0) as u32).max(MIN_CPU_MILLIS),
memory_bytes: ((memory_gib * GIB).round().max(0.0) as u64).max(MIN_MEMORY_BYTES),
gpus,
}
}
pub fn cpus(&self) -> f64 {
self.cpu_millis as f64 / 1000.0
}
pub fn cpus_flag(&self) -> String {
cpu_millis_flag(self.cpu_millis)
}
pub fn memory_flag(&self) -> String {
format!("{}m", (self.memory_bytes / (1024 * 1024)).max(6))
}
pub fn split(&self, n: u32) -> Limits {
let n = n.max(1);
Limits {
cpu_millis: (self.cpu_millis / n).max(MIN_CPU_MILLIS),
memory_bytes: (self.memory_bytes / n as u64).max(MIN_MEMORY_BYTES),
gpus: self.gpus.clone(),
}
}
pub fn clamp_request(&self, req: &Request) -> Limits {
let cpu_millis = if req.cpus > 0.0 {
((req.cpus * 1000.0).round() as u32)
.clamp(MIN_CPU_MILLIS, self.cpu_millis.max(MIN_CPU_MILLIS))
} else {
self.cpu_millis
};
let memory_bytes = if req.memory_gib > 0.0 {
((req.memory_gib * GIB).round() as u64)
.clamp(MIN_MEMORY_BYTES, self.memory_bytes.max(MIN_MEMORY_BYTES))
} else {
self.memory_bytes
};
let gpus = match (req.gpus.filter(|n| *n > 0), &self.gpus) {
(_, Gpus::None) => Gpus::None,
(None, budget) => budget.clone(),
(Some(n), Gpus::All) => Gpus::Count(n),
(Some(n), Gpus::Count(m)) => Gpus::Count(n.min(*m)),
(Some(_), pinned @ Gpus::Spec(_)) => pinned.clone(),
};
Limits {
cpu_millis,
memory_bytes,
gpus,
}
}
}
impl fmt::Display for Limits {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} cpu, {:.1} GiB memory, gpu {}",
self.cpus_flag(),
self.memory_bytes as f64 / GIB,
self.gpus.flag_value().unwrap_or_else(|| "off".into())
)
}
}
pub const BUDGET_LABEL: &str = "zakuro.budget";
impl Limits {
pub fn budget_label(&self) -> String {
format!("{}m,{}", self.cpu_millis, self.memory_bytes)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Allocation {
pub name: String,
pub cpu_millis: u32,
pub memory_bytes: u64,
}
impl Allocation {
pub fn from_label(name: &str, label: &str) -> Option<Self> {
let (cpu, mem) = label.trim().split_once(',')?;
Some(Self {
name: name.to_string(),
cpu_millis: cpu.trim().strip_suffix('m')?.parse().ok()?,
memory_bytes: mem.trim().parse().ok()?,
})
}
}
pub fn remaining(budget: &Limits, running: &[Allocation]) -> (u32, u64) {
let cpu: u64 = running.iter().map(|a| a.cpu_millis as u64).sum();
let mem: u64 = running.iter().map(|a| a.memory_bytes).sum();
(
(budget.cpu_millis as u64).saturating_sub(cpu) as u32,
budget.memory_bytes.saturating_sub(mem),
)
}
pub fn admit(budget: &Limits, running: &[Allocation], want: &Limits) -> Result<(), String> {
let (cpu_left, mem_left) = remaining(budget, running);
let mib = |b: u64| b / (1024 * 1024);
let mut short = Vec::new();
if want.cpu_millis > cpu_left {
short.push(format!(
"cpu: asks {} of the {} left",
want.cpus_flag(),
cpu_millis_flag(cpu_left)
));
}
if want.memory_bytes > mem_left {
short.push(format!(
"memory: asks {} MiB of the {} MiB left",
mib(want.memory_bytes),
mib(mem_left)
));
}
if short.is_empty() {
return Ok(());
}
Err(format!(
"node budget exhausted ({}; {} running workload(s) hold {} of {} cpu and {} of {} MiB)",
short.join(", "),
running.len(),
cpu_millis_flag(budget.cpu_millis - cpu_left),
budget.cpus_flag(),
mib(budget.memory_bytes - mem_left),
mib(budget.memory_bytes),
))
}
pub fn lease_limits(budget: &Limits, running: &[Allocation], req: &Request) -> Limits {
let (cpu_left, mem_left) = remaining(budget, running);
let mut l = budget.clamp_request(req);
if req.cpus <= 0.0 {
l.cpu_millis = cpu_left.max(MIN_CPU_MILLIS);
}
if req.memory_gib <= 0.0 {
l.memory_bytes = mem_left.max(MIN_MEMORY_BYTES);
}
l
}
pub const DEFAULT_DEPLOY_SLICE_PERCENT: f64 = 25.0;
pub fn deployment_limits(
budget: &Limits,
get: &dyn Fn(&str) -> Option<String>,
) -> Result<Limits, String> {
let pct = match get("ZAKURO_DEPLOY_BUDGET_SLICE")
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
{
None => DEFAULT_DEPLOY_SLICE_PERCENT,
Some(v) => match parse_percent(&v) {
Some(Ok(Amount::Percent(p))) => p,
_ => {
return Err(format!(
"ZAKURO_DEPLOY_BUDGET_SLICE={v:?} is not a percentage in (0, 100]"
))
}
},
};
Ok(Limits {
cpu_millis: ((budget.cpu_millis as f64 * pct / 100.0) as u32).max(MIN_CPU_MILLIS),
memory_bytes: ((budget.memory_bytes as f64 * pct / 100.0) as u64).max(MIN_MEMORY_BYTES),
gpus: budget.gpus.clone(),
})
}
fn cpu_millis_flag(millis: u32) -> String {
let whole = millis / 1000;
let frac = millis % 1000;
if frac == 0 {
whole.to_string()
} else {
let f = format!("{frac:03}");
format!("{whole}.{}", f.trim_end_matches('0'))
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Request {
pub cpus: f64,
pub memory_gib: f64,
pub gpus: Option<u32>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Amount {
Absolute(f64),
Percent(f64),
}
impl Amount {
fn of(&self, host_total: f64) -> f64 {
match *self {
Amount::Absolute(v) => v.min(host_total),
Amount::Percent(p) => host_total * p / 100.0,
}
}
fn describe(&self, unit: &str) -> String {
match *self {
Amount::Percent(p) => format!("{p}%"),
Amount::Absolute(v) if unit == "bytes" => format!("{:.1} GiB", v / GIB),
Amount::Absolute(v) => format!("{v} {unit}"),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Budget {
pub cpu: Amount,
pub memory: Amount,
pub gpu: Gpus,
pub disk_bytes: u64,
}
impl Default for Budget {
fn default() -> Self {
Self {
cpu: Amount::Percent(50.0),
memory: Amount::Percent(50.0),
gpu: Gpus::None,
disk_bytes: lxc::DEFAULT_DISK_BYTES,
}
}
}
fn parse_percent(v: &str) -> Option<Result<Amount, String>> {
let p = v.strip_suffix('%')?;
Some(match p.trim().parse::<f64>() {
Ok(p) if p > 0.0 && p <= 100.0 => Ok(Amount::Percent(p)),
_ => Err(format!("{v:?} is not a percentage in (0, 100]")),
})
}
pub fn parse_cpu(v: &str) -> Result<Amount, String> {
let v = v.trim();
if let Some(p) = parse_percent(v) {
return p;
}
match v.parse::<f64>() {
Ok(c) if c > 0.0 && c.is_finite() => Ok(Amount::Absolute(c)),
_ => Err(format!("{v:?} is not a cpu count or a percentage")),
}
}
pub fn parse_memory(v: &str) -> Result<Amount, String> {
let v = v.trim();
if let Some(p) = parse_percent(v) {
return p;
}
let low = v.to_ascii_lowercase();
let digits_end = low
.find(|c: char| !(c.is_ascii_digit() || c == '.'))
.unwrap_or(low.len());
let (num, unit) = low.split_at(digits_end);
let mult: f64 = match unit.trim() {
"" | "b" => 1.0,
"k" | "kb" | "kib" => 1024.0,
"m" | "mb" | "mib" => 1024.0 * 1024.0,
"g" | "gb" | "gib" => GIB,
"t" | "tb" | "tib" => GIB * 1024.0,
_ => return Err(format!("{v:?} is not a memory size or a percentage")),
};
match num.parse::<f64>() {
Ok(n) if n > 0.0 && n.is_finite() => Ok(Amount::Absolute(n * mult)),
_ => Err(format!("{v:?} is not a memory size or a percentage")),
}
}
impl Budget {
pub fn from_env(get: &dyn Fn(&str) -> Option<String>) -> Result<Self, String> {
Self::from_env_over(Budget::default(), get)
}
pub fn from_env_over(
base: Budget,
get: &dyn Fn(&str) -> Option<String>,
) -> Result<Self, String> {
let val = |k: &str| {
get(k)
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
};
let mut b = base;
if let Some(v) = val("ZAKURO_BUDGET_CPU") {
b.cpu = parse_cpu(&v).map_err(|e| format!("ZAKURO_BUDGET_CPU={e}"))?;
}
if let Some(v) = val("ZAKURO_BUDGET_MEMORY") {
b.memory = parse_memory(&v).map_err(|e| format!("ZAKURO_BUDGET_MEMORY={e}"))?;
}
if let Some(v) = val("ZAKURO_BUDGET_DISK") {
b.disk_bytes = match parse_memory(&v) {
Ok(Amount::Absolute(n)) if n >= GIB => n as u64,
Ok(Amount::Absolute(_)) => {
return Err(format!(
"ZAKURO_BUDGET_DISK={v:?} is below the 1 GiB minimum for the sandbox disk"
))
}
Ok(Amount::Percent(_)) => {
return Err(format!(
"ZAKURO_BUDGET_DISK={v:?}: give a size such as 20g, not a percentage"
))
}
Err(e) => return Err(format!("ZAKURO_BUDGET_DISK={e}")),
};
}
let deploy_gpus = val("ZAKURO_DEPLOY_GPUS");
b.gpu = match val("ZAKURO_BUDGET_GPU").map(|v| v.to_ascii_lowercase()) {
Some(v) if matches!(v.as_str(), "1" | "true" | "yes" | "on") => {
deploy_gpus.map(Gpus::Spec).unwrap_or(Gpus::All)
}
Some(v) if matches!(v.as_str(), "0" | "false" | "no" | "off") => Gpus::None,
Some(v) => return Err(format!("ZAKURO_BUDGET_GPU={v:?} is not a boolean")),
None => deploy_gpus.map(Gpus::Spec).unwrap_or(b.gpu),
};
Ok(b)
}
pub fn total(&self, host: &HostResources) -> Result<Limits, String> {
let cpus = self.cpu.of(host.logical_cpus.max(1) as f64);
let memory_bytes = match (self.memory, host.memory_bytes) {
(Amount::Absolute(b), Some(total)) => b.min(total as f64),
(Amount::Absolute(b), None) => b,
(Amount::Percent(_), Some(total)) => self.memory.of(total as f64),
(Amount::Percent(p), None) => {
return Err(format!(
"cannot size the memory budget ({p}% of an unknown total): \
set ZAKURO_BUDGET_MEMORY to an absolute size such as 4g"
))
}
};
Ok(Limits::new(cpus, memory_bytes / GIB, self.gpu.clone()))
}
pub fn describe(&self, host: &HostResources) -> String {
let host_mem = host
.memory_bytes
.map(|b| format!("{:.1} GiB", b as f64 / GIB))
.unwrap_or_else(|| "unknown".into());
format!(
"cpu {} of {} logical, memory {} of {host_mem}, gpu {}, disk {:.1} GiB",
self.cpu.describe("cores"),
host.logical_cpus,
self.memory.describe("bytes"),
self.gpu.flag_value().unwrap_or_else(|| "off".into()),
self.disk_bytes as f64 / GIB,
)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HostResources {
pub logical_cpus: usize,
pub memory_bytes: Option<u64>,
}
impl HostResources {
pub fn detect() -> Self {
Self {
logical_cpus: std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1),
memory_bytes: crate::setup::total_memory(),
}
}
}
pub fn node_limits() -> Result<Limits, String> {
node_limits_from(&|k| std::env::var(k).ok())
}
pub fn node_budget_from(get: &dyn Fn(&str) -> Option<String>) -> Result<Budget, String> {
crate::policy::node_budget(crate::credentials::dir().as_deref(), get)
}
pub fn node_limits_from(get: &dyn Fn(&str) -> Option<String>) -> Result<Limits, String> {
node_budget_from(get)?.total(&HostResources::detect())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RuntimeKind {
Lima,
Colima,
Lxc,
Docker,
Host,
}
impl RuntimeKind {
pub fn as_str(&self) -> &'static str {
match self {
RuntimeKind::Lima => "lima",
RuntimeKind::Colima => "colima",
RuntimeKind::Lxc => "lxc",
RuntimeKind::Docker => "docker",
RuntimeKind::Host => "host",
}
}
pub fn parse(s: &str) -> Option<Self> {
match s.trim().to_ascii_lowercase().as_str() {
"lima" => Some(Self::Lima),
"colima" => Some(Self::Colima),
"lxc" | "incus" | "lxd" => Some(Self::Lxc),
"docker" => Some(Self::Docker),
"host" => Some(Self::Host),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Probe {
pub kind: RuntimeKind,
pub available: bool,
pub detail: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StartSpec {
pub name: String,
pub image: String,
pub network: Option<String>,
pub create_network: bool,
pub labels: Vec<(String, String)>,
pub mounts: Vec<(String, String)>,
pub publish: Vec<String>,
pub env: Vec<(String, String)>,
pub cmd: Vec<String>,
pub limits: Limits,
pub remove_on_exit: bool,
pub hardened: bool,
}
pub(crate) use crate::up::ContainerState as State;
pub trait Runtime: Send + Sync {
fn kind(&self) -> RuntimeKind;
fn probe(&self) -> Probe;
fn verify_image(&self, image: &str, expected_digest: Option<&str>) -> Result<(), String>;
fn start(&self, spec: &StartSpec) -> Result<String, String>;
fn exec(&self, name: &str, argv: &[String], detach: bool) -> Result<String, String>;
fn stop(&self, name: &str) -> Result<(), String>;
fn limits(&self, name: &str) -> Result<Limits, String>;
fn health(&self, name: &str) -> Result<State, String>;
fn allocations(&self) -> Result<Vec<Allocation>, String> {
Ok(Vec::new())
}
}
pub fn preference(get: &dyn Fn(&str) -> Option<String>, macos: bool) -> Vec<RuntimeKind> {
if let Some(v) = get("ZAKURO_RUNTIME_PREFER").filter(|v| !v.trim().is_empty()) {
let list: Vec<RuntimeKind> = v
.split(',')
.filter_map(RuntimeKind::parse)
.filter(|k| *k != RuntimeKind::Host)
.collect();
if !list.is_empty() {
return list;
}
}
if macos {
vec![RuntimeKind::Lima, RuntimeKind::Colima, RuntimeKind::Docker]
} else {
vec![RuntimeKind::Lxc, RuntimeKind::Docker]
}
}
pub fn select(prefer: &[RuntimeKind], probe: &dyn Fn(RuntimeKind) -> Probe) -> Selection {
let mut tried = Vec::new();
for &k in prefer {
let p = probe(k);
let ok = p.available;
tried.push(p);
if ok {
return Selection {
chosen: Some(k),
tried,
};
}
}
Selection {
chosen: None,
tried,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Selection {
pub chosen: Option<RuntimeKind>,
pub tried: Vec<Probe>,
}
pub fn unimplemented_probe(kind: RuntimeKind, macos: bool) -> Probe {
let detail = match kind {
RuntimeKind::Lima | RuntimeKind::Colima if !macos => "macOS only".to_string(),
RuntimeKind::Lima | RuntimeKind::Colima => "not probed".to_string(),
RuntimeKind::Lxc => "Linux only".to_string(),
RuntimeKind::Docker => "not probed".to_string(),
RuntimeKind::Host => "no isolation".to_string(),
};
Probe {
kind,
available: false,
detail,
}
}
pub fn doctor_report(
sel: &Selection,
budget: Result<&Budget, &String>,
host: &HostResources,
) -> String {
let chosen = sel
.chosen
.map(|k| k.as_str())
.unwrap_or("host (no isolation)");
let mut out = format!("sandbox runtime: {chosen}");
for p in &sel.tried {
let mark = if p.available { "OK" } else { "skipped" };
out.push_str(&format!(
"\n {:<7} {mark}: {}",
format!("{}:", p.kind.as_str()),
p.detail
));
}
if sel.chosen.is_none() {
out.push_str(
"\n fallback: workers run as host processes, with no cpu/memory cap; \
install Incus or start Docker (or set ZAKURO_RUNTIME_PREFER) to enforce the budget",
);
}
match budget {
Ok(b) => {
out.push_str(&format!("\n budget: {}", b.describe(host)));
match b.total(host) {
Ok(l) => out.push_str(&format!(
"\n limits: {l} per node (split across share workers)"
)),
Err(e) => out.push_str(&format!("\n limits: UNRESOLVED: {e}")),
}
}
Err(e) => out.push_str(&format!("\n budget: INVALID: {e}")),
}
out
}
pub fn allocation_line(budget: &Limits, running: &[Allocation]) -> String {
let (cpu_left, mem_left) = remaining(budget, running);
format!(
" allocated: {} of {} cpu, {:.1} of {:.1} GiB memory across {} workload(s); left {} cpu, {:.1} GiB",
cpu_millis_flag(budget.cpu_millis - cpu_left),
budget.cpus_flag(),
(budget.memory_bytes - mem_left) as f64 / GIB,
budget.memory_bytes as f64 / GIB,
running.len(),
cpu_millis_flag(cpu_left),
mem_left as f64 / GIB,
)
}
pub fn probe_host() -> Selection {
let macos = cfg!(target_os = "macos");
let prefer = preference(&|k| std::env::var(k).ok(), macos);
select(&prefer, &|k| match k {
RuntimeKind::Docker => {
let p = crate::broker::deploy::runtime_probe();
let detail = match p.next_step() {
Some(step) => format!("{} ({step})", p.detail),
None => p.detail.clone(),
};
Probe {
kind: k,
available: p.available,
detail,
}
}
RuntimeKind::Lxc if cfg!(target_os = "linux") => lxc::LxcRuntime::shell().probe(),
RuntimeKind::Lima if macos => lima::LimaRuntime::shell().probe(),
RuntimeKind::Colima if macos => colima::ColimaRuntime::shell().probe(),
other => unimplemented_probe(other, macos),
})
}
#[cfg(test)]
mod tests {
use super::*;
fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
move |k| {
pairs
.iter()
.find(|(ek, _)| *ek == k)
.map(|(_, v)| v.to_string())
}
}
const HOST: HostResources = HostResources {
logical_cpus: 8,
memory_bytes: Some(16 * 1024 * 1024 * 1024),
};
#[test]
fn the_default_budget_is_half_the_host_and_no_gpu() {
let b = Budget::from_env(&env(&[])).unwrap();
assert_eq!(b, Budget::default());
let l = b.total(&HOST).unwrap();
assert_eq!(l.cpus_flag(), "4");
assert_eq!(l.memory_flag(), "8192m");
assert_eq!(l.gpus, Gpus::None);
}
#[test]
fn absolute_and_percent_budgets_parse() {
let b = Budget::from_env(&env(&[
("ZAKURO_BUDGET_CPU", "1.5"),
("ZAKURO_BUDGET_MEMORY", "512m"),
("ZAKURO_BUDGET_GPU", "yes"),
]))
.unwrap();
let l = b.total(&HOST).unwrap();
assert_eq!(l.cpus_flag(), "1.5");
assert_eq!(l.memory_flag(), "512m");
assert_eq!(l.gpus, Gpus::All);
assert_eq!(b.disk_bytes, lxc::DEFAULT_DISK_BYTES);
let d = Budget::from_env(&env(&[("ZAKURO_BUDGET_DISK", "40g")])).unwrap();
assert_eq!(d.disk_bytes, 40 << 30);
assert!(
d.describe(&HOST).ends_with("disk 40.0 GiB"),
"{}",
d.describe(&HOST)
);
let p = Budget::from_env(&env(&[
("ZAKURO_BUDGET_CPU", "25%"),
("ZAKURO_BUDGET_MEMORY", "25%"),
]))
.unwrap()
.total(&HOST)
.unwrap();
assert_eq!(
(p.cpus_flag(), p.memory_flag()),
("2".into(), "4096m".into())
);
}
#[test]
fn a_budget_never_exceeds_the_host() {
let l = Budget::from_env(&env(&[
("ZAKURO_BUDGET_CPU", "64"),
("ZAKURO_BUDGET_MEMORY", "1t"),
]))
.unwrap()
.total(&HOST)
.unwrap();
assert_eq!(
(l.cpus_flag(), l.memory_flag()),
("8".into(), "16384m".into())
);
}
#[test]
fn an_invalid_budget_is_an_error_not_a_default() {
for (k, v) in [
("ZAKURO_BUDGET_CPU", "lots"),
("ZAKURO_BUDGET_CPU", "150%"),
("ZAKURO_BUDGET_MEMORY", "8 parsecs"),
("ZAKURO_BUDGET_MEMORY", "-1g"),
("ZAKURO_BUDGET_GPU", "maybe"),
("ZAKURO_BUDGET_DISK", "50%"),
("ZAKURO_BUDGET_DISK", "big"),
("ZAKURO_BUDGET_DISK", "100m"),
] {
let e = Budget::from_env(&move |q: &str| (q == k).then(|| v.to_string()));
assert!(e.unwrap_err().starts_with(k), "{k}={v}");
}
}
#[test]
fn deploy_gpus_keeps_the_gpu_fleet_passing_its_value() {
let b = Budget::from_env(&env(&[("ZAKURO_DEPLOY_GPUS", "device=0")])).unwrap();
assert_eq!(b.gpu, Gpus::Spec("device=0".into()));
let off = Budget::from_env(&env(&[
("ZAKURO_DEPLOY_GPUS", "all"),
("ZAKURO_BUDGET_GPU", "0"),
]))
.unwrap();
assert_eq!(off.gpu, Gpus::None);
}
#[test]
fn an_unknown_host_memory_refuses_a_percentage_but_not_a_size() {
let host = HostResources {
logical_cpus: 4,
memory_bytes: None,
};
assert!(Budget::default().total(&host).is_err());
let b = Budget::from_env(&env(&[("ZAKURO_BUDGET_MEMORY", "2g")])).unwrap();
assert_eq!(b.total(&host).unwrap().memory_flag(), "2048m");
}
#[test]
fn splitting_the_budget_never_sums_past_it() {
let total = Limits::new(3.0, 6.0, Gpus::All);
let one = total.split(4);
assert_eq!(one.cpus_flag(), "0.75");
assert_eq!(one.memory_flag(), "1536m");
assert_eq!(one.gpus, Gpus::All);
assert!(one.cpu_millis * 4 <= total.cpu_millis);
let tiny = Limits::new(0.01, 0.0, Gpus::None).split(100);
assert_eq!(tiny.cpu_millis, MIN_CPU_MILLIS);
assert_eq!(tiny.memory_bytes, MIN_MEMORY_BYTES);
assert_eq!(total.split(0), total);
}
#[test]
fn a_request_is_clamped_to_the_budget_and_a_missing_one_takes_it_all() {
let budget = Limits::new(2.0, 4.0, Gpus::None);
let big = budget.clamp_request(&Request {
cpus: 16.0,
memory_gib: 64.0,
gpus: None,
});
assert_eq!(big, budget);
let small = budget.clamp_request(&Request {
cpus: 0.5,
memory_gib: 1.0,
gpus: None,
});
assert_eq!(
(small.cpus_flag(), small.memory_flag(), small.gpus),
("0.5".into(), "1024m".into(), Gpus::None)
);
assert_eq!(budget.clamp_request(&Request::default()), budget);
}
#[test]
fn a_gpu_request_is_granted_only_from_the_budget() {
let with = |gpus: Gpus| Limits::new(2.0, 4.0, gpus);
let ask = |n: u32| Request {
gpus: Some(n),
..Request::default()
};
assert_eq!(with(Gpus::None).clamp_request(&ask(8)).gpus, Gpus::None);
assert_eq!(with(Gpus::All).clamp_request(&ask(2)).gpus, Gpus::Count(2));
assert_eq!(
with(Gpus::Count(1)).clamp_request(&ask(8)).gpus,
Gpus::Count(1)
);
assert_eq!(
with(Gpus::Count(4)).clamp_request(&ask(2)).gpus,
Gpus::Count(2)
);
let pin = Gpus::Spec("device=0".into());
assert_eq!(with(pin.clone()).clamp_request(&ask(2)).gpus, pin);
for b in [Gpus::None, Gpus::All, Gpus::Count(3), pin.clone()] {
assert_eq!(with(b.clone()).clamp_request(&ask(0)).gpus, b);
assert_eq!(with(b.clone()).clamp_request(&Request::default()).gpus, b);
}
}
#[test]
fn the_preference_follows_the_os_and_the_override() {
let none = env(&[]);
assert_eq!(
preference(&none, true),
vec![RuntimeKind::Lima, RuntimeKind::Colima, RuntimeKind::Docker]
);
let colima = env(&[("ZAKURO_RUNTIME_PREFER", "colima, docker")]);
assert_eq!(
preference(&colima, true),
vec![RuntimeKind::Colima, RuntimeKind::Docker]
);
assert_eq!(RuntimeKind::Colima.as_str(), "colima");
assert_eq!(
preference(&none, false),
vec![RuntimeKind::Lxc, RuntimeKind::Docker]
);
let docker_only = env(&[("ZAKURO_RUNTIME_PREFER", "docker, host")]);
assert_eq!(preference(&docker_only, false), vec![RuntimeKind::Docker]);
let junk = env(&[("ZAKURO_RUNTIME_PREFER", "podman")]);
assert_eq!(
preference(&junk, false),
vec![RuntimeKind::Lxc, RuntimeKind::Docker]
);
}
#[test]
fn selection_falls_back_and_keeps_every_reason() {
let probe = |k: RuntimeKind| match k {
RuntimeKind::Docker => Probe {
kind: k,
available: true,
detail: "docker daemon responding".into(),
},
k => Probe {
kind: k,
available: false,
detail: "incus is not installed".into(),
},
};
let sel = select(&[RuntimeKind::Lxc, RuntimeKind::Docker], &probe);
assert_eq!(sel.chosen, Some(RuntimeKind::Docker));
assert_eq!(sel.tried.len(), 2);
let report = doctor_report(&sel, Ok(&Budget::default()), &HOST);
assert!(report.starts_with("sandbox runtime: docker"), "{report}");
assert!(
report.contains("lxc: skipped: incus is not installed"),
"{report}"
);
assert!(
report.contains("docker: OK: docker daemon responding"),
"{report}"
);
assert!(
report.contains("limits: 4 cpu, 8.0 GiB memory, gpu off"),
"{report}"
);
let down = |k: RuntimeKind| Probe {
kind: k,
available: false,
detail: "docker daemon is not responding".into(),
};
let sel = select(&[RuntimeKind::Docker], &down);
assert_eq!(sel.chosen, None);
let report = doctor_report(&sel, Err(&"ZAKURO_BUDGET_CPU=bad".to_string()), &HOST);
assert!(
report.contains("fallback: workers run as host processes"),
"{report}"
);
assert!(
report.contains("budget: INVALID: ZAKURO_BUDGET_CPU=bad"),
"{report}"
);
}
fn alloc(name: &str, cpus: f64, gib: f64) -> Allocation {
let l = Limits::new(cpus, gib, Gpus::None);
Allocation::from_label(name, &l.budget_label()).unwrap()
}
#[test]
fn the_budget_label_round_trips() {
let l = Limits::new(1.5, 2.0, Gpus::All);
assert_eq!(l.budget_label(), "1500m,2147483648");
assert_eq!(
Allocation::from_label("w", &l.budget_label()),
Some(Allocation {
name: "w".into(),
cpu_millis: 1500,
memory_bytes: 2 << 30,
})
);
for junk in ["", "1500,2", "m,2", "1500m", "1500m,lots"] {
assert_eq!(Allocation::from_label("w", junk), None, "{junk:?}");
}
}
#[test]
fn admission_refuses_a_start_past_the_sum_of_what_runs() {
let budget = Limits::new(4.0, 8.0, Gpus::None);
let running = [alloc("a", 2.0, 2.0), alloc("b", 1.0, 4.0)];
assert_eq!(remaining(&budget, &running), (1000, 2 << 30));
assert!(admit(&budget, &running, &Limits::new(1.0, 2.0, Gpus::None)).is_ok());
let e = admit(&budget, &running, &Limits::new(1.5, 1.0, Gpus::None)).unwrap_err();
assert!(e.contains("cpu: asks 1.5 of the 1 left"), "{e}");
assert!(e.contains("2 running workload(s) hold 3 of 4 cpu"), "{e}");
let e = admit(&budget, &running, &Limits::new(0.5, 3.0, Gpus::None)).unwrap_err();
assert!(
e.contains("memory: asks 3072 MiB of the 2048 MiB left"),
"{e}"
);
assert!(!e.contains("cpu: asks"), "{e}");
assert!(admit(&budget, &[], &budget).is_ok());
let full = [alloc("a", 4.0, 8.0), alloc("b", 1.0, 1.0)];
assert_eq!(remaining(&budget, &full), (0, 0));
assert!(admit(&budget, &full, &Limits::new(0.01, 0.0, Gpus::None)).is_err());
}
#[test]
fn a_deployment_takes_a_slice_of_the_budget() {
let budget = Limits::new(2.0, 4.0, Gpus::All);
let d = deployment_limits(&budget, &env(&[])).unwrap();
assert_eq!(
(d.cpus_flag(), d.memory_flag(), d.gpus),
("0.5".into(), "1024m".into(), Gpus::All)
);
let half =
deployment_limits(&budget, &env(&[("ZAKURO_DEPLOY_BUDGET_SLICE", "50%")])).unwrap();
assert_eq!(half.cpus_flag(), "1");
for bad in ["0.5", "0%", "150%", "lots"] {
let e = deployment_limits(&budget, &move |k: &str| {
(k == "ZAKURO_DEPLOY_BUDGET_SLICE").then(|| bad.to_string())
})
.unwrap_err();
assert!(e.starts_with("ZAKURO_DEPLOY_BUDGET_SLICE="), "{bad}: {e}");
}
}
#[test]
fn an_unsized_lease_takes_what_is_left_and_a_sized_one_its_request() {
let budget = Limits::new(4.0, 8.0, Gpus::None);
let running = [alloc("w", 3.0, 6.0)];
let unsized_lease = lease_limits(&budget, &running, &Request::default());
assert_eq!(
(unsized_lease.cpus_flag(), unsized_lease.memory_flag()),
("1".into(), "2048m".into())
);
assert!(admit(&budget, &running, &unsized_lease).is_ok());
let sized = lease_limits(
&budget,
&running,
&Request {
cpus: 2.0,
memory_gib: 1.0,
gpus: None,
},
);
assert_eq!(sized.cpus_flag(), "2");
assert!(
admit(&budget, &running, &sized).is_err(),
"2 cpu asked, 1 left"
);
let full = [alloc("w", 4.0, 8.0)];
let none = lease_limits(&budget, &full, &Request::default());
assert_eq!(none.cpu_millis, MIN_CPU_MILLIS);
assert!(admit(&budget, &full, &none).is_err());
}
#[test]
fn the_doctor_line_shows_allocated_and_left() {
let budget = Limits::new(4.0, 8.0, Gpus::None);
let line = allocation_line(&budget, &[alloc("a", 1.5, 2.0)]);
assert_eq!(
line,
" allocated: 1.5 of 4 cpu, 2.0 of 8.0 GiB memory across 1 workload(s); left 2.5 cpu, 6.0 GiB"
);
}
#[test]
fn cpu_flags_have_no_trailing_zeros() {
for (c, want) in [(1.0, "1"), (0.5, "0.5"), (1.25, "1.25"), (0.125, "0.125")] {
assert_eq!(Limits::new(c, 1.0, Gpus::None).cpus_flag(), want);
}
}
}