use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use super::lxc::{self, IncusCli, SandboxSize};
use super::{docker, Allocation, Gpus, Limits, StartSpec};
pub const JOB_PREFIX: &str = "zk-job-";
pub const GOLDEN_ALIAS: &str = "zakuro-sandbox-base";
pub const BUILD_INSTANCE: &str = "zk-golden-build";
pub const PROVISION_PROP: &str = "zakuro.provision";
pub const CFG_BUDGET: &str = "user.zakuro.budget";
pub const CFG_OWNER: &str = "user.zakuro.owner";
pub const CFG_GPUS: &str = "user.zakuro.gpus";
pub const CFG_LEASE: &str = "user.zakuro.lease";
const MIB: u64 = 1024 * 1024;
const OVERHEAD_BYTES: u64 = 256 * MIB;
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|a| a.to_string()).collect()
}
pub fn provision_version(base_image: &str) -> String {
let material = format!(
"{base_image}\n{}\n{}",
lxc::NERDCTL_VERSION,
lxc::provision_script()
);
crate::dataset::sha256_hex(material.as_bytes())[..12].to_string()
}
pub fn instance_name(job: &str) -> String {
format!(
"{JOB_PREFIX}{}",
&crate::dataset::sha256_hex(job.as_bytes())[..8]
)
}
pub fn inner_limits(l: &Limits, gpu_attached: bool) -> Limits {
let memory_bytes = if l.memory_bytes > 3 * OVERHEAD_BYTES {
l.memory_bytes - OVERHEAD_BYTES
} else {
(l.memory_bytes / 4 * 3).max(super::MIN_MEMORY_BYTES)
};
Limits {
cpu_millis: l.cpu_millis,
memory_bytes,
gpus: if gpu_attached { Gpus::All } else { Gpus::None },
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GpuInfo {
pub index: u32,
pub uuid: String,
pub pci: String,
}
pub fn normalize_pci(addr: &str) -> String {
let a = addr.trim().to_ascii_lowercase();
match a.split_once(':') {
Some((domain, rest)) if domain.len() > 4 => {
format!("{}:{rest}", &domain[domain.len() - 4..])
}
_ => a,
}
}
pub fn parse_nvidia_smi(out: &str) -> Vec<GpuInfo> {
out.lines()
.filter_map(|l| {
let mut f = l.split(',').map(str::trim);
let index = f.next()?.parse().ok()?;
let uuid = f.next()?.to_string();
let pci = normalize_pci(f.next()?);
(!uuid.is_empty() && !pci.is_empty()).then_some(GpuInfo { index, uuid, pci })
})
.collect()
}
pub fn pick_gpus(
want: &Gpus,
all: &[GpuInfo],
taken: &HashSet<String>,
) -> Result<Vec<GpuInfo>, String> {
let free: Vec<&GpuInfo> = all.iter().filter(|g| !taken.contains(&g.pci)).collect();
let by_id = |id: &str| -> Result<GpuInfo, String> {
let hit = all.iter().find(|g| {
g.uuid.eq_ignore_ascii_case(id)
|| g.pci == normalize_pci(id)
|| id.parse::<u32>().ok() == Some(g.index)
});
let g = hit.ok_or_else(|| format!("no GPU {id:?} on this node"))?;
if taken.contains(&g.pci) {
return Err(format!("GPU {id:?} is held by another job"));
}
Ok(g.clone())
};
let n_free = |n: usize| -> Result<Vec<GpuInfo>, String> {
if free.len() < n {
return Err(format!(
"asks for {n} GPU(s), {} free of {} (one job per GPU)",
free.len(),
all.len()
));
}
Ok(free.iter().take(n).map(|g| (*g).clone()).collect())
};
match want {
Gpus::None | Gpus::Count(0) => Ok(vec![]),
Gpus::All => {
if all.is_empty() {
return Err("asks for GPUs but this node has none".into());
}
n_free(all.len())
}
Gpus::Count(n) => n_free(*n as usize),
Gpus::Spec(spec) => {
let v = spec.trim().trim_matches('"').trim();
if v == "all" {
return pick_gpus(&Gpus::All, all, taken);
}
if let Ok(n) = v.parse::<u32>() {
return n_free(n as usize);
}
let ids = v.strip_prefix("device=").unwrap_or(v);
let mut out: Vec<GpuInfo> = Vec::new();
for id in ids.split(',').map(str::trim).filter(|i| !i.is_empty()) {
let g = by_id(id)?;
if !out.contains(&g) {
out.push(g);
}
}
if out.is_empty() {
return Err(format!("GPU spec {spec:?} names no GPU"));
}
Ok(out)
}
}
}
pub fn docker_config_json(registry: &str, user: &str, password: &str) -> String {
use base64::Engine as _;
let key = match registry {
"docker.io" | "index.docker.io" | "registry-1.docker.io" => {
"https://index.docker.io/v1/".to_string()
}
r => r.to_string(),
};
let auth = base64::engine::general_purpose::STANDARD.encode(format!("{user}:{password}"));
serde_json::json!({ "auths": { key: { "auth": auth } } }).to_string()
}
#[derive(Debug, Clone)]
pub struct InstanceSpec {
pub job: String,
pub limits: Limits,
pub disk_bytes: u64,
pub gpus: Vec<String>,
pub mounts: Vec<(String, String)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobInstance {
pub name: String,
pub status: String,
pub owner_pid: Option<u32>,
pub budget: Option<String>,
pub gpus: Vec<String>,
}
pub fn parse_job_instances(json: &str) -> Vec<JobInstance> {
let Ok(serde_json::Value::Array(items)) = serde_json::from_str::<serde_json::Value>(json)
else {
return vec![];
};
items
.iter()
.filter_map(|i| {
let name = i.get("name")?.as_str()?.to_string();
if !name.starts_with(JOB_PREFIX) {
return None;
}
let cfg = i.get("config");
let get = |k: &str| {
cfg.and_then(|c| c.get(k))
.and_then(|v| v.as_str())
.map(str::to_string)
};
Some(JobInstance {
status: i
.get("status")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
owner_pid: get(CFG_OWNER).and_then(|p| p.parse().ok()),
budget: get(CFG_BUDGET),
gpus: get(CFG_GPUS)
.map(|g| {
g.split(',')
.filter(|x| !x.is_empty())
.map(str::to_string)
.collect()
})
.unwrap_or_default(),
name,
})
})
.collect()
}
pub fn parse_ipv4(csv: &str) -> Option<String> {
csv.split(['\n', '"']).map(str::trim).find_map(|l| {
let ip = l.split_whitespace().next()?;
(ip.split('.').count() == 4 && ip.split('.').all(|o| o.parse::<u8>().is_ok()))
.then(|| ip.to_string())
})
}
fn live_names() -> &'static Mutex<HashSet<String>> {
static LIVE: std::sync::OnceLock<Mutex<HashSet<String>>> = std::sync::OnceLock::new();
LIVE.get_or_init(|| Mutex::new(HashSet::new()))
}
#[cfg(unix)]
fn pid_alive(pid: u32) -> bool {
let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
rc == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
}
#[cfg(not(unix))]
fn pid_alive(_pid: u32) -> bool {
true
}
#[derive(Clone)]
pub struct Ephemeral {
cli: Arc<dyn IncusCli>,
base_image: String,
size: Result<SandboxSize, String>,
poll: Duration,
polls: u32,
pid: u32,
}
impl Ephemeral {
pub fn new(
cli: Arc<dyn IncusCli>,
base_image: &str,
size: Result<SandboxSize, String>,
) -> Self {
Self {
cli,
base_image: base_image.to_string(),
size,
poll: Duration::from_secs(1),
polls: 90,
pid: std::process::id(),
}
}
pub fn from_env(get: &dyn Fn(&str) -> Option<String>) -> Self {
let rt = lxc::LxcRuntime::from_env(Arc::new(lxc::IncusShell), get);
Self::new(Arc::new(lxc::IncusShell), rt.image(), rt.size().clone())
}
#[cfg(test)]
fn fast(mut self) -> Self {
self.poll = Duration::ZERO;
self.polls = 3;
self.pid = 4242;
self
}
fn run(&self, rest: &[&str]) -> Result<String, String> {
let mut argv = vec!["incus".to_string()];
argv.extend(rest.iter().map(|a| a.to_string()));
self.cli.run(&argv)
}
pub fn ensure_golden(&self) -> Result<String, String> {
let want = provision_version(&self.base_image);
if self.golden_version().as_deref() == Some(want.as_str()) {
return Ok(GOLDEN_ALIAS.to_string());
}
let _guard = super::lock::provision_lock(Duration::from_secs(900))?;
if self.golden_version().as_deref() == Some(want.as_str()) {
return Ok(GOLDEN_ALIAS.to_string());
}
self.build_golden(&want)?;
Ok(GOLDEN_ALIAS.to_string())
}
fn golden_version(&self) -> Option<String> {
self.run(&["image", "get-property", GOLDEN_ALIAS, PROVISION_PROP])
.ok()
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
}
fn build_golden(&self, version: &str) -> Result<(), String> {
let b = BUILD_INSTANCE;
let _ = self.run(&["delete", "-f", b]);
let _ = self.run(&["image", "delete", GOLDEN_ALIAS]);
self.run(&[
"launch",
&self.base_image,
b,
"-c",
"security.privileged=false",
"-c",
"security.nesting=true",
])
.map_err(|e| format!("golden image: launching {b} failed: {e}"))?;
let built = (|| {
self.run(&["exec", b, "--", "sh", "-c", &lxc::provision_script()])
.map_err(|e| format!("provisioning containerd in {b} failed: {e}"))?;
let _ = self.run(&[
"exec",
b,
"--",
"sh",
"-c",
"apt-get clean; rm -rf /var/lib/apt/lists/* /tmp/*",
]);
self.run(&["stop", b])
.map_err(|e| format!("stopping {b} failed: {e}"))?;
self.run(&[
"publish",
b,
"--alias",
GOLDEN_ALIAS,
&format!("{PROVISION_PROP}={version}"),
])
.map_err(|e| format!("publishing {GOLDEN_ALIAS} failed: {e}"))?;
Ok::<(), String>(())
})();
let _ = self.run(&["delete", "-f", b]);
built
}
fn config_args(&self, spec: &InstanceSpec) -> Vec<String> {
let mut a: Vec<String> = Vec::new();
let mut limits = spec.limits.clone();
limits.gpus = if spec.gpus.is_empty() {
Gpus::None
} else {
Gpus::All
};
for (k, v) in lxc::sandbox_config(&limits) {
a.push("-c".into());
a.push(format!("{k}={v}"));
}
for (k, v) in [
(CFG_BUDGET, spec.limits.budget_label()),
(CFG_OWNER, self.pid.to_string()),
(CFG_GPUS, spec.gpus.join(",")),
(CFG_LEASE, spec.job.clone()),
] {
a.push("-c".into());
a.push(format!("{k}={v}"));
}
a
}
pub fn launch(&self, spec: &InstanceSpec) -> Result<String, String> {
let node = self.size.clone()?;
let image = self.ensure_golden()?;
let name = instance_name(&spec.job);
live_names().lock().unwrap().insert(name.clone());
let r = self.launch_inner(spec, &image, &name, node.disk_bytes);
if r.is_err() {
let _ = self.destroy(&name);
}
r.map(|_| name)
}
fn launch_inner(
&self,
spec: &InstanceSpec,
image: &str,
name: &str,
node_disk: u64,
) -> Result<(), String> {
let disk = spec.disk_bytes.min(node_disk).max(1 << 30);
let _ = self.run(&["delete", "-f", name]);
let mut init = s(&["incus", "init", image, name, "--ephemeral"]);
init.extend(self.config_args(spec));
init.push("-d".into());
init.push(format!("root,size={}MiB", (disk / MIB).max(6)));
self.cli
.run(&init)
.map_err(|e| format!("incus init {name} failed: {e}"))?;
for (i, pci) in spec.gpus.iter().enumerate() {
self.run(&[
"config",
"device",
"add",
name,
&format!("zakuro-gpu-{i}"),
"gpu",
"gputype=physical",
&format!("pci={pci}"),
])
.map_err(|e| format!("attaching GPU {pci} to {name} failed: {e}"))?;
}
for (i, (host, inner)) in spec.mounts.iter().enumerate() {
self.run(&[
"config",
"device",
"add",
name,
&format!("zakuro-mnt-{i}"),
"disk",
&format!("source={host}"),
&format!("path={inner}"),
"shift=true",
])
.map_err(|e| format!("mounting {host} into {name} failed: {e}"))?;
}
self.run(&["start", name])
.map_err(|e| format!("incus start {name} failed: {e}"))?;
self.wait_ready(name)
}
fn wait_ready(&self, name: &str) -> Result<(), String> {
let mut last = String::new();
for _ in 0..self.polls {
match self.run(&[
"exec",
name,
"--",
"sh",
"-c",
"systemctl is-active --quiet containerd && nerdctl version >/dev/null 2>&1",
]) {
Ok(_) => return Ok(()),
Err(e) => last = e,
}
if !self.poll.is_zero() {
std::thread::sleep(self.poll);
}
}
Err(format!(
"{name} did not become ready (containerd): {}",
last.lines().next().unwrap_or("no answer")
))
}
pub fn exec_argv(name: &str, docker_args: &[String]) -> Vec<String> {
let mut a = s(&["incus", "exec", name, "--", "nerdctl"]);
let rest = match docker_args.first().map(String::as_str) {
Some("docker") => &docker_args[1..],
_ => docker_args,
};
a.extend(rest.iter().cloned());
a
}
pub fn nerdctl(&self, name: &str, docker_args: &[String]) -> Result<String, String> {
self.cli.run(&Self::exec_argv(name, docker_args))
}
pub fn push_docker_config(
&self,
name: &str,
host_file: &std::path::Path,
) -> Result<(), String> {
self.run(&[
"file",
"push",
"--create-dirs",
"--mode",
"0600",
&host_file.display().to_string(),
&format!("{name}/root/.docker/config.json"),
])
.map(|_| ())
}
pub fn run_container(&self, name: &str, spec: &StartSpec) -> Result<String, String> {
let mut run = vec!["docker".to_string()];
run.extend(docker::run_argv(spec));
self.nerdctl(name, &run)
}
pub fn destroy(&self, name: &str) -> Result<(), String> {
live_names().lock().unwrap().remove(name);
match self.run(&["delete", "-f", name]) {
Ok(_) => Ok(()),
Err(e)
if e.contains("not found")
|| e.contains("Not Found")
|| e.contains("does not exist") =>
{
Ok(())
}
Err(e) => Err(e),
}
}
pub fn list(&self) -> Result<Vec<JobInstance>, String> {
let out = self.run(&["list", JOB_PREFIX, "-f", "json"])?;
Ok(parse_job_instances(&out))
}
pub fn allocations(&self) -> Result<Vec<Allocation>, String> {
Ok(self
.list()?
.iter()
.filter_map(|i| Allocation::from_label(&i.name, i.budget.as_deref()?))
.collect())
}
pub fn taken_gpus(&self) -> Result<HashSet<String>, String> {
Ok(self.list()?.into_iter().flat_map(|i| i.gpus).collect())
}
pub fn address(&self, name: &str) -> Option<String> {
parse_ipv4(&self.run(&["list", name, "-f", "csv", "-c", "4"]).ok()?)
}
pub fn reap(&self, alive: &dyn Fn(u32) -> bool, live: &HashSet<String>) -> Vec<String> {
let Ok(all) = self.list() else {
return vec![];
};
let mut gone = Vec::new();
for i in all {
let orphan = match i.owner_pid {
None => true,
Some(p) if p == self.pid => !live.contains(&i.name),
Some(p) => !alive(p),
};
if orphan && self.destroy(&i.name).is_ok() {
gone.push(i.name);
}
}
gone
}
pub fn reap_leaked(&self) -> Vec<String> {
let live = live_names().lock().unwrap().clone();
self.reap(&pid_alive, &live)
}
}
pub const JOB_ID_PREFIX: &str = "t2:";
pub fn job_id(instance: &str, container: &str) -> String {
format!("{JOB_ID_PREFIX}{instance}/{container}")
}
pub fn parse_job_id(id: &str) -> Option<(&str, &str)> {
let (inst, name) = id.strip_prefix(JOB_ID_PREFIX)?.split_once('/')?;
(!inst.is_empty() && !name.is_empty()).then_some((inst, name))
}
#[derive(Debug, Clone)]
pub struct JobRun {
pub name: String,
pub image: String,
pub labels: Vec<(String, String)>,
pub mounts: Vec<(String, String)>,
pub env: Vec<(String, String)>,
pub cmd: Vec<String>,
pub limits: Limits,
pub disk_bytes: u64,
pub hardened: bool,
pub pull_config: Option<std::path::PathBuf>,
}
pub fn real_nvidia_smi() -> Result<String, String> {
let out = std::process::Command::new("nvidia-smi")
.args(["--query-gpu=index,uuid,pci.bus_id", "--format=csv,noheader"])
.output()
.map_err(|e| format!("nvidia-smi not available: {e}"))?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
pub fn start_job(
eph: &Ephemeral,
run: &JobRun,
nvidia_smi: &dyn Fn() -> Result<String, String>,
) -> Result<String, String> {
let picked = if run.limits.gpus.flag_value().is_some() {
let all = parse_nvidia_smi(&nvidia_smi()?);
pick_gpus(&run.limits.gpus, &all, &eph.taken_gpus()?)
.map_err(|e| format!("{}: {e}", run.name))?
} else {
vec![]
};
let mut inner_mounts = Vec::new();
let mut inst_mounts = Vec::new();
for (host, container) in &run.mounts {
let at = format!(
"/zakuro/mnt/{}",
&crate::dataset::sha256_hex(host.as_bytes())[..16]
);
inst_mounts.push((host.clone(), at.clone()));
inner_mounts.push((at, container.clone()));
}
let instance = eph.launch(&InstanceSpec {
job: run.name.clone(),
limits: run.limits.clone(),
disk_bytes: run.disk_bytes,
gpus: picked.iter().map(|g| g.pci.clone()).collect(),
mounts: inst_mounts,
})?;
let started = (|| {
if let Some(dir) = &run.pull_config {
let f = dir.join("config.json");
if f.exists() {
eph.push_docker_config(&instance, &f)?;
}
}
eph.nerdctl(&instance, &s(&["pull", "--quiet", &run.image]))
.map_err(|e| format!("pull {} failed: {e}", run.image))?;
eph.run_container(
&instance,
&StartSpec {
name: run.name.clone(),
image: run.image.clone(),
network: None,
create_network: false,
labels: run.labels.clone(),
mounts: inner_mounts,
publish: vec![],
env: run.env.clone(),
cmd: run.cmd.clone(),
limits: inner_limits(&run.limits, !picked.is_empty()),
remove_on_exit: false,
hardened: run.hardened,
},
)
})();
match started {
Ok(_) => Ok(job_id(&instance, &run.name)),
Err(e) => {
let _ = eph.destroy(&instance);
Err(e)
}
}
}
pub const REAP_EVERY: Duration = Duration::from_secs(120);
pub fn start_reaper_once(eph: &Ephemeral) {
static STARTED: std::sync::Once = std::sync::Once::new();
let eph = eph.clone();
let first = eph.clone();
STARTED.call_once(move || {
for n in first.reap_leaked() {
eprintln!(" [SANDBOX] removed leaked job instance {n}");
}
let _ = std::thread::Builder::new()
.name("sandbox-reaper".into())
.spawn(move || loop {
std::thread::sleep(REAP_EVERY);
for n in eph.reap_leaked() {
eprintln!(" [SANDBOX] removed leaked job instance {n}");
}
});
});
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sandbox::lxc::DEFAULT_IMAGE;
#[derive(Default)]
struct Rec {
calls: Mutex<Vec<String>>,
answer: Mutex<Vec<(&'static str, Result<String, String>)>>,
}
impl Rec {
fn on(&self, needle: &'static str, a: Result<&str, &str>) {
self.answer
.lock()
.unwrap()
.push((needle, a.map(String::from).map_err(String::from)));
}
fn calls(&self) -> Vec<String> {
self.calls.lock().unwrap().clone()
}
}
impl IncusCli for Rec {
fn run(&self, argv: &[String]) -> Result<String, String> {
let j = argv.join(" ");
self.calls.lock().unwrap().push(j.clone());
for (needle, a) in self.answer.lock().unwrap().iter() {
if j.contains(needle) {
return a.clone();
}
}
Ok(String::new())
}
}
fn eph(rec: &Arc<Rec>) -> Ephemeral {
Ephemeral::new(
rec.clone(),
DEFAULT_IMAGE,
Ok(SandboxSize {
limits: Limits::new(8.0, 16.0, Gpus::None),
disk_bytes: 20 << 30,
}),
)
.fast()
}
fn spec(gpus: Vec<String>) -> InstanceSpec {
spec_for("zk-space-abc", gpus)
}
fn spec_for(job: &str, gpus: Vec<String>) -> InstanceSpec {
InstanceSpec {
job: job.into(),
limits: Limits::new(1.5, 2.0, Gpus::None),
disk_bytes: 10 << 30,
gpus,
mounts: vec![(
"/home/u/.zakuro/spaces/abc.sock".into(),
"/zakuro/sock".into(),
)],
}
}
fn golden_ok(rec: &Rec) {
rec.on(
"image get-property zakuro-sandbox-base",
Ok(Box::leak(provision_version(DEFAULT_IMAGE).into_boxed_str())),
);
}
#[test]
fn a_job_instance_is_ephemeral_limited_and_owned() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let name = eph(&rec).launch(&spec(vec![])).unwrap();
assert_eq!(name, instance_name("zk-space-abc"));
assert!(name.starts_with("zk-job-") && name.len() == 15, "{name}");
let calls = rec.calls();
let init = calls.iter().find(|c| c.starts_with("incus init")).unwrap();
assert_eq!(
init,
&format!(
"incus init zakuro-sandbox-base {name} --ephemeral \
-c limits.cpu=2 -c limits.cpu.allowance=150ms/100ms -c limits.memory=2048MiB \
-c limits.memory.enforce=hard -c limits.memory.swap=false \
-c security.privileged=false -c security.nesting=true -c nvidia.runtime=false \
-c user.zakuro.budget=1500m,2147483648 -c user.zakuro.owner=4242 \
-c user.zakuro.gpus= -c user.zakuro.lease=zk-space-abc -d root,size=10240MiB"
)
);
let mnt = calls
.iter()
.position(|c| c.contains("zakuro-mnt-0"))
.unwrap();
let start = calls
.iter()
.position(|c| c == &format!("incus start {name}"))
.unwrap();
assert!(mnt < start, "{calls:#?}");
assert!(calls[mnt]
.contains("source=/home/u/.zakuro/spaces/abc.sock path=/zakuro/sock shift=true"));
assert!(!calls.iter().any(|c| c.contains(" gpu ")), "no GPU unasked");
assert!(calls
.iter()
.any(|c| c.contains("systemctl is-active --quiet containerd")));
live_names().lock().unwrap().remove(&name);
}
#[test]
fn a_picked_gpu_is_attached_by_pci_address_and_enables_the_nvidia_runtime() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let name = eph(&rec)
.launch(&spec(vec!["0000:01:00.0".into()]))
.unwrap();
let calls = rec.calls();
assert!(calls.iter().any(|c| c
== &format!(
"incus config device add {name} zakuro-gpu-0 gpu gputype=physical pci=0000:01:00.0"
)));
let init = calls.iter().find(|c| c.starts_with("incus init")).unwrap();
assert!(init.contains("-c nvidia.runtime=true"), "{init}");
assert!(init.contains("-c user.zakuro.gpus=0000:01:00.0"), "{init}");
live_names().lock().unwrap().remove(&name);
}
#[test]
fn a_failed_start_deletes_the_instance_and_says_why() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("incus start", Err("no space left on device"));
let e = eph(&rec)
.launch(&spec_for("zk-space-fail", vec![]))
.unwrap_err();
assert!(
e.contains("incus start") && e.contains("no space left"),
"{e}"
);
let calls = rec.calls();
assert!(
calls.last().unwrap().starts_with("incus delete -f zk-job-"),
"{calls:#?}"
);
assert!(!live_names()
.lock()
.unwrap()
.contains(&instance_name("zk-space-fail")));
}
#[test]
fn an_instance_that_never_gets_containerd_is_deleted() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("systemctl is-active", Err("Failed to connect to bus"));
let e = eph(&rec).launch(&spec(vec![])).unwrap_err();
assert!(e.contains("did not become ready"), "{e}");
assert!(rec.calls().last().unwrap().starts_with("incus delete -f"));
}
#[test]
fn an_invalid_budget_refuses_every_instance() {
let rec = Arc::new(Rec::default());
let e = Ephemeral::new(
rec.clone(),
DEFAULT_IMAGE,
Err("ZAKURO_BUDGET_CPU=bad".into()),
)
.launch(&spec(vec![]))
.unwrap_err();
assert_eq!(e, "ZAKURO_BUDGET_CPU=bad");
assert!(rec.calls().is_empty());
}
#[test]
fn the_golden_image_is_built_once_from_the_provisioning_and_versioned() {
let rec = Arc::new(Rec::default());
rec.on("image get-property", Err("Image not found"));
eph(&rec).ensure_golden().unwrap();
let calls = rec.calls();
let ver = provision_version(DEFAULT_IMAGE);
let pos = |needle: &str| {
calls
.iter()
.position(|c| c.contains(needle))
.unwrap_or_else(|| panic!("{needle} missing in {calls:#?}"))
};
assert!(pos("incus launch images:ubuntu/24.04 zk-golden-build") < pos("sh -c set -e;"));
assert!(calls[pos("sh -c set -e;")].contains("sha256sum -c"));
assert!(pos("sh -c set -e;") < pos("incus stop zk-golden-build"));
assert_eq!(
calls[pos("incus publish")],
format!(
"incus publish zk-golden-build --alias zakuro-sandbox-base zakuro.provision={ver}"
)
);
assert!(calls.last().unwrap() == "incus delete -f zk-golden-build");
}
#[test]
fn a_current_golden_image_is_reused_and_an_old_one_rebuilt() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
eph(&rec).ensure_golden().unwrap();
assert_eq!(rec.calls().len(), 1, "{:#?}", rec.calls());
let old = Arc::new(Rec::default());
old.on("image get-property", Ok("0123456789ab"));
eph(&old).ensure_golden().unwrap();
assert!(old
.calls()
.iter()
.any(|c| c == "incus image delete zakuro-sandbox-base"));
assert!(old.calls().iter().any(|c| c.starts_with("incus publish")));
assert_ne!(
provision_version("images:debian/12"),
provision_version(DEFAULT_IMAGE)
);
}
#[test]
fn a_failed_golden_build_leaves_no_scratch_instance() {
let rec = Arc::new(Rec::default());
rec.on("image get-property", Err("not found"));
rec.on("sha256sum", Err("curl: (6) Could not resolve host"));
let e = eph(&rec).ensure_golden().unwrap_err();
assert!(e.contains("provisioning containerd"), "{e}");
assert_eq!(
rec.calls().last().unwrap(),
"incus delete -f zk-golden-build"
);
assert!(!rec.calls().iter().any(|c| c.starts_with("incus publish")));
}
#[test]
fn the_inner_container_is_capped_below_the_boundary() {
let l = Limits::new(2.0, 4.0, Gpus::None);
let i = inner_limits(&l, false);
assert_eq!(i.cpu_millis, 2000);
assert_eq!(i.memory_bytes, 4 * 1024 * MIB - 256 * MIB);
let tiny = inner_limits(&Limits::new(1.0, 0.5, Gpus::None), true);
assert_eq!(tiny.memory_bytes, 384 * MIB);
assert_eq!(tiny.gpus, Gpus::All);
}
#[test]
fn allocations_come_from_the_instances_own_budget_key() {
let rec = Arc::new(Rec::default());
rec.on(
"list zk-job- -f json",
Ok(r#"[{"name":"zk-job-aaaaaaaa","status":"Running","config":{"user.zakuro.budget":"1500m,2147483648","user.zakuro.owner":"7","user.zakuro.gpus":"0000:01:00.0,0000:02:00.0"}},
{"name":"zk-job-bbbbbbbb","status":"Running","config":{}},
{"name":"other","status":"Running","config":{"user.zakuro.budget":"1m,1"}}]"#),
);
let e = eph(&rec);
assert_eq!(
e.allocations().unwrap(),
vec![Allocation {
name: "zk-job-aaaaaaaa".into(),
cpu_millis: 1500,
memory_bytes: 2 << 30
}]
);
assert_eq!(
e.taken_gpus().unwrap(),
HashSet::from(["0000:01:00.0".to_string(), "0000:02:00.0".to_string()])
);
}
#[test]
fn the_reaper_removes_dead_owners_and_unowned_but_not_live_jobs() {
let rec = Arc::new(Rec::default());
rec.on(
"list zk-job- -f json",
Ok(r#"[{"name":"zk-job-dead0000","status":"Running","config":{"user.zakuro.owner":"111"}},
{"name":"zk-job-live1111","status":"Running","config":{"user.zakuro.owner":"222"}},
{"name":"zk-job-mine2222","status":"Running","config":{"user.zakuro.owner":"4242"}},
{"name":"zk-job-mine3333","status":"Running","config":{"user.zakuro.owner":"4242"}},
{"name":"zk-job-noown444","status":"Stopped","config":{}}]"#),
);
let e = eph(&rec);
let live = HashSet::from(["zk-job-mine2222".to_string()]);
let gone = e.reap(&|p| p == 222, &live);
assert_eq!(
gone,
vec!["zk-job-dead0000", "zk-job-mine3333", "zk-job-noown444"]
);
let calls = rec.calls();
assert!(!calls
.iter()
.any(|c| c.contains("delete -f zk-job-live1111")));
assert!(!calls
.iter()
.any(|c| c.contains("delete -f zk-job-mine2222")));
}
#[test]
fn destroying_a_gone_instance_is_not_an_error() {
let rec = Arc::new(Rec::default());
rec.on("incus delete", Err("Instance not found"));
assert!(eph(&rec).destroy("zk-job-x").is_ok());
let rec = Arc::new(Rec::default());
rec.on("incus delete", Err("permission denied"));
assert!(eph(&rec).destroy("zk-job-x").is_err());
}
fn gpus() -> Vec<GpuInfo> {
parse_nvidia_smi("0, GPU-aaaa-1111, 00000000:01:00.0\n1, GPU-bbbb-2222, 00000000:41:00.0\n")
}
#[test]
fn nvidia_smi_output_gives_index_uuid_and_a_sysfs_pci_address() {
let g = gpus();
assert_eq!(g.len(), 2);
assert_eq!(g[1].pci, "0000:41:00.0");
assert_eq!(g[0].uuid, "GPU-aaaa-1111");
assert!(parse_nvidia_smi("garbage\n,,\n").is_empty());
}
#[test]
fn a_uuid_pin_resolves_to_its_card_and_one_job_gets_one_gpu() {
let g = gpus();
let none = HashSet::new();
let one = pick_gpus(&Gpus::Spec("device=GPU-bbbb-2222".into()), &g, &none).unwrap();
assert_eq!(one.len(), 1);
assert_eq!(one[0].pci, "0000:41:00.0");
let by_index = pick_gpus(&Gpus::Spec("device=0".into()), &g, &none).unwrap();
assert_eq!(by_index[0].uuid, "GPU-aaaa-1111");
let taken = HashSet::from(["0000:01:00.0".to_string()]);
let c = pick_gpus(&Gpus::Count(1), &g, &taken).unwrap();
assert_eq!(c[0].pci, "0000:41:00.0");
assert!(pick_gpus(&Gpus::Count(2), &g, &taken)
.unwrap_err()
.contains("one job per GPU"));
assert!(pick_gpus(&Gpus::Spec("device=0".into()), &g, &taken)
.unwrap_err()
.contains("held by another job"));
assert!(pick_gpus(&Gpus::Spec("device=GPU-nope".into()), &g, &none).is_err());
assert!(pick_gpus(&Gpus::None, &g, &none).unwrap().is_empty());
assert_eq!(pick_gpus(&Gpus::All, &g, &none).unwrap().len(), 2);
assert!(pick_gpus(&Gpus::All, &[], &none).is_err());
}
#[test]
fn the_registry_login_is_a_docker_config_not_an_argv() {
let j = docker_config_json("docker.io", "zak", "s3cret");
let v: serde_json::Value = serde_json::from_str(&j).unwrap();
let auth = v["auths"]["https://index.docker.io/v1/"]["auth"]
.as_str()
.unwrap();
use base64::Engine as _;
assert_eq!(
base64::engine::general_purpose::STANDARD
.decode(auth)
.unwrap(),
b"zak:s3cret"
);
let r = docker_config_json("registry.example.com", "u", "p");
assert!(r.contains("registry.example.com"));
let rec = Arc::new(Rec::default());
eph(&rec)
.push_docker_config("zk-job-1", std::path::Path::new("/tmp/cfg/config.json"))
.unwrap();
let c = rec.calls().pop().unwrap();
assert_eq!(
c,
"incus file push --create-dirs --mode 0600 /tmp/cfg/config.json zk-job-1/root/.docker/config.json"
);
}
#[test]
fn instance_addresses_and_names() {
assert_eq!(
parse_ipv4("10.20.30.4 (eth0)\nfd42::1 (eth0)"),
Some("10.20.30.4".into())
);
assert_eq!(
parse_ipv4("\"10.0.0.9 (eth0)\n172.17.0.1 (docker0)\""),
Some("10.0.0.9".into())
);
assert_eq!(parse_ipv4(""), None);
assert_eq!(instance_name("a"), instance_name("a"));
assert_ne!(instance_name("a"), instance_name("b"));
}
#[test]
fn the_container_runs_under_nerdctl_in_its_instance_with_the_same_flags() {
let rec = Arc::new(Rec::default());
let e = eph(&rec);
let spec = StartSpec {
name: "zk-space-abc".into(),
image: "img@sha256:ab".into(),
network: None,
create_network: false,
labels: vec![],
mounts: vec![("/zakuro/sock".into(), "/run/zakuro/space.sock".into())],
publish: vec![],
env: vec![],
cmd: vec![],
limits: inner_limits(&Limits::new(1.0, 2.0, Gpus::None), false),
remove_on_exit: false,
hardened: true,
};
e.run_container("zk-job-1", &spec).unwrap();
let c = rec.calls().pop().unwrap();
assert!(
c.starts_with("incus exec zk-job-1 -- nerdctl run -d --name zk-space-abc"),
"{c}"
);
assert!(c.contains("--cpus 1 --memory 1792m"), "{c}");
assert!(
c.contains("--pids-limit 4096 --security-opt no-new-privileges"),
"{c}"
);
}
#[test]
fn a_job_start_pulls_inside_runs_capped_and_returns_a_routable_id() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let run = JobRun {
name: "zk-space-run1".into(),
image: "reg.example.com/space@sha256:ab".into(),
labels: vec![("zakuro.space".into(), "s1".into())],
mounts: vec![("/h/s1.sock".into(), "/run/zakuro/space.sock".into())],
env: vec![("ZAKURO_SPACE_ID".into(), "s1".into())],
cmd: vec![],
limits: Limits::new(1.0, 2.0, Gpus::None),
disk_bytes: 5 << 30,
hardened: true,
pull_config: Some(std::path::PathBuf::from("/nonexistent-cfg")),
};
let id = start_job(&eph(&rec), &run, &|| Err("no gpus asked".into())).unwrap();
let (inst, name) = parse_job_id(&id).unwrap();
assert_eq!(
(inst, name),
(instance_name("zk-space-run1").as_str(), "zk-space-run1")
);
let calls = rec.calls();
let pull = calls
.iter()
.position(|c| c.contains("nerdctl pull --quiet reg.example.com/space@sha256:ab"))
.unwrap();
let run_at = calls
.iter()
.position(|c| c.contains("nerdctl run -d --name zk-space-run1"))
.unwrap();
assert!(pull < run_at, "{calls:#?}");
let r = &calls[run_at];
assert!(r.contains("--cpus 1 --memory 1792m"), "{r}");
assert!(r.contains("--pids-limit 4096"), "{r}");
assert!(!r.contains("--gpus"), "{r}");
assert!(r.contains("-v /zakuro/mnt/"), "{r}");
assert!(r.contains(":/run/zakuro/space.sock"), "{r}");
live_names().lock().unwrap().remove(inst);
}
#[test]
fn a_pull_that_fails_inside_deletes_the_instance() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("nerdctl pull", Err("manifest unknown"));
let run = JobRun {
name: "zk-space-pullfail".into(),
image: "img".into(),
labels: vec![],
mounts: vec![],
env: vec![],
cmd: vec![],
limits: Limits::new(1.0, 1.0, Gpus::None),
disk_bytes: 5 << 30,
hardened: true,
pull_config: None,
};
let e = start_job(&eph(&rec), &run, &|| Err(String::new())).unwrap_err();
assert!(e.contains("pull img failed: manifest unknown"), "{e}");
assert!(rec
.calls()
.last()
.unwrap()
.starts_with("incus delete -f zk-job-"));
}
#[test]
fn a_gpu_job_gets_a_free_card_by_pci_and_a_refusal_when_none_is_free() {
let smi =
|| Ok("0, GPU-aaaa, 00000000:01:00.0\n1, GPU-bbbb, 00000000:41:00.0\n".to_string());
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on(
"list zk-job- -f json",
Ok(r#"[{"name":"zk-job-other000","status":"Running","config":{"user.zakuro.gpus":"0000:01:00.0","user.zakuro.owner":"1"}}]"#),
);
let mut run = JobRun {
name: "zk-space-gpu1".into(),
image: "img".into(),
labels: vec![],
mounts: vec![],
env: vec![],
cmd: vec![],
limits: Limits::new(1.0, 2.0, Gpus::Spec("device=GPU-bbbb".into())),
disk_bytes: 5 << 30,
hardened: true,
pull_config: None,
};
let id = start_job(&eph(&rec), &run, &smi).unwrap();
let calls = rec.calls();
assert!(
calls
.iter()
.any(|c| c.contains("gputype=physical pci=0000:41:00.0")),
"{calls:#?}"
);
assert!(calls
.iter()
.any(|c| c.contains("nerdctl run") && c.contains("--gpus all")));
live_names()
.lock()
.unwrap()
.remove(parse_job_id(&id).unwrap().0);
run.name = "zk-space-gpu2".into();
run.limits = Limits::new(1.0, 2.0, Gpus::Spec("device=0".into()));
let before = rec.calls().len();
let e = start_job(&eph(&rec), &run, &smi).unwrap_err();
assert!(e.contains("held by another job"), "{e}");
assert!(!rec.calls()[before..]
.iter()
.any(|c| c.starts_with("incus init")));
}
#[test]
fn job_ids_round_trip_and_plain_ids_are_not_jobs() {
let id = job_id("zk-job-abcd1234", "zk-space-x");
assert_eq!(parse_job_id(&id), Some(("zk-job-abcd1234", "zk-space-x")));
for plain in ["abc123", "t2:", "t2:/x", "t2:i/", ""] {
assert_eq!(parse_job_id(plain), None, "{plain:?}");
}
}
#[test]
#[ignore]
fn real_incus_job_instance_end_to_end() {
if std::env::var("ZAKURO_TEST_REAL_INCUS").ok().as_deref() != Some("1") {
eprintln!("skipped: set ZAKURO_TEST_REAL_INCUS=1");
return;
}
let cli: Arc<dyn IncusCli> = Arc::new(lxc::IncusShell);
let image = std::env::var("ZAKURO_SANDBOX_IMAGE")
.unwrap_or_else(|_| lxc::default_image(lxc::cli_binary()).to_string());
let e = Ephemeral::new(
cli.clone(),
&image,
Ok(SandboxSize {
limits: Limits::new(2.0, 4.0, Gpus::None),
disk_bytes: 10 << 30,
}),
);
let t = std::time::Instant::now();
e.ensure_golden().expect("golden image");
println!("REAL golden image ready in {:?}", t.elapsed());
let job = format!("zk-real-{}", std::process::id());
let limits = Limits::new(1.0, 1.0, Gpus::None);
let name = e
.launch(&InstanceSpec {
job: job.clone(),
limits: limits.clone(),
disk_bytes: 5 << 30,
gpus: vec![],
mounts: vec![],
})
.expect("launch");
println!("REAL instance {name} running");
let inner = inner_limits(&limits, false);
let spec = StartSpec {
name: "real-job".into(),
image: "docker.io/library/busybox:latest".into(),
network: None,
create_network: false,
labels: vec![],
mounts: vec![],
publish: vec![],
env: vec![],
cmd: vec!["sleep".into(), "120".into()],
limits: inner.clone(),
remove_on_exit: false,
hardened: true,
};
e.nerdctl(&name, &s(&["pull", "--quiet", &spec.image]))
.expect("pull inside");
e.run_container(&name, &spec).expect("run container");
let cg = e
.nerdctl(
&name,
&s(&[
"exec",
"real-job",
"sh",
"-c",
"cat /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max",
]),
)
.expect("read cgroup");
let read = lxc::parse_cgroup_limits(&cg).expect("limits are set");
println!("REAL container limits read back from inside: {read}");
assert_eq!(read.cpu_millis, inner.cpu_millis);
assert_eq!(read.memory_bytes, inner.memory_bytes);
let inst_mem = e.run(&["config", "get", &name, "limits.memory"]).unwrap();
println!("REAL instance limits.memory={inst_mem}");
assert_eq!(inst_mem.trim(), "1024MiB");
let out = e
.nerdctl(
&name,
&s(&[
"run",
"--rm",
"docker.io/library/busybox:latest",
"echo",
"hello-from-t2",
]),
)
.expect("echo");
println!("REAL echo: {out}");
assert!(out.contains("hello-from-t2"));
assert!(e.allocations().unwrap().iter().any(|a| a.name == name));
e.destroy(&name).expect("destroy");
assert!(!e.list().unwrap().iter().any(|i| i.name == name));
println!("REAL instance {name} gone after destroy");
let leaked = format!("{JOB_PREFIX}leak{:04}", std::process::id() % 10000);
e.run(&[
"init",
GOLDEN_ALIAS,
&leaked,
"--ephemeral",
"-c",
"user.zakuro.owner=2147483646",
"-c",
"user.zakuro.budget=1000m,1073741824",
])
.expect("create leaked");
assert!(e.list().unwrap().iter().any(|i| i.name == leaked));
let gone = e.reap_leaked();
println!("REAL reaper removed {gone:?}");
assert!(gone.contains(&leaked));
assert!(!e.list().unwrap().iter().any(|i| i.name == leaked));
}
}