use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use super::lxc::{self, IncusCli, SandboxSize};
use super::{docker, Allocation, Gpus, Limits, StartSpec};
pub const JOB_PREFIX: &str = "zk-job-";
pub const GOLDEN_ALIAS: &str = "zakuro-sandbox-base";
pub const BUILD_INSTANCE: &str = "zk-golden-build";
pub const PROVISION_PROP: &str = "zakuro.provision";
pub const CFG_BUDGET: &str = "user.zakuro.budget";
pub const CFG_OWNER: &str = "user.zakuro.owner";
pub const CFG_GPUS: &str = "user.zakuro.gpus";
pub const CFG_LEASE: &str = "user.zakuro.lease";
pub const CFG_KIND: &str = "user.zakuro.kind";
pub const CFG_WARM: &str = "user.zakuro.warm";
pub const WARM_PREFIX: &str = "zk-job-w";
pub const IMG_PREFIX: &str = "zk-img-";
const MIB: u64 = 1024 * 1024;
const OVERHEAD_BYTES: u64 = 256 * MIB;
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|a| a.to_string()).collect()
}
pub fn provision_version(base_image: &str) -> String {
let material = format!(
"{base_image}\n{}\n{}\n{}",
lxc::NERDCTL_VERSION,
lxc::provision_script(),
job_egress_script()
);
crate::dataset::sha256_hex(material.as_bytes())[..12].to_string()
}
pub fn instance_name(job: &str) -> String {
format!(
"{JOB_PREFIX}{}",
&crate::dataset::sha256_hex(job.as_bytes())[..8]
)
}
pub fn job_egress_script() -> String {
"set -e; C=${1:-10.4.0.0/16}; \
NS=$(awk '/^nameserver/{print $2}' /etc/resolv.conf /run/systemd/resolve/resolv.conf 2>/dev/null | grep -E '^[0-9.]+$' | sort -u); \
DEV=$(ip -4 route show default | awk '{print $5; exit}'); \
GW=$(ip -4 route show default | awk '{print $3; exit}'); \
SUB=$( [ -n \"$DEV\" ] && ip -4 -o addr show dev $DEV | awk '{print $4; exit}' || true); \
iptables -N ZAKURO-EGRESS 2>/dev/null || true; iptables -F ZAKURO-EGRESS; \
iptables -A ZAKURO-EGRESS -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN; \
for ns in $NS; do \
iptables -A ZAKURO-EGRESS -d $ns -p udp --dport 53 -j RETURN; \
iptables -A ZAKURO-EGRESS -d $ns -p tcp --dport 53 -j RETURN; \
done; \
[ -z \"$GW\" ] || iptables -A ZAKURO-EGRESS -d $GW -j REJECT; \
[ -z \"$SUB\" ] || iptables -A ZAKURO-EGRESS -d $SUB -j REJECT; \
iptables -A ZAKURO-EGRESS -d $C -j RETURN; \
for r in 10.13.13.0/24 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10; do \
iptables -A ZAKURO-EGRESS -d $r -j REJECT; \
done; \
iptables -N CNI-ADMIN 2>/dev/null || true; \
iptables -C CNI-ADMIN -s $C -j ZAKURO-EGRESS 2>/dev/null || iptables -A CNI-ADMIN -s $C -j ZAKURO-EGRESS; \
iptables -C FORWARD -s $C -j ZAKURO-EGRESS 2>/dev/null || iptables -I FORWARD 1 -s $C -j ZAKURO-EGRESS; \
iptables -N ZAKURO-IN 2>/dev/null || true; iptables -F ZAKURO-IN; \
for ns in $NS; do \
iptables -A ZAKURO-IN -d $ns -p udp --dport 53 -j RETURN; \
iptables -A ZAKURO-IN -d $ns -p tcp --dport 53 -j RETURN; \
done; \
iptables -A ZAKURO-IN -d $C -j RETURN; \
iptables -A ZAKURO-IN -m addrtype --dst-type LOCAL -j REJECT; \
iptables -C INPUT -s $C -j ZAKURO-IN 2>/dev/null || iptables -I INPUT 1 -s $C -j ZAKURO-IN"
.to_string()
}
pub fn inner_limits(l: &Limits, gpu_attached: bool) -> Limits {
let memory_bytes = if l.memory_bytes > 3 * OVERHEAD_BYTES {
l.memory_bytes - OVERHEAD_BYTES
} else {
(l.memory_bytes / 4 * 3).max(super::MIN_MEMORY_BYTES)
};
Limits {
cpu_millis: l.cpu_millis,
memory_bytes,
gpus: if gpu_attached { Gpus::All } else { Gpus::None },
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GpuInfo {
pub index: u32,
pub uuid: String,
pub pci: String,
}
pub fn normalize_pci(addr: &str) -> String {
let a = addr.trim().to_ascii_lowercase();
match a.split_once(':') {
Some((domain, rest)) if domain.len() > 4 => {
format!("{}:{rest}", &domain[domain.len() - 4..])
}
_ => a,
}
}
pub fn parse_nvidia_smi(out: &str) -> Vec<GpuInfo> {
out.lines()
.filter_map(|l| {
let mut f = l.split(',').map(str::trim);
let index = f.next()?.parse().ok()?;
let uuid = f.next()?.to_string();
let pci = normalize_pci(f.next()?);
(!uuid.is_empty() && !pci.is_empty()).then_some(GpuInfo { index, uuid, pci })
})
.collect()
}
pub fn pick_gpus(
want: &Gpus,
all: &[GpuInfo],
taken: &HashSet<String>,
) -> Result<Vec<GpuInfo>, String> {
let free: Vec<&GpuInfo> = all.iter().filter(|g| !taken.contains(&g.pci)).collect();
let by_id = |id: &str| -> Result<GpuInfo, String> {
let hit = all.iter().find(|g| {
g.uuid.eq_ignore_ascii_case(id)
|| g.pci == normalize_pci(id)
|| id.parse::<u32>().ok() == Some(g.index)
});
let g = hit.ok_or_else(|| format!("no GPU {id:?} on this node"))?;
if taken.contains(&g.pci) {
return Err(format!("GPU {id:?} is held by another job"));
}
Ok(g.clone())
};
let n_free = |n: usize| -> Result<Vec<GpuInfo>, String> {
if free.len() < n {
return Err(format!(
"asks for {n} GPU(s), {} free of {} (one job per GPU)",
free.len(),
all.len()
));
}
Ok(free.iter().take(n).map(|g| (*g).clone()).collect())
};
match want {
Gpus::None | Gpus::Count(0) => Ok(vec![]),
Gpus::All => {
if all.is_empty() {
return Err("asks for GPUs but this node has none".into());
}
n_free(all.len())
}
Gpus::Count(n) => n_free(*n as usize),
Gpus::Spec(spec) => {
let v = spec.trim().trim_matches('"').trim();
if v == "all" {
return pick_gpus(&Gpus::All, all, taken);
}
if let Ok(n) = v.parse::<u32>() {
return n_free(n as usize);
}
let ids = v.strip_prefix("device=").unwrap_or(v);
let mut out: Vec<GpuInfo> = Vec::new();
for id in ids.split(',').map(str::trim).filter(|i| !i.is_empty()) {
let g = by_id(id)?;
if !out.contains(&g) {
out.push(g);
}
}
if out.is_empty() {
return Err(format!("GPU spec {spec:?} names no GPU"));
}
Ok(out)
}
}
}
pub fn docker_config_json(registry: &str, user: &str, password: &str) -> String {
use base64::Engine as _;
let key = match registry {
"docker.io" | "index.docker.io" | "registry-1.docker.io" => {
"https://index.docker.io/v1/".to_string()
}
r => r.to_string(),
};
let auth = base64::engine::general_purpose::STANDARD.encode(format!("{user}:{password}"));
serde_json::json!({ "auths": { key: { "auth": auth } } }).to_string()
}
#[derive(Debug, Clone)]
pub struct InstanceSpec {
pub job: String,
pub limits: Limits,
pub disk_bytes: u64,
pub gpus: Vec<String>,
pub mounts: Vec<(String, String)>,
pub image: Option<String>,
pub persistent: bool,
pub config: Vec<(String, String)>,
pub shift: bool,
}
impl InstanceSpec {
pub fn new(
job: &str,
limits: Limits,
disk_bytes: u64,
gpus: Vec<String>,
mounts: Vec<(String, String)>,
) -> Self {
Self {
job: job.to_string(),
limits,
disk_bytes,
gpus,
mounts,
image: None,
persistent: false,
config: vec![],
shift: true,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobInstance {
pub name: String,
pub status: String,
pub owner_pid: Option<u32>,
pub budget: Option<String>,
pub gpus: Vec<String>,
pub kind: Option<String>,
pub lease: Option<String>,
pub warm: Option<String>,
}
pub fn parse_job_instances(json: &str) -> Vec<JobInstance> {
let Ok(serde_json::Value::Array(items)) = serde_json::from_str::<serde_json::Value>(json)
else {
return vec![];
};
items
.iter()
.filter_map(|i| {
let name = i.get("name")?.as_str()?.to_string();
if !name.starts_with(JOB_PREFIX) {
return None;
}
let cfg = i.get("config");
let get = |k: &str| {
cfg.and_then(|c| c.get(k))
.and_then(|v| v.as_str())
.map(str::to_string)
};
Some(JobInstance {
status: i
.get("status")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
owner_pid: get(CFG_OWNER).and_then(|p| p.parse().ok()),
budget: get(CFG_BUDGET),
kind: get(CFG_KIND).filter(|v| !v.is_empty()),
lease: get(CFG_LEASE).filter(|v| !v.is_empty()),
warm: get(CFG_WARM).filter(|v| !v.is_empty()),
gpus: get(CFG_GPUS)
.map(|g| {
g.split(',')
.filter(|x| !x.is_empty())
.map(str::to_string)
.collect()
})
.unwrap_or_default(),
name,
})
})
.collect()
}
pub fn parse_ipv4(csv: &str) -> Option<String> {
csv.split(['\n', '"']).map(str::trim).find_map(|l| {
let ip = l.split_whitespace().next()?;
(ip.split('.').count() == 4 && ip.split('.').all(|o| o.parse::<u8>().is_ok()))
.then(|| ip.to_string())
})
}
fn warm_suffix() -> String {
use std::sync::atomic::{AtomicU64, Ordering};
static N: AtomicU64 = AtomicU64::new(0);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let m = format!(
"{now}-{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::Relaxed)
);
crate::dataset::sha256_hex(m.as_bytes())[..7].to_string()
}
fn live_names() -> &'static Mutex<HashSet<String>> {
static LIVE: std::sync::OnceLock<Mutex<HashSet<String>>> = std::sync::OnceLock::new();
LIVE.get_or_init(|| Mutex::new(HashSet::new()))
}
#[cfg(unix)]
fn pid_alive(pid: u32) -> bool {
let rc = unsafe { libc::kill(pid as libc::pid_t, 0) };
rc == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
}
#[cfg(not(unix))]
fn pid_alive(_pid: u32) -> bool {
true
}
#[derive(Clone)]
pub struct Ephemeral {
cli: Arc<dyn IncusCli>,
base_image: String,
size: Result<SandboxSize, String>,
poll: Duration,
polls: u32,
pid: u32,
warm: usize,
inline_topup: bool,
cache: CacheCaps,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CacheCaps {
pub max_images: usize,
pub max_bytes: u64,
}
pub const DEFAULT_CACHE_IMAGES: usize = 8;
pub const DEFAULT_CACHE_BYTES: u64 = 60 << 30;
impl CacheCaps {
pub fn from_env(get: &dyn Fn(&str) -> Option<String>) -> Self {
let n = get("ZAKURO_SANDBOX_IMG_CACHE_MAX")
.and_then(|v| v.trim().parse::<usize>().ok())
.unwrap_or(DEFAULT_CACHE_IMAGES);
let gb = get("ZAKURO_SANDBOX_IMG_CACHE_GB")
.and_then(|v| v.trim().parse::<f64>().ok())
.filter(|g| *g >= 0.0)
.map(|g| (g * (1u64 << 30) as f64) as u64)
.unwrap_or(DEFAULT_CACHE_BYTES);
Self {
max_images: n,
max_bytes: gb,
}
}
}
pub fn warm_pool_size(get: &dyn Fn(&str) -> Option<String>) -> usize {
get("ZAKURO_SANDBOX_WARM_POOL")
.and_then(|v| v.trim().parse::<usize>().ok())
.unwrap_or(0)
.min(8)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CachedImage {
pub alias: String,
pub size: u64,
pub last_used: String,
pub golden: String,
pub digest: String,
pub tag: String,
pub pinned: bool,
pub cached_at: u64,
}
pub fn parse_cached_images(json: &str) -> Vec<CachedImage> {
let Ok(serde_json::Value::Array(items)) = serde_json::from_str::<serde_json::Value>(json)
else {
return vec![];
};
items
.iter()
.filter_map(|i| {
let alias = i
.get("aliases")?
.as_array()?
.iter()
.filter_map(|a| a.get("name")?.as_str())
.find(|n| n.starts_with(IMG_PREFIX))?
.to_string();
let prop = |k: &str| {
i.get("properties")
.and_then(|p| p.get(k))
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string()
};
let when = |k: &str| {
i.get(k)
.and_then(|v| v.as_str())
.filter(|v| !v.starts_with("0001-"))
.map(str::to_string)
};
Some(CachedImage {
size: i.get("size").and_then(|v| v.as_u64()).unwrap_or(0),
last_used: when("last_used_at")
.or_else(|| when("uploaded_at"))
.or_else(|| when("created_at"))
.unwrap_or_default(),
golden: prop("zakuro.golden"),
digest: prop("zakuro.digest"),
tag: prop("zakuro.tag"),
pinned: prop("zakuro.pinned") == "1",
cached_at: prop("zakuro.cached_at").parse().unwrap_or(0),
alias,
})
})
.collect()
}
pub fn lru_victims(images: &[CachedImage], caps: CacheCaps, keep: &str) -> Vec<String> {
let mut by_age: Vec<&CachedImage> = images.iter().collect();
by_age.sort_by(|a, b| a.last_used.cmp(&b.last_used).then(a.alias.cmp(&b.alias)));
let mut count = images.len();
let mut bytes: u64 = images.iter().map(|i| i.size).sum();
let mut out = vec![];
for i in by_age {
if count <= caps.max_images && bytes <= caps.max_bytes {
break;
}
if i.alias == keep {
continue;
}
count -= 1;
bytes = bytes.saturating_sub(i.size);
out.push(i.alias.clone());
}
out
}
impl Ephemeral {
pub fn new(
cli: Arc<dyn IncusCli>,
base_image: &str,
size: Result<SandboxSize, String>,
) -> Self {
Self {
cli,
base_image: base_image.to_string(),
size,
poll: Duration::from_secs(1),
polls: 90,
pid: std::process::id(),
warm: 0,
inline_topup: false,
cache: CacheCaps {
max_images: DEFAULT_CACHE_IMAGES,
max_bytes: DEFAULT_CACHE_BYTES,
},
}
}
pub fn with_warm_pool(mut self, n: usize) -> Self {
self.warm = n;
self
}
pub fn with_cache_caps(mut self, c: CacheCaps) -> Self {
self.cache = c;
self
}
pub fn from_env(get: &dyn Fn(&str) -> Option<String>) -> Self {
let rt = lxc::LxcRuntime::from_env(Arc::new(lxc::IncusShell), get);
Self::new(Arc::new(lxc::IncusShell), rt.image(), rt.size().clone())
.with_warm_pool(warm_pool_size(get))
.with_cache_caps(CacheCaps::from_env(get))
}
#[cfg(test)]
pub(crate) fn fast_for_tests(&mut self) {
self.poll = Duration::ZERO;
self.polls = 3;
self.pid = 4242;
self.inline_topup = true;
}
#[cfg(test)]
fn fast(mut self) -> Self {
self.poll = Duration::ZERO;
self.polls = 3;
self.pid = 4242;
self.inline_topup = true;
self
}
pub fn run(&self, rest: &[&str]) -> Result<String, String> {
let mut argv = vec!["incus".to_string()];
argv.extend(rest.iter().map(|a| a.to_string()));
self.cli.run(&argv)
}
pub fn ensure_golden(&self) -> Result<String, String> {
let want = provision_version(&self.base_image);
if self.golden_version().as_deref() == Some(want.as_str()) {
return Ok(GOLDEN_ALIAS.to_string());
}
let _guard = super::lock::provision_lock(Duration::from_secs(900))?;
if self.golden_version().as_deref() == Some(want.as_str()) {
return Ok(GOLDEN_ALIAS.to_string());
}
self.build_golden(&want)?;
Ok(GOLDEN_ALIAS.to_string())
}
fn golden_version(&self) -> Option<String> {
self.run(&["image", "get-property", GOLDEN_ALIAS, PROVISION_PROP])
.ok()
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
}
fn build_golden(&self, version: &str) -> Result<(), String> {
let b = BUILD_INSTANCE;
let _ = self.run(&["delete", "-f", b]);
let _ = self.run(&["image", "delete", GOLDEN_ALIAS]);
self.run(&[
"launch",
&self.base_image,
b,
"-c",
"security.privileged=false",
"-c",
"security.nesting=true",
])
.map_err(|e| format!("golden image: launching {b} failed: {e}"))?;
let built = (|| {
self.run(&["exec", b, "--", "sh", "-c", &lxc::provision_script()])
.map_err(|e| format!("provisioning containerd in {b} failed: {e}"))?;
let _ = self.run(&[
"exec",
b,
"--",
"sh",
"-c",
"apt-get clean; rm -rf /var/lib/apt/lists/* /tmp/*",
]);
self.run(&["stop", b])
.map_err(|e| format!("stopping {b} failed: {e}"))?;
self.run(&[
"publish",
b,
"--alias",
GOLDEN_ALIAS,
&format!("{PROVISION_PROP}={version}"),
])
.map_err(|e| format!("publishing {GOLDEN_ALIAS} failed: {e}"))?;
Ok::<(), String>(())
})();
let _ = self.run(&["delete", "-f", b]);
built
}
fn config_pairs(&self, spec: &InstanceSpec) -> Vec<(String, String)> {
let mut limits = spec.limits.clone();
limits.gpus = if spec.gpus.is_empty() {
Gpus::None
} else {
Gpus::All
};
let mut v: Vec<(String, String)> = lxc::sandbox_config(&limits);
for (k, val) in [
(CFG_BUDGET, spec.limits.budget_label()),
(CFG_OWNER, self.pid.to_string()),
(CFG_GPUS, spec.gpus.join(",")),
(CFG_LEASE, spec.job.clone()),
] {
v.push((k.to_string(), val));
}
v.extend(spec.config.iter().cloned());
v
}
fn config_args(&self, spec: &InstanceSpec) -> Vec<String> {
let mut a: Vec<String> = Vec::new();
for (k, v) in self.config_pairs(spec) {
a.push("-c".into());
a.push(format!("{k}={v}"));
}
a
}
pub fn launch(&self, spec: &InstanceSpec) -> Result<String, String> {
let node = self.size.clone()?;
let golden = self.ensure_golden()?;
let image = spec.image.clone().unwrap_or_else(|| golden.clone());
let name = instance_name(&spec.job);
live_names().lock().unwrap().insert(name.clone());
let r = self.launch_inner(spec, &image, &name, node.disk_bytes);
if r.is_err() {
let _ = self.destroy(&name);
}
r.map(|_| name)
}
fn launch_inner(
&self,
spec: &InstanceSpec,
image: &str,
name: &str,
node_disk: u64,
) -> Result<(), String> {
let disk = spec.disk_bytes.min(node_disk).max(1 << 30);
let _ = self.run(&["delete", "-f", name]);
let warm = spec.image.is_none() && !spec.persistent && self.warm > 0;
if !(warm && self.take_warm(spec, name, disk)) {
let mut init = s(&["incus", "init", image, name]);
if !spec.persistent {
init.push("--ephemeral".into());
}
init.extend(self.config_args(spec));
init.push("-d".into());
init.push(format!("root,size={}MiB", (disk / MIB).max(6)));
self.cli
.run(&init)
.map_err(|e| format!("incus init {name} failed: {e}"))?;
}
for (i, pci) in spec.gpus.iter().enumerate() {
self.run(&[
"config",
"device",
"add",
name,
&format!("zakuro-gpu-{i}"),
"gpu",
"gputype=physical",
&format!("pci={pci}"),
])
.map_err(|e| format!("attaching GPU {pci} to {name} failed: {e}"))?;
}
for (i, (host, inner)) in spec.mounts.iter().enumerate() {
self.run(&[
"config",
"device",
"add",
name,
&format!("zakuro-mnt-{i}"),
"disk",
&format!("source={host}"),
&format!("path={inner}"),
&format!("shift={}", spec.shift),
])
.map_err(|e| format!("mounting {host} into {name} failed: {e}"))?;
}
self.run(&["start", name])
.map_err(|e| format!("incus start {name} failed: {e}"))?;
self.wait_ready(name)?;
self.apply_egress(name)?;
if warm {
self.spawn_topup();
}
Ok(())
}
fn wait_ready(&self, name: &str) -> Result<(), String> {
let mut last = String::new();
for _ in 0..self.polls {
match self.run(&[
"exec",
name,
"--",
"sh",
"-c",
"systemctl is-active --quiet containerd && nerdctl version >/dev/null 2>&1",
]) {
Ok(_) => return Ok(()),
Err(e) => last = e,
}
if !self.poll.is_zero() {
std::thread::sleep(self.poll);
}
}
Err(format!(
"{name} did not become ready (containerd): {}",
last.lines().next().unwrap_or("no answer")
))
}
pub fn apply_egress(&self, name: &str) -> Result<(), String> {
self.run(&[
"exec",
name,
"--",
"sh",
"-c",
&job_egress_script(),
"zakuro-egress",
])
.map(|_| ())
.map_err(|e| format!("egress rules in {name} failed (the job is not started): {e}"))
}
fn take_warm(&self, spec: &InstanceSpec, name: &str, disk: u64) -> bool {
let want = provision_version(&self.base_image);
let Ok(all) = self.list() else {
return false;
};
for w in all.iter().filter(|i| {
i.warm.as_deref() == Some(want.as_str()) && i.status.eq_ignore_ascii_case("stopped")
}) {
if self.run(&["rename", &w.name, name]).is_err() {
continue;
}
let owner = format!("{CFG_OWNER}={}", self.pid);
let applied = (|| {
self.run(&["config", "set", name, &owner])?;
self.run(&["config", "unset", name, CFG_WARM])?;
for (k, v) in self.config_pairs(spec) {
self.run(&["config", "set", name, &format!("{k}={v}")])?;
}
self.run(&[
"config",
"device",
"set",
name,
"root",
&format!("size={}MiB", (disk / MIB).max(6)),
])?;
Ok::<(), String>(())
})();
match applied {
Ok(()) => return true,
Err(_) => {
let _ = self.destroy(name);
return false;
}
}
}
false
}
fn spawn_topup(&self) {
if self.inline_topup {
self.topup();
} else {
let me = self.clone();
let _ = std::thread::Builder::new()
.name("sandbox-warm-pool".into())
.spawn(move || me.topup());
}
}
pub fn topup(&self) {
static BUSY: Mutex<()> = Mutex::new(());
let Ok(_one_at_a_time) = BUSY.try_lock() else {
return;
};
let want = provision_version(&self.base_image);
let Ok(all) = self.list() else {
return;
};
let mut have = 0usize;
for w in all.iter().filter(|i| i.warm.is_some()) {
if w.warm.as_deref() == Some(want.as_str()) && have < self.warm {
have += 1;
} else {
let _ = self.destroy(&w.name);
}
}
if self.warm == 0 || have >= self.warm || self.golden_version().as_deref() != Some(&want) {
return;
}
let disk = self.size.as_ref().map(|n| n.disk_bytes).unwrap_or(0);
for _ in have..self.warm {
let name = format!("{WARM_PREFIX}{}", warm_suffix());
let mut init = s(&["incus", "init", GOLDEN_ALIAS, &name]);
for (k, v) in [
("security.privileged", "false".to_string()),
("security.nesting", "true".to_string()),
(CFG_WARM, want.clone()),
] {
init.push("-c".into());
init.push(format!("{k}={v}"));
}
if disk > 0 {
init.push("-d".into());
init.push(format!("root,size={}MiB", (disk / MIB).max(6)));
}
if self.cli.run(&init).is_err() {
let _ = self.destroy(&name);
return;
}
}
}
pub fn cached_images(&self) -> Vec<CachedImage> {
self.run(&["image", "list", "-f", "json"])
.map(|o| parse_cached_images(&o))
.unwrap_or_default()
}
pub fn cached_image(&self, alias: &str, ttl: Duration, now_unix: u64) -> Option<CachedImage> {
let want = provision_version(&self.base_image);
let hit = self
.cached_images()
.into_iter()
.find(|i| i.alias == alias)?;
let fresh = hit.pinned || now_unix.saturating_sub(hit.cached_at) <= ttl.as_secs();
if hit.golden == want && fresh {
Some(hit)
} else {
let _ = self.run(&["image", "delete", alias]);
None
}
}
pub fn publish_derived(
&self,
from: &str,
alias: &str,
digest: &str,
tag: &str,
pinned: bool,
now_unix: u64,
) -> Result<(), String> {
let _ = self.run(&["image", "delete", alias]);
self.run(&[
"publish",
from,
"--alias",
alias,
&format!("zakuro.golden={}", provision_version(&self.base_image)),
&format!("zakuro.digest={digest}"),
&format!("zakuro.tag={tag}"),
&format!("zakuro.pinned={}", u8::from(pinned)),
&format!("zakuro.cached_at={now_unix}"),
])
.map_err(|e| format!("publishing {alias} failed: {e}"))?;
self.evict(alias);
Ok(())
}
pub fn evict(&self, keep: &str) -> Vec<String> {
let mut gone = vec![];
for alias in lru_victims(&self.cached_images(), self.cache, keep) {
if self.run(&["image", "delete", &alias]).is_ok() {
gone.push(alias);
}
}
gone
}
pub fn node_disk(&self) -> Result<u64, String> {
self.size.clone().map(|n| n.disk_bytes)
}
pub fn exec_argv(name: &str, docker_args: &[String]) -> Vec<String> {
let mut a = s(&["incus", "exec", name, "--", "nerdctl"]);
let rest = match docker_args.first().map(String::as_str) {
Some("docker") => &docker_args[1..],
_ => docker_args,
};
a.extend(rest.iter().cloned());
a
}
pub fn nerdctl(&self, name: &str, docker_args: &[String]) -> Result<String, String> {
self.cli.run(&Self::exec_argv(name, docker_args))
}
pub fn push_docker_config(
&self,
name: &str,
host_file: &std::path::Path,
) -> Result<(), String> {
self.run(&[
"file",
"push",
"--create-dirs",
"--mode",
"0600",
&host_file.display().to_string(),
&format!("{name}/root/.docker/config.json"),
])
.map(|_| ())
}
pub fn run_container(&self, name: &str, spec: &StartSpec) -> Result<String, String> {
let mut run = vec!["docker".to_string()];
run.extend(docker::run_argv(spec));
self.nerdctl(name, &run)
}
pub fn destroy(&self, name: &str) -> Result<(), String> {
live_names().lock().unwrap().remove(name);
match self.run(&["delete", "-f", name]) {
Ok(_) => Ok(()),
Err(e)
if e.contains("not found")
|| e.contains("Not Found")
|| e.contains("does not exist") =>
{
Ok(())
}
Err(e) => Err(e),
}
}
pub fn list(&self) -> Result<Vec<JobInstance>, String> {
let out = self.run(&["list", JOB_PREFIX, "-f", "json"])?;
Ok(parse_job_instances(&out))
}
pub fn allocations(&self) -> Result<Vec<Allocation>, String> {
Ok(self
.list()?
.iter()
.filter_map(|i| Allocation::from_label(&i.name, i.budget.as_deref()?))
.collect())
}
pub fn taken_gpus(&self) -> Result<HashSet<String>, String> {
Ok(self.list()?.into_iter().flat_map(|i| i.gpus).collect())
}
pub fn address(&self, name: &str) -> Option<String> {
parse_ipv4(&self.run(&["list", name, "-f", "csv", "-c", "4"]).ok()?)
}
pub fn reap(&self, alive: &dyn Fn(u32) -> bool, live: &HashSet<String>) -> Vec<String> {
let Ok(all) = self.list() else {
return vec![];
};
let mut gone = Vec::new();
for i in all.into_iter().filter(|i| i.warm.is_none()) {
let orphan = match i.owner_pid {
None => true,
Some(p) if p == self.pid => !live.contains(&i.name),
Some(p) => !alive(p),
};
if orphan && self.destroy(&i.name).is_ok() {
gone.push(i.name);
}
}
gone
}
pub fn reap_leaked(&self) -> Vec<String> {
let live = live_names().lock().unwrap().clone();
let gone = self.reap(&pid_alive, &live);
self.topup();
gone
}
}
pub const JOB_ID_PREFIX: &str = "t2:";
pub fn job_id(instance: &str, container: &str) -> String {
format!("{JOB_ID_PREFIX}{instance}/{container}")
}
pub fn parse_job_id(id: &str) -> Option<(&str, &str)> {
let (inst, name) = id.strip_prefix(JOB_ID_PREFIX)?.split_once('/')?;
(!inst.is_empty() && !name.is_empty()).then_some((inst, name))
}
#[derive(Debug, Clone)]
pub struct JobRun {
pub name: String,
pub image: String,
pub labels: Vec<(String, String)>,
pub mounts: Vec<(String, String)>,
pub env: Vec<(String, String)>,
pub cmd: Vec<String>,
pub limits: Limits,
pub disk_bytes: u64,
pub hardened: bool,
pub pull_config: Option<std::path::PathBuf>,
}
pub fn real_nvidia_smi() -> Result<String, String> {
let out = std::process::Command::new("nvidia-smi")
.args(["--query-gpu=index,uuid,pci.bus_id", "--format=csv,noheader"])
.output()
.map_err(|e| format!("nvidia-smi not available: {e}"))?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
pub fn start_job(
eph: &Ephemeral,
run: &JobRun,
nvidia_smi: &dyn Fn() -> Result<String, String>,
) -> Result<String, String> {
let picked = if run.limits.gpus.flag_value().is_some() {
let all = parse_nvidia_smi(&nvidia_smi()?);
pick_gpus(&run.limits.gpus, &all, &eph.taken_gpus()?)
.map_err(|e| format!("{}: {e}", run.name))?
} else {
vec![]
};
let mut inner_mounts = Vec::new();
let mut inst_mounts = Vec::new();
for (host, container) in &run.mounts {
let at = format!(
"/zakuro/mnt/{}",
&crate::dataset::sha256_hex(host.as_bytes())[..16]
);
inst_mounts.push((host.clone(), at.clone()));
inner_mounts.push((at, container.clone()));
}
let instance = eph.launch(&InstanceSpec::new(
&run.name,
run.limits.clone(),
run.disk_bytes,
picked.iter().map(|g| g.pci.clone()).collect(),
inst_mounts,
))?;
let started = (|| {
if let Some(dir) = &run.pull_config {
let f = dir.join("config.json");
if f.exists() {
eph.push_docker_config(&instance, &f)?;
}
}
eph.nerdctl(&instance, &s(&["pull", "--quiet", &run.image]))
.map_err(|e| format!("pull {} failed: {e}", run.image))?;
eph.run_container(
&instance,
&StartSpec {
name: run.name.clone(),
image: run.image.clone(),
network: None,
create_network: false,
labels: run.labels.clone(),
mounts: inner_mounts,
publish: vec![],
env: run.env.clone(),
cmd: run.cmd.clone(),
limits: inner_limits(&run.limits, !picked.is_empty()),
remove_on_exit: false,
hardened: run.hardened,
},
)
})();
match started {
Ok(_) => Ok(job_id(&instance, &run.name)),
Err(e) => {
let _ = eph.destroy(&instance);
Err(e)
}
}
}
pub const REAP_EVERY: Duration = Duration::from_secs(120);
pub fn start_reaper_once(eph: &Ephemeral) {
static STARTED: std::sync::Once = std::sync::Once::new();
let eph = eph.clone();
let first = eph.clone();
STARTED.call_once(move || {
for n in first.reap_leaked() {
eprintln!(" [SANDBOX] removed leaked job instance {n}");
}
let _ = std::thread::Builder::new()
.name("sandbox-reaper".into())
.spawn(move || loop {
std::thread::sleep(REAP_EVERY);
for n in eph.reap_leaked() {
eprintln!(" [SANDBOX] removed leaked job instance {n}");
}
});
});
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sandbox::lxc::DEFAULT_IMAGE;
#[derive(Default)]
struct Rec {
calls: Mutex<Vec<String>>,
answer: Mutex<Vec<(&'static str, Result<String, String>)>>,
}
impl Rec {
fn on(&self, needle: &'static str, a: Result<&str, &str>) {
self.answer
.lock()
.unwrap()
.push((needle, a.map(String::from).map_err(String::from)));
}
fn calls(&self) -> Vec<String> {
self.calls.lock().unwrap().clone()
}
}
impl IncusCli for Rec {
fn run(&self, argv: &[String]) -> Result<String, String> {
let j = argv.join(" ");
self.calls.lock().unwrap().push(j.clone());
for (needle, a) in self.answer.lock().unwrap().iter() {
if j.contains(needle) {
return a.clone();
}
}
Ok(String::new())
}
}
fn eph(rec: &Arc<Rec>) -> Ephemeral {
Ephemeral::new(
rec.clone(),
DEFAULT_IMAGE,
Ok(SandboxSize {
limits: Limits::new(8.0, 16.0, Gpus::None),
disk_bytes: 20 << 30,
}),
)
.fast()
}
fn spec(gpus: Vec<String>) -> InstanceSpec {
spec_for("zk-space-abc", gpus)
}
fn spec_for(job: &str, gpus: Vec<String>) -> InstanceSpec {
InstanceSpec::new(
job,
Limits::new(1.5, 2.0, Gpus::None),
10 << 30,
gpus,
vec![(
"/home/u/.zakuro/spaces/abc.sock".into(),
"/zakuro/sock".into(),
)],
)
}
fn golden_ok(rec: &Rec) {
rec.on(
"image get-property zakuro-sandbox-base",
Ok(Box::leak(provision_version(DEFAULT_IMAGE).into_boxed_str())),
);
}
#[test]
fn a_job_instance_is_ephemeral_limited_and_owned() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let name = eph(&rec).launch(&spec(vec![])).unwrap();
assert_eq!(name, instance_name("zk-space-abc"));
assert!(name.starts_with("zk-job-") && name.len() == 15, "{name}");
let calls = rec.calls();
let init = calls.iter().find(|c| c.starts_with("incus init")).unwrap();
assert_eq!(
init,
&format!(
"incus init zakuro-sandbox-base {name} --ephemeral \
-c limits.cpu=2 -c limits.cpu.allowance=150ms/100ms -c limits.memory=2048MiB \
-c limits.memory.enforce=hard -c limits.memory.swap=false \
-c security.privileged=false -c security.nesting=true -c nvidia.runtime=false \
-c user.zakuro.budget=1500m,2147483648 -c user.zakuro.owner=4242 \
-c user.zakuro.gpus= -c user.zakuro.lease=zk-space-abc -d root,size=10240MiB"
)
);
let mnt = calls
.iter()
.position(|c| c.contains("zakuro-mnt-0"))
.unwrap();
let start = calls
.iter()
.position(|c| c == &format!("incus start {name}"))
.unwrap();
assert!(mnt < start, "{calls:#?}");
assert!(calls[mnt]
.contains("source=/home/u/.zakuro/spaces/abc.sock path=/zakuro/sock shift=true"));
assert!(!calls.iter().any(|c| c.contains(" gpu ")), "no GPU unasked");
assert!(calls
.iter()
.any(|c| c.contains("systemctl is-active --quiet containerd")));
live_names().lock().unwrap().remove(&name);
}
#[test]
fn a_picked_gpu_is_attached_by_pci_address_and_enables_the_nvidia_runtime() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let name = eph(&rec)
.launch(&spec(vec!["0000:01:00.0".into()]))
.unwrap();
let calls = rec.calls();
assert!(calls.iter().any(|c| c
== &format!(
"incus config device add {name} zakuro-gpu-0 gpu gputype=physical pci=0000:01:00.0"
)));
let init = calls.iter().find(|c| c.starts_with("incus init")).unwrap();
assert!(init.contains("-c nvidia.runtime=true"), "{init}");
assert!(init.contains("-c user.zakuro.gpus=0000:01:00.0"), "{init}");
live_names().lock().unwrap().remove(&name);
}
#[test]
fn a_failed_start_deletes_the_instance_and_says_why() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("incus start", Err("no space left on device"));
let e = eph(&rec)
.launch(&spec_for("zk-space-fail", vec![]))
.unwrap_err();
assert!(
e.contains("incus start") && e.contains("no space left"),
"{e}"
);
let calls = rec.calls();
assert!(
calls.last().unwrap().starts_with("incus delete -f zk-job-"),
"{calls:#?}"
);
assert!(!live_names()
.lock()
.unwrap()
.contains(&instance_name("zk-space-fail")));
}
#[test]
fn an_instance_that_never_gets_containerd_is_deleted() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("systemctl is-active", Err("Failed to connect to bus"));
let e = eph(&rec).launch(&spec(vec![])).unwrap_err();
assert!(e.contains("did not become ready"), "{e}");
assert!(rec.calls().last().unwrap().starts_with("incus delete -f"));
}
#[test]
fn an_invalid_budget_refuses_every_instance() {
let rec = Arc::new(Rec::default());
let e = Ephemeral::new(
rec.clone(),
DEFAULT_IMAGE,
Err("ZAKURO_BUDGET_CPU=bad".into()),
)
.launch(&spec(vec![]))
.unwrap_err();
assert_eq!(e, "ZAKURO_BUDGET_CPU=bad");
assert!(rec.calls().is_empty());
}
#[test]
fn the_golden_image_is_built_once_from_the_provisioning_and_versioned() {
let rec = Arc::new(Rec::default());
rec.on("image get-property", Err("Image not found"));
eph(&rec).ensure_golden().unwrap();
let calls = rec.calls();
let ver = provision_version(DEFAULT_IMAGE);
let pos = |needle: &str| {
calls
.iter()
.position(|c| c.contains(needle))
.unwrap_or_else(|| panic!("{needle} missing in {calls:#?}"))
};
assert!(pos("incus launch images:ubuntu/24.04 zk-golden-build") < pos("sh -c set -e;"));
assert!(calls[pos("sh -c set -e;")].contains("sha256sum -c"));
assert!(pos("sh -c set -e;") < pos("incus stop zk-golden-build"));
assert_eq!(
calls[pos("incus publish")],
format!(
"incus publish zk-golden-build --alias zakuro-sandbox-base zakuro.provision={ver}"
)
);
assert!(calls.last().unwrap() == "incus delete -f zk-golden-build");
}
#[test]
fn a_current_golden_image_is_reused_and_an_old_one_rebuilt() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
eph(&rec).ensure_golden().unwrap();
assert_eq!(rec.calls().len(), 1, "{:#?}", rec.calls());
let old = Arc::new(Rec::default());
old.on("image get-property", Ok("0123456789ab"));
eph(&old).ensure_golden().unwrap();
assert!(old
.calls()
.iter()
.any(|c| c == "incus image delete zakuro-sandbox-base"));
assert!(old.calls().iter().any(|c| c.starts_with("incus publish")));
assert_ne!(
provision_version("images:debian/12"),
provision_version(DEFAULT_IMAGE)
);
}
#[test]
fn a_failed_golden_build_leaves_no_scratch_instance() {
let rec = Arc::new(Rec::default());
rec.on("image get-property", Err("not found"));
rec.on("sha256sum", Err("curl: (6) Could not resolve host"));
let e = eph(&rec).ensure_golden().unwrap_err();
assert!(e.contains("provisioning containerd"), "{e}");
assert_eq!(
rec.calls().last().unwrap(),
"incus delete -f zk-golden-build"
);
assert!(!rec.calls().iter().any(|c| c.starts_with("incus publish")));
}
#[test]
fn the_inner_container_is_capped_below_the_boundary() {
let l = Limits::new(2.0, 4.0, Gpus::None);
let i = inner_limits(&l, false);
assert_eq!(i.cpu_millis, 2000);
assert_eq!(i.memory_bytes, 4 * 1024 * MIB - 256 * MIB);
let tiny = inner_limits(&Limits::new(1.0, 0.5, Gpus::None), true);
assert_eq!(tiny.memory_bytes, 384 * MIB);
assert_eq!(tiny.gpus, Gpus::All);
}
#[test]
fn allocations_come_from_the_instances_own_budget_key() {
let rec = Arc::new(Rec::default());
rec.on(
"list zk-job- -f json",
Ok(r#"[{"name":"zk-job-aaaaaaaa","status":"Running","config":{"user.zakuro.budget":"1500m,2147483648","user.zakuro.owner":"7","user.zakuro.gpus":"0000:01:00.0,0000:02:00.0"}},
{"name":"zk-job-bbbbbbbb","status":"Running","config":{}},
{"name":"other","status":"Running","config":{"user.zakuro.budget":"1m,1"}}]"#),
);
let e = eph(&rec);
assert_eq!(
e.allocations().unwrap(),
vec![Allocation {
name: "zk-job-aaaaaaaa".into(),
cpu_millis: 1500,
memory_bytes: 2 << 30
}]
);
assert_eq!(
e.taken_gpus().unwrap(),
HashSet::from(["0000:01:00.0".to_string(), "0000:02:00.0".to_string()])
);
}
#[test]
fn the_reaper_removes_dead_owners_and_unowned_but_not_live_jobs() {
let rec = Arc::new(Rec::default());
rec.on(
"list zk-job- -f json",
Ok(r#"[{"name":"zk-job-dead0000","status":"Running","config":{"user.zakuro.owner":"111"}},
{"name":"zk-job-live1111","status":"Running","config":{"user.zakuro.owner":"222"}},
{"name":"zk-job-mine2222","status":"Running","config":{"user.zakuro.owner":"4242"}},
{"name":"zk-job-mine3333","status":"Running","config":{"user.zakuro.owner":"4242"}},
{"name":"zk-job-noown444","status":"Stopped","config":{}}]"#),
);
let e = eph(&rec);
let live = HashSet::from(["zk-job-mine2222".to_string()]);
let gone = e.reap(&|p| p == 222, &live);
assert_eq!(
gone,
vec!["zk-job-dead0000", "zk-job-mine3333", "zk-job-noown444"]
);
let calls = rec.calls();
assert!(!calls
.iter()
.any(|c| c.contains("delete -f zk-job-live1111")));
assert!(!calls
.iter()
.any(|c| c.contains("delete -f zk-job-mine2222")));
}
#[test]
fn destroying_a_gone_instance_is_not_an_error() {
let rec = Arc::new(Rec::default());
rec.on("incus delete", Err("Instance not found"));
assert!(eph(&rec).destroy("zk-job-x").is_ok());
let rec = Arc::new(Rec::default());
rec.on("incus delete", Err("permission denied"));
assert!(eph(&rec).destroy("zk-job-x").is_err());
}
fn gpus() -> Vec<GpuInfo> {
parse_nvidia_smi("0, GPU-aaaa-1111, 00000000:01:00.0\n1, GPU-bbbb-2222, 00000000:41:00.0\n")
}
#[test]
fn nvidia_smi_output_gives_index_uuid_and_a_sysfs_pci_address() {
let g = gpus();
assert_eq!(g.len(), 2);
assert_eq!(g[1].pci, "0000:41:00.0");
assert_eq!(g[0].uuid, "GPU-aaaa-1111");
assert!(parse_nvidia_smi("garbage\n,,\n").is_empty());
}
#[test]
fn a_uuid_pin_resolves_to_its_card_and_one_job_gets_one_gpu() {
let g = gpus();
let none = HashSet::new();
let one = pick_gpus(&Gpus::Spec("device=GPU-bbbb-2222".into()), &g, &none).unwrap();
assert_eq!(one.len(), 1);
assert_eq!(one[0].pci, "0000:41:00.0");
let by_index = pick_gpus(&Gpus::Spec("device=0".into()), &g, &none).unwrap();
assert_eq!(by_index[0].uuid, "GPU-aaaa-1111");
let taken = HashSet::from(["0000:01:00.0".to_string()]);
let c = pick_gpus(&Gpus::Count(1), &g, &taken).unwrap();
assert_eq!(c[0].pci, "0000:41:00.0");
assert!(pick_gpus(&Gpus::Count(2), &g, &taken)
.unwrap_err()
.contains("one job per GPU"));
assert!(pick_gpus(&Gpus::Spec("device=0".into()), &g, &taken)
.unwrap_err()
.contains("held by another job"));
assert!(pick_gpus(&Gpus::Spec("device=GPU-nope".into()), &g, &none).is_err());
assert!(pick_gpus(&Gpus::None, &g, &none).unwrap().is_empty());
assert_eq!(pick_gpus(&Gpus::All, &g, &none).unwrap().len(), 2);
assert!(pick_gpus(&Gpus::All, &[], &none).is_err());
}
#[test]
fn the_registry_login_is_a_docker_config_not_an_argv() {
let j = docker_config_json("docker.io", "zak", "s3cret");
let v: serde_json::Value = serde_json::from_str(&j).unwrap();
let auth = v["auths"]["https://index.docker.io/v1/"]["auth"]
.as_str()
.unwrap();
use base64::Engine as _;
assert_eq!(
base64::engine::general_purpose::STANDARD
.decode(auth)
.unwrap(),
b"zak:s3cret"
);
let r = docker_config_json("registry.example.com", "u", "p");
assert!(r.contains("registry.example.com"));
let rec = Arc::new(Rec::default());
eph(&rec)
.push_docker_config("zk-job-1", std::path::Path::new("/tmp/cfg/config.json"))
.unwrap();
let c = rec.calls().pop().unwrap();
assert_eq!(
c,
"incus file push --create-dirs --mode 0600 /tmp/cfg/config.json zk-job-1/root/.docker/config.json"
);
}
#[test]
fn instance_addresses_and_names() {
assert_eq!(
parse_ipv4("10.20.30.4 (eth0)\nfd42::1 (eth0)"),
Some("10.20.30.4".into())
);
assert_eq!(
parse_ipv4("\"10.0.0.9 (eth0)\n172.17.0.1 (docker0)\""),
Some("10.0.0.9".into())
);
assert_eq!(parse_ipv4(""), None);
assert_eq!(instance_name("a"), instance_name("a"));
assert_ne!(instance_name("a"), instance_name("b"));
}
#[test]
fn the_container_runs_under_nerdctl_in_its_instance_with_the_same_flags() {
let rec = Arc::new(Rec::default());
let e = eph(&rec);
let spec = StartSpec {
name: "zk-space-abc".into(),
image: "img@sha256:ab".into(),
network: None,
create_network: false,
labels: vec![],
mounts: vec![("/zakuro/sock".into(), "/run/zakuro/space.sock".into())],
publish: vec![],
env: vec![],
cmd: vec![],
limits: inner_limits(&Limits::new(1.0, 2.0, Gpus::None), false),
remove_on_exit: false,
hardened: true,
};
e.run_container("zk-job-1", &spec).unwrap();
let c = rec.calls().pop().unwrap();
assert!(
c.starts_with("incus exec zk-job-1 -- nerdctl run -d --name zk-space-abc"),
"{c}"
);
assert!(c.contains("--cpus 1 --memory 1792m"), "{c}");
assert!(
c.contains("--pids-limit 4096 --security-opt no-new-privileges"),
"{c}"
);
}
#[test]
fn a_job_start_pulls_inside_runs_capped_and_returns_a_routable_id() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let run = JobRun {
name: "zk-space-run1".into(),
image: "reg.example.com/space@sha256:ab".into(),
labels: vec![("zakuro.space".into(), "s1".into())],
mounts: vec![("/h/s1.sock".into(), "/run/zakuro/space.sock".into())],
env: vec![("ZAKURO_SPACE_ID".into(), "s1".into())],
cmd: vec![],
limits: Limits::new(1.0, 2.0, Gpus::None),
disk_bytes: 5 << 30,
hardened: true,
pull_config: Some(std::path::PathBuf::from("/nonexistent-cfg")),
};
let id = start_job(&eph(&rec), &run, &|| Err("no gpus asked".into())).unwrap();
let (inst, name) = parse_job_id(&id).unwrap();
assert_eq!(
(inst, name),
(instance_name("zk-space-run1").as_str(), "zk-space-run1")
);
let calls = rec.calls();
let pull = calls
.iter()
.position(|c| c.contains("nerdctl pull --quiet reg.example.com/space@sha256:ab"))
.unwrap();
let run_at = calls
.iter()
.position(|c| c.contains("nerdctl run -d --name zk-space-run1"))
.unwrap();
assert!(pull < run_at, "{calls:#?}");
let r = &calls[run_at];
assert!(r.contains("--cpus 1 --memory 1792m"), "{r}");
assert!(r.contains("--pids-limit 4096"), "{r}");
assert!(!r.contains("--gpus"), "{r}");
assert!(r.contains("-v /zakuro/mnt/"), "{r}");
assert!(r.contains(":/run/zakuro/space.sock"), "{r}");
live_names().lock().unwrap().remove(inst);
}
#[test]
fn a_pull_that_fails_inside_deletes_the_instance() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("nerdctl pull", Err("manifest unknown"));
let run = JobRun {
name: "zk-space-pullfail".into(),
image: "img".into(),
labels: vec![],
mounts: vec![],
env: vec![],
cmd: vec![],
limits: Limits::new(1.0, 1.0, Gpus::None),
disk_bytes: 5 << 30,
hardened: true,
pull_config: None,
};
let e = start_job(&eph(&rec), &run, &|| Err(String::new())).unwrap_err();
assert!(e.contains("pull img failed: manifest unknown"), "{e}");
assert!(rec
.calls()
.last()
.unwrap()
.starts_with("incus delete -f zk-job-"));
}
#[test]
fn a_gpu_job_gets_a_free_card_by_pci_and_a_refusal_when_none_is_free() {
let smi =
|| Ok("0, GPU-aaaa, 00000000:01:00.0\n1, GPU-bbbb, 00000000:41:00.0\n".to_string());
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on(
"list zk-job- -f json",
Ok(r#"[{"name":"zk-job-other000","status":"Running","config":{"user.zakuro.gpus":"0000:01:00.0","user.zakuro.owner":"1"}}]"#),
);
let mut run = JobRun {
name: "zk-space-gpu1".into(),
image: "img".into(),
labels: vec![],
mounts: vec![],
env: vec![],
cmd: vec![],
limits: Limits::new(1.0, 2.0, Gpus::Spec("device=GPU-bbbb".into())),
disk_bytes: 5 << 30,
hardened: true,
pull_config: None,
};
let id = start_job(&eph(&rec), &run, &smi).unwrap();
let calls = rec.calls();
assert!(
calls
.iter()
.any(|c| c.contains("gputype=physical pci=0000:41:00.0")),
"{calls:#?}"
);
assert!(calls
.iter()
.any(|c| c.contains("nerdctl run") && c.contains("--gpus all")));
live_names()
.lock()
.unwrap()
.remove(parse_job_id(&id).unwrap().0);
run.name = "zk-space-gpu2".into();
run.limits = Limits::new(1.0, 2.0, Gpus::Spec("device=0".into()));
let before = rec.calls().len();
let e = start_job(&eph(&rec), &run, &smi).unwrap_err();
assert!(e.contains("held by another job"), "{e}");
assert!(!rec.calls()[before..]
.iter()
.any(|c| c.starts_with("incus init")));
}
#[test]
fn job_ids_round_trip_and_plain_ids_are_not_jobs() {
let id = job_id("zk-job-abcd1234", "zk-space-x");
assert_eq!(parse_job_id(&id), Some(("zk-job-abcd1234", "zk-space-x")));
for plain in ["abc123", "t2:", "t2:/x", "t2:i/", ""] {
assert_eq!(parse_job_id(plain), None, "{plain:?}");
}
}
#[test]
fn the_egress_script_fences_the_instance_host_the_lan_and_the_mesh() {
let s = job_egress_script();
assert!(s.contains("CNI-ADMIN"), "{s}");
assert!(s.contains("iptables -F ZAKURO-EGRESS"), "idempotent refill");
for range in [
"10.13.13.0/24",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"169.254.0.0/16",
"100.64.0.0/10",
] {
assert!(s.contains(range), "{range} missing");
}
let dns = s.find("--dport 53 -j RETURN").unwrap();
let first_reject = s.find("-j REJECT").unwrap();
assert!(dns < first_reject);
assert!(s.find("-d $GW -j REJECT").unwrap() < s.find("-d $C -j RETURN").unwrap());
assert!(s.contains("--dst-type LOCAL -j REJECT"));
assert!(s.contains("iptables -I INPUT 1"));
assert_ne!(provision_version("a"), provision_version("b"));
}
#[test]
fn every_job_instance_gets_the_egress_rules_after_boot_and_before_any_container() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let name = eph(&rec).launch(&spec(vec![])).unwrap();
let calls = rec.calls();
let ready = calls
.iter()
.position(|c| c.contains("systemctl is-active --quiet containerd"))
.unwrap();
let egress = calls
.iter()
.position(|c| c.starts_with(&format!("incus exec {name} -- sh -c set -e; C=")))
.unwrap_or_else(|| panic!("{calls:#?}"));
assert!(ready < egress, "rules go in once the instance is up");
assert!(calls[egress].ends_with("zakuro-egress"));
live_names().lock().unwrap().remove(&name);
let bad = Arc::new(Rec::default());
golden_ok(&bad);
bad.on("ZAKURO-EGRESS", Err("iptables: Operation not permitted"));
let e = eph(&bad)
.launch(&spec_for("zk-space-nofence", vec![]))
.unwrap_err();
assert!(
e.contains("egress rules") && e.contains("not started"),
"{e}"
);
assert!(bad.calls().last().unwrap().starts_with("incus delete -f"));
}
fn warm_json(rows: &[(&str, &str, &str)]) -> &'static str {
let items: Vec<String> = rows
.iter()
.map(|(n, st, ver)| {
format!(
r#"{{"name":"{n}","status":"{st}","config":{{"user.zakuro.warm":"{ver}"}}}}"#
)
})
.collect();
Box::leak(format!("[{}]", items.join(",")).into_boxed_str())
}
#[test]
fn the_warm_pool_size_is_opt_in_and_bounded() {
let g = |v: &'static str| {
move |k: &str| (k == "ZAKURO_SANDBOX_WARM_POOL").then(|| v.to_string())
};
assert_eq!(warm_pool_size(&|_| None), 0, "off by default");
assert_eq!(warm_pool_size(&g("2")), 2);
assert_eq!(warm_pool_size(&g("junk")), 0);
assert_eq!(warm_pool_size(&g("500")), 8, "a pool is not a fleet");
}
#[test]
fn a_job_takes_a_warm_instance_instead_of_creating_one_and_the_pool_is_refilled() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let ver = provision_version(DEFAULT_IMAGE);
rec.on(
"list zk-job-",
Ok(warm_json(&[
("zk-job-wold0000", "Stopped", "0123456789ab"),
(
"zk-job-wnew1111",
"Stopped",
Box::leak(ver.clone().into_boxed_str()),
),
])),
);
let name = eph(&rec).with_warm_pool(1).launch(&spec(vec![])).unwrap();
let calls = rec.calls();
assert!(
!calls
.iter()
.any(|c| c.starts_with("incus init") && c.contains(&name)),
"no cold create when a warm one is there: {calls:#?}"
);
let pos = |n: &str| {
calls
.iter()
.position(|c| c.contains(n))
.unwrap_or_else(|| panic!("{n} missing: {calls:#?}"))
};
assert_eq!(
calls[pos("incus rename")],
format!("incus rename zk-job-wnew1111 {name}")
);
assert!(pos("config set") < pos("config unset"));
assert!(calls[pos("config set")].contains("user.zakuro.owner=4242"));
assert!(calls[pos("config unset")].ends_with(&format!("{name} user.zakuro.warm")));
assert!(calls
.iter()
.any(|c| c.contains("config set") && c.contains("limits.memory=2048MiB")));
assert!(calls
.iter()
.any(|c| c.contains("config set") && c.contains("user.zakuro.lease=zk-space-abc")));
assert!(calls
.iter()
.any(|c| c == &format!("incus config device set {name} root size=10240MiB")));
assert!(pos("incus rename") < pos(&format!("incus start {name}")));
assert!(calls.iter().any(|c| c == "incus delete -f zk-job-wold0000"));
assert!(!calls.iter().any(|c| c.starts_with("incus publish")));
live_names().lock().unwrap().remove(&name);
}
#[test]
fn without_a_warm_instance_the_job_launches_cold_and_a_failed_take_falls_back() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("list zk-job-", Ok("[]"));
let name = eph(&rec).with_warm_pool(2).launch(&spec(vec![])).unwrap();
let calls = rec.calls();
assert!(calls.iter().any(|c| c.starts_with(&format!(
"incus init zakuro-sandbox-base {name} --ephemeral"
))));
let ver = provision_version(DEFAULT_IMAGE);
let warm: Vec<&String> = calls
.iter()
.filter(|c| c.starts_with("incus init zakuro-sandbox-base zk-job-w"))
.collect();
assert_eq!(warm.len(), 2, "{calls:#?}");
assert!(warm[0].contains(&format!("-c user.zakuro.warm={ver}")));
assert!(warm[0].contains("-c security.privileged=false -c security.nesting=true"));
assert!(
!warm[0].contains("--ephemeral"),
"a stopped ephemeral instance would be deleted"
);
live_names().lock().unwrap().remove(&name);
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let ver2 = Box::leak(ver.into_boxed_str());
rec.on(
"list zk-job-",
Ok(warm_json(&[("zk-job-wnew2222", "Stopped", ver2)])),
);
rec.on("device set", Err("cannot shrink"));
let name = eph(&rec)
.with_warm_pool(1)
.launch(&spec_for("zk-space-two", vec![]))
.unwrap();
let calls = rec.calls();
assert!(
calls.iter().any(|c| c.starts_with(&format!(
"incus init zakuro-sandbox-base {name} --ephemeral"
))),
"{calls:#?}"
);
live_names().lock().unwrap().remove(&name);
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let name = eph(&rec)
.launch(&spec_for("zk-space-three", vec![]))
.unwrap();
assert!(!rec
.calls()
.iter()
.any(|c| c.contains("rename") || c.contains("zk-job-w")));
live_names().lock().unwrap().remove(&name);
}
#[test]
fn the_pool_is_kept_at_size_and_the_reaper_never_takes_a_warm_instance_for_a_leak() {
let ver = provision_version(DEFAULT_IMAGE);
let ver: &'static str = Box::leak(ver.into_boxed_str());
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on(
"list zk-job-",
Ok(warm_json(&[
("zk-job-wa000000", "Stopped", ver),
("zk-job-wb000000", "Stopped", ver),
("zk-job-wc000000", "Stopped", "0123456789ab"),
])),
);
let e = eph(&rec).with_warm_pool(1);
assert!(e.reap(&|_| false, &HashSet::new()).is_empty());
assert!(!rec.calls().iter().any(|c| c.contains("delete")));
e.topup();
let calls = rec.calls();
assert!(
calls.contains(&"incus delete -f zk-job-wb000000".to_string()),
"{calls:#?}"
);
assert!(calls.contains(&"incus delete -f zk-job-wc000000".to_string()));
assert!(!calls.contains(&"incus delete -f zk-job-wa000000".to_string()));
assert!(
!calls.iter().any(|c| c.starts_with("incus init")),
"already at size"
);
let off = Arc::new(Rec::default());
golden_ok(&off);
off.on(
"list zk-job-",
Ok(warm_json(&[("zk-job-wa000000", "Stopped", ver)])),
);
eph(&off).topup();
assert!(off
.calls()
.contains(&"incus delete -f zk-job-wa000000".to_string()));
}
fn image_row(alias: &str, size: u64, used: &str) -> CachedImage {
CachedImage {
alias: alias.into(),
size,
last_used: used.into(),
golden: "g".into(),
digest: String::new(),
tag: String::new(),
pinned: true,
cached_at: 0,
}
}
#[test]
fn the_derived_image_cache_evicts_least_recently_used_by_count_and_by_bytes() {
let imgs = vec![
image_row("zk-img-a", 10 << 30, "2026-11-01T00:00:00Z"),
image_row("zk-img-b", 10 << 30, "2026-11-03T00:00:00Z"),
image_row("zk-img-c", 10 << 30, "2026-11-02T00:00:00Z"),
];
let caps = |n, gb: u64| CacheCaps {
max_images: n,
max_bytes: gb << 30,
};
assert!(
lru_victims(&imgs, caps(3, 30), "").is_empty(),
"at the caps is fine"
);
assert_eq!(lru_victims(&imgs, caps(2, 100), ""), vec!["zk-img-a"]);
assert_eq!(
lru_victims(&imgs, caps(8, 15), ""),
vec!["zk-img-a", "zk-img-c"]
);
assert_eq!(
lru_victims(&imgs, caps(2, 100), "zk-img-a"),
vec!["zk-img-c"]
);
assert_eq!(
lru_victims(&imgs, caps(0, 0), "zk-img-b"),
vec!["zk-img-a", "zk-img-c"]
);
}
#[test]
fn derived_images_are_listed_from_incus_json_and_published_then_evicted() {
let json = r#"[
{"size":100,"last_used_at":"0001-01-01T00:00:00Z","uploaded_at":"2026-11-05T01:00:00Z",
"aliases":[{"name":"zk-img-aaaaaaaaaaaa"}],
"properties":{"zakuro.golden":"g1","zakuro.digest":"sha256:d","zakuro.tag":"r:t","zakuro.pinned":"1","zakuro.cached_at":"77"}},
{"size":5,"aliases":[{"name":"zakuro-sandbox-base"}],"properties":{}}
]"#;
let got = parse_cached_images(json);
assert_eq!(got.len(), 1, "only zk-img-* images");
assert_eq!(got[0].alias, "zk-img-aaaaaaaaaaaa");
assert_eq!(
got[0].last_used, "2026-11-05T01:00:00Z",
"never launched: upload time"
);
assert!(got[0].pinned && got[0].cached_at == 77 && got[0].digest == "sha256:d");
assert!(parse_cached_images("not json").is_empty());
let rec = Arc::new(Rec::default());
rec.on(
"image list -f json",
Ok(Box::leak(
r#"[{"size":1,"last_used_at":"2026-11-01T00:00:00Z","aliases":[{"name":"zk-img-old"}],"properties":{}},
{"size":1,"last_used_at":"2026-11-09T00:00:00Z","aliases":[{"name":"zk-img-new"}],"properties":{}}]"#
.to_string()
.into_boxed_str(),
)),
);
let e = eph(&rec).with_cache_caps(CacheCaps {
max_images: 1,
max_bytes: 1 << 30,
});
e.publish_derived("b", "zk-img-new", "sha256:d", "r:t", true, 9)
.unwrap();
let calls = rec.calls();
let publish = calls
.iter()
.position(|c| c.starts_with("incus publish b --alias zk-img-new"))
.unwrap();
assert!(calls[publish].contains("zakuro.golden="));
assert!(
calls
.iter()
.position(|c| c == "incus image delete zk-img-old")
.unwrap()
> publish
);
assert!(!calls.iter().any(|c| c == "incus image delete zk-img-new"
&& calls.iter().position(|x| x == c).unwrap() > publish));
}
#[test]
fn cache_caps_read_the_environment() {
let get = |k: &str| match k {
"ZAKURO_SANDBOX_IMG_CACHE_MAX" => Some("3".to_string()),
"ZAKURO_SANDBOX_IMG_CACHE_GB" => Some("1.5".to_string()),
_ => None,
};
assert_eq!(
CacheCaps::from_env(&get),
CacheCaps {
max_images: 3,
max_bytes: 3 << 29
}
);
assert_eq!(
CacheCaps::from_env(&|_| None),
CacheCaps {
max_images: DEFAULT_CACHE_IMAGES,
max_bytes: DEFAULT_CACHE_BYTES
}
);
}
#[test]
#[ignore]
fn real_incus_job_instance_end_to_end() {
if std::env::var("ZAKURO_TEST_REAL_INCUS").ok().as_deref() != Some("1") {
eprintln!("skipped: set ZAKURO_TEST_REAL_INCUS=1");
return;
}
let cli: Arc<dyn IncusCli> = Arc::new(lxc::IncusShell);
let image = std::env::var("ZAKURO_SANDBOX_IMAGE")
.unwrap_or_else(|_| lxc::default_image(lxc::cli_binary()).to_string());
let e = Ephemeral::new(
cli.clone(),
&image,
Ok(SandboxSize {
limits: Limits::new(2.0, 4.0, Gpus::None),
disk_bytes: 10 << 30,
}),
);
let t = std::time::Instant::now();
e.ensure_golden().expect("golden image");
println!("REAL golden image ready in {:?}", t.elapsed());
let job = format!("zk-real-{}", std::process::id());
let limits = Limits::new(1.0, 1.0, Gpus::None);
let name = e
.launch(&InstanceSpec::new(
&job,
limits.clone(),
5 << 30,
vec![],
vec![],
))
.expect("launch");
println!("REAL instance {name} running");
let inner = inner_limits(&limits, false);
let spec = StartSpec {
name: "real-job".into(),
image: "docker.io/library/busybox:latest".into(),
network: None,
create_network: false,
labels: vec![],
mounts: vec![],
publish: vec![],
env: vec![],
cmd: vec!["sleep".into(), "120".into()],
limits: inner.clone(),
remove_on_exit: false,
hardened: true,
};
e.nerdctl(&name, &s(&["pull", "--quiet", &spec.image]))
.expect("pull inside");
e.run_container(&name, &spec).expect("run container");
let cg = e
.nerdctl(
&name,
&s(&[
"exec",
"real-job",
"sh",
"-c",
"cat /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max",
]),
)
.expect("read cgroup");
let read = lxc::parse_cgroup_limits(&cg).expect("limits are set");
println!("REAL container limits read back from inside: {read}");
assert_eq!(read.cpu_millis, inner.cpu_millis);
assert_eq!(read.memory_bytes, inner.memory_bytes);
let inst_mem = e.run(&["config", "get", &name, "limits.memory"]).unwrap();
println!("REAL instance limits.memory={inst_mem}");
assert_eq!(inst_mem.trim(), "1024MiB");
let out = e
.nerdctl(
&name,
&s(&[
"run",
"--rm",
"docker.io/library/busybox:latest",
"echo",
"hello-from-t2",
]),
)
.expect("echo");
println!("REAL echo: {out}");
assert!(out.contains("hello-from-t2"));
assert!(e.allocations().unwrap().iter().any(|a| a.name == name));
use std::sync::atomic::{AtomicUsize, Ordering};
let listener = std::net::TcpListener::bind("0.0.0.0:0").expect("listener");
let port = listener.local_addr().unwrap().port();
let hits = Arc::new(AtomicUsize::new(0));
let h2 = hits.clone();
std::thread::spawn(move || {
for c in listener.incoming().flatten() {
h2.fetch_add(1, Ordering::SeqCst);
drop(c);
}
});
let gw = e
.run(&[
"exec",
&name,
"--",
"sh",
"-c",
"ip -4 route show default | awk '{print $3; exit}'",
])
.expect("gateway");
let bb = "docker.io/library/busybox:latest";
let from_instance = e.run(&[
"exec",
&name,
"--",
"bash",
"-c",
&format!("timeout 5 bash -c 'exec 3<>/dev/tcp/{gw}/{port}'"),
]);
println!(
"REAL egress: instance -> {gw}:{port} {}",
if from_instance.is_ok() {
"reachable"
} else {
"BLOCKED"
}
);
assert!(
from_instance.is_ok(),
"the instance reaches its host: {from_instance:?}"
);
let before = hits.load(Ordering::SeqCst);
assert!(before >= 1, "the listener saw the instance");
let nc = |target: &str, p: &str| {
e.nerdctl(
&name,
&s(&[
"run",
"--rm",
bb,
"sh",
"-c",
&format!("nc -w 4 {target} {p} </dev/null"),
]),
)
};
let to_gw = nc(&gw, &port.to_string());
println!(
"REAL egress: job container -> {gw}:{port} {}",
if to_gw.is_ok() {
"REACHABLE"
} else {
"blocked"
}
);
assert!(
to_gw.is_err(),
"a job container must not reach the instance's host"
);
assert_eq!(
hits.load(Ordering::SeqCst),
before,
"the listener saw no job traffic"
);
for lan in ["10.13.13.1", "192.168.1.1", "169.254.169.254"] {
let r = nc(lan, "80");
println!(
"REAL egress: job container -> {lan}:80 {}",
if r.is_ok() { "REACHABLE" } else { "blocked" }
);
assert!(r.is_err(), "{lan} must be unreachable");
}
e.run(&[
"exec",
&name,
"--",
"sh",
"-c",
"iptables -F ZAKURO-EGRESS; iptables -F ZAKURO-IN",
])
.expect("flush");
let open = nc(&gw, &port.to_string());
println!(
"REAL egress control (rules flushed): job container -> {gw}:{port} {}",
if open.is_ok() { "reachable" } else { "BLOCKED" }
);
assert!(open.is_ok(), "without the rules it connects: {open:?}");
assert!(hits.load(Ordering::SeqCst) > before);
e.apply_egress(&name).expect("re-apply");
let again = nc(&gw, &port.to_string());
println!(
"REAL egress re-applied: job container -> {gw}:{port} {}",
if again.is_ok() {
"REACHABLE"
} else {
"blocked"
}
);
assert!(again.is_err());
let net = nc("1.1.1.1", "443");
println!(
"REAL egress: job container -> 1.1.1.1:443 {}",
if net.is_ok() { "reachable" } else { "BLOCKED" }
);
assert!(net.is_ok(), "the internet works: {net:?}");
let dns = e.nerdctl(&name, &s(&["run", "--rm", bb, "nslookup", "example.com"]));
println!(
"REAL egress: job container DNS example.com {}",
if dns.is_ok() { "resolves" } else { "FAILS" }
);
assert!(dns.is_ok(), "DNS works: {dns:?}");
e.destroy(&name).expect("destroy");
assert!(!e.list().unwrap().iter().any(|i| i.name == name));
println!("REAL instance {name} gone after destroy");
let leaked = format!("{JOB_PREFIX}leak{:04}", std::process::id() % 10000);
e.run(&[
"init",
GOLDEN_ALIAS,
&leaked,
"--ephemeral",
"-c",
"user.zakuro.owner=2147483646",
"-c",
"user.zakuro.budget=1000m,1073741824",
])
.expect("create leaked");
assert!(e.list().unwrap().iter().any(|i| i.name == leaked));
let gone = e.reap_leaked();
println!("REAL reaper removed {gone:?}");
assert!(gone.contains(&leaked));
assert!(!e.list().unwrap().iter().any(|i| i.name == leaked));
}
}