use std::process::Command;
use std::sync::Arc;
use super::{
docker, Budget, Gpus, HostResources, Limits, Probe, Runtime, RuntimeKind, StartSpec, State,
};
pub const SANDBOX: &str = "zakuro-sandbox";
pub const DEFAULT_IMAGE: &str = "images:ubuntu/24.04";
pub const NERDCTL_VERSION: &str = "2.0.3";
const NERDCTL_SHA256: [(&str, &str); 2] = [
(
"amd64",
"95ff850688a73eace7453f19e74bf4cc8a1f3e458eeb97ef7a6b74de9825df16",
),
(
"arm64",
"f2c3f12c99e112cd82ba19ca9b875045c44b2f5a19cecc295ed8d61d415e8851",
),
];
pub const DEFAULT_DISK_BYTES: u64 = 20 * 1024 * 1024 * 1024;
const GPU_DEVICE: &str = "zakuro-gpu";
const PORT_DEVICE: &str = "zakuro-port-";
const MOUNT_DEVICE: &str = "zakuro-mnt-";
const MIB: u64 = 1024 * 1024;
pub const NOT_INSTALLED: &str = "incus is not installed";
pub const DEFAULT_IMAGE_LXD: &str = "ubuntu:24.04";
fn on_path(bin: &str) -> bool {
std::env::var_os("PATH")
.is_some_and(|p| std::env::split_paths(&p).any(|d| d.join(bin).is_file()))
}
pub fn cli_binary() -> &'static str {
static CLI: std::sync::OnceLock<&'static str> = std::sync::OnceLock::new();
CLI.get_or_init(|| {
if on_path("incus") {
"incus"
} else if on_path("lxc") {
"lxc"
} else {
"incus"
}
})
}
pub fn default_image(cli: &str) -> &'static str {
if cli == "lxc" {
DEFAULT_IMAGE_LXD
} else {
DEFAULT_IMAGE
}
}
pub trait IncusCli: Send + Sync {
fn run(&self, argv: &[String]) -> Result<String, String>;
}
pub struct IncusShell;
impl IncusCli for IncusShell {
fn run(&self, argv: &[String]) -> Result<String, String> {
let (prog, rest) = argv.split_first().ok_or("empty incus command")?;
let prog = if prog == "incus" { cli_binary() } else { prog };
let out = Command::new(prog).args(rest).output().map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
format!("{NOT_INSTALLED} ({e})")
} else {
format!("incus failed to run: {e}")
}
})?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SandboxSize {
pub limits: Limits,
pub disk_bytes: u64,
}
impl SandboxSize {
pub fn from_budget(b: &Budget, host: &HostResources) -> Result<Self, String> {
Ok(Self {
limits: b.total(host)?,
disk_bytes: b.disk_bytes,
})
}
}
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|a| a.to_string()).collect()
}
pub fn sandbox_config(l: &Limits) -> Vec<(String, String)> {
let cores = l.cpu_millis.div_ceil(1000).max(1);
let allowance = (l.cpu_millis / 10).max(1);
let gpu = l.gpus.flag_value().is_some();
vec![
("limits.cpu".into(), cores.to_string()),
(
"limits.cpu.allowance".into(),
format!("{allowance}ms/100ms"),
),
("limits.memory".into(), mib(l.memory_bytes)),
("limits.memory.enforce".into(), "hard".into()),
("limits.memory.swap".into(), "false".into()),
("security.privileged".into(), "false".into()),
("security.nesting".into(), "true".into()),
("nvidia.runtime".into(), gpu.to_string()),
]
}
fn mib(bytes: u64) -> String {
format!("{}MiB", (bytes / MIB).max(6))
}
pub fn launch_argv(instance: &str, image: &str, size: &SandboxSize) -> Vec<String> {
let mut a = s(&["incus", "launch", image, instance]);
for (k, v) in sandbox_config(&size.limits) {
a.push("-c".into());
a.push(format!("{k}={v}"));
}
a.push("-d".into());
a.push(format!("root,size={}", mib(size.disk_bytes)));
a
}
pub fn gpu_devices(g: &Gpus) -> Result<Vec<(String, Vec<String>)>, String> {
let dev = |id: Option<&str>| {
let mut kv = s(&["gputype=physical"]);
if let Some(id) = id {
kv.push(format!("id={id}"));
}
(format!("{GPU_DEVICE}-{}", id.unwrap_or("all")), kv)
};
Ok(match g {
Gpus::None | Gpus::Count(0) => vec![],
Gpus::All => vec![dev(None)],
Gpus::Count(n) => (0..*n).map(|i| dev(Some(&i.to_string()))).collect(),
Gpus::Spec(spec) => {
let v = spec.trim().trim_matches('"').trim();
if v == "all" {
vec![dev(None)]
} else if let Ok(n) = v.parse::<u32>() {
return gpu_devices(&Gpus::Count(n));
} else if let Some(ids) = v.strip_prefix("device=") {
let ids: Vec<&str> = ids
.split(',')
.map(str::trim)
.filter(|i| !i.is_empty())
.collect();
if ids.is_empty() || ids.iter().any(|i| i.parse::<u32>().is_err()) {
return Err(format!(
"GPU spec {spec:?}: the lxc backend passes GPUs by index (device=0,1)"
));
}
ids.into_iter().map(|i| dev(Some(i))).collect()
} else {
return Err(format!(
"GPU spec {spec:?} is not understood by the lxc backend"
));
}
}
})
}
pub fn provision_script() -> String {
let sums: String = NERDCTL_SHA256
.iter()
.map(|(arch, sum)| format!("{arch}) sum={sum} ;; "))
.collect();
format!(
"set -e; export DEBIAN_FRONTEND=noninteractive; \
for i in $(seq 60); do getent hosts github.com >/dev/null 2>&1 && break; sleep 1; done; \
apt-get update -q >/dev/null; \
apt-get install -y -q containerd containernetworking-plugins iptables curl ca-certificates >/dev/null; \
arch=$(dpkg --print-architecture); \
case $arch in {sums}*) echo \"no pinned nerdctl for $arch\" >&2; exit 1 ;; esac; \
curl -fsSL https://github.com/containerd/nerdctl/releases/download/v{v}/nerdctl-{v}-linux-$arch.tar.gz -o /tmp/nerdctl.tgz; \
echo \"$sum /tmp/nerdctl.tgz\" | sha256sum -c - >/dev/null; \
tar -C /usr/local/bin -xzf /tmp/nerdctl.tgz nerdctl; rm -f /tmp/nerdctl.tgz; \
systemctl enable --now containerd >/dev/null 2>&1; nerdctl version >/dev/null",
v = NERDCTL_VERSION
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Publish {
pub ip: String,
pub host: Option<u16>,
pub container: String,
pub proto: String,
}
pub fn parse_publish(p: &str) -> Result<Publish, String> {
let parts: Vec<&str> = p.split(':').collect();
let port = |v: &str| {
v.parse::<u16>()
.map_err(|_| format!("bad publish spec {p:?}"))
};
let (ip, host, cont) = match parts.as_slice() {
[c] => ("0.0.0.0", "", *c),
[h, c] => ("0.0.0.0", *h, *c),
[ip, h, c] => (*ip, *h, *c),
_ => return Err(format!("bad publish spec {p:?}")),
};
let host = if host.is_empty() {
None
} else {
Some(port(host)?)
};
let (cport, proto) = cont.split_once('/').unwrap_or((cont, "tcp"));
port(cport)?;
let proto = proto.to_ascii_lowercase();
if proto != "tcp" && proto != "udp" {
return Err(format!(
"publish {p:?}: the lxc backend publishes tcp or udp, not {proto}"
));
}
Ok(Publish {
ip: ip.to_string(),
host,
container: cont.to_string(),
proto,
})
}
fn tag(v: &str) -> String {
crate::dataset::sha256_hex(v.as_bytes())[..16].to_string()
}
pub fn parse_cgroup_limits(out: &str) -> Result<Limits, String> {
let bad = || format!("unexpected cgroup limits output: {out:?}");
let mut lines = out.lines().map(str::trim).filter(|l| !l.is_empty());
let cpu = lines.next().ok_or_else(bad)?;
let mem = lines.next().ok_or_else(bad)?;
let gpus: u32 = lines.next().unwrap_or("0").parse().map_err(|_| bad())?;
let mut c = cpu.split_whitespace();
let quota = c.next().ok_or_else(bad)?;
let period: u64 = c.next().ok_or_else(bad)?.parse().map_err(|_| bad())?;
if quota == "max" {
return Err("workload runs without a cpu limit".into());
}
if mem == "max" {
return Err("workload runs without a memory limit".into());
}
let quota: u64 = quota.parse().map_err(|_| bad())?;
let mem: u64 = mem.parse().map_err(|_| bad())?;
if period == 0 {
return Err(bad());
}
Ok(Limits {
cpu_millis: (quota * 1000 / period) as u32,
memory_bytes: mem,
gpus: if gpus > 0 {
Gpus::Count(gpus)
} else {
Gpus::None
},
})
}
#[derive(Clone)]
pub struct LxcRuntime {
cli: Arc<dyn IncusCli>,
instance: String,
image: String,
size: Result<SandboxSize, String>,
}
impl LxcRuntime {
pub fn new(
cli: Arc<dyn IncusCli>,
instance: &str,
image: &str,
size: Result<SandboxSize, String>,
) -> Self {
Self {
cli,
instance: instance.to_string(),
image: image.to_string(),
size,
}
}
pub fn from_env(cli: Arc<dyn IncusCli>, get: &dyn Fn(&str) -> Option<String>) -> Self {
let image = get("ZAKURO_SANDBOX_IMAGE")
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty())
.unwrap_or_else(|| default_image(cli_binary()).to_string());
let size = Budget::from_env(get)
.and_then(|b| SandboxSize::from_budget(&b, &HostResources::detect()));
Self::new(cli, SANDBOX, &image, size)
}
pub fn shell() -> Self {
Self::from_env(Arc::new(IncusShell), &|k| std::env::var(k).ok())
}
pub fn image(&self) -> &str {
&self.image
}
pub fn size(&self) -> &Result<SandboxSize, String> {
&self.size
}
pub fn instance(&self) -> &str {
&self.instance
}
fn incus(&self, rest: &[&str]) -> Result<String, String> {
let mut argv = vec!["incus".to_string()];
argv.extend(rest.iter().map(|a| a.to_string()));
self.cli.run(&argv)
}
pub fn in_sandbox(&self, docker_argv: &[String]) -> Vec<String> {
let mut a = s(&["incus", "exec", &self.instance, "--", "nerdctl"]);
let rest = match docker_argv.first().map(String::as_str) {
Some("docker") => &docker_argv[1..],
_ => docker_argv,
};
a.extend(rest.iter().cloned());
a
}
fn nerdctl(&self, docker_argv: &[String]) -> Result<String, String> {
self.cli.run(&self.in_sandbox(docker_argv))
}
pub fn state(&self) -> Result<Option<String>, String> {
let out = self.incus(&["list", &self.instance, "-f", "csv", "-c", "ns"])?;
Ok(out.lines().find_map(|l| {
let (n, st) = l.split_once(',')?;
(n.trim() == self.instance).then(|| st.trim().to_string())
}))
}
fn privileged(&self) -> Result<bool, String> {
Ok(self
.incus(&["config", "get", &self.instance, "security.privileged"])?
.trim()
.eq_ignore_ascii_case("true"))
}
fn devices(&self) -> Result<Vec<String>, String> {
Ok(self
.incus(&["config", "device", "list", &self.instance])?
.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.collect())
}
pub fn sandbox_limits(&self) -> Result<Limits, String> {
let get = |k: &str| self.incus(&["config", "get", &self.instance, k]);
let allowance = get("limits.cpu.allowance")?;
let cpu_millis = allowance
.strip_suffix("ms/100ms")
.and_then(|v| v.parse::<u32>().ok())
.map(|ms| ms * 10)
.ok_or_else(|| {
format!(
"{} has no hard cpu allowance ({allowance:?})",
self.instance
)
})?;
let mem = get("limits.memory")?;
let memory_bytes = mem
.strip_suffix("MiB")
.and_then(|v| v.parse::<u64>().ok())
.map(|m| m * MIB)
.ok_or_else(|| format!("{} has no memory limit ({mem:?})", self.instance))?;
let gpus = self
.devices()?
.iter()
.filter(|d| d.starts_with(GPU_DEVICE))
.count() as u32;
Ok(Limits {
cpu_millis,
memory_bytes,
gpus: if gpus > 0 {
Gpus::Count(gpus)
} else {
Gpus::None
},
})
}
pub fn ensure(&self) -> Result<(), String> {
let size = self.size.clone()?;
let inst = self.instance.as_str();
let wanted_gpus = gpu_devices(&size.limits.gpus)?;
match self.state()? {
None => {
self.cli
.run(&launch_argv(inst, &self.image, &size))
.map_err(|e| format!("incus launch {inst} failed: {e}"))?;
}
Some(state) => {
if self.privileged()? {
return Err(privileged_refusal(inst));
}
let nv_before = self.incus(&["config", "get", inst, "nvidia.runtime"])?;
let mut set = s(&["incus", "config", "set", inst]);
let cfg = sandbox_config(&size.limits);
let nv_after = cfg
.iter()
.find(|(k, _)| k == "nvidia.runtime")
.map(|(_, v)| v.clone())
.unwrap_or_default();
set.extend(cfg.into_iter().map(|(k, v)| format!("{k}={v}")));
self.cli.run(&set)?;
let disk = format!("size={}", mib(size.disk_bytes));
if self
.incus(&["config", "device", "set", inst, "root", &disk])
.is_err()
{
self.incus(&["config", "device", "override", inst, "root", &disk])?;
}
let nv_changed =
nv_before.trim().eq_ignore_ascii_case("true") != (nv_after == "true");
if !state.eq_ignore_ascii_case("RUNNING") {
self.incus(&["start", inst])?;
} else if nv_changed {
self.incus(&["restart", inst])?;
}
}
}
let have: Vec<String> = self
.devices()?
.into_iter()
.filter(|d| d.starts_with(GPU_DEVICE))
.collect();
for d in &have {
if !wanted_gpus.iter().any(|(n, _)| n == d) {
self.incus(&["config", "device", "remove", inst, d])?;
}
}
for (name, kv) in &wanted_gpus {
if !have.contains(name) {
let mut a = s(&["incus", "config", "device", "add", inst, name, "gpu"]);
a.extend(kv.iter().cloned());
self.cli.run(&a)?;
}
}
let ready = self.incus(&[
"exec",
inst,
"--",
"sh",
"-c",
"command -v nerdctl >/dev/null && systemctl is-active --quiet containerd",
]);
if ready.is_err() {
self.incus(&["exec", inst, "--", "sh", "-c", &provision_script()])
.map_err(|e| format!("provisioning containerd in {inst} failed: {e}"))?;
}
Ok(())
}
fn mount(&self, host: &str, have: &[String]) -> Result<String, String> {
let t = tag(host);
let dev = format!("{MOUNT_DEVICE}{t}");
let inner = format!("/zakuro/mnt/{t}");
if have.contains(&dev) {
let source =
self.incus(&["config", "device", "get", &self.instance, &dev, "source"])?;
if source.trim() != host {
return Err(format!(
"{} device {dev} shares {:?}, not {host:?}; refusing to reuse it \
(remove it with `incus config device remove {} {dev}`)",
self.instance,
source.trim(),
self.instance
));
}
} else {
self.cli.run(&s(&[
"incus",
"config",
"device",
"add",
&self.instance,
&dev,
"disk",
&format!("source={host}"),
&format!("path={inner}"),
"shift=true",
]))?;
}
Ok(inner)
}
fn remove_port_devices(&self, name: &str) -> Result<(), String> {
let suffix = format!("-{}", tag(name));
for d in self.devices()? {
if d.starts_with(PORT_DEVICE) && d.ends_with(&suffix) {
self.incus(&["config", "device", "remove", &self.instance, &d])?;
}
}
Ok(())
}
}
fn privileged_refusal(inst: &str) -> String {
format!(
"{inst} is a privileged container, so the budget would not hold; \
delete it (`incus delete -f {inst}`) and zc recreates it unprivileged"
)
}
fn server_version(out: &str) -> Option<String> {
out.lines()
.find_map(|l| l.trim().strip_prefix("Server version:"))
.map(|v| v.trim().to_string())
.filter(|v| !v.is_empty() && v != "unreachable")
}
impl Runtime for LxcRuntime {
fn kind(&self) -> RuntimeKind {
RuntimeKind::Lxc
}
fn probe(&self) -> Probe {
let no = |detail: String| Probe {
kind: RuntimeKind::Lxc,
available: false,
detail,
};
let version = match self.incus(&["version"]) {
Ok(v) => v,
Err(e) if e.starts_with(NOT_INSTALLED) => {
return no(format!(
"{NOT_INSTALLED} (apt install incus; the zakuro .deb recommends it, \
Zabbly repo: https://github.com/zabbly/incus)"
))
}
Err(e) => {
return no(e
.lines()
.next()
.unwrap_or("incus version failed")
.to_string())
}
};
let Some(server) = server_version(&version) else {
return no(
"incus daemon unreachable (start incus, and add this user to the incus-admin group)"
.into(),
);
};
let sandbox = match self.state() {
Err(e) => return no(e.lines().next().unwrap_or("incus list failed").to_string()),
Ok(None) => "not created yet (created at the first start)".to_string(),
Ok(Some(st)) => match self.privileged() {
Ok(true) => return no(privileged_refusal(&self.instance)),
Ok(false) => match self.sandbox_limits() {
Ok(l) => format!("{} ({l})", st.to_lowercase()),
Err(e) => format!("{} ({e})", st.to_lowercase()),
},
Err(e) => return no(e),
},
};
Probe {
kind: RuntimeKind::Lxc,
available: true,
detail: format!(
"incus {server}, unprivileged sandbox {} {sandbox}",
self.instance
),
}
}
fn verify_image(&self, image: &str, expected_digest: Option<&str>) -> Result<(), String> {
let out = self
.nerdctl(&s(&[
"docker",
"image",
"inspect",
"--format",
"{{.Id}}|{{json .RepoDigests}}",
image,
]))
.map_err(|e| format!("image {image} is not in {}: {e}", self.instance))?;
let Some(want) = expected_digest else {
return Ok(());
};
docker::check_digest(image, &out, want)
}
fn start(&self, spec: &StartSpec) -> Result<String, String> {
let size = self.size.clone()?;
if spec.limits.gpus.flag_value().is_some() && size.limits.gpus.flag_value().is_none() {
return Err(format!(
"{} asks for a GPU but this node's budget passes none through \
(set ZAKURO_BUDGET_GPU=1 to opt in)",
spec.name
));
}
let publish = spec
.publish
.iter()
.map(|p| parse_publish(p))
.collect::<Result<Vec<_>, _>>()?;
if let Some(p) = publish.iter().position(|p| p.host.is_none()) {
return Err(format!(
"publish {:?}: the lxc backend needs a fixed host port",
spec.publish[p]
));
}
self.ensure()?;
let have = self.devices()?;
let mut inner = spec.clone();
inner.mounts = spec
.mounts
.iter()
.map(|(h, c)| Ok((self.mount(h, &have)?, c.clone())))
.collect::<Result<_, String>>()?;
inner.publish = publish
.iter()
.map(|p| format!("{}:{}", p.host.unwrap_or_default(), p.container))
.collect();
if spec.create_network {
if let Some(net) = &spec.network {
let _ = self.nerdctl(&s(&["docker", "network", "create", net]));
}
}
self.remove_port_devices(&spec.name)?;
inner.remove_on_exit = false;
let _ = self.nerdctl(&s(&["docker", "rm", "-f", &spec.name]));
let mut run = vec!["docker".to_string()];
run.extend(docker::run_argv(&inner));
let id = self.nerdctl(&run)?;
let t = tag(&spec.name);
for p in &publish {
let (ip, proto) = (&p.ip, &p.proto);
let h = p.host.unwrap_or_default();
let r = self.cli.run(&s(&[
"incus",
"config",
"device",
"add",
&self.instance,
&format!("{PORT_DEVICE}{proto}{h}-{t}"),
"proxy",
&format!("listen={proto}:{ip}:{h}"),
&format!("connect={proto}:127.0.0.1:{h}"),
]));
if let Err(e) = r {
let _ = self.stop(&spec.name);
return Err(format!(
"publishing {proto} {ip}:{h} for {} failed: {e}",
spec.name
));
}
}
Ok(id)
}
fn exec(&self, name: &str, argv: &[String], detach: bool) -> Result<String, String> {
let mut full = s(&["docker", "exec"]);
if detach {
full.push("-d".into());
}
full.push(name.to_string());
full.extend(argv.iter().cloned());
self.nerdctl(&full)
}
fn stop(&self, name: &str) -> Result<(), String> {
let Some(state) = self.state()? else {
return Ok(());
};
self.remove_port_devices(name)?;
if !state.eq_ignore_ascii_case("RUNNING") {
return Ok(());
}
match self.nerdctl(&crate::up::docker_rm_argv(name)) {
Ok(_) => Ok(()),
Err(e) if e.contains("no such container") || e.contains("no such object") => Ok(()),
Err(e) => Err(e),
}
}
fn limits(&self, name: &str) -> Result<Limits, String> {
parse_cgroup_limits(&self.nerdctl(&s(&[
"docker",
"exec",
name,
"sh",
"-c",
"cat /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max; \
ls -d /dev/nvidia[0-9]* 2>/dev/null | wc -l",
]))?)
}
fn health(&self, name: &str) -> Result<State, String> {
let out = self.nerdctl(&crate::up::docker_state_argv(name))?;
crate::up::parse_container_state(&out)
.ok_or_else(|| format!("unexpected nerdctl inspect output: {out:?}"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
#[derive(Default)]
struct Rec {
calls: Mutex<Vec<String>>,
answer: Mutex<Vec<(&'static str, Result<String, String>)>>,
}
impl Rec {
fn on(&self, needle: &'static str, a: Result<&str, &str>) {
self.answer
.lock()
.unwrap()
.push((needle, a.map(String::from).map_err(String::from)));
}
fn calls(&self) -> Vec<String> {
self.calls.lock().unwrap().clone()
}
}
impl IncusCli for Rec {
fn run(&self, argv: &[String]) -> Result<String, String> {
let j = argv.join(" ");
self.calls.lock().unwrap().push(j.clone());
for (needle, a) in self.answer.lock().unwrap().iter() {
if j.contains(needle) {
return a.clone();
}
}
Ok(String::new())
}
}
fn size(gpus: Gpus) -> SandboxSize {
SandboxSize {
limits: Limits::new(1.5, 2.0, gpus),
disk_bytes: 10 * 1024 * MIB,
}
}
fn rt(rec: &Arc<Rec>, gpus: Gpus) -> LxcRuntime {
LxcRuntime::new(rec.clone(), SANDBOX, DEFAULT_IMAGE, Ok(size(gpus)))
}
fn spec() -> StartSpec {
StartSpec {
name: "zc-share-w0".into(),
image: "img".into(),
network: Some("net".into()),
create_network: true,
labels: vec![("zakuro.share".into(), "1".into())],
mounts: vec![],
publish: vec!["127.0.0.1:3960:3960".into()],
env: vec![],
cmd: vec![],
limits: Limits::new(0.5, 1.0, Gpus::None),
remove_on_exit: true,
hardened: false,
}
}
#[test]
fn a_new_sandbox_is_launched_unprivileged_and_sized_to_the_budget() {
let d = Arc::new(Rec::default());
rt(&d, Gpus::None).ensure().unwrap();
let calls = d.calls();
assert_eq!(calls[0], "incus list zakuro-sandbox -f csv -c ns");
assert_eq!(
calls[1],
"incus launch images:ubuntu/24.04 zakuro-sandbox \
-c limits.cpu=2 -c limits.cpu.allowance=150ms/100ms -c limits.memory=2048MiB \
-c limits.memory.enforce=hard -c limits.memory.swap=false \
-c security.privileged=false -c security.nesting=true -c nvidia.runtime=false \
-d root,size=10240MiB"
);
assert!(
!calls.iter().any(|c| c.contains(" gpu")),
"no GPU device without the opt-in: {calls:#?}"
);
let d = Arc::new(Rec::default());
d.on("command -v nerdctl", Err("not found"));
rt(&d, Gpus::None).ensure().unwrap();
let last = d.calls().pop().unwrap();
assert!(
last.starts_with("incus exec zakuro-sandbox -- sh -c set -e;"),
"{last}"
);
assert!(
last.contains("containerd containernetworking-plugins"),
"{last}"
);
assert!(
last.contains(&format!("nerdctl-{NERDCTL_VERSION}-linux-$arch")),
"{last}"
);
assert!(last.contains("sha256sum -c"), "{last}");
}
#[test]
fn an_existing_sandbox_is_brought_to_the_current_budget() {
let d = Arc::new(Rec::default());
d.on(
"incus list zakuro-sandbox",
Ok("zakuro-sandbox-old,RUNNING\nzakuro-sandbox,STOPPED"),
);
d.on("config get zakuro-sandbox security.privileged", Ok(""));
d.on("config get zakuro-sandbox nvidia.runtime", Ok("false"));
d.on(
"device set zakuro-sandbox root",
Err("Device from profile(s) cannot be modified"),
);
rt(&d, Gpus::None).ensure().unwrap();
let calls = d.calls();
assert!(
calls.contains(
&"incus config set zakuro-sandbox limits.cpu=2 limits.cpu.allowance=150ms/100ms \
limits.memory=2048MiB limits.memory.enforce=hard limits.memory.swap=false \
security.privileged=false security.nesting=true nvidia.runtime=false"
.to_string()
),
"{calls:#?}"
);
assert!(calls.contains(
&"incus config device override zakuro-sandbox root size=10240MiB".to_string()
));
assert!(calls.contains(&"incus start zakuro-sandbox".to_string()));
assert!(!calls.iter().any(|c| c.starts_with("incus launch")));
}
#[test]
fn a_privileged_sandbox_is_refused() {
let d = Arc::new(Rec::default());
d.on("incus list zakuro-sandbox", Ok("zakuro-sandbox,RUNNING"));
d.on("security.privileged", Ok("true"));
d.on(
"version",
Ok("Client version: 6.0.0\nServer version: 6.0.0"),
);
let r = rt(&d, Gpus::None);
assert!(r.ensure().unwrap_err().contains("privileged"));
let p = r.probe();
assert!(!p.available);
assert!(p.detail.contains("privileged"), "{}", p.detail);
assert!(r.start(&spec()).is_err());
assert!(!d.calls().iter().any(|c| c.contains("nerdctl run")));
}
#[test]
fn the_gpu_is_passed_through_only_after_opt_in() {
let d = Arc::new(Rec::default());
rt(&d, Gpus::All).ensure().unwrap();
let calls = d.calls();
assert!(calls[1].contains("-c nvidia.runtime=true"), "{}", calls[1]);
assert!(
calls.contains(
&"incus config device add zakuro-sandbox zakuro-gpu-all gpu gputype=physical"
.to_string()
),
"{calls:#?}"
);
let d = Arc::new(Rec::default());
d.on("incus list zakuro-sandbox", Ok("zakuro-sandbox,RUNNING"));
d.on("security.privileged", Ok("false"));
d.on("nvidia.runtime", Ok("true"));
d.on(
"device list",
Ok("root\nzakuro-gpu-all\nzakuro-port-3960-1"),
);
rt(&d, Gpus::None).ensure().unwrap();
let calls = d.calls();
assert!(
calls.contains(&"incus config device remove zakuro-sandbox zakuro-gpu-all".to_string()),
"{calls:#?}"
);
assert!(calls.contains(&"incus restart zakuro-sandbox".to_string()));
assert!(
!calls.iter().any(|c| c.contains("device add")),
"{calls:#?}"
);
let d = Arc::new(Rec::default());
let gpu_job = StartSpec {
limits: Limits::new(1.0, 1.0, Gpus::Count(1)),
..spec()
};
let e = rt(&d, Gpus::None).start(&gpu_job).unwrap_err();
assert!(e.contains("ZAKURO_BUDGET_GPU=1"), "{e}");
assert!(d.calls().is_empty(), "{:#?}", d.calls());
let d = Arc::new(Rec::default());
rt(&d, Gpus::All).start(&gpu_job).unwrap();
assert!(d
.calls()
.iter()
.any(|c| c.contains("nerdctl run") && c.contains("--gpus 1")));
}
#[test]
fn gpu_specs_map_to_devices_by_index() {
let names = |g: Gpus| -> Vec<String> {
gpu_devices(&g)
.unwrap()
.into_iter()
.map(|(n, _)| n)
.collect()
};
assert!(names(Gpus::None).is_empty());
assert!(names(Gpus::Count(0)).is_empty());
assert_eq!(names(Gpus::Count(2)), ["zakuro-gpu-0", "zakuro-gpu-1"]);
assert_eq!(
names(Gpus::Spec("\"device=1,3\"".into())),
["zakuro-gpu-1", "zakuro-gpu-3"]
);
assert_eq!(names(Gpus::Spec("all".into())), ["zakuro-gpu-all"]);
assert_eq!(
gpu_devices(&Gpus::Spec("device=0".into())).unwrap()[0].1,
["gputype=physical", "id=0"]
);
assert!(gpu_devices(&Gpus::Spec("device=GPU-abc".into())).is_err());
}
#[test]
fn a_workload_runs_through_nerdctl_with_its_own_limits_and_a_host_proxy() {
let d = Arc::new(Rec::default());
d.on("nerdctl run", Ok("cid"));
let job = StartSpec {
mounts: vec![("/home/me/data".into(), "/data".into())],
..spec()
};
assert_eq!(rt(&d, Gpus::None).start(&job).unwrap(), "cid");
let calls = d.calls();
let t = tag("/home/me/data");
assert!(
calls.contains(&format!(
"incus config device add zakuro-sandbox zakuro-mnt-{t} disk \
source=/home/me/data path=/zakuro/mnt/{t} shift=true"
)),
"{calls:#?}"
);
assert!(
calls.contains(&"incus exec zakuro-sandbox -- nerdctl network create net".to_string())
);
assert!(
calls.contains(&"incus exec zakuro-sandbox -- nerdctl rm -f zc-share-w0".to_string())
);
let run = calls.iter().find(|c| c.contains("nerdctl run")).unwrap();
assert_eq!(
run,
&format!(
"incus exec zakuro-sandbox -- nerdctl run -d --name zc-share-w0 \
--network net --label zakuro.share=1 --label zakuro.budget=500m,1073741824 \
-v /zakuro/mnt/{t}:/data \
--cpus 0.5 --memory 1024m --memory-swap 1024m -p 3960:3960 img"
)
);
let w = tag("zc-share-w0");
assert_eq!(
calls.last().unwrap(),
&format!(
"incus config device add zakuro-sandbox zakuro-port-tcp3960-{w} proxy \
listen=tcp:127.0.0.1:3960 connect=tcp:127.0.0.1:3960"
)
);
}
#[test]
fn a_udp_publish_is_carried_as_udp_to_the_host() {
let d = Arc::new(Rec::default());
let job = StartSpec {
publish: vec!["127.0.0.1:5353:53/udp".into()],
..spec()
};
rt(&d, Gpus::None).start(&job).unwrap();
let calls = d.calls();
let run = calls.iter().find(|c| c.contains("nerdctl run")).unwrap();
assert!(run.contains(" -p 5353:53/udp "), "{run}");
assert_eq!(
calls.last().unwrap(),
&format!(
"incus config device add zakuro-sandbox zakuro-port-udp5353-{} proxy \
listen=udp:127.0.0.1:5353 connect=udp:127.0.0.1:5353",
tag("zc-share-w0")
)
);
let d = Arc::new(Rec::default());
let sctp = StartSpec {
publish: vec!["9000:9000/sctp".into()],
..spec()
};
let e = rt(&d, Gpus::None).start(&sctp).unwrap_err();
assert!(e.contains("tcp or udp, not sctp"), "{e}");
assert!(d.calls().is_empty(), "{:#?}", d.calls());
}
#[test]
fn a_mount_device_is_reused_only_for_the_same_host_directory() {
let t = tag("/home/me/data");
let job = StartSpec {
mounts: vec![("/home/me/data".into(), "/data".into())],
..spec()
};
let devs = format!("root\nzakuro-mnt-{t}");
let d = Arc::new(Rec::default());
d.on("device list", Ok(&devs));
d.on("source", Ok("/home/me/data"));
rt(&d, Gpus::None).start(&job).unwrap();
assert!(
!d.calls()
.iter()
.any(|c| c.contains("add zakuro-sandbox zakuro-mnt-")),
"{:#?}",
d.calls()
);
let d = Arc::new(Rec::default());
d.on("device list", Ok(&devs));
d.on("source", Ok("/home/someone-else"));
let e = rt(&d, Gpus::None).start(&job).unwrap_err();
assert!(e.contains("refusing to reuse"), "{e}");
assert!(e.contains("/home/someone-else"), "{e}");
assert!(
!d.calls().iter().any(|c| c.contains("nerdctl run")),
"{:#?}",
d.calls()
);
assert!(d.calls().contains(&format!(
"incus config device get zakuro-sandbox zakuro-mnt-{t} source"
)));
assert_eq!(t.len(), 16);
}
#[test]
fn an_ephemeral_publish_or_a_bad_budget_is_refused_up_front() {
let d = Arc::new(Rec::default());
let job = StartSpec {
publish: vec!["127.0.0.1::8888".into()],
..spec()
};
assert!(rt(&d, Gpus::None)
.start(&job)
.unwrap_err()
.contains("fixed host port"));
let bad = LxcRuntime::new(
d.clone(),
SANDBOX,
DEFAULT_IMAGE,
Err("ZAKURO_BUDGET_CPU=bad".into()),
);
assert_eq!(bad.start(&spec()).unwrap_err(), "ZAKURO_BUDGET_CPU=bad");
assert!(d.calls().is_empty(), "{:#?}", d.calls());
assert_eq!(
parse_publish("127.0.0.1:3960:3960").unwrap(),
Publish {
ip: "127.0.0.1".into(),
host: Some(3960),
container: "3960".into(),
proto: "tcp".into(),
}
);
let p = parse_publish("8080:80/tcp").unwrap();
assert_eq!((p.host, p.proto.as_str()), (Some(8080), "tcp"));
assert_eq!(parse_publish("53:53/UDP").unwrap().proto, "udp");
assert!(parse_publish("53:x/udp").is_err());
assert!(parse_publish("a:b:c:d").is_err());
}
#[test]
fn exec_stop_health_and_limits_go_through_nerdctl() {
let d = Arc::new(Rec::default());
d.on("incus list zakuro-sandbox", Ok("zakuro-sandbox,RUNNING"));
let devs = format!(
"root\nzakuro-port-tcp3960-{}\nzakuro-port-tcp4000-0000000000000000",
tag("n1")
);
d.on("device list", Ok(&devs));
d.on("inspect -f {{.State", Ok("true|2026-11-07T00:00:00Z|0"));
d.on("cpu.max", Ok("50000 100000\n67108864\n0"));
let r = rt(&d, Gpus::None);
r.exec("n1", &s(&["sh", "-c", "true"]), true).unwrap();
r.stop("n1").unwrap();
assert!(r.health("n1").unwrap().running);
assert_eq!(
r.limits("n1").unwrap(),
Limits::new(0.5, 64.0 / 1024.0, Gpus::None)
);
let calls = d.calls();
assert_eq!(
calls[0],
"incus exec zakuro-sandbox -- nerdctl exec -d n1 sh -c true"
);
assert!(calls.contains(&format!(
"incus config device remove zakuro-sandbox zakuro-port-tcp3960-{}",
tag("n1")
)));
assert!(
!calls.iter().any(|c| c.contains("zakuro-port-tcp4000")),
"other workload's port kept"
);
assert!(calls.contains(&"incus exec zakuro-sandbox -- nerdctl rm -f n1".to_string()));
let gone = Arc::new(Rec::default());
gone.on("incus list", Ok("zakuro-sandbox,RUNNING"));
gone.on(
"rm -f",
Err("time=... level=error msg=\"1 errors:\\nno such container: n1\""),
);
assert!(rt(&gone, Gpus::None).stop("n1").is_ok());
let none = Arc::new(Rec::default());
assert!(rt(&none, Gpus::None).stop("n1").is_ok());
assert_eq!(none.calls().len(), 1);
let stopped = Arc::new(Rec::default());
stopped.on("incus list", Ok("zakuro-sandbox,STOPPED"));
stopped.on("device list", Ok(&devs));
stopped.on("incus exec", Err("Error: Instance is not running"));
assert!(rt(&stopped, Gpus::None).stop("n1").is_ok());
let calls = stopped.calls();
assert!(calls.contains(&format!(
"incus config device remove zakuro-sandbox zakuro-port-tcp3960-{}",
tag("n1")
)));
assert!(
!calls.iter().any(|c| c.starts_with("incus exec")),
"{calls:#?}"
);
}
#[test]
fn cgroup_limits_parse_and_an_uncapped_workload_is_an_error() {
assert_eq!(
parse_cgroup_limits("150000 100000\n2147483648\n2").unwrap(),
Limits::new(1.5, 2.0, Gpus::Count(2))
);
assert!(parse_cgroup_limits("max 100000\n1024\n0")
.unwrap_err()
.contains("cpu"));
assert!(parse_cgroup_limits("1000 100000\nmax\n0")
.unwrap_err()
.contains("memory"));
assert!(parse_cgroup_limits("garbage").is_err());
}
#[test]
fn the_sandbox_limits_are_read_back_from_incus() {
let d = Arc::new(Rec::default());
d.on("limits.cpu.allowance", Ok("150ms/100ms"));
d.on("limits.memory", Ok("2048MiB"));
d.on("device list", Ok("root\nzakuro-gpu-all"));
assert_eq!(
rt(&d, Gpus::None).sandbox_limits().unwrap(),
Limits::new(1.5, 2.0, Gpus::Count(1))
);
let soft = Arc::new(Rec::default());
soft.on("limits.cpu.allowance", Ok("50%"));
assert!(rt(&soft, Gpus::None)
.sandbox_limits()
.unwrap_err()
.contains("hard cpu"));
}
#[test]
fn the_probe_says_why_and_docker_takes_over_without_incus() {
let missing = Arc::new(Rec::default());
missing.on(
"version",
Err("incus is not installed (No such file or directory (os error 2))"),
);
let p = rt(&missing, Gpus::None).probe();
assert!(!p.available);
assert!(
p.detail.contains("zakuro .deb recommends it"),
"{}",
p.detail
);
let down = Arc::new(Rec::default());
down.on(
"version",
Ok("Client version: 6.0.0\nServer version: unreachable"),
);
let p = rt(&down, Gpus::None).probe();
assert!(!p.available);
assert!(p.detail.contains("incus-admin"), "{}", p.detail);
let up = Arc::new(Rec::default());
up.on(
"version",
Ok("Client version: 6.0.0\nServer version: 6.0.0"),
);
let p = rt(&up, Gpus::None).probe();
assert!(p.available, "{}", p.detail);
assert_eq!(
p.detail,
"incus 6.0.0, unprivileged sandbox zakuro-sandbox not created yet (created at the first start)"
);
let lxc = rt(&missing, Gpus::None);
let sel = super::super::select(&[RuntimeKind::Lxc, RuntimeKind::Docker], &|k| match k {
RuntimeKind::Lxc => lxc.probe(),
k => Probe {
kind: k,
available: true,
detail: "docker daemon responding".into(),
},
});
assert_eq!(sel.chosen, Some(RuntimeKind::Docker));
assert!(!sel.tried[0].available);
let host = HostResources {
logical_cpus: 8,
memory_bytes: Some(16 << 30),
};
let report = super::super::doctor_report(&sel, Ok(&Budget::default()), &host);
assert!(
report.contains("lxc: skipped: incus is not installed"),
"{report}"
);
}
#[test]
#[ignore = "needs an Incus daemon; set ZC_LIVE_INCUS=1"]
fn live_incus_sandbox_enforces_the_budget() {
if std::env::var("ZC_LIVE_INCUS").ok().as_deref() != Some("1") {
eprintln!("ZC_LIVE_INCUS != 1: skipped");
return;
}
let inst = format!("zc-live-sandbox-{}", std::process::id());
let sandbox = SandboxSize {
limits: Limits::new(1.0, 0.5, Gpus::None),
disk_bytes: 4 << 30,
};
let rt = LxcRuntime::new(
Arc::new(IncusShell),
&inst,
DEFAULT_IMAGE,
Ok(sandbox.clone()),
);
assert!(rt.probe().available, "{}", rt.probe().detail);
struct Cleanup(String);
impl Drop for Cleanup {
fn drop(&mut self) {
let _ = IncusShell.run(&s(&["incus", "delete", "-f", &self.0]));
}
}
let _cleanup = Cleanup(inst.clone());
let t0 = std::time::Instant::now();
rt.ensure().expect("ensure sandbox");
eprintln!("sandbox ready in {:?}", t0.elapsed());
assert_eq!(rt.sandbox_limits().unwrap(), sandbox.limits);
let inside =
|cmd: &str| IncusShell.run(&s(&["incus", "exec", &inst, "--", "sh", "-c", cmd]));
assert_eq!(
inside("cat /sys/fs/cgroup/cpu.max").unwrap(),
"100000 100000"
);
assert_eq!(
inside("cat /sys/fs/cgroup/memory.max").unwrap(),
(512 * MIB).to_string()
);
assert_eq!(
inside("cat /proc/self/uid_map")
.unwrap()
.split_whitespace()
.next(),
Some("0")
);
assert_ne!(
inside("cat /proc/self/uid_map")
.unwrap()
.split_whitespace()
.nth(1),
Some("0"),
"sandbox root must not be host root"
);
let budget = Limits::new(0.5, 64.0 / 1024.0, Gpus::None);
let name = "zc-live-stress".to_string();
let spec = StartSpec {
name: name.clone(),
image: "docker.io/library/alpine:3.20".into(),
network: None,
create_network: false,
labels: vec![("zakuro.test".into(), "stress".into())],
mounts: vec![],
publish: vec![],
env: vec![],
cmd: vec![
"sh".into(),
"-c".into(),
"a=$(awk '/usage_usec/{print $2}' /sys/fs/cgroup/cpu.stat); \
(timeout 3 sh -c 'while :; do :; done' &); \
(timeout 3 sh -c 'while :; do :; done' &); sleep 3.2; \
b=$(awk '/usage_usec/{print $2}' /sys/fs/cgroup/cpu.stat); \
echo cpu_usec=$((b-a)) > /tmp/cpu; sleep 600"
.into(),
],
limits: budget.clone(),
remove_on_exit: false,
hardened: true,
};
rt.start(&spec).expect("start");
assert_eq!(
rt.limits(&name).unwrap(),
budget,
"nerdctl applied the budget"
);
assert!(rt.health(&name).unwrap().running);
std::thread::sleep(std::time::Duration::from_secs(5));
let cpu = rt
.exec(&name, &s(&["cat", "/tmp/cpu"]), false)
.expect("cpu sample");
let usec: u64 = cpu.trim().trim_start_matches("cpu_usec=").parse().unwrap();
eprintln!("workload cpu over 3.2 s: {usec} usec");
assert!(usec < 2_400_000, "cpu not capped: {usec} usec in 3.2 s");
let mem = rt.exec(
&name,
&s(&[
"sh",
"-c",
"dd if=/dev/zero of=/dev/null bs=256M count=1 2>/dev/null && echo survived",
]),
false,
);
assert!(
!mem.as_deref().unwrap_or("").contains("survived"),
"memory not capped: {mem:?}"
);
assert!(!rt
.devices()
.unwrap()
.iter()
.any(|d| d.starts_with(GPU_DEVICE)));
let gpu = StartSpec {
name: "zc-live-gpu".into(),
limits: Limits::new(0.5, 0.0625, Gpus::Count(1)),
..spec.clone()
};
assert!(rt.start(&gpu).unwrap_err().contains("ZAKURO_BUDGET_GPU"));
rt.stop(&name).unwrap();
rt.stop(&name).unwrap();
}
}