use std::process::Command;
use std::sync::Arc;
use super::{Gpus, Limits, Probe, Runtime, RuntimeKind, StartSpec, State};
pub trait DockerCli: Send + Sync {
fn run(&self, argv: &[String]) -> Result<String, String>;
}
pub struct DockerShell;
impl DockerCli for DockerShell {
fn run(&self, argv: &[String]) -> Result<String, String> {
let (prog, rest) = argv.split_first().ok_or("empty docker command")?;
let out = Command::new(prog)
.args(rest)
.output()
.map_err(|e| format!("docker not available: {e}"))?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
}
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|a| a.to_string()).collect()
}
pub fn run_argv(spec: &StartSpec) -> Vec<String> {
let mut a = vec!["run".to_string()];
if spec.remove_on_exit {
a.push("--rm".into());
}
a.extend(s(&["-d", "--name"]));
a.push(spec.name.clone());
if let Some(net) = &spec.network {
a.push("--network".into());
a.push(net.clone());
}
for (k, v) in &spec.labels {
a.push("--label".into());
a.push(format!("{k}={v}"));
}
a.push("--label".into());
a.push(format!(
"{}={}",
super::BUDGET_LABEL,
spec.limits.budget_label()
));
if let Some(node) = crate::nodes::explicit_id() {
a.push("--label".into());
a.push(format!("{}={node}", crate::nodes::NODE_LABEL));
}
for (host, container) in &spec.mounts {
a.push("-v".into());
a.push(format!("{host}:{container}"));
}
if spec.hardened {
a.extend(s(&[
"--pids-limit",
"4096",
"--security-opt",
"no-new-privileges",
]));
}
a.extend(limit_args(&spec.limits));
for p in &spec.publish {
a.push("-p".into());
a.push(p.clone());
}
for (k, v) in &spec.env {
a.push("-e".into());
a.push(format!("{k}={v}"));
}
a.push(spec.image.clone());
a.extend(spec.cmd.iter().cloned());
a
}
pub fn limit_args(l: &Limits) -> Vec<String> {
let mut a = vec![
"--cpus".to_string(),
l.cpus_flag(),
"--memory".to_string(),
l.memory_flag(),
"--memory-swap".to_string(),
l.memory_flag(),
];
if let Some(g) = l.gpus.flag_value() {
a.push("--gpus".into());
a.push(g);
}
a
}
pub fn allocations_args() -> Vec<String> {
vec![
"ps".to_string(),
"--filter".to_string(),
format!("label={}", super::BUDGET_LABEL),
"--format".to_string(),
format!(
"{{{{.Names}}}}|{{{{.Label \"{}\"}}}}|{{{{.Label \"{}\"}}}}",
super::BUDGET_LABEL,
crate::nodes::NODE_LABEL
),
]
}
pub fn parse_allocations(out: &str) -> Vec<super::Allocation> {
parse_allocations_for(out, &crate::nodes::current_id())
}
pub fn parse_allocations_for(out: &str, me: &str) -> Vec<super::Allocation> {
out.lines()
.filter_map(|l| {
let mut f = l.trim().splitn(3, '|');
let (name, label) = (f.next()?, f.next()?);
let node = f.next().map(str::trim).filter(|n| !n.is_empty());
if node.unwrap_or(crate::nodes::MAIN) != me {
return None;
}
super::Allocation::from_label(name.trim(), label)
})
.collect()
}
pub fn limits_inspect_argv(name: &str) -> Vec<String> {
s(&[
"docker",
"inspect",
"-f",
"{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}|{{json .HostConfig.DeviceRequests}}",
name,
])
}
pub fn parse_inspected_limits(out: &str) -> Result<Limits, String> {
let mut f = out.trim().splitn(3, '|');
let bad = || format!("unexpected docker inspect output: {out:?}");
let nano: u64 = f
.next()
.ok_or_else(bad)?
.trim()
.parse()
.map_err(|_| bad())?;
let mem: u64 = f
.next()
.ok_or_else(bad)?
.trim()
.parse()
.map_err(|_| bad())?;
let dev = f.next().unwrap_or("null").trim();
if nano == 0 || mem == 0 {
return Err(format!(
"container runs without a {} limit",
if nano == 0 { "cpu" } else { "memory" }
));
}
let gpus = match serde_json::from_str::<serde_json::Value>(dev) {
Ok(serde_json::Value::Array(reqs)) if !reqs.is_empty() => {
let r = &reqs[0];
let ids: Vec<String> = r["DeviceIDs"]
.as_array()
.map(|v| {
v.iter()
.filter_map(|x| x.as_str().map(String::from))
.collect()
})
.unwrap_or_default();
match r["Count"].as_i64() {
_ if !ids.is_empty() => Gpus::Spec(format!("device={}", ids.join(","))),
Some(-1) => Gpus::All,
Some(n) if n > 0 => Gpus::Count(n as u32),
_ => Gpus::None,
}
}
_ => Gpus::None,
};
Ok(Limits {
cpu_millis: (nano / 1_000_000) as u32,
memory_bytes: mem,
gpus,
})
}
pub fn parse_sha256_digest(v: &str) -> Option<String> {
let d = v.trim().rsplit('@').next()?.to_ascii_lowercase();
let hex = d.strip_prefix("sha256:")?;
(hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())).then_some(d)
}
pub fn check_digest(image: &str, inspected: &str, expected: &str) -> Result<(), String> {
let want = parse_sha256_digest(expected).ok_or_else(|| {
format!("expected digest {expected:?} for {image} is not a full sha256:<64 hex> digest")
})?;
let (id, repo_digests) = inspected.split_once('|').unwrap_or((inspected, ""));
let mut have: Vec<String> = parse_sha256_digest(id).into_iter().collect();
if let Ok(list) = serde_json::from_str::<Vec<String>>(repo_digests.trim()) {
have.extend(list.iter().filter_map(|d| parse_sha256_digest(d)));
}
if have.contains(&want) {
Ok(())
} else {
Err(format!(
"image {image} does not match the expected digest {want}"
))
}
}
#[derive(Clone)]
pub struct DockerRuntime {
cli: Arc<dyn DockerCli>,
}
impl DockerRuntime {
pub fn new(cli: Arc<dyn DockerCli>) -> Self {
Self { cli }
}
pub fn shell() -> Self {
Self::new(Arc::new(DockerShell))
}
fn docker(&self, rest: &[&str]) -> Result<String, String> {
let mut argv = vec!["docker".to_string()];
argv.extend(rest.iter().map(|a| a.to_string()));
self.cli.run(&argv)
}
}
impl Runtime for DockerRuntime {
fn kind(&self) -> RuntimeKind {
RuntimeKind::Docker
}
fn probe(&self) -> Probe {
match self.docker(&["version", "--format", "{{.Server.Version}}"]) {
Ok(v) => Probe {
kind: RuntimeKind::Docker,
available: true,
detail: format!("docker daemon responding (server {v})"),
},
Err(e) => Probe {
kind: RuntimeKind::Docker,
available: false,
detail: e
.lines()
.next()
.unwrap_or("docker version failed")
.to_string(),
},
}
}
fn verify_image(&self, image: &str, expected_digest: Option<&str>) -> Result<(), String> {
let out = self
.docker(&[
"image",
"inspect",
"--format",
"{{.Id}}|{{json .RepoDigests}}",
image,
])
.map_err(|e| format!("image {image} is not on this machine: {e}"))?;
let Some(want) = expected_digest else {
return Ok(());
};
check_digest(image, &out, want)
}
fn start(&self, spec: &StartSpec) -> Result<String, String> {
if spec.create_network {
if let Some(net) = &spec.network {
let _ = self.docker(&["network", "create", net]);
}
}
let mut argv = vec!["docker".to_string()];
argv.extend(run_argv(spec));
self.cli.run(&argv)
}
fn exec(&self, name: &str, argv: &[String], detach: bool) -> Result<String, String> {
let mut full = vec!["docker".to_string(), "exec".to_string()];
if detach {
full.push("-d".into());
}
full.push(name.to_string());
full.extend(argv.iter().cloned());
self.cli.run(&full)
}
fn stop(&self, name: &str) -> Result<(), String> {
self.cli.run(&crate::up::docker_rm_argv(name)).map(|_| ())
}
fn limits(&self, name: &str) -> Result<Limits, String> {
parse_inspected_limits(&self.cli.run(&limits_inspect_argv(name))?)
}
fn allocations(&self) -> Result<Vec<super::Allocation>, String> {
let mut argv = vec!["docker".to_string()];
argv.extend(allocations_args());
Ok(parse_allocations(&self.cli.run(&argv)?))
}
fn health(&self, name: &str) -> Result<State, String> {
let out = self.cli.run(&crate::up::docker_state_argv(name))?;
crate::up::parse_container_state(&out)
.ok_or_else(|| format!("unexpected docker inspect output: {out:?}"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
#[derive(Default)]
struct Rec {
calls: Mutex<Vec<String>>,
answer: Mutex<Vec<(&'static str, Result<String, String>)>>,
}
impl DockerCli for Rec {
fn run(&self, argv: &[String]) -> Result<String, String> {
let j = argv.join(" ");
self.calls.lock().unwrap().push(j.clone());
for (needle, a) in self.answer.lock().unwrap().iter() {
if j.contains(needle) {
return a.clone();
}
}
Ok(String::new())
}
}
fn spec() -> StartSpec {
StartSpec {
name: "n".into(),
image: "img".into(),
network: Some("net".into()),
create_network: true,
labels: vec![("zakuro.share".into(), "1".into())],
mounts: vec![],
publish: vec!["127.0.0.1:3960:3960".into()],
env: vec![],
cmd: vec![],
limits: Limits::new(1.5, 2.0, Gpus::None),
remove_on_exit: true,
hardened: false,
}
}
#[test]
fn every_start_carries_cpu_and_memory() {
assert_eq!(
run_argv(&spec()).join(" "),
"run --rm -d --name n --network net --label zakuro.share=1 \
--label zakuro.budget=1500m,2147483648 \
--cpus 1.5 --memory 2048m --memory-swap 2048m -p 127.0.0.1:3960:3960 img"
);
let gpu = StartSpec {
limits: Limits::new(1.0, 1.0, Gpus::All),
..spec()
};
assert!(run_argv(&gpu)
.join(" ")
.contains("--cpus 1 --memory 1024m --memory-swap 1024m --gpus all"));
}
#[test]
fn start_creates_the_network_then_runs() {
let d = Arc::new(Rec::default());
let rt = DockerRuntime::new(d.clone());
rt.start(&spec()).unwrap();
let calls = d.calls.lock().unwrap().clone();
assert_eq!(calls[0], "docker network create net");
assert!(calls[1].starts_with("docker run --rm -d --name n"));
}
#[test]
fn allocations_are_summed_back_from_the_running_containers() {
let d = Arc::new(Rec::default());
d.answer.lock().unwrap().push((
"docker ps --filter label=zakuro.budget --format {{.Names}}|{{.Label \"zakuro.budget\"}}",
Ok("zk-dep-a-v1|500m,1073741824\nzakuro-share-w0|1000m,2147483648\nodd|garbage".into()),
));
let rt = DockerRuntime::new(d.clone());
let got = rt.allocations().unwrap();
assert_eq!(
got,
vec![
super::super::Allocation {
name: "zk-dep-a-v1".into(),
cpu_millis: 500,
memory_bytes: 1 << 30,
},
super::super::Allocation {
name: "zakuro-share-w0".into(),
cpu_millis: 1000,
memory_bytes: 2 << 30,
},
]
);
d.answer
.lock()
.unwrap()
.insert(0, ("docker ps", Err("Cannot connect".into())));
assert!(rt.allocations().is_err());
}
#[test]
fn applied_limits_are_read_back() {
let l = parse_inspected_limits("1500000000|2147483648|null").unwrap();
assert_eq!(l, Limits::new(1.5, 2.0, Gpus::None));
let all = parse_inspected_limits(
r#"1000000000|1073741824|[{"Driver":"","Count":-1,"DeviceIDs":null,"Capabilities":[["gpu"]]}]"#,
)
.unwrap();
assert_eq!(all.gpus, Gpus::All);
let dev = parse_inspected_limits(
r#"1000000000|1073741824|[{"Driver":"","Count":0,"DeviceIDs":["0"],"Capabilities":[["gpu"]]}]"#,
)
.unwrap();
assert_eq!(dev.gpus, Gpus::Spec("device=0".into()));
assert!(parse_inspected_limits("0|0|null")
.unwrap_err()
.contains("without a cpu limit"));
assert!(parse_inspected_limits("1000|0|null")
.unwrap_err()
.contains("without a memory limit"));
assert!(parse_inspected_limits("garbage").is_err());
}
const D_ID: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
const D_REPO: &str = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
#[test]
fn verify_image_checks_presence_and_digest() {
let d = Arc::new(Rec::default());
d.answer
.lock()
.unwrap()
.push(("image inspect", Ok(format!(r#"{D_ID}|["img@{D_REPO}"]"#))));
let rt = DockerRuntime::new(d.clone());
assert!(rt.verify_image("img", None).is_ok());
assert!(rt
.verify_image("img", Some(&format!("img@{D_REPO}")))
.is_ok());
assert!(rt.verify_image("img", Some(D_REPO)).is_ok());
assert!(rt.verify_image("img", Some(D_ID)).is_ok());
let other = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
assert!(rt
.verify_image("img", Some(other))
.unwrap_err()
.contains("does not match"));
let missing = Arc::new(Rec::default());
missing
.answer
.lock()
.unwrap()
.push(("image inspect", Err("No such image".into())));
assert!(DockerRuntime::new(missing)
.verify_image("img", None)
.unwrap_err()
.contains("not on this machine"));
}
#[test]
fn a_digest_is_compared_whole_never_as_a_substring() {
let inspected = format!(r#"{D_ID}|["img@{D_REPO}"]"#);
let hex = D_REPO.trim_start_matches("sha256:");
for bad in [
&D_REPO[..40],
&D_REPO[..D_REPO.len() - 1],
&format!("sha512:{hex}"),
hex,
"bbbb",
"",
"latest",
&format!("sha256:{hex}0"),
] {
let e = check_digest("img", &inspected, bad).unwrap_err();
assert!(
e.contains("not a full sha256") || e.contains("does not match"),
"{bad:?}: {e}"
);
}
let embedded = format!("{D_REPO}ffff");
assert!(check_digest("img", &inspected, &embedded).is_err());
assert!(check_digest(
"img",
&inspected,
&D_REPO.to_ascii_uppercase().replace("SHA256", "sha256")
)
.is_ok());
assert!(check_digest("img", &inspected, &format!("registry.io/x/img@{D_REPO}")).is_ok());
assert_eq!(parse_sha256_digest(D_ID).as_deref(), Some(D_ID));
assert_eq!(parse_sha256_digest("sha256:abc"), None);
}
#[test]
fn exec_stop_and_health_use_the_shared_argv() {
let d = Arc::new(Rec::default());
d.answer.lock().unwrap().push((
"inspect -f {{.State",
Ok("true|2026-11-07T00:00:00Z|0".into()),
));
let rt = DockerRuntime::new(d.clone());
rt.exec("n", &["sh".into(), "-c".into(), "true".into()], true)
.unwrap();
rt.stop("n").unwrap();
assert!(rt.health("n").unwrap().running);
let calls = d.calls.lock().unwrap().clone();
assert_eq!(calls[0], "docker exec -d n sh -c true");
assert_eq!(calls[1], "docker rm -f n");
}
#[test]
fn a_failed_probe_says_why() {
let d = Arc::new(Rec::default());
d.answer.lock().unwrap().push((
"version",
Err("Cannot connect to the Docker daemon at unix:///var/run/docker.sock".into()),
));
let p = DockerRuntime::new(d).probe();
assert!(!p.available);
assert!(p.detail.contains("Cannot connect"), "{}", p.detail);
}
#[test]
#[ignore = "needs a Docker daemon; set ZC_LIVE_DOCKER=1"]
fn live_stress_is_capped_at_the_budget() {
if std::env::var("ZC_LIVE_DOCKER").ok().as_deref() != Some("1") {
eprintln!("ZC_LIVE_DOCKER != 1: skipped");
return;
}
let rt = DockerRuntime::shell();
assert!(rt.probe().available, "{}", rt.probe().detail);
let name = format!("zc-live-stress-{}", std::process::id());
let budget = Limits::new(0.5, 64.0 / 1024.0, Gpus::None);
let spec = StartSpec {
name: name.clone(),
image: "alpine:3.20".into(),
network: None,
create_network: false,
labels: vec![("zakuro.test".into(), "stress".into())],
mounts: vec![],
publish: vec![],
env: vec![],
cmd: vec![
"sh".into(),
"-c".into(),
"a=$(awk '/usage_usec/{print $2}' /sys/fs/cgroup/cpu.stat); \
(timeout 3 sh -c 'while :; do :; done' &); \
(timeout 3 sh -c 'while :; do :; done' &); sleep 3.2; \
b=$(awk '/usage_usec/{print $2}' /sys/fs/cgroup/cpu.stat); \
echo cpu_usec=$((b-a)) > /tmp/cpu; sleep 600"
.into(),
],
limits: budget.clone(),
remove_on_exit: false,
hardened: true,
};
let _ = rt.stop(&name);
rt.start(&spec).expect("start");
let cleanup = scopeguard(&rt, &name);
assert_eq!(
rt.limits(&name).unwrap(),
budget,
"docker applied the budget"
);
std::thread::sleep(std::time::Duration::from_secs(5));
let cpu = rt
.exec(&name, &["cat".into(), "/tmp/cpu".into()], false)
.expect("cpu sample");
let usec: u64 = cpu.trim().trim_start_matches("cpu_usec=").parse().unwrap();
assert!(usec < 2_400_000, "cpu not capped: {usec} usec in 3.2 s");
let mem = rt.exec(
&name,
&[
"sh".into(),
"-c".into(),
"dd if=/dev/zero of=/dev/null bs=256M count=1 2>/dev/null && echo survived".into(),
],
false,
);
assert!(
!mem.as_deref().unwrap_or("").contains("survived"),
"memory not capped: {mem:?}"
);
drop(cleanup);
}
struct Guard<'a>(&'a DockerRuntime, String);
impl Drop for Guard<'_> {
fn drop(&mut self) {
let _ = self.0.stop(&self.1);
}
}
fn scopeguard<'a>(rt: &'a DockerRuntime, name: &str) -> Guard<'a> {
Guard(rt, name.to_string())
}
}