use std::collections::HashMap;
use std::process::Command;
use std::time::Duration;
use super::{Healthcheck, Port};
pub struct RunSpec<'a> {
pub name: String,
pub image: &'a str,
pub cmd: Option<&'a str>,
pub ports: &'a [Port],
pub env: &'a HashMap<String, String>,
pub labels: Vec<(String, String)>,
pub mounts: Vec<(String, String)>,
pub limits: crate::sandbox::Limits,
pub network: Option<String>,
pub hardened: bool,
}
pub fn build_run_args(spec: &RunSpec, default_network: &str) -> Vec<String> {
let network = spec
.network
.clone()
.unwrap_or_else(|| default_network.to_string());
let mut env: Vec<(String, String)> = spec
.env
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
env.sort();
crate::sandbox::docker::run_argv(&crate::sandbox::StartSpec {
name: spec.name.clone(),
image: spec.image.to_string(),
network: Some(network),
create_network: false,
labels: spec.labels.clone(),
mounts: spec.mounts.clone(),
publish: Vec::new(),
env,
cmd: spec
.cmd
.map(|c| c.split_whitespace().map(str::to_string).collect())
.unwrap_or_default(),
limits: spec.limits.clone(),
remove_on_exit: false,
hardened: spec.hardened,
})
}
pub fn deploy_network() -> String {
std::env::var("ZAKURO_DEPLOY_NETWORK")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "bridge".to_string())
}
pub fn publish_loopback() -> bool {
publish_loopback_from(
std::env::var("ZAKURO_DEPLOY_PUBLISH_LOOPBACK")
.ok()
.as_deref(),
cfg!(any(target_os = "macos", target_os = "windows")),
)
}
fn publish_loopback_from(var: Option<&str>, os_default: bool) -> bool {
match var.map(|v| v.trim().to_ascii_lowercase()).as_deref() {
Some("1") | Some("true") => true,
Some("0") | Some("false") => false,
_ => os_default,
}
}
fn loopback_publish_args(ports: &[u16], bind: &str) -> Vec<String> {
let mut seen = Vec::new();
let mut args = Vec::new();
for &p in ports {
if seen.contains(&p) {
continue;
}
seen.push(p);
args.push("-p".to_string());
args.push(format!("{bind}::{p}"));
}
args
}
fn add_loopback_publish(args: &mut Vec<String>, spec: &RunSpec, sandbox: bool) {
let ports: Vec<u16> = spec.ports.iter().map(|p| p.container).collect();
let at = args.len() - spec.cmd.map_or(0, |c| c.split_whitespace().count()) - 1;
let bind = if sandbox { "0.0.0.0" } else { "127.0.0.1" };
args.splice(at..at, loopback_publish_args(&ports, bind));
}
fn parse_published_port(out: &str) -> Option<(String, u16)> {
let parsed: Vec<std::net::SocketAddr> =
out.lines().filter_map(|l| l.trim().parse().ok()).collect();
let addr = parsed
.iter()
.find(|a| a.is_ipv4())
.or_else(|| parsed.first())?;
let host = if addr.ip().is_unspecified() {
"127.0.0.1".to_string()
} else {
addr.ip().to_string()
};
Some((host, addr.port()))
}
pub fn isolated_network() -> String {
std::env::var("ZAKURO_DEPLOY_ISOLATED_NETWORK")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "zc-isolated".to_string())
}
pub trait ContainerRuntime {
fn available(&self) -> bool;
fn pull(&self, image: &str) -> Result<(), String>;
fn run(&self, spec: &RunSpec) -> Result<String, String>;
fn run_leased(
&self,
spec: &RunSpec,
_pull_config: Option<&std::path::Path>,
) -> Result<String, String> {
self.run(spec)
}
fn stop(&self, container_id: &str) -> Result<(), String>;
fn rm(&self, container_id: &str) -> Result<(), String>;
fn restart(&self, container_id: &str) -> Result<(), String>;
fn is_running(&self, container_id: &str) -> bool;
fn container_ip(&self, container_id: &str) -> Result<String, String>;
fn container_addr(&self, container_id: &str, port: u16) -> Result<(String, u16), String> {
self.container_ip(container_id).map(|ip| (ip, port))
}
fn health_check(&self, container_id: &str, hc: &Healthcheck) -> bool;
fn health_probe(&self, container_id: &str, hc: &Healthcheck) -> Result<(), String> {
if self.health_check(container_id, hc) {
Ok(())
} else {
Err("health check failed".to_string())
}
}
fn logs_tail(&self, container_id: &str, lines: u32) -> String;
fn list_by_label(&self, id: &str) -> Vec<String>;
fn list_by_label_key(&self, _key: &str) -> Vec<String> {
Vec::new()
}
fn login_in(
&self,
_config_dir: &std::path::Path,
registry: &str,
username: &str,
password: &str,
) -> Result<(), String> {
self.login(registry, username, password)
}
fn pull_in(&self, _config_dir: &std::path::Path, image: &str) -> Result<(), String> {
self.pull(image)
}
fn login(&self, _registry: &str, _username: &str, _password: &str) -> Result<(), String> {
Ok(())
}
fn image_digest(&self, _container_id: &str) -> Result<String, String> {
Err("image_digest not supported by this runtime".to_string())
}
fn budget(&self) -> Result<crate::sandbox::Limits, String> {
crate::sandbox::node_limits()
}
fn admit_guest(&self, _d: &super::Desired) -> Result<(), crate::policy::admit::Refusal> {
Ok(())
}
fn allocations(&self) -> Result<Vec<crate::sandbox::Allocation>, String> {
Ok(Vec::new())
}
}
pub fn container_name(id: &str, version: u64) -> String {
format!("zk-dep-{id}-v{version}")
}
pub fn deployment_labels(id: &str, version: u64) -> Vec<(String, String)> {
vec![
("zakuro.deployment".to_string(), id.to_string()),
("zakuro.version".to_string(), version.to_string()),
]
}
pub fn health_http_url(host: &str, port: u16, hc: &Healthcheck) -> String {
let path = hc.path.as_deref().unwrap_or("/");
let path = if path.starts_with('/') {
path.to_string()
} else {
format!("/{path}")
};
format!("http://{host}:{port}{path}")
}
pub struct DockerCli;
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub enum RuntimeError {
SocketMissing,
PermissionDenied,
DaemonDown,
NotInstalled,
Other,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct RuntimeProbe {
pub available: bool,
pub runtime: Option<&'static str>,
pub error: Option<RuntimeError>,
pub detail: String,
pub checked_at: String,
}
#[derive(Debug, Clone)]
pub enum CliOutcome {
SpawnFailed(std::io::ErrorKind, String),
Exited { success: bool, stderr: String },
TimedOut,
}
pub const PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
pub fn run_bounded(cmd: &mut Command, timeout: std::time::Duration) -> (CliOutcome, String) {
use std::io::Read;
use std::process::Stdio;
let mut child = match cmd
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
{
Ok(c) => c,
Err(e) => {
return (
CliOutcome::SpawnFailed(e.kind(), e.to_string()),
String::new(),
)
}
};
fn drain<R: Read + Send + 'static>(r: Option<R>) -> std::thread::JoinHandle<String> {
std::thread::spawn(move || {
let mut buf = Vec::new();
if let Some(mut r) = r {
let _ = r.read_to_end(&mut buf);
}
String::from_utf8_lossy(&buf).into_owned()
})
}
let out = drain(child.stdout.take());
let err = drain(child.stderr.take());
let deadline = std::time::Instant::now() + timeout;
let status = loop {
match child.try_wait() {
Ok(Some(st)) => break Some(st),
Ok(None) if std::time::Instant::now() >= deadline => {
let _ = child.kill();
let _ = child.wait();
break None;
}
Ok(None) => std::thread::sleep(std::time::Duration::from_millis(20)),
Err(e) => {
return (
CliOutcome::SpawnFailed(e.kind(), e.to_string()),
String::new(),
);
}
}
};
match status {
Some(st) => (
CliOutcome::Exited {
success: st.success(),
stderr: err.join().unwrap_or_default().trim().to_string(),
},
out.join().unwrap_or_default(),
),
None => (CliOutcome::TimedOut, String::new()),
}
}
#[derive(Debug, Clone)]
pub struct HostFacts {
pub socket_path: String,
pub socket_exists: bool,
pub containerd_socket: bool,
}
fn default_socket_path() -> String {
match std::env::var("DOCKER_HOST") {
Ok(h) if h.starts_with("unix://") => h["unix://".len()..].to_string(),
_ => "/var/run/docker.sock".to_string(),
}
}
fn host_facts() -> HostFacts {
let socket_path = default_socket_path();
HostFacts {
socket_exists: std::path::Path::new(&socket_path).exists(),
socket_path,
containerd_socket: std::path::Path::new("/run/containerd/containerd.sock").exists(),
}
}
pub fn classify_probe(outcome: CliOutcome, facts: &HostFacts, checked_at: String) -> RuntimeProbe {
let fail = |runtime, error, detail: String| RuntimeProbe {
available: false,
runtime,
error: Some(error),
detail,
checked_at: checked_at.clone(),
};
match outcome {
CliOutcome::Exited { success: true, .. } => RuntimeProbe {
available: true,
runtime: Some("docker"),
error: None,
detail: "docker daemon responding".into(),
checked_at,
},
CliOutcome::TimedOut => fail(
Some("docker"),
RuntimeError::DaemonDown,
format!(
"docker version did not answer within {} s",
PROBE_TIMEOUT.as_secs()
),
),
CliOutcome::SpawnFailed(kind, msg) => {
if kind == std::io::ErrorKind::NotFound {
let (rt, extra) = if facts.containerd_socket {
(
Some("containerd"),
" (containerd is present, but zc deploys through docker)",
)
} else {
(None, "")
};
fail(
rt,
RuntimeError::NotInstalled,
format!("`docker` binary not found on PATH{extra}"),
)
} else {
fail(
None,
RuntimeError::Other,
format!("could not run `docker`: {msg}"),
)
}
}
CliOutcome::Exited { stderr, .. } => {
let low = stderr.to_lowercase();
let sock = &facts.socket_path;
if low.contains("permission denied") {
fail(
Some("docker"),
RuntimeError::PermissionDenied,
format!("permission denied on {sock}"),
)
} else if !facts.socket_exists || low.contains("no such file or directory") {
fail(
None,
RuntimeError::SocketMissing,
format!("docker socket {sock} does not exist"),
)
} else if low.contains("cannot connect")
|| low.contains("daemon running")
|| low.contains("connection refused")
{
fail(
Some("docker"),
RuntimeError::DaemonDown,
format!("docker daemon is not responding on {sock}"),
)
} else {
let first = stderr.lines().next().unwrap_or("docker version failed");
fail(
Some("docker"),
RuntimeError::Other,
format!("docker version failed: {first}"),
)
}
}
}
}
impl RuntimeProbe {
pub fn next_step(&self) -> Option<String> {
Some(match self.error? {
RuntimeError::PermissionDenied => {
"add the zc service user to the docker group (usermod -aG docker <user>) and restart zc".into()
}
RuntimeError::SocketMissing => "start the docker service (systemctl enable --now docker) or set DOCKER_HOST".into(),
RuntimeError::DaemonDown => "start the docker daemon (systemctl start docker)".into(),
RuntimeError::NotInstalled => "install docker on this host".into(),
RuntimeError::Other => "run `zc doctor` on this host for details".into(),
})
}
pub fn failure_message(&self) -> String {
match self.next_step() {
None => "container runtime available".into(),
Some(step) => format!(
"no container runtime on this host: {} — {step}",
self.detail
),
}
}
pub fn to_json(&self) -> serde_json::Value {
serde_json::to_value(self).unwrap_or(serde_json::Value::Null)
}
}
pub fn doctor_report(p: &RuntimeProbe) -> String {
let mut out = format!(
"container runtime: {}\n runtime: {}\n detail: {}\n checked_at: {}",
if p.available { "OK" } else { "UNAVAILABLE" },
p.runtime.unwrap_or("none"),
p.detail,
p.checked_at
);
if let Some(e) = p.error {
out.push_str(&format!(
"\n error: {}",
serde_json::to_value(e)
.ok()
.and_then(|v| v.as_str().map(String::from))
.unwrap_or_default()
));
}
if let Some(step) = p.next_step() {
out.push_str(&format!("\n next step: {step}"));
}
out
}
pub fn runtime_probe() -> RuntimeProbe {
let (outcome, _) = run_bounded(
Command::new("docker").args(["version", "--format", "{{.Server.Version}}"]),
PROBE_TIMEOUT,
);
classify_probe(outcome, &host_facts(), chrono::Utc::now().to_rfc3339())
}
pub fn runtime_available() -> bool {
runtime_probe().available
}
fn docker(args: &[&str]) -> Result<String, String> {
crate::sandbox::engine::run(args)
}
enum Where<'a> {
Engine,
Job(&'a str),
}
fn locate(id: &str) -> (Where<'_>, &str) {
match crate::sandbox::ephemeral::parse_job_id(id) {
Some((inst, name)) => (Where::Job(inst), name),
None => (Where::Engine, id),
}
}
fn ephemeral() -> &'static crate::sandbox::ephemeral::Ephemeral {
crate::sandbox::engine::ephemeral()
}
fn docker_at(w: &Where, args: &[&str]) -> Result<String, String> {
match w {
Where::Engine => docker(args),
Where::Job(inst) => {
use crate::sandbox::lxc::IncusCli as _;
let argv: Vec<String> = args.iter().map(|a| a.to_string()).collect();
crate::sandbox::lxc::IncusShell.run(&crate::sandbox::ephemeral::Ephemeral::exec_argv(
inst, &argv,
))
}
}
}
fn docker_logs(container_id: &str, lines: u32) -> String {
let (w, name) = locate(container_id);
let n = lines.to_string();
let args = ["logs", "--tail", n.as_str(), name];
let argv: Vec<String> = args.iter().map(|a| a.to_string()).collect();
let cmd = match w {
Where::Engine => crate::sandbox::engine::command(&argv).ok(),
Where::Job(inst) => {
let full = crate::sandbox::ephemeral::Ephemeral::exec_argv(inst, &argv);
let mut c = Command::new(crate::sandbox::lxc::cli_binary());
c.args(&full[1..]);
Some(c)
}
};
let Some(Ok(out)) = cmd.map(|mut c| c.output()) else {
return String::new();
};
merge_logs(&out.stdout, &out.stderr)
}
fn merge_logs(stdout: &[u8], stderr: &[u8]) -> String {
let out = String::from_utf8_lossy(stdout);
let err = String::from_utf8_lossy(stderr);
[out.trim_end(), err.trim_end()]
.iter()
.filter(|s| !s.is_empty())
.copied()
.collect::<Vec<_>>()
.join("\n")
}
fn describe_io_error(e: &std::io::Error) -> String {
match e.kind() {
std::io::ErrorKind::ConnectionRefused => "connection refused".to_string(),
std::io::ErrorKind::TimedOut => "timed out".to_string(),
_ => e.to_string(),
}
}
fn describe_ureq_error(e: &ureq::Error) -> String {
match e {
ureq::Error::Io(io) => describe_io_error(io),
ureq::Error::Timeout(_) => "timed out".to_string(),
other => other.to_string(),
}
}
fn docker_login(
config_dir: Option<&std::path::Path>,
registry: &str,
username: &str,
password: &str,
) -> Result<(), String> {
use std::io::Write;
let mut argv: Vec<String> = Vec::new();
if let Some(d) = config_dir {
argv.push("--config".into());
argv.push(d.display().to_string());
}
argv.extend(
["login", "-u", username, "--password-stdin", registry]
.iter()
.map(|a| a.to_string()),
);
let mut child = crate::sandbox::engine::command(&argv)?
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped())
.spawn()
.map_err(|e| format!("container runtime not available: {e}"))?;
if let Some(mut si) = child.stdin.take() {
let _ = si.write_all(password.as_bytes());
}
let out = child.wait_with_output().map_err(|e| e.to_string())?;
if out.status.success() {
Ok(())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
fn write_job_login(
config_dir: &std::path::Path,
registry: &str,
username: &str,
password: &str,
) -> Result<(), String> {
use std::io::Write;
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
let mut o = std::fs::OpenOptions::new();
o.write(true).create(true).truncate(true);
#[cfg(unix)]
o.mode(0o600);
o.open(config_dir.join("config.json"))
.and_then(|mut f| {
f.write_all(
crate::sandbox::ephemeral::docker_config_json(registry, username, password)
.as_bytes(),
)
})
.map_err(|e| format!("docker config: {e}"))
}
const CONTAINER_IP_RETRIES: u32 = 10;
const CONTAINER_IP_RETRY_DELAY: Duration = Duration::from_millis(300);
fn in_sandbox() -> bool {
crate::sandbox::engine::active()
.engine
.as_ref()
.is_some_and(|e| e.is_lxc())
}
fn per_job_isolation() -> bool {
crate::sandbox::engine::active_job_tier() == crate::sandbox::engine::Tier::PerJob
}
fn ephemeral_disk_bytes() -> u64 {
crate::sandbox::Budget::from_env(&|k| std::env::var(k).ok())
.map(|b| b.disk_bytes)
.unwrap_or(crate::sandbox::lxc::DEFAULT_DISK_BYTES)
}
fn sandbox_address() -> Option<String> {
use crate::sandbox::lxc::IncusCli as _;
let out = crate::sandbox::lxc::IncusShell
.run(&[
"incus".to_string(),
"list".to_string(),
crate::sandbox::lxc::SANDBOX.to_string(),
"-f".to_string(),
"csv".to_string(),
"-c".to_string(),
"4".to_string(),
])
.ok()?;
crate::sandbox::ephemeral::parse_ipv4(&out)
}
impl ContainerRuntime for DockerCli {
fn available(&self) -> bool {
crate::sandbox::engine::available()
}
fn admit_guest(&self, d: &super::Desired) -> Result<(), crate::policy::admit::Refusal> {
crate::broker::admission::guest_deployment(d)
}
fn pull(&self, image: &str) -> Result<(), String> {
docker(&["pull", image]).map(|_| ())
}
fn run(&self, spec: &RunSpec) -> Result<String, String> {
let _ = docker(&["rm", "-f", &spec.name]);
if spec.network.as_deref() == Some(isolated_network().as_str())
&& docker(&["network", "inspect", &isolated_network()]).is_err()
{
docker(&["network", "create", "--internal", &isolated_network()])
.map_err(|e| format!("cannot create isolated network: {e}"))?;
}
let mut args = build_run_args(spec, &deploy_network());
if (publish_loopback() || in_sandbox())
&& spec.network.as_deref() != Some(isolated_network().as_str())
{
add_loopback_publish(&mut args, spec, in_sandbox());
}
let arg_refs: Vec<&str> = args.iter().map(|s| s.as_str()).collect();
docker(&arg_refs)
}
fn run_leased(
&self,
spec: &RunSpec,
pull_config: Option<&std::path::Path>,
) -> Result<String, String> {
if !per_job_isolation() {
return self.run(spec);
}
use crate::sandbox::ephemeral;
let eph = ephemeral();
ephemeral::start_reaper_once(eph);
ephemeral::start_job(
eph,
&ephemeral::JobRun {
name: spec.name.clone(),
image: spec.image.to_string(),
labels: spec.labels.clone(),
mounts: spec.mounts.clone(),
env: {
let mut e: Vec<(String, String)> = spec
.env
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
e.sort();
e
},
cmd: spec
.cmd
.map(|c| c.split_whitespace().map(str::to_string).collect())
.unwrap_or_default(),
limits: spec.limits.clone(),
disk_bytes: ephemeral_disk_bytes(),
hardened: spec.hardened,
pull_config: pull_config.map(std::path::Path::to_path_buf),
},
&ephemeral::real_nvidia_smi,
)
}
fn stop(&self, container_id: &str) -> Result<(), String> {
match locate(container_id) {
(Where::Job(inst), _) => ephemeral().destroy(inst),
(w, name) => docker_at(&w, &["stop", name]).map(|_| ()),
}
}
fn rm(&self, container_id: &str) -> Result<(), String> {
match locate(container_id) {
(Where::Job(inst), _) => ephemeral().destroy(inst),
(w, name) => docker_at(&w, &["rm", "-f", name]).map(|_| ()),
}
}
fn restart(&self, container_id: &str) -> Result<(), String> {
let (w, name) = locate(container_id);
docker_at(&w, &["start", name]).map(|_| ())
}
fn is_running(&self, container_id: &str) -> bool {
let (w, name) = locate(container_id);
docker_at(&w, &["inspect", "-f", "{{.State.Running}}", name])
.map(|out| out == "true")
.unwrap_or(false)
}
fn container_ip(&self, container_id: &str) -> Result<String, String> {
let (w, container_id) = locate(container_id);
for attempt in 0..CONTAINER_IP_RETRIES {
let out = docker_at(
&w,
&[
"inspect",
"-f",
"{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}",
container_id,
],
)?;
if let Some(ip) = out.split_whitespace().find(|s| !s.is_empty()) {
return Ok(ip.to_string());
}
if attempt + 1 < CONTAINER_IP_RETRIES {
std::thread::sleep(CONTAINER_IP_RETRY_DELAY);
}
}
Err(format!(
"container {container_id} has no network IP after {CONTAINER_IP_RETRIES} attempts \
(host network mode, or the container exited)"
))
}
fn container_addr(&self, container_id: &str, port: u16) -> Result<(String, u16), String> {
if let (Where::Job(_), _) = locate(container_id) {
return Err(format!(
"{container_id} runs in its own instance and publishes no port"
));
}
if publish_loopback() || in_sandbox() {
let spec = format!("{port}/tcp");
for attempt in 0..CONTAINER_IP_RETRIES {
if let Some((host, mapped)) = docker(&["port", container_id, &spec])
.ok()
.and_then(|out| parse_published_port(&out))
{
return match sandbox_address().filter(|_| in_sandbox()) {
Some(ip) => Ok((ip, mapped)),
None => Ok((host, mapped)),
};
}
if attempt + 1 < CONTAINER_IP_RETRIES {
std::thread::sleep(CONTAINER_IP_RETRY_DELAY);
}
}
}
self.container_ip(container_id).map(|ip| (ip, port))
}
fn health_check(&self, container_id: &str, hc: &Healthcheck) -> bool {
self.health_probe(container_id, hc).is_ok()
}
fn health_probe(&self, container_id: &str, hc: &Healthcheck) -> Result<(), String> {
let (host, port) = self
.container_addr(container_id, hc.port)
.unwrap_or_else(|_| ("127.0.0.1".to_string(), hc.port));
match hc.kind.as_str() {
"tcp" => std::net::TcpStream::connect((host.as_str(), port))
.map(|_| ())
.map_err(|e| format!("TCP {host}:{port} → {}", describe_io_error(&e))),
_ => {
let url = health_http_url(&host, port, hc);
match ureq::get(&url)
.config()
.timeout_global(Some(Duration::from_secs(3)))
.http_status_as_error(false)
.build()
.call()
{
Ok(r) if (200..300).contains(&r.status().as_u16()) => Ok(()),
Ok(r) => Err(format!("GET {url} → HTTP {}", r.status().as_u16())),
Err(e) => Err(format!("GET {url} → {}", describe_ureq_error(&e))),
}
}
}
}
fn logs_tail(&self, container_id: &str, lines: u32) -> String {
docker_logs(container_id, lines)
}
fn allocations(&self) -> Result<Vec<crate::sandbox::Allocation>, String> {
crate::sandbox::engine::allocations()
}
fn list_by_label_key(&self, key: &str) -> Vec<String> {
docker(&["ps", "-a", "-q", "--filter", &format!("label={key}")])
.map(|out| {
out.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.collect()
})
.unwrap_or_default()
}
fn login_in(
&self,
config_dir: &std::path::Path,
registry: &str,
username: &str,
password: &str,
) -> Result<(), String> {
if per_job_isolation() {
return write_job_login(config_dir, registry, username, password);
}
docker_login(Some(config_dir), registry, username, password)
}
fn pull_in(&self, config_dir: &std::path::Path, image: &str) -> Result<(), String> {
if per_job_isolation() {
return Ok(());
}
docker(&["--config", &config_dir.display().to_string(), "pull", image]).map(|_| ())
}
fn login(&self, registry: &str, username: &str, password: &str) -> Result<(), String> {
docker_login(None, registry, username, password)
}
fn image_digest(&self, container_id: &str) -> Result<String, String> {
let (w, container_id) = locate(container_id);
let image_id = docker_at(&w, &["inspect", "-f", "{{.Image}}", container_id])?;
let digests = docker_at(
&w,
&[
"image",
"inspect",
"-f",
"{{range .RepoDigests}}{{.}} {{end}}",
&image_id,
],
)
.unwrap_or_default();
let found: Vec<String> = digests
.split_whitespace()
.filter_map(|d| d.rsplit_once('@').map(|(_, h)| h.to_string()))
.collect();
if found.is_empty() {
Ok(image_id)
} else {
Ok(found.join(","))
}
}
fn list_by_label(&self, id: &str) -> Vec<String> {
docker(&[
"ps",
"-a",
"-q",
"--filter",
&format!("label=zakuro.deployment={id}"),
])
.map(|out| out.lines().map(|l| l.trim().to_string()).collect())
.unwrap_or_default()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn run_spec_new_fields_map_to_docker_args() {
let env = HashMap::from([("K".to_string(), "V".to_string())]);
let spec = RunSpec {
name: "n".into(),
image: "img@sha256:ab",
cmd: None,
ports: &[],
env: &env,
labels: vec![("a".into(), "b".into())],
mounts: vec![("/h/s.sock".into(), "/run/zakuro/space.sock".into())],
limits: crate::sandbox::Limits::new(2.5, 8.0, crate::sandbox::Gpus::Count(1)),
network: Some("egress".into()),
hardened: true,
};
let args = build_run_args(&spec, "bridge");
let joined = args.join(" ");
assert!(joined.contains("--network egress"));
assert!(joined.contains("-v /h/s.sock:/run/zakuro/space.sock"));
assert!(joined.contains("--cpus 2.5"));
assert!(joined.contains("--memory 8192m"));
assert!(joined.contains("--gpus 1"));
assert!(joined.contains("--pids-limit 4096"));
assert!(joined.contains("--security-opt no-new-privileges"));
assert!(joined.contains("--label a=b"));
assert!(joined.contains("-e K=V"));
assert_eq!(args.last().unwrap(), "img@sha256:ab");
let plain = RunSpec {
mounts: vec![],
limits: crate::sandbox::Limits::new(1.0, 1.0, crate::sandbox::Gpus::Count(0)),
network: None,
hardened: false,
..spec
};
let j = build_run_args(&plain, "bridge").join(" ");
assert!(j.contains("--network bridge"));
assert!(!j.contains("--gpus"));
assert!(
j.contains("--cpus 1 --memory 1024m --memory-swap 1024m"),
"{j}"
);
assert!(!j.contains(" -v "));
assert!(!j.contains("--pids-limit") && !j.contains("no-new-privileges"));
}
#[test]
fn container_name_is_unique_per_version() {
assert_eq!(container_name("dep_1", 3), "zk-dep-dep_1-v3");
assert_ne!(container_name("dep_1", 3), container_name("dep_1", 4));
}
#[test]
fn deployment_labels_carry_id_and_version() {
let labels = deployment_labels("dep_1", 3);
assert!(labels.contains(&("zakuro.deployment".to_string(), "dep_1".to_string())));
assert!(labels.contains(&("zakuro.version".to_string(), "3".to_string())));
}
fn hc(kind: &str, path: Option<&str>) -> Healthcheck {
Healthcheck {
kind: kind.to_string(),
port: 8000,
path: path.map(|s| s.to_string()),
timeout_s: 60,
}
}
#[test]
fn merge_logs_keeps_stderr_when_stdout_is_empty() {
assert_eq!(
merge_logs(b"", b"jupyter: bind failed\n"),
"jupyter: bind failed"
);
assert_eq!(merge_logs(b"out\n", b"err\n"), "out\nerr");
assert_eq!(merge_logs(b"", b""), "");
}
#[test]
fn connect_errors_read_as_plain_words() {
let refused = std::io::Error::from(std::io::ErrorKind::ConnectionRefused);
assert_eq!(describe_io_error(&refused), "connection refused");
let timeout = std::io::Error::from(std::io::ErrorKind::TimedOut);
assert_eq!(describe_io_error(&timeout), "timed out");
}
#[test]
fn health_http_url_defaults_path_to_root() {
assert_eq!(
health_http_url("172.17.0.5", 8000, &hc("http", None)),
"http://172.17.0.5:8000/"
);
}
#[test]
fn health_http_url_tolerates_a_path_without_leading_slash() {
assert_eq!(
health_http_url("172.17.0.5", 8000, &hc("http", Some("healthz"))),
"http://172.17.0.5:8000/healthz"
);
}
#[test]
fn health_http_url_keeps_an_already_leading_slash() {
assert_eq!(
health_http_url("172.17.0.5", 8000, &hc("http", Some("/api/health"))),
"http://172.17.0.5:8000/api/health"
);
}
#[test]
fn health_http_url_uses_the_container_ip_not_loopback() {
let url = health_http_url("10.13.13.10", 8000, &hc("http", None));
assert!(url.starts_with("http://10.13.13.10:"));
assert!(!url.contains("127.0.0.1"));
}
#[test]
fn health_http_url_dials_the_given_port_not_the_container_port() {
assert_eq!(
health_http_url("127.0.0.1", 49153, &hc("http", Some("/api"))),
"http://127.0.0.1:49153/api"
);
}
#[test]
fn loopback_publish_args_bind_each_port_once_on_loopback_only() {
assert_eq!(
loopback_publish_args(&[8888, 8000, 8888], "127.0.0.1"),
vec!["-p", "127.0.0.1::8888", "-p", "127.0.0.1::8000"]
);
assert!(loopback_publish_args(&[], "127.0.0.1").is_empty());
assert_eq!(
loopback_publish_args(&[80], "0.0.0.0"),
vec!["-p".to_string(), "0.0.0.0::80".to_string()]
);
}
#[test]
fn loopback_publish_goes_before_the_image_and_its_command() {
let env = HashMap::new();
let ports = [Port {
container: 8888,
protocol: "http".into(),
}];
let spec = RunSpec {
name: "n".into(),
image: "jupyter/base-notebook:latest",
cmd: Some("jupyter lab"),
ports: &ports,
env: &env,
labels: vec![],
mounts: vec![],
limits: crate::sandbox::Limits::new(1.0, 1.0, crate::sandbox::Gpus::None),
network: None,
hardened: false,
};
let mut args = build_run_args(&spec, "bridge");
add_loopback_publish(&mut args, &spec, false);
assert!(args
.join(" ")
.ends_with("-p 127.0.0.1::8888 jupyter/base-notebook:latest jupyter lab"));
}
#[test]
fn parse_published_port_prefers_the_ipv4_loopback_mapping() {
assert_eq!(
parse_published_port("[::1]:49154\n127.0.0.1:49153"),
Some(("127.0.0.1".to_string(), 49153))
);
assert_eq!(
parse_published_port("0.0.0.0:49153"),
Some(("127.0.0.1".to_string(), 49153))
);
assert_eq!(parse_published_port(""), None);
assert_eq!(parse_published_port("garbage"), None);
}
#[test]
fn publish_loopback_defaults_per_os_and_honours_the_override() {
assert!(publish_loopback_from(None, true));
assert!(!publish_loopback_from(None, false));
assert!(!publish_loopback_from(Some(" 0 "), true));
assert!(!publish_loopback_from(Some("false"), true));
assert!(publish_loopback_from(Some("1"), false));
assert!(publish_loopback_from(Some("true"), false));
assert!(publish_loopback_from(Some(""), true));
}
#[test]
fn isolated_spec_runs_on_the_isolated_network() {
let env = HashMap::new();
let spec = RunSpec {
name: "n".into(),
image: "img",
cmd: None,
ports: &[],
env: &env,
labels: vec![],
mounts: vec![],
limits: crate::sandbox::Limits::new(1.0, 1.0, crate::sandbox::Gpus::None),
network: Some("zc-isolated".into()),
hardened: false,
};
assert!(build_run_args(&spec, "bridge")
.join(" ")
.contains("--network zc-isolated"));
}
#[test]
fn deploy_network_defaults_to_bridge_when_unset() {
let _lock = crate::credentials::HOME_ENV_LOCK.lock();
let prev = std::env::var_os("ZAKURO_DEPLOY_NETWORK");
std::env::remove_var("ZAKURO_DEPLOY_NETWORK");
assert_eq!(deploy_network(), "bridge");
if let Some(v) = prev {
std::env::set_var("ZAKURO_DEPLOY_NETWORK", v);
}
}
#[test]
fn deploy_network_honours_the_env_override() {
let _lock = crate::credentials::HOME_ENV_LOCK.lock();
let prev = std::env::var_os("ZAKURO_DEPLOY_NETWORK");
std::env::set_var("ZAKURO_DEPLOY_NETWORK", "hubnet");
assert_eq!(deploy_network(), "hubnet");
match prev {
Some(v) => std::env::set_var("ZAKURO_DEPLOY_NETWORK", v),
None => std::env::remove_var("ZAKURO_DEPLOY_NETWORK"),
}
}
#[test]
fn deploy_network_blank_value_falls_back_to_bridge() {
let _lock = crate::credentials::HOME_ENV_LOCK.lock();
let prev = std::env::var_os("ZAKURO_DEPLOY_NETWORK");
std::env::set_var("ZAKURO_DEPLOY_NETWORK", " ");
assert_eq!(deploy_network(), "bridge");
match prev {
Some(v) => std::env::set_var("ZAKURO_DEPLOY_NETWORK", v),
None => std::env::remove_var("ZAKURO_DEPLOY_NETWORK"),
}
}
}
#[cfg(test)]
mod probe_tests {
use super::*;
fn facts(exists: bool, containerd: bool) -> HostFacts {
HostFacts {
socket_path: "/var/run/docker.sock".into(),
socket_exists: exists,
containerd_socket: containerd,
}
}
fn exited(stderr: &str) -> CliOutcome {
CliOutcome::Exited {
success: false,
stderr: stderr.into(),
}
}
fn probe(o: CliOutcome, f: HostFacts) -> RuntimeProbe {
classify_probe(o, &f, "t".into())
}
#[test]
fn available_when_info_succeeds() {
let p = probe(
CliOutcome::Exited {
success: true,
stderr: String::new(),
},
facts(true, false),
);
assert!(p.available && p.error.is_none() && p.runtime == Some("docker"));
assert!(p.next_step().is_none());
}
#[test]
fn binary_missing_is_not_installed() {
let p = probe(
CliOutcome::SpawnFailed(std::io::ErrorKind::NotFound, "nf".into()),
facts(false, false),
);
assert_eq!(p.error, Some(RuntimeError::NotInstalled));
assert_eq!(p.runtime, None);
}
#[test]
fn containerd_only_is_reported_as_containerd_not_installed() {
let p = probe(
CliOutcome::SpawnFailed(std::io::ErrorKind::NotFound, "nf".into()),
facts(false, true),
);
assert_eq!(p.error, Some(RuntimeError::NotInstalled));
assert_eq!(p.runtime, Some("containerd"));
assert!(!p.available);
}
#[test]
fn socket_missing() {
let p = probe(exited("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?"), facts(false, false));
assert_eq!(p.error, Some(RuntimeError::SocketMissing));
}
#[test]
fn permission_denied_names_socket_and_group_fix() {
let p = probe(exited("permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock"), facts(true, false));
assert_eq!(p.error, Some(RuntimeError::PermissionDenied));
let m = p.failure_message();
assert!(m.starts_with("no container runtime on this host"));
assert!(m.contains("permission denied on /var/run/docker.sock"));
assert!(m.contains("docker group"));
}
#[test]
fn daemon_down_when_socket_exists_but_refuses() {
let p = probe(exited("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?"), facts(true, false));
assert_eq!(p.error, Some(RuntimeError::DaemonDown));
}
#[test]
fn hung_daemon_is_daemon_down_with_timeout_detail() {
let p = probe(CliOutcome::TimedOut, facts(true, false));
assert!(!p.available);
assert_eq!(p.error, Some(RuntimeError::DaemonDown));
assert_eq!(p.detail, "docker version did not answer within 5 s");
assert!(p.next_step().is_some());
}
#[cfg(unix)]
#[test]
fn run_bounded_kills_a_hanging_process() {
let t = std::time::Instant::now();
let (o, _) = run_bounded(
Command::new("sleep").arg("30"),
std::time::Duration::from_millis(200),
);
assert!(matches!(o, CliOutcome::TimedOut));
assert!(t.elapsed() < std::time::Duration::from_secs(10));
}
#[cfg(unix)]
#[test]
fn run_bounded_returns_output_of_a_quick_process() {
let (o, out) = run_bounded(
Command::new("sh").args(["-c", "echo hi; echo oops >&2; exit 3"]),
std::time::Duration::from_secs(5),
);
assert_eq!(out.trim(), "hi");
assert!(matches!(o, CliOutcome::Exited { success: false, ref stderr } if stderr == "oops"));
}
#[test]
fn unknown_failure_is_other_with_first_line() {
let p = probe(exited("boom\nsecond"), facts(true, false));
assert_eq!(p.error, Some(RuntimeError::Other));
assert!(p.detail.contains("boom") && !p.detail.contains("second"));
}
#[test]
fn recovery_flips_to_available() {
let down = probe(
exited("Cannot connect to the Docker daemon"),
facts(true, false),
);
let up = probe(
CliOutcome::Exited {
success: true,
stderr: String::new(),
},
facts(true, false),
);
assert!(!down.available && up.available);
}
#[test]
fn doctor_report_prints_reason_and_next_step() {
let p = probe(exited("permission denied"), facts(true, false));
let r = doctor_report(&p);
assert!(
r.contains("UNAVAILABLE") && r.contains("permission_denied") && r.contains("next step")
);
}
}