use std::path::{Path, PathBuf};
use std::time::Duration;
use super::ephemeral::{
inner_limits, instance_name, parse_nvidia_smi, pick_gpus, Ephemeral, InstanceSpec, CFG_KIND,
IMG_PREFIX,
};
use super::job::{layer_dockerfile, layer_tag, JobHandle, JobSpec};
use super::Limits;
pub const KIND_ATELIER: &str = "atelier";
pub const DEFAULT_TAG_TTL: Duration = Duration::from_secs(24 * 3600);
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|a| a.to_string()).collect()
}
fn unix_now() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
pub fn tag_ttl(get: &dyn Fn(&str) -> Option<String>) -> Duration {
get("ZAKURO_SANDBOX_IMG_TTL_HOURS")
.and_then(|v| v.trim().parse::<f64>().ok())
.filter(|h| *h >= 0.0)
.map(|h| Duration::from_secs((h * 3600.0) as u64))
.unwrap_or(DEFAULT_TAG_TTL)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Plan {
pub alias: String,
pub base: String,
pub pinned_digest: Option<String>,
}
pub fn plan(spec: &JobSpec) -> Plan {
let pinned_digest = spec
.image
.split_once('@')
.map(|(_, d)| d.to_string())
.filter(|d| d.starts_with("sha256:"));
let layer = spec.layer.as_ref().map(|l| {
let mut apt = l.apt.clone();
apt.sort();
let mut pip = l.pip.clone();
pip.sort();
format!(
"{}|{}|{}|{}|{}",
l.venv,
apt.join(" "),
pip.join(" "),
spec.layer_repo.as_deref().unwrap_or("atelier-custom"),
spec.owner.as_deref().unwrap_or("local")
)
});
let material = format!("{}\n{}", spec.image, layer.unwrap_or_default());
Plan {
alias: format!(
"{IMG_PREFIX}{}",
&crate::dataset::sha256_hex(material.as_bytes())[..12]
),
base: spec.image.clone(),
pinned_digest,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Prepared {
pub alias: String,
pub image: String,
pub digest: String,
pub layer_tag: Option<String>,
pub cached: bool,
}
impl Prepared {
pub fn handle(&self, name: &str) -> JobHandle {
JobHandle {
name: name.to_string(),
image: self.image.clone(),
image_digest: self.digest.clone(),
layer_tag: self.layer_tag.clone(),
}
}
}
pub struct InstanceJobs {
eph: Ephemeral,
scratch: PathBuf,
nvidia_smi: Box<dyn Fn() -> Result<String, String> + Send + Sync>,
now: fn() -> u64,
ttl: Duration,
#[cfg(test)]
dockerfile: Option<String>,
#[cfg(test)]
shift: bool,
}
impl InstanceJobs {
pub fn new(eph: Ephemeral, scratch: PathBuf, ttl: Duration) -> Self {
Self {
eph,
scratch,
nvidia_smi: Box::new(super::ephemeral::real_nvidia_smi),
now: unix_now,
ttl,
#[cfg(test)]
dockerfile: None,
#[cfg(test)]
shift: true,
}
}
#[cfg(test)]
fn with(
mut self,
smi: Box<dyn Fn() -> Result<String, String> + Send + Sync>,
now: fn() -> u64,
) -> Self {
self.nvidia_smi = smi;
self.now = now;
self
}
pub fn ephemeral(&self) -> &Ephemeral {
&self.eph
}
pub fn ready(&self, spec: &JobSpec, plan: &Plan) -> Option<Prepared> {
if spec.push_to.is_some() {
return None;
}
self.cached(spec, plan)
}
fn cached(&self, spec: &JobSpec, plan: &Plan) -> Option<Prepared> {
let hit = self.eph.cached_image(&plan.alias, self.ttl, (self.now)())?;
if hit.digest.is_empty() {
return None;
}
let tag = Some(hit.tag.clone()).filter(|t| !t.is_empty() && spec.layer.is_some());
Some(Prepared {
alias: plan.alias.clone(),
image: tag.clone().unwrap_or_else(|| spec.image.clone()),
digest: hit.digest,
layer_tag: tag,
cached: true,
})
}
pub fn launch_builder(
&self,
spec: &JobSpec,
plan: &Plan,
limits: &Limits,
) -> Result<String, String> {
let mut l = limits.clone();
l.gpus = super::Gpus::None;
let from = self.cached(spec, plan).map(|p| p.alias);
let mut is = InstanceSpec::new(
&format!("{}-build", spec.name),
l,
self.eph.node_disk()?,
vec![],
vec![],
);
is.image = from;
is.persistent = true;
is.config = vec![(CFG_KIND.to_string(), KIND_ATELIER.to_string())];
self.eph.launch(&is)
}
fn nerdctl(&self, inst: &str, args: &[&str]) -> Result<String, String> {
self.eph.nerdctl(inst, &s(args))
}
fn present(&self, inst: &str, image: &str) -> bool {
self.nerdctl(inst, &["image", "inspect", "--format", "{{.Id}}", image])
.is_ok()
}
pub fn finish_prepare(
&self,
builder: &str,
spec: &JobSpec,
plan: &Plan,
) -> Result<Prepared, String> {
let r = self.prepare_in(builder, spec, plan);
let _ = self.eph.destroy(builder);
r
}
fn prepare_in(&self, inst: &str, spec: &JobSpec, plan: &Plan) -> Result<Prepared, String> {
let had = self
.eph
.cached_images()
.iter()
.any(|i| i.alias == plan.alias);
if let Some(auth) = &spec.push_to {
let f = self.scratch_file(&spec.name, "config.json")?;
let body =
super::ephemeral::docker_config_json(&auth.registry, &auth.username, &auth.token);
let pushed = write_private(&f, &body)
.map_err(|e| format!("registry config: {e}"))
.and_then(|_| self.eph.push_docker_config(inst, &f));
let _ = std::fs::remove_file(&f);
pushed?;
}
if !self.present(inst, &spec.image) {
self.nerdctl(inst, &["pull", "--quiet", &spec.image])
.map_err(|e| format!("pull {}: {e}", spec.image))?;
}
let out = self.nerdctl(
inst,
&[
"image",
"inspect",
"--format",
"{{json .RepoDigests}} {{.Id}}",
&spec.image,
],
)?;
let digest = super::job::digest_from_inspect(&out);
let (image, tag) = match &spec.layer {
None => (spec.image.clone(), None),
Some(layer) => {
let repo = spec.layer_repo.as_deref().unwrap_or("atelier-custom");
let owner = spec.owner.as_deref().unwrap_or("local");
let tag = layer_tag(repo, owner, &digest, layer);
if !self.present(inst, &tag) {
self.build(inst, spec, &spec.image, layer, &tag)?;
}
(tag.clone(), Some(tag))
}
};
if spec.push_to.is_some() {
if let Some(t) = &tag {
self.nerdctl(inst, &["push", "--quiet", t])
.map_err(|e| format!("push {t}: {e}"))?;
}
}
if !had {
let _ = self.eph.run(&[
"exec",
inst,
"--",
"rm",
"-rf",
"/root/.docker",
"/root/Dockerfile",
"/root/ctx",
]);
self.eph
.run(&["stop", inst])
.map_err(|e| format!("stopping {inst} failed: {e}"))?;
self.eph.publish_derived(
inst,
&plan.alias,
&digest,
tag.as_deref().unwrap_or(""),
plan.pinned_digest.is_some(),
(self.now)(),
)?;
}
Ok(Prepared {
alias: plan.alias.clone(),
image,
digest,
layer_tag: tag,
cached: false,
})
}
fn build(
&self,
inst: &str,
spec: &JobSpec,
base: &str,
layer: &super::job::Layer,
tag: &str,
) -> Result<(), String> {
let dockerfile = layer_dockerfile(base, layer)?;
#[cfg(test)]
let dockerfile = self
.dockerfile
.as_ref()
.map(|d| d.replace("{base}", base))
.unwrap_or(dockerfile);
let f = self.scratch_file(&spec.name, "Dockerfile")?;
let pushed = write_private(&f, &dockerfile)
.map_err(|e| format!("Dockerfile: {e}"))
.and_then(|_| {
self.eph.run(&[
"file",
"push",
"--create-dirs",
&f.display().to_string(),
&format!("{inst}/root/Dockerfile"),
])
});
let _ = std::fs::remove_file(&f);
pushed?;
self.eph
.run(&["exec", inst, "--", "mkdir", "-p", "/root/ctx"])?;
self.nerdctl(
inst,
&[
"build",
"--quiet",
"-t",
tag,
"-f",
"/root/Dockerfile",
"/root/ctx",
],
)
.map(|_| ())
.map_err(|e| format!("layer build failed: {}", last_lines(&e, 20)))
}
fn scratch_file(&self, job: &str, what: &str) -> Result<PathBuf, String> {
std::fs::create_dir_all(&self.scratch)
.map_err(|e| format!("{}: {e}", self.scratch.display()))?;
Ok(self.scratch.join(format!("{job}.{what}")))
}
pub fn run(
&self,
spec: &JobSpec,
prepared: &Prepared,
limits: &Limits,
user: Option<(u32, u32)>,
) -> Result<(), String> {
let picked = if limits.gpus.flag_value().is_some() {
let all = parse_nvidia_smi(&(self.nvidia_smi)()?);
pick_gpus(&limits.gpus, &all, &self.eph.taken_gpus()?)
.map_err(|e| format!("{}: {e}", spec.name))?
} else {
vec![]
};
let inner_dir = format!(
"/zakuro/mnt/{}",
&crate::dataset::sha256_hex(spec.workdir.display().to_string().as_bytes())[..16]
);
let mut is = InstanceSpec::new(
&spec.name,
limits.clone(),
self.eph.node_disk()?,
picked.iter().map(|g| g.pci.clone()).collect(),
vec![(spec.workdir.display().to_string(), inner_dir.clone())],
);
is.image = Some(prepared.alias.clone());
#[cfg(test)]
{
is.shift = self.shift;
}
is.config = vec![(CFG_KIND.to_string(), KIND_ATELIER.to_string())];
let inst = self.eph.launch(&is)?;
let mut inner_spec = spec.clone();
inner_spec.workdir = PathBuf::from(&inner_dir);
let inner = inner_limits(limits, !picked.is_empty());
let argv = super::job::start_argv(&inner_spec, &prepared.image, user, &inner);
match self.eph.nerdctl(&inst, &argv) {
Ok(_) => Ok(()),
Err(e) => {
let _ = self.eph.destroy(&inst);
Err(format!("start {}: {e}", spec.name))
}
}
}
pub fn inspect(&self, name: &str) -> Result<String, String> {
self.nerdctl(
&instance_name(name),
&[
"inspect",
"--format",
"{{.State.Status}} {{.State.ExitCode}}",
name,
],
)
}
pub fn logs(&self, name: &str) -> Result<String, String> {
self.nerdctl(&instance_name(name), &["logs", name])
}
pub fn stop(&self, name: &str) -> Result<(), String> {
self.eph.destroy(&instance_name(name))
}
pub fn reconcile(&self, known: &[String]) -> Result<Vec<String>, String> {
let mut removed = vec![];
for i in self.eph.list()? {
if i.kind.as_deref() != Some(KIND_ATELIER) {
continue;
}
let job = i.lease.clone().unwrap_or_default();
let owner = job.strip_suffix("-build").unwrap_or(&job);
if !known.iter().any(|k| k == owner) {
self.eph.destroy(&i.name)?;
removed.push(job);
}
}
Ok(removed)
}
pub fn exists(&self, name: &str) -> bool {
let inst = instance_name(name);
self.eph
.list()
.map(|l| l.iter().any(|i| i.name == inst))
.unwrap_or(false)
}
}
fn last_lines(text: &str, n: usize) -> String {
let lines: Vec<&str> = text.lines().collect();
lines[lines.len().saturating_sub(n)..].join("\n")
}
fn write_private(path: &Path, body: &str) -> std::io::Result<()> {
use std::io::Write;
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
let mut o = std::fs::OpenOptions::new();
o.write(true).create(true).truncate(true);
#[cfg(unix)]
o.mode(0o600);
o.open(path)?.write_all(body.as_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sandbox::ephemeral::provision_version;
use crate::sandbox::job::{Layer, RegistryAuth};
use crate::sandbox::lxc::{IncusCli, SandboxSize, DEFAULT_IMAGE};
use crate::sandbox::{Gpus, Limits};
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
#[derive(Default)]
struct Rec {
calls: Mutex<Vec<String>>,
answer: Mutex<Vec<(&'static str, Result<String, String>)>>,
}
impl Rec {
fn on(&self, needle: &'static str, a: Result<&str, &str>) {
self.answer
.lock()
.unwrap()
.push((needle, a.map(String::from).map_err(String::from)));
}
fn calls(&self) -> Vec<String> {
self.calls.lock().unwrap().clone()
}
}
impl IncusCli for Rec {
fn run(&self, argv: &[String]) -> Result<String, String> {
let j = argv.join(" ");
self.calls.lock().unwrap().push(j.clone());
for (needle, a) in self.answer.lock().unwrap().iter() {
if j.contains(needle) {
return a.clone();
}
}
Ok(String::new())
}
}
fn golden_ok(rec: &Rec) {
rec.on(
"image get-property zakuro-sandbox-base",
Ok(Box::leak(provision_version(DEFAULT_IMAGE).into_boxed_str())),
);
}
fn jobs(rec: &Arc<Rec>, dir: &Path) -> InstanceJobs {
let eph = Ephemeral::new(
rec.clone(),
DEFAULT_IMAGE,
Ok(SandboxSize {
limits: Limits::new(8.0, 16.0, Gpus::None),
disk_bytes: 20 << 30,
}),
);
let eph = {
let mut e = eph;
e.fast_for_tests();
e
};
InstanceJobs::new(eph, dir.to_path_buf(), DEFAULT_TAG_TTL)
.with(Box::new(|| Ok(String::new())), || 1_000_000)
}
fn spec(layer: bool) -> JobSpec {
JobSpec {
name: "atl-1".into(),
image: "docker.io/zakuroai/atelier-vision@sha256:aaaa".into(),
layer: layer.then(|| Layer {
apt: vec![],
pip: vec!["timm==1.0.15".into()],
venv: "/opt/v".into(),
}),
layer_repo: Some("r".into()),
owner: Some("4".into()),
push_to: None,
workdir: PathBuf::from("/var/lib/atelier/work/atl-1"),
mount_at: None,
prepare_only: false,
env: BTreeMap::new(),
gpus: String::new(),
cpus: None,
memory_gb: None,
cmd: s(&["python", "-m", "sakura.atelier", "run", "/work/spec.yaml"]),
}
}
fn lim() -> Limits {
Limits::new(2.0, 4.0, Gpus::None)
}
fn tmp(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("zc-jobinst-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
d
}
fn images_json(
alias: &str,
golden: &str,
digest: &str,
tag: &str,
pinned: bool,
at: u64,
) -> String {
format!(
r#"[{{"fingerprint":"ff","size":1000,"last_used_at":"2026-11-10T00:00:00Z",
"aliases":[{{"name":"{alias}"}}],
"properties":{{"zakuro.golden":"{golden}","zakuro.digest":"{digest}","zakuro.tag":"{tag}",
"zakuro.pinned":"{}","zakuro.cached_at":"{at}"}}}}]"#,
u8::from(pinned)
)
}
#[test]
fn the_cache_key_is_the_image_and_the_layer_whatever_the_package_order() {
let a = plan(&spec(true));
assert!(
a.alias.starts_with("zk-img-") && a.alias.len() == 7 + 12,
"{a:?}"
);
assert_eq!(a.pinned_digest.as_deref(), Some("sha256:aaaa"));
let mut b = spec(true);
b.layer.as_mut().unwrap().pip = vec!["timm==1.0.15".into()];
assert_eq!(plan(&b).alias, a.alias);
b.layer.as_mut().unwrap().pip.push("numpy==2.0.0".into());
assert_ne!(plan(&b).alias, a.alias, "another layer is another image");
assert_ne!(
plan(&spec(false)).alias,
a.alias,
"no layer is another image"
);
let mut c = spec(false);
c.image = "docker.io/zakuroai/atelier-vision:abc-cpu".into();
assert_eq!(plan(&c).pinned_digest, None, "a tag is not a pin");
let mut d = spec(true);
d.owner = Some("5".into());
assert_ne!(plan(&d).alias, a.alias, "the owner is in the layer tag");
}
#[test]
fn a_cached_image_is_a_hit_that_pulls_nothing_and_a_stale_one_is_not() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let p = plan(&spec(true));
let ver = provision_version(DEFAULT_IMAGE);
rec.on(
"image list -f json",
Ok(Box::leak(
images_json(&p.alias, &ver, "sha256:aaaa", "r:4-1234", true, 5).into_boxed_str(),
)),
);
let j = jobs(&rec, &tmp("hit"));
let got = j.ready(&spec(true), &p).expect("hit");
assert!(got.cached);
assert_eq!(got.image, "r:4-1234");
assert_eq!(got.layer_tag.as_deref(), Some("r:4-1234"));
assert_eq!(got.digest, "sha256:aaaa");
assert!(!rec.calls().iter().any(|c| c.contains("pull")));
let mut push = spec(true);
push.push_to = Some(RegistryAuth {
registry: "docker.io".into(),
username: "u".into(),
token: "t".into(),
});
assert!(j.ready(&push, &p).is_none());
let old = Arc::new(Rec::default());
golden_ok(&old);
old.on(
"image list -f json",
Ok(Box::leak(
images_json(&p.alias, "0123456789ab", "sha256:aaaa", "", true, 5).into_boxed_str(),
)),
);
assert!(jobs(&old, &tmp("old")).ready(&spec(false), &p).is_none());
assert!(old
.calls()
.iter()
.any(|c| c == &format!("incus image delete {}", p.alias)));
let tagged = Arc::new(Rec::default());
golden_ok(&tagged);
tagged.on(
"image list -f json",
Ok(Box::leak(
images_json(&p.alias, &ver, "sha256:b", "", false, 5).into_boxed_str(),
)),
);
let jt = InstanceJobs::new(
jobs(&tagged, &tmp("tag")).eph.clone(),
tmp("tag"),
Duration::from_secs(60),
)
.with(Box::new(|| Ok(String::new())), || 1_000_000);
assert!(
jt.ready(&spec(false), &p).is_none(),
"tag-built image past its TTL"
);
}
#[test]
fn prepare_pulls_builds_publishes_the_derived_image_and_deletes_the_builder() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("image list -f json", Ok("[]"));
rec.on(
"image inspect --format {{.Id}} docker.io",
Err("no such image"),
);
rec.on("image inspect --format {{.Id}} r:4-", Err("no such image"));
rec.on(
"{{json .RepoDigests}}",
Ok(r#"["docker.io/zakuroai/atelier-vision@sha256:aaaa"] sha256:id"#),
);
let dir = tmp("prep");
let j = jobs(&rec, &dir);
let sp = spec(true);
let p = plan(&sp);
let builder = j.launch_builder(&sp, &p, &lim()).unwrap();
let got = j.finish_prepare(&builder, &sp, &p).unwrap();
assert!(!got.cached);
assert_eq!(got.digest, "sha256:aaaa");
let tag = got.layer_tag.clone().unwrap();
assert!(tag.starts_with("r:4-"), "{tag}");
let calls = rec.calls();
let pos = |needle: &str| {
calls
.iter()
.position(|c| c.contains(needle))
.unwrap_or_else(|| panic!("{needle} missing in {calls:#?}"))
};
let init = &calls[pos("incus init zakuro-sandbox-base")];
assert!(!init.contains("--ephemeral"), "{init}");
assert!(init.contains("-c user.zakuro.kind=atelier"), "{init}");
assert!(init.contains("-c user.zakuro.lease=atl-1-build"), "{init}");
assert!(pos("nerdctl pull --quiet docker.io/zakuroai") < pos("nerdctl build --quiet -t"));
assert!(calls[pos("nerdctl build")].contains("-f /root/Dockerfile /root/ctx"));
assert!(pos("file push") < pos("nerdctl build"));
assert!(pos("nerdctl build") < pos("incus stop"));
let publish = &calls[pos("incus publish")];
assert!(
publish.starts_with(&format!("incus publish {builder} --alias {}", p.alias)),
"{publish}"
);
assert!(publish.contains("zakuro.digest=sha256:aaaa"));
assert!(publish.contains(&format!("zakuro.tag={tag}")));
assert!(publish.contains("zakuro.pinned=1"));
assert!(publish.contains("zakuro.cached_at=1000000"));
assert!(pos("incus publish") < calls.len() - 1);
assert_eq!(calls.last().unwrap(), &format!("incus delete -f {builder}"));
assert!(
!dir.join("atl-1.Dockerfile").exists(),
"no Dockerfile left on the host"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_failed_pull_or_build_deletes_the_builder_and_publishes_nothing() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("image list -f json", Ok("[]"));
rec.on("image inspect", Err("no such image"));
rec.on("nerdctl pull", Err("manifest unknown"));
let j = jobs(&rec, &tmp("fail"));
let sp = spec(false);
let p = plan(&sp);
let b = j.launch_builder(&sp, &p, &lim()).unwrap();
let e = j.finish_prepare(&b, &sp, &p).unwrap_err();
assert!(e.contains("pull") && e.contains("manifest unknown"), "{e}");
assert!(!rec.calls().iter().any(|c| c.starts_with("incus publish")));
assert_eq!(rec.calls().last().unwrap(), &format!("incus delete -f {b}"));
}
#[test]
fn a_push_logs_in_with_a_pushed_config_never_an_argv() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("image list -f json", Ok("[]"));
rec.on("image inspect --format {{.Id}}", Err("no such image"));
rec.on("{{json .RepoDigests}}", Ok(r#"["x@sha256:aaaa"] sha256:i"#));
let dir = tmp("push");
let j = jobs(&rec, &dir);
let mut sp = spec(true);
sp.push_to = Some(RegistryAuth {
registry: "docker.io".into(),
username: "bot".into(),
token: "s3cret-token".into(),
});
let p = plan(&sp);
let b = j.launch_builder(&sp, &p, &lim()).unwrap();
j.finish_prepare(&b, &sp, &p).unwrap();
let calls = rec.calls();
assert!(
!calls.iter().any(|c| c.contains("s3cret-token")),
"{calls:#?}"
);
assert!(calls
.iter()
.any(|c| c.contains(&format!("{b}/root/.docker/config.json"))));
let push = calls
.iter()
.position(|c| c.contains("nerdctl push --quiet r:4-"))
.unwrap();
let build = calls
.iter()
.position(|c| c.contains("nerdctl build"))
.unwrap();
assert!(build < push);
let rm = calls
.iter()
.position(|c| c.contains("rm -rf /root/.docker"))
.unwrap();
let publish = calls
.iter()
.position(|c| c.starts_with("incus publish"))
.unwrap();
assert!(rm < publish, "the login is not part of the published image");
assert!(!dir.join("atl-1.config.json").exists());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_job_runs_in_its_own_instance_from_the_derived_image_with_the_workdir_as_a_disk() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let j = jobs(&rec, &tmp("run"));
let sp = spec(true);
let prep = Prepared {
alias: plan(&sp).alias,
image: "r:4-1234".into(),
digest: "sha256:aaaa".into(),
layer_tag: Some("r:4-1234".into()),
cached: true,
};
j.run(&sp, &prep, &lim(), Some((1000, 1000))).unwrap();
let name = instance_name("atl-1");
let calls = rec.calls();
let init = calls.iter().find(|c| c.starts_with("incus init")).unwrap();
assert!(
init.starts_with(&format!("incus init {} {name} --ephemeral", prep.alias)),
"{init}"
);
assert!(init.contains("-c user.zakuro.kind=atelier"));
assert!(init.contains("-c limits.cpu=2"));
assert!(init.contains("-d root,size=20480MiB"), "{init}");
let mnt = calls.iter().find(|c| c.contains("zakuro-mnt-0")).unwrap();
assert!(
mnt.contains("source=/var/lib/atelier/work/atl-1 path=/zakuro/mnt/"),
"{mnt}"
);
assert!(mnt.contains("shift=true"));
let egress = calls
.iter()
.position(|c| c.contains("ZAKURO-EGRESS"))
.unwrap();
let run = calls
.iter()
.position(|c| {
c.starts_with(&format!("incus exec {name} -- nerdctl run -d --name atl-1"))
})
.unwrap_or_else(|| panic!("{calls:#?}"));
assert!(egress < run);
let r = &calls[run];
assert!(r.contains("--user 1000:1000"));
assert!(r.contains("--memory "), "{r}");
assert!(
r.contains(" r:4-1234 python -m sakura.atelier run /work/spec.yaml"),
"{r}"
);
assert!(
r.contains(":/work"),
"the container sees the dir at the job's mount: {r}"
);
assert!(
!calls.iter().any(|c| c.contains("nerdctl pull")),
"the image is already inside"
);
let _ = std::fs::remove_dir_all(tmp("run"));
}
#[test]
fn a_start_that_fails_inside_deletes_the_instance() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("nerdctl run -d", Err("no such image"));
let j = jobs(&rec, &tmp("runfail"));
let sp = spec(false);
let prep = Prepared {
alias: plan(&sp).alias,
image: sp.image.clone(),
digest: "sha256:aaaa".into(),
layer_tag: None,
cached: true,
};
let e = j.run(&sp, &prep, &lim(), None).unwrap_err();
assert!(e.contains("no such image"), "{e}");
assert_eq!(
rec.calls().last().unwrap(),
&format!("incus delete -f {}", instance_name("atl-1"))
);
}
#[test]
fn a_gpu_job_gets_a_free_card_and_none_when_all_are_held() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
let eph = jobs(&rec, &tmp("gpu")).eph.clone();
let j = InstanceJobs::new(eph, tmp("gpu"), DEFAULT_TAG_TTL).with(
Box::new(|| Ok("0, GPU-aaa, 00000000:01:00.0\n".to_string())),
|| 1,
);
let sp = spec(false);
let prep = Prepared {
alias: plan(&sp).alias,
image: sp.image.clone(),
digest: "sha256:aaaa".into(),
layer_tag: None,
cached: true,
};
let mut l = lim();
l.gpus = Gpus::All;
j.run(&sp, &prep, &l, None).unwrap();
let calls = rec.calls();
assert!(
calls
.iter()
.any(|c| c.contains("gputype=physical pci=0000:01:00.0")),
"{calls:#?}"
);
assert!(calls.iter().any(|c| c.contains("--gpus all")), "{calls:#?}");
let held = Arc::new(Rec::default());
golden_ok(&held);
held.on(
"list zk-job-",
Ok(r#"[{"name":"zk-job-aaaaaaaa","status":"Running","config":{"user.zakuro.gpus":"0000:01:00.0"}}]"#),
);
let eph = jobs(&held, &tmp("gpu2")).eph.clone();
let j = InstanceJobs::new(eph, tmp("gpu2"), DEFAULT_TAG_TTL).with(
Box::new(|| Ok("0, GPU-aaa, 00000000:01:00.0\n".to_string())),
|| 1,
);
let e = j.run(&sp, &prep, &l, None).unwrap_err();
assert!(e.contains("free"), "{e}");
assert!(!held.calls().iter().any(|c| c.starts_with("incus init")));
}
#[test]
fn inspect_logs_and_stop_address_the_jobs_instance() {
let rec = Arc::new(Rec::default());
let j = jobs(&rec, &tmp("ops"));
j.inspect("atl-1").unwrap();
j.logs("atl-1").unwrap();
j.stop("atl-1").unwrap();
let name = instance_name("atl-1");
let calls = rec.calls();
assert_eq!(
calls[0],
format!("incus exec {name} -- nerdctl inspect --format {{{{.State.Status}}}} {{{{.State.ExitCode}}}} atl-1")
);
assert_eq!(calls[1], format!("incus exec {name} -- nerdctl logs atl-1"));
assert_eq!(calls[2], format!("incus delete -f {name}"));
}
#[test]
fn reconcile_removes_only_atelier_instances_no_job_owns() {
let rec = Arc::new(Rec::default());
rec.on(
"list zk-job-",
Ok(r#"[
{"name":"zk-job-11111111","status":"Running","config":{"user.zakuro.kind":"atelier","user.zakuro.lease":"keep"}},
{"name":"zk-job-22222222","status":"Running","config":{"user.zakuro.kind":"atelier","user.zakuro.lease":"gone"}},
{"name":"zk-job-33333333","status":"Running","config":{"user.zakuro.kind":"atelier","user.zakuro.lease":"keep-build"}},
{"name":"zk-job-44444444","status":"Running","config":{"user.zakuro.lease":"zk-space-x"}}
]"#),
);
let j = jobs(&rec, &tmp("rec"));
let gone = j.reconcile(&["keep".to_string()]).unwrap();
assert_eq!(gone, vec!["gone".to_string()]);
let calls = rec.calls();
assert!(calls.contains(&"incus delete -f zk-job-22222222".to_string()));
assert!(
!calls
.iter()
.any(|c| c.contains("zk-job-44444444") && c.contains("delete")),
"a Space lease is not an Atelier job"
);
assert!(!calls
.iter()
.any(|c| c.contains("zk-job-11111111") && c.contains("delete")));
assert!(!calls
.iter()
.any(|c| c.contains("zk-job-33333333") && c.contains("delete")));
}
struct NoCli;
impl crate::sandbox::job::JobCli for NoCli {
fn run(&self, argv: &[String]) -> Result<String, String> {
panic!("the shared tier was touched: {argv:?}")
}
fn run_stdin(&self, argv: &[String], _: &str) -> Result<String, String> {
panic!("the shared tier was touched: {argv:?}")
}
}
fn runtime(j: InstanceJobs, budget: Limits) -> crate::sandbox::job::JobRuntime<NoCli> {
let j = Arc::new(j);
crate::sandbox::job::JobRuntime::new(
NoCli,
PathBuf::from("/nonexistent"),
crate::sandbox::job::Admission {
budget: Box::new(move || Ok(budget.clone())),
allocations: Box::new(|| Ok(vec![])),
lock_path: None,
},
)
.with_tier(Box::new(move || Some(j.clone())))
}
#[test]
fn the_runtime_prepares_once_and_a_second_job_hits_the_cache_without_a_pull() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("image list -f json", Ok("[]"));
rec.on("image inspect --format {{.Id}}", Err("no such image"));
rec.on("{{json .RepoDigests}}", Ok(r#"["x@sha256:aaaa"] sha256:i"#));
let rt = runtime(jobs(&rec, &tmp("rt1")), Limits::new(4.0, 8.0, Gpus::None));
let mut sp = spec(true);
sp.prepare_only = true;
let h = rt.start(&sp).unwrap();
assert!(h.layer_tag.as_deref().unwrap().starts_with("r:4-"));
assert_eq!(h.image_digest, "sha256:aaaa");
let calls = rec.calls();
assert!(calls.iter().any(|c| c.starts_with("incus publish")));
assert!(
!calls
.iter()
.any(|c| c.contains(&instance_name("atl-1")) && c.starts_with("incus init")),
"prepare-only runs nothing"
);
let p = plan(&spec(true));
let ver = provision_version(DEFAULT_IMAGE);
let rec2 = Arc::new(Rec::default());
golden_ok(&rec2);
rec2.on(
"image list -f json",
Ok(Box::leak(
images_json(
&p.alias,
&ver,
"sha256:aaaa",
&h.layer_tag.clone().unwrap(),
true,
5,
)
.into_boxed_str(),
)),
);
let rt2 = runtime(jobs(&rec2, &tmp("rt2")), Limits::new(4.0, 8.0, Gpus::None));
let h2 = rt2.start(&spec(true)).unwrap();
assert_eq!(h2.image, h.image, "same derived image");
let calls = rec2.calls();
assert!(
!calls.iter().any(|c| c.contains("pull")
|| c.contains("nerdctl build")
|| c.starts_with("incus publish")),
"{calls:#?}"
);
assert!(
!calls.iter().any(|c| c.contains("-build")),
"no builder on a hit"
);
let init = calls.iter().find(|c| c.starts_with("incus init")).unwrap();
assert!(
init.starts_with(&format!(
"incus init {} {} --ephemeral",
p.alias,
instance_name("atl-1")
)),
"{init}"
);
assert!(calls
.iter()
.any(|c| c.contains("nerdctl run -d --name atl-1")));
}
#[test]
fn a_job_past_the_budget_is_refused_before_any_instance_exists() {
let rec = Arc::new(Rec::default());
golden_ok(&rec);
rec.on("image list -f json", Ok("[]"));
let mut sp = spec(false);
sp.cpus = Some(64.0);
sp.gpus = "1".into();
let rt = runtime(jobs(&rec, &tmp("rt3")), Limits::new(4.0, 8.0, Gpus::None));
let e = rt.start(&sp).unwrap_err();
assert!(e.starts_with("budget:"), "{e}");
assert!(
!rec.calls().iter().any(|c| c.starts_with("incus init")),
"{:#?}",
rec.calls()
);
}
#[test]
fn status_logs_and_stop_of_a_per_instance_job_go_to_its_instance() {
let rec = Arc::new(Rec::default());
let name = instance_name("atl-1");
rec.on(
"list zk-job-",
Ok(Box::leak(
format!(r#"[{{"name":"{name}","status":"Running","config":{{}}}}]"#)
.into_boxed_str(),
)),
);
rec.on("nerdctl inspect", Ok("exited 3"));
rec.on("nerdctl logs", Ok("a\nb"));
let rt = runtime(jobs(&rec, &tmp("rt4")), Limits::new(4.0, 8.0, Gpus::None));
assert_eq!(rt.inspect("atl-1").unwrap(), (false, Some(3)));
assert_eq!(rt.logs("atl-1", 1).unwrap(), (vec!["b".to_string()], 2));
rt.stop("atl-1").unwrap();
assert!(rec.calls().contains(&format!("incus delete -f {name}")));
assert!(rt.inspect("nope").is_err());
}
#[test]
#[ignore]
fn real_incus_atelier_job_in_its_own_instance() {
if std::env::var("ZAKURO_TEST_REAL_INCUS").ok().as_deref() != Some("1") {
eprintln!("skipped: set ZAKURO_TEST_REAL_INCUS=1");
return;
}
struct Spy(Mutex<Vec<String>>);
impl IncusCli for Spy {
fn run(&self, argv: &[String]) -> Result<String, String> {
self.0.lock().unwrap().push(argv.join(" "));
crate::sandbox::lxc::IncusShell.run(argv)
}
}
let spy = Arc::new(Spy(Mutex::new(vec![])));
let image = std::env::var("ZAKURO_SANDBOX_IMAGE").unwrap_or_else(|_| {
crate::sandbox::lxc::default_image(crate::sandbox::lxc::cli_binary()).to_string()
});
let eph = Ephemeral::new(
spy.clone(),
&image,
Ok(SandboxSize {
limits: Limits::new(2.0, 4.0, Gpus::None),
disk_bytes: 12 << 30,
}),
);
let t = std::time::Instant::now();
eph.ensure_golden().expect("golden");
println!("REAL golden ready in {:?}", t.elapsed());
let pid = std::process::id();
let root = std::env::temp_dir().join(format!("zc-real-atelier-{pid}"));
let _ = std::fs::remove_dir_all(&root);
let mut ij = InstanceJobs::new(eph.clone(), root.join("scratch"), DEFAULT_TAG_TTL);
ij.shift = std::env::var("ZAKURO_TEST_NO_IDMAP").ok().as_deref() != Some("1");
ij.dockerfile = Some("FROM {base}\nRUN echo layered-by-buildkit > /layer.txt\n".into());
let ij = Arc::new(ij);
let ij2 = ij.clone();
let eph2 = eph.clone();
let rt = crate::sandbox::job::JobRuntime::new(
NoCli,
root.join("docker"),
crate::sandbox::job::Admission {
budget: Box::new(|| Ok(Limits::new(2.0, 4.0, Gpus::None))),
allocations: Box::new(move || eph2.allocations()),
lock_path: None,
},
)
.with_tier(Box::new(move || Some(ij2.clone())));
let job = |n: &str| {
let work = root.join(n);
std::fs::create_dir_all(&work).unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&work, std::fs::Permissions::from_mode(0o777)).unwrap();
}
let mut sp = spec(true);
sp.name = format!("real-{n}-{pid}");
sp.image = "docker.io/library/busybox:latest".into();
sp.layer.as_mut().unwrap().pip.clear();
sp.layer.as_mut().unwrap().venv = "/opt/none".into();
sp.workdir = work.clone();
sp.cmd = s(&[
"sh",
"-c",
"cat /layer.txt > /work/out.txt; echo ran-$(hostname) >> /work/out.txt",
]);
(sp, work)
};
let alias = plan(&job("p").0).alias;
let _ = eph.run(&["image", "delete", &alias]);
let wait_exit = |name: &str| {
for _ in 0..60 {
if let Ok((false, code)) = rt.inspect(name) {
return code;
}
std::thread::sleep(Duration::from_secs(1));
}
panic!("job {name} did not finish");
};
let (sp, work) = job("a");
let t = std::time::Instant::now();
let h = rt.start(&sp).expect("start (miss)");
println!(
"REAL job {} started in {:?} (image {})",
sp.name,
t.elapsed(),
h.image
);
assert_eq!(wait_exit(&sp.name), Some(0));
let out = std::fs::read_to_string(work.join("out.txt")).expect("output on the host");
println!("REAL output on host: {}", out.replace('\n', " | "));
assert!(out.contains("layered-by-buildkit"));
let (logs, _) = rt.logs(&sp.name, 0).unwrap();
println!("REAL logs readable: {} line(s)", logs.len());
assert!(eph.cached_images().iter().any(|i| i.alias == alias));
println!("REAL derived image {alias} cached");
rt.stop(&sp.name).unwrap();
let live: Vec<String> = eph.list().unwrap().into_iter().map(|i| i.name).collect();
println!("REAL instances after job 1: {live:?}");
assert!(live.is_empty(), "no instance left: {live:?}");
let before = spy.0.lock().unwrap().len();
let (sp2, work2) = job("b");
let t = std::time::Instant::now();
rt.start(&sp2).expect("start (hit)");
println!(
"REAL job {} started in {:?} (cache hit)",
sp2.name,
t.elapsed()
);
assert_eq!(wait_exit(&sp2.name), Some(0));
assert!(std::fs::read_to_string(work2.join("out.txt"))
.unwrap()
.contains("layered-by-buildkit"));
let calls: Vec<String> = spy.0.lock().unwrap()[before..].to_vec();
let pulled = calls.iter().any(|c| {
c.contains("nerdctl pull")
|| c.contains("nerdctl build")
|| c.starts_with("incus publish")
});
println!("REAL second run pulled/built/published: {pulled}");
assert!(!pulled, "{calls:#?}");
rt.stop(&sp2.name).unwrap();
assert!(eph.list().unwrap().is_empty());
println!("REAL instances after job 2: none");
let _ = eph.run(&["image", "delete", &alias]);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn the_tag_ttl_is_tunable() {
assert_eq!(tag_ttl(&|_| None), DEFAULT_TAG_TTL);
assert_eq!(
tag_ttl(&|k| (k == "ZAKURO_SANDBOX_IMG_TTL_HOURS").then(|| "2".into())),
Duration::from_secs(7200)
);
}
}