use std::collections::{BTreeMap, HashMap};
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::{Arc, Mutex};
use serde::{Deserialize, Serialize};
use super::{docker, Allocation, Gpus, Limits, Request};
pub const JOB_LABEL: &str = "ai.zakuro.atelier.job";
pub const WORK_MOUNT: &str = "/work";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
pub struct Layer {
#[serde(default)]
pub apt: Vec<String>,
#[serde(default)]
pub pip: Vec<String>,
pub venv: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RegistryAuth {
pub registry: String,
pub username: String,
pub token: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct JobSpec {
pub name: String,
pub image: String,
#[serde(default)]
pub layer: Option<Layer>,
#[serde(default)]
pub layer_repo: Option<String>,
#[serde(default)]
pub owner: Option<String>,
#[serde(default)]
pub push_to: Option<RegistryAuth>,
pub workdir: PathBuf,
#[serde(default)]
pub mount_at: Option<String>,
#[serde(default)]
pub prepare_only: bool,
#[serde(default)]
pub env: BTreeMap<String, String>,
#[serde(default)]
pub gpus: String,
#[serde(default)]
pub cpus: Option<f64>,
#[serde(default)]
pub memory_gb: Option<f64>,
pub cmd: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobHandle {
pub name: String,
pub image: String,
pub image_digest: String,
pub layer_tag: Option<String>,
}
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|a| a.to_string()).collect()
}
fn pip_ok(p: &str) -> bool {
let Some((name, ver)) = p.split_once("==") else {
return false;
};
let name_ok = !name.is_empty()
&& name
.chars()
.next()
.is_some_and(|c| c.is_ascii_alphanumeric())
&& name
.chars()
.all(|c| c.is_ascii_alphanumeric() || "._-[],".contains(c));
let ver_ok = !ver.is_empty()
&& ver
.chars()
.all(|c| c.is_ascii_alphanumeric() || ".+!_-".contains(c));
name_ok && ver_ok
}
fn apt_ok(p: &str) -> bool {
let Some((name, ver)) = p.split_once('=') else {
return false;
};
!name.is_empty()
&& name
.chars()
.next()
.is_some_and(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
&& name
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "+.-".contains(c))
&& !ver.is_empty()
&& ver
.chars()
.all(|c| c.is_ascii_alphanumeric() || ".+:~-".contains(c))
}
pub fn validate_layer(layer: &Layer) -> Result<(), String> {
if !layer.venv.starts_with('/') || layer.venv.contains(char::is_whitespace) {
return Err(format!(
"layer venv must be an absolute path, got {:?}",
layer.venv
));
}
for p in &layer.pip {
if !pip_ok(p) {
return Err(format!(
"pip package {p:?} is not an exact name==version pin"
));
}
}
for p in &layer.apt {
if !apt_ok(p) {
return Err(format!(
"apt package {p:?} is not an exact name=version pin"
));
}
}
Ok(())
}
pub fn layer_dockerfile(base: &str, layer: &Layer) -> Result<String, String> {
validate_layer(layer)?;
let mut f = format!("FROM {base}\n");
if !layer.apt.is_empty() {
f.push_str(&format!(
"RUN apt-get update && apt-get install -y --no-install-recommends {} && rm -rf /var/lib/apt/lists/*\n",
layer.apt.join(" ")
));
}
if !layer.pip.is_empty() {
f.push_str(&format!(
"RUN uv pip install --python {}/bin/python {}\n",
layer.venv,
layer.pip.join(" ")
));
}
Ok(f)
}
pub fn layer_tag(repo: &str, owner: &str, base_digest: &str, layer: &Layer) -> String {
let mut apt = layer.apt.clone();
apt.sort();
let mut pip = layer.pip.clone();
pip.sort();
let material = format!(
"{base_digest}\n{}\n{}\n{}",
layer.venv,
apt.join(" "),
pip.join(" ")
);
let d = ring::digest::digest(&ring::digest::SHA256, material.as_bytes());
format!("{repo}:{owner}-{}", &hex::encode(d.as_ref())[..16])
}
#[cfg(unix)]
pub fn workdir_owner(workdir: &std::path::Path) -> Option<(u32, u32)> {
use std::os::unix::fs::MetadataExt;
let m = std::fs::metadata(workdir).ok()?;
(m.uid() != 0).then(|| (m.uid(), m.gid()))
}
#[cfg(not(unix))]
pub fn workdir_owner(_workdir: &std::path::Path) -> Option<(u32, u32)> {
None
}
pub fn start_argv(
spec: &JobSpec,
image: &str,
user: Option<(u32, u32)>,
limits: &Limits,
) -> Vec<String> {
let mut a = s(&["run", "-d", "--name"]);
a.push(spec.name.clone());
a.push("--label".into());
a.push(format!("{JOB_LABEL}={}", spec.name));
a.push("--label".into());
a.push(format!("{}={}", super::BUDGET_LABEL, limits.budget_label()));
a.push("-v".into());
let at = spec.mount_at.as_deref().unwrap_or(WORK_MOUNT);
a.push(format!("{}:{at}", spec.workdir.display()));
a.push("-w".into());
a.push(at.into());
a.extend(s(&[
"--security-opt",
"no-new-privileges",
"--pids-limit",
"4096",
]));
if let Some((uid, gid)) = user {
a.push("--user".into());
a.push(format!("{uid}:{gid}"));
if !spec.env.contains_key("HOME") {
a.push("-e".into());
a.push("HOME=/tmp".into());
}
}
a.extend(docker::limit_args(limits));
for (k, v) in &spec.env {
a.push("-e".into());
a.push(format!("{k}={v}"));
}
a.push(image.to_string());
a.extend(spec.cmd.iter().cloned());
a
}
pub trait JobCli: Send + Sync {
fn run(&self, argv: &[String]) -> Result<String, String>;
fn run_stdin(&self, argv: &[String], input: &str) -> Result<String, String>;
}
pub struct DockerJobCli;
impl JobCli for DockerJobCli {
fn run(&self, argv: &[String]) -> Result<String, String> {
self.run_stdin(argv, "")
}
fn run_stdin(&self, argv: &[String], input: &str) -> Result<String, String> {
use std::io::Write;
let mut child = super::engine::command(argv)?
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| format!("container runtime not available: {e}"))?;
if let Some(mut stdin) = child.stdin.take() {
let _ = stdin.write_all(input.as_bytes());
}
let out = child.wait_with_output().map_err(|e| e.to_string())?;
if out.status.success() {
Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
} else {
Err(String::from_utf8_lossy(&out.stderr).trim().to_string())
}
}
}
pub struct Admission {
pub budget: Box<dyn Fn() -> Result<Limits, String> + Send + Sync>,
pub allocations: Box<dyn Fn() -> Result<Vec<Allocation>, String> + Send + Sync>,
pub lock_path: Option<PathBuf>,
}
impl Admission {
pub fn node() -> Self {
Self {
budget: Box::new(super::node_limits),
allocations: Box::new(super::engine::allocations),
lock_path: super::lock::default_lock_path(),
}
}
fn lock(&self) -> super::AdmissionGuard {
super::lock::lock_at(self.lock_path.as_deref(), super::lock::FILE_LOCK_WAIT)
.or_else(|e| {
eprintln!(" [SANDBOX] {e}; admitting under the in-process lock only");
super::lock::lock_at(None, super::lock::FILE_LOCK_WAIT)
})
.expect("the in-process lock cannot fail")
}
fn limits_for(&self, spec: &JobSpec) -> Result<Limits, String> {
let budget = (self.budget)()?;
let running =
(self.allocations)().map_err(|e| format!("cannot read the node's allocations: {e}"))?;
let mut l = super::lease_limits(
&budget,
&running,
&Request {
cpus: spec.cpus.unwrap_or(0.0),
memory_gib: spec.memory_gb.unwrap_or(0.0),
gpus: None,
},
);
l.gpus =
job_gpus(&spec.gpus, &budget.gpus).map_err(|e| format!("job {}: {e}", spec.name))?;
super::admit(&budget, &running, &l)?;
Ok(l)
}
}
pub fn job_gpus(ask: &str, budget: &Gpus) -> Result<Gpus, String> {
let ask = ask.trim().trim_matches('"');
if ask.is_empty() || ask == "0" {
return Ok(Gpus::None);
}
if matches!(budget, Gpus::None | Gpus::Count(0)) {
return Err("asks for a GPU but this node's budget passes none through \
(the owner opts in with ZAKURO_BUDGET_GPU=1)"
.to_string());
}
Ok(match ask.parse::<u32>() {
Ok(n) => {
Limits::new(1.0, 1.0, budget.clone())
.clamp_request(&Request {
gpus: Some(n),
..Request::default()
})
.gpus
}
Err(_) => budget.clone(),
})
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum JobState {
Pulling,
Building,
Running,
Exited,
Failed,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct JobStatus {
pub state: JobState,
pub exit_code: Option<i32>,
pub image_digest: Option<String>,
pub layer_tag: Option<String>,
pub error: Option<String>,
}
pub struct JobRuntime<C: JobCli> {
cli: C,
config_dir: PathBuf,
admission: Admission,
}
impl<C: JobCli> JobRuntime<C> {
pub fn new(cli: C, config_dir: PathBuf, admission: Admission) -> Self {
Self {
cli,
config_dir,
admission,
}
}
fn cfg(&self) -> Vec<String> {
vec!["--config".into(), self.config_dir.display().to_string()]
}
fn login(&self, auth: &RegistryAuth) -> Result<(), String> {
let mut a = self.cfg();
a.extend(s(&["login", "--username"]));
a.push(auth.username.clone());
a.push("--password-stdin".into());
a.push(auth.registry.clone());
self.cli.run_stdin(&a, &auth.token).map(|_| ())
}
fn image_present(&self, image: &str) -> bool {
self.cli
.run(&s(&["image", "inspect", "--format", "{{.Id}}", image]))
.is_ok()
}
pub fn ensure_image(&self, image: &str) -> Result<String, String> {
if !self.image_present(image) {
let mut a = self.cfg();
a.extend(s(&["pull", "--quiet", image]));
self.cli.run(&a).map_err(|e| format!("pull {image}: {e}"))?;
}
let out = self.cli.run(&s(&[
"image",
"inspect",
"--format",
"{{json .RepoDigests}} {{.Id}}",
image,
]))?;
let (digests, id) = out.rsplit_once(' ').unwrap_or((out.as_str(), ""));
let first: Option<String> = serde_json::from_str::<Vec<String>>(digests)
.ok()
.and_then(|v| v.into_iter().next())
.and_then(|d| d.split_once('@').map(|(_, h)| h.to_string()));
Ok(first.unwrap_or_else(|| id.to_string()))
}
pub fn build_layer(
&self,
base: &str,
layer: &Layer,
tag: &str,
push: Option<&RegistryAuth>,
) -> Result<(), String> {
let dockerfile = layer_dockerfile(base, layer)?;
if !self.image_present(tag) {
let mut a = self.cfg();
a.extend(s(&["build", "--quiet", "-t"]));
a.push(tag.to_string());
a.push("-".into());
self.cli
.run_stdin(&a, &dockerfile)
.map_err(|e| format!("layer build failed: {}", last_lines(&e, 20)))?;
}
if let Some(auth) = push {
self.login(auth)?;
let mut a = self.cfg();
a.extend(s(&["push", "--quiet", tag]));
self.cli.run(&a).map_err(|e| format!("push {tag}: {e}"))?;
}
Ok(())
}
pub fn start(&self, spec: &JobSpec) -> Result<JobHandle, String> {
if let Some(auth) = &spec.push_to {
self.login(auth)?;
}
let digest = self.ensure_image(&spec.image)?;
let (image, layer_tag) = match &spec.layer {
None => (spec.image.clone(), None),
Some(layer) => {
validate_layer(layer)?;
let repo = spec.layer_repo.as_deref().unwrap_or("atelier-custom");
let owner = spec.owner.as_deref().unwrap_or("local");
let tag = layer_tag(repo, owner, &digest, layer);
self.build_layer(&spec.image, layer, &tag, spec.push_to.as_ref())?;
(tag.clone(), Some(tag))
}
};
if !spec.prepare_only {
let _admission = self.admission.lock();
let limits = self
.admission
.limits_for(spec)
.map_err(|e| format!("budget: {e}"))?;
self.cli
.run(&start_argv(
spec,
&image,
workdir_owner(&spec.workdir),
&limits,
))
.map_err(|e| format!("start {}: {e}", spec.name))?;
}
Ok(JobHandle {
name: spec.name.clone(),
image,
image_digest: digest,
layer_tag,
})
}
pub fn logs(&self, name: &str, since: usize) -> Result<(Vec<String>, usize), String> {
let out = self.cli.run(&s(&["logs", name]))?;
let lines: Vec<String> = out.lines().map(str::to_string).collect();
let n = lines.len();
Ok((lines.into_iter().skip(since).collect(), n))
}
pub fn inspect(&self, name: &str) -> Result<(bool, Option<i32>), String> {
let out = self.cli.run(&s(&[
"inspect",
"--format",
"{{.State.Status}} {{.State.ExitCode}}",
name,
]))?;
let mut it = out.split_whitespace();
let status = it.next().unwrap_or("");
let code = it.next().and_then(|c| c.parse().ok());
Ok(match status {
"running" | "created" | "restarting" => (true, None),
_ => (false, code),
})
}
pub fn stop(&self, name: &str) -> Result<(), String> {
let _ = self.cli.run(&s(&["stop", "--time", "10", name]));
self.cli.run(&s(&["rm", "-f", name])).map(|_| ())
}
pub fn reconcile(&self, known: &[String]) -> Result<Vec<String>, String> {
let out = self.cli.run(&s(&[
"ps",
"-a",
"--filter",
&format!("label={JOB_LABEL}"),
"--format",
"{{.Names}}",
]))?;
let mut removed = vec![];
for name in out.lines().map(str::trim).filter(|n| !n.is_empty()) {
if !known.iter().any(|k| k == name) {
self.stop(name)?;
removed.push(name.to_string());
}
}
Ok(removed)
}
}
fn last_lines(text: &str, n: usize) -> String {
let lines: Vec<&str> = text.lines().collect();
lines[lines.len().saturating_sub(n)..].join("\n")
}
#[derive(Debug, Clone)]
struct Entry {
prepare_only: bool,
state: JobState,
handle: Option<JobHandle>,
error: Option<String>,
}
pub struct JobManager<C: JobCli + 'static> {
rt: Arc<JobRuntime<C>>,
jobs: Arc<Mutex<HashMap<String, Entry>>>,
}
impl<C: JobCli + 'static> JobManager<C> {
pub fn new(rt: JobRuntime<C>) -> Self {
Self {
rt: Arc::new(rt),
jobs: Arc::new(Mutex::new(HashMap::new())),
}
}
fn set(jobs: &Mutex<HashMap<String, Entry>>, name: &str, f: impl FnOnce(&mut Entry)) {
if let Some(e) = jobs.lock().unwrap().get_mut(name) {
f(e)
}
}
pub fn submit(&self, spec: JobSpec) -> Result<(), String> {
let name_ok = !spec.name.is_empty()
&& spec.name.len() <= 64
&& spec
.name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
if !name_ok {
return Err(format!("bad job name {:?}", spec.name));
}
if let Some(l) = &spec.layer {
validate_layer(l)?;
}
{
let mut jobs = self.jobs.lock().unwrap();
if jobs.contains_key(&spec.name) {
return Err(format!("job {} exists", spec.name));
}
jobs.insert(
spec.name.clone(),
Entry {
prepare_only: spec.prepare_only,
state: JobState::Pulling,
handle: None,
error: None,
},
);
}
let (rt, jobs) = (self.rt.clone(), self.jobs.clone());
std::thread::spawn(move || {
if spec.layer.is_some() {
Self::set(&jobs, &spec.name, |e| e.state = JobState::Building);
}
match rt.start(&spec) {
Ok(h) => Self::set(&jobs, &spec.name, |e| {
e.state = JobState::Running;
e.handle = Some(h);
}),
Err(err) => Self::set(&jobs, &spec.name, |e| {
e.state = JobState::Failed;
e.error = Some(last_lines(&err, 20));
}),
}
});
Ok(())
}
#[cfg(test)]
pub fn submit_sync(&self, spec: JobSpec) -> Result<(), String> {
self.jobs.lock().unwrap().insert(
spec.name.clone(),
Entry {
prepare_only: spec.prepare_only,
state: JobState::Pulling,
handle: None,
error: None,
},
);
match self.rt.start(&spec) {
Ok(h) => Self::set(&self.jobs, &spec.name, |e| {
e.state = JobState::Running;
e.handle = Some(h)
}),
Err(err) => Self::set(&self.jobs, &spec.name, |e| {
e.state = JobState::Failed;
e.error = Some(err)
}),
}
Ok(())
}
pub fn status(&self, name: &str) -> Option<JobStatus> {
let e = self.jobs.lock().unwrap().get(name).cloned()?;
let mut st = JobStatus {
state: e.state.clone(),
exit_code: None,
image_digest: e.handle.as_ref().map(|h| h.image_digest.clone()),
layer_tag: e.handle.as_ref().and_then(|h| h.layer_tag.clone()),
error: e.error.clone(),
};
if e.state == JobState::Running && e.prepare_only {
st.state = JobState::Exited;
st.exit_code = Some(0);
} else if e.state == JobState::Running {
if let Ok((running, code)) = self.rt.inspect(name) {
if !running {
st.state = JobState::Exited;
st.exit_code = code;
}
}
}
Some(st)
}
pub fn logs(&self, name: &str, since: usize) -> Option<Result<(Vec<String>, usize), String>> {
let known = self.jobs.lock().unwrap().contains_key(name);
known.then(|| self.rt.logs(name, since))
}
pub fn stop(&self, name: &str) -> Result<(), String> {
self.jobs.lock().unwrap().remove(name);
self.rt.stop(name)
}
pub fn reconcile(&self, known: &[String]) -> Result<Vec<String>, String> {
let removed = self.rt.reconcile(known)?;
let mut jobs = self.jobs.lock().unwrap();
for k in known {
if !jobs.contains_key(k) && self.rt.inspect(k).is_ok() {
jobs.insert(
k.clone(),
Entry {
prepare_only: false,
state: JobState::Running,
handle: None,
error: None,
},
);
}
}
Ok(removed)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Default)]
struct Fake {
calls: Mutex<Vec<(Vec<String>, String)>>,
answers: Vec<(&'static str, Result<String, String>)>,
}
impl JobCli for Arc<Fake> {
fn run(&self, argv: &[String]) -> Result<String, String> {
self.run_stdin(argv, "")
}
fn run_stdin(&self, argv: &[String], input: &str) -> Result<String, String> {
self.calls
.lock()
.unwrap()
.push((argv.to_vec(), input.to_string()));
let joined = argv.join(" ");
for (needle, ans) in &self.answers {
if joined.contains(needle) {
return ans.clone();
}
}
Ok(String::new())
}
}
fn adm() -> Admission {
adm_with(Limits::new(4.0, 8.0, Gpus::None), vec![])
}
fn adm_with(budget: Limits, running: Vec<Allocation>) -> Admission {
Admission {
budget: Box::new(move || Ok(budget.clone())),
allocations: Box::new(move || Ok(running.clone())),
lock_path: None,
}
}
fn lim() -> Limits {
Limits::new(2.0, 4.0, Gpus::None)
}
fn layer() -> Layer {
Layer {
apt: vec!["libsndfile1=1.2.2-1".into()],
pip: vec!["timm==1.0.15".into()],
venv: "/opt/atelier/envs/hf_vision".into(),
}
}
fn spec(layer: Option<Layer>) -> JobSpec {
JobSpec {
name: "job-1".into(),
image: "docker.io/zakuroai/atelier-vision:abc-cpu".into(),
layer,
layer_repo: Some("docker.io/zakuroai/atelier-custom".into()),
owner: Some("4".into()),
push_to: None,
workdir: PathBuf::from("/var/lib/atelier/work/job-1"),
env: BTreeMap::from([("PYTHONUNBUFFERED".into(), "1".into())]),
gpus: String::new(),
cpus: None,
memory_gb: None,
mount_at: None,
prepare_only: false,
cmd: s(&[
"/opt/atelier/envs/hf_vision/bin/python",
"-m",
"sakura.atelier",
"run",
"/work/spec.yaml",
]),
}
}
#[test]
fn layer_tag_is_order_independent() {
let a = layer();
let mut b = layer();
b.pip = vec!["timm==1.0.15".into()];
b.apt.reverse();
let mut c = a.clone();
c.pip.push("open_clip_torch==2.32.0".into());
let mut d = c.clone();
d.pip.reverse();
assert_eq!(
layer_tag("r", "4", "sha256:x", &c),
layer_tag("r", "4", "sha256:x", &d)
);
assert_eq!(
layer_tag("r", "4", "sha256:x", &a),
layer_tag("r", "4", "sha256:x", &b)
);
assert_ne!(
layer_tag("r", "4", "sha256:x", &a),
layer_tag("r", "4", "sha256:y", &a),
"base digest is in the hash"
);
assert!(layer_tag("r", "4", "sha256:x", &a).starts_with("r:4-"));
}
#[test]
fn layer_dockerfile_pins_only() {
let f = layer_dockerfile("img@sha256:1", &layer()).unwrap();
assert!(f.starts_with("FROM img@sha256:1\n"));
assert!(f.contains("apt-get install -y --no-install-recommends libsndfile1=1.2.2-1"));
assert!(f.contains(
"uv pip install --python /opt/atelier/envs/hf_vision/bin/python timm==1.0.15"
));
for bad in [
"timm",
"timm>=1.0",
"git+https://github.com/x/y",
"https://evil/x.whl",
"-e .",
"timm==1.0; rm -rf /",
] {
let mut l = layer();
l.pip = vec![bad.into()];
assert!(layer_dockerfile("img", &l).is_err(), "{bad} accepted");
}
let mut l = layer();
l.apt = vec!["curl".into()];
assert!(
layer_dockerfile("img", &l).is_err(),
"unpinned apt accepted"
);
let mut l = layer();
l.venv = "relative/venv".into();
assert!(layer_dockerfile("img", &l).is_err());
}
#[test]
fn start_argv_mounts_work_labels_and_carries_the_admitted_limits() {
let gpu = Limits::new(2.0, 4.0, Gpus::All);
let a = start_argv(&spec(None), "img", None, &gpu);
let j = a.join(" ");
assert!(j.starts_with("run -d --name job-1 --label ai.zakuro.atelier.job=job-1"));
assert!(j.contains("--label zakuro.budget=2000m,4294967296"), "{j}");
assert!(j.contains("-v /var/lib/atelier/work/job-1:/work -w /work"));
assert!(
j.contains("--cpus 2 --memory 4096m --memory-swap 4096m --gpus all"),
"{j}"
);
assert!(j.contains(
"-e PYTHONUNBUFFERED=1 img /opt/atelier/envs/hf_vision/bin/python -m sakura.atelier"
));
let plain = start_argv(&spec(None), "img", None, &lim()).join(" ");
assert!(!plain.contains("--gpus"), "{plain}");
assert!(
plain.contains("--cpus 2 --memory 4096m --memory-swap 4096m"),
"{plain}"
);
assert!(plain.contains("--pids-limit 4096 --cpus"), "{plain}");
}
fn run_argv_of(fake: &Arc<Fake>) -> Option<String> {
fake.calls
.lock()
.unwrap()
.iter()
.find(|c| c.0.first().map(String::as_str) == Some("run"))
.map(|c| c.0.join(" "))
}
#[test]
fn a_job_is_started_under_its_request_clamped_to_the_budget() {
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), adm());
let mut sp = spec(None);
sp.cpus = Some(16.0);
sp.memory_gb = Some(64.0);
rt.start(&sp).unwrap();
let j = run_argv_of(&fake).unwrap();
assert!(
j.contains("--cpus 4 --memory 8192m --memory-swap 8192m"),
"{j}"
);
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), adm());
sp.cpus = Some(1.0);
sp.memory_gb = Some(2.0);
rt.start(&sp).unwrap();
let j = run_argv_of(&fake).unwrap();
assert!(
j.contains("--cpus 1 --memory 2048m --memory-swap 2048m"),
"{j}"
);
}
#[test]
fn an_unsized_job_takes_what_the_other_workloads_leave() {
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let busy = vec![Allocation {
name: "zc-share-w0".into(),
cpu_millis: 3000,
memory_bytes: 6 << 30,
}];
let rt = JobRuntime::new(
fake.clone(),
PathBuf::from("/tmp/cfg"),
adm_with(Limits::new(4.0, 8.0, Gpus::None), busy),
);
rt.start(&spec(None)).unwrap();
let j = run_argv_of(&fake).unwrap();
assert!(j.contains("--cpus 1 --memory 2048m"), "{j}");
}
#[test]
fn a_start_past_the_budget_is_an_error_with_a_reason_and_runs_nothing() {
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let full = vec![Allocation {
name: "zc-share-w0".into(),
cpu_millis: 4000,
memory_bytes: 8 << 30,
}];
let rt = JobRuntime::new(
fake.clone(),
PathBuf::from("/tmp/cfg"),
adm_with(Limits::new(4.0, 8.0, Gpus::None), full),
);
let e = rt.start(&spec(None)).unwrap_err();
assert!(e.starts_with("budget: node budget exhausted"), "{e}");
assert!(run_argv_of(&fake).is_none(), "nothing was started");
let some = vec![Allocation {
name: "w".into(),
cpu_millis: 3000,
memory_bytes: 1 << 30,
}];
let rt = JobRuntime::new(
fake.clone(),
PathBuf::from("/tmp/cfg"),
adm_with(Limits::new(4.0, 8.0, Gpus::None), some),
);
let mut sp = spec(None);
sp.cpus = Some(2.0);
let e = rt.start(&sp).unwrap_err();
assert!(e.contains("cpu: asks 2 of the 1 left"), "{e}");
assert!(run_argv_of(&fake).is_none());
}
#[test]
fn an_invalid_budget_or_unreadable_allocations_refuse_the_start() {
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let bad = Admission {
budget: Box::new(|| Err("ZAKURO_BUDGET_CPU=bad".into())),
allocations: Box::new(|| Ok(vec![])),
lock_path: None,
};
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), bad);
assert_eq!(
rt.start(&spec(None)).unwrap_err(),
"budget: ZAKURO_BUDGET_CPU=bad"
);
let blind = Admission {
budget: Box::new(|| Ok(Limits::new(4.0, 8.0, Gpus::None))),
allocations: Box::new(|| Err("daemon down".into())),
lock_path: None,
};
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), blind);
let e = rt.start(&spec(None)).unwrap_err();
assert!(
e.contains("cannot read the node's allocations: daemon down"),
"{e}"
);
assert!(run_argv_of(&fake).is_none());
}
#[test]
fn a_gpu_is_granted_only_from_the_budget_and_a_refusal_says_so() {
assert_eq!(job_gpus("", &Gpus::All).unwrap(), Gpus::None);
assert_eq!(job_gpus("0", &Gpus::All).unwrap(), Gpus::None);
assert_eq!(job_gpus("all", &Gpus::All).unwrap(), Gpus::All);
assert_eq!(job_gpus("2", &Gpus::All).unwrap(), Gpus::Count(2));
assert_eq!(job_gpus("8", &Gpus::Count(1)).unwrap(), Gpus::Count(1));
let pin = Gpus::Spec("device=0".into());
assert_eq!(job_gpus("all", &pin).unwrap(), pin);
let e = job_gpus("all", &Gpus::None).unwrap_err();
assert!(
e.contains("budget passes none") && e.contains("ZAKURO_BUDGET_GPU=1"),
"{e}"
);
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), adm());
let mut sp = spec(None);
sp.gpus = "all".into();
let e = rt.start(&sp).unwrap_err();
assert!(e.starts_with("budget: job job-1: asks for a GPU"), "{e}");
assert!(run_argv_of(&fake).is_none());
let rt = JobRuntime::new(
fake.clone(),
PathBuf::from("/tmp/cfg"),
adm_with(Limits::new(4.0, 8.0, Gpus::All), vec![]),
);
rt.start(&sp).unwrap();
assert!(run_argv_of(&fake).unwrap().contains("--gpus all"));
}
#[test]
fn preparing_a_job_is_not_admitted_it_starts_nothing() {
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let blind = Admission {
budget: Box::new(|| Err("must not be read".into())),
allocations: Box::new(|| Err("must not be read".into())),
lock_path: None,
};
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), blind);
let mut sp = spec(None);
sp.prepare_only = true;
rt.start(&sp).unwrap();
}
#[test]
fn reaps_unknown_job_containers() {
let fake = Arc::new(Fake {
answers: vec![("ps -a", Ok("job-1\norphan-7\n".into()))],
..Default::default()
});
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), adm());
let removed = rt.reconcile(&["job-1".into()]).unwrap();
assert_eq!(removed, vec!["orphan-7".to_string()]);
let calls: Vec<String> = fake
.calls
.lock()
.unwrap()
.iter()
.map(|c| c.0.join(" "))
.collect();
assert!(calls.iter().any(|c| c == "rm -f orphan-7"));
assert!(!calls.iter().any(|c| c.contains("rm -f job-1")));
}
#[test]
fn layer_build_failure_is_reported_not_started() {
let fake = Arc::new(Fake {
answers: vec![
(
"image inspect --format {{.Id}} docker.io/zakuroai/atelier-custom",
Err("No such image".into()),
),
(
"image inspect --format {{json",
Ok("[\"docker.io/zakuroai/atelier-vision@sha256:abc\"] sha256:id".into()),
),
(
"build",
Err("step 2\nERROR: No matching distribution found for timm==1.0.15".into()),
),
],
..Default::default()
});
let m = JobManager::new(JobRuntime::new(
fake.clone(),
PathBuf::from("/tmp/cfg"),
adm(),
));
m.submit_sync(spec(Some(layer()))).unwrap();
let st = m.status("job-1").unwrap();
assert_eq!(st.state, JobState::Failed);
assert!(st.error.unwrap().contains("No matching distribution"));
assert!(!fake
.calls
.lock()
.unwrap()
.iter()
.any(|c| c.0.first().map(String::as_str) == Some("run")));
let build = fake
.calls
.lock()
.unwrap()
.iter()
.find(|c| c.0.contains(&"build".to_string()))
.cloned()
.unwrap();
assert!(
build
.1
.starts_with("FROM docker.io/zakuroai/atelier-vision:abc-cpu\n"),
"dockerfile goes on stdin"
);
}
#[test]
fn started_job_reports_digest_and_exit() {
let fake = Arc::new(Fake {
answers: vec![
(
"image inspect --format {{json",
Ok("[\"docker.io/zakuroai/atelier-vision@sha256:abc\"] sha256:id".into()),
),
("inspect --format {{.State.Status}}", Ok("exited 0".into())),
("logs job-1", Ok("a\nb\nc".into())),
],
..Default::default()
});
let m = JobManager::new(JobRuntime::new(fake, PathBuf::from("/tmp/cfg"), adm()));
m.submit_sync(spec(None)).unwrap();
let st = m.status("job-1").unwrap();
assert_eq!(
(st.state, st.exit_code, st.image_digest.as_deref()),
(JobState::Exited, Some(0), Some("sha256:abc"))
);
assert_eq!(
m.logs("job-1", 1).unwrap().unwrap(),
(vec!["b".to_string(), "c".to_string()], 3)
);
assert!(m.logs("nope", 0).is_none());
}
#[test]
fn prepare_only_pulls_and_builds_but_starts_nothing() {
let fake = Arc::new(Fake {
answers: vec![
(
"image inspect --format {{.Id}} docker.io/zakuroai/atelier-custom",
Err("No such image".into()),
),
(
"image inspect --format {{json",
Ok("[\"docker.io/zakuroai/atelier-vision@sha256:abc\"] sha256:id".into()),
),
],
..Default::default()
});
let m = JobManager::new(JobRuntime::new(
fake.clone(),
PathBuf::from("/tmp/cfg"),
adm(),
));
let mut sp = spec(Some(layer()));
sp.prepare_only = true;
m.submit_sync(sp).unwrap();
let st = m.status("job-1").unwrap();
assert_eq!((st.state, st.exit_code), (JobState::Exited, Some(0)));
assert!(st
.layer_tag
.unwrap()
.starts_with("docker.io/zakuroai/atelier-custom:4-"));
let calls = fake.calls.lock().unwrap();
assert!(calls.iter().any(|c| c.0.contains(&"build".to_string())));
assert!(!calls
.iter()
.any(|c| c.0.first().map(String::as_str) == Some("run")));
}
#[test]
fn a_job_runs_as_the_owner_of_its_workdir() {
let j = start_argv(&spec(None), "img", Some((999, 988)), &lim()).join(" ");
assert!(j.contains("--user 999:988"));
assert!(
j.contains("-e HOME=/tmp"),
"uid 999 has no home in the image"
);
assert!(j.find("--user").unwrap() < j.find(" img ").unwrap());
let root = start_argv(&spec(None), "img", None, &lim()).join(" ");
assert!(!root.contains("--user") && !root.contains("HOME=/tmp"));
}
#[test]
fn a_home_the_spec_sets_is_kept() {
let mut sp = spec(None);
sp.env.insert("HOME".into(), "/work/.home".into());
let j = start_argv(&sp, "img", Some((999, 988)), &lim()).join(" ");
assert!(j.contains("-e HOME=/work/.home") && !j.contains("HOME=/tmp"));
}
#[test]
fn the_workdir_owner_is_read_from_disk() {
let d = std::env::temp_dir().join(format!("zc-job-owner-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let me = workdir_owner(&d);
std::fs::remove_dir(&d).unwrap();
let uid = unsafe { libc::geteuid() };
assert_eq!(me.map(|(u, _)| u), if uid == 0 { None } else { Some(uid) });
assert_eq!(
workdir_owner(std::path::Path::new("/")),
None,
"/ is root's"
);
assert_eq!(
workdir_owner(std::path::Path::new("/nonexistent/zc-job")),
None
);
}
#[test]
fn mount_at_keeps_host_paths() {
let mut sp = spec(None);
sp.workdir = PathBuf::from("/var/lib/atelier/work");
sp.mount_at = Some("/var/lib/atelier/work".into());
let j = start_argv(&sp, "img", None, &lim()).join(" ");
assert!(
j.contains("-v /var/lib/atelier/work:/var/lib/atelier/work -w /var/lib/atelier/work")
);
}
#[test]
fn registry_password_never_in_argv() {
let fake = Arc::new(Fake {
answers: vec![("image inspect --format {{json", Ok("[] sha256:id".into()))],
..Default::default()
});
let rt = JobRuntime::new(fake.clone(), PathBuf::from("/tmp/cfg"), adm());
let mut sp = spec(Some(layer()));
sp.push_to = Some(RegistryAuth {
registry: "docker.io".into(),
username: "zakuroai".into(),
token: "s3cret".into(),
});
let _ = rt.start(&sp);
for (argv, _) in fake.calls.lock().unwrap().iter() {
assert!(!argv.join(" ").contains("s3cret"));
}
assert!(fake
.calls
.lock()
.unwrap()
.iter()
.any(|(a, i)| a.contains(&"login".to_string()) && i == "s3cret"));
}
}