use serde_json::{json, Value};
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{Read, Write};
use std::os::unix::net::UnixStream;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
pub const DEFAULT_SOCKET: &str = "/run/zakuro/space.sock";
pub const MAX_OUTPUTS: usize = 64;
pub const STDERR_TAIL_MAX: usize = 4096;
#[derive(Debug, Clone)]
pub struct Config {
pub socket: PathBuf,
pub in_dir: PathBuf,
pub out_dir: PathBuf,
pub cmd: Option<Cmd>,
pub poll_min: Duration,
pub poll_max: Duration,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Cmd {
Shell(String),
Argv(Vec<String>),
}
pub fn parse_cli(
args: &[String],
env: &dyn Fn(&str) -> Option<String>,
) -> Result<(String, Cmd), String> {
let mut flag_id: Option<String> = None;
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--" => {
i += 1;
break;
}
"--space-id" => {
flag_id = Some(
args.get(i + 1)
.cloned()
.ok_or("--space-id requires a value")?,
);
i += 2;
}
a if a.starts_with("--space-id=") => {
flag_id = Some(a["--space-id=".len()..].to_string());
i += 1;
}
_ => break,
}
}
let rest = &args[i..];
let space_id = flag_id
.or_else(|| env("ZAKURO_SPACE_ID"))
.filter(|s| !s.trim().is_empty())
.ok_or("no Space ID: pass --space-id or set ZAKURO_SPACE_ID")?;
let cmd = match env("ZAKURO_SPACE_CMD").filter(|c| !c.trim().is_empty()) {
Some(c) => Cmd::Shell(c),
None if !rest.is_empty() => Cmd::Argv(rest.to_vec()),
None => return Err("no command: set ZAKURO_SPACE_CMD or pass one after `--`".into()),
};
Ok((space_id, cmd))
}
pub fn run_cli(args: &[String]) -> i32 {
let Some(args) = worker_args(args) else {
eprintln!("usage: zc space worker [--space-id ID] [--] [cmd…]");
return 2;
};
let (space_id, cmd) = match parse_cli(args, &|k| std::env::var(k).ok()) {
Ok(r) => r,
Err(e) => {
eprintln!("zc space worker: {e}");
return 2;
}
};
let cfg = Config {
socket: std::env::var("ZAKURO_SPACE_SOCKET")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from(DEFAULT_SOCKET)),
in_dir: PathBuf::from("/work/in"),
out_dir: PathBuf::from("/work/out"),
cmd: Some(cmd),
poll_min: Duration::from_secs(1),
poll_max: Duration::from_secs(15),
};
eprintln!(
"zc space worker: space={} socket={}",
space_id,
cfg.socket.display()
);
run_loop(&cfg, &AtomicBool::new(false));
0
}
pub fn worker_args(args: &[String]) -> Option<&[String]> {
match args.first().map(String::as_str) {
Some("worker") => Some(&args[1..]),
_ => None,
}
}
pub fn run_loop(cfg: &Config, stop: &AtomicBool) {
let mut delay = cfg.poll_min;
while !stop.load(Ordering::SeqCst) {
match run_once(cfg) {
Ok(true) => {
delay = cfg.poll_min;
continue;
}
Ok(false) => {}
Err(e) => eprintln!("zc space worker: {e}"),
}
if stop.load(Ordering::SeqCst) {
break;
}
std::thread::sleep(delay);
delay = next_delay(delay, cfg.poll_max);
}
}
pub fn next_delay(cur: Duration, max: Duration) -> Duration {
(cur * 2).min(max)
}
pub fn run_once(cfg: &Config) -> Result<bool, String> {
let (status, body) = sock_request(&cfg.socket, "GET", "/next", None)?;
if status == 204 {
return Ok(false);
}
if status != 200 {
return Err(format!("/next returned HTTP {status}"));
}
let batch: Value = serde_json::from_slice(&body).map_err(|e| format!("/next body: {e}"))?;
let batch_id = batch["batch_id"]
.as_i64()
.ok_or("/next: missing batch_id")?;
let seq = batch["seq"].as_i64().unwrap_or(0);
let outcome = process_batch(cfg, &batch, batch_id, seq);
let _ = reset_dir(&cfg.in_dir);
let _ = reset_dir(&cfg.out_dir);
let (hashes, exit_code, tail) = outcome;
let done = json!({
"batch_id": batch_id,
"output_hashes": hashes,
"exit_code": exit_code,
"stderr_tail": tail,
});
let (st, _) = sock_request(&cfg.socket, "POST", "/done", Some(&done))?;
if st != 200 && st != 202 {
return Err(format!("/done returned HTTP {st} for batch {batch_id}"));
}
Ok(true)
}
fn process_batch(cfg: &Config, batch: &Value, batch_id: i64, seq: i64) -> (Hashes, i32, String) {
let fail = |code: i32, msg: String| {
(
Hashes::new(),
code,
tail_utf8(msg.as_bytes(), STDERR_TAIL_MAX),
)
};
if let Err(e) = reset_dir(&cfg.in_dir).and_then(|_| reset_dir(&cfg.out_dir)) {
return fail(1, format!("cannot prepare work dirs: {e}"));
}
let agent = http_agent();
for inp in batch["inputs"].as_array().into_iter().flatten() {
let (Some(name), Some(url)) = (inp["name"].as_str(), inp["url"].as_str()) else {
return fail(1, "malformed input entry".into());
};
if let Err(e) = download(&agent, url, &cfg.in_dir, name) {
return fail(1, format!("download {name}: {e}"));
}
}
let Some(cmd) = cfg.cmd.as_ref() else {
return fail(2, "no command configured (ZAKURO_SPACE_CMD)".into());
};
let timeout = batch["max_runtime_s"]
.as_u64()
.filter(|t| *t > 0)
.map(Duration::from_secs);
let (code, tail) = match run_command(cfg, cmd, batch_id, seq, timeout) {
Ok(r) => r,
Err(e) => return fail(127, format!("cannot start command: {e}")),
};
if code != 0 {
return (Hashes::new(), code, tail);
}
let mut names = match list_outputs(&cfg.out_dir) {
Ok(n) => n,
Err(e) => return fail(1, format!("cannot list outputs: {e}")),
};
names.sort();
if names.len() > MAX_OUTPUTS {
return fail(
2,
format!("too many outputs: {} (max {MAX_OUTPUTS})", names.len()),
);
}
if names.is_empty() {
return (Hashes::new(), 0, tail);
}
match upload_outputs(cfg, &agent, batch_id, &names) {
Ok(h) => (h, 0, tail),
Err(e) => fail(1, format!("upload: {e}")),
}
}
fn upload_outputs(
cfg: &Config,
agent: &ureq::Agent,
batch_id: i64,
names: &[String],
) -> Result<Hashes, String> {
let req = json!({"batch_id": batch_id, "names": names});
let (st, body) = sock_request(&cfg.socket, "POST", "/outputs", Some(&req))?;
if st != 200 {
return Err(format!("/outputs returned HTTP {st}"));
}
let v: Value = serde_json::from_slice(&body).map_err(|e| format!("/outputs body: {e}"))?;
let mut hashes = Hashes::new();
for n in names {
let url = v["urls"][n.as_str()]
.as_str()
.ok_or_else(|| format!("no presigned URL for {n}"))?;
let path = cfg.out_dir.join(n);
let hash = sha256_file(&path).map_err(|e| format!("hash {n}: {e}"))?;
let f = File::open(&path).map_err(|e| format!("open {n}: {e}"))?;
agent
.put(url)
.send(f)
.map_err(|e| format!("PUT {n}: {e}"))?;
hashes.insert(n.clone(), hash);
}
Ok(hashes)
}
fn http_agent() -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(3600)))
.build()
.into()
}
fn safe_name(name: &str) -> bool {
!name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0'])
}
fn download(agent: &ureq::Agent, url: &str, dir: &Path, name: &str) -> Result<(), String> {
if !safe_name(name) {
return Err("unsafe file name".into());
}
let resp = agent.get(url).call().map_err(|e| e.to_string())?;
let mut rd = resp.into_body().into_reader();
let mut f = File::create(dir.join(name)).map_err(|e| e.to_string())?;
std::io::copy(&mut rd, &mut f).map_err(|e| e.to_string())?;
Ok(())
}
fn reset_dir(dir: &Path) -> std::io::Result<()> {
if dir.exists() {
std::fs::remove_dir_all(dir)?;
}
std::fs::create_dir_all(dir)
}
fn list_outputs(dir: &Path) -> std::io::Result<Vec<String>> {
let bad = |m: String| std::io::Error::new(std::io::ErrorKind::InvalidData, m);
let mut out = Vec::new();
for e in std::fs::read_dir(dir)? {
let e = e?;
let ft = e.file_type()?;
let name = e
.file_name()
.into_string()
.map_err(|n| bad(format!("output name is not UTF-8: {n:?}")))?;
if !ft.is_file() {
let kind = if ft.is_dir() {
"a directory"
} else {
"not a regular file (symlink?)"
};
return Err(bad(format!(
"output '{name}' is {kind}; outputs must be flat regular files in {}",
dir.display()
)));
}
out.push(name);
}
Ok(out)
}
fn sha256_file(p: &Path) -> std::io::Result<String> {
let mut f = File::open(p)?;
let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
let mut buf = vec![0u8; 64 * 1024];
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
ctx.update(&buf[..n]);
}
Ok(hex::encode(ctx.finish()))
}
fn run_command(
cfg: &Config,
cmd: &Cmd,
batch_id: i64,
seq: i64,
timeout: Option<Duration>,
) -> std::io::Result<(i32, String)> {
use std::os::unix::process::CommandExt;
use std::sync::{Arc, Mutex};
let mut c = match cmd {
Cmd::Shell(s) => {
let mut c = Command::new("sh");
c.arg("-c").arg(s);
c
}
Cmd::Argv(v) => {
let mut c = Command::new(&v[0]);
c.args(&v[1..]);
c
}
};
let mut child = c
.env("ZAKURO_IN_DIR", &cfg.in_dir)
.env("ZAKURO_OUT_DIR", &cfg.out_dir)
.env("ZAKURO_BATCH_ID", batch_id.to_string())
.env("ZAKURO_BATCH_SEQ", seq.to_string())
.stdin(Stdio::null())
.stderr(Stdio::piped())
.process_group(0)
.spawn()?;
let mut err = child.stderr.take().expect("piped");
let tail: Arc<Mutex<Vec<u8>>> = Arc::new(Mutex::new(Vec::new()));
let t2 = tail.clone();
let reader = std::thread::spawn(move || {
let mut buf = [0u8; 8192];
loop {
match err.read(&mut buf) {
Ok(0) | Err(_) => break,
Ok(n) => {
let _ = std::io::stderr().write_all(&buf[..n]);
let mut t = t2.lock().unwrap();
t.extend_from_slice(&buf[..n]);
if t.len() > 2 * STDERR_TAIL_MAX {
let cut = t.len() - STDERR_TAIL_MAX;
t.drain(..cut);
}
}
}
}
});
let start = std::time::Instant::now();
let mut timed_out = false;
let status = loop {
if let Some(st) = child.try_wait()? {
break st;
}
if timeout.is_some_and(|t| start.elapsed() >= t) {
timed_out = true;
unsafe {
libc::kill(-(child.id() as i32), libc::SIGKILL);
}
let _ = child.kill();
break child.wait()?;
}
std::thread::sleep(Duration::from_millis(20));
};
let grace = std::time::Instant::now();
while !reader.is_finished() && grace.elapsed() < Duration::from_millis(500) {
std::thread::sleep(Duration::from_millis(10));
}
let mut bytes = tail.lock().unwrap().clone();
if timed_out {
bytes.extend_from_slice(
format!(
"\nzc space worker: command exceeded max runtime ({}s); killed\n",
timeout.map(|t| t.as_secs()).unwrap_or(0)
)
.as_bytes(),
);
return Ok((124, tail_utf8(&bytes, STDERR_TAIL_MAX)));
}
let code = status.code().unwrap_or(-1);
Ok((code, tail_utf8(&bytes, STDERR_TAIL_MAX)))
}
pub fn tail_utf8(b: &[u8], max: usize) -> String {
let mut s = if b.len() > max {
&b[b.len() - max..]
} else {
b
};
while !s.is_empty() && (s[0] & 0xC0) == 0x80 {
s = &s[1..];
}
String::from_utf8_lossy(s).into_owned()
}
fn sock_request(
sock: &Path,
method: &str,
path: &str,
body: Option<&Value>,
) -> Result<(u16, Vec<u8>), String> {
let mut s =
UnixStream::connect(sock).map_err(|e| format!("connect {}: {e}", sock.display()))?;
let _ = s.set_read_timeout(Some(Duration::from_secs(120)));
let payload = body.map(|b| b.to_string()).unwrap_or_default();
let mut req = format!("{method} {path} HTTP/1.1\r\nHost: zakuro-space\r\nConnection: close\r\nAccept: application/json\r\n");
if body.is_some() {
req.push_str(&format!(
"Content-Type: application/json\r\nContent-Length: {}\r\n",
payload.len()
));
}
req.push_str("\r\n");
s.write_all(req.as_bytes())
.and_then(|_| s.write_all(payload.as_bytes()))
.map_err(|e| format!("write {path}: {e}"))?;
let mut raw = Vec::new();
s.read_to_end(&mut raw)
.map_err(|e| format!("read {path}: {e}"))?;
parse_response(&raw)
}
fn parse_response(raw: &[u8]) -> Result<(u16, Vec<u8>), String> {
let split = raw
.windows(4)
.position(|w| w == b"\r\n\r\n")
.ok_or("malformed HTTP response")?;
let head = String::from_utf8_lossy(&raw[..split]).into_owned();
let body = &raw[split + 4..];
let mut lines = head.lines();
let status: u16 = lines
.next()
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|c| c.parse().ok())
.ok_or("malformed HTTP status line")?;
let chunked = lines.any(|l| {
let l = l.to_ascii_lowercase();
l.starts_with("transfer-encoding:") && l.contains("chunked")
});
if !chunked {
return Ok((status, body.to_vec()));
}
let mut out = Vec::new();
let mut rest = body;
loop {
let eol = rest
.windows(2)
.position(|w| w == b"\r\n")
.ok_or("bad chunk framing")?;
let size = usize::from_str_radix(
String::from_utf8_lossy(&rest[..eol])
.split(';')
.next()
.unwrap_or("")
.trim(),
16,
)
.map_err(|_| "bad chunk size")?;
rest = &rest[eol + 2..];
if size == 0 {
break;
}
if rest.len() < size {
return Err("truncated chunk".into());
}
out.extend_from_slice(&rest[..size]);
rest = rest.get(size + 2..).ok_or("bad chunk framing")?;
}
Ok((status, out))
}
type Hashes = BTreeMap<String, String>;
#[cfg(test)]
mod tests {
use super::*;
use serde_json::{json, Value};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpListener;
use std::os::unix::net::UnixListener;
use std::sync::atomic::Ordering;
use std::sync::{Arc, Mutex};
fn tmp(tag: &str) -> PathBuf {
let d = PathBuf::from("/tmp").join(format!(
"zcs-{}-{}",
tag,
&uuid::Uuid::new_v4().simple().to_string()[..8]
));
std::fs::create_dir_all(&d).unwrap();
d
}
type Log = Arc<Mutex<Vec<(String, String, Vec<u8>)>>>;
fn read_req<R: Read>(r: R) -> Option<(String, String, Vec<u8>)> {
let mut br = BufReader::new(r);
let mut line = String::new();
if br.read_line(&mut line).ok()? == 0 {
return None;
}
let mut it = line.split_whitespace();
let method = it.next()?.to_string();
let path = it.next()?.to_string();
let mut len = 0usize;
loop {
let mut h = String::new();
br.read_line(&mut h).ok()?;
if h == "\r\n" || h.is_empty() {
break;
}
if let Some((k, v)) = h.split_once(':') {
if k.eq_ignore_ascii_case("content-length") {
len = v.trim().parse().unwrap();
}
}
}
let mut body = vec![0u8; len];
br.read_exact(&mut body).ok()?;
Some((method, path, body))
}
fn respond<W: Write>(mut w: W, status: u16, body: &[u8]) {
let reason = if status == 204 { "No Content" } else { "OK" };
let _ = write!(
w,
"HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = w.write_all(body);
}
fn fake_broker(
sock: &std::path::Path,
h: impl Fn(&str, &str, &Value) -> (u16, Value) + Send + 'static,
) -> Log {
let l = UnixListener::bind(sock).unwrap();
let log: Log = Arc::default();
let log2 = log.clone();
std::thread::spawn(move || {
for s in l.incoming() {
let Ok(s) = s else { break };
let Some((m, p, b)) = read_req(&s) else {
continue;
};
let v: Value = serde_json::from_slice(&b).unwrap_or(Value::Null);
let (st, body) = h(&m, &p, &v);
log2.lock().unwrap().push((m, p, b));
respond(&s, st, body.to_string().as_bytes());
}
});
log
}
type Puts = Arc<Mutex<BTreeMap<String, Vec<u8>>>>;
fn s3(files: Vec<(&'static str, &'static str)>) -> (String, Puts) {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}", l.local_addr().unwrap());
let puts: Arc<Mutex<BTreeMap<String, Vec<u8>>>> = Arc::default();
let puts2 = puts.clone();
std::thread::spawn(move || {
for s in l.incoming() {
let Ok(s) = s else { break };
let Some((m, p, b)) = read_req(&s) else {
continue;
};
if m == "PUT" {
puts2.lock().unwrap().insert(p, b);
respond(&s, 200, b"");
} else {
let name = p.trim_start_matches("/in/");
match files.iter().find(|(n, _)| *n == name) {
Some((_, c)) => respond(&s, 200, c.as_bytes()),
None => respond(&s, 404, b""),
}
}
}
});
(base, puts)
}
fn cfg(root: &std::path::Path, cmd: &str) -> Config {
Config {
socket: root.join("s.sock"),
in_dir: root.join("work/in"),
out_dir: root.join("work/out"),
cmd: Some(Cmd::Shell(cmd.to_string())),
poll_min: Duration::from_millis(10),
poll_max: Duration::from_millis(40),
}
}
fn sha(b: &[u8]) -> String {
hex::encode(ring::digest::digest(&ring::digest::SHA256, b))
}
fn done_body(log: &Log) -> Value {
let g = log.lock().unwrap();
let (_, _, b) = g.iter().find(|(_, p, _)| p == "/done").expect("no /done");
serde_json::from_slice(b).unwrap()
}
#[test]
fn full_batch_hashes_and_uploads() {
let root = tmp("full");
let (base, puts) = s3(vec![("a.txt", "hello"), ("b.txt", "world")]);
let c = cfg(&root, "sh -c 'cat /dev/null; sha256sum \"$ZAKURO_IN_DIR\"/a.txt > \"$ZAKURO_OUT_DIR\"/hashes.txt; echo hi > \"$ZAKURO_OUT_DIR\"/x.txt'");
let served = Arc::new(Mutex::new(false));
let base2 = base.clone();
let log = fake_broker(&c.socket, move |m, p, _| match (m, p) {
("GET", "/next") => {
let mut s = served.lock().unwrap();
if *s {
(204, Value::Null)
} else {
*s = true;
(
200,
json!({"batch_id":123,"seq":7,"inputs":[
{"name":"a.txt","url":format!("{base2}/in/a.txt")},
{"name":"b.txt","url":format!("{base2}/in/b.txt")}]}),
)
}
}
("POST", "/outputs") => (
200,
json!({"urls":{
"hashes.txt":format!("{base2}/out/hashes.txt"),
"x.txt":format!("{base2}/out/x.txt")}}),
),
("POST", "/done") => (200, json!({"accepted":true})),
_ => (404, Value::Null),
});
assert_eq!(run_once(&c), Ok(true));
let d = done_body(&log);
assert_eq!(d["batch_id"], 123);
assert_eq!(d["exit_code"], 0);
let up = puts.lock().unwrap();
let hashes = up.get("/out/hashes.txt").expect("hashes uploaded");
assert!(String::from_utf8_lossy(hashes).contains(&sha(b"hello")));
assert_eq!(up.get("/out/x.txt").unwrap(), b"hi\n");
assert_eq!(d["output_hashes"]["hashes.txt"], sha(hashes));
assert_eq!(d["output_hashes"]["x.txt"], sha(b"hi\n"));
let g = log.lock().unwrap();
let o: Value =
serde_json::from_slice(&g.iter().find(|(_, p, _)| p == "/outputs").unwrap().2).unwrap();
assert_eq!(o["batch_id"], 123);
assert_eq!(o["names"], json!(["hashes.txt", "x.txt"]));
}
#[test]
fn next_204_backs_off_and_polls_again() {
let root = tmp("poll");
let c = cfg(&root, "true");
let n = Arc::new(Mutex::new(0u32));
let n2 = n.clone();
let stop = Arc::new(AtomicBool::new(false));
let stop2 = stop.clone();
let log = fake_broker(&c.socket, move |_, _, _| {
let mut k = n2.lock().unwrap();
*k += 1;
if *k >= 4 {
stop2.store(true, Ordering::SeqCst);
}
(204, Value::Null)
});
assert_eq!(run_once(&c), Ok(false));
run_loop(&c, &stop);
assert!(log.lock().unwrap().len() >= 4);
assert!(log.lock().unwrap().iter().all(|(_, p, _)| p == "/next"));
}
#[test]
fn backoff_doubles_and_caps() {
let max = Duration::from_millis(40);
assert_eq!(
next_delay(Duration::from_millis(10), max),
Duration::from_millis(20)
);
assert_eq!(next_delay(Duration::from_millis(30), max), max);
assert_eq!(next_delay(max, max), max);
}
#[test]
fn failing_command_reports_exit_and_stderr_tail() {
let root = tmp("fail");
let c = cfg(
&root,
"sh -c 'yes ERRLINE | head -c 20000 >&2; echo END >&2; exit 3'",
);
let served = Arc::new(Mutex::new(false));
let log = fake_broker(&c.socket, move |m, p, _| match (m, p) {
("GET", "/next") => {
let mut s = served.lock().unwrap();
let first = !*s;
*s = true;
if first {
(200, json!({"batch_id":9,"seq":1,"inputs":[]}))
} else {
(204, Value::Null)
}
}
("POST", "/done") => (200, json!({"accepted":true})),
_ => (404, Value::Null),
});
assert_eq!(run_once(&c), Ok(true));
let d = done_body(&log);
assert_eq!(d["exit_code"], 3);
let t = d["stderr_tail"].as_str().unwrap();
assert!(t.len() <= STDERR_TAIL_MAX && t.len() > 1000, "{}", t.len());
assert!(t.trim_end().ends_with("END"));
assert_eq!(d["output_hashes"], json!({}));
assert!(!log.lock().unwrap().iter().any(|(_, p, _)| p == "/outputs"));
}
fn one_batch_broker(c: &Config, batch: Value) -> Log {
let served = Arc::new(Mutex::new(false));
fake_broker(&c.socket, move |m, p, _| match (m, p) {
("GET", "/next") => {
let mut s = served.lock().unwrap();
let first = !*s;
*s = true;
if first {
(200, batch.clone())
} else {
(204, Value::Null)
}
}
("POST", "/done") => (202, json!({"accepted":"pending"})),
_ => (404, Value::Null),
})
}
#[test]
fn command_is_killed_at_lease_max_runtime() {
let root = tmp("timeout");
let c = cfg(&root, "sleep 30 & sleep 30");
let log = one_batch_broker(
&c,
json!({"batch_id":4,"seq":1,"inputs":[],"max_runtime_s":1}),
);
let t = std::time::Instant::now();
assert_eq!(run_once(&c), Ok(true), "202 from /done is success");
assert!(t.elapsed() < Duration::from_secs(10), "{:?}", t.elapsed());
let d = done_body(&log);
assert_eq!(d["exit_code"], 124);
assert!(d["stderr_tail"].as_str().unwrap().contains("max runtime"));
assert_eq!(d["output_hashes"], json!({}));
}
#[test]
fn subdirectory_output_fails_loudly() {
let root = tmp("subdir");
let c = cfg(
&root,
"mkdir \"$ZAKURO_OUT_DIR/results\"; echo x > \"$ZAKURO_OUT_DIR/results/a.json\"; echo y > \"$ZAKURO_OUT_DIR/ok.txt\"",
);
let log = one_batch_broker(&c, json!({"batch_id":6,"seq":1,"inputs":[]}));
assert_eq!(run_once(&c), Ok(true));
let d = done_body(&log);
assert_eq!(d["exit_code"], 1);
assert!(d["stderr_tail"].as_str().unwrap().contains("results"));
assert!(!log.lock().unwrap().iter().any(|(_, p, _)| p == "/outputs"));
}
#[test]
fn symlink_output_is_an_error() {
let root = tmp("symlink");
std::fs::create_dir_all(&root).unwrap();
std::os::unix::fs::symlink("/etc/hosts", root.join("link")).unwrap();
assert!(list_outputs(&root).is_err());
std::fs::remove_file(root.join("link")).unwrap();
std::fs::write(root.join("f"), b"x").unwrap();
assert_eq!(list_outputs(&root).unwrap(), vec!["f".to_string()]);
}
#[test]
fn too_many_outputs_exit_2_no_upload() {
let root = tmp("many");
let c = cfg(
&root,
"sh -c 'for i in $(seq 1 65); do echo $i > \"$ZAKURO_OUT_DIR/f$i\"; done'",
);
let served = Arc::new(Mutex::new(false));
let log = fake_broker(&c.socket, move |m, p, _| match (m, p) {
("GET", "/next") => {
let mut s = served.lock().unwrap();
let first = !*s;
*s = true;
if first {
(200, json!({"batch_id":5,"seq":1,"inputs":[]}))
} else {
(204, Value::Null)
}
}
("POST", "/done") => (200, json!({"accepted":true})),
_ => (404, Value::Null),
});
assert_eq!(run_once(&c), Ok(true));
let d = done_body(&log);
assert_eq!(d["exit_code"], 2);
assert!(d["stderr_tail"]
.as_str()
.unwrap()
.contains("too many outputs"));
assert!(!log.lock().unwrap().iter().any(|(_, p, _)| p == "/outputs"));
}
#[test]
fn dirs_are_cleaned_between_batches() {
let root = tmp("clean");
let c = cfg(
&root,
"sh -c 'ls \"$ZAKURO_IN_DIR\" \"$ZAKURO_OUT_DIR\" >&2; echo z > \"$ZAKURO_OUT_DIR/o\"'",
);
let (base, _puts) = s3(vec![("a", "1")]);
let n = Arc::new(Mutex::new(0u32));
let base2 = base.clone();
let log = fake_broker(&c.socket, move |m, p, _| match (m, p) {
("GET", "/next") => {
let mut k = n.lock().unwrap();
*k += 1;
if *k <= 2 {
(
200,
json!({"batch_id":*k,"seq":*k,"inputs":[{"name":"a","url":format!("{base2}/in/a")}]}),
)
} else {
(204, Value::Null)
}
}
("POST", "/outputs") => (200, json!({"urls":{"o":format!("{base2}/out/o")}})),
("POST", "/done") => (200, json!({"accepted":true})),
_ => (404, Value::Null),
});
std::fs::create_dir_all(&c.out_dir).unwrap();
std::fs::write(c.out_dir.join("stale"), "x").unwrap();
assert_eq!(run_once(&c), Ok(true));
assert_eq!(run_once(&c), Ok(true));
let g = log.lock().unwrap();
let dones: Vec<Value> = g
.iter()
.filter(|(_, p, _)| p == "/done")
.map(|(_, _, b)| serde_json::from_slice(b).unwrap())
.collect();
assert_eq!(dones.len(), 2);
for d in &dones {
assert_eq!(d["output_hashes"].as_object().unwrap().len(), 1, "{d}");
let t = d["stderr_tail"].as_str().unwrap();
assert!(!t.contains("stale"), "{t}");
assert!(!t.contains("\no"), "{t}");
}
assert!(std::fs::read_dir(&c.out_dir).unwrap().next().is_none());
assert!(std::fs::read_dir(&c.in_dir).unwrap().next().is_none());
}
fn envf<'a>(pairs: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option<String> + 'a {
move |k| {
pairs
.iter()
.find(|(a, _)| *a == k)
.map(|(_, v)| v.to_string())
}
}
fn sv(a: &[&str]) -> Vec<String> {
a.iter().map(|s| s.to_string()).collect()
}
#[test]
fn cli_env_cmd_wins_over_argv() {
let e = envf(&[("ZAKURO_SPACE_ID", "e1"), ("ZAKURO_SPACE_CMD", "echo hi")]);
let (id, c) = parse_cli(&sv(&["--", "ls"]), &e).unwrap();
assert_eq!(id, "e1");
assert_eq!(c, Cmd::Shell("echo hi".into()));
}
#[test]
fn cli_argv_command_used_when_env_unset() {
let e = envf(&[("ZAKURO_SPACE_ID", "e1")]);
let (_, c) = parse_cli(&sv(&["--", "python", "-u", "x.py"]), &e).unwrap();
assert_eq!(c, Cmd::Argv(sv(&["python", "-u", "x.py"])));
}
#[test]
fn cli_missing_command_is_error() {
let e = envf(&[("ZAKURO_SPACE_ID", "e1")]);
assert!(parse_cli(&[], &e).unwrap_err().contains("no command"));
let e = envf(&[("ZAKURO_SPACE_ID", "e1"), ("ZAKURO_SPACE_CMD", " ")]);
assert!(parse_cli(&[], &e).is_err());
}
#[test]
fn cli_missing_space_id_is_error() {
let e = envf(&[("ZAKURO_SPACE_CMD", "true")]);
assert!(parse_cli(&[], &e).unwrap_err().contains("Space ID"));
}
#[test]
fn cli_flag_overrides_env_space_id() {
let e = envf(&[("ZAKURO_SPACE_ID", "env"), ("ZAKURO_SPACE_CMD", "true")]);
assert_eq!(
parse_cli(&sv(&["--space-id", "flag"]), &e).unwrap().0,
"flag"
);
assert_eq!(parse_cli(&sv(&["--space-id=f2"]), &e).unwrap().0, "f2");
assert!(parse_cli(&sv(&["--space-id"]), &e).is_err());
}
#[test]
fn argv_command_runs_directly() {
let root = tmp("argv");
let mut c = cfg(&root, "");
c.cmd = Some(Cmd::Argv(sv(&["sh", "-c", "echo boom >&2; exit 3"])));
let served = Arc::new(Mutex::new(false));
let log = fake_broker(&c.socket, move |m, p, _| match (m, p) {
("GET", "/next") => {
let mut s = served.lock().unwrap();
let first = !*s;
*s = true;
if first {
(200, json!({"batch_id":5,"seq":1,"inputs":[]}))
} else {
(204, Value::Null)
}
}
("POST", "/done") => (200, json!({"accepted":true})),
_ => (404, Value::Null),
});
assert_eq!(run_once(&c), Ok(true));
let d = done_body(&log);
assert_eq!(d["exit_code"], 3);
assert!(d["stderr_tail"].as_str().unwrap().contains("boom"));
}
#[test]
fn tail_keeps_last_bytes_on_char_boundary() {
let s = "あ".repeat(3000);
let t = tail_utf8(s.as_bytes(), STDERR_TAIL_MAX);
assert!(t.len() <= STDERR_TAIL_MAX);
assert!(t.chars().all(|c| c == 'あ'));
assert_eq!(tail_utf8(b"abc", 10), "abc");
}
#[test]
fn worker_args_strips_the_subcommand_so_it_is_not_run_as_the_command() {
let argv: Vec<String> = ["worker", "--", "sha256sum", "/work/in/a"]
.iter()
.map(|s| s.to_string())
.collect();
let rest = worker_args(&argv).expect("worker subcommand");
let (_, cmd) = parse_cli(rest, &|k| (k == "ZAKURO_SPACE_ID").then(|| "s1".into())).unwrap();
match cmd {
Cmd::Argv(a) => assert_eq!(a, vec!["sha256sum", "/work/in/a"]),
other => panic!("unexpected {other:?}"),
}
assert!(worker_args(&["other".to_string()]).is_none());
assert!(worker_args(&[]).is_none());
}
}