use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use crate::atelier_upload::{self, DatasetArg, FilePlan, Target};
use crate::credentials;
const TERMINAL: [&str; 3] = ["delivered", "failed", "cancelled"];
const POLL: Duration = Duration::from_secs(2);
const DEFAULT_TIMEOUT_SECS: u64 = 3600;
const HUB_OUTAGE_TOLERANCE: Duration = Duration::from_secs(300);
const USAGE: &str = "usage: zc atelier run|submit SPEC.yaml [--dataset REF|FILE] [--timeout SECS] [--out DIR] [--no-artifacts]
zc atelier status|logs|cancel JOB
zc atelier results JOB [--out DIR]
--dataset REF a dataset already on the hub (its id), or a URL
--dataset FILE a file on this machine: uploaded as a private dataset, then used";
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Args {
pub verb: String,
pub target: String,
pub dataset: Option<String>,
pub timeout_secs: Option<u64>,
pub out: Option<PathBuf>,
pub no_artifacts: bool,
}
pub fn parse_args(args: &[String]) -> Result<Args, String> {
let mut out = Args::default();
let mut positional = Vec::new();
let mut it = args.iter();
while let Some(a) = it.next() {
let mut value = |name: &str| {
it.next()
.cloned()
.ok_or_else(|| format!("{name} needs a value"))
};
match a.as_str() {
"--dataset" => out.dataset = Some(value("--dataset")?),
"--timeout" => {
let v = value("--timeout")?;
out.timeout_secs = Some(
v.parse()
.map_err(|_| format!("--timeout takes seconds, got {v:?}"))?,
);
}
"--out" => out.out = Some(PathBuf::from(value("--out")?)),
"--no-artifacts" => out.no_artifacts = true,
"--json" => {}
flag if flag.starts_with("--") => return Err(format!("unknown option {flag}")),
_ => positional.push(a.clone()),
}
}
let mut positional = positional.into_iter();
out.verb = positional.next().ok_or("missing verb")?;
if !matches!(
out.verb.as_str(),
"run" | "submit" | "status" | "logs" | "results" | "cancel"
) {
return Err(format!("unknown verb {:?}", out.verb));
}
out.target = positional
.next()
.ok_or_else(|| format!("zc atelier {} needs an argument", out.verb))?;
if let Some(extra) = positional.next() {
return Err(format!("unexpected argument {extra:?}"));
}
Ok(out)
}
pub fn hub_error(text: &str) -> String {
let Ok(v) = serde_json::from_str::<Value>(text) else {
return text.trim().chars().take(400).collect();
};
let detail = v.get("detail").unwrap_or(&v);
if let Some(errors) = detail.get("errors").and_then(Value::as_array) {
return errors
.iter()
.map(|e| e.as_str().map(str::to_string).unwrap_or(e.to_string()))
.collect::<Vec<_>>()
.join("; ");
}
match detail {
Value::String(s) => s.clone(),
other => other.to_string().chars().take(400).collect(),
}
}
pub fn artifact_formats(job: &Value) -> Vec<String> {
let mut formats: Vec<String> = job
.get("atelier_artifacts")
.and_then(Value::as_array)
.map(|rows| {
rows.iter()
.filter_map(|r| r.get("format").and_then(Value::as_str))
.map(str::to_string)
.collect()
})
.unwrap_or_default();
formats.dedup();
formats
}
pub fn safe_filename(name: &str, fallback: &str) -> String {
let base = name.rsplit(['/', '\\']).next().unwrap_or("");
if base.is_empty() || base == "." || base == ".." {
fallback.to_string()
} else {
base.to_string()
}
}
pub fn same_origin(base: &str, url: &str) -> bool {
url.strip_prefix(base)
.is_some_and(|rest| rest.is_empty() || rest.starts_with('/') || rest.starts_with('?'))
}
#[derive(Debug, PartialEq, Eq)]
struct Uploaded {
dataset_id: String,
uri: String,
}
struct Hub {
base: String,
key: String,
agent: ureq::Agent,
}
impl Hub {
fn from_env() -> Result<Self, String> {
credentials::load_into_env();
let base = std::env::var("ZAKURO_API_URL")
.ok()
.filter(|u| !u.trim().is_empty())
.unwrap_or_else(credentials::default_api_url);
let key = std::env::var("ZAKURO_API_KEY")
.ok()
.filter(|k| !k.trim().is_empty())
.ok_or_else(|| "no API key; run `zc login` first".to_string())?;
let agent = ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_connect(Some(Duration::from_secs(10)))
.timeout_recv_response(Some(Duration::from_secs(120)))
.http_status_as_error(false)
.build(),
);
Ok(Self {
base: base.trim_end_matches('/').to_string(),
key: key.trim().to_string(),
agent,
})
}
fn call(&self, method: &str, path: &str, body: Option<Value>) -> Result<Value, String> {
self.api(method, &format!("/api/forge{path}"), body)
}
fn api(&self, method: &str, path: &str, body: Option<Value>) -> Result<Value, String> {
let url = format!("{}{path}", self.base);
let auth = format!("Bearer {}", self.key);
let response = match (method, body) {
("GET", _) => self.agent.get(&url).header("Authorization", &auth).call(),
(_, Some(b)) => self
.agent
.post(&url)
.header("Authorization", &auth)
.send_json(b),
(_, None) => self
.agent
.post(&url)
.header("Authorization", &auth)
.send_empty(),
}
.map_err(|e| format!("the hub did not answer: {e}"))?;
let status = response.status();
let text = response
.into_body()
.read_to_string()
.map_err(|e| format!("reading the hub's answer: {e}"))?;
if !status.is_success() {
return Err(format!("HTTP {}: {}", status.as_u16(), hub_error(&text)));
}
serde_json::from_str(&text).or(Ok(Value::String(text)))
}
fn job(&self, id: &str) -> Result<Value, String> {
self.call("GET", &format!("/jobs/{id}"), None)
}
fn submit(&self, spec_yaml: &str, dataset: Option<&str>) -> Result<String, String> {
let mut create = json!({"spec_yaml": spec_yaml});
if let Some(d) = dataset {
create["dataset_ref"] = json!(d);
}
let job = self
.call("POST", "/jobs", Some(create))
.map_err(|e| format!("create: {e}"))?;
let id = job
.get("id")
.and_then(Value::as_str)
.ok_or("create: the hub returned no job id")?
.to_string();
self.call(
"POST",
&format!("/jobs/{id}/confirm"),
Some(json!({"spec_yaml": spec_yaml})),
)
.map_err(|e| format!("confirm {id}: {e}"))?;
self.call("POST", &format!("/jobs/{id}/accept"), None)
.map_err(|e| format!("accept {id}: {e}"))?;
Ok(id)
}
fn put_presigned(
&self,
url: &str,
sha256: &str,
bytes: &[u8],
) -> Result<Option<String>, String> {
let full = if url.starts_with("http://") || url.starts_with("https://") {
url.to_string()
} else {
format!("{}{url}", self.base)
};
let response = self
.agent
.put(&full)
.header(
"x-amz-checksum-sha256",
&atelier_upload::checksum_header(sha256)?,
)
.send(bytes)
.map_err(|e| format!("storage did not answer: {}", transport_reason(&e)))?;
let status = response.status();
let etag = response
.headers()
.get("etag")
.and_then(|v| v.to_str().ok())
.map(str::to_string);
if !status.is_success() {
let text = response.into_body().read_to_string().unwrap_or_default();
return Err(format!(
"storage answered HTTP {}: {}",
status.as_u16(),
hub_error(&text)
));
}
Ok(etag)
}
fn upload_dataset(&self, path: &Path) -> Result<Uploaded, String> {
let plan = atelier_upload::plan_file(path)?;
self.upload_planned(path, &plan)
}
fn upload_planned(&self, path: &Path, plan: &FilePlan) -> Result<Uploaded, String> {
let what = &plan.path;
let fail = |step: &str, e: String| format!("uploading {what}: {step}: {e}");
let answer = self
.api(
"POST",
"/api/datasets/uploads",
Some(atelier_upload::open_body(plan)),
)
.map_err(|e| fail("open", e))?;
let opened = atelier_upload::parse_open(&answer, plan).map_err(|e| fail("open", e))?;
let uploads = format!("/api/datasets/uploads/{}", opened.upload_id);
let read = |offset: u64, length: u64| -> Result<Vec<u8>, String> {
use std::io::{Read, Seek, SeekFrom};
let mut file = std::fs::File::open(path).map_err(|e| e.to_string())?;
file.seek(SeekFrom::Start(offset))
.map_err(|e| e.to_string())?;
let mut bytes = Vec::with_capacity(length as usize);
file.take(length)
.read_to_end(&mut bytes)
.map_err(|e| e.to_string())?;
if bytes.len() as u64 != length {
return Err("the file changed while it was uploaded".to_string());
}
Ok(bytes)
};
match &opened.target {
Target::Landed => {}
Target::Single(url) => {
let bytes = read(0, plan.size).map_err(|e| fail("read", e))?;
self.put_presigned(url, &plan.sha256, &bytes)
.map_err(|e| fail("put", e))?;
}
Target::Parts(parts) => {
let ranges = atelier_upload::part_ranges(plan.size, plan.part_size.unwrap_or(1));
let mut landed = Vec::new();
for ((number, url), (_, offset, length)) in parts.iter().zip(ranges) {
let step = format!("part {number} of {}", parts.len());
let bytes = read(offset, length).map_err(|e| fail(&step, e))?;
let sha = &plan.part_sha256s[(*number - 1) as usize];
let etag = self
.put_presigned(url, sha, &bytes)
.map_err(|e| fail(&step, e))?
.ok_or_else(|| fail(&step, "storage answered without an ETag".into()))?;
landed.push(json!({"part_number": number, "etag": etag}));
}
let segments: Vec<String> =
what.split('/').map(crate::dataset::urlencode).collect();
self.api(
"POST",
&format!("{uploads}/files/{}/complete", segments.join("/")),
Some(json!({"parts": landed})),
)
.map_err(|e| fail("assemble", e))?;
}
}
let version = self
.api("POST", &format!("{uploads}/complete"), Some(json!({})))
.map_err(|e| fail("complete", e))?;
let status = version.get("status").and_then(Value::as_str).unwrap_or("-");
if status != "ready" {
return Err(fail(
"complete",
format!("the hub left the dataset {status}, not ready"),
));
}
let digest = version.get("digest").and_then(Value::as_str).unwrap_or("");
if digest.len() != 64 || !digest.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(fail(
"complete",
"the hub returned no version digest".to_string(),
));
}
let uploaded = Uploaded {
uri: atelier_upload::dataset_uri(&opened.dataset_id, digest),
dataset_id: opened.dataset_id,
};
let verb = if opened.target == Target::Landed {
"already on the hub"
} else {
"uploaded"
};
eprintln!(
"{verb}: {what} ({} bytes) as private dataset {} ({})\n {}",
plan.size, uploaded.dataset_id, plan.name, uploaded.uri
);
Ok(uploaded)
}
fn logs(&self, id: &str) -> Result<String, String> {
let v = self.call("GET", &format!("/jobs/{id}/logs"), None)?;
Ok(v.get("text")
.and_then(Value::as_str)
.unwrap_or("")
.to_string())
}
fn download(&self, url: &str, dest: &Path, sha256: &str) -> Result<u64, String> {
let (full, same_origin) = if url.starts_with("http://") || url.starts_with("https://") {
(url.to_string(), same_origin(&self.base, url))
} else {
(format!("{}{url}", self.base), true)
};
let mut req = self.agent.get(&full);
if same_origin {
req = req.header("Authorization", &format!("Bearer {}", self.key));
}
let response = req.call().map_err(|e| format!("download: {e}"))?;
if !response.status().is_success() {
return Err(format!("download: HTTP {}", response.status().as_u16()));
}
let mut reader = response.into_body().into_reader();
let mut file =
std::fs::File::create(dest).map_err(|e| format!("{}: {e}", dest.display()))?;
let mut ctx = ring::digest::Context::new(&ring::digest::SHA256);
let mut buf = vec![0u8; 1 << 16];
let mut total = 0u64;
loop {
let n =
std::io::Read::read(&mut reader, &mut buf).map_err(|e| format!("download: {e}"))?;
if n == 0 {
break;
}
ctx.update(&buf[..n]);
file.write_all(&buf[..n])
.map_err(|e| format!("{}: {e}", dest.display()))?;
total += n as u64;
}
let got = hex::encode(ctx.finish());
if !sha256.is_empty() && got != sha256 {
let _ = std::fs::remove_file(dest);
return Err(format!(
"{}: sha256 mismatch (hub {sha256}, received {got})",
dest.display()
));
}
Ok(total)
}
}
fn transport_reason(e: &ureq::Error) -> String {
match e {
ureq::Error::Io(io) => io.to_string(),
ureq::Error::Timeout(t) => format!("timeout ({t})"),
ureq::Error::HostNotFound => "host not found".to_string(),
ureq::Error::ConnectionFailed => "connection failed".to_string(),
ureq::Error::Tls(m) => format!("TLS: {m}"),
_ => "request failed".to_string(),
}
}
fn results_dir(job_id: &str, out: Option<&Path>) -> PathBuf {
match out {
Some(p) => p.to_path_buf(),
None => credentials::dir()
.unwrap_or_else(|| PathBuf::from("."))
.join("results")
.join(job_id),
}
}
fn fetch_results(
hub: &Hub,
job: &Value,
dir: &Path,
artifacts: bool,
) -> Result<Vec<Value>, String> {
let id = job.get("id").and_then(Value::as_str).unwrap_or("job");
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let write = |name: &str, bytes: &[u8]| {
std::fs::write(dir.join(name), bytes).map_err(|e| format!("{name}: {e}"))
};
write(
"job.json",
serde_json::to_string_pretty(job)
.unwrap_or_default()
.as_bytes(),
)?;
if let Some(spec) = job.get("spec_yaml").and_then(Value::as_str) {
write("spec.yaml", spec.as_bytes())?;
}
if let Ok(log) = hub.logs(id) {
write("log.txt", log.as_bytes())?;
}
let mut files = Vec::new();
if !artifacts || job.get("status").and_then(Value::as_str) != Some("delivered") {
return Ok(files);
}
for format in artifact_formats(job) {
let view = hub.call(
"GET",
&format!("/jobs/{id}/artifacts/url?format={format}"),
None,
)?;
let url = view.get("url").and_then(Value::as_str).unwrap_or("");
let sha = view.get("sha256").and_then(Value::as_str).unwrap_or("");
let name = safe_filename(
view.get("filename").and_then(Value::as_str).unwrap_or(""),
&format!("artifact.{format}"),
);
let bytes = hub.download(url, &dir.join(&name), sha)?;
files.push(json!({"format": format, "file": name, "bytes": bytes, "sha256": sha}));
}
Ok(files)
}
fn status_line(job: &Value) -> String {
let s = |k: &str| job.get(k).and_then(Value::as_str).unwrap_or("-");
let stage = job
.pointer("/progress/stage")
.and_then(Value::as_str)
.unwrap_or("-");
format!("{} {} stage={}", s("id"), s("status"), stage)
}
fn is_terminal(job: &Value) -> bool {
job.get("status")
.and_then(Value::as_str)
.is_some_and(|s| TERMINAL.contains(&s))
}
fn wait(hub: &Hub, id: &str, timeout: Duration, quiet: bool) -> Result<Option<Value>, String> {
let started = Instant::now();
let mut seen = String::new();
let mut first_miss: Option<Instant> = None;
loop {
match hub.job(id) {
Ok(job) => {
first_miss = None;
let line = status_line(&job);
if line != seen {
if !quiet {
eprintln!("[{:>5.0}s] {line}", started.elapsed().as_secs_f64());
}
seen = line;
}
if is_terminal(&job) {
return Ok(Some(job));
}
}
Err(e) => {
let since = *first_miss.get_or_insert_with(Instant::now);
if since.elapsed() > HUB_OUTAGE_TOLERANCE {
return Err(format!(
"lost the hub for {}s while waiting: {e}",
HUB_OUTAGE_TOLERANCE.as_secs()
));
}
}
}
if started.elapsed() > timeout {
return Ok(None);
}
std::thread::sleep(POLL);
}
}
fn exit_for(job: &Value) -> i32 {
match job.get("status").and_then(Value::as_str) {
Some("delivered") => 0,
_ => 1,
}
}
fn run_inner(args: &Args, json_out: bool) -> Result<i32, String> {
let hub = Hub::from_env()?;
match args.verb.as_str() {
"submit" | "run" => {
let spec = std::fs::read_to_string(&args.target)
.map_err(|e| format!("{}: {e}", args.target))?;
let mut spec = spec;
let dataset = match args
.dataset
.as_deref()
.map(atelier_upload::classify_dataset)
{
Some(DatasetArg::File(path)) => {
let up = hub.upload_dataset(&path)?;
match atelier_upload::set_data_uri(&spec, &up.uri) {
Some(pointed) => {
eprintln!("spec: data.uri set to the uploaded dataset");
spec = pointed;
}
None => eprintln!(
"spec: no data.uri to set; the job names the dataset by id only"
),
}
Some(up.dataset_id)
}
Some(DatasetArg::Ref(reference)) => Some(reference),
None => None,
};
let id = hub.submit(&spec, dataset.as_deref())?;
if args.verb == "submit" {
if json_out {
println!("{}", json!({"job_id": id}));
} else {
println!("{id}");
}
return Ok(0);
}
let timeout = Duration::from_secs(args.timeout_secs.unwrap_or(DEFAULT_TIMEOUT_SECS));
let Some(job) = wait(&hub, &id, timeout, json_out)? else {
eprintln!(
"zc atelier: {id} is still running after {}s",
timeout.as_secs()
);
if json_out {
println!("{}", json!({"job_id": id, "status": "timeout"}));
}
return Ok(3);
};
let dir = results_dir(&id, args.out.as_deref());
let files = fetch_results(&hub, &job, &dir, !args.no_artifacts)?;
let status = job.get("status").and_then(Value::as_str).unwrap_or("-");
if json_out {
println!(
"{}",
json!({"job_id": id, "status": status, "error": job.get("error"),
"results": dir.display().to_string(), "files": files})
);
} else {
println!("{id} {status}");
if let Some(e) = job.get("error").and_then(Value::as_str) {
println!("error: {e}");
}
println!("results: {}", dir.display());
for f in &files {
println!(
" {} ({} bytes)",
f["file"].as_str().unwrap_or("?"),
f["bytes"]
);
}
}
Ok(exit_for(&job))
}
"status" => {
let job = hub.job(&args.target)?;
if json_out {
println!("{job}");
} else {
println!("{}", status_line(&job));
if let Some(e) = job.get("error").and_then(Value::as_str) {
println!("error: {e}");
}
}
Ok(0)
}
"logs" => {
print!("{}", hub.logs(&args.target)?);
Ok(0)
}
"results" => {
let job = hub.job(&args.target)?;
let dir = results_dir(&args.target, args.out.as_deref());
let files = fetch_results(&hub, &job, &dir, !args.no_artifacts)?;
if json_out {
println!(
"{}",
json!({"job_id": args.target, "results": dir.display().to_string(), "files": files})
);
} else {
println!("results: {} ({} artifact(s))", dir.display(), files.len());
}
Ok(0)
}
"cancel" => {
let job = hub.call("POST", &format!("/jobs/{}/cancel", args.target), None)?;
println!("{}", status_line(&job));
Ok(0)
}
_ => unreachable!("parse_args admits known verbs only"),
}
}
pub fn run(args: &[String], json_out: bool) -> i32 {
let parsed = match parse_args(args) {
Ok(a) => a,
Err(e) => {
eprintln!("zc atelier: {e}\n{USAGE}");
return 2;
}
};
match run_inner(&parsed, json_out) {
Ok(code) => code,
Err(e) => {
eprintln!("zc atelier: {e}");
2
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn argv(s: &str) -> Vec<String> {
s.split_whitespace().map(str::to_string).collect()
}
#[test]
fn run_takes_a_spec_and_options_in_any_order() {
let a = parse_args(&argv(
"run --timeout 90 spec.yaml --dataset d1 --out /tmp/x --no-artifacts",
))
.unwrap();
assert_eq!(a.verb, "run");
assert_eq!(a.target, "spec.yaml");
assert_eq!(a.dataset.as_deref(), Some("d1"));
assert_eq!(a.timeout_secs, Some(90));
assert_eq!(a.out, Some(PathBuf::from("/tmp/x")));
assert!(a.no_artifacts);
}
#[test]
fn bad_invocations_are_refused_with_a_reason() {
assert!(parse_args(&argv("")).unwrap_err().contains("missing verb"));
assert!(parse_args(&argv("fly spec.yaml"))
.unwrap_err()
.contains("unknown verb"));
assert!(parse_args(&argv("status"))
.unwrap_err()
.contains("needs an argument"));
assert!(parse_args(&argv("run a b"))
.unwrap_err()
.contains("unexpected"));
assert!(parse_args(&argv("run a --timeout soon"))
.unwrap_err()
.contains("seconds"));
assert!(parse_args(&argv("run a --dataset"))
.unwrap_err()
.contains("needs a value"));
assert!(parse_args(&argv("run a --fast"))
.unwrap_err()
.contains("unknown option"));
}
#[test]
fn hub_errors_read_as_one_line_in_every_envelope() {
assert_eq!(hub_error(r#"{"detail":{"errors":["a","b"]}}"#), "a; b");
assert_eq!(hub_error(r#"{"errors":["only"]}"#), "only");
assert_eq!(
hub_error(r#"{"detail":"Not authenticated"}"#),
"Not authenticated"
);
assert_eq!(hub_error(" plain text \n"), "plain text");
}
#[test]
fn formats_come_from_the_published_artifact_rows() {
let job =
json!({"atelier_artifacts": [{"format": "onnx"}, {"format": "joblib"}, {"path": "x"}]});
assert_eq!(artifact_formats(&job), vec!["onnx", "joblib"]);
assert!(artifact_formats(&json!({"atelier_artifacts": null})).is_empty());
assert!(artifact_formats(&json!({})).is_empty());
}
#[test]
fn a_suggested_filename_cannot_leave_the_results_directory() {
assert_eq!(safe_filename("model.onnx", "f"), "model.onnx");
assert_eq!(safe_filename("../../etc/passwd", "f"), "passwd");
assert_eq!(safe_filename("a\\b\\c.bin", "f"), "c.bin");
assert_eq!(safe_filename("..", "f"), "f");
assert_eq!(safe_filename("", "f"), "f");
assert_eq!(safe_filename("dir/", "f"), "f");
}
#[test]
fn the_key_goes_to_the_hub_and_to_no_host_that_merely_starts_like_it() {
let hub = "https://hub.example.test";
assert!(same_origin(
hub,
"https://hub.example.test/api/forge/jobs/j/spec.yaml"
));
assert!(same_origin(hub, "https://hub.example.test"));
assert!(!same_origin(hub, "https://hub.example.test.evil.test/x"));
assert!(!same_origin(hub, "https://hub.example.test:8443/x"));
assert!(!same_origin(hub, "https://storage.example.test/presigned"));
}
#[derive(Debug, Clone)]
struct Seen {
method: String,
url: String,
auth: Option<String>,
checksum: Option<String>,
body: Vec<u8>,
}
type Log = std::sync::Arc<std::sync::Mutex<Vec<Seen>>>;
fn stub(
answer: impl Fn(&Seen) -> (u16, String, Option<String>) + Send + 'static,
) -> (String, Log) {
let server = tiny_http::Server::http("127.0.0.1:0").unwrap();
let base = format!("http://{}", server.server_addr().to_ip().unwrap());
let log: Log = Log::default();
let seen = log.clone();
std::thread::spawn(move || {
for mut req in server.incoming_requests() {
let header = |name: &'static str| {
req.headers()
.iter()
.find(|h| h.field.equiv(name))
.map(|h| h.value.to_string())
};
let (auth, checksum) = (header("Authorization"), header("x-amz-checksum-sha256"));
let mut body = Vec::new();
let _ = std::io::Read::read_to_end(req.as_reader(), &mut body);
let one = Seen {
method: req.method().to_string(),
url: req.url().to_string(),
auth,
checksum,
body,
};
let (status, text, etag) = answer(&one);
seen.lock().unwrap().push(one);
let mut resp = tiny_http::Response::from_string(text).with_status_code(status);
if let Some(etag) = etag {
resp.add_header(
tiny_http::Header::from_bytes(&b"ETag"[..], etag.as_bytes()).unwrap(),
);
}
let _ = req.respond(resp);
}
});
(base, log)
}
const KEY: &str = "zk_1000000001_dddddddddddddddddddddddddddddddd";
fn hub_at(base: &str) -> Hub {
Hub {
base: base.to_string(),
key: KEY.to_string(),
agent: ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.http_status_as_error(false)
.build(),
),
}
}
fn data_file(tag: &str, bytes: &[u8]) -> PathBuf {
let dir = std::env::temp_dir().join(format!("zc-atelier-cli-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("train.csv");
std::fs::write(&path, bytes).unwrap();
path
}
fn sha(bytes: &[u8]) -> String {
hex::encode(ring::digest::digest(&ring::digest::SHA256, bytes))
}
const UPLOAD: &str = "11111111-2222-3333-4444-555555555555";
const DATASET: &str = "e629e662-1d32-4e52-88e9-b0e83416c852";
const DIGEST: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789";
#[test]
fn a_local_file_is_opened_put_to_storage_without_the_key_and_completed() {
let bytes = b"id,score\n1,0.5\n2,0.7\n";
let path = data_file("single", bytes);
let (storage, stored) = stub(|_| (200, String::new(), Some("\"etag-1\"".into())));
let put = format!("{storage}/bucket/datasets/7/blobs/x?X-Amz-Signature=s3cr3t");
let (base, hub_log) = stub(move |r| match (r.method.as_str(), r.url.as_str()) {
("POST", "/api/datasets/uploads") => (
201,
json!({"upload_id": UPLOAD, "dataset_id": DATASET, "version_id": UPLOAD,
"digest": "d", "expires_at": null,
"files": [{"path": "train.csv", "url": put, "expires_at": "t"}]})
.to_string(),
None,
),
("POST", "/api/datasets/uploads/11111111-2222-3333-4444-555555555555/complete") => (
200,
json!({"id": UPLOAD, "dataset_id": DATASET, "status": "ready", "digest": DIGEST})
.to_string(),
None,
),
_ => (404, "{}".into(), None),
});
let hub = hub_at(&base);
assert_eq!(
hub.upload_dataset(&path).unwrap(),
Uploaded {
dataset_id: DATASET.to_string(),
uri: format!("zc://dataset/{DATASET}@sha256:{DIGEST}"),
}
);
let calls = hub_log.lock().unwrap().clone();
let bearer = format!("Bearer {KEY}");
assert_eq!(
calls
.iter()
.map(|c| (c.method.as_str(), c.url.as_str(), c.auth.as_deref()))
.collect::<Vec<_>>(),
vec![
("POST", "/api/datasets/uploads", Some(bearer.as_str())),
(
"POST",
"/api/datasets/uploads/11111111-2222-3333-4444-555555555555/complete",
Some(bearer.as_str())
),
]
);
let opened: Value = serde_json::from_slice(&calls[0].body).unwrap();
assert_eq!(
opened,
json!({
"name": format!("train-{}", &sha(bytes)[..12]),
"visibility": "private",
"files": [{"path": "train.csv", "sha256": sha(bytes), "size_bytes": bytes.len(),
"meta": {"records": 2, "columns": ["id", "score"]}}],
})
);
let puts = stored.lock().unwrap().clone();
assert_eq!(puts.len(), 1);
assert_eq!(puts[0].method, "PUT");
assert_eq!(
puts[0].url,
"/bucket/datasets/7/blobs/x?X-Amz-Signature=s3cr3t"
);
assert_eq!(puts[0].auth, None, "the key went to storage");
assert_eq!(
puts[0].checksum.as_deref(),
Some(
atelier_upload::checksum_header(&sha(bytes))
.unwrap()
.as_str()
)
);
assert_eq!(puts[0].body, bytes);
}
#[test]
fn a_file_in_parts_sends_each_part_then_assembles_with_the_etags() {
let bytes: Vec<u8> = (0..2500u32).map(|i| b'a' + (i % 23) as u8).collect();
let path = data_file("parts", &bytes);
let plan = atelier_upload::plan_file_with(&path, 1000, 1000).unwrap();
let (storage, stored) = stub(|r| (200, String::new(), Some(format!("\"e{}\"", r.url))));
let urls: Vec<Value> = (1..=3)
.map(|n| json!({"part_number": n, "url": format!("{storage}/p{n}")}))
.collect();
let (base, hub_log) = stub(move |r| match (r.method.as_str(), r.url.as_str()) {
("POST", "/api/datasets/uploads") => (
201,
json!({"upload_id": UPLOAD, "dataset_id": DATASET,
"files": [{"path": "train.csv", "part_size": 1000, "parts": urls}]})
.to_string(),
None,
),
("POST", u) if u.ends_with("/complete") => (
200,
json!({"status": "ready", "landed": true, "digest": DIGEST}).to_string(),
None,
),
_ => (404, "{}".into(), None),
});
assert_eq!(
hub_at(&base)
.upload_planned(&path, &plan)
.unwrap()
.dataset_id,
DATASET
);
let puts = stored.lock().unwrap().clone();
assert_eq!(
puts.iter().map(|p| p.body.clone()).collect::<Vec<_>>(),
vec![
bytes[..1000].to_vec(),
bytes[1000..2000].to_vec(),
bytes[2000..].to_vec()
]
);
for (put, part) in puts.iter().zip(&plan.part_sha256s) {
assert_eq!(put.auth, None);
assert_eq!(
put.checksum.as_deref(),
Some(atelier_upload::checksum_header(part).unwrap().as_str())
);
}
let calls = hub_log.lock().unwrap().clone();
assert_eq!(
calls.iter().map(|c| c.url.as_str()).collect::<Vec<_>>(),
vec![
"/api/datasets/uploads",
"/api/datasets/uploads/11111111-2222-3333-4444-555555555555/files/train.csv/complete",
"/api/datasets/uploads/11111111-2222-3333-4444-555555555555/complete",
]
);
assert_eq!(
serde_json::from_slice::<Value>(&calls[1].body).unwrap(),
json!({"parts": [
{"part_number": 1, "etag": "\"e/p1\""},
{"part_number": 2, "etag": "\"e/p2\""},
{"part_number": 3, "etag": "\"e/p3\""}]})
);
}
#[test]
fn bytes_already_on_the_hub_are_not_sent_again() {
let path = data_file("landed", b"a,b\n1,2\n");
let (base, hub_log) = stub(|r| match r.url.as_str() {
"/api/datasets/uploads" => (
200,
json!({"upload_id": UPLOAD, "dataset_id": DATASET, "files": []}).to_string(),
None,
),
_ => (
200,
json!({"status": "ready", "digest": DIGEST}).to_string(),
None,
),
});
assert_eq!(
hub_at(&base).upload_dataset(&path).unwrap().dataset_id,
DATASET
);
assert_eq!(hub_log.lock().unwrap().len(), 2);
}
#[test]
fn a_refused_upload_names_the_step_the_status_and_the_hubs_reason_not_the_key() {
let path = data_file("refused", b"a,b\n1,2\n");
let (base, _) = stub(|_| {
(
403,
json!({"detail": {"errors": ["an API or device key uploads private datasets only"]}})
.to_string(),
None,
)
});
let err = hub_at(&base).upload_dataset(&path).unwrap_err();
assert_eq!(
err,
"uploading train.csv: open: HTTP 403: an API or device key uploads private datasets only"
);
let (storage, _) = stub(|_| (400, "<Error><Code>BadDigest</Code></Error>".into(), None));
let put = format!("{storage}/k?X-Amz-Signature=s3cr3t");
let (base, hub_log) = stub(move |_| {
(
201,
json!({"upload_id": UPLOAD, "dataset_id": DATASET,
"files": [{"path": "train.csv", "url": put}]})
.to_string(),
None,
)
});
let err = hub_at(&base).upload_dataset(&path).unwrap_err();
assert!(
err.contains("put: storage answered HTTP 400") && err.contains("BadDigest"),
"{err}"
);
assert!(!err.contains("s3cr3t") && !err.contains(KEY), "{err}");
assert_eq!(hub_log.lock().unwrap().len(), 1);
let (base, _) = stub(|r| match r.url.as_str() {
"/api/datasets/uploads" => (
200,
json!({"upload_id": UPLOAD, "dataset_id": DATASET, "files": []}).to_string(),
None,
),
_ => (200, json!({"status": "pending"}).to_string(), None),
});
let err = hub_at(&base).upload_dataset(&path).unwrap_err();
assert!(err.contains("complete") && err.contains("pending"), "{err}");
assert!(!err.contains(KEY), "{err}");
}
#[test]
fn an_uploaded_file_is_submitted_by_its_dataset_id_like_any_reference() {
let (base, hub_log) = stub(|r| match r.url.as_str() {
"/api/forge/jobs" => (200, json!({"id": "j1"}).to_string(), None),
_ => (200, "{}".into(), None),
});
assert_eq!(
hub_at(&base).submit("spec: 1\n", Some(DATASET)).unwrap(),
"j1"
);
let calls = hub_log.lock().unwrap().clone();
assert_eq!(
serde_json::from_slice::<Value>(&calls[0].body).unwrap(),
json!({"spec_yaml": "spec: 1\n", "dataset_ref": DATASET})
);
assert_eq!(calls.len(), 3);
}
#[test]
fn only_a_delivered_job_exits_zero() {
assert_eq!(exit_for(&json!({"status": "delivered"})), 0);
assert_eq!(exit_for(&json!({"status": "failed"})), 1);
assert_eq!(exit_for(&json!({"status": "cancelled"})), 1);
assert!(is_terminal(&json!({"status": "failed"})));
assert!(!is_terminal(&json!({"status": "running"})));
}
}