use super::spec::{self, Format};
use crate::agent::client::{AgentClient, SseMessage};
use serde_json::Value;
use std::path::{Path, PathBuf};
use std::time::Duration;
const USAGE: &str = "usage: zc job submit <spec.toml|spec.json|-> [--json]
zc job ls [--json]
zc job status <id> [--json]
zc job watch <id> [--json]
zc job results <id> [--out <dir>] [--json]
zc job cancel <id> [--json]";
const NO_AGENT: &str =
"the zc agent is not running. A job outlives the terminal only when the agent does: \
start it with `zc agent run` (or `zc agent install` to keep it running at login)";
#[derive(Debug, PartialEq)]
pub enum Cmd {
Submit { spec: String },
Ls,
Status { id: String },
Watch { id: String },
Results { id: String, out: Option<PathBuf> },
Cancel { id: String },
Help,
}
pub fn parse(args: &[String]) -> Result<Cmd, String> {
let (verb, rest) = args.split_first().ok_or("which verb?")?;
let mut out: Option<PathBuf> = None;
let mut positional: Vec<&String> = vec![];
let mut it = rest.iter();
while let Some(a) = it.next() {
match a.as_str() {
"--json" => {}
"--out" | "-o" => {
out = Some(PathBuf::from(it.next().ok_or("--out needs a directory")?))
}
s if s.starts_with("--out=") => out = Some(PathBuf::from(&s["--out=".len()..])),
"-h" | "--help" => return Ok(Cmd::Help),
"-" => positional.push(a),
s if s.starts_with('-') => return Err(format!("unknown argument '{s}'")),
_ => positional.push(a),
}
}
let one = |what: &str| -> Result<String, String> {
match positional.as_slice() {
[x] => Ok((*x).clone()),
[] => Err(format!("{verb} needs {what}")),
_ => Err(format!("{verb} takes one {what}")),
}
};
let none = || -> Result<(), String> {
if positional.is_empty() {
Ok(())
} else {
Err(format!("{verb} takes no argument"))
}
};
if out.is_some() && verb != "results" {
return Err("--out belongs to `zc job results`".into());
}
match verb.as_str() {
"submit" => Ok(Cmd::Submit {
spec: one("a spec file")?,
}),
"ls" | "list" => none().map(|_| Cmd::Ls),
"status" => Ok(Cmd::Status {
id: one("a job id")?,
}),
"watch" => Ok(Cmd::Watch {
id: one("a job id")?,
}),
"results" => Ok(Cmd::Results {
id: one("a job id")?,
out,
}),
"cancel" => Ok(Cmd::Cancel {
id: one("a job id")?,
}),
"help" | "-h" | "--help" => Ok(Cmd::Help),
other => Err(format!("unknown verb '{other}'")),
}
}
pub fn run(args: &[String], json: bool) -> i32 {
let cmd = match parse(args) {
Ok(c) => c,
Err(e) => {
eprintln!("zc job: {e}\n{USAGE}");
return 2;
}
};
if cmd == Cmd::Help {
println!("{USAGE}");
return 0;
}
let Some(dir) = crate::credentials::dir().map(|d| d.join("agent")) else {
eprintln!("zc job: no HOME or ZAKURO_HOME");
return 1;
};
let client = match AgentClient::from_dir(&dir) {
Some(c) if c.summary(Duration::from_secs(2)).is_ok() => c,
_ => {
eprintln!("zc job: {NO_AGENT}");
return 1;
}
};
execute(&client, &dir, cmd, json)
}
pub fn execute(client: &AgentClient, agent_dir: &Path, cmd: Cmd, json: bool) -> i32 {
match cmd {
Cmd::Submit { spec } => submit(client, &spec, json),
Cmd::Ls => ls(client, json),
Cmd::Status { id } => status(client, &id, json),
Cmd::Watch { id } => watch(client, agent_dir, &id, json),
Cmd::Results { id, out } => results(client, &id, out.as_deref(), json),
Cmd::Cancel { id } => cancel(client, &id, json),
Cmd::Help => 0,
}
}
fn fail(what: &str, e: impl std::fmt::Display) -> i32 {
eprintln!("zc job {what}: {e}");
1
}
fn pretty(v: &Value) -> String {
serde_json::to_string_pretty(v).unwrap_or_default()
}
fn api_error(status: u16, body: &Value) -> String {
body["error"]["message"]
.as_str()
.map(str::to_string)
.unwrap_or_else(|| format!("the agent answered HTTP {status}"))
}
fn submit(client: &AgentClient, path: &str, json: bool) -> i32 {
let (text, base) = if path == "-" {
let mut t = String::new();
if let Err(e) = std::io::Read::read_to_string(&mut std::io::stdin(), &mut t) {
return fail("submit", format!("stdin: {e}"));
}
(t, std::env::current_dir().unwrap_or_default())
} else {
let p = Path::new(path);
match std::fs::read_to_string(p) {
Ok(t) => {
let base = p
.canonicalize()
.ok()
.and_then(|c| c.parent().map(Path::to_path_buf))
.unwrap_or_default();
(t, base)
}
Err(e) => return fail("submit", format!("{path}: {e}")),
}
};
let format = match Path::new(path).extension().and_then(|e| e.to_str()) {
Some("json") => Some(Format::Json),
Some("toml") => Some(Format::Toml),
_ => None,
};
let mut job = match spec::parse(&text, format) {
Ok(s) => s,
Err(e) => return fail("submit", format!("{path}: {e}")),
};
spec::resolve_paths(&mut job, &base);
let n = job.batch.len();
let ceiling = job.max_price_per_batch * n as f64;
let body = serde_json::to_value(&job).unwrap_or_default();
match client.post_raw("/v2/submissions", Some(body), Duration::from_secs(600)) {
Ok((200..=299, v)) => {
if json {
println!("{}", pretty(&v));
} else {
println!(
"{} {}: {} batch{}, the hub holds at most {} yen ({} per batch)",
if v["duplicate"] == true {
"already submitted as"
} else {
"submitted"
},
v["id"].as_str().unwrap_or("?"),
n,
if n == 1 { "" } else { "es" },
trim_num(ceiling),
trim_num(job.max_price_per_batch),
);
println!(
" watch it with `zc job watch {}`; results land in the agent's results directory",
v["id"].as_str().unwrap_or("<id>")
);
}
0
}
Ok((status, v)) => fail("submit", api_error(status, &v)),
Err(e) => fail("submit", format!("the agent is not reachable: {e}")),
}
}
fn trim_num(n: f64) -> String {
if n.fract() == 0.0 {
format!("{}", n as i64)
} else {
format!("{n}")
}
}
fn progress(s: &Value) -> String {
let total = s["batches"].as_u64().unwrap_or(0);
match s["counts"].as_object() {
Some(c) => format!(
"{}/{}",
c.get("succeeded").and_then(Value::as_u64).unwrap_or(0),
total
),
None => format!("-/{total}"),
}
}
pub fn format_row(s: &Value) -> String {
format!(
"{:<14} {:<11} {:>7} {}{}",
s["id"].as_str().unwrap_or("?"),
s["state"].as_str().unwrap_or("?"),
progress(s),
s["name"].as_str().unwrap_or(""),
match (s["error"]["message"].as_str(), s["waiting"].as_str()) {
(Some(e), _) => format!(" -- {e}"),
(None, Some(w)) => format!(" (waiting: {w})"),
_ => String::new(),
}
)
}
fn ls(client: &AgentClient, json: bool) -> i32 {
match client.get("/v2/submissions", Duration::from_secs(10)) {
Ok(v) if json => {
println!("{}", pretty(&v));
0
}
Ok(v) => {
let rows = v["submissions"].as_array().cloned().unwrap_or_default();
if rows.is_empty() {
println!("No submitted jobs. Send one with `zc job submit <spec>`.");
}
for r in &rows {
println!("{}", format_row(r));
}
0
}
Err(e) => fail("ls", e),
}
}
pub fn render_status(d: &Value) -> String {
let mut o = String::new();
let line = |o: &mut String, k: &str, v: String| o.push_str(&format!("{k:<12} {v}\n"));
line(
&mut o,
"job",
format!(
"{} {}",
d["id"].as_str().unwrap_or("?"),
d["name"].as_str().unwrap_or("")
),
);
let mut state = d["state"].as_str().unwrap_or("?").to_string();
if let Some(c) = d["counts"].as_object() {
let g = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
state.push_str(&format!(
" ({} done, {} running, {} waiting, {} failed of {})",
g("succeeded"),
g("claimed"),
g("pending"),
g("failed"),
d["batches"].as_u64().unwrap_or(0)
));
}
if d["cancel_requested"] == true {
state.push_str(" (cancel requested)");
}
line(&mut o, "state", state);
if let Some(w) = d["waiting"].as_str() {
line(&mut o, "waiting", w.to_string());
}
if let Some(e) = d["error"].as_object() {
line(
&mut o,
"error",
format!(
"{}: {}",
e.get("code").and_then(Value::as_str).unwrap_or("?"),
e.get("message").and_then(Value::as_str).unwrap_or("")
),
);
}
let img = &d["image"];
let mut image = img["as_given"].as_str().unwrap_or("?").to_string();
if let Some(dg) = img["hub_resolved_digest"]
.as_str()
.or(img["pinned_digest"].as_str())
{
if !image.contains('@') {
image.push_str(&format!(" -> {dg}"));
}
}
line(&mut o, "image", image);
if let Some(p) = d["max_price_per_batch"].as_f64() {
let n = d["batches"].as_u64().unwrap_or(0) as f64;
line(
&mut o,
"price",
format!(
"at most {} yen per batch, {} yen held in all",
trim_num(p),
trim_num(p * n)
),
);
}
if let Some(s) = d["space_id"].as_str() {
line(&mut o, "hub space", s.to_string());
}
let results = d["result_batches"].as_array().cloned().unwrap_or_default();
if !results.is_empty() {
line(
&mut o,
"results",
format!(
"{} batch(es) in {}; receipts verified: {}",
results.len(),
d["results_dir"].as_str().unwrap_or("?"),
d["receipts_verified"].as_u64().unwrap_or(0)
),
);
for r in &results {
if r["receipt"]["status"] != "verified" {
line(
&mut o,
"",
format!(
"batch {}: receipt {}{}",
r["seq"],
r["receipt"]["status"].as_str().unwrap_or("?"),
r["receipt"]["detail"]
.as_str()
.map(|x| format!(" ({x})"))
.unwrap_or_default()
),
);
}
}
}
o
}
fn get_job(client: &AgentClient, id: &str) -> Result<Value, String> {
client.get(&format!("/v2/submissions/{id}"), Duration::from_secs(10))
}
fn status(client: &AgentClient, id: &str, json: bool) -> i32 {
match get_job(client, id) {
Ok(d) => {
if json {
println!("{}", pretty(&d));
} else {
print!("{}", render_status(&d));
}
0
}
Err(e) => fail("status", e),
}
}
fn is_terminal(state: &str) -> bool {
matches!(state, "done" | "failed" | "cancelled")
}
fn exit_for(state: &str) -> i32 {
if state == "done" {
0
} else {
1
}
}
pub fn render_event(m: &SseMessage, id: &str, json: bool) -> Option<(String, String)> {
if m.event != "submission.state" {
return None;
}
let v: Value = serde_json::from_str(&m.data).ok()?;
let s = &v["data"];
if s["id"].as_str() != Some(id) {
return None;
}
let line = if json { s.to_string() } else { format_row(s) };
Some((s["state"].as_str().unwrap_or("").to_string(), line))
}
fn watch(client: &AgentClient, agent_dir: &Path, id: &str, json: bool) -> i32 {
let first = match get_job(client, id) {
Ok(d) => d,
Err(e) => return fail("watch", e),
};
println!(
"{}",
if json {
first.to_string()
} else {
format_row(&first)
}
);
if let Some(s) = first["state"].as_str() {
if is_terminal(s) {
return exit_for(s);
}
}
let mut last_id: Option<u64> = None;
let mut backoff = Duration::from_millis(500);
let mut printed = first["state"].as_str().unwrap_or("").to_string();
loop {
let mut finished: Option<String> = None;
let result = match AgentClient::from_dir(agent_dir) {
None => Err("no agent.json".to_string()),
Some(c) => c.events(last_id, |m| {
if let Some(i) = m.id.as_deref().and_then(|i| i.parse().ok()) {
last_id = Some(i);
}
backoff = Duration::from_millis(500);
if let Some((state, line)) = render_event(&m, id, json) {
println!("{line}");
printed = state.clone();
if is_terminal(&state) {
finished = Some(state);
return false;
}
}
true
}),
};
if let Some(s) = finished {
return exit_for(&s);
}
if let Some(c) = AgentClient::from_dir(agent_dir) {
if let Ok(d) = get_job(&c, id) {
if let Some(s) = d["state"].as_str() {
if s != printed && is_terminal(s) {
println!("{}", if json { d.to_string() } else { format_row(&d) });
return exit_for(s);
}
}
}
}
if let Err(e) = result {
eprintln!(" (the agent is not reachable: {e}; retrying)");
}
std::thread::sleep(backoff);
backoff = (backoff * 2).min(Duration::from_secs(10));
}
}
pub fn copy_tree(from: &Path, to: &Path) -> std::io::Result<usize> {
std::fs::create_dir_all(to)?;
let mut n = 0;
for e in std::fs::read_dir(from)? {
let e = e?;
let dest = to.join(e.file_name());
if e.file_type()?.is_dir() {
n += copy_tree(&e.path(), &dest)?;
} else if !e.file_name().to_string_lossy().starts_with('.') {
std::fs::copy(e.path(), &dest)?;
n += 1;
}
}
Ok(n)
}
fn results(client: &AgentClient, id: &str, out: Option<&Path>, json: bool) -> i32 {
let d = match get_job(client, id) {
Ok(d) => d,
Err(e) => return fail("results", e),
};
let batches = d["result_batches"].as_array().cloned().unwrap_or_default();
let state = d["state"].as_str().unwrap_or("?");
if batches.is_empty() {
return fail(
"results",
format!("{id} has no results yet (it is {state}); see `zc job status {id}`"),
);
}
let dir = PathBuf::from(d["results_dir"].as_str().unwrap_or(""));
let mut copied = None;
if let Some(out) = out {
match copy_tree(&dir, out) {
Ok(n) => copied = Some(n),
Err(e) => return fail("results", format!("copying to {}: {e}", out.display())),
}
}
if json {
println!(
"{}",
pretty(&serde_json::json!({
"id": id,
"state": state,
"dir": out.map(|o| o.display().to_string()).unwrap_or_else(|| dir.display().to_string()),
"batches": batches,
}))
);
} else {
let shown = out.unwrap_or(&dir);
match copied {
Some(n) => println!("copied {n} file(s) to {}", shown.display()),
None => println!("{}", shown.display()),
}
for b in &batches {
let names: Vec<&str> = b["files"]
.as_array()
.into_iter()
.flatten()
.filter_map(|f| f["name"].as_str())
.collect();
println!(
" batch {}: {} (receipt {})",
b["seq"],
names.join(", "),
b["receipt"]["status"].as_str().unwrap_or("?")
);
}
if state != "done" {
eprintln!("note: the job is {state}, so these are the batches that finished");
}
}
0
}
fn cancel(client: &AgentClient, id: &str, json: bool) -> i32 {
match client.delete_raw(&format!("/v2/submissions/{id}"), Duration::from_secs(10)) {
Ok((200..=299, v)) => {
if json {
println!("{}", pretty(&v));
} else {
println!("cancel requested for {id}; the hub refunds what was not used");
}
0
}
Ok((status, v)) => fail("cancel", api_error(status, &v)),
Err(e) => fail("cancel", format!("the agent is not reachable: {e}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn a(v: &[&str]) -> Vec<String> {
v.iter().map(|s| s.to_string()).collect()
}
#[test]
fn verbs_parse_and_refuse_what_they_do_not_take() {
assert_eq!(
parse(&a(&["submit", "job.toml"])).unwrap(),
Cmd::Submit {
spec: "job.toml".into()
}
);
assert_eq!(
parse(&a(&["submit", "-"])).unwrap(),
Cmd::Submit { spec: "-".into() }
);
assert_eq!(parse(&a(&["ls"])).unwrap(), Cmd::Ls);
assert_eq!(
parse(&a(&["results", "j_1", "--out", "/tmp/x"])).unwrap(),
Cmd::Results {
id: "j_1".into(),
out: Some("/tmp/x".into())
}
);
assert_eq!(
parse(&a(&["results", "j_1", "--out=/tmp/x"])).unwrap(),
Cmd::Results {
id: "j_1".into(),
out: Some("/tmp/x".into())
}
);
assert!(parse(&a(&["status"])).is_err());
assert!(parse(&a(&["status", "a", "b"])).is_err());
assert!(parse(&a(&["ls", "x"])).is_err());
assert!(parse(&a(&["cancel", "j", "--out", "d"])).is_err());
assert!(parse(&a(&["bogus"])).is_err());
assert!(parse(&a(&["status", "j", "--nope"])).is_err());
assert!(parse(&a(&[])).is_err());
assert_eq!(
parse(&a(&["watch", "j_1", "--json"])).unwrap(),
Cmd::Watch { id: "j_1".into() }
);
}
#[test]
fn a_row_shows_progress_and_why_a_job_is_stuck_or_failed() {
let running = json!({"id":"j_a","state":"running","batches":10,"name":"digest",
"counts":{"pending":5,"claimed":2,"succeeded":3,"failed":0}});
assert!(
format_row(&running).contains("3/10"),
"{}",
format_row(&running)
);
let waiting = json!({"id":"j_a","state":"uploading","batches":2,"name":"d",
"waiting":"could not reach the hub"});
assert!(format_row(&waiting).contains("(waiting: could not reach the hub)"));
let failed = json!({"id":"j_a","state":"failed","batches":2,"name":"d",
"error":{"code":"insufficient_funds","message":"the hub refused the hold"}});
assert!(format_row(&failed).contains("-- the hub refused the hold"));
}
#[test]
fn status_says_the_ceiling_the_digest_and_unverified_receipts() {
let d = json!({"id":"j_a","name":"digest","state":"done","batches":2,
"counts":{"pending":0,"claimed":0,"succeeded":2,"failed":0},
"image":{"as_given":"r.io/app:1","pinned_digest":"sha256:ab","hub_resolved_digest":"sha256:ab"},
"max_price_per_batch":50.0,"space_id":"sp-1","results_dir":"/h/results/j_a",
"receipts_verified":1,
"result_batches":[
{"seq":1,"files":[],"receipt":{"status":"verified"}},
{"seq":2,"files":[],"receipt":{"status":"unavailable","detail":"no hash"}}]});
let t = render_status(&d);
assert!(t.contains("2 done"), "{t}");
assert!(t.contains("r.io/app:1 -> sha256:ab"), "{t}");
assert!(t.contains("at most 50 yen per batch, 100 yen held"), "{t}");
assert!(t.contains("batch 2: receipt unavailable (no hash)"), "{t}");
assert!(t.contains("/h/results/j_a"), "{t}");
}
#[test]
fn watch_follows_only_its_job_and_stops_on_a_final_state() {
let ev = |id: &str, state: &str| SseMessage {
id: Some("1".into()),
event: "submission.state".into(),
data: json!({"id":1,"type":"submission.state","at":"x","data":{
"id":id,"state":state,"batches":1,"name":"n"}})
.to_string(),
};
assert!(render_event(&ev("j_other", "done"), "j_a", false).is_none());
let (state, line) = render_event(&ev("j_a", "done"), "j_a", false).unwrap();
assert_eq!(state, "done");
assert!(line.contains("j_a"));
let other = SseMessage {
event: "summary.changed".into(),
..ev("j_a", "done")
};
assert!(render_event(&other, "j_a", false).is_none());
assert!(is_terminal("failed") && is_terminal("cancelled") && !is_terminal("running"));
assert_eq!((exit_for("done"), exit_for("failed")), (0, 1));
}
#[test]
fn results_are_copied_without_the_partial_download_files() {
let src = super::super::spec::test_dir("cli-copy-src");
let dst = super::super::spec::test_dir("cli-copy-dst").join("out");
std::fs::create_dir_all(src.join("1")).unwrap();
std::fs::write(src.join("1/o.txt"), "x").unwrap();
std::fs::write(src.join("1/receipt.json"), "{}").unwrap();
std::fs::write(src.join("1/.o.txt.part"), "half").unwrap();
assert_eq!(copy_tree(&src, &dst).unwrap(), 2);
assert!(dst.join("1/o.txt").exists() && !dst.join("1/.o.txt.part").exists());
}
}