use super::spaces_hub::{HubErr, SpacesHub};
use super::spec::{self, visibility_and_squad};
use super::store::{
now_rfc3339, write_atomic, BatchResult, Counts, Job, JobError, OutFile, ReceiptState,
Retention, State, Store,
};
use crate::receipt_verify as rv;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use tokio::sync::{watch, Notify};
pub type Creds = Option<(String, String)>;
pub type Emit = Arc<dyn Fn(&str, Value) + Send + Sync>;
pub struct Config {
pub tick: Duration,
pub poll: Duration,
pub backoff_base: Duration,
pub backoff_cap: Duration,
pub login_retry: Duration,
pub retention: Retention,
}
impl Default for Config {
fn default() -> Self {
Self {
tick: Duration::from_secs(2),
poll: Duration::from_secs(10),
backoff_base: Duration::from_secs(2),
backoff_cap: Duration::from_secs(300),
login_retry: Duration::from_secs(15),
retention: Retention::from_env(),
}
}
}
pub struct Dispatcher {
store: Store,
creds: Arc<dyn Fn() -> Creds + Send + Sync>,
emit: Emit,
cfg: Config,
wake: Arc<Notify>,
inflight: Mutex<HashSet<String>>,
due: Mutex<HashMap<String, Instant>>,
last_prune: Mutex<Option<Instant>>,
}
enum Flow {
Next,
Wait(Duration),
Stop,
}
enum StepErr {
Transient(String),
NoCreds,
Permanent { code: String, message: String },
}
impl From<HubErr> for StepErr {
fn from(e: HubErr) -> Self {
match e {
HubErr::Transient(m) => StepErr::Transient(m),
HubErr::Permanent { status, message } => {
let (code, message) = match status {
401 => (
"hub_credential_refused",
format!(
"the hub refused this agent's credential (401: {message}). The Spaces \
routes take a hub session token; the `zk_` key `zc login` saves is \
not accepted there"
),
),
402 => (
"insufficient_funds",
format!("the hub refused the hold: {message}"),
),
403 => ("hub_forbidden", format!("the hub forbids this: {message}")),
404 => (
"hub_not_found",
format!("the hub has no such route or Space: {message}"),
),
409 => ("hub_conflict", format!("the hub refused (409): {message}")),
400 | 413 | 422 => (
"hub_rejected",
format!("the hub rejected the job ({status}): {message}"),
),
_ => (
"hub_refused",
format!("the hub refused ({status}): {message}"),
),
};
StepErr::Permanent {
code: code.into(),
message,
}
}
}
}
}
fn permanent(code: &str, message: impl Into<String>) -> StepErr {
StepErr::Permanent {
code: code.into(),
message: message.into(),
}
}
fn local(e: std::io::Error) -> StepErr {
StepErr::Transient(format!("local disk: {e}"))
}
fn counts_of(space: &Value) -> Counts {
let n = |k: &str| space["counts"][k].as_u64().unwrap_or(0);
Counts {
pending: n("pending"),
claimed: n("claimed"),
succeeded: n("succeeded"),
failed: n("failed"),
}
}
fn safe_name(n: &str) -> bool {
!n.is_empty() && n.len() <= 255 && !n.starts_with('.') && !n.contains(['/', '\\', '\0'])
}
impl Dispatcher {
pub fn new(
store: Store,
creds: Arc<dyn Fn() -> Creds + Send + Sync>,
emit: Emit,
cfg: Config,
wake: Arc<Notify>,
) -> Arc<Self> {
Arc::new(Self {
store,
creds,
emit,
cfg,
wake,
inflight: Mutex::new(HashSet::new()),
due: Mutex::new(HashMap::new()),
last_prune: Mutex::new(None),
})
}
pub async fn run(self: Arc<Self>, mut stop: watch::Receiver<bool>) {
loop {
self.tick().await;
tokio::select! {
biased;
_ = stop.wait_for(|s| *s) => break,
_ = self.wake.notified() => {}
_ = tokio::time::sleep(self.cfg.tick) => {}
}
}
}
pub async fn tick(self: &Arc<Self>) {
let now = Instant::now();
for job in self.store.list() {
let live = !job.state.terminal() || (job.cleanup_cancel && !job.hub.cancel_done);
if !live {
continue;
}
if self
.due
.lock()
.unwrap()
.get(&job.id)
.is_some_and(|t| *t > now)
{
continue;
}
if !self.inflight.lock().unwrap().insert(job.id.clone()) {
continue;
}
let me = self.clone();
let id = job.id.clone();
tokio::spawn(async move {
me.advance(&id).await;
me.inflight.lock().unwrap().remove(&id);
});
}
let prune_now = {
let mut last = self.last_prune.lock().unwrap();
let due = last.is_none_or(|t| t.elapsed() > Duration::from_secs(3600));
if due {
*last = Some(now);
}
due
};
if prune_now {
let store = self.store.clone();
let r = self.cfg.retention;
let _ = tokio::task::spawn_blocking(move || store.prune(r, chrono::Utc::now())).await;
}
}
pub async fn advance(&self, id: &str) {
let Some(mut job) = self.store.load(id) else {
return;
};
loop {
match self.step(&mut job).await {
Ok(Flow::Next) => {
self.clear_waiting(&mut job);
}
Ok(Flow::Wait(d)) => {
self.clear_waiting(&mut job);
self.set_due(id, d);
return;
}
Ok(Flow::Stop) => return,
Err(StepErr::Transient(m)) => {
job.attempts += 1;
let shift = (job.attempts - 1).min(16);
let delay = self
.cfg
.backoff_base
.checked_mul(1u32 << shift)
.unwrap_or(self.cfg.backoff_cap)
.min(self.cfg.backoff_cap);
self.set_due(id, delay);
self.set_waiting(&mut job, m);
return;
}
Err(StepErr::NoCreds) => {
self.set_due(id, self.cfg.login_retry);
self.set_waiting(&mut job, "not signed in to the hub: run `zc login`".into());
return;
}
Err(StepErr::Permanent { code, message }) => {
self.fail(&mut job, &code, message);
return;
}
}
}
}
fn set_due(&self, id: &str, d: Duration) {
self.due
.lock()
.unwrap()
.insert(id.to_string(), Instant::now() + d);
}
fn emit(&self, job: &Job) {
(self.emit)(
"submission.state",
job.summary(self.store.cancel_requested(&job.id)),
);
}
fn save_emit(&self, job: &mut Job) -> Result<(), StepErr> {
job.updated_at = now_rfc3339();
self.store.save(job).map_err(local)?;
self.emit(job);
Ok(())
}
fn save(&self, job: &mut Job) -> Result<(), StepErr> {
job.updated_at = now_rfc3339();
self.store.save(job).map_err(local)
}
fn set_waiting(&self, job: &mut Job, why: String) {
let changed = job.waiting.as_deref() != Some(why.as_str());
job.waiting = Some(why);
job.updated_at = now_rfc3339();
let _ = self.store.save(job);
if changed {
self.emit(job);
}
}
fn clear_waiting(&self, job: &mut Job) {
if job.waiting.is_some() || job.attempts > 0 {
let had = job.waiting.take().is_some();
job.attempts = 0;
let _ = self.save(job);
if had {
self.emit(job);
}
}
}
fn set_state(&self, job: &mut Job, s: State) -> Result<(), StepErr> {
job.state = s;
job.waiting = None;
job.attempts = 0;
if s.terminal() {
job.finished_at = Some(now_rfc3339());
}
self.save_emit(job)
}
fn fail(&self, job: &mut Job, code: &str, message: String) {
job.error = Some(JobError {
code: code.into(),
message,
});
let before_run = matches!(job.state, State::Uploading | State::Publishing);
let may_have_space = job.hub.space_id.is_some() || job.hub.create_intent;
if before_run && may_have_space && code != "hub_changed" {
job.cleanup_cancel = true;
}
let _ = self.set_state(job, State::Failed);
}
fn hub(&self, job: &mut Job) -> Result<SpacesHub, StepErr> {
let (base, token) = (self.creds)().ok_or(StepErr::NoCreds)?;
let base = base.trim_end_matches('/').to_string();
match &job.hub.base {
Some(b) if *b != base => {
return Err(permanent(
"hub_changed",
format!(
"this job was made on {b}, but the agent is now signed in to {base}; \
it is not moved between hubs"
),
))
}
Some(_) => {}
None => {
job.hub.base = Some(base.clone());
self.save(job)?;
}
}
Ok(SpacesHub::new(&base, &token))
}
async fn step(&self, job: &mut Job) -> Result<Flow, StepErr> {
if job.state.terminal() {
return self.cleanup(job).await;
}
if job.state != State::Collecting && self.store.cancel_requested(&job.id) {
return self.cancel(job).await;
}
match job.state {
State::Queued => {
self.set_state(job, State::Uploading)?;
Ok(Flow::Next)
}
State::Uploading => self.uploading(job).await,
State::Publishing => self.publishing(job).await,
State::Running => self.running(job).await,
State::Collecting => self.collecting(job).await,
State::Done | State::Failed | State::Cancelled => Ok(Flow::Stop),
}
}
fn space_body(&self, job: &Job) -> Value {
let (visibility, squad) = visibility_and_squad(&job.spec);
let r = &job.spec.resources;
let desc = format!(
"{}\n\nzc-job:{}",
job.spec.description.clone().unwrap_or_default().trim(),
job.id
);
let mut body = serde_json::json!({
"name": job.spec.name,
"description": desc.trim(),
"image_ref": job.image.as_given,
"requirements": {
"cpus": r.cpus,
"memory_gb": r.memory_gb,
"gpus": r.gpus,
"min_vram_gb": r.min_vram_gb,
"max_runtime_s": r.max_runtime_s,
"est_runtime_s": r.est_runtime_s.unwrap_or(r.max_runtime_s),
},
"price_per_batch": job.spec.max_price_per_batch,
"data_mode": "hub",
"visibility": visibility,
});
if let Some(s) = squad {
body["squad_id"] = Value::String(s);
}
body
}
async fn find_by_marker(&self, job: &Job, hub: &SpacesHub) -> Result<Option<Value>, StepErr> {
let marker = format!("zc-job:{}", job.id);
Ok(hub.list_spaces().await?.into_iter().find(|s| {
s["mine"].as_bool().unwrap_or(true)
&& s["description"]
.as_str()
.is_some_and(|d| d.contains(&marker))
}))
}
fn adopt_space(&self, job: &mut Job, space: &Value) -> Result<String, StepErr> {
let id = space["id"]
.as_str()
.ok_or_else(|| StepErr::Transient("the hub's Space has no id".into()))?
.to_string();
job.hub.space_id = Some(id.clone());
job.hub.image_digest = space["image_digest"].as_str().map(str::to_string);
if job.image.pinned_digest.is_none() {
job.image.pinned_digest = job.hub.image_digest.clone();
}
self.save_emit(job)?;
if let (Some(want), Some(got)) = (&job.image.pinned_digest, &job.hub.image_digest) {
if want != got {
return Err(permanent(
"image_digest_mismatch",
format!("the job pins {want} but the hub resolved the image to {got}"),
));
}
}
Ok(id)
}
async fn ensure_space(&self, job: &mut Job, hub: &SpacesHub) -> Result<String, StepErr> {
if let Some(id) = &job.hub.space_id {
return Ok(id.clone());
}
if job.hub.create_intent {
if let Some(s) = self.find_by_marker(job, hub).await? {
return self.adopt_space(job, &s);
}
} else {
job.hub.create_intent = true;
self.save(job)?;
}
let space = hub.create_space(&self.space_body(job)).await?;
self.adopt_space(job, &space)
}
async fn uploading(&self, job: &mut Job) -> Result<Flow, StepErr> {
let hub = self.hub(job)?;
let space_id = self.ensure_space(job, &hub).await?;
let todo: Vec<usize> = (0..job.files.len())
.filter(|i| !job.files[*i].uploaded)
.collect();
for chunk in todo.chunks(500) {
let req: Vec<(String, String, u64)> = chunk
.iter()
.map(|i| {
let f = &job.files[*i];
(f.upload_name.clone(), f.sha256.clone(), f.size)
})
.collect();
let uploads = hub.presign(&space_id, &req).await?;
for i in chunk {
if self.store.cancel_requested(&job.id) {
return Ok(Flow::Next);
}
let (name, path, sha) = {
let f = &job.files[*i];
(f.upload_name.clone(), f.path.clone(), f.sha256.clone())
};
let up = uploads
.iter()
.find(|u| u.name == name)
.ok_or_else(|| StepErr::Transient(format!("the hub did not presign {name}")))?;
let bytes = read_verified(&path, &sha).await?;
hub.put_object(&up.url, &sha, bytes).await?;
job.files[*i].key = Some(up.key.clone());
job.files[*i].uploaded = true;
self.save(job)?;
}
}
if !job.hub.batches_added {
let n = job.batches.len() as u64;
let mut post = true;
if job.hub.batches_intent {
let have = counts_of(&hub.get_space(&space_id).await?).total();
if have == n {
post = false;
} else if have != 0 {
return Err(permanent(
"batch_count_mismatch",
format!("the Space holds {have} batches but this job has {n}"),
));
}
} else {
job.hub.batches_intent = true;
self.save(job)?;
}
if post {
let mut nd = String::new();
for b in &job.batches {
let inputs: serde_json::Map<String, Value> = b
.iter()
.map(|(name, idx)| {
(
name.clone(),
Value::String(job.files[*idx].key.clone().unwrap_or_default()),
)
})
.collect();
nd.push_str(&serde_json::json!({ "inputs": inputs }).to_string());
nd.push('\n');
}
let res = hub.add_batches(&space_id, nd).await?;
if res["added"].as_u64() != Some(n) {
return Err(permanent(
"batch_count_mismatch",
format!("the hub added {} of {n} batches", res["added"]),
));
}
}
job.hub.batches_added = true;
self.save(job)?;
}
self.set_state(job, State::Publishing)?;
Ok(Flow::Next)
}
async fn publishing(&self, job: &mut Job) -> Result<Flow, StepErr> {
let hub = self.hub(job)?;
let space_id = job.hub.space_id.clone().ok_or_else(|| {
permanent(
"internal",
"a job in `publishing` has no Space; this is a bug",
)
})?;
let space = hub.get_space(&space_id).await?;
let status = space["status"].as_str().unwrap_or("");
if status != "draft" {
job.hub.published = true;
job.counts = Some(counts_of(&space));
self.save(job)?;
self.set_state(job, State::Running)?;
return Ok(Flow::Next);
}
match space["mirror"]["status"].as_str() {
Some("ready") => {}
Some("failed") => {
return Err(permanent(
"mirror_failed",
format!(
"the hub could not mirror the image: {}",
space["mirror"]["error"]
.as_str()
.unwrap_or("no reason given")
),
))
}
_ => return Ok(Flow::Wait(self.cfg.poll)),
}
job.hub.publish_intent = true;
self.save(job)?;
match hub.publish(&space_id).await {
Ok(_) => {}
Err(HubErr::Permanent { status: 409, .. }) => {
let again = hub.get_space(&space_id).await?;
if again["status"].as_str() == Some("draft") {
return Err(HubErr::Permanent {
status: 409,
message: "the Space cannot be published".into(),
}
.into());
}
}
Err(e) => return Err(e.into()),
}
job.hub.published = true;
self.save(job)?;
self.set_state(job, State::Running)?;
Ok(Flow::Next)
}
async fn running(&self, job: &mut Job) -> Result<Flow, StepErr> {
let hub = self.hub(job)?;
let space_id = job.hub.space_id.clone().unwrap_or_default();
let space = hub.get_space(&space_id).await?;
let counts = counts_of(&space);
if job.counts.as_ref() != Some(&counts) {
job.counts = Some(counts.clone());
self.save_emit(job)?;
}
let n = job.batches.len() as u64;
match space["status"].as_str().unwrap_or("") {
"cancelled" => {
job.error = Some(JobError {
code: "cancelled_by_hub".into(),
message: "the Space was cancelled on the hub".into(),
});
self.set_state(job, State::Cancelled)?;
Ok(Flow::Stop)
}
status => {
let settled = counts.pending == 0 && counts.claimed == 0 && counts.total() == n;
if status == "done" || settled {
self.set_state(job, State::Collecting)?;
return Ok(Flow::Next);
}
let why =
(status == "paused").then(|| "the Space is paused on the hub".to_string());
if job.waiting != why {
job.waiting = why;
self.save_emit(job)?;
}
Ok(Flow::Wait(self.cfg.poll))
}
}
}
async fn all_succeeded(&self, hub: &SpacesHub, id: &str) -> Result<Vec<Value>, StepErr> {
let mut out = vec![];
let mut cursor: Option<String> = None;
loop {
let page = hub
.list_batches(id, Some("succeeded"), cursor.as_deref())
.await?;
out.extend(page["items"].as_array().cloned().unwrap_or_default());
match page["next_cursor"].as_str() {
Some(c) => cursor = Some(c.to_string()),
None => return Ok(out),
}
}
}
async fn collecting(&self, job: &mut Job) -> Result<Flow, StepErr> {
let hub = self.hub(job)?;
let space_id = job.hub.space_id.clone().unwrap_or_default();
let items = self.all_succeeded(&hub, &space_id).await?;
let mut keys: Option<(Vec<rv::TrustedKey>, rv::KeySource)> = None;
for item in &items {
let seq = item["seq"].as_u64().unwrap_or(0);
if job.results.iter().any(|r| r.seq == seq) {
continue;
}
let res = self.collect_batch(job, &hub, item, &mut keys).await?;
job.results.push(res);
self.save_emit(job)?;
}
let succeeded = job.counts.as_ref().map(|c| c.succeeded).unwrap_or(0);
if (job.results.len() as u64) < succeeded {
return Ok(Flow::Wait(self.cfg.poll));
}
let failed = job.counts.as_ref().map(|c| c.failed).unwrap_or(0);
if failed > 0 {
let dir = self.store.results_dir(&job.id);
let message = format!(
"{failed} of {} batches failed on the hub; the outputs of the other {} are in {}",
job.batches.len(),
job.results.len(),
dir.display()
);
self.fail(job, "batches_failed", message);
return Ok(Flow::Stop);
}
self.set_state(job, State::Done)?;
Ok(Flow::Stop)
}
async fn collect_batch(
&self,
job: &Job,
hub: &SpacesHub,
item: &Value,
keys: &mut Option<(Vec<rv::TrustedKey>, rv::KeySource)>,
) -> Result<BatchResult, StepErr> {
let seq = item["seq"].as_u64().unwrap_or(0);
let dir = self.store.results_dir(&job.id).join(seq.to_string());
std::fs::create_dir_all(&dir).map_err(local)?;
let declared = item["output_hashes"]
.as_object()
.cloned()
.unwrap_or_default();
let mut names: Vec<&String> = declared.keys().collect();
names.sort();
let mut files = vec![];
for name in names {
if !safe_name(name) {
return Err(permanent(
"hub_output_name",
format!("the hub named an output `{name}`, which is not a plain file name"),
));
}
let want = declared[name].as_str().unwrap_or("").to_ascii_lowercase();
let dest = dir.join(name);
if let Ok((sha, size)) = spec::sha256_file(&dest) {
if sha == want {
files.push(OutFile {
name: name.clone(),
sha256: sha,
size,
});
continue;
}
}
let url = item["outputs"][name.as_str()].as_str().ok_or_else(|| {
permanent(
"hub_output_missing",
format!("the hub lists output `{name}` but gave no link to it"),
)
})?;
let tmp = dir.join(format!(".{name}.part"));
let (sha, size) = hub.download(url, &tmp).await?;
if sha != want {
let _ = std::fs::remove_file(&tmp);
return Err(permanent(
"output_digest_mismatch",
format!(
"batch {seq} output `{name}` hashes to {sha}, but the hub declared {want}"
),
));
}
std::fs::rename(&tmp, &dest).map_err(local)?;
files.push(OutFile {
name: name.clone(),
sha256: sha,
size,
});
}
let receipt = self.receipt_for(job, hub, item, &dir, &files, keys).await?;
Ok(BatchResult {
seq,
dir: dir.display().to_string(),
files,
receipt,
})
}
async fn receipt_for(
&self,
job: &Job,
hub: &SpacesHub,
item: &Value,
dir: &Path,
files: &[OutFile],
keys: &mut Option<(Vec<rv::TrustedKey>, rv::KeySource)>,
) -> Result<ReceiptState, StepErr> {
let seq = item["seq"].as_u64().unwrap_or(0);
let unavailable = |detail: String| -> Result<ReceiptState, StepErr> {
if job.spec.require_receipts {
return Err(permanent(
"receipt_unavailable",
format!("batch {seq}: {detail}, and this job requires receipts"),
));
}
Ok(ReceiptState {
status: "unavailable".into(),
hash: None,
file: None,
detail: Some(detail),
key_source: None,
})
};
let Some(hash) = item["receipt_hash"].as_str().map(str::to_string) else {
return unavailable(
"the hub's batch listing carries no receipt hash, and the hub has no route that \
finds a batch's receipt"
.into(),
);
};
let base = hub.base().to_string();
let h = hash.clone();
let fetched = tokio::task::spawn_blocking(move || rv::fetch_receipt(&base, &h))
.await
.map_err(|e| StepErr::Transient(e.to_string()))?;
let signed = match fetched {
Ok(s) => s,
Err(e) if e.permanent() => return unavailable(format!("receipt {hash}: {e}")),
Err(e) => return Err(StepErr::Transient(format!("receipt {hash}: {e}"))),
};
if keys.is_none() {
let base = hub.base().to_string();
let got = tokio::task::spawn_blocking(move || rv::fetch_pubkeys(&base))
.await
.map_err(|e| StepErr::Transient(e.to_string()))?;
match got {
Ok(k) => *keys = Some(k),
Err(e) if e.permanent() => {
return unavailable(format!("the hub's receipt keys: {e}"))
}
Err(e) => return Err(StepErr::Transient(format!("the hub's receipt keys: {e}"))),
}
}
let (trusted, source) = keys.as_ref().expect("fetched above");
let verified = rv::verify(&signed, trusted).map_err(|e| {
permanent(
"receipt_invalid",
format!("batch {seq}: receipt {hash} does not verify: {e}"),
)
})?;
if verified.receipt_hash != hash {
return Err(permanent(
"receipt_invalid",
format!("batch {seq}: the receipt does not hash to {hash}"),
));
}
let local_digests: Vec<String> = files.iter().map(|f| f.sha256.clone()).collect();
rv::check_output_digests(&verified.body, &local_digests).map_err(|e| {
permanent(
"receipt_mismatch",
format!("batch {seq}: receipt {hash}: {e}"),
)
})?;
let doc = serde_json::json!({
"receipt_hash": verified.receipt_hash,
"receipt_json": signed.receipt_json,
"signature": signed.signature,
});
write_atomic(
&dir.join("receipt.json"),
serde_json::to_string_pretty(&doc)
.unwrap_or_default()
.as_bytes(),
)
.map_err(local)?;
Ok(ReceiptState {
status: "verified".into(),
hash: Some(verified.receipt_hash),
file: Some("receipt.json".into()),
detail: if verified.legacy {
Some("legacy v1 receipt".into())
} else {
None
},
key_source: Some(match source {
rv::KeySource::Given => "given".into(),
rv::KeySource::Fetched(_) => "fetched".into(),
}),
})
}
async fn cancel_space(&self, job: &mut Job) -> Result<(), StepErr> {
if job.hub.cancel_done {
return Ok(());
}
if job.hub.space_id.is_some() || job.hub.create_intent {
let hub = self.hub(job)?;
let id = match job.hub.space_id.clone() {
Some(i) => Some(i),
None => self
.find_by_marker(job, &hub)
.await?
.and_then(|s| s["id"].as_str().map(str::to_string)),
};
if let Some(id) = id {
match hub.cancel(&id).await {
Ok(_) => {}
Err(HubErr::Permanent {
status: 404 | 409, ..
}) => {}
Err(e) => return Err(e.into()),
}
}
}
job.hub.cancel_done = true;
self.save(job)
}
async fn cancel(&self, job: &mut Job) -> Result<Flow, StepErr> {
self.cancel_space(job).await?;
self.set_state(job, State::Cancelled)?;
Ok(Flow::Stop)
}
async fn cleanup(&self, job: &mut Job) -> Result<Flow, StepErr> {
if !job.cleanup_cancel || job.hub.cancel_done {
return Ok(Flow::Stop);
}
match self.cancel_space(job).await {
Ok(()) => {
self.save_emit(job)?;
Ok(Flow::Stop)
}
Err(StepErr::Permanent { message, .. }) => {
job.hub.cancel_done = true;
if let Some(e) = &mut job.error {
e.message.push_str(&format!(
" (the Space could not be cancelled afterwards: {message})"
));
}
self.save_emit(job)?;
Ok(Flow::Stop)
}
Err(e) => Err(e),
}
}
}
async fn read_verified(path: &str, sha256: &str) -> Result<Vec<u8>, StepErr> {
let p = path.to_string();
let want = sha256.to_string();
let res = tokio::task::spawn_blocking(move || -> std::io::Result<Option<Vec<u8>>> {
let bytes = std::fs::read(&p)?;
let got = rv::sha256_hex(&bytes);
Ok((got == want).then_some(bytes))
})
.await
.map_err(|e| StepErr::Transient(e.to_string()))?;
match res {
Ok(Some(b)) => Ok(b),
Ok(None) => Err(permanent(
"input_changed",
format!("{path} changed after the job was submitted"),
)),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(permanent(
"input_changed",
format!("{path} is gone: it was there when the job was submitted"),
)),
Err(e) => Err(StepErr::Transient(format!("{path}: {e}"))),
}
}
#[cfg(test)]
mod tests {
use super::super::fakehub::FakeHub;
use super::super::spec::{sample_spec, test_dir};
use super::super::Outbox;
use super::*;
use crate::receipt_verify::testkit::Hub as Signer;
type Cell = Arc<Mutex<Creds>>;
type Events = Arc<Mutex<Vec<(String, Value)>>>;
struct Rig {
hub: FakeHub,
home: std::path::PathBuf,
inputs: std::path::PathBuf,
outbox: Outbox,
creds: Cell,
events: Events,
}
fn fast() -> Config {
Config {
tick: Duration::from_millis(5),
poll: Duration::from_millis(2),
backoff_base: Duration::from_millis(1),
backoff_cap: Duration::from_millis(4),
login_retry: Duration::from_millis(1),
retention: Retention::default(),
}
}
fn rig() -> Rig {
let hub = FakeHub::start(Some("tok"));
let home = test_dir("disp-home");
let creds: Cell = Arc::new(Mutex::new(Some((hub.base.clone(), "tok".to_string()))));
Rig {
outbox: Outbox::new(&home),
hub,
home,
inputs: test_dir("disp-in"),
creds,
events: Arc::new(Mutex::new(vec![])),
}
}
impl Rig {
fn dispatcher(&self) -> Arc<Dispatcher> {
let c = self.creds.clone();
let ev = self.events.clone();
Dispatcher::new(
Store::new(&self.home),
Arc::new(move || c.lock().unwrap().clone()),
Arc::new(move |k, d| ev.lock().unwrap().push((k.to_string(), d))),
fast(),
Arc::new(Notify::new()),
)
}
fn submit(&self) -> String {
self.submit_with(|_| {})
}
fn submit_with(&self, f: impl FnOnce(&mut spec::JobSpec)) -> String {
let mut s = sample_spec(&self.inputs);
f(&mut s);
self.outbox.submit(s).unwrap().0.id
}
fn job(&self, id: &str) -> Job {
Store::new(&self.home).load(id).unwrap()
}
async fn drive(&self, d: &Dispatcher, id: &str, until: impl Fn(&Job) -> bool) -> Job {
for _ in 0..300 {
d.advance(id).await;
let j = self.job(id);
if until(&j) {
return j;
}
tokio::time::sleep(Duration::from_millis(3)).await;
}
panic!("job never reached the state: {:?}", self.job(id));
}
async fn done(&self, d: &Dispatcher, id: &str) -> Job {
self.drive(d, id, |j| j.state.terminal()).await
}
}
#[tokio::test]
async fn a_two_batch_job_goes_through_every_state_to_done_with_verified_receipts() {
let r = rig();
let d = r.dispatcher();
let id = r.submit();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Done, "{:?}", j.error);
assert_eq!(j.results.len(), 2);
for (i, res) in j.results.iter().enumerate() {
let seq = i as u64 + 1;
assert_eq!(res.seq, seq);
let dir = Store::new(&r.home).results_dir(&id).join(seq.to_string());
assert_eq!(
std::fs::read_to_string(dir.join("out.txt")).unwrap(),
format!("result-{seq}")
);
assert_eq!(res.receipt.status, "verified");
assert_eq!(res.receipt.key_source.as_deref(), Some("fetched"));
let doc = std::fs::read_to_string(dir.join("receipt.json")).unwrap();
let signed = rv::parse_signed(&doc).unwrap();
let key = r.hub.with(|s| s.signer.trusted());
rv::verify(&signed, &[key]).unwrap();
}
for (k, n) in [
("create", 1),
("publish", 1),
("batches_post", 1),
("put", 2),
] {
assert_eq!(r.hub.count(k), n, "{k}");
}
let body = r.hub.with(|s| s.spaces[0].req.clone());
assert_eq!(body["price_per_batch"], 50.0);
assert_eq!(body["image_ref"], "registry.example/app:1");
assert!(j
.image
.pinned_digest
.as_deref()
.unwrap()
.starts_with("sha256:cccc"));
assert!(!r.hub.with(|s| s.object_auth_seen));
let states: Vec<String> = r
.events
.lock()
.unwrap()
.iter()
.filter(|(k, _)| k == "submission.state")
.map(|(_, d)| d["state"].as_str().unwrap().to_string())
.collect();
let mut order: Vec<&str> = vec![];
for s in &states {
if order.last() != Some(&s.as_str()) {
order.push(s);
}
}
assert_eq!(
order,
["uploading", "publishing", "running", "collecting", "done"]
);
}
#[tokio::test]
async fn running_carries_the_hubs_batch_counts() {
let r = rig();
r.hub.with(|s| s.auto_complete = false);
let d = r.dispatcher();
let id = r.submit();
let j = r
.drive(&d, &id, |j| j.state == State::Running && j.counts.is_some())
.await;
assert_eq!(
j.counts.unwrap(),
Counts {
pending: 2,
claimed: 0,
succeeded: 0,
failed: 0
}
);
r.hub.with(|s| s.complete_all(0));
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Done);
assert_eq!(j.counts.unwrap().succeeded, 2);
}
#[tokio::test]
async fn a_restart_resumes_and_never_repeats_a_completed_step() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("presign".into(), [503, 503].into());
});
let d1 = r.dispatcher();
let id = r.submit();
d1.advance(&id).await;
let j = r.job(&id);
assert_eq!(j.state, State::Uploading);
assert!(
j.waiting.as_deref().unwrap().contains("503"),
"{:?}",
j.waiting
);
assert_eq!(j.attempts, 1);
let space = j.hub.space_id.clone().unwrap();
let d2 = r.dispatcher();
let j = r.done(&d2, &id).await;
assert_eq!(j.state, State::Done, "{:?}", j.error);
assert_eq!(j.hub.space_id.unwrap(), space);
assert_eq!(r.hub.count("create"), 1, "never a second Space");
assert_eq!(r.hub.count("publish"), 1);
assert_eq!(r.hub.with(|s| s.spaces.len()), 1);
}
#[tokio::test]
async fn a_lost_create_answer_adopts_the_space_by_its_marker() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("create".into(), [503].into());
s.apply_then_fail.insert("create".into());
});
let id = r.submit();
let d1 = r.dispatcher();
d1.advance(&id).await;
assert!(r.job(&id).hub.space_id.is_none());
assert!(r.job(&id).hub.create_intent, "the intent was on disk first");
let d2 = r.dispatcher();
let j = r.done(&d2, &id).await;
assert_eq!(j.state, State::Done, "{:?}", j.error);
assert_eq!(r.hub.with(|s| s.spaces.len()), 1, "never a second Space");
assert_eq!(r.hub.count("create"), 1);
}
#[tokio::test]
async fn a_lost_batches_answer_does_not_post_twice() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("batches_post".into(), [503].into());
s.apply_then_fail.insert("batches_post".into());
});
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Done, "{:?}", j.error);
assert_eq!(r.hub.count("batches_post"), 1);
assert_eq!(r.hub.with(|s| s.spaces[0].batches.len()), 2);
}
#[tokio::test]
async fn a_lost_publish_answer_does_not_publish_twice() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("publish".into(), [503].into());
s.apply_then_fail.insert("publish".into());
});
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Done, "{:?}", j.error);
assert_eq!(r.hub.count("publish"), 1);
}
#[tokio::test]
async fn a_hub_that_is_down_keeps_the_job_in_its_state_and_it_recovers() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("create".into(), [503, 500, 429, 408].into());
});
let id = r.submit();
let d = r.dispatcher();
for _ in 0..4 {
d.advance(&id).await;
let j = r.job(&id);
assert_eq!(j.state, State::Uploading, "transient never fails a job");
assert!(j.waiting.is_some());
}
assert_eq!(r.job(&id).attempts, 4);
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Done);
assert_eq!(j.attempts, 0);
assert!(j.waiting.is_none());
}
#[tokio::test]
async fn a_refusal_fails_the_job_at_once_and_the_hubs_words_are_kept() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("publish".into(), [402].into());
});
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
let e = j.error.unwrap();
assert_eq!(e.code, "insufficient_funds");
assert_eq!(r.hub.count("publish"), 1, "not retried");
d.advance(&id).await;
assert_eq!(r.hub.count("cancel"), 1);
assert_eq!(r.hub.with(|s| s.spaces[0].status.clone()), "cancelled");
d.advance(&id).await;
assert_eq!(r.hub.count("cancel"), 1, "cleanup runs once");
}
#[tokio::test]
async fn the_hub_refusing_the_agents_credential_is_a_clear_permanent_failure() {
let r = rig();
*r.creds.lock().unwrap() = Some((r.hub.base.clone(), "zk_1_not-a-session".into()));
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
let e = j.error.unwrap();
assert_eq!(e.code, "hub_credential_refused");
assert!(
e.message.contains("session token") && e.message.contains("zk_"),
"{}",
e.message
);
assert_eq!(r.hub.with(|s| s.spaces.len()), 0);
}
#[tokio::test]
async fn without_a_login_the_job_waits_and_goes_on_after_one() {
let r = rig();
*r.creds.lock().unwrap() = None;
let id = r.submit();
let d = r.dispatcher();
d.advance(&id).await;
let j = r.job(&id);
assert_eq!(j.state, State::Uploading);
assert!(j.waiting.unwrap().contains("zc login"));
assert_eq!(r.hub.with(|s| s.calls.len()), 0);
*r.creds.lock().unwrap() = Some((r.hub.base.clone(), "tok".into()));
assert_eq!(r.done(&d, &id).await.state, State::Done);
}
#[tokio::test]
async fn a_job_is_not_moved_to_another_hub() {
let r = rig();
r.hub.with(|s| s.auto_complete = false);
let d = r.dispatcher();
let id = r.submit();
r.drive(&d, &id, |j| j.state == State::Running).await;
let other = FakeHub::start(Some("tok"));
*r.creds.lock().unwrap() = Some((other.base.clone(), "tok".into()));
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "hub_changed");
assert_eq!(other.with(|s| s.calls.len()), 0);
}
#[tokio::test]
async fn cancel_while_running_cancels_the_space_once() {
let r = rig();
r.hub.with(|s| s.auto_complete = false);
let d = r.dispatcher();
let id = r.submit();
r.drive(&d, &id, |j| j.state == State::Running).await;
Store::new(&r.home).request_cancel(&id).unwrap();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Cancelled);
assert_eq!(r.hub.count("cancel"), 1);
assert_eq!(r.hub.with(|s| s.spaces[0].status.clone()), "cancelled");
d.advance(&id).await;
assert_eq!(r.hub.count("cancel"), 1);
}
#[tokio::test]
async fn cancel_before_anything_ran_never_touches_the_hub() {
let r = rig();
let id = r.submit();
Store::new(&r.home).request_cancel(&id).unwrap();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Cancelled);
assert_eq!(r.hub.with(|s| s.calls.len()), 0);
}
#[tokio::test]
async fn a_digest_the_hub_resolves_differently_fails_and_cancels_the_space() {
let r = rig();
let id =
r.submit_with(|s| s.image = format!("registry.example/app@sha256:{}", "a".repeat(64)));
let want = format!("sha256:{}", "b".repeat(64));
let mut job = r.job(&id);
job.image.pinned_digest = Some(want.clone());
Store::new(&r.home).save(&job).unwrap();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "image_digest_mismatch");
assert_eq!(r.hub.count("put"), 0, "nothing uploaded");
d.advance(&id).await;
assert_eq!(r.hub.with(|s| s.spaces[0].status.clone()), "cancelled");
}
#[tokio::test]
async fn an_input_edited_after_submit_is_not_uploaded() {
let r = rig();
let id = r.submit();
std::fs::write(r.inputs.join("b.txt"), "changed after submit").unwrap();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
let e = j.error.unwrap();
assert_eq!(e.code, "input_changed");
assert!(e.message.contains("b.txt"));
d.advance(&id).await;
assert_eq!(r.hub.with(|s| s.spaces[0].status.clone()), "cancelled");
}
#[tokio::test]
async fn an_output_that_does_not_match_the_hubs_digest_fails_the_job() {
let r = rig();
r.hub.with(|s| s.tamper_output = true);
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "output_digest_mismatch");
assert!(!Store::new(&r.home)
.results_dir(&id)
.join("1/out.txt")
.exists());
}
#[tokio::test]
async fn a_receipt_that_does_not_cover_the_outputs_blocks_done() {
let r = rig();
r.hub.with(|s| s.bad_receipt_outputs = true);
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "receipt_mismatch");
}
#[tokio::test]
async fn a_receipt_signed_by_an_unknown_key_blocks_done() {
let r = rig();
r.hub.with(|s| s.auto_complete = false);
let d = r.dispatcher();
let id = r.submit();
r.drive(&d, &id, |j| j.state == State::Running).await;
r.hub.with(|s| {
s.complete_all(0);
s.signer = Signer::new();
});
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "receipt_invalid");
}
#[tokio::test]
async fn without_a_receipt_hash_the_job_is_done_and_says_the_receipt_is_unavailable() {
let r = rig();
r.hub.with(|s| s.with_receipt_hash = false);
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Done);
assert!(j.results.iter().all(|x| x.receipt.status == "unavailable"));
assert!(j.results[0]
.receipt
.detail
.as_deref()
.unwrap()
.contains("no route"));
assert_eq!(j.summary(false)["receipts_verified"], 0);
}
#[tokio::test]
async fn require_receipts_makes_a_missing_receipt_a_failure() {
let r = rig();
r.hub.with(|s| s.with_receipt_hash = false);
let id = r.submit_with(|s| s.require_receipts = true);
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "receipt_unavailable");
}
#[tokio::test]
async fn a_failed_batch_fails_the_job_but_keeps_what_finished() {
let r = rig();
r.hub.with(|s| s.fail_second_batch = true);
let id = r.submit();
let d = r.dispatcher();
let j = r.done(&d, &id).await;
assert_eq!(j.state, State::Failed);
assert_eq!(j.error.unwrap().code, "batches_failed");
assert_eq!(j.results.len(), 1);
assert!(Store::new(&r.home)
.results_dir(&id)
.join("1/out.txt")
.exists());
d.advance(&id).await;
assert_eq!(r.hub.count("cancel"), 0, "finished work is not cancelled");
}
#[tokio::test]
async fn a_half_collected_job_resumes_after_a_restart_without_a_second_space() {
let r = rig();
r.hub.with(|s| {
s.fail.insert("download".into(), [503].into());
});
let id = r.submit();
let d = r.dispatcher();
let j = r
.drive(&d, &id, |j| {
j.state == State::Collecting && j.waiting.is_some()
})
.await;
assert!(j.results.is_empty());
let d2 = r.dispatcher();
let j = r.done(&d2, &id).await;
assert_eq!(j.state, State::Done, "{:?}", j.error);
assert_eq!(j.results.len(), 2);
assert_eq!(r.hub.count("create"), 1);
assert_eq!(r.hub.count("publish"), 1);
}
#[tokio::test]
#[ignore = "needs a real hub: ZAKURO_TEST_HUB_URL, ZAKURO_TEST_HUB_TOKEN, ZAKURO_TEST_JOB_IMAGE"]
async fn end_to_end_against_a_real_hub_two_batches() {
let var = |k: &str| std::env::var(k).unwrap_or_else(|_| panic!("{k} is not set"));
let (url, token, image) = (
var("ZAKURO_TEST_HUB_URL"),
var("ZAKURO_TEST_HUB_TOKEN"),
var("ZAKURO_TEST_JOB_IMAGE"),
);
let home = test_dir("e2e-home");
let inputs = test_dir("e2e-in");
let outbox = Outbox::new(&home);
let id = outbox
.submit({
let mut s = sample_spec(&inputs);
s.name = "zc-e2e".into();
s.image = image;
s.max_price_per_batch = 1.0;
s
})
.unwrap()
.0
.id;
let d = Dispatcher::new(
Store::new(&home),
Arc::new(move || Some((url.clone(), token.clone()))),
Arc::new(|_, _| {}),
Config {
poll: Duration::from_secs(5),
..Config::default()
},
Arc::new(Notify::new()),
);
let store = Store::new(&home);
let deadline = Instant::now() + Duration::from_secs(20 * 60);
let job = loop {
d.advance(&id).await;
let j = store.load(&id).unwrap();
if j.state.terminal() {
break j;
}
assert!(Instant::now() < deadline, "not done in 20 minutes: {j:?}");
tokio::time::sleep(Duration::from_secs(5)).await;
};
assert_eq!(job.state, State::Done, "{:?}", job.error);
assert_eq!(job.results.len(), 2);
for r in &job.results {
assert!(!r.files.is_empty(), "batch {} has outputs", r.seq);
for f in &r.files {
let p = std::path::Path::new(&r.dir).join(&f.name);
assert_eq!(spec::sha256_file(&p).unwrap().0, f.sha256);
}
match r.receipt.status.as_str() {
"verified" => assert!(std::path::Path::new(&r.dir).join("receipt.json").exists()),
other => {
assert_eq!(other, "unavailable", "{:?}", r.receipt);
eprintln!(
"batch {}: receipt unavailable: {:?}",
r.seq, r.receipt.detail
);
}
}
}
}
#[test]
fn backoff_doubles_and_is_capped() {
let cfg = Config::default();
let at = |n: u32| {
cfg.backoff_base
.checked_mul(1u32 << (n - 1).min(16))
.unwrap_or(cfg.backoff_cap)
.min(cfg.backoff_cap)
};
assert_eq!(at(1), Duration::from_secs(2));
assert_eq!(at(2), Duration::from_secs(4));
assert_eq!(at(3), Duration::from_secs(8));
assert_eq!(at(30), Duration::from_secs(300));
}
}