use super::spec::{Checked, ImageRef, JobSpec};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
pub const JOB_FILE: &str = "job.json";
const CANCEL_FILE: &str = "cancel";
const SCHEMA: u32 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum State {
Queued,
Uploading,
Publishing,
Running,
Collecting,
Done,
Failed,
Cancelled,
}
impl State {
pub fn terminal(self) -> bool {
matches!(self, State::Done | State::Failed | State::Cancelled)
}
pub fn as_str(self) -> &'static str {
match self {
State::Queued => "queued",
State::Uploading => "uploading",
State::Publishing => "publishing",
State::Running => "running",
State::Collecting => "collecting",
State::Done => "done",
State::Failed => "failed",
State::Cancelled => "cancelled",
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct FileEntry {
pub path: String,
pub sha256: String,
pub size: u64,
pub upload_name: String,
#[serde(default)]
pub key: Option<String>,
#[serde(default)]
pub uploaded: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct HubRefs {
#[serde(default)]
pub base: Option<String>,
#[serde(default)]
pub create_intent: bool,
#[serde(default)]
pub space_id: Option<String>,
#[serde(default)]
pub image_digest: Option<String>,
#[serde(default)]
pub batches_intent: bool,
#[serde(default)]
pub batches_added: bool,
#[serde(default)]
pub publish_intent: bool,
#[serde(default)]
pub published: bool,
#[serde(default)]
pub cancel_done: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct Counts {
pub pending: u64,
pub claimed: u64,
pub succeeded: u64,
pub failed: u64,
}
impl Counts {
pub fn total(&self) -> u64 {
self.pending + self.claimed + self.succeeded + self.failed
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct OutFile {
pub name: String,
pub sha256: String,
pub size: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ReceiptState {
pub status: String,
#[serde(default)]
pub hash: Option<String>,
#[serde(default)]
pub file: Option<String>,
#[serde(default)]
pub detail: Option<String>,
#[serde(default)]
pub key_source: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct BatchResult {
pub seq: u64,
pub dir: String,
pub files: Vec<OutFile>,
pub receipt: ReceiptState,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct JobError {
pub code: String,
pub message: String,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Job {
pub v: u32,
pub id: String,
pub created_at: String,
pub updated_at: String,
#[serde(default)]
pub finished_at: Option<String>,
pub state: State,
pub spec: JobSpec,
pub image: ImageRef,
pub files: Vec<FileEntry>,
pub batches: Vec<BTreeMap<String, usize>>,
#[serde(default)]
pub hub: HubRefs,
#[serde(default)]
pub counts: Option<Counts>,
#[serde(default)]
pub results: Vec<BatchResult>,
#[serde(default)]
pub error: Option<JobError>,
#[serde(default)]
pub waiting: Option<String>,
#[serde(default)]
pub attempts: u32,
#[serde(default)]
pub cleanup_cancel: bool,
}
pub fn now_rfc3339() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
pub fn new_id() -> String {
format!("j_{}", &uuid::Uuid::new_v4().simple().to_string()[..12])
}
pub fn valid_id(id: &str) -> bool {
id.len() == 14
&& id.starts_with("j_")
&& id[2..]
.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
pub fn upload_name(sha256: &str, path: &str) -> String {
let base = Path::new(path)
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("input");
let clean: String = base
.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') {
c
} else {
'_'
}
})
.take(100)
.collect();
format!("{}-{}", &sha256[..16], clean)
}
impl Job {
pub fn new(spec: JobSpec, checked: Checked) -> Job {
let now = now_rfc3339();
let files = checked
.files
.into_iter()
.map(|f| FileEntry {
upload_name: upload_name(&f.sha256, &f.path),
path: f.path,
sha256: f.sha256,
size: f.size,
key: None,
uploaded: false,
})
.collect();
Job {
v: SCHEMA,
id: new_id(),
created_at: now.clone(),
updated_at: now,
finished_at: None,
state: State::Queued,
image: checked.image,
files,
batches: checked.batches,
spec,
hub: HubRefs::default(),
counts: None,
results: vec![],
error: None,
waiting: None,
attempts: 0,
cleanup_cancel: false,
}
}
pub fn summary(&self, cancel_requested: bool) -> serde_json::Value {
let verified = self
.results
.iter()
.filter(|r| r.receipt.status == "verified")
.count();
serde_json::json!({
"id": self.id,
"name": self.spec.name,
"state": self.state.as_str(),
"created_at": self.created_at,
"updated_at": self.updated_at,
"finished_at": self.finished_at,
"batches": self.batches.len(),
"counts": self.counts,
"space_id": self.hub.space_id,
"results": self.results.len(),
"receipts_verified": verified,
"error": self.error,
"waiting": self.waiting,
"cancel_requested": cancel_requested,
})
}
pub fn detail(&self, cancel_requested: bool, results_root: &Path) -> serde_json::Value {
let mut v = self.summary(cancel_requested);
let o = v.as_object_mut().expect("summary is an object");
o.insert(
"image".into(),
serde_json::json!({
"as_given": self.image.as_given,
"pinned_digest": self.image.pinned_digest,
"hub_resolved_digest": self.hub.image_digest,
}),
);
o.insert(
"max_price_per_batch".into(),
serde_json::json!(self.spec.max_price_per_batch),
);
o.insert("resources".into(), serde_json::json!(self.spec.resources));
o.insert(
"results_dir".into(),
serde_json::json!(results_root.join(&self.id).display().to_string()),
);
o.insert("result_batches".into(), serde_json::json!(self.results));
v
}
}
#[derive(Debug, Clone, Copy)]
pub struct Retention {
pub max_age_days: u64,
pub max_finished: usize,
}
impl Default for Retention {
fn default() -> Self {
Self {
max_age_days: 30,
max_finished: 200,
}
}
}
impl Retention {
pub fn from_env() -> Self {
let num = |k: &str| {
std::env::var(k)
.ok()
.and_then(|v| v.trim().parse::<u64>().ok())
};
let d = Self::default();
Self {
max_age_days: num("ZAKURO_OUTBOX_KEEP_DAYS").unwrap_or(d.max_age_days),
max_finished: num("ZAKURO_OUTBOX_KEEP_MAX")
.map(|n| n as usize)
.unwrap_or(d.max_finished),
}
}
}
#[derive(Clone)]
pub struct Store {
outbox: PathBuf,
results: PathBuf,
}
static TMP: AtomicU64 = AtomicU64::new(0);
fn fsync_dir(dir: &Path) -> std::io::Result<()> {
#[cfg(unix)]
{
std::fs::File::open(dir)?.sync_all()
}
#[cfg(not(unix))]
{
let _ = dir;
Ok(())
}
}
pub fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
let dir = path.parent().unwrap_or(Path::new("."));
let tmp = dir.join(format!(
".{}.tmp.{}.{}",
path.file_name().and_then(|n| n.to_str()).unwrap_or("f"),
std::process::id(),
TMP.fetch_add(1, Ordering::Relaxed)
));
{
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
let mut f = opts.open(&tmp)?;
f.write_all(bytes)?;
f.sync_all()?;
}
if let Err(e) = std::fs::rename(&tmp, path) {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
fsync_dir(dir)
}
impl Store {
pub fn new(home: &Path) -> Self {
Self {
outbox: home.join("outbox"),
results: home.join("results"),
}
}
pub fn results_root(&self) -> &Path {
&self.results
}
pub fn results_dir(&self, id: &str) -> PathBuf {
self.results.join(id)
}
fn job_dir(&self, id: &str) -> PathBuf {
self.outbox.join(id)
}
pub fn create(&self, job: &Job) -> std::io::Result<()> {
crate::agent::files::ensure_dir(&self.outbox)?;
std::fs::create_dir(self.job_dir(&job.id))?;
self.save(job)?;
fsync_dir(&self.outbox)
}
pub fn save(&self, job: &Job) -> std::io::Result<()> {
let bytes = serde_json::to_vec_pretty(job).map_err(std::io::Error::other)?;
write_atomic(&self.job_dir(&job.id).join(JOB_FILE), &bytes)
}
pub fn load(&self, id: &str) -> Option<Job> {
if !valid_id(id) {
return None;
}
serde_json::from_slice(&std::fs::read(self.job_dir(id).join(JOB_FILE)).ok()?).ok()
}
pub fn list(&self) -> Vec<Job> {
let mut jobs: Vec<Job> = std::fs::read_dir(&self.outbox)
.into_iter()
.flatten()
.flatten()
.filter_map(|e| self.load(e.file_name().to_str()?))
.collect();
jobs.sort_by(|a, b| (&a.created_at, &a.id).cmp(&(&b.created_at, &b.id)));
jobs
}
pub fn cancel_requested(&self, id: &str) -> bool {
valid_id(id) && self.job_dir(id).join(CANCEL_FILE).exists()
}
pub fn request_cancel(&self, id: &str) -> std::io::Result<()> {
if !valid_id(id) || !self.job_dir(id).is_dir() {
return Err(std::io::ErrorKind::NotFound.into());
}
write_atomic(&self.job_dir(id).join(CANCEL_FILE), b"1")
}
pub fn remove(&self, id: &str) {
if valid_id(id) {
let _ = std::fs::remove_dir_all(self.job_dir(id));
let _ = std::fs::remove_dir_all(self.results_dir(id));
}
}
pub fn prune(&self, r: Retention, now: chrono::DateTime<chrono::Utc>) -> Vec<String> {
let mut finished: Vec<(chrono::DateTime<chrono::Utc>, String)> = self
.list()
.into_iter()
.filter(|j| j.state.terminal() && (!j.cleanup_cancel || j.hub.cancel_done))
.map(|j| {
let at = j
.finished_at
.as_deref()
.or(Some(&j.updated_at))
.and_then(|t| chrono::DateTime::parse_from_rfc3339(t).ok())
.map(|t| t.with_timezone(&chrono::Utc))
.unwrap_or(now);
(at, j.id)
})
.collect();
finished.sort();
let cutoff = now - chrono::Duration::days(r.max_age_days as i64);
let mut removed = vec![];
let keep_from = finished.len().saturating_sub(r.max_finished);
for (i, (at, id)) in finished.iter().enumerate() {
if *at < cutoff || i < keep_from {
self.remove(id);
removed.push(id.clone());
}
}
removed
}
}
#[cfg(test)]
pub(crate) mod tests {
use super::super::spec::{sample_spec, test_dir, validate};
use super::*;
pub(crate) fn sample_job(dir: &Path) -> Job {
let spec = sample_spec(dir);
let checked = validate(&spec).unwrap();
Job::new(spec, checked)
}
#[test]
fn a_created_job_round_trips_and_is_listed() {
let home = test_dir("store-rt");
let store = Store::new(&home);
let inputs = test_dir("store-rt-in");
let job = sample_job(&inputs);
store.create(&job).unwrap();
assert_eq!(store.load(&job.id).unwrap(), job);
assert_eq!(store.list().len(), 1);
assert_eq!(job.state, State::Queued);
assert!(std::fs::metadata(home.join("outbox").join(&job.id).join(JOB_FILE)).is_ok());
assert!(
store.create(&job).is_err(),
"an id is never reused: the directory exists"
);
}
#[test]
fn save_is_atomic_and_leaves_no_temp_files() {
let home = test_dir("store-atomic");
let store = Store::new(&home);
let mut job = sample_job(&test_dir("store-atomic-in"));
store.create(&job).unwrap();
for i in 0..5 {
job.attempts = i;
store.save(&job).unwrap();
}
assert_eq!(store.load(&job.id).unwrap().attempts, 4);
let names: Vec<String> = std::fs::read_dir(home.join("outbox").join(&job.id))
.unwrap()
.map(|e| e.unwrap().file_name().into_string().unwrap())
.collect();
assert_eq!(names, vec![JOB_FILE.to_string()], "{names:?}");
}
#[test]
fn a_torn_temp_file_never_hides_the_last_good_job() {
let home = test_dir("store-torn");
let store = Store::new(&home);
let job = sample_job(&test_dir("store-torn-in"));
store.create(&job).unwrap();
std::fs::write(
home.join("outbox").join(&job.id).join(".job.json.tmp.1.1"),
b"{ half",
)
.unwrap();
assert_eq!(store.load(&job.id).unwrap(), job);
assert_eq!(store.list().len(), 1);
}
#[test]
fn ids_cannot_escape_the_outbox() {
let store = Store::new(&test_dir("store-ids"));
for bad in [
"../x",
"j_../../etc",
"",
"j_ZZZZZZZZZZZZ",
"j_0123456789abc",
] {
assert!(!valid_id(bad), "{bad}");
assert!(store.load(bad).is_none());
assert!(store.request_cancel(bad).is_err());
}
assert!(valid_id(&new_id()));
}
#[test]
fn cancel_is_a_marker_that_does_not_touch_the_job_file() {
let store = Store::new(&test_dir("store-cancel"));
let job = sample_job(&test_dir("store-cancel-in"));
store.create(&job).unwrap();
assert!(!store.cancel_requested(&job.id));
store.request_cancel(&job.id).unwrap();
assert!(store.cancel_requested(&job.id));
assert_eq!(store.load(&job.id).unwrap(), job);
}
#[test]
fn upload_names_are_content_addressed_and_fit_the_hub_pattern() {
let sha = "ab".repeat(32);
let n = upload_name(&sha, "/data/my file (1).txt");
assert_eq!(n, format!("{}-my_file__1_.txt", &sha[..16]));
assert!(n.chars().next().unwrap().is_ascii_alphanumeric());
let long = upload_name(&sha, &format!("/d/{}", "x".repeat(400)));
assert!(long.len() <= 255);
}
#[test]
fn retention_removes_old_and_excess_finished_jobs_and_keeps_live_ones() {
let home = test_dir("store-prune");
let store = Store::new(&home);
let inputs = test_dir("store-prune-in");
let now = chrono::Utc::now();
let mut ids = vec![];
for (age_days, state) in [
(40, State::Done),
(10, State::Done),
(5, State::Failed),
(1, State::Done),
(90, State::Running), ] {
let mut j = sample_job(&inputs);
j.state = state;
let at = (now - chrono::Duration::days(age_days))
.to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
j.created_at = at.clone();
if state.terminal() {
j.finished_at = Some(at);
}
store.create(&j).unwrap();
std::fs::create_dir_all(store.results_dir(&j.id)).unwrap();
ids.push(j.id);
}
let removed = store.prune(
Retention {
max_age_days: 30,
max_finished: 2,
},
now,
);
assert_eq!(removed, vec![ids[0].clone(), ids[1].clone()]);
assert!(store.load(&ids[0]).is_none());
assert!(!store.results_dir(&ids[0]).exists(), "results go with it");
assert!(store.load(&ids[2]).is_some());
assert!(store.load(&ids[3]).is_some());
assert!(store.load(&ids[4]).is_some(), "a live job is never pruned");
}
}