use ring::digest::{Context, SHA256};
use serde::{Deserialize, Deserializer, Serialize};
use std::collections::BTreeMap;
use std::io::Read;
use std::path::{Path, PathBuf};
pub const MAX_INPUT_BYTES: u64 = 512 * 1024 * 1024;
pub const MAX_BATCHES: usize = 100_000;
#[derive(Debug, Clone, PartialEq)]
pub struct SpecError {
pub field: String,
pub message: String,
}
impl SpecError {
pub fn new(field: impl Into<String>, message: impl Into<String>) -> Self {
Self {
field: field.into(),
message: message.into(),
}
}
}
impl std::fmt::Display for SpecError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.field.is_empty() {
write!(f, "{}", self.message)
} else {
write!(f, "{}: {}", self.field, self.message)
}
}
}
fn de_num<'de, D: Deserializer<'de>>(d: D) -> Result<f64, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum N {
I(i64),
F(f64),
}
Ok(match N::deserialize(d)? {
N::I(i) => i as f64,
N::F(f) => f,
})
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct Resources {
pub cpus: u32,
#[serde(deserialize_with = "de_num")]
pub memory_gb: f64,
#[serde(default)]
pub gpus: u32,
#[serde(default, deserialize_with = "de_num")]
pub min_vram_gb: f64,
pub max_runtime_s: u64,
#[serde(default)]
pub est_runtime_s: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct BatchSpec {
pub inputs: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct JobSpec {
pub name: String,
#[serde(default)]
pub description: Option<String>,
pub image: String,
#[serde(default)]
pub cmd: Option<Vec<String>>,
#[serde(default)]
pub squad: Option<String>,
#[serde(deserialize_with = "de_num")]
pub max_price_per_batch: f64,
pub resources: Resources,
#[serde(default, rename = "batch", alias = "batches")]
pub batch: Vec<BatchSpec>,
#[serde(default)]
pub require_receipts: bool,
#[serde(default)]
pub request_id: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Format {
Json,
Toml,
}
pub fn parse(text: &str, format: Option<Format>) -> Result<JobSpec, SpecError> {
let format = format.unwrap_or_else(|| {
if text.trim_start().starts_with('{') {
Format::Json
} else {
Format::Toml
}
});
match format {
Format::Json => serde_json::from_str(text).map_err(|e| SpecError::new("", e.to_string())),
Format::Toml => toml::from_str(text).map_err(|e| SpecError::new("", e.to_string())),
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ImageRef {
pub as_given: String,
pub pinned_digest: Option<String>,
}
pub fn parse_image(image: &str) -> Result<ImageRef, SpecError> {
let f = "image";
if image.is_empty() {
return Err(SpecError::new(f, "must not be empty"));
}
if image.len() > 512 || image.chars().any(char::is_whitespace) {
return Err(SpecError::new(
f,
"must be one reference of at most 512 characters, without spaces",
));
}
match image.split_once('@') {
None => Ok(ImageRef {
as_given: image.into(),
pinned_digest: None,
}),
Some((repo, digest)) => {
let hex = digest.strip_prefix("sha256:").unwrap_or("");
if repo.is_empty() || hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(SpecError::new(
f,
"a digest-pinned image is `repo@sha256:` followed by 64 hex characters",
));
}
Ok(ImageRef {
as_given: image.into(),
pinned_digest: Some(format!("sha256:{}", hex.to_ascii_lowercase())),
})
}
}
}
pub fn visibility_and_squad(spec: &JobSpec) -> (&'static str, Option<String>) {
match spec.squad.as_deref().map(str::trim) {
None | Some("") | Some("public") => ("public", None),
Some("private") => ("private", None),
Some(id) => ("public", Some(id.to_string())),
}
}
fn at_most_six_places(p: f64) -> bool {
((p * 1e6).round() / 1e6 - p).abs() < 1e-9
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct InputFile {
pub path: String,
pub sha256: String,
pub size: u64,
}
#[derive(Debug)]
pub struct Checked {
pub image: ImageRef,
pub files: Vec<InputFile>,
pub batches: Vec<BTreeMap<String, usize>>,
}
pub fn sha256_file(path: &Path) -> std::io::Result<(String, u64)> {
let mut f = std::fs::File::open(path)?;
let mut ctx = Context::new(&SHA256);
let mut buf = vec![0u8; 64 * 1024];
let mut size = 0u64;
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
ctx.update(&buf[..n]);
size += n as u64;
}
Ok((hex::encode(ctx.finish().as_ref()), size))
}
pub fn resolve_paths(spec: &mut JobSpec, base: &Path) {
for b in &mut spec.batch {
for p in b.inputs.values_mut() {
let path = PathBuf::from(&*p);
if path.is_relative() {
*p = base.join(path).display().to_string();
}
}
}
}
fn valid_input_name(n: &str) -> bool {
!n.is_empty() && n.len() <= 255 && !n.contains('/') && n != "." && n != ".."
}
pub fn validate(spec: &JobSpec) -> Result<Checked, SpecError> {
fn e(f: impl Into<String>, m: impl Into<String>) -> SpecError {
SpecError::new(f, m)
}
let name = spec.name.trim();
if name.is_empty() || spec.name.chars().count() > 128 || spec.name.chars().any(char::is_control)
{
return Err(e("name", "1 to 128 characters, no control characters"));
}
if spec.description.as_ref().is_some_and(|d| d.len() > 3000) {
return Err(e("description", "at most 3000 characters"));
}
let image = parse_image(&spec.image)?;
if spec.cmd.is_some() {
return Err(e(
"cmd",
"the hub's Spaces queue has no command override (a Space is an image plus batches); \
bake the command into the image's entrypoint",
));
}
if let Some(s) = &spec.squad {
if s.len() > 36 {
return Err(e(
"squad",
"`public`, `private` or a squad id (at most 36 characters)",
));
}
}
let p = spec.max_price_per_batch;
if !p.is_finite() || p <= 0.0 {
return Err(e("max_price_per_batch", "must be a price above 0 (yen)"));
}
if p > 1e9 || !at_most_six_places(p) {
return Err(e(
"max_price_per_batch",
"at most 1000000000, with at most 6 decimal places",
));
}
let r = &spec.resources;
if !(1..=4096).contains(&r.cpus) {
return Err(e("resources.cpus", "between 1 and 4096"));
}
if !r.memory_gb.is_finite() || r.memory_gb <= 0.0 || r.memory_gb > 65536.0 {
return Err(e("resources.memory_gb", "above 0 and at most 65536"));
}
if r.gpus > 64 {
return Err(e("resources.gpus", "at most 64"));
}
if !r.min_vram_gb.is_finite() || !(0.0..=4096.0).contains(&r.min_vram_gb) {
return Err(e("resources.min_vram_gb", "between 0 and 4096"));
}
if r.max_runtime_s == 0 || r.max_runtime_s > 7 * 86400 {
return Err(e("resources.max_runtime_s", "between 1 second and 7 days"));
}
if let Some(est) = r.est_runtime_s {
if est == 0 || est > r.max_runtime_s {
return Err(e(
"resources.est_runtime_s",
"above 0 and not over max_runtime_s",
));
}
}
if let Some(id) = &spec.request_id {
if id.is_empty() || id.len() > 128 {
return Err(e("request_id", "1 to 128 characters"));
}
}
if spec.batch.is_empty() {
return Err(e("batch", "at least one [[batch]] is needed"));
}
if spec.batch.len() > MAX_BATCHES {
return Err(e("batch", format!("at most {MAX_BATCHES} batches")));
}
let mut files: Vec<InputFile> = Vec::new();
let mut by_path: BTreeMap<String, usize> = BTreeMap::new();
let mut batches = Vec::with_capacity(spec.batch.len());
for (i, b) in spec.batch.iter().enumerate() {
if b.inputs.is_empty() {
return Err(e(format!("batch[{i}].inputs"), "at least one input file"));
}
let mut resolved = BTreeMap::new();
for (name, path) in &b.inputs {
let field = format!("batch[{i}].inputs.{name}");
if !valid_input_name(name) {
return Err(e(
format!("batch[{i}].inputs"),
format!("the name `{name}` must be non-empty, without `/`"),
));
}
let pb = Path::new(path);
if !pb.is_absolute() {
return Err(e(
field,
format!(
"`{path}` is not an absolute path (the agent does not share your working directory)"
),
));
}
let idx = match by_path.get(path) {
Some(i) => *i,
None => {
let meta = std::fs::metadata(pb)
.map_err(|err| e(field.clone(), format!("`{path}`: {err}")))?;
if !meta.is_file() {
return Err(e(field, format!("`{path}` is not a regular file")));
}
if meta.len() > MAX_INPUT_BYTES {
return Err(e(
field,
format!("`{path}` is over {} MiB", MAX_INPUT_BYTES >> 20),
));
}
let (sha256, size) = sha256_file(pb).map_err(|err| {
e(field.clone(), format!("`{path}` is not readable: {err}"))
})?;
files.push(InputFile {
path: path.clone(),
sha256,
size,
});
by_path.insert(path.clone(), files.len() - 1);
files.len() - 1
}
};
resolved.insert(name.clone(), idx);
}
batches.push(resolved);
}
Ok(Checked {
image,
files,
batches,
})
}
#[cfg(test)]
pub(crate) fn test_dir(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("zc-outbox-{tag}-{}-{}", std::process::id(), {
use std::sync::atomic::{AtomicU64, Ordering};
static N: AtomicU64 = AtomicU64::new(0);
N.fetch_add(1, Ordering::Relaxed)
}));
std::fs::create_dir_all(&d).unwrap();
d
}
#[cfg(test)]
pub(crate) fn sample_spec(dir: &Path) -> JobSpec {
for (n, body) in [("a.txt", "alpha\n"), ("b.txt", "beta\n")] {
std::fs::write(dir.join(n), body).unwrap();
}
let toml = format!(
r#"
name = "digest-run"
image = "registry.example/app:1"
max_price_per_batch = 50
[resources]
cpus = 1
memory_gb = 1
max_runtime_s = 180
[[batch]]
inputs = {{ "part.txt" = "{a}" }}
[[batch]]
inputs = {{ "part.txt" = "{b}" }}
"#,
a = dir.join("a.txt").display(),
b = dir.join("b.txt").display()
);
parse(&toml, None).unwrap()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn toml_and_json_give_the_same_spec() {
let dir = test_dir("spec-fmt");
let t = sample_spec(&dir);
let j = serde_json::to_string(&t).unwrap();
assert_eq!(parse(&j, None).unwrap(), t);
assert_eq!(parse(&j, Some(Format::Json)).unwrap(), t);
assert_eq!(t.max_price_per_batch, 50.0, "an integer price is a price");
assert_eq!(t.batch.len(), 2);
}
#[test]
fn a_valid_spec_hashes_each_distinct_file_once() {
let dir = test_dir("spec-ok");
let mut s = sample_spec(&dir);
s.batch.push(s.batch[0].clone()); let c = validate(&s).unwrap();
assert_eq!(c.files.len(), 2);
assert_eq!(c.batches.len(), 3);
assert_eq!(c.batches[0], c.batches[2]);
let (sha, _) = sha256_file(&dir.join("a.txt")).unwrap();
assert_eq!((c.files[0].sha256.clone(), c.files[0].size), (sha, 6));
assert_eq!(c.image.pinned_digest, None);
assert_eq!(c.image.as_given, "registry.example/app:1");
}
#[test]
fn errors_name_the_field() {
let dir = test_dir("spec-err");
let ok = sample_spec(&dir);
let field = |s: &JobSpec| validate(s).unwrap_err().field;
let mut s = ok.clone();
s.name = " ".into();
assert_eq!(field(&s), "name");
let mut s = ok.clone();
s.max_price_per_batch = 0.0;
assert_eq!(field(&s), "max_price_per_batch");
let mut s = ok.clone();
s.max_price_per_batch = 0.0000001;
assert_eq!(field(&s), "max_price_per_batch");
let mut s = ok.clone();
s.resources.cpus = 0;
assert_eq!(field(&s), "resources.cpus");
let mut s = ok.clone();
s.resources.est_runtime_s = Some(999);
assert_eq!(field(&s), "resources.est_runtime_s");
let mut s = ok.clone();
s.batch.clear();
assert_eq!(field(&s), "batch");
let mut s = ok.clone();
s.batch[1].inputs.clear();
assert_eq!(field(&s), "batch[1].inputs");
let mut s = ok.clone();
s.batch[0].inputs.insert("a/b".into(), "/x".into());
assert_eq!(field(&s), "batch[0].inputs");
let mut s = ok.clone();
s.cmd = Some(vec!["true".into()]);
let err = validate(&s).unwrap_err();
assert_eq!(err.field, "cmd");
assert!(err.message.contains("entrypoint"));
let mut s = ok.clone();
s.image = "a b".into();
assert_eq!(field(&s), "image");
}
#[test]
fn input_files_must_exist_be_regular_and_be_absolute() {
let dir = test_dir("spec-files");
let ok = sample_spec(&dir);
let mut s = ok.clone();
s.batch[1].inputs.insert(
"part.txt".into(),
dir.join("nope.txt").display().to_string(),
);
let e = validate(&s).unwrap_err();
assert_eq!(e.field, "batch[1].inputs.part.txt");
assert!(e.message.contains("nope.txt"), "{e}");
let mut s = ok.clone();
s.batch[0]
.inputs
.insert("part.txt".into(), dir.display().to_string());
assert!(validate(&s).unwrap_err().message.contains("regular file"));
let mut s = ok.clone();
s.batch[0].inputs.insert("part.txt".into(), "a.txt".into());
assert!(validate(&s).unwrap_err().message.contains("absolute"));
resolve_paths(&mut s, &dir);
validate(&s).unwrap();
}
#[test]
fn an_unreadable_input_is_refused_at_submit() {
use std::os::unix::fs::PermissionsExt;
let dir = test_dir("spec-perm");
let s = sample_spec(&dir);
let a = dir.join("a.txt");
std::fs::set_permissions(&a, std::fs::Permissions::from_mode(0o000)).unwrap();
if std::fs::File::open(&a).is_err() {
let e = validate(&s).unwrap_err();
assert_eq!(e.field, "batch[0].inputs.part.txt");
assert!(e.message.contains("not readable"), "{e}");
}
std::fs::set_permissions(&a, std::fs::Permissions::from_mode(0o644)).unwrap();
}
#[test]
fn images_are_digest_pinned_or_recorded_as_given() {
let d = "a".repeat(64);
let p = parse_image(&format!("r.io/app@sha256:{d}")).unwrap();
assert_eq!(p.pinned_digest, Some(format!("sha256:{d}")));
let t = parse_image("r.io/app:latest").unwrap();
assert_eq!(
(t.as_given.as_str(), t.pinned_digest),
("r.io/app:latest", None)
);
assert!(parse_image("r.io/app@sha256:abc").is_err());
assert!(parse_image("r.io/app@md5:abc").is_err());
assert!(parse_image("").is_err());
}
#[test]
fn unknown_keys_are_refused_so_a_typo_cannot_drop_the_price_ceiling() {
let e = parse(
"name='x'\nimage='i'\nmax_price='5'\nmax_price_per_batch=1\n[resources]\ncpus=1\nmemory_gb=1\nmax_runtime_s=1\n",
None,
)
.unwrap_err();
assert!(e.message.contains("max_price"), "{e}");
}
#[test]
fn squad_maps_to_visibility_or_a_squad_id() {
let dir = test_dir("spec-squad");
let mut s = sample_spec(&dir);
assert_eq!(visibility_and_squad(&s), ("public", None));
s.squad = Some("public".into());
assert_eq!(visibility_and_squad(&s), ("public", None));
s.squad = Some("private".into());
assert_eq!(visibility_and_squad(&s), ("private", None));
s.squad = Some("sq-1".into());
assert_eq!(visibility_and_squad(&s), ("public", Some("sq-1".into())));
}
}