use std::collections::HashSet;
use std::sync::OnceLock;
use jsonschema::{Draft, Validator};
use serde_json::Value;
use evalhub_schema::RecordKind;
use evalhub_schema::error::{ErrorCode, ErrorEntry};
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RelationTarget {
Version {
ns: String,
name: String,
seq: u32,
},
External(String),
Hf(String),
}
pub fn is_valid_id(s: &str) -> bool {
let Some((ns, name)) = s.split_once('/') else {
return false;
};
is_slug(ns) && is_slug(name)
}
fn is_slug(part: &str) -> bool {
let mut chars = part.chars();
match chars.next() {
Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
_ => return false,
}
chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
}
pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
if let Some(rest) = s.strip_prefix("external:") {
return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
}
if let Some(rest) = s.strip_prefix("hf:") {
return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
}
let (id, seq) = s.rsplit_once('@')?;
let (ns, name) = id.split_once('/')?;
if !is_slug(ns) || !is_slug(name) {
return None;
}
if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let seq: u32 = seq.parse().ok()?;
if seq == 0 {
return None;
}
Some(RelationTarget::Version {
ns: ns.to_string(),
name: name.to_string(),
seq,
})
}
pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
let mut errors = Vec::new();
structural(kind, value, &mut errors);
numbers("", value, &mut errors);
semantic(kind, value, &mut errors);
errors.sort_by(|a, b| {
a.path
.cmp(&b.path)
.then_with(|| (a.code as u8).cmp(&(b.code as u8)))
});
errors
}
fn validator(kind: RecordKind) -> &'static Validator {
static CARD: OnceLock<Validator> = OnceLock::new();
static EVAL: OnceLock<Validator> = OnceLock::new();
let cell = match kind {
RecordKind::Card => &CARD,
RecordKind::Eval => &EVAL,
};
cell.get_or_init(|| {
let schema = kind.schema().to_value();
#[allow(clippy::expect_used)]
jsonschema::options()
.with_draft(Draft::Draft202012)
.offline()
.build(&schema)
.expect("the generated record schema compiles")
})
}
fn structural(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
for e in validator(kind).iter_errors(value) {
out.push(ErrorEntry {
path: e.instance_path().as_str().to_string(),
code: ErrorCode::Schema,
hint: Some(e.to_string()),
});
}
}
fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
match value {
Value::Number(n) => {
let too_large = if let Some(i) = n.as_i64() {
i.unsigned_abs() > MAX_SAFE_INTEGER as u64
} else if let Some(u) = n.as_u64() {
u > MAX_SAFE_INTEGER as u64
} else if let Some(f) = n.as_f64() {
f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
} else {
false
};
if too_large {
out.push(ErrorEntry {
path: path.to_string(),
code: ErrorCode::NumberTooLarge,
hint: Some(format!(
"{n} is an integer outside ±2^53; send it as a string"
)),
});
}
}
Value::Array(items) => {
for (i, item) in items.iter().enumerate() {
numbers(&format!("{path}/{i}"), item, out);
}
}
Value::Object(map) => {
for (k, v) in map {
numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
}
}
_ => {}
}
}
fn escape_pointer(key: &str) -> String {
key.replace('~', "~0").replace('/', "~1")
}
fn semantic(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
let Some(obj) = value.as_object() else {
return;
};
let push = |out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String| {
out.push(ErrorEntry {
path,
code,
hint: Some(hint),
})
};
let mut paths: HashSet<&str> = HashSet::new();
if let Some(attachments) = obj.get("attachments").and_then(Value::as_array) {
for (i, a) in attachments.iter().enumerate() {
let Some(p) = a.get("path").and_then(Value::as_str) else {
continue;
};
let pointer = format!("/attachments/{i}/path");
let invalid = if p.is_empty() {
Some("empty")
} else if p.starts_with('/') {
Some("absolute")
} else if p.split('/').any(|seg| seg == "..") {
Some("contains `..`")
} else if !paths.insert(p) {
Some("duplicate")
} else {
None
};
if let Some(why) = invalid {
push(
out,
pointer,
ErrorCode::AttachmentPathInvalid,
format!("attachment path {p:?} is {why}"),
);
}
}
}
let ref_unknown = |out: &mut Vec<ErrorEntry>, pointer: String, p: &str| {
if !paths.contains(p) {
push(
out,
pointer,
ErrorCode::AttachmentRefUnknown,
format!("{p:?} names no attachments[].path"),
);
}
};
if kind == RecordKind::Card {
let results = obj.get("results").and_then(Value::as_array);
if let Some(results) = results {
for (i, r) in results.iter().enumerate() {
if let Some(m) = r.get("metric").and_then(Value::as_str)
&& !is_valid_id(m)
{
push(
out,
format!("/results/{i}/metric"),
ErrorCode::MetricIdInvalid,
format!("{m:?} is not of the form {{ns}}/{{name}}"),
);
}
if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
ref_unknown(out, format!("/results/{i}/samples_ref"), s);
}
}
let counts = obj.get("counts").filter(|c| !c.is_null());
match counts {
None if !results.is_empty() => push(
out,
"/counts".to_string(),
ErrorCode::CountsMissing,
"results are present, so counts is required".to_string(),
),
Some(c) => {
let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
let sum = n("completed") + n("failed") + n("skipped") + n("errored");
if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
&& attempted < sum
{
push(
out,
"/counts".to_string(),
ErrorCode::CountsInconsistent,
format!(
"attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
),
);
}
}
None => {}
}
}
}
if kind == RecordKind::Eval
&& let Some(runs) = obj.get("runs").and_then(Value::as_array)
{
for (i, run) in runs.iter().enumerate() {
if let Some(c) = run.get("calls").and_then(Value::as_str) {
ref_unknown(out, format!("/runs/{i}/calls"), c);
}
if let Some(artifacts) = run.get("artifacts").and_then(Value::as_array) {
for (j, a) in artifacts.iter().enumerate() {
if let Some(a) = a.as_str() {
ref_unknown(out, format!("/runs/{i}/artifacts/{j}"), a);
}
}
}
}
}
if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
for (i, rel) in relations.iter().enumerate() {
if let Some(t) = rel.get("type").and_then(Value::as_str)
&& !is_valid_id(t)
{
push(
out,
format!("/relations/{i}/type"),
ErrorCode::Schema,
format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
);
}
if let Some(to) = rel.get("to").and_then(Value::as_str)
&& parse_relation_target(to).is_none()
{
push(
out,
format!("/relations/{i}/to"),
ErrorCode::Schema,
format!(
"{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
),
);
}
}
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
#[test]
fn ids() {
assert!(is_valid_id("core/pass_rate"));
assert!(is_valid_id("alice/qwen-loop.v2"));
assert!(!is_valid_id("pass_rate"));
assert!(!is_valid_id("Core/pass_rate"));
assert!(!is_valid_id("core/"));
assert!(!is_valid_id("/x"));
assert!(!is_valid_id("a/b/c"));
assert!(!is_valid_id("-a/b"));
}
#[test]
fn relation_targets() {
assert_eq!(
parse_relation_target("alice/single2-k4@3"),
Some(RelationTarget::Version {
ns: "alice".into(),
name: "single2-k4".into(),
seq: 3
})
);
assert_eq!(
parse_relation_target("external:https://x/y"),
Some(RelationTarget::External("https://x/y".into()))
);
assert_eq!(
parse_relation_target("hf:org/repo@sha"),
Some(RelationTarget::Hf("org/repo@sha".into()))
);
for bad in [
"alice/x",
"alice/x@0",
"alice/x@+1",
"alice/x@label",
"alice@3",
"external:",
"hf:",
"",
] {
assert_eq!(parse_relation_target(bad), None, "{bad}");
}
}
#[test]
fn validators_build_offline() {
let _ = validator(RecordKind::Card);
let _ = validator(RecordKind::Eval);
}
#[test]
fn pointer_escaping() {
let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
let mut out = Vec::new();
numbers("", &v, &mut out);
assert_eq!(out.len(), 1);
assert_eq!(out[0].path, "/ext/a~1b/n");
assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
}
}