use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;
use jsonschema::{Draft, Validator};
use serde_json::{Map, Value};
use evalhub_schema::error::{ErrorCode, ErrorEntry};
use evalhub_schema::{EVAL_SCHEMA, RecordKind};
const EVAL_SCHEMA_1_0: &str = "evalhub.eval/1.0";
const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
pub const MAX_RUN_ID_BYTES: usize = 200;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RelationTarget {
Version {
ns: String,
name: String,
seq: u32,
},
External(String),
Hf(String),
}
pub fn is_valid_id(s: &str) -> bool {
let Some((ns, name)) = s.split_once('/') else {
return false;
};
is_slug(ns) && is_slug(name)
}
pub fn is_valid_run_id(s: &str) -> bool {
!s.is_empty() && !s.contains('/') && s.len() <= MAX_RUN_ID_BYTES
}
fn is_slug(part: &str) -> bool {
let mut chars = part.chars();
match chars.next() {
Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
_ => return false,
}
chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
}
pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
if let Some(rest) = s.strip_prefix("external:") {
return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
}
if let Some(rest) = s.strip_prefix("hf:") {
return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
}
let (id, seq) = s.rsplit_once('@')?;
let (ns, name) = id.split_once('/')?;
if !is_slug(ns) || !is_slug(name) {
return None;
}
if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let seq: u32 = seq.parse().ok()?;
if seq == 0 {
return None;
}
Some(RelationTarget::Version {
ns: ns.to_string(),
name: name.to_string(),
seq,
})
}
pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
let declared = value.get("schema").and_then(Value::as_str);
let runs_moved_here = kind == RecordKind::Eval
&& match declared {
Some(EVAL_SCHEMA_1_0) => true,
Some(EVAL_SCHEMA) => value.get("runs").is_some(),
_ => false,
};
if runs_moved_here {
return vec![runs_moved()];
}
let mut errors = Vec::new();
structural(record_validator(kind, declared), value, &mut errors);
numbers("", value, &mut errors);
semantic(kind, value, &mut errors);
sort(&mut errors);
errors
}
pub fn run(run_id: &str, value: &Value) -> Vec<ErrorEntry> {
let mut errors = Vec::new();
structural(run_validator(), value, &mut errors);
numbers("", value, &mut errors);
run_semantic(run_id, value, &mut errors);
sort(&mut errors);
errors
}
fn runs_moved() -> ErrorEntry {
ErrorEntry {
path: "/runs".to_string(),
code: ErrorCode::RunsMoved,
hint: Some(
"an Eval body is an evalhub.eval/2.0 header, the only Eval schema \
accepted (evalhub.eval/1.0 is not), and it does not carry runs; \
write each run with PUT /evals/{ns}/{name}/runs/{run_id}, or \
several with POST /evals/{ns}/{name}/runs:batch"
.to_string(),
),
}
}
fn sort(errors: &mut [ErrorEntry]) {
errors.sort_by(|a, b| {
a.path
.cmp(&b.path)
.then_with(|| (a.code as u8).cmp(&(b.code as u8)))
});
}
fn compile(schema: &Value) -> Validator {
#[allow(clippy::expect_used)]
jsonschema::options()
.with_draft(Draft::Draft202012)
.should_validate_formats(true)
.should_ignore_unknown_formats(true)
.offline()
.build(schema)
.expect("the generated schema compiles")
}
fn record_validators() -> &'static HashMap<&'static str, Validator> {
static CACHE: OnceLock<HashMap<&'static str, Validator>> = OnceLock::new();
CACHE.get_or_init(|| {
let mut map = HashMap::new();
for kind in [RecordKind::Card, RecordKind::Eval] {
for &id in kind.schema_ids() {
if let Some(schema) = kind.schema_for(id) {
map.insert(id, compile(schema.as_value()));
}
}
}
map
})
}
fn record_validator(kind: RecordKind, declared: Option<&str>) -> &'static Validator {
let cache = record_validators();
let accepted = declared.filter(|id| kind.schema_ids().contains(id));
let id = accepted.unwrap_or(kind.schema_id());
#[allow(clippy::expect_used)]
cache
.get(id)
.expect("every accepted schema identifier has a document")
}
fn run_validator() -> &'static Validator {
static RUN: OnceLock<Validator> = OnceLock::new();
RUN.get_or_init(|| compile(evalhub_schema::schema_for_run().as_value()))
}
fn structural(validator: &Validator, value: &Value, out: &mut Vec<ErrorEntry>) {
for e in validator.iter_errors(value) {
out.push(ErrorEntry {
path: e.instance_path().as_str().to_string(),
code: ErrorCode::Schema,
hint: Some(e.to_string()),
});
}
}
fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
match value {
Value::Number(n) => {
let too_large = if let Some(i) = n.as_i64() {
i.unsigned_abs() > MAX_SAFE_INTEGER as u64
} else if let Some(u) = n.as_u64() {
u > MAX_SAFE_INTEGER as u64
} else if let Some(f) = n.as_f64() {
f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
} else {
false
};
if too_large {
out.push(ErrorEntry {
path: path.to_string(),
code: ErrorCode::NumberTooLarge,
hint: Some(format!(
"{n} is an integer outside ±2^53; send it as a string"
)),
});
}
}
Value::Array(items) => {
for (i, item) in items.iter().enumerate() {
numbers(&format!("{path}/{i}"), item, out);
}
}
Value::Object(map) => {
for (k, v) in map {
numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
}
}
_ => {}
}
}
fn escape_pointer(key: &str) -> String {
key.replace('~', "~0").replace('/', "~1")
}
fn push(out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String) {
out.push(ErrorEntry {
path,
code,
hint: Some(hint),
});
}
fn attachment_paths<'a>(
obj: &'a Map<String, Value>,
out: &mut Vec<ErrorEntry>,
) -> HashSet<&'a str> {
let mut paths: HashSet<&str> = HashSet::new();
let Some(attachments) = obj.get("attachments").and_then(Value::as_array) else {
return paths;
};
for (i, a) in attachments.iter().enumerate() {
let Some(p) = a.get("path").and_then(Value::as_str) else {
continue;
};
let invalid = if p.is_empty() {
Some("empty")
} else if p.starts_with('/') {
Some("absolute")
} else if p.split('/').any(|seg| seg == "..") {
Some("contains `..`")
} else if !paths.insert(p) {
Some("duplicate")
} else {
None
};
if let Some(why) = invalid {
push(
out,
format!("/attachments/{i}/path"),
ErrorCode::AttachmentPathInvalid,
format!("attachment path {p:?} is {why}"),
);
}
}
paths
}
fn ref_known(paths: &HashSet<&str>, out: &mut Vec<ErrorEntry>, pointer: String, p: &str) {
if !paths.contains(p) {
push(
out,
pointer,
ErrorCode::AttachmentRefUnknown,
format!("{p:?} names no attachments[].path"),
);
}
}
fn metric_id(out: &mut Vec<ErrorEntry>, pointer: String, m: &str) {
if !is_valid_id(m) {
push(
out,
pointer,
ErrorCode::MetricIdInvalid,
format!("{m:?} is not of the form {{ns}}/{{name}}"),
);
}
}
const FACETS_WITH_EXT: [&str; 7] = [
"model",
"task",
"harness",
"generation",
"trial",
"grading",
"env",
];
fn ext_keys(out: &mut Vec<ErrorEntry>, prefix: &str, obj: &Map<String, Value>) {
let Some(ext) = obj.get("ext").and_then(Value::as_object) else {
return;
};
for key in ext.keys() {
if !is_valid_id(key) {
push(
out,
format!("{prefix}/ext/{}", escape_pointer(key)),
ErrorCode::ExtKeyInvalid,
format!("ext key {key:?} is not of the form {{ns}}/{{name}}"),
);
}
}
}
fn ext_keys_everywhere(out: &mut Vec<ErrorEntry>, obj: &Map<String, Value>) {
ext_keys(out, "", obj);
for facet in FACETS_WITH_EXT {
if let Some(f) = obj.get(facet).and_then(Value::as_object) {
ext_keys(out, &format!("/{facet}"), f);
}
}
}
fn given(obj: &Map<String, Value>, key: &str) -> bool {
obj.get(key).is_some_and(|v| !v.is_null())
}
fn semantic(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
let Some(obj) = value.as_object() else {
return;
};
let paths = attachment_paths(obj, out);
ext_keys_everywhere(out, obj);
if kind == RecordKind::Card {
let results = obj.get("results").and_then(Value::as_array);
if let Some(results) = results {
for (i, r) in results.iter().enumerate() {
if let Some(m) = r.get("metric").and_then(Value::as_str) {
metric_id(out, format!("/results/{i}/metric"), m);
}
if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
ref_known(&paths, out, format!("/results/{i}/samples_ref"), s);
}
}
let counts = obj.get("counts").filter(|c| !c.is_null());
match counts {
None if !results.is_empty() => push(
out,
"/counts".to_string(),
ErrorCode::CountsMissing,
"results are present, so counts is required".to_string(),
),
Some(c) => {
let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
let sum = n("completed") + n("failed") + n("skipped") + n("errored");
if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
&& attempted < sum
{
push(
out,
"/counts".to_string(),
ErrorCode::CountsInconsistent,
format!(
"attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
),
);
}
}
None => {}
}
}
run_results(obj, out);
}
if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
for (i, rel) in relations.iter().enumerate() {
if let Some(t) = rel.get("type").and_then(Value::as_str)
&& !is_valid_id(t)
{
push(
out,
format!("/relations/{i}/type"),
ErrorCode::Schema,
format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
);
}
if let Some(to) = rel.get("to").and_then(Value::as_str)
&& parse_relation_target(to).is_none()
{
push(
out,
format!("/relations/{i}/to"),
ErrorCode::Schema,
format!(
"{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
),
);
}
}
}
}
fn used_eval_records(obj: &Map<String, Value>) -> HashSet<String> {
let Some(relations) = obj.get("relations").and_then(Value::as_array) else {
return HashSet::new();
};
relations
.iter()
.filter(|rel| rel.get("type").and_then(Value::as_str) == Some("core/uses_eval"))
.filter_map(|rel| rel.get("to").and_then(Value::as_str))
.filter_map(|to| match parse_relation_target(to)? {
RelationTarget::Version { ns, name, .. } => Some(format!("{ns}/{name}")),
RelationTarget::External(_) | RelationTarget::Hf(_) => None,
})
.collect()
}
fn run_results(obj: &Map<String, Value>, out: &mut Vec<ErrorEntry>) {
let Some(items) = obj.get("run_results").and_then(Value::as_array) else {
return;
};
let evals = used_eval_records(obj);
for (i, item) in items.iter().enumerate() {
let Some(item) = item.as_object() else {
continue;
};
if let Some(m) = item.get("metric").and_then(Value::as_str) {
metric_id(out, format!("/run_results/{i}/metric"), m);
}
if !given(item, "value") && !given(item, "label") {
push(
out,
format!("/run_results/{i}"),
ErrorCode::RunResultValueOrLabel,
"a run result needs a value, a label, or both".to_string(),
);
}
if let Some(e) = item.get("eval").and_then(Value::as_str)
&& !evals.contains(e)
{
push(
out,
format!("/run_results/{i}/eval"),
ErrorCode::RunResultsEvalUnknown,
format!(
"{e:?} is not the {{ns}}/{{name}} (without @seq) of any core/uses_eval target of this Card"
),
);
}
}
}
fn run_semantic(run_id: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
if !is_valid_run_id(run_id) {
push(
out,
"/run_id".to_string(),
ErrorCode::RunIdInvalid,
format!(
"run_id {run_id:?} must be non-empty, contain no `/`, and be at most {MAX_RUN_ID_BYTES} bytes"
),
);
}
let Some(obj) = value.as_object() else {
return;
};
if let Some(body_id) = obj.get("run_id").and_then(Value::as_str)
&& body_id != run_id
{
push(
out,
"/run_id".to_string(),
ErrorCode::RunIdMismatch,
format!("the body says run_id {body_id:?}, but it is written as {run_id:?}"),
);
}
let status = obj.get("status").and_then(Value::as_str);
let expects = match status {
Some("ok") => Some((false, false)),
Some("error") => Some((true, false)),
Some("skipped") => Some((false, true)),
_ => None,
};
if let (Some(status), Some((wants_error, wants_skip))) = (status, expects) {
for (key, wanted) in [("error", wants_error), ("skip_reason", wants_skip)] {
let has = given(obj, key);
if has != wanted {
let hint = if wanted {
format!("status {status:?} requires {key}")
} else {
format!(
"{key} is only allowed when status is {:?}",
if key == "error" { "error" } else { "skipped" }
)
};
push(out, format!("/{key}"), ErrorCode::RunStatusDetail, hint);
}
}
}
let paths = attachment_paths(obj, out);
if let Some(c) = obj.get("calls").and_then(Value::as_str) {
ref_known(&paths, out, "/calls".to_string(), c);
}
if let Some(artifacts) = obj.get("artifacts").and_then(Value::as_array) {
for (j, a) in artifacts.iter().enumerate() {
if let Some(a) = a.as_str() {
ref_known(&paths, out, format!("/artifacts/{j}"), a);
}
}
}
if let Some(log) = obj
.get("error")
.and_then(|e| e.get("log"))
.and_then(Value::as_str)
{
ref_known(&paths, out, "/error/log".to_string(), log);
}
if let Some(metrics) = obj.get("metrics").and_then(Value::as_object) {
for key in metrics.keys() {
metric_id(out, format!("/metrics/{}", escape_pointer(key)), key);
}
}
ext_keys_everywhere(out, obj);
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
#[test]
fn ids() {
assert!(is_valid_id("core/pass_rate"));
assert!(is_valid_id("alice/qwen-loop.v2"));
assert!(!is_valid_id("pass_rate"));
assert!(!is_valid_id("Core/pass_rate"));
assert!(!is_valid_id("core/"));
assert!(!is_valid_id("/x"));
assert!(!is_valid_id("a/b/c"));
assert!(!is_valid_id("-a/b"));
}
#[test]
fn run_ids() {
assert!(is_valid_run_id("r1"));
assert!(is_valid_run_id("task 7, trial 2"));
assert!(is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES)));
assert!(!is_valid_run_id(""));
assert!(!is_valid_run_id("a/b"));
assert!(!is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES + 1)));
assert!(!is_valid_run_id(&"あ".repeat(67)));
assert!(is_valid_run_id(&"あ".repeat(66)));
}
#[test]
fn relation_targets() {
assert_eq!(
parse_relation_target("alice/single2-k4@3"),
Some(RelationTarget::Version {
ns: "alice".into(),
name: "single2-k4".into(),
seq: 3
})
);
assert_eq!(
parse_relation_target("external:https://x/y"),
Some(RelationTarget::External("https://x/y".into()))
);
assert_eq!(
parse_relation_target("hf:org/repo@sha"),
Some(RelationTarget::Hf("org/repo@sha".into()))
);
for bad in [
"alice/x",
"alice/x@0",
"alice/x@+1",
"alice/x@label",
"alice@3",
"external:",
"hf:",
"",
] {
assert_eq!(parse_relation_target(bad), None, "{bad}");
}
}
#[test]
fn validators_build_offline_one_per_accepted_identifier() {
let cache = record_validators();
for kind in [RecordKind::Card, RecordKind::Eval] {
for id in kind.schema_ids() {
assert!(cache.contains_key(id), "{id}");
}
}
assert_eq!(
cache.len(),
RecordKind::Card.schema_ids().len() + RecordKind::Eval.schema_ids().len()
);
let _ = run_validator();
}
#[test]
fn pointer_escaping() {
let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
let mut out = Vec::new();
numbers("", &v, &mut out);
assert_eq!(out.len(), 1);
assert_eq!(out[0].path, "/ext/a~1b/n");
assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
}
}