Skip to main content

evalhub_core/
validate.rs

1//! Validation: everything the hub checks before it stores a record.
2//!
3//! Validation runs in two passes and collects every failure from both into
4//! one list, so a producer sees all problems in one round trip.
5//!
6//! # Pass 1: structural
7//!
8//! The record is checked against the JSON Schema for its declared `schema`
9//! (`evalhub.card/1.0` or `evalhub.eval/1.0`) using `jsonschema` with draft
10//! 2020-12. The core is closed, so an unknown key anywhere outside `ext` is
11//! a `schema` error. In the same pass every number is inspected: an integer
12//! outside ±2^53 is `number_too_large`.
13//!
14//! The schema is the generated one from `evalhub_schema` (the same
15//! generator output that is committed under `schemas/`), built once per
16//! process. The validator is built in offline mode: every `$ref` in the
17//! schema is local (`#/$defs/…`), and a reference that would need fetching
18//! is a build error, not a fetch.
19//!
20//! # Pass 2: semantic
21//!
22//! Rules the schema language cannot express:
23//!
24//! | Rule                                                                                   | Code                       |
25//! | -------------------------------------------------------------------------------------- | -------------------------- |
26//! | `results` non-empty ⇒ `counts` present                                                 | `counts_missing`           |
27//! | `counts.attempted >= completed + failed + skipped + errored`                           | `counts_inconsistent`      |
28//! | every `results[].samples_ref`, `runs[].calls`, `runs[].artifacts[]` is an `attachments[].path` | `attachment_ref_unknown` |
29//! | `attachments[].path` unique, relative, contains no `..`                                | `attachment_path_invalid`  |
30//! | `results[].metric` matches `{ns}/{name}`                                               | `metric_id_invalid`        |
31//! | `relations[].type` matches `{ns}/{name}`                                               | `schema`                   |
32//! | `relations[].to` is `{ns}/{name}@{seq}`, `external:…`, or `hf:…`                       | `schema`                   |
33//!
34//! The semantic pass runs on whatever is present even when the structural
35//! pass failed, so a producer sees both kinds of problem at once. Errors
36//! are sorted by `(path, code)`; the order is deterministic.
37//!
38//! # What is deliberately not an error
39//!
40//! - A `relations[].to` that does not resolve. The record is accepted; the
41//!   `refs_resolved` badge is withheld. A Card may legitimately be published
42//!   before the Eval it cites.
43//! - A `harness` or `metric` the registry does not know. Accepted; the
44//!   `harness_registered` / `metric_registered` badges are withheld. The hub
45//!   does not gate on vocabulary.
46//! - An `attachments[].sha256` that has not been uploaded. This is not a
47//!   validation failure but a state conflict, `409 attachment_missing`,
48//!   raised by the store, because only the store knows.
49//!
50//! # Output
51//!
52//! `Vec<Error { path, code, hint }>`, empty on success. `path` is a JSON
53//! pointer into the submitted record (`/results/0/metric`; the root is
54//! `""`). `hint` is prose and not part of the contract.
55
56use std::collections::HashSet;
57use std::sync::OnceLock;
58
59use jsonschema::{Draft, Validator};
60use serde_json::Value;
61
62use evalhub_schema::RecordKind;
63use evalhub_schema::error::{ErrorCode, ErrorEntry};
64
65/// Largest integer a JSON number may carry without losing precision as a
66/// double: 2^53.
67const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
68
69/// Where a relation points.
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub enum RelationTarget {
72    /// A version on this hub, `{ns}/{name}@{seq}`.
73    Version {
74        /// Namespace of the target record.
75        ns: String,
76        /// Name of the target record.
77        name: String,
78        /// Sequence number of the target version, from 1.
79        seq: u32,
80    },
81    /// Something outside the hub, written `external:<anything>`.
82    External(String),
83    /// A Hugging Face reference, written `hf:<org>/<repo>@<sha>`.
84    Hf(String),
85}
86
87/// Whether `s` has the form `{ns}/{name}`: exactly one `/`, both parts
88/// non-empty, each starting with `[a-z0-9]` and continuing with
89/// `[a-z0-9._-]`.
90pub fn is_valid_id(s: &str) -> bool {
91    let Some((ns, name)) = s.split_once('/') else {
92        return false;
93    };
94    is_slug(ns) && is_slug(name)
95}
96
97fn is_slug(part: &str) -> bool {
98    let mut chars = part.chars();
99    match chars.next() {
100        Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
101        _ => return false,
102    }
103    chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
104}
105
106/// Parse a `relations[].to` value. `None` when it has none of the accepted
107/// forms.
108pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
109    if let Some(rest) = s.strip_prefix("external:") {
110        return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
111    }
112    if let Some(rest) = s.strip_prefix("hf:") {
113        return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
114    }
115    let (id, seq) = s.rsplit_once('@')?;
116    let (ns, name) = id.split_once('/')?;
117    if !is_slug(ns) || !is_slug(name) {
118        return None;
119    }
120    // `parse::<u32>` accepts a leading `+`; the address form does not.
121    if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
122        return None;
123    }
124    let seq: u32 = seq.parse().ok()?;
125    if seq == 0 {
126        return None;
127    }
128    Some(RelationTarget::Version {
129        ns: ns.to_string(),
130        name: name.to_string(),
131        seq,
132    })
133}
134
135/// Run both passes and return every failure, sorted by `(path, code)`.
136pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
137    let mut errors = Vec::new();
138    structural(kind, value, &mut errors);
139    numbers("", value, &mut errors);
140    semantic(kind, value, &mut errors);
141    // `ErrorCode` is a plain enum; its declaration order is the tie-break.
142    errors.sort_by(|a, b| {
143        a.path
144            .cmp(&b.path)
145            .then_with(|| (a.code as u8).cmp(&(b.code as u8)))
146    });
147    errors
148}
149
150fn validator(kind: RecordKind) -> &'static Validator {
151    static CARD: OnceLock<Validator> = OnceLock::new();
152    static EVAL: OnceLock<Validator> = OnceLock::new();
153    let cell = match kind {
154        RecordKind::Card => &CARD,
155        RecordKind::Eval => &EVAL,
156    };
157    cell.get_or_init(|| {
158        let schema = kind.schema().to_value();
159        // The generated schema only references `#/$defs/*`; `offline()`
160        // turns any other reference into a build failure here, which the
161        // unit tests exercise, rather than a fetch in production.
162        #[allow(clippy::expect_used)]
163        jsonschema::options()
164            .with_draft(Draft::Draft202012)
165            .offline()
166            .build(&schema)
167            .expect("the generated record schema compiles")
168    })
169}
170
171fn structural(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
172    for e in validator(kind).iter_errors(value) {
173        out.push(ErrorEntry {
174            path: e.instance_path().as_str().to_string(),
175            code: ErrorCode::Schema,
176            hint: Some(e.to_string()),
177        });
178    }
179}
180
181fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
182    match value {
183        Value::Number(n) => {
184            let too_large = if let Some(i) = n.as_i64() {
185                i.unsigned_abs() > MAX_SAFE_INTEGER as u64
186            } else if let Some(u) = n.as_u64() {
187                u > MAX_SAFE_INTEGER as u64
188            } else if let Some(f) = n.as_f64() {
189                f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
190            } else {
191                false
192            };
193            if too_large {
194                out.push(ErrorEntry {
195                    path: path.to_string(),
196                    code: ErrorCode::NumberTooLarge,
197                    hint: Some(format!(
198                        "{n} is an integer outside ±2^53; send it as a string"
199                    )),
200                });
201            }
202        }
203        Value::Array(items) => {
204            for (i, item) in items.iter().enumerate() {
205                numbers(&format!("{path}/{i}"), item, out);
206            }
207        }
208        Value::Object(map) => {
209            for (k, v) in map {
210                numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
211            }
212        }
213        _ => {}
214    }
215}
216
217/// RFC 6901 escaping for one reference token.
218fn escape_pointer(key: &str) -> String {
219    key.replace('~', "~0").replace('/', "~1")
220}
221
222fn semantic(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
223    let Some(obj) = value.as_object() else {
224        return;
225    };
226    let push = |out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String| {
227        out.push(ErrorEntry {
228            path,
229            code,
230            hint: Some(hint),
231        })
232    };
233
234    // Attachment paths: unique, relative, no `..`.
235    let mut paths: HashSet<&str> = HashSet::new();
236    if let Some(attachments) = obj.get("attachments").and_then(Value::as_array) {
237        for (i, a) in attachments.iter().enumerate() {
238            let Some(p) = a.get("path").and_then(Value::as_str) else {
239                continue;
240            };
241            let pointer = format!("/attachments/{i}/path");
242            let invalid = if p.is_empty() {
243                Some("empty")
244            } else if p.starts_with('/') {
245                Some("absolute")
246            } else if p.split('/').any(|seg| seg == "..") {
247                Some("contains `..`")
248            } else if !paths.insert(p) {
249                Some("duplicate")
250            } else {
251                None
252            };
253            if let Some(why) = invalid {
254                push(
255                    out,
256                    pointer,
257                    ErrorCode::AttachmentPathInvalid,
258                    format!("attachment path {p:?} is {why}"),
259                );
260            }
261        }
262    }
263    let ref_unknown = |out: &mut Vec<ErrorEntry>, pointer: String, p: &str| {
264        if !paths.contains(p) {
265            push(
266                out,
267                pointer,
268                ErrorCode::AttachmentRefUnknown,
269                format!("{p:?} names no attachments[].path"),
270            );
271        }
272    };
273
274    if kind == RecordKind::Card {
275        let results = obj.get("results").and_then(Value::as_array);
276        if let Some(results) = results {
277            for (i, r) in results.iter().enumerate() {
278                if let Some(m) = r.get("metric").and_then(Value::as_str)
279                    && !is_valid_id(m)
280                {
281                    push(
282                        out,
283                        format!("/results/{i}/metric"),
284                        ErrorCode::MetricIdInvalid,
285                        format!("{m:?} is not of the form {{ns}}/{{name}}"),
286                    );
287                }
288                if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
289                    ref_unknown(out, format!("/results/{i}/samples_ref"), s);
290                }
291            }
292            let counts = obj.get("counts").filter(|c| !c.is_null());
293            match counts {
294                None if !results.is_empty() => push(
295                    out,
296                    "/counts".to_string(),
297                    ErrorCode::CountsMissing,
298                    "results are present, so counts is required".to_string(),
299                ),
300                Some(c) => {
301                    let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
302                    let sum = n("completed") + n("failed") + n("skipped") + n("errored");
303                    if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
304                        && attempted < sum
305                    {
306                        push(
307                            out,
308                            "/counts".to_string(),
309                            ErrorCode::CountsInconsistent,
310                            format!(
311                                "attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
312                            ),
313                        );
314                    }
315                }
316                None => {}
317            }
318        }
319    }
320
321    if kind == RecordKind::Eval
322        && let Some(runs) = obj.get("runs").and_then(Value::as_array)
323    {
324        for (i, run) in runs.iter().enumerate() {
325            if let Some(c) = run.get("calls").and_then(Value::as_str) {
326                ref_unknown(out, format!("/runs/{i}/calls"), c);
327            }
328            if let Some(artifacts) = run.get("artifacts").and_then(Value::as_array) {
329                for (j, a) in artifacts.iter().enumerate() {
330                    if let Some(a) = a.as_str() {
331                        ref_unknown(out, format!("/runs/{i}/artifacts/{j}"), a);
332                    }
333                }
334            }
335        }
336    }
337
338    if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
339        for (i, rel) in relations.iter().enumerate() {
340            if let Some(t) = rel.get("type").and_then(Value::as_str)
341                && !is_valid_id(t)
342            {
343                push(
344                    out,
345                    format!("/relations/{i}/type"),
346                    ErrorCode::Schema,
347                    format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
348                );
349            }
350            if let Some(to) = rel.get("to").and_then(Value::as_str)
351                && parse_relation_target(to).is_none()
352            {
353                push(
354                    out,
355                    format!("/relations/{i}/to"),
356                    ErrorCode::Schema,
357                    format!(
358                        "{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
359                    ),
360                );
361            }
362        }
363    }
364}
365
366#[cfg(test)]
367mod tests {
368    #![allow(clippy::unwrap_used)]
369
370    use super::*;
371
372    #[test]
373    fn ids() {
374        assert!(is_valid_id("core/pass_rate"));
375        assert!(is_valid_id("alice/qwen-loop.v2"));
376        assert!(!is_valid_id("pass_rate"));
377        assert!(!is_valid_id("Core/pass_rate"));
378        assert!(!is_valid_id("core/"));
379        assert!(!is_valid_id("/x"));
380        assert!(!is_valid_id("a/b/c"));
381        assert!(!is_valid_id("-a/b"));
382    }
383
384    #[test]
385    fn relation_targets() {
386        assert_eq!(
387            parse_relation_target("alice/single2-k4@3"),
388            Some(RelationTarget::Version {
389                ns: "alice".into(),
390                name: "single2-k4".into(),
391                seq: 3
392            })
393        );
394        assert_eq!(
395            parse_relation_target("external:https://x/y"),
396            Some(RelationTarget::External("https://x/y".into()))
397        );
398        assert_eq!(
399            parse_relation_target("hf:org/repo@sha"),
400            Some(RelationTarget::Hf("org/repo@sha".into()))
401        );
402        for bad in [
403            "alice/x",
404            "alice/x@0",
405            "alice/x@+1",
406            "alice/x@label",
407            "alice@3",
408            "external:",
409            "hf:",
410            "",
411        ] {
412            assert_eq!(parse_relation_target(bad), None, "{bad}");
413        }
414    }
415
416    #[test]
417    fn validators_build_offline() {
418        let _ = validator(RecordKind::Card);
419        let _ = validator(RecordKind::Eval);
420    }
421
422    #[test]
423    fn pointer_escaping() {
424        let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
425        let mut out = Vec::new();
426        numbers("", &v, &mut out);
427        assert_eq!(out.len(), 1);
428        assert_eq!(out[0].path, "/ext/a~1b/n");
429        assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
430    }
431}