Skip to main content

evalhub_core/
validate.rs

1//! Validation: everything the hub checks before it stores a record or a
2//! run.
3//!
4//! Validation runs in two passes and collects every failure from both into
5//! one list, so a producer sees all problems in one round trip. There are
6//! two entry points:
7//!
8//! - [`validate`] for a record (a Card, or an Eval header), keyed by
9//!   [`RecordKind`];
10//! - [`run()`] for one run of an Eval, given the `run_id` it is written
11//!   under.
12//!
13//! A run is not a [`RecordKind`]. `RecordKind` is the two kinds that appear
14//! in URLs and in the store's `type` column, and every match on it is
15//! exhaustive; a run is a row of an Eval record, addressed under it, so it
16//! has its own entry point rather than a third variant that every one of
17//! those matches would have to reject.
18//!
19//! # Pass 1: structural
20//!
21//! The value is checked against a JSON Schema using `jsonschema` with
22//! draft 2020-12. The core is closed, so an unknown key anywhere outside
23//! `ext` is a `schema` error. In the same pass every number is inspected:
24//! an integer outside ±2^53 is `number_too_large`.
25//!
26//! Which schema: a record is checked against the document for the
27//! identifier its own `schema` key declares, through
28//! [`RecordKind::schema_for`]:
29//!
30//! | Declared `schema`                      | Document               |
31//! | -------------------------------------- | ---------------------- |
32//! | `evalhub.card/1.1`, `evalhub.card/1.0` | `schemas/card.json`    |
33//! | `evalhub.eval/2.0`                     | `schemas/eval-2.json`  |
34//! | `evalhub.eval/1.0`                     | `schemas/eval.json`    |
35//! | anything else, or no `schema` key      | the kind's current one |
36//!
37//! The last row is what makes an unknown identifier an error: the current
38//! document's `schema` key is a constant, so the value fails it at
39//! `/schema` (`schema`), exactly as it did before a kind had two
40//! identifiers. A run is checked against `schemas/run.json`.
41//!
42//! The schemas are the generated ones from `evalhub_schema` (the same
43//! generator output that is committed under `schemas/`). Each is compiled
44//! once per process, one validator per accepted identifier plus one for
45//! the run, on first use. The validators are built in offline mode: every
46//! `$ref` in the schema is local (`#/$defs/…`), and a reference that would
47//! need fetching is a build error, not a fetch.
48//!
49//! # The two Eval arms
50//!
51//! Release 0.2.0 accepts two majors of the Eval schema
52//! ([`crate::eval::EvalSchema`]):
53//!
54//! - **2.0** (`evalhub.eval/2.0`) is a header. Runs are not part of it; they
55//!   are written one by one (`PUT /evals/{ns}/{name}/runs/{run_id}`) or in
56//!   a batch (`POST /evals/{ns}/{name}/runs:batch`). A 2.0 body that has a
57//!   `runs` key at all, whatever its value, is answered with one error,
58//!   `runs_moved` at `/runs`, *before* the structural pass, and nothing
59//!   else: every other error the body might have would be noise next to
60//!   "this is the wrong endpoint for your runs", and the producer that
61//!   sends `runs` is typically a 1.0 producer that bumped the identifier.
62//! - **1.0** (`evalhub.eval/1.0`) is the old shape with `runs[]` in the
63//!   body, validated as 0.1.x validated it, including the check that each
64//!   `runs[].calls` and `runs[].artifacts[]` names an `attachments[].path`
65//!   of the body. That check exists for the 1.0 arm only. A 1.0 body that
66//!   passes is converted into a 2.0 header and its runs by
67//!   [`crate::eval::split_v1`]; release 0.3.0 removes the arm.
68//!
69//! # Pass 2: semantic
70//!
71//! Rules the schema language cannot express. On a record:
72//!
73//! | Rule                                                                                   | Code                       |
74//! | -------------------------------------------------------------------------------------- | -------------------------- |
75//! | `results` non-empty ⇒ `counts` present                                                 | `counts_missing`           |
76//! | `counts.attempted >= completed + failed + skipped + errored`                           | `counts_inconsistent`      |
77//! | every `results[].samples_ref` is an `attachments[].path`                               | `attachment_ref_unknown`   |
78//! | Eval 1.0 only: every `runs[].calls`, `runs[].artifacts[]` is an `attachments[].path`   | `attachment_ref_unknown`   |
79//! | `attachments[].path` unique, relative, contains no `..`                                | `attachment_path_invalid`  |
80//! | `results[].metric` matches `{ns}/{name}`                                               | `metric_id_invalid`        |
81//! | `run_results[].metric` matches `{ns}/{name}`                                           | `metric_id_invalid`        |
82//! | `run_results[]` element has `value` or `label` (or both)                               | `run_result_value_or_label`|
83//! | `run_results[].eval` is the `{ns}/{name}` of a `core/uses_eval` relation's target      | `run_results_eval_unknown` |
84//! | `relations[].type` matches `{ns}/{name}`                                               | `schema`                   |
85//! | `relations[].to` is `{ns}/{name}@{seq}`, `external:…`, or `hf:…`                       | `schema`                   |
86//! | every key of `ext`, and of each facet's `ext`, matches `{ns}/{name}`                   | `ext_key_invalid`          |
87//!
88//! `run_results[].eval` is compared with the target of each of the Card's
89//! `core/uses_eval` relations *without* its `@{seq}`: runs belong to the
90//! Eval record, not to a version, so a verdict names the record. Only a
91//! hub target (`{ns}/{name}@{seq}`) can match; an `external:` or `hf:`
92//! target has no runs on this hub. Whether that target resolves, and
93//! whether the writer may see it, is not this crate's question.
94//!
95//! Three more rules about `run_results` need data this crate does not
96//! have, and are checked elsewhere:
97//!
98//! | Rule                                                                                   | Code                       | Checked by |
99//! | -------------------------------------------------------------------------------------- | -------------------------- | ---------- |
100//! | every `run_id` of a used set has a row in its Eval (archived and deleted rows count)   | `run_unknown`              | the store  |
101//! | every `run_results[].run_id` is in the used set of its `eval`                          | `run_not_in_used_set`      | the store  |
102//! | `run_results[]` is no longer than the configured limit                                 | `too_many_run_results`     | the server |
103//!
104//! On a run ([`run()`]), everything is checked against the run itself; a
105//! run never points into the header:
106//!
107//! | Rule                                                                                   | Code                       |
108//! | -------------------------------------------------------------------------------------- | -------------------------- |
109//! | the given `run_id` is non-empty, has no `/`, is at most 200 bytes                      | `run_id_invalid`           |
110//! | the body's `run_id` equals the given one                                               | `run_id_mismatch`          |
111//! | `error` present iff `status` is `error`; `skip_reason` iff `status` is `skipped`       | `run_status_detail`        |
112//! | `calls`, `artifacts[]`, `error.log` each name an `attachments[].path` of the run       | `attachment_ref_unknown`   |
113//! | the run's `attachments[].path` unique, relative, contains no `..`                      | `attachment_path_invalid`  |
114//! | every key of `metrics` matches `{ns}/{name}`                                           | `metric_id_invalid`        |
115//! | every key of `ext`, and of each facet's `ext`, matches `{ns}/{name}`                   | `ext_key_invalid`          |
116//!
117//! `ext` is treated on a run as on a record: the map itself is open, and
118//! only its keys are checked, at the top level and inside each facet that
119//! carries one (`model`, `task`, `harness`, `generation`, `trial`,
120//! `grading`, `env`), one error per key at `/ext/{key}` or
121//! `/{facet}/ext/{key}`. The values are never looked into, and neither is
122//! `meta`.
123//!
124//! The semantic pass runs on whatever is present even when the structural
125//! pass failed, so a producer sees both kinds of problem at once. Errors
126//! are sorted by `(path, code)`; the order is deterministic.
127//!
128//! # What is deliberately not an error
129//!
130//! - A `relations[].to` that does not resolve. The record is accepted; the
131//!   `refs_resolved` badge is withheld. A Card may legitimately be published
132//!   before the Eval it cites.
133//! - A `harness` or `metric` the registry does not know, including a run's
134//!   `metrics` key. Accepted; the `harness_registered` /
135//!   `metric_registered` badges are withheld. The hub does not gate on
136//!   vocabulary.
137//! - An `attachments[].sha256` that has not been uploaded. This is not a
138//!   validation failure but a state conflict, `409 attachment_missing`,
139//!   raised by the store, because only the store knows.
140//! - An Eval header with zero runs, of any `eval_kind`. A header is
141//!   complete on its own; its runs are written afterwards.
142//!
143//! # Output
144//!
145//! `Vec<ErrorEntry { path, code, hint }>`, empty on success. `path` is a
146//! JSON pointer into the submitted value (`/results/0/metric`; the root is
147//! `""`). `hint` is prose and not part of the contract.
148
149use std::collections::{HashMap, HashSet};
150use std::sync::OnceLock;
151
152use jsonschema::{Draft, Validator};
153use serde_json::{Map, Value};
154
155use evalhub_schema::RecordKind;
156use evalhub_schema::error::{ErrorCode, ErrorEntry};
157
158use crate::eval::{EvalSchema, declared_schema};
159
160/// Largest integer a JSON number may carry without losing precision as a
161/// double: 2^53.
162const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
163
164/// Longest `run_id` accepted, in bytes of UTF-8.
165pub const MAX_RUN_ID_BYTES: usize = 200;
166
167/// Where a relation points.
168#[derive(Debug, Clone, PartialEq, Eq)]
169pub enum RelationTarget {
170    /// A version on this hub, `{ns}/{name}@{seq}`.
171    Version {
172        /// Namespace of the target record.
173        ns: String,
174        /// Name of the target record.
175        name: String,
176        /// Sequence number of the target version, from 1.
177        seq: u32,
178    },
179    /// Something outside the hub, written `external:<anything>`.
180    External(String),
181    /// A Hugging Face reference, written `hf:<org>/<repo>@<sha>`.
182    Hf(String),
183}
184
185/// Whether `s` has the form `{ns}/{name}`: exactly one `/`, both parts
186/// non-empty, each starting with `[a-z0-9]` and continuing with
187/// `[a-z0-9._-]`.
188pub fn is_valid_id(s: &str) -> bool {
189    let Some((ns, name)) = s.split_once('/') else {
190        return false;
191    };
192    is_slug(ns) && is_slug(name)
193}
194
195/// Whether `s` is a valid `run_id`: non-empty, no `/`, at most
196/// [`MAX_RUN_ID_BYTES`] bytes of UTF-8. Nothing else is imposed (see
197/// `evalhub_schema::run`); every `run_id` a 0.1.x producer wrote passes.
198pub fn is_valid_run_id(s: &str) -> bool {
199    !s.is_empty() && !s.contains('/') && s.len() <= MAX_RUN_ID_BYTES
200}
201
202fn is_slug(part: &str) -> bool {
203    let mut chars = part.chars();
204    match chars.next() {
205        Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
206        _ => return false,
207    }
208    chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
209}
210
211/// Parse a `relations[].to` value. `None` when it has none of the accepted
212/// forms.
213pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
214    if let Some(rest) = s.strip_prefix("external:") {
215        return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
216    }
217    if let Some(rest) = s.strip_prefix("hf:") {
218        return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
219    }
220    let (id, seq) = s.rsplit_once('@')?;
221    let (ns, name) = id.split_once('/')?;
222    if !is_slug(ns) || !is_slug(name) {
223        return None;
224    }
225    // `parse::<u32>` accepts a leading `+`; the address form does not.
226    if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
227        return None;
228    }
229    let seq: u32 = seq.parse().ok()?;
230    if seq == 0 {
231        return None;
232    }
233    Some(RelationTarget::Version {
234        ns: ns.to_string(),
235        name: name.to_string(),
236        seq,
237    })
238}
239
240/// Validate a record of `kind` and return every failure, sorted by
241/// `(path, code)`; empty when the record is accepted.
242///
243/// The JSON Schema is the one for the identifier the body declares in
244/// `schema` (see the module doc). An `evalhub.eval/2.0` body with a `runs`
245/// key returns exactly one error, `runs_moved` at `/runs`, and nothing
246/// else. Cost: one pass of the compiled validator plus one walk of the
247/// value; the validator is compiled on the first call for its identifier.
248pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
249    let eval_schema = match kind {
250        RecordKind::Eval => declared_schema(value),
251        RecordKind::Card => None,
252    };
253    if eval_schema == Some(EvalSchema::V2) && value.get("runs").is_some() {
254        return vec![runs_moved()];
255    }
256    let declared = value.get("schema").and_then(Value::as_str);
257    let mut errors = Vec::new();
258    structural(record_validator(kind, declared), value, &mut errors);
259    numbers("", value, &mut errors);
260    semantic(kind, eval_schema, value, &mut errors);
261    sort(&mut errors);
262    errors
263}
264
265/// Validate one run of an Eval, written under `run_id` (the path segment
266/// of `PUT /evals/{ns}/{name}/runs/{run_id}`, or the element's own
267/// `run_id` in a batch). Returns every failure, sorted by `(path, code)`;
268/// empty when the run is accepted.
269///
270/// Both passes run: `schemas/run.json`, the ±2^53 check, then the run
271/// rules in the module doc. `run_id_invalid` and `run_id_mismatch` are
272/// reported at `/run_id`. The run's pointers are checked against its own
273/// `attachments[]` only. Cost: as for [`validate`].
274pub fn run(run_id: &str, value: &Value) -> Vec<ErrorEntry> {
275    let mut errors = Vec::new();
276    structural(run_validator(), value, &mut errors);
277    numbers("", value, &mut errors);
278    run_semantic(run_id, value, &mut errors);
279    sort(&mut errors);
280    errors
281}
282
283/// The one error a 2.0 header carrying `runs` gets.
284fn runs_moved() -> ErrorEntry {
285    ErrorEntry {
286        path: "/runs".to_string(),
287        code: ErrorCode::RunsMoved,
288        hint: Some(
289            "an evalhub.eval/2.0 header does not carry runs; write each run with \
290             PUT /evals/{ns}/{name}/runs/{run_id}, or several with \
291             POST /evals/{ns}/{name}/runs:batch"
292                .to_string(),
293        ),
294    }
295}
296
297fn sort(errors: &mut [ErrorEntry]) {
298    // `ErrorCode` is a plain enum; its declaration order is the tie-break.
299    errors.sort_by(|a, b| {
300        a.path
301            .cmp(&b.path)
302            .then_with(|| (a.code as u8).cmp(&(b.code as u8)))
303    });
304}
305
306/// Compile a generated schema. The generated schemas only reference
307/// `#/$defs/*`; `offline()` turns any other reference into a build failure
308/// here, which the unit tests exercise, rather than a fetch in production.
309fn compile(schema: &Value) -> Validator {
310    #[allow(clippy::expect_used)]
311    jsonschema::options()
312        .with_draft(Draft::Draft202012)
313        .offline()
314        .build(schema)
315        .expect("the generated schema compiles")
316}
317
318/// One compiled validator per accepted identifier of every kind, keyed by
319/// the identifier. Two Card identifiers share a document and are compiled
320/// twice; that is two compilations per process, and it keeps the cache a
321/// plain map from what the body says to what checks it.
322fn record_validators() -> &'static HashMap<&'static str, Validator> {
323    static CACHE: OnceLock<HashMap<&'static str, Validator>> = OnceLock::new();
324    CACHE.get_or_init(|| {
325        let mut map = HashMap::new();
326        for kind in [RecordKind::Card, RecordKind::Eval] {
327            for &id in kind.schema_ids() {
328                if let Some(schema) = kind.schema_for(id) {
329                    map.insert(id, compile(schema.as_value()));
330                }
331            }
332        }
333        map
334    })
335}
336
337/// The validator for a record of `kind` declaring `declared`: the one for
338/// that identifier when `kind` accepts it, otherwise the kind's current one.
339fn record_validator(kind: RecordKind, declared: Option<&str>) -> &'static Validator {
340    let cache = record_validators();
341    let accepted = declared.filter(|id| kind.schema_ids().contains(id));
342    let id = accepted.unwrap_or(kind.schema_id());
343    // Every accepted identifier has a document (a unit test asserts it);
344    // a miss here is a bug in `evalhub_schema`, not a property of the input.
345    #[allow(clippy::expect_used)]
346    cache
347        .get(id)
348        .expect("every accepted schema identifier has a document")
349}
350
351fn run_validator() -> &'static Validator {
352    static RUN: OnceLock<Validator> = OnceLock::new();
353    RUN.get_or_init(|| compile(evalhub_schema::schema_for_run().as_value()))
354}
355
356fn structural(validator: &Validator, value: &Value, out: &mut Vec<ErrorEntry>) {
357    for e in validator.iter_errors(value) {
358        out.push(ErrorEntry {
359            path: e.instance_path().as_str().to_string(),
360            code: ErrorCode::Schema,
361            hint: Some(e.to_string()),
362        });
363    }
364}
365
366fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
367    match value {
368        Value::Number(n) => {
369            let too_large = if let Some(i) = n.as_i64() {
370                i.unsigned_abs() > MAX_SAFE_INTEGER as u64
371            } else if let Some(u) = n.as_u64() {
372                u > MAX_SAFE_INTEGER as u64
373            } else if let Some(f) = n.as_f64() {
374                f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
375            } else {
376                false
377            };
378            if too_large {
379                out.push(ErrorEntry {
380                    path: path.to_string(),
381                    code: ErrorCode::NumberTooLarge,
382                    hint: Some(format!(
383                        "{n} is an integer outside ±2^53; send it as a string"
384                    )),
385                });
386            }
387        }
388        Value::Array(items) => {
389            for (i, item) in items.iter().enumerate() {
390                numbers(&format!("{path}/{i}"), item, out);
391            }
392        }
393        Value::Object(map) => {
394            for (k, v) in map {
395                numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
396            }
397        }
398        _ => {}
399    }
400}
401
402/// RFC 6901 escaping for one reference token.
403fn escape_pointer(key: &str) -> String {
404    key.replace('~', "~0").replace('/', "~1")
405}
406
407fn push(out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String) {
408    out.push(ErrorEntry {
409        path,
410        code,
411        hint: Some(hint),
412    });
413}
414
415/// The `attachments[].path` set of one object (a record or a run), after
416/// reporting every path that is empty, absolute, contains `..`, or repeats
417/// an earlier one. The set holds the first occurrence of each valid path.
418fn attachment_paths<'a>(
419    obj: &'a Map<String, Value>,
420    out: &mut Vec<ErrorEntry>,
421) -> HashSet<&'a str> {
422    let mut paths: HashSet<&str> = HashSet::new();
423    let Some(attachments) = obj.get("attachments").and_then(Value::as_array) else {
424        return paths;
425    };
426    for (i, a) in attachments.iter().enumerate() {
427        let Some(p) = a.get("path").and_then(Value::as_str) else {
428            continue;
429        };
430        let invalid = if p.is_empty() {
431            Some("empty")
432        } else if p.starts_with('/') {
433            Some("absolute")
434        } else if p.split('/').any(|seg| seg == "..") {
435            Some("contains `..`")
436        } else if !paths.insert(p) {
437            Some("duplicate")
438        } else {
439            None
440        };
441        if let Some(why) = invalid {
442            push(
443                out,
444                format!("/attachments/{i}/path"),
445                ErrorCode::AttachmentPathInvalid,
446                format!("attachment path {p:?} is {why}"),
447            );
448        }
449    }
450    paths
451}
452
453/// `attachment_ref_unknown` at `pointer` unless `p` is in `paths`.
454fn ref_known(paths: &HashSet<&str>, out: &mut Vec<ErrorEntry>, pointer: String, p: &str) {
455    if !paths.contains(p) {
456        push(
457            out,
458            pointer,
459            ErrorCode::AttachmentRefUnknown,
460            format!("{p:?} names no attachments[].path"),
461        );
462    }
463}
464
465/// `metric_id_invalid` at `pointer` unless `m` is `{ns}/{name}`.
466fn metric_id(out: &mut Vec<ErrorEntry>, pointer: String, m: &str) {
467    if !is_valid_id(m) {
468        push(
469            out,
470            pointer,
471            ErrorCode::MetricIdInvalid,
472            format!("{m:?} is not of the form {{ns}}/{{name}}"),
473        );
474    }
475}
476
477/// The facets of a record or a run that carry an `ext` map of their own.
478/// `grading` exists on a Card only; on an Eval or a run it is simply absent.
479const FACETS_WITH_EXT: [&str; 7] = [
480    "model",
481    "task",
482    "harness",
483    "generation",
484    "trial",
485    "grading",
486    "env",
487];
488
489/// `ext_key_invalid` at `{prefix}/ext/{key}` for every key of `obj.ext`
490/// that is not `{ns}/{name}`. `prefix` is the pointer of `obj` itself
491/// (`""` for the body, `"/model"` for a facet).
492fn ext_keys(out: &mut Vec<ErrorEntry>, prefix: &str, obj: &Map<String, Value>) {
493    let Some(ext) = obj.get("ext").and_then(Value::as_object) else {
494        return;
495    };
496    for key in ext.keys() {
497        if !is_valid_id(key) {
498            push(
499                out,
500                format!("{prefix}/ext/{}", escape_pointer(key)),
501                ErrorCode::ExtKeyInvalid,
502                format!("ext key {key:?} is not of the form {{ns}}/{{name}}"),
503            );
504        }
505    }
506}
507
508/// [`ext_keys`] on the body and on each of its [`FACETS_WITH_EXT`].
509fn ext_keys_everywhere(out: &mut Vec<ErrorEntry>, obj: &Map<String, Value>) {
510    ext_keys(out, "", obj);
511    for facet in FACETS_WITH_EXT {
512        if let Some(f) = obj.get(facet).and_then(Value::as_object) {
513            ext_keys(out, &format!("/{facet}"), f);
514        }
515    }
516}
517
518/// Whether a key is absent or `null`: both mean "not given" for the
519/// pairings checked here.
520fn given(obj: &Map<String, Value>, key: &str) -> bool {
521    obj.get(key).is_some_and(|v| !v.is_null())
522}
523
524fn semantic(
525    kind: RecordKind,
526    eval_schema: Option<EvalSchema>,
527    value: &Value,
528    out: &mut Vec<ErrorEntry>,
529) {
530    let Some(obj) = value.as_object() else {
531        return;
532    };
533
534    let paths = attachment_paths(obj, out);
535    ext_keys_everywhere(out, obj);
536
537    if kind == RecordKind::Card {
538        let results = obj.get("results").and_then(Value::as_array);
539        if let Some(results) = results {
540            for (i, r) in results.iter().enumerate() {
541                if let Some(m) = r.get("metric").and_then(Value::as_str) {
542                    metric_id(out, format!("/results/{i}/metric"), m);
543                }
544                if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
545                    ref_known(&paths, out, format!("/results/{i}/samples_ref"), s);
546                }
547            }
548            let counts = obj.get("counts").filter(|c| !c.is_null());
549            match counts {
550                None if !results.is_empty() => push(
551                    out,
552                    "/counts".to_string(),
553                    ErrorCode::CountsMissing,
554                    "results are present, so counts is required".to_string(),
555                ),
556                Some(c) => {
557                    let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
558                    let sum = n("completed") + n("failed") + n("skipped") + n("errored");
559                    if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
560                        && attempted < sum
561                    {
562                        push(
563                            out,
564                            "/counts".to_string(),
565                            ErrorCode::CountsInconsistent,
566                            format!(
567                                "attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
568                            ),
569                        );
570                    }
571                }
572                None => {}
573            }
574        }
575        run_results(obj, out);
576    }
577
578    // The 1.0 arm only: runs in the body point into the body's attachments.
579    if eval_schema == Some(EvalSchema::V1)
580        && let Some(runs) = obj.get("runs").and_then(Value::as_array)
581    {
582        for (i, run) in runs.iter().enumerate() {
583            if let Some(c) = run.get("calls").and_then(Value::as_str) {
584                ref_known(&paths, out, format!("/runs/{i}/calls"), c);
585            }
586            if let Some(artifacts) = run.get("artifacts").and_then(Value::as_array) {
587                for (j, a) in artifacts.iter().enumerate() {
588                    if let Some(a) = a.as_str() {
589                        ref_known(&paths, out, format!("/runs/{i}/artifacts/{j}"), a);
590                    }
591                }
592            }
593        }
594    }
595
596    if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
597        for (i, rel) in relations.iter().enumerate() {
598            if let Some(t) = rel.get("type").and_then(Value::as_str)
599                && !is_valid_id(t)
600            {
601                push(
602                    out,
603                    format!("/relations/{i}/type"),
604                    ErrorCode::Schema,
605                    format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
606                );
607            }
608            if let Some(to) = rel.get("to").and_then(Value::as_str)
609                && parse_relation_target(to).is_none()
610            {
611                push(
612                    out,
613                    format!("/relations/{i}/to"),
614                    ErrorCode::Schema,
615                    format!(
616                        "{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
617                    ),
618                );
619            }
620        }
621    }
622}
623
624/// The `{ns}/{name}` of every hub target of the Card's `core/uses_eval`
625/// relations, without the `@{seq}`.
626fn used_eval_records(obj: &Map<String, Value>) -> HashSet<String> {
627    let Some(relations) = obj.get("relations").and_then(Value::as_array) else {
628        return HashSet::new();
629    };
630    relations
631        .iter()
632        .filter(|rel| rel.get("type").and_then(Value::as_str) == Some("core/uses_eval"))
633        .filter_map(|rel| rel.get("to").and_then(Value::as_str))
634        .filter_map(|to| match parse_relation_target(to)? {
635            RelationTarget::Version { ns, name, .. } => Some(format!("{ns}/{name}")),
636            RelationTarget::External(_) | RelationTarget::Hf(_) => None,
637        })
638        .collect()
639}
640
641/// The Card's `run_results[]` rules that need nothing but the Card.
642fn run_results(obj: &Map<String, Value>, out: &mut Vec<ErrorEntry>) {
643    let Some(items) = obj.get("run_results").and_then(Value::as_array) else {
644        return;
645    };
646    let evals = used_eval_records(obj);
647    for (i, item) in items.iter().enumerate() {
648        let Some(item) = item.as_object() else {
649            continue;
650        };
651        if let Some(m) = item.get("metric").and_then(Value::as_str) {
652            metric_id(out, format!("/run_results/{i}/metric"), m);
653        }
654        if !given(item, "value") && !given(item, "label") {
655            push(
656                out,
657                format!("/run_results/{i}"),
658                ErrorCode::RunResultValueOrLabel,
659                "a run result needs a value, a label, or both".to_string(),
660            );
661        }
662        if let Some(e) = item.get("eval").and_then(Value::as_str)
663            && !evals.contains(e)
664        {
665            push(
666                out,
667                format!("/run_results/{i}/eval"),
668                ErrorCode::RunResultsEvalUnknown,
669                format!(
670                    "{e:?} is not the {{ns}}/{{name}} (without @seq) of any core/uses_eval target of this Card"
671                ),
672            );
673        }
674    }
675}
676
677fn run_semantic(run_id: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
678    if !is_valid_run_id(run_id) {
679        push(
680            out,
681            "/run_id".to_string(),
682            ErrorCode::RunIdInvalid,
683            format!(
684                "run_id {run_id:?} must be non-empty, contain no `/`, and be at most {MAX_RUN_ID_BYTES} bytes"
685            ),
686        );
687    }
688    let Some(obj) = value.as_object() else {
689        return;
690    };
691    if let Some(body_id) = obj.get("run_id").and_then(Value::as_str)
692        && body_id != run_id
693    {
694        push(
695            out,
696            "/run_id".to_string(),
697            ErrorCode::RunIdMismatch,
698            format!("the body says run_id {body_id:?}, but it is written as {run_id:?}"),
699        );
700    }
701
702    // status ⇔ detail. Only a recognised status is paired; an unknown one
703    // is already a `schema` error.
704    let status = obj.get("status").and_then(Value::as_str);
705    let expects = match status {
706        Some("ok") => Some((false, false)),
707        Some("error") => Some((true, false)),
708        Some("skipped") => Some((false, true)),
709        _ => None,
710    };
711    if let (Some(status), Some((wants_error, wants_skip))) = (status, expects) {
712        for (key, wanted) in [("error", wants_error), ("skip_reason", wants_skip)] {
713            let has = given(obj, key);
714            if has != wanted {
715                let hint = if wanted {
716                    format!("status {status:?} requires {key}")
717                } else {
718                    format!(
719                        "{key} is only allowed when status is {:?}",
720                        if key == "error" { "error" } else { "skipped" }
721                    )
722                };
723                push(out, format!("/{key}"), ErrorCode::RunStatusDetail, hint);
724            }
725        }
726    }
727
728    let paths = attachment_paths(obj, out);
729    if let Some(c) = obj.get("calls").and_then(Value::as_str) {
730        ref_known(&paths, out, "/calls".to_string(), c);
731    }
732    if let Some(artifacts) = obj.get("artifacts").and_then(Value::as_array) {
733        for (j, a) in artifacts.iter().enumerate() {
734            if let Some(a) = a.as_str() {
735                ref_known(&paths, out, format!("/artifacts/{j}"), a);
736            }
737        }
738    }
739    if let Some(log) = obj
740        .get("error")
741        .and_then(|e| e.get("log"))
742        .and_then(Value::as_str)
743    {
744        ref_known(&paths, out, "/error/log".to_string(), log);
745    }
746
747    if let Some(metrics) = obj.get("metrics").and_then(Value::as_object) {
748        for key in metrics.keys() {
749            metric_id(out, format!("/metrics/{}", escape_pointer(key)), key);
750        }
751    }
752    ext_keys_everywhere(out, obj);
753}
754
755#[cfg(test)]
756mod tests {
757    #![allow(clippy::unwrap_used)]
758
759    use super::*;
760
761    #[test]
762    fn ids() {
763        assert!(is_valid_id("core/pass_rate"));
764        assert!(is_valid_id("alice/qwen-loop.v2"));
765        assert!(!is_valid_id("pass_rate"));
766        assert!(!is_valid_id("Core/pass_rate"));
767        assert!(!is_valid_id("core/"));
768        assert!(!is_valid_id("/x"));
769        assert!(!is_valid_id("a/b/c"));
770        assert!(!is_valid_id("-a/b"));
771    }
772
773    #[test]
774    fn run_ids() {
775        assert!(is_valid_run_id("r1"));
776        assert!(is_valid_run_id("task 7, trial 2"));
777        assert!(is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES)));
778        assert!(!is_valid_run_id(""));
779        assert!(!is_valid_run_id("a/b"));
780        assert!(!is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES + 1)));
781        // Bytes, not characters: 67 three-byte characters are 201 bytes.
782        assert!(!is_valid_run_id(&"あ".repeat(67)));
783        assert!(is_valid_run_id(&"あ".repeat(66)));
784    }
785
786    #[test]
787    fn relation_targets() {
788        assert_eq!(
789            parse_relation_target("alice/single2-k4@3"),
790            Some(RelationTarget::Version {
791                ns: "alice".into(),
792                name: "single2-k4".into(),
793                seq: 3
794            })
795        );
796        assert_eq!(
797            parse_relation_target("external:https://x/y"),
798            Some(RelationTarget::External("https://x/y".into()))
799        );
800        assert_eq!(
801            parse_relation_target("hf:org/repo@sha"),
802            Some(RelationTarget::Hf("org/repo@sha".into()))
803        );
804        for bad in [
805            "alice/x",
806            "alice/x@0",
807            "alice/x@+1",
808            "alice/x@label",
809            "alice@3",
810            "external:",
811            "hf:",
812            "",
813        ] {
814            assert_eq!(parse_relation_target(bad), None, "{bad}");
815        }
816    }
817
818    #[test]
819    fn validators_build_offline_one_per_accepted_identifier() {
820        let cache = record_validators();
821        for kind in [RecordKind::Card, RecordKind::Eval] {
822            for id in kind.schema_ids() {
823                assert!(cache.contains_key(id), "{id}");
824            }
825        }
826        assert_eq!(
827            cache.len(),
828            RecordKind::Card.schema_ids().len() + RecordKind::Eval.schema_ids().len()
829        );
830        let _ = run_validator();
831    }
832
833    #[test]
834    fn pointer_escaping() {
835        let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
836        let mut out = Vec::new();
837        numbers("", &v, &mut out);
838        assert_eq!(out.len(), 1);
839        assert_eq!(out[0].path, "/ext/a~1b/n");
840        assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
841    }
842}