Skip to main content

evalhub_core/
validate.rs

1//! Validation: everything the hub checks before it stores a record or a
2//! run.
3//!
4//! Validation runs in two passes and collects every failure from both into
5//! one list, so a producer sees all problems in one round trip. There are
6//! two entry points:
7//!
8//! - [`validate`] for a record (a Card, or an Eval header), keyed by
9//!   [`RecordKind`];
10//! - [`run()`] for one run of an Eval, given the `run_id` it is written
11//!   under.
12//!
13//! A run is not a [`RecordKind`]. `RecordKind` is the two kinds that appear
14//! in URLs and in the store's `type` column, and every match on it is
15//! exhaustive; a run is a row of an Eval record, addressed under it, so it
16//! has its own entry point rather than a third variant that every one of
17//! those matches would have to reject.
18//!
19//! # Pass 1: structural
20//!
21//! The value is checked against a JSON Schema using `jsonschema` with
22//! draft 2020-12. The core is closed, so an unknown key anywhere outside
23//! `ext` is a `schema` error. In the same pass every number is inspected:
24//! an integer outside ±2^53 is `number_too_large`.
25//!
26//! Which schema: a record is checked against the document for the
27//! identifier its own `schema` key declares, through
28//! [`RecordKind::schema_for`]:
29//!
30//! | Declared `schema`                      | Document               |
31//! | -------------------------------------- | ---------------------- |
32//! | `evalhub.card/1.1`, `evalhub.card/1.0` | `schemas/card.json`    |
33//! | `evalhub.eval/2.0`                     | `schemas/eval-2.json`  |
34//! | `evalhub.eval/1.0`                     | `schemas/eval.json`    |
35//! | anything else, or no `schema` key      | the kind's current one |
36//!
37//! The last row is what makes an unknown identifier an error: the current
38//! document's `schema` key is a constant, so the value fails it at
39//! `/schema` (`schema`), exactly as it did before a kind had two
40//! identifiers. A run is checked against `schemas/run.json`.
41//!
42//! The schemas are the generated ones from `evalhub_schema` (the same
43//! generator output that is committed under `schemas/`). Each is compiled
44//! once per process, one validator per accepted identifier plus one for
45//! the run, on first use. The validators are built in offline mode: every
46//! `$ref` in the schema is local (`#/$defs/…`), and a reference that would
47//! need fetching is a build error, not a fetch.
48//!
49//! # The two Eval arms
50//!
51//! Release 0.2.0 accepts two majors of the Eval schema
52//! ([`crate::eval::EvalSchema`]):
53//!
54//! - **2.0** (`evalhub.eval/2.0`) is a header. Runs are not part of it; they
55//!   are written one by one (`PUT /evals/{ns}/{name}/runs/{run_id}`) or in
56//!   a batch (`POST /evals/{ns}/{name}/runs:batch`). A 2.0 body that has a
57//!   `runs` key at all, whatever its value, is answered with one error,
58//!   `runs_moved` at `/runs`, *before* the structural pass, and nothing
59//!   else: every other error the body might have would be noise next to
60//!   "this is the wrong endpoint for your runs", and the producer that
61//!   sends `runs` is typically a 1.0 producer that bumped the identifier.
62//! - **1.0** (`evalhub.eval/1.0`) is the old shape with `runs[]` in the
63//!   body, validated as 0.1.x validated it, including the check that each
64//!   `runs[].calls` and `runs[].artifacts[]` names an `attachments[].path`
65//!   of the body. That check exists for the 1.0 arm only. A 1.0 body that
66//!   passes is converted into a 2.0 header and its runs by
67//!   [`crate::eval::split_v1`]; release 0.3.0 removes the arm.
68//!
69//! # Pass 2: semantic
70//!
71//! Rules the schema language cannot express. On a record:
72//!
73//! | Rule                                                                                   | Code                       |
74//! | -------------------------------------------------------------------------------------- | -------------------------- |
75//! | `results` non-empty ⇒ `counts` present                                                 | `counts_missing`           |
76//! | `counts.attempted >= completed + failed + skipped + errored`                           | `counts_inconsistent`      |
77//! | every `results[].samples_ref` is an `attachments[].path`                               | `attachment_ref_unknown`   |
78//! | Eval 1.0 only: every `runs[].calls`, `runs[].artifacts[]` is an `attachments[].path`   | `attachment_ref_unknown`   |
79//! | `attachments[].path` unique, relative, contains no `..`                                | `attachment_path_invalid`  |
80//! | `results[].metric` matches `{ns}/{name}`                                               | `metric_id_invalid`        |
81//! | `run_results[].metric` matches `{ns}/{name}`                                           | `metric_id_invalid`        |
82//! | `run_results[]` element has `value` or `label` (or both)                               | `run_result_value_or_label`|
83//! | `run_results[].eval` is the `{ns}/{name}` of a `core/uses_eval` relation's target      | `run_results_eval_unknown` |
84//! | `relations[].type` matches `{ns}/{name}`                                               | `schema`                   |
85//! | `relations[].to` is `{ns}/{name}@{seq}`, `external:…`, or `hf:…`                       | `schema`                   |
86//!
87//! `run_results[].eval` is compared with the target of each of the Card's
88//! `core/uses_eval` relations *without* its `@{seq}`: runs belong to the
89//! Eval record, not to a version, so a verdict names the record. Only a
90//! hub target (`{ns}/{name}@{seq}`) can match; an `external:` or `hf:`
91//! target has no runs on this hub. Whether that target resolves, and
92//! whether the writer may see it, is not this crate's question.
93//!
94//! Three more rules about `run_results` need data this crate does not
95//! have, and are checked elsewhere:
96//!
97//! | Rule                                                                                   | Code                       | Checked by |
98//! | -------------------------------------------------------------------------------------- | -------------------------- | ---------- |
99//! | every `run_id` of a used set has a row in its Eval (archived and deleted rows count)   | `run_unknown`              | the store  |
100//! | every `run_results[].run_id` is in the used set of its `eval`                          | `run_not_in_used_set`      | the store  |
101//! | `run_results[]` is no longer than the configured limit                                 | `too_many_run_results`     | the server |
102//!
103//! On a run ([`run()`]), everything is checked against the run itself; a
104//! run never points into the header:
105//!
106//! | Rule                                                                                   | Code                       |
107//! | -------------------------------------------------------------------------------------- | -------------------------- |
108//! | the given `run_id` is non-empty, has no `/`, is at most 200 bytes                      | `run_id_invalid`           |
109//! | the body's `run_id` equals the given one                                               | `run_id_mismatch`          |
110//! | `error` present iff `status` is `error`; `skip_reason` iff `status` is `skipped`       | `run_status_detail`        |
111//! | `calls`, `artifacts[]`, `error.log` each name an `attachments[].path` of the run       | `attachment_ref_unknown`   |
112//! | the run's `attachments[].path` unique, relative, contains no `..`                      | `attachment_path_invalid`  |
113//! | every key of `metrics` matches `{ns}/{name}`                                           | `metric_id_invalid`        |
114//!
115//! `ext` is treated on a run as on a record: an object whose keys are
116//! meant to be namespaces, accepted as the JSON Schema describes it, with
117//! no semantic rule of its own in this crate today. `meta` is never looked
118//! into.
119//!
120//! The semantic pass runs on whatever is present even when the structural
121//! pass failed, so a producer sees both kinds of problem at once. Errors
122//! are sorted by `(path, code)`; the order is deterministic.
123//!
124//! # What is deliberately not an error
125//!
126//! - A `relations[].to` that does not resolve. The record is accepted; the
127//!   `refs_resolved` badge is withheld. A Card may legitimately be published
128//!   before the Eval it cites.
129//! - A `harness` or `metric` the registry does not know, including a run's
130//!   `metrics` key. Accepted; the `harness_registered` /
131//!   `metric_registered` badges are withheld. The hub does not gate on
132//!   vocabulary.
133//! - An `attachments[].sha256` that has not been uploaded. This is not a
134//!   validation failure but a state conflict, `409 attachment_missing`,
135//!   raised by the store, because only the store knows.
136//! - An Eval header with zero runs, of any `eval_kind`. A header is
137//!   complete on its own; its runs are written afterwards.
138//!
139//! # Output
140//!
141//! `Vec<ErrorEntry { path, code, hint }>`, empty on success. `path` is a
142//! JSON pointer into the submitted value (`/results/0/metric`; the root is
143//! `""`). `hint` is prose and not part of the contract.
144
145use std::collections::{HashMap, HashSet};
146use std::sync::OnceLock;
147
148use jsonschema::{Draft, Validator};
149use serde_json::{Map, Value};
150
151use evalhub_schema::RecordKind;
152use evalhub_schema::error::{ErrorCode, ErrorEntry};
153
154use crate::eval::{EvalSchema, declared_schema};
155
156/// Largest integer a JSON number may carry without losing precision as a
157/// double: 2^53.
158const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
159
160/// Longest `run_id` accepted, in bytes of UTF-8.
161pub const MAX_RUN_ID_BYTES: usize = 200;
162
163/// Where a relation points.
164#[derive(Debug, Clone, PartialEq, Eq)]
165pub enum RelationTarget {
166    /// A version on this hub, `{ns}/{name}@{seq}`.
167    Version {
168        /// Namespace of the target record.
169        ns: String,
170        /// Name of the target record.
171        name: String,
172        /// Sequence number of the target version, from 1.
173        seq: u32,
174    },
175    /// Something outside the hub, written `external:<anything>`.
176    External(String),
177    /// A Hugging Face reference, written `hf:<org>/<repo>@<sha>`.
178    Hf(String),
179}
180
181/// Whether `s` has the form `{ns}/{name}`: exactly one `/`, both parts
182/// non-empty, each starting with `[a-z0-9]` and continuing with
183/// `[a-z0-9._-]`.
184pub fn is_valid_id(s: &str) -> bool {
185    let Some((ns, name)) = s.split_once('/') else {
186        return false;
187    };
188    is_slug(ns) && is_slug(name)
189}
190
191/// Whether `s` is a valid `run_id`: non-empty, no `/`, at most
192/// [`MAX_RUN_ID_BYTES`] bytes of UTF-8. Nothing else is imposed (see
193/// `evalhub_schema::run`); every `run_id` a 0.1.x producer wrote passes.
194pub fn is_valid_run_id(s: &str) -> bool {
195    !s.is_empty() && !s.contains('/') && s.len() <= MAX_RUN_ID_BYTES
196}
197
198fn is_slug(part: &str) -> bool {
199    let mut chars = part.chars();
200    match chars.next() {
201        Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
202        _ => return false,
203    }
204    chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
205}
206
207/// Parse a `relations[].to` value. `None` when it has none of the accepted
208/// forms.
209pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
210    if let Some(rest) = s.strip_prefix("external:") {
211        return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
212    }
213    if let Some(rest) = s.strip_prefix("hf:") {
214        return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
215    }
216    let (id, seq) = s.rsplit_once('@')?;
217    let (ns, name) = id.split_once('/')?;
218    if !is_slug(ns) || !is_slug(name) {
219        return None;
220    }
221    // `parse::<u32>` accepts a leading `+`; the address form does not.
222    if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
223        return None;
224    }
225    let seq: u32 = seq.parse().ok()?;
226    if seq == 0 {
227        return None;
228    }
229    Some(RelationTarget::Version {
230        ns: ns.to_string(),
231        name: name.to_string(),
232        seq,
233    })
234}
235
236/// Validate a record of `kind` and return every failure, sorted by
237/// `(path, code)`; empty when the record is accepted.
238///
239/// The JSON Schema is the one for the identifier the body declares in
240/// `schema` (see the module doc). An `evalhub.eval/2.0` body with a `runs`
241/// key returns exactly one error, `runs_moved` at `/runs`, and nothing
242/// else. Cost: one pass of the compiled validator plus one walk of the
243/// value; the validator is compiled on the first call for its identifier.
244pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
245    let eval_schema = match kind {
246        RecordKind::Eval => declared_schema(value),
247        RecordKind::Card => None,
248    };
249    if eval_schema == Some(EvalSchema::V2) && value.get("runs").is_some() {
250        return vec![runs_moved()];
251    }
252    let declared = value.get("schema").and_then(Value::as_str);
253    let mut errors = Vec::new();
254    structural(record_validator(kind, declared), value, &mut errors);
255    numbers("", value, &mut errors);
256    semantic(kind, eval_schema, value, &mut errors);
257    sort(&mut errors);
258    errors
259}
260
261/// Validate one run of an Eval, written under `run_id` (the path segment
262/// of `PUT /evals/{ns}/{name}/runs/{run_id}`, or the element's own
263/// `run_id` in a batch). Returns every failure, sorted by `(path, code)`;
264/// empty when the run is accepted.
265///
266/// Both passes run: `schemas/run.json`, the ±2^53 check, then the run
267/// rules in the module doc. `run_id_invalid` and `run_id_mismatch` are
268/// reported at `/run_id`. The run's pointers are checked against its own
269/// `attachments[]` only. Cost: as for [`validate`].
270pub fn run(run_id: &str, value: &Value) -> Vec<ErrorEntry> {
271    let mut errors = Vec::new();
272    structural(run_validator(), value, &mut errors);
273    numbers("", value, &mut errors);
274    run_semantic(run_id, value, &mut errors);
275    sort(&mut errors);
276    errors
277}
278
279/// The one error a 2.0 header carrying `runs` gets.
280fn runs_moved() -> ErrorEntry {
281    ErrorEntry {
282        path: "/runs".to_string(),
283        code: ErrorCode::RunsMoved,
284        hint: Some(
285            "an evalhub.eval/2.0 header does not carry runs; write each run with \
286             PUT /evals/{ns}/{name}/runs/{run_id}, or several with \
287             POST /evals/{ns}/{name}/runs:batch"
288                .to_string(),
289        ),
290    }
291}
292
293fn sort(errors: &mut [ErrorEntry]) {
294    // `ErrorCode` is a plain enum; its declaration order is the tie-break.
295    errors.sort_by(|a, b| {
296        a.path
297            .cmp(&b.path)
298            .then_with(|| (a.code as u8).cmp(&(b.code as u8)))
299    });
300}
301
302/// Compile a generated schema. The generated schemas only reference
303/// `#/$defs/*`; `offline()` turns any other reference into a build failure
304/// here, which the unit tests exercise, rather than a fetch in production.
305fn compile(schema: &Value) -> Validator {
306    #[allow(clippy::expect_used)]
307    jsonschema::options()
308        .with_draft(Draft::Draft202012)
309        .offline()
310        .build(schema)
311        .expect("the generated schema compiles")
312}
313
314/// One compiled validator per accepted identifier of every kind, keyed by
315/// the identifier. Two Card identifiers share a document and are compiled
316/// twice; that is two compilations per process, and it keeps the cache a
317/// plain map from what the body says to what checks it.
318fn record_validators() -> &'static HashMap<&'static str, Validator> {
319    static CACHE: OnceLock<HashMap<&'static str, Validator>> = OnceLock::new();
320    CACHE.get_or_init(|| {
321        let mut map = HashMap::new();
322        for kind in [RecordKind::Card, RecordKind::Eval] {
323            for &id in kind.schema_ids() {
324                if let Some(schema) = kind.schema_for(id) {
325                    map.insert(id, compile(schema.as_value()));
326                }
327            }
328        }
329        map
330    })
331}
332
333/// The validator for a record of `kind` declaring `declared`: the one for
334/// that identifier when `kind` accepts it, otherwise the kind's current one.
335fn record_validator(kind: RecordKind, declared: Option<&str>) -> &'static Validator {
336    let cache = record_validators();
337    let accepted = declared.filter(|id| kind.schema_ids().contains(id));
338    let id = accepted.unwrap_or(kind.schema_id());
339    // Every accepted identifier has a document (a unit test asserts it);
340    // a miss here is a bug in `evalhub_schema`, not a property of the input.
341    #[allow(clippy::expect_used)]
342    cache
343        .get(id)
344        .expect("every accepted schema identifier has a document")
345}
346
347fn run_validator() -> &'static Validator {
348    static RUN: OnceLock<Validator> = OnceLock::new();
349    RUN.get_or_init(|| compile(evalhub_schema::schema_for_run().as_value()))
350}
351
352fn structural(validator: &Validator, value: &Value, out: &mut Vec<ErrorEntry>) {
353    for e in validator.iter_errors(value) {
354        out.push(ErrorEntry {
355            path: e.instance_path().as_str().to_string(),
356            code: ErrorCode::Schema,
357            hint: Some(e.to_string()),
358        });
359    }
360}
361
362fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
363    match value {
364        Value::Number(n) => {
365            let too_large = if let Some(i) = n.as_i64() {
366                i.unsigned_abs() > MAX_SAFE_INTEGER as u64
367            } else if let Some(u) = n.as_u64() {
368                u > MAX_SAFE_INTEGER as u64
369            } else if let Some(f) = n.as_f64() {
370                f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
371            } else {
372                false
373            };
374            if too_large {
375                out.push(ErrorEntry {
376                    path: path.to_string(),
377                    code: ErrorCode::NumberTooLarge,
378                    hint: Some(format!(
379                        "{n} is an integer outside ±2^53; send it as a string"
380                    )),
381                });
382            }
383        }
384        Value::Array(items) => {
385            for (i, item) in items.iter().enumerate() {
386                numbers(&format!("{path}/{i}"), item, out);
387            }
388        }
389        Value::Object(map) => {
390            for (k, v) in map {
391                numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
392            }
393        }
394        _ => {}
395    }
396}
397
398/// RFC 6901 escaping for one reference token.
399fn escape_pointer(key: &str) -> String {
400    key.replace('~', "~0").replace('/', "~1")
401}
402
403fn push(out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String) {
404    out.push(ErrorEntry {
405        path,
406        code,
407        hint: Some(hint),
408    });
409}
410
411/// The `attachments[].path` set of one object (a record or a run), after
412/// reporting every path that is empty, absolute, contains `..`, or repeats
413/// an earlier one. The set holds the first occurrence of each valid path.
414fn attachment_paths<'a>(
415    obj: &'a Map<String, Value>,
416    out: &mut Vec<ErrorEntry>,
417) -> HashSet<&'a str> {
418    let mut paths: HashSet<&str> = HashSet::new();
419    let Some(attachments) = obj.get("attachments").and_then(Value::as_array) else {
420        return paths;
421    };
422    for (i, a) in attachments.iter().enumerate() {
423        let Some(p) = a.get("path").and_then(Value::as_str) else {
424            continue;
425        };
426        let invalid = if p.is_empty() {
427            Some("empty")
428        } else if p.starts_with('/') {
429            Some("absolute")
430        } else if p.split('/').any(|seg| seg == "..") {
431            Some("contains `..`")
432        } else if !paths.insert(p) {
433            Some("duplicate")
434        } else {
435            None
436        };
437        if let Some(why) = invalid {
438            push(
439                out,
440                format!("/attachments/{i}/path"),
441                ErrorCode::AttachmentPathInvalid,
442                format!("attachment path {p:?} is {why}"),
443            );
444        }
445    }
446    paths
447}
448
449/// `attachment_ref_unknown` at `pointer` unless `p` is in `paths`.
450fn ref_known(paths: &HashSet<&str>, out: &mut Vec<ErrorEntry>, pointer: String, p: &str) {
451    if !paths.contains(p) {
452        push(
453            out,
454            pointer,
455            ErrorCode::AttachmentRefUnknown,
456            format!("{p:?} names no attachments[].path"),
457        );
458    }
459}
460
461/// `metric_id_invalid` at `pointer` unless `m` is `{ns}/{name}`.
462fn metric_id(out: &mut Vec<ErrorEntry>, pointer: String, m: &str) {
463    if !is_valid_id(m) {
464        push(
465            out,
466            pointer,
467            ErrorCode::MetricIdInvalid,
468            format!("{m:?} is not of the form {{ns}}/{{name}}"),
469        );
470    }
471}
472
473/// Whether a key is absent or `null`: both mean "not given" for the
474/// pairings checked here.
475fn given(obj: &Map<String, Value>, key: &str) -> bool {
476    obj.get(key).is_some_and(|v| !v.is_null())
477}
478
479fn semantic(
480    kind: RecordKind,
481    eval_schema: Option<EvalSchema>,
482    value: &Value,
483    out: &mut Vec<ErrorEntry>,
484) {
485    let Some(obj) = value.as_object() else {
486        return;
487    };
488
489    let paths = attachment_paths(obj, out);
490
491    if kind == RecordKind::Card {
492        let results = obj.get("results").and_then(Value::as_array);
493        if let Some(results) = results {
494            for (i, r) in results.iter().enumerate() {
495                if let Some(m) = r.get("metric").and_then(Value::as_str) {
496                    metric_id(out, format!("/results/{i}/metric"), m);
497                }
498                if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
499                    ref_known(&paths, out, format!("/results/{i}/samples_ref"), s);
500                }
501            }
502            let counts = obj.get("counts").filter(|c| !c.is_null());
503            match counts {
504                None if !results.is_empty() => push(
505                    out,
506                    "/counts".to_string(),
507                    ErrorCode::CountsMissing,
508                    "results are present, so counts is required".to_string(),
509                ),
510                Some(c) => {
511                    let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
512                    let sum = n("completed") + n("failed") + n("skipped") + n("errored");
513                    if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
514                        && attempted < sum
515                    {
516                        push(
517                            out,
518                            "/counts".to_string(),
519                            ErrorCode::CountsInconsistent,
520                            format!(
521                                "attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
522                            ),
523                        );
524                    }
525                }
526                None => {}
527            }
528        }
529        run_results(obj, out);
530    }
531
532    // The 1.0 arm only: runs in the body point into the body's attachments.
533    if eval_schema == Some(EvalSchema::V1)
534        && let Some(runs) = obj.get("runs").and_then(Value::as_array)
535    {
536        for (i, run) in runs.iter().enumerate() {
537            if let Some(c) = run.get("calls").and_then(Value::as_str) {
538                ref_known(&paths, out, format!("/runs/{i}/calls"), c);
539            }
540            if let Some(artifacts) = run.get("artifacts").and_then(Value::as_array) {
541                for (j, a) in artifacts.iter().enumerate() {
542                    if let Some(a) = a.as_str() {
543                        ref_known(&paths, out, format!("/runs/{i}/artifacts/{j}"), a);
544                    }
545                }
546            }
547        }
548    }
549
550    if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
551        for (i, rel) in relations.iter().enumerate() {
552            if let Some(t) = rel.get("type").and_then(Value::as_str)
553                && !is_valid_id(t)
554            {
555                push(
556                    out,
557                    format!("/relations/{i}/type"),
558                    ErrorCode::Schema,
559                    format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
560                );
561            }
562            if let Some(to) = rel.get("to").and_then(Value::as_str)
563                && parse_relation_target(to).is_none()
564            {
565                push(
566                    out,
567                    format!("/relations/{i}/to"),
568                    ErrorCode::Schema,
569                    format!(
570                        "{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
571                    ),
572                );
573            }
574        }
575    }
576}
577
578/// The `{ns}/{name}` of every hub target of the Card's `core/uses_eval`
579/// relations, without the `@{seq}`.
580fn used_eval_records(obj: &Map<String, Value>) -> HashSet<String> {
581    let Some(relations) = obj.get("relations").and_then(Value::as_array) else {
582        return HashSet::new();
583    };
584    relations
585        .iter()
586        .filter(|rel| rel.get("type").and_then(Value::as_str) == Some("core/uses_eval"))
587        .filter_map(|rel| rel.get("to").and_then(Value::as_str))
588        .filter_map(|to| match parse_relation_target(to)? {
589            RelationTarget::Version { ns, name, .. } => Some(format!("{ns}/{name}")),
590            RelationTarget::External(_) | RelationTarget::Hf(_) => None,
591        })
592        .collect()
593}
594
595/// The Card's `run_results[]` rules that need nothing but the Card.
596fn run_results(obj: &Map<String, Value>, out: &mut Vec<ErrorEntry>) {
597    let Some(items) = obj.get("run_results").and_then(Value::as_array) else {
598        return;
599    };
600    let evals = used_eval_records(obj);
601    for (i, item) in items.iter().enumerate() {
602        let Some(item) = item.as_object() else {
603            continue;
604        };
605        if let Some(m) = item.get("metric").and_then(Value::as_str) {
606            metric_id(out, format!("/run_results/{i}/metric"), m);
607        }
608        if !given(item, "value") && !given(item, "label") {
609            push(
610                out,
611                format!("/run_results/{i}"),
612                ErrorCode::RunResultValueOrLabel,
613                "a run result needs a value, a label, or both".to_string(),
614            );
615        }
616        if let Some(e) = item.get("eval").and_then(Value::as_str)
617            && !evals.contains(e)
618        {
619            push(
620                out,
621                format!("/run_results/{i}/eval"),
622                ErrorCode::RunResultsEvalUnknown,
623                format!(
624                    "{e:?} is not the {{ns}}/{{name}} (without @seq) of any core/uses_eval target of this Card"
625                ),
626            );
627        }
628    }
629}
630
631fn run_semantic(run_id: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
632    if !is_valid_run_id(run_id) {
633        push(
634            out,
635            "/run_id".to_string(),
636            ErrorCode::RunIdInvalid,
637            format!(
638                "run_id {run_id:?} must be non-empty, contain no `/`, and be at most {MAX_RUN_ID_BYTES} bytes"
639            ),
640        );
641    }
642    let Some(obj) = value.as_object() else {
643        return;
644    };
645    if let Some(body_id) = obj.get("run_id").and_then(Value::as_str)
646        && body_id != run_id
647    {
648        push(
649            out,
650            "/run_id".to_string(),
651            ErrorCode::RunIdMismatch,
652            format!("the body says run_id {body_id:?}, but it is written as {run_id:?}"),
653        );
654    }
655
656    // status ⇔ detail. Only a recognised status is paired; an unknown one
657    // is already a `schema` error.
658    let status = obj.get("status").and_then(Value::as_str);
659    let expects = match status {
660        Some("ok") => Some((false, false)),
661        Some("error") => Some((true, false)),
662        Some("skipped") => Some((false, true)),
663        _ => None,
664    };
665    if let (Some(status), Some((wants_error, wants_skip))) = (status, expects) {
666        for (key, wanted) in [("error", wants_error), ("skip_reason", wants_skip)] {
667            let has = given(obj, key);
668            if has != wanted {
669                let hint = if wanted {
670                    format!("status {status:?} requires {key}")
671                } else {
672                    format!(
673                        "{key} is only allowed when status is {:?}",
674                        if key == "error" { "error" } else { "skipped" }
675                    )
676                };
677                push(out, format!("/{key}"), ErrorCode::RunStatusDetail, hint);
678            }
679        }
680    }
681
682    let paths = attachment_paths(obj, out);
683    if let Some(c) = obj.get("calls").and_then(Value::as_str) {
684        ref_known(&paths, out, "/calls".to_string(), c);
685    }
686    if let Some(artifacts) = obj.get("artifacts").and_then(Value::as_array) {
687        for (j, a) in artifacts.iter().enumerate() {
688            if let Some(a) = a.as_str() {
689                ref_known(&paths, out, format!("/artifacts/{j}"), a);
690            }
691        }
692    }
693    if let Some(log) = obj
694        .get("error")
695        .and_then(|e| e.get("log"))
696        .and_then(Value::as_str)
697    {
698        ref_known(&paths, out, "/error/log".to_string(), log);
699    }
700
701    if let Some(metrics) = obj.get("metrics").and_then(Value::as_object) {
702        for key in metrics.keys() {
703            metric_id(out, format!("/metrics/{}", escape_pointer(key)), key);
704        }
705    }
706}
707
708#[cfg(test)]
709mod tests {
710    #![allow(clippy::unwrap_used)]
711
712    use super::*;
713
714    #[test]
715    fn ids() {
716        assert!(is_valid_id("core/pass_rate"));
717        assert!(is_valid_id("alice/qwen-loop.v2"));
718        assert!(!is_valid_id("pass_rate"));
719        assert!(!is_valid_id("Core/pass_rate"));
720        assert!(!is_valid_id("core/"));
721        assert!(!is_valid_id("/x"));
722        assert!(!is_valid_id("a/b/c"));
723        assert!(!is_valid_id("-a/b"));
724    }
725
726    #[test]
727    fn run_ids() {
728        assert!(is_valid_run_id("r1"));
729        assert!(is_valid_run_id("task 7, trial 2"));
730        assert!(is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES)));
731        assert!(!is_valid_run_id(""));
732        assert!(!is_valid_run_id("a/b"));
733        assert!(!is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES + 1)));
734        // Bytes, not characters: 67 three-byte characters are 201 bytes.
735        assert!(!is_valid_run_id(&"あ".repeat(67)));
736        assert!(is_valid_run_id(&"あ".repeat(66)));
737    }
738
739    #[test]
740    fn relation_targets() {
741        assert_eq!(
742            parse_relation_target("alice/single2-k4@3"),
743            Some(RelationTarget::Version {
744                ns: "alice".into(),
745                name: "single2-k4".into(),
746                seq: 3
747            })
748        );
749        assert_eq!(
750            parse_relation_target("external:https://x/y"),
751            Some(RelationTarget::External("https://x/y".into()))
752        );
753        assert_eq!(
754            parse_relation_target("hf:org/repo@sha"),
755            Some(RelationTarget::Hf("org/repo@sha".into()))
756        );
757        for bad in [
758            "alice/x",
759            "alice/x@0",
760            "alice/x@+1",
761            "alice/x@label",
762            "alice@3",
763            "external:",
764            "hf:",
765            "",
766        ] {
767            assert_eq!(parse_relation_target(bad), None, "{bad}");
768        }
769    }
770
771    #[test]
772    fn validators_build_offline_one_per_accepted_identifier() {
773        let cache = record_validators();
774        for kind in [RecordKind::Card, RecordKind::Eval] {
775            for id in kind.schema_ids() {
776                assert!(cache.contains_key(id), "{id}");
777            }
778        }
779        assert_eq!(
780            cache.len(),
781            RecordKind::Card.schema_ids().len() + RecordKind::Eval.schema_ids().len()
782        );
783        let _ = run_validator();
784    }
785
786    #[test]
787    fn pointer_escaping() {
788        let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
789        let mut out = Vec::new();
790        numbers("", &v, &mut out);
791        assert_eq!(out.len(), 1);
792        assert_eq!(out[0].path, "/ext/a~1b/n");
793        assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
794    }
795}