evalhub_core/validate.rs
1//! Validation: everything the hub checks before it stores a record or a
2//! run.
3//!
4//! Validation runs in two passes and collects every failure from both into
5//! one list, so a producer sees all problems in one round trip. There are
6//! two entry points:
7//!
8//! - [`validate`] for a record (a Card, or an Eval header), keyed by
9//! [`RecordKind`];
10//! - [`run()`] for one run of an Eval, given the `run_id` it is written
11//! under.
12//!
13//! A run is not a [`RecordKind`]. `RecordKind` is the two kinds that appear
14//! in URLs and in the store's `type` column, and every match on it is
15//! exhaustive; a run is a row of an Eval record, addressed under it, so it
16//! has its own entry point rather than a third variant that every one of
17//! those matches would have to reject.
18//!
19//! # Pass 1: structural
20//!
21//! The value is checked against a JSON Schema using `jsonschema` with
22//! draft 2020-12. The core is closed, so an unknown key anywhere outside
23//! `ext` is a `schema` error. In the same pass every number is inspected:
24//! an integer outside ±2^53 is `number_too_large`.
25//!
26//! Which schema: a record is checked against the document for the
27//! identifier its own `schema` key declares, through
28//! [`RecordKind::schema_for`]:
29//!
30//! | Declared `schema` | Document |
31//! | -------------------------------------- | ---------------------- |
32//! | `evalhub.card/1.1`, `evalhub.card/1.0` | `schemas/card.json` |
33//! | `evalhub.eval/2.0` | `schemas/eval-2.json` |
34//! | anything else, or no `schema` key | the kind's current one |
35//!
36//! The last row is what makes an unknown identifier an error: the current
37//! document's `schema` key is a constant, so the value fails it at
38//! `/schema` (`schema`), exactly as it did before a kind had two
39//! identifiers. A run is checked against `schemas/run.json`.
40//!
41//! The schemas are the generated ones from `evalhub_schema` (the same
42//! generator output that is committed under `schemas/`). Each is compiled
43//! once per process, one validator per accepted identifier plus one for
44//! the run, on first use. The validators are built in offline mode: every
45//! `$ref` in the schema is local (`#/$defs/…`), and a reference that would
46//! need fetching is a build error, not a fetch.
47//!
48//! ## Formats
49//!
50//! `format` is an assertion. Draft 2020-12 makes it an annotation unless
51//! the validator opts in, and the validators opt in, so a value the served
52//! schema marks `"format": "date-time"` must be an RFC 3339 date-time, or
53//! it is a `schema` error at its path. The fields that carry it are a run's
54//! `started_at` and `ended_at` (`schemas/run.json`). Card and Eval have no
55//! `date-time` field.
56//!
57//! The check is the `jsonschema` crate's `date-time`: a `T` (or `t`)
58//! between date and time, seconds present, and a `Z` or `±hh:mm` offset. A
59//! value that passes it also parses with `DateTime::parse_from_rfc3339`,
60//! which is what fills the store's `runs.started_at` / `ended_at` columns,
61//! so an accepted run with a timestamp always has the column. The
62//! separator between date and time must be `T` (or `t`), as RFC 3339 §5.6
63//! has it; a space is refused, although `parse_from_rfc3339` would parse
64//! it.
65//!
66//! The other `format` values in the generated schemas are the numeric
67//! hints schemars emits for Rust integer and float types (`double`,
68//! `int64`, `uint32`, `uint64`). They are not formats the crate knows, and
69//! unknown formats are ignored, so they stay annotations; `type` and
70//! `minimum` already carry what they say.
71//!
72//! # Runs are not part of an Eval body
73//!
74//! An Eval body is an `evalhub.eval/2.0` header. Runs are not part of it;
75//! they are written one by one (`PUT /evals/{ns}/{name}/runs/{run_id}`) or
76//! in a batch (`POST /evals/{ns}/{name}/runs:batch`). Two bodies are
77//! answered with one error, `runs_moved` at `/runs`, *before* the
78//! structural pass, and nothing else:
79//!
80//! - an `evalhub.eval/2.0` body that has a `runs` key at all, whatever its
81//! value;
82//! - a body that declares `evalhub.eval/1.0`, with or without `runs`. That
83//! is the 0.1.x shape, header and `runs[]` in one body. Releases 0.2.0
84//! and 0.3.0 accepted it and converted it at ingest; release 0.4.0
85//! refuses it.
86//!
87//! Every other error such a body might have would be noise next to "this
88//! is the wrong endpoint for your runs", and the producer that sends it is
89//! typically a 1.0 producer, whether or not it bumped the identifier.
90//!
91//! # Pass 2: semantic
92//!
93//! Rules the schema language cannot express. On a record:
94//!
95//! | Rule | Code |
96//! | -------------------------------------------------------------------------------------- | -------------------------- |
97//! | `results` non-empty ⇒ `counts` present | `counts_missing` |
98//! | `counts.attempted >= completed + failed + skipped + errored` | `counts_inconsistent` |
99//! | every `results[].samples_ref` is an `attachments[].path` | `attachment_ref_unknown` |
100//! | `attachments[].path` unique, relative, contains no `..` | `attachment_path_invalid` |
101//! | `results[].metric` matches `{ns}/{name}` | `metric_id_invalid` |
102//! | `run_results[].metric` matches `{ns}/{name}` | `metric_id_invalid` |
103//! | `run_results[]` element has `value` or `label` (or both) | `run_result_value_or_label`|
104//! | `run_results[].eval` is the `{ns}/{name}` of a `core/uses_eval` relation's target | `run_results_eval_unknown` |
105//! | `relations[].type` matches `{ns}/{name}` | `schema` |
106//! | `relations[].to` is `{ns}/{name}@{seq}`, `external:…`, or `hf:…` | `schema` |
107//! | every key of `ext`, and of each facet's `ext`, matches `{ns}/{name}` | `ext_key_invalid` |
108//!
109//! `run_results[].eval` is compared with the target of each of the Card's
110//! `core/uses_eval` relations *without* its `@{seq}`: runs belong to the
111//! Eval record, not to a version, so a verdict names the record. Only a
112//! hub target (`{ns}/{name}@{seq}`) can match; an `external:` or `hf:`
113//! target has no runs on this hub. Whether that target resolves, and
114//! whether the writer may see it, is not this crate's question.
115//!
116//! Three more rules about `run_results` need data this crate does not
117//! have, and are checked elsewhere:
118//!
119//! | Rule | Code | Checked by |
120//! | -------------------------------------------------------------------------------------- | -------------------------- | ---------- |
121//! | every `run_id` of a used set has a row in its Eval (archived and deleted rows count) | `run_unknown` | the store |
122//! | every `run_results[].run_id` is in the used set of its `eval` | `run_not_in_used_set` | the store |
123//! | `run_results[]` is no longer than the configured limit | `too_many_run_results` | the server |
124//!
125//! On a run ([`run()`]), everything is checked against the run itself; a
126//! run never points into the header:
127//!
128//! | Rule | Code |
129//! | -------------------------------------------------------------------------------------- | -------------------------- |
130//! | the given `run_id` is non-empty, has no `/`, is at most 200 bytes | `run_id_invalid` |
131//! | the body's `run_id` equals the given one | `run_id_mismatch` |
132//! | `error` present iff `status` is `error`; `skip_reason` iff `status` is `skipped` | `run_status_detail` |
133//! | `calls`, `artifacts[]`, `error.log` each name an `attachments[].path` of the run | `attachment_ref_unknown` |
134//! | the run's `attachments[].path` unique, relative, contains no `..` | `attachment_path_invalid` |
135//! | every key of `metrics` matches `{ns}/{name}` | `metric_id_invalid` |
136//! | every key of `ext`, and of each facet's `ext`, matches `{ns}/{name}` | `ext_key_invalid` |
137//!
138//! `ext` is treated on a run as on a record: the map itself is open, and
139//! only its keys are checked, at the top level and inside each facet that
140//! carries one (`model`, `task`, `harness`, `generation`, `trial`,
141//! `grading`, `env`), one error per key at `/ext/{key}` or
142//! `/{facet}/ext/{key}`. The values are never looked into, and neither is
143//! `meta`.
144//!
145//! The semantic pass runs on whatever is present even when the structural
146//! pass failed, so a producer sees both kinds of problem at once. Errors
147//! are sorted by `(path, code)`; the order is deterministic.
148//!
149//! # What is deliberately not an error
150//!
151//! - A `relations[].to` that does not resolve. The record is accepted; the
152//! `refs_resolved` badge is withheld. A Card may legitimately be published
153//! before the Eval it cites.
154//! - A `harness` or `metric` the registry does not know, including a run's
155//! `metrics` key. Accepted; the `harness_registered` /
156//! `metric_registered` badges are withheld. The hub does not gate on
157//! vocabulary.
158//! - An `attachments[].sha256` that has not been uploaded. This is not a
159//! validation failure but a state conflict, `409 attachment_missing`,
160//! raised by the store, because only the store knows.
161//! - An Eval header with zero runs, of any `eval_kind`. A header is
162//! complete on its own; its runs are written afterwards.
163//!
164//! # Output
165//!
166//! `Vec<ErrorEntry { path, code, hint }>`, empty on success. `path` is a
167//! JSON pointer into the submitted value (`/results/0/metric`; the root is
168//! `""`). `hint` is prose and not part of the contract.
169
170use std::collections::{HashMap, HashSet};
171use std::sync::OnceLock;
172
173use jsonschema::{Draft, Validator};
174use serde_json::{Map, Value};
175
176use evalhub_schema::error::{ErrorCode, ErrorEntry};
177use evalhub_schema::{EVAL_SCHEMA, RecordKind};
178
179/// The Eval identifier of the 0.1.x shape, header and `runs[]` in one
180/// body. Refused with `runs_moved` since release 0.4.0; see "Runs are not
181/// part of an Eval body" in the module doc.
182const EVAL_SCHEMA_1_0: &str = "evalhub.eval/1.0";
183
184/// Largest integer a JSON number may carry without losing precision as a
185/// double: 2^53.
186const MAX_SAFE_INTEGER: f64 = 9_007_199_254_740_992.0;
187
188/// Longest `run_id` accepted, in bytes of UTF-8.
189pub const MAX_RUN_ID_BYTES: usize = 200;
190
191/// Where a relation points.
192#[derive(Debug, Clone, PartialEq, Eq)]
193pub enum RelationTarget {
194 /// A version on this hub, `{ns}/{name}@{seq}`.
195 Version {
196 /// Namespace of the target record.
197 ns: String,
198 /// Name of the target record.
199 name: String,
200 /// Sequence number of the target version, from 1.
201 seq: u32,
202 },
203 /// Something outside the hub, written `external:<anything>`.
204 External(String),
205 /// A Hugging Face reference, written `hf:<org>/<repo>@<sha>`.
206 Hf(String),
207}
208
209/// Whether `s` has the form `{ns}/{name}`: exactly one `/`, both parts
210/// non-empty, each starting with `[a-z0-9]` and continuing with
211/// `[a-z0-9._-]`.
212pub fn is_valid_id(s: &str) -> bool {
213 let Some((ns, name)) = s.split_once('/') else {
214 return false;
215 };
216 is_slug(ns) && is_slug(name)
217}
218
219/// Whether `s` is a valid `run_id`: non-empty, no `/`, at most
220/// [`MAX_RUN_ID_BYTES`] bytes of UTF-8. Nothing else is imposed (see
221/// `evalhub_schema::run`); every `run_id` a 0.1.x producer wrote passes.
222pub fn is_valid_run_id(s: &str) -> bool {
223 !s.is_empty() && !s.contains('/') && s.len() <= MAX_RUN_ID_BYTES
224}
225
226fn is_slug(part: &str) -> bool {
227 let mut chars = part.chars();
228 match chars.next() {
229 Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {}
230 _ => return false,
231 }
232 chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-'))
233}
234
235/// Parse a `relations[].to` value. `None` when it has none of the accepted
236/// forms.
237pub fn parse_relation_target(s: &str) -> Option<RelationTarget> {
238 if let Some(rest) = s.strip_prefix("external:") {
239 return (!rest.is_empty()).then(|| RelationTarget::External(rest.to_string()));
240 }
241 if let Some(rest) = s.strip_prefix("hf:") {
242 return (!rest.is_empty()).then(|| RelationTarget::Hf(rest.to_string()));
243 }
244 let (id, seq) = s.rsplit_once('@')?;
245 let (ns, name) = id.split_once('/')?;
246 if !is_slug(ns) || !is_slug(name) {
247 return None;
248 }
249 // `parse::<u32>` accepts a leading `+`; the address form does not.
250 if seq.is_empty() || !seq.bytes().all(|b| b.is_ascii_digit()) {
251 return None;
252 }
253 let seq: u32 = seq.parse().ok()?;
254 if seq == 0 {
255 return None;
256 }
257 Some(RelationTarget::Version {
258 ns: ns.to_string(),
259 name: name.to_string(),
260 seq,
261 })
262}
263
264/// Validate a record of `kind` and return every failure, sorted by
265/// `(path, code)`; empty when the record is accepted.
266///
267/// The JSON Schema is the one for the identifier the body declares in
268/// `schema` (see the module doc). An `evalhub.eval/2.0` body with a `runs`
269/// key, and an Eval body declaring `evalhub.eval/1.0`, return exactly one
270/// error, `runs_moved` at `/runs`, and nothing else. Cost: one pass of the
271/// compiled validator plus one walk of the value; the validator is
272/// compiled on the first call for its identifier.
273pub fn validate(kind: RecordKind, value: &Value) -> Vec<ErrorEntry> {
274 let declared = value.get("schema").and_then(Value::as_str);
275 let runs_moved_here = kind == RecordKind::Eval
276 && match declared {
277 Some(EVAL_SCHEMA_1_0) => true,
278 Some(EVAL_SCHEMA) => value.get("runs").is_some(),
279 _ => false,
280 };
281 if runs_moved_here {
282 return vec![runs_moved()];
283 }
284 let mut errors = Vec::new();
285 structural(record_validator(kind, declared), value, &mut errors);
286 numbers("", value, &mut errors);
287 semantic(kind, value, &mut errors);
288 sort(&mut errors);
289 errors
290}
291
292/// Validate one run of an Eval, written under `run_id` (the path segment
293/// of `PUT /evals/{ns}/{name}/runs/{run_id}`, or the element's own
294/// `run_id` in a batch). Returns every failure, sorted by `(path, code)`;
295/// empty when the run is accepted.
296///
297/// Both passes run: `schemas/run.json`, the ±2^53 check, then the run
298/// rules in the module doc. `run_id_invalid` and `run_id_mismatch` are
299/// reported at `/run_id`. The run's pointers are checked against its own
300/// `attachments[]` only. Cost: as for [`validate`].
301pub fn run(run_id: &str, value: &Value) -> Vec<ErrorEntry> {
302 let mut errors = Vec::new();
303 structural(run_validator(), value, &mut errors);
304 numbers("", value, &mut errors);
305 run_semantic(run_id, value, &mut errors);
306 sort(&mut errors);
307 errors
308}
309
310/// The one error a 2.0 header carrying `runs`, or a body declaring
311/// `evalhub.eval/1.0`, gets.
312fn runs_moved() -> ErrorEntry {
313 ErrorEntry {
314 path: "/runs".to_string(),
315 code: ErrorCode::RunsMoved,
316 hint: Some(
317 "an Eval body is an evalhub.eval/2.0 header, the only Eval schema \
318 accepted (evalhub.eval/1.0 is not), and it does not carry runs; \
319 write each run with PUT /evals/{ns}/{name}/runs/{run_id}, or \
320 several with POST /evals/{ns}/{name}/runs:batch"
321 .to_string(),
322 ),
323 }
324}
325
326fn sort(errors: &mut [ErrorEntry]) {
327 // `ErrorCode` is a plain enum; its declaration order is the tie-break.
328 errors.sort_by(|a, b| {
329 a.path
330 .cmp(&b.path)
331 .then_with(|| (a.code as u8).cmp(&(b.code as u8)))
332 });
333}
334
335/// Compile a generated schema. The generated schemas only reference
336/// `#/$defs/*`; `offline()` turns any other reference into a build failure
337/// here, which the unit tests exercise, rather than a fetch in production.
338///
339/// `format` is asserted, not annotated (see "Formats" in the module doc).
340/// Unknown formats stay ignored: that is the crate's default, and it is set
341/// here explicitly because schemars emits numeric hints (`double`, `int64`,
342/// `uint32`, `uint64`) that are not formats the crate knows.
343fn compile(schema: &Value) -> Validator {
344 #[allow(clippy::expect_used)]
345 jsonschema::options()
346 .with_draft(Draft::Draft202012)
347 .should_validate_formats(true)
348 .should_ignore_unknown_formats(true)
349 .offline()
350 .build(schema)
351 .expect("the generated schema compiles")
352}
353
354/// One compiled validator per accepted identifier of every kind, keyed by
355/// the identifier. Two Card identifiers share a document and are compiled
356/// twice; that is two compilations per process, and it keeps the cache a
357/// plain map from what the body says to what checks it.
358fn record_validators() -> &'static HashMap<&'static str, Validator> {
359 static CACHE: OnceLock<HashMap<&'static str, Validator>> = OnceLock::new();
360 CACHE.get_or_init(|| {
361 let mut map = HashMap::new();
362 for kind in [RecordKind::Card, RecordKind::Eval] {
363 for &id in kind.schema_ids() {
364 if let Some(schema) = kind.schema_for(id) {
365 map.insert(id, compile(schema.as_value()));
366 }
367 }
368 }
369 map
370 })
371}
372
373/// The validator for a record of `kind` declaring `declared`: the one for
374/// that identifier when `kind` accepts it, otherwise the kind's current one.
375fn record_validator(kind: RecordKind, declared: Option<&str>) -> &'static Validator {
376 let cache = record_validators();
377 let accepted = declared.filter(|id| kind.schema_ids().contains(id));
378 let id = accepted.unwrap_or(kind.schema_id());
379 // Every accepted identifier has a document (a unit test asserts it);
380 // a miss here is a bug in `evalhub_schema`, not a property of the input.
381 #[allow(clippy::expect_used)]
382 cache
383 .get(id)
384 .expect("every accepted schema identifier has a document")
385}
386
387fn run_validator() -> &'static Validator {
388 static RUN: OnceLock<Validator> = OnceLock::new();
389 RUN.get_or_init(|| compile(evalhub_schema::schema_for_run().as_value()))
390}
391
392fn structural(validator: &Validator, value: &Value, out: &mut Vec<ErrorEntry>) {
393 for e in validator.iter_errors(value) {
394 out.push(ErrorEntry {
395 path: e.instance_path().as_str().to_string(),
396 code: ErrorCode::Schema,
397 hint: Some(e.to_string()),
398 });
399 }
400}
401
402fn numbers(path: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
403 match value {
404 Value::Number(n) => {
405 let too_large = if let Some(i) = n.as_i64() {
406 i.unsigned_abs() > MAX_SAFE_INTEGER as u64
407 } else if let Some(u) = n.as_u64() {
408 u > MAX_SAFE_INTEGER as u64
409 } else if let Some(f) = n.as_f64() {
410 f.is_finite() && f.fract() == 0.0 && f.abs() > MAX_SAFE_INTEGER
411 } else {
412 false
413 };
414 if too_large {
415 out.push(ErrorEntry {
416 path: path.to_string(),
417 code: ErrorCode::NumberTooLarge,
418 hint: Some(format!(
419 "{n} is an integer outside ±2^53; send it as a string"
420 )),
421 });
422 }
423 }
424 Value::Array(items) => {
425 for (i, item) in items.iter().enumerate() {
426 numbers(&format!("{path}/{i}"), item, out);
427 }
428 }
429 Value::Object(map) => {
430 for (k, v) in map {
431 numbers(&format!("{path}/{}", escape_pointer(k)), v, out);
432 }
433 }
434 _ => {}
435 }
436}
437
438/// RFC 6901 escaping for one reference token.
439fn escape_pointer(key: &str) -> String {
440 key.replace('~', "~0").replace('/', "~1")
441}
442
443fn push(out: &mut Vec<ErrorEntry>, path: String, code: ErrorCode, hint: String) {
444 out.push(ErrorEntry {
445 path,
446 code,
447 hint: Some(hint),
448 });
449}
450
451/// The `attachments[].path` set of one object (a record or a run), after
452/// reporting every path that is empty, absolute, contains `..`, or repeats
453/// an earlier one. The set holds the first occurrence of each valid path.
454fn attachment_paths<'a>(
455 obj: &'a Map<String, Value>,
456 out: &mut Vec<ErrorEntry>,
457) -> HashSet<&'a str> {
458 let mut paths: HashSet<&str> = HashSet::new();
459 let Some(attachments) = obj.get("attachments").and_then(Value::as_array) else {
460 return paths;
461 };
462 for (i, a) in attachments.iter().enumerate() {
463 let Some(p) = a.get("path").and_then(Value::as_str) else {
464 continue;
465 };
466 let invalid = if p.is_empty() {
467 Some("empty")
468 } else if p.starts_with('/') {
469 Some("absolute")
470 } else if p.split('/').any(|seg| seg == "..") {
471 Some("contains `..`")
472 } else if !paths.insert(p) {
473 Some("duplicate")
474 } else {
475 None
476 };
477 if let Some(why) = invalid {
478 push(
479 out,
480 format!("/attachments/{i}/path"),
481 ErrorCode::AttachmentPathInvalid,
482 format!("attachment path {p:?} is {why}"),
483 );
484 }
485 }
486 paths
487}
488
489/// `attachment_ref_unknown` at `pointer` unless `p` is in `paths`.
490fn ref_known(paths: &HashSet<&str>, out: &mut Vec<ErrorEntry>, pointer: String, p: &str) {
491 if !paths.contains(p) {
492 push(
493 out,
494 pointer,
495 ErrorCode::AttachmentRefUnknown,
496 format!("{p:?} names no attachments[].path"),
497 );
498 }
499}
500
501/// `metric_id_invalid` at `pointer` unless `m` is `{ns}/{name}`.
502fn metric_id(out: &mut Vec<ErrorEntry>, pointer: String, m: &str) {
503 if !is_valid_id(m) {
504 push(
505 out,
506 pointer,
507 ErrorCode::MetricIdInvalid,
508 format!("{m:?} is not of the form {{ns}}/{{name}}"),
509 );
510 }
511}
512
513/// The facets of a record or a run that carry an `ext` map of their own.
514/// `grading` exists on a Card only; on an Eval or a run it is simply absent.
515const FACETS_WITH_EXT: [&str; 7] = [
516 "model",
517 "task",
518 "harness",
519 "generation",
520 "trial",
521 "grading",
522 "env",
523];
524
525/// `ext_key_invalid` at `{prefix}/ext/{key}` for every key of `obj.ext`
526/// that is not `{ns}/{name}`. `prefix` is the pointer of `obj` itself
527/// (`""` for the body, `"/model"` for a facet).
528fn ext_keys(out: &mut Vec<ErrorEntry>, prefix: &str, obj: &Map<String, Value>) {
529 let Some(ext) = obj.get("ext").and_then(Value::as_object) else {
530 return;
531 };
532 for key in ext.keys() {
533 if !is_valid_id(key) {
534 push(
535 out,
536 format!("{prefix}/ext/{}", escape_pointer(key)),
537 ErrorCode::ExtKeyInvalid,
538 format!("ext key {key:?} is not of the form {{ns}}/{{name}}"),
539 );
540 }
541 }
542}
543
544/// [`ext_keys`] on the body and on each of its [`FACETS_WITH_EXT`].
545fn ext_keys_everywhere(out: &mut Vec<ErrorEntry>, obj: &Map<String, Value>) {
546 ext_keys(out, "", obj);
547 for facet in FACETS_WITH_EXT {
548 if let Some(f) = obj.get(facet).and_then(Value::as_object) {
549 ext_keys(out, &format!("/{facet}"), f);
550 }
551 }
552}
553
554/// Whether a key is absent or `null`: both mean "not given" for the
555/// pairings checked here.
556fn given(obj: &Map<String, Value>, key: &str) -> bool {
557 obj.get(key).is_some_and(|v| !v.is_null())
558}
559
560fn semantic(kind: RecordKind, value: &Value, out: &mut Vec<ErrorEntry>) {
561 let Some(obj) = value.as_object() else {
562 return;
563 };
564
565 let paths = attachment_paths(obj, out);
566 ext_keys_everywhere(out, obj);
567
568 if kind == RecordKind::Card {
569 let results = obj.get("results").and_then(Value::as_array);
570 if let Some(results) = results {
571 for (i, r) in results.iter().enumerate() {
572 if let Some(m) = r.get("metric").and_then(Value::as_str) {
573 metric_id(out, format!("/results/{i}/metric"), m);
574 }
575 if let Some(s) = r.get("samples_ref").and_then(Value::as_str) {
576 ref_known(&paths, out, format!("/results/{i}/samples_ref"), s);
577 }
578 }
579 let counts = obj.get("counts").filter(|c| !c.is_null());
580 match counts {
581 None if !results.is_empty() => push(
582 out,
583 "/counts".to_string(),
584 ErrorCode::CountsMissing,
585 "results are present, so counts is required".to_string(),
586 ),
587 Some(c) => {
588 let n = |k: &str| c.get(k).and_then(Value::as_u64).unwrap_or(0);
589 let sum = n("completed") + n("failed") + n("skipped") + n("errored");
590 if let Some(attempted) = c.get("attempted").and_then(Value::as_u64)
591 && attempted < sum
592 {
593 push(
594 out,
595 "/counts".to_string(),
596 ErrorCode::CountsInconsistent,
597 format!(
598 "attempted ({attempted}) < completed + failed + skipped + errored ({sum})"
599 ),
600 );
601 }
602 }
603 None => {}
604 }
605 }
606 run_results(obj, out);
607 }
608
609 if let Some(relations) = obj.get("relations").and_then(Value::as_array) {
610 for (i, rel) in relations.iter().enumerate() {
611 if let Some(t) = rel.get("type").and_then(Value::as_str)
612 && !is_valid_id(t)
613 {
614 push(
615 out,
616 format!("/relations/{i}/type"),
617 ErrorCode::Schema,
618 format!("{t:?} is not a registry id of the form {{ns}}/{{name}}"),
619 );
620 }
621 if let Some(to) = rel.get("to").and_then(Value::as_str)
622 && parse_relation_target(to).is_none()
623 {
624 push(
625 out,
626 format!("/relations/{i}/to"),
627 ErrorCode::Schema,
628 format!(
629 "{to:?} is not `{{ns}}/{{name}}@{{seq}}`, `external:<ref>` or `hf:<ref>`"
630 ),
631 );
632 }
633 }
634 }
635}
636
637/// The `{ns}/{name}` of every hub target of the Card's `core/uses_eval`
638/// relations, without the `@{seq}`.
639fn used_eval_records(obj: &Map<String, Value>) -> HashSet<String> {
640 let Some(relations) = obj.get("relations").and_then(Value::as_array) else {
641 return HashSet::new();
642 };
643 relations
644 .iter()
645 .filter(|rel| rel.get("type").and_then(Value::as_str) == Some("core/uses_eval"))
646 .filter_map(|rel| rel.get("to").and_then(Value::as_str))
647 .filter_map(|to| match parse_relation_target(to)? {
648 RelationTarget::Version { ns, name, .. } => Some(format!("{ns}/{name}")),
649 RelationTarget::External(_) | RelationTarget::Hf(_) => None,
650 })
651 .collect()
652}
653
654/// The Card's `run_results[]` rules that need nothing but the Card.
655fn run_results(obj: &Map<String, Value>, out: &mut Vec<ErrorEntry>) {
656 let Some(items) = obj.get("run_results").and_then(Value::as_array) else {
657 return;
658 };
659 let evals = used_eval_records(obj);
660 for (i, item) in items.iter().enumerate() {
661 let Some(item) = item.as_object() else {
662 continue;
663 };
664 if let Some(m) = item.get("metric").and_then(Value::as_str) {
665 metric_id(out, format!("/run_results/{i}/metric"), m);
666 }
667 if !given(item, "value") && !given(item, "label") {
668 push(
669 out,
670 format!("/run_results/{i}"),
671 ErrorCode::RunResultValueOrLabel,
672 "a run result needs a value, a label, or both".to_string(),
673 );
674 }
675 if let Some(e) = item.get("eval").and_then(Value::as_str)
676 && !evals.contains(e)
677 {
678 push(
679 out,
680 format!("/run_results/{i}/eval"),
681 ErrorCode::RunResultsEvalUnknown,
682 format!(
683 "{e:?} is not the {{ns}}/{{name}} (without @seq) of any core/uses_eval target of this Card"
684 ),
685 );
686 }
687 }
688}
689
690fn run_semantic(run_id: &str, value: &Value, out: &mut Vec<ErrorEntry>) {
691 if !is_valid_run_id(run_id) {
692 push(
693 out,
694 "/run_id".to_string(),
695 ErrorCode::RunIdInvalid,
696 format!(
697 "run_id {run_id:?} must be non-empty, contain no `/`, and be at most {MAX_RUN_ID_BYTES} bytes"
698 ),
699 );
700 }
701 let Some(obj) = value.as_object() else {
702 return;
703 };
704 if let Some(body_id) = obj.get("run_id").and_then(Value::as_str)
705 && body_id != run_id
706 {
707 push(
708 out,
709 "/run_id".to_string(),
710 ErrorCode::RunIdMismatch,
711 format!("the body says run_id {body_id:?}, but it is written as {run_id:?}"),
712 );
713 }
714
715 // status ⇔ detail. Only a recognised status is paired; an unknown one
716 // is already a `schema` error.
717 let status = obj.get("status").and_then(Value::as_str);
718 let expects = match status {
719 Some("ok") => Some((false, false)),
720 Some("error") => Some((true, false)),
721 Some("skipped") => Some((false, true)),
722 _ => None,
723 };
724 if let (Some(status), Some((wants_error, wants_skip))) = (status, expects) {
725 for (key, wanted) in [("error", wants_error), ("skip_reason", wants_skip)] {
726 let has = given(obj, key);
727 if has != wanted {
728 let hint = if wanted {
729 format!("status {status:?} requires {key}")
730 } else {
731 format!(
732 "{key} is only allowed when status is {:?}",
733 if key == "error" { "error" } else { "skipped" }
734 )
735 };
736 push(out, format!("/{key}"), ErrorCode::RunStatusDetail, hint);
737 }
738 }
739 }
740
741 let paths = attachment_paths(obj, out);
742 if let Some(c) = obj.get("calls").and_then(Value::as_str) {
743 ref_known(&paths, out, "/calls".to_string(), c);
744 }
745 if let Some(artifacts) = obj.get("artifacts").and_then(Value::as_array) {
746 for (j, a) in artifacts.iter().enumerate() {
747 if let Some(a) = a.as_str() {
748 ref_known(&paths, out, format!("/artifacts/{j}"), a);
749 }
750 }
751 }
752 if let Some(log) = obj
753 .get("error")
754 .and_then(|e| e.get("log"))
755 .and_then(Value::as_str)
756 {
757 ref_known(&paths, out, "/error/log".to_string(), log);
758 }
759
760 if let Some(metrics) = obj.get("metrics").and_then(Value::as_object) {
761 for key in metrics.keys() {
762 metric_id(out, format!("/metrics/{}", escape_pointer(key)), key);
763 }
764 }
765 ext_keys_everywhere(out, obj);
766}
767
768#[cfg(test)]
769mod tests {
770 #![allow(clippy::unwrap_used)]
771
772 use super::*;
773
774 #[test]
775 fn ids() {
776 assert!(is_valid_id("core/pass_rate"));
777 assert!(is_valid_id("alice/qwen-loop.v2"));
778 assert!(!is_valid_id("pass_rate"));
779 assert!(!is_valid_id("Core/pass_rate"));
780 assert!(!is_valid_id("core/"));
781 assert!(!is_valid_id("/x"));
782 assert!(!is_valid_id("a/b/c"));
783 assert!(!is_valid_id("-a/b"));
784 }
785
786 #[test]
787 fn run_ids() {
788 assert!(is_valid_run_id("r1"));
789 assert!(is_valid_run_id("task 7, trial 2"));
790 assert!(is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES)));
791 assert!(!is_valid_run_id(""));
792 assert!(!is_valid_run_id("a/b"));
793 assert!(!is_valid_run_id(&"x".repeat(MAX_RUN_ID_BYTES + 1)));
794 // Bytes, not characters: 67 three-byte characters are 201 bytes.
795 assert!(!is_valid_run_id(&"あ".repeat(67)));
796 assert!(is_valid_run_id(&"あ".repeat(66)));
797 }
798
799 #[test]
800 fn relation_targets() {
801 assert_eq!(
802 parse_relation_target("alice/single2-k4@3"),
803 Some(RelationTarget::Version {
804 ns: "alice".into(),
805 name: "single2-k4".into(),
806 seq: 3
807 })
808 );
809 assert_eq!(
810 parse_relation_target("external:https://x/y"),
811 Some(RelationTarget::External("https://x/y".into()))
812 );
813 assert_eq!(
814 parse_relation_target("hf:org/repo@sha"),
815 Some(RelationTarget::Hf("org/repo@sha".into()))
816 );
817 for bad in [
818 "alice/x",
819 "alice/x@0",
820 "alice/x@+1",
821 "alice/x@label",
822 "alice@3",
823 "external:",
824 "hf:",
825 "",
826 ] {
827 assert_eq!(parse_relation_target(bad), None, "{bad}");
828 }
829 }
830
831 #[test]
832 fn validators_build_offline_one_per_accepted_identifier() {
833 let cache = record_validators();
834 for kind in [RecordKind::Card, RecordKind::Eval] {
835 for id in kind.schema_ids() {
836 assert!(cache.contains_key(id), "{id}");
837 }
838 }
839 assert_eq!(
840 cache.len(),
841 RecordKind::Card.schema_ids().len() + RecordKind::Eval.schema_ids().len()
842 );
843 let _ = run_validator();
844 }
845
846 #[test]
847 fn pointer_escaping() {
848 let v = serde_json::json!({"ext": {"a/b": {"n": 9007199254740993u64}}});
849 let mut out = Vec::new();
850 numbers("", &v, &mut out);
851 assert_eq!(out.len(), 1);
852 assert_eq!(out[0].path, "/ext/a~1b/n");
853 assert_eq!(out[0].code, ErrorCode::NumberTooLarge);
854 }
855}