eval_magic/validation/
schema.rs

1//! Schema embedding + the generic `validate_against_schema` entry point.
2//!
3//! The portable-artifact schemas are the single source of truth for each
4//! artifact's shape. They are embedded at compile time with `include_str!` (so
5//! the binary is self-contained, with no `schema/` directory to ship alongside)
6//! and compiled once into reusable `jsonschema` validators.
7
8use std::collections::HashMap;
9use std::sync::LazyLock;
10
11use jsonschema::Validator;
12use serde::de::DeserializeOwned;
13use serde_json::Value;
14
15use crate::validation::error::ValidationError;
16
17/// Names the portable-artifact schemas. `benchmark` and `judge-tasks` are
18/// first-class here, schema-gated like every other pipeline output.
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
20pub enum SchemaName {
21    RunRecord,
22    Evals,
23    Grading,
24    StrayWrites,
25    Benchmark,
26    JudgeTasks,
27    HarnessDescriptor,
28}
29
30impl SchemaName {
31    /// Every schema, for building the validator cache.
32    const ALL: [SchemaName; 7] = [
33        SchemaName::RunRecord,
34        SchemaName::Evals,
35        SchemaName::Grading,
36        SchemaName::StrayWrites,
37        SchemaName::Benchmark,
38        SchemaName::JudgeTasks,
39        SchemaName::HarnessDescriptor,
40    ];
41
42    /// The schema's kebab-case name, as used in error messages and the on-disk
43    /// `<name>.schema.json` filenames.
44    pub fn as_str(self) -> &'static str {
45        match self {
46            SchemaName::RunRecord => "run-record",
47            SchemaName::Evals => "evals",
48            SchemaName::Grading => "grading",
49            SchemaName::StrayWrites => "stray-writes",
50            SchemaName::Benchmark => "benchmark",
51            SchemaName::JudgeTasks => "judge-tasks",
52            SchemaName::HarnessDescriptor => "harness-descriptor",
53        }
54    }
55
56    /// The embedded schema JSON source.
57    fn source(self) -> &'static str {
58        match self {
59            SchemaName::RunRecord => include_str!("../../schema/run-record.schema.json"),
60            SchemaName::Evals => include_str!("../../schema/evals.schema.json"),
61            SchemaName::Grading => include_str!("../../schema/grading.schema.json"),
62            SchemaName::StrayWrites => include_str!("../../schema/stray-writes.schema.json"),
63            SchemaName::Benchmark => include_str!("../../schema/benchmark.schema.json"),
64            SchemaName::JudgeTasks => include_str!("../../schema/judge-tasks.schema.json"),
65            SchemaName::HarnessDescriptor => {
66                include_str!("../../schema/harness-descriptor.schema.json")
67            }
68        }
69    }
70}
71
72/// Compiled validators, built once on first use. The schemas are embedded and
73/// known-valid, so a failure here is a programmer error (a malformed bundled
74/// schema) and panics rather than being surfaced as a runtime validation error.
75static VALIDATORS: LazyLock<HashMap<SchemaName, Validator>> = LazyLock::new(|| {
76    SchemaName::ALL
77        .iter()
78        .map(|&name| {
79            let schema: Value = serde_json::from_str(name.source()).unwrap_or_else(|e| {
80                panic!("bundled {} schema is not valid JSON: {e}", name.as_str())
81            });
82            let validator = jsonschema::validator_for(&schema).unwrap_or_else(|e| {
83                panic!("bundled {} schema does not compile: {e}", name.as_str())
84            });
85            (name, validator)
86        })
87        .collect()
88});
89
90/// Validate `data` against the named schema. Returns it deserialized into `T` on
91/// success; on mismatch, returns a `source`-prefixed [`ValidationError`] listing
92/// every failure.
93///
94/// Deserializing into `T` makes the typed result honest: the schema gate and
95/// the Rust type agree, or the call fails.
96pub fn validate_against_schema<T: DeserializeOwned>(
97    name: SchemaName,
98    data: &Value,
99    source: &str,
100) -> Result<T, ValidationError> {
101    let validator = &VALIDATORS[&name];
102
103    if !validator.is_valid(data) {
104        let details = validator
105            .iter_errors(data)
106            .map(|e| {
107                let instance = e.instance_path().to_string();
108                let instance = if instance.is_empty() {
109                    "/".to_string()
110                } else {
111                    instance
112                };
113                format!("  {instance} {e}")
114            })
115            .collect::<Vec<_>>()
116            .join("\n");
117        return Err(ValidationError::SchemaMismatch {
118            path: source.to_string(),
119            schema: name.as_str().to_string(),
120            details,
121        });
122    }
123
124    serde_json::from_value(data.clone()).map_err(|e| ValidationError::Deserialize {
125        path: source.to_string(),
126        message: e.to_string(),
127    })
128}
129
130#[cfg(test)]
131mod tests {
132    use super::{SchemaName, validate_against_schema};
133    use serde_json::{Value, json};
134
135    /// The canonical valid run-record the cases below mutate.
136    fn valid_run_record() -> Value {
137        json!({
138            "eval_id": "e1",
139            "condition": "with_skill",
140            "skill_path": null,
141            "prompt": "do the thing",
142            "files": [],
143            "final_message": "done",
144            "tool_invocations": [],
145            "total_tokens": 100,
146            "duration_ms": 1000
147        })
148    }
149
150    #[test]
151    fn returns_data_when_it_matches_the_run_record_schema() {
152        let data = valid_run_record();
153        let out: Value =
154            validate_against_schema(SchemaName::RunRecord, &data, "/tmp/run.json").unwrap();
155        assert_eq!(out, data);
156    }
157
158    #[test]
159    fn accepts_an_empty_tool_invocations_array() {
160        let mut data = valid_run_record();
161        data["tool_invocations"] = json!([]);
162        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
163        assert!(r.is_ok());
164    }
165
166    #[test]
167    fn accepts_skill_path_null_on_the_without_skill_arm() {
168        let mut data = valid_run_record();
169        data["condition"] = json!("without_skill");
170        data["skill_path"] = Value::Null;
171        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
172        assert!(r.is_ok());
173    }
174
175    #[test]
176    fn source_prefixed_error_when_required_field_missing() {
177        let mut data = valid_run_record();
178        data.as_object_mut().unwrap().remove("eval_id");
179        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "/tmp/run.json")
180            .unwrap_err()
181            .to_string();
182        assert!(err.contains("/tmp/run.json"), "error was: {err}");
183    }
184
185    #[test]
186    fn requires_skill_path() {
187        let mut data = valid_run_record();
188        data.as_object_mut().unwrap().remove("skill_path");
189        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
190            .unwrap_err()
191            .to_string();
192        assert!(err.contains("skill_path"), "error was: {err}");
193    }
194
195    #[test]
196    fn requires_files() {
197        let mut data = valid_run_record();
198        data.as_object_mut().unwrap().remove("files");
199        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
200            .unwrap_err()
201            .to_string();
202        assert!(err.contains("files"), "error was: {err}");
203    }
204
205    #[test]
206    fn rejects_unknown_extra_property() {
207        let mut data = valid_run_record();
208        data["surprise"] = json!(true);
209        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
210        assert!(r.is_err());
211    }
212
213    #[test]
214    fn tool_invocation_ordinal_must_be_an_integer() {
215        let mut data = valid_run_record();
216        data["tool_invocations"] = json!([{ "name": "Bash", "ordinal": "zero" }]);
217        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
218        assert!(r.is_err());
219    }
220
221    #[test]
222    fn validates_a_well_formed_benchmark() {
223        let benchmark = json!({
224            "generated": "2026-06-08T00:00:00.000Z",
225            "mode": "new-skill",
226            "conditions_compared": ["with_skill", "without_skill"],
227            "missing_gradings": 0,
228            "validity_warnings": [],
229            "run_summary": {
230                "with_skill": {
231                    "pass_rate": { "mean": 1.0, "stddev": 0.0, "n": 1 },
232                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
233                    "total_tokens": { "mean": 5000.0, "stddev": 0.0, "n": 1 },
234                    "skill_invocation_n": 0,
235                    "skill_invocation_rate": null
236                },
237                "without_skill": {
238                    "pass_rate": { "mean": 0.0, "stddev": 0.0, "n": 1 },
239                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
240                    "total_tokens": { "mean": 3000.0, "stddev": 0.0, "n": 1 }
241                }
242            },
243            "delta": { "direction": "with_skill - without_skill", "pass_rate": 1.0, "duration_ms": 0.0, "total_tokens": 2000.0 }
244        });
245        let r: Result<Value, _> =
246            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
247        assert!(r.is_ok(), "benchmark should validate: {r:?}");
248    }
249
250    #[test]
251    fn rejects_a_benchmark_missing_delta() {
252        let mut benchmark = json!({
253            "generated": "t", "mode": "new-skill",
254            "conditions_compared": ["a", "b"], "missing_gradings": 0,
255            "validity_warnings": [], "run_summary": {},
256            "delta": { "direction": "a - b", "pass_rate": 0.0, "duration_ms": 0.0, "total_tokens": 0.0 }
257        });
258        benchmark.as_object_mut().unwrap().remove("delta");
259        let r: Result<Value, _> =
260            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
261        assert!(r.is_err());
262    }
263
264    #[test]
265    fn validates_a_well_formed_judge_tasks_file() {
266        let tasks = json!({
267            "generated": "2026-06-08T00:00:00.000Z",
268            "total_tasks": 1,
269            "meta_tasks_injected": 1,
270            "skipped_transcript_checks": 0,
271            "tasks": [{
272                "eval_id": "e1", "condition": "with_skill", "assertion_id": "__skill_invoked",
273                "rubric": "did it apply the skill?", "model": null, "is_meta": true,
274                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
275                "response_path": "/w/judge-responses/__skill_invoked.json",
276                "dispatch_prompt_path": "/w/judge-prompts/__skill_invoked.txt"
277            }]
278        });
279        let r: Result<Value, _> =
280            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
281        assert!(r.is_ok(), "judge-tasks should validate: {r:?}");
282    }
283
284    #[test]
285    fn rejects_a_judge_task_with_an_inlined_dispatch_prompt() {
286        // dispatch_prompt is stripped before write; the schema forbids extras.
287        let tasks = json!({
288            "generated": "t", "total_tasks": 1, "meta_tasks_injected": 0,
289            "skipped_transcript_checks": 0,
290            "tasks": [{
291                "eval_id": "e1", "condition": "with_skill", "assertion_id": "a1",
292                "rubric": "r", "model": null, "is_meta": false,
293                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
294                "response_path": "/w/r.json", "dispatch_prompt_path": "/w/p.txt",
295                "dispatch_prompt": "SHOULD NOT BE HERE"
296            }]
297        });
298        let r: Result<Value, _> =
299            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
300        assert!(r.is_err());
301    }
302
303    #[test]
304    fn compiles_and_validates_the_grading_schema_too() {
305        let grading = json!({
306            "assertion_results": [
307                {
308                    "id": "a1",
309                    "passed": true,
310                    "evidence": "quoted output",
311                    "grader": "transcript_check"
312                }
313            ],
314            "summary": { "passed": 1, "failed": 0, "total": 1, "pass_rate": 1.0 }
315        });
316        let r: Result<Value, _> =
317            validate_against_schema(SchemaName::Grading, &grading, "grading.json");
318        assert!(r.is_ok(), "grading should validate");
319    }
320}
eval_magic/validation/schema.rs

eval_magic/validation/
schema.rs
