eval_magic/validation/
schema.rs

1//! Schema embedding + the generic `validate_against_schema` entry point.
2//!
3//! The portable-artifact schemas are the single source of truth for each
4//! artifact's shape. They are embedded at compile time with `include_str!` (so
5//! the binary is self-contained, with no `schema/` directory to ship alongside)
6//! and compiled once into reusable `jsonschema` validators.
7
8use std::collections::HashMap;
9use std::sync::LazyLock;
10
11use jsonschema::Validator;
12use serde::de::DeserializeOwned;
13use serde_json::Value;
14
15use crate::validation::error::ValidationError;
16
17/// Names the portable-artifact schemas. `benchmark` and `judge-tasks` are
18/// first-class here, schema-gated like every other pipeline output.
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
20pub enum SchemaName {
21    RunRecord,
22    Evals,
23    Grading,
24    StrayWrites,
25    Benchmark,
26    JudgeTasks,
27}
28
29impl SchemaName {
30    /// Every schema, for building the validator cache.
31    const ALL: [SchemaName; 6] = [
32        SchemaName::RunRecord,
33        SchemaName::Evals,
34        SchemaName::Grading,
35        SchemaName::StrayWrites,
36        SchemaName::Benchmark,
37        SchemaName::JudgeTasks,
38    ];
39
40    /// The schema's kebab-case name, as used in error messages and the on-disk
41    /// `<name>.schema.json` filenames.
42    pub fn as_str(self) -> &'static str {
43        match self {
44            SchemaName::RunRecord => "run-record",
45            SchemaName::Evals => "evals",
46            SchemaName::Grading => "grading",
47            SchemaName::StrayWrites => "stray-writes",
48            SchemaName::Benchmark => "benchmark",
49            SchemaName::JudgeTasks => "judge-tasks",
50        }
51    }
52
53    /// The embedded schema JSON source.
54    fn source(self) -> &'static str {
55        match self {
56            SchemaName::RunRecord => include_str!("../../schema/run-record.schema.json"),
57            SchemaName::Evals => include_str!("../../schema/evals.schema.json"),
58            SchemaName::Grading => include_str!("../../schema/grading.schema.json"),
59            SchemaName::StrayWrites => include_str!("../../schema/stray-writes.schema.json"),
60            SchemaName::Benchmark => include_str!("../../schema/benchmark.schema.json"),
61            SchemaName::JudgeTasks => include_str!("../../schema/judge-tasks.schema.json"),
62        }
63    }
64}
65
66/// Compiled validators, built once on first use. The schemas are embedded and
67/// known-valid, so a failure here is a programmer error (a malformed bundled
68/// schema) and panics rather than being surfaced as a runtime validation error.
69static VALIDATORS: LazyLock<HashMap<SchemaName, Validator>> = LazyLock::new(|| {
70    SchemaName::ALL
71        .iter()
72        .map(|&name| {
73            let schema: Value = serde_json::from_str(name.source()).unwrap_or_else(|e| {
74                panic!("bundled {} schema is not valid JSON: {e}", name.as_str())
75            });
76            let validator = jsonschema::validator_for(&schema).unwrap_or_else(|e| {
77                panic!("bundled {} schema does not compile: {e}", name.as_str())
78            });
79            (name, validator)
80        })
81        .collect()
82});
83
84/// Validate `data` against the named schema. Returns it deserialized into `T` on
85/// success; on mismatch, returns a `source`-prefixed [`ValidationError`] listing
86/// every failure.
87///
88/// Deserializing into `T` makes the typed result honest: the schema gate and
89/// the Rust type agree, or the call fails.
90pub fn validate_against_schema<T: DeserializeOwned>(
91    name: SchemaName,
92    data: &Value,
93    source: &str,
94) -> Result<T, ValidationError> {
95    let validator = &VALIDATORS[&name];
96
97    if !validator.is_valid(data) {
98        let details = validator
99            .iter_errors(data)
100            .map(|e| {
101                let instance = e.instance_path().to_string();
102                let instance = if instance.is_empty() {
103                    "/".to_string()
104                } else {
105                    instance
106                };
107                format!("  {instance} {e}")
108            })
109            .collect::<Vec<_>>()
110            .join("\n");
111        return Err(ValidationError::SchemaMismatch {
112            path: source.to_string(),
113            schema: name.as_str().to_string(),
114            details,
115        });
116    }
117
118    serde_json::from_value(data.clone()).map_err(|e| ValidationError::Deserialize {
119        path: source.to_string(),
120        message: e.to_string(),
121    })
122}
123
124#[cfg(test)]
125mod tests {
126    use super::{SchemaName, validate_against_schema};
127    use serde_json::{Value, json};
128
129    /// The canonical valid run-record the cases below mutate.
130    fn valid_run_record() -> Value {
131        json!({
132            "eval_id": "e1",
133            "condition": "with_skill",
134            "skill_path": null,
135            "prompt": "do the thing",
136            "files": [],
137            "final_message": "done",
138            "tool_invocations": [],
139            "total_tokens": 100,
140            "duration_ms": 1000
141        })
142    }
143
144    #[test]
145    fn returns_data_when_it_matches_the_run_record_schema() {
146        let data = valid_run_record();
147        let out: Value =
148            validate_against_schema(SchemaName::RunRecord, &data, "/tmp/run.json").unwrap();
149        assert_eq!(out, data);
150    }
151
152    #[test]
153    fn accepts_an_empty_tool_invocations_array() {
154        let mut data = valid_run_record();
155        data["tool_invocations"] = json!([]);
156        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
157        assert!(r.is_ok());
158    }
159
160    #[test]
161    fn accepts_skill_path_null_on_the_without_skill_arm() {
162        let mut data = valid_run_record();
163        data["condition"] = json!("without_skill");
164        data["skill_path"] = Value::Null;
165        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
166        assert!(r.is_ok());
167    }
168
169    #[test]
170    fn source_prefixed_error_when_required_field_missing() {
171        let mut data = valid_run_record();
172        data.as_object_mut().unwrap().remove("eval_id");
173        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "/tmp/run.json")
174            .unwrap_err()
175            .to_string();
176        assert!(err.contains("/tmp/run.json"), "error was: {err}");
177    }
178
179    #[test]
180    fn requires_skill_path() {
181        let mut data = valid_run_record();
182        data.as_object_mut().unwrap().remove("skill_path");
183        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
184            .unwrap_err()
185            .to_string();
186        assert!(err.contains("skill_path"), "error was: {err}");
187    }
188
189    #[test]
190    fn requires_files() {
191        let mut data = valid_run_record();
192        data.as_object_mut().unwrap().remove("files");
193        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
194            .unwrap_err()
195            .to_string();
196        assert!(err.contains("files"), "error was: {err}");
197    }
198
199    #[test]
200    fn rejects_unknown_extra_property() {
201        let mut data = valid_run_record();
202        data["surprise"] = json!(true);
203        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
204        assert!(r.is_err());
205    }
206
207    #[test]
208    fn tool_invocation_ordinal_must_be_an_integer() {
209        let mut data = valid_run_record();
210        data["tool_invocations"] = json!([{ "name": "Bash", "ordinal": "zero" }]);
211        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
212        assert!(r.is_err());
213    }
214
215    #[test]
216    fn validates_a_well_formed_benchmark() {
217        let benchmark = json!({
218            "generated": "2026-06-08T00:00:00.000Z",
219            "mode": "new-skill",
220            "conditions_compared": ["with_skill", "without_skill"],
221            "missing_gradings": 0,
222            "validity_warnings": [],
223            "run_summary": {
224                "with_skill": {
225                    "pass_rate": { "mean": 1.0, "stddev": 0.0, "n": 1 },
226                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
227                    "total_tokens": { "mean": 5000.0, "stddev": 0.0, "n": 1 },
228                    "skill_invocation_n": 0,
229                    "skill_invocation_rate": null
230                },
231                "without_skill": {
232                    "pass_rate": { "mean": 0.0, "stddev": 0.0, "n": 1 },
233                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
234                    "total_tokens": { "mean": 3000.0, "stddev": 0.0, "n": 1 }
235                }
236            },
237            "delta": { "direction": "with_skill - without_skill", "pass_rate": 1.0, "duration_ms": 0.0, "total_tokens": 2000.0 }
238        });
239        let r: Result<Value, _> =
240            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
241        assert!(r.is_ok(), "benchmark should validate: {r:?}");
242    }
243
244    #[test]
245    fn rejects_a_benchmark_missing_delta() {
246        let mut benchmark = json!({
247            "generated": "t", "mode": "new-skill",
248            "conditions_compared": ["a", "b"], "missing_gradings": 0,
249            "validity_warnings": [], "run_summary": {},
250            "delta": { "direction": "a - b", "pass_rate": 0.0, "duration_ms": 0.0, "total_tokens": 0.0 }
251        });
252        benchmark.as_object_mut().unwrap().remove("delta");
253        let r: Result<Value, _> =
254            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
255        assert!(r.is_err());
256    }
257
258    #[test]
259    fn validates_a_well_formed_judge_tasks_file() {
260        let tasks = json!({
261            "generated": "2026-06-08T00:00:00.000Z",
262            "total_tasks": 1,
263            "meta_tasks_injected": 1,
264            "skipped_transcript_checks": 0,
265            "tasks": [{
266                "eval_id": "e1", "condition": "with_skill", "assertion_id": "__skill_invoked",
267                "rubric": "did it apply the skill?", "model": null, "is_meta": true,
268                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
269                "response_path": "/w/judge-responses/__skill_invoked.json",
270                "dispatch_prompt_path": "/w/judge-prompts/__skill_invoked.txt"
271            }]
272        });
273        let r: Result<Value, _> =
274            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
275        assert!(r.is_ok(), "judge-tasks should validate: {r:?}");
276    }
277
278    #[test]
279    fn rejects_a_judge_task_with_an_inlined_dispatch_prompt() {
280        // dispatch_prompt is stripped before write; the schema forbids extras.
281        let tasks = json!({
282            "generated": "t", "total_tasks": 1, "meta_tasks_injected": 0,
283            "skipped_transcript_checks": 0,
284            "tasks": [{
285                "eval_id": "e1", "condition": "with_skill", "assertion_id": "a1",
286                "rubric": "r", "model": null, "is_meta": false,
287                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
288                "response_path": "/w/r.json", "dispatch_prompt_path": "/w/p.txt",
289                "dispatch_prompt": "SHOULD NOT BE HERE"
290            }]
291        });
292        let r: Result<Value, _> =
293            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
294        assert!(r.is_err());
295    }
296
297    #[test]
298    fn compiles_and_validates_the_grading_schema_too() {
299        let grading = json!({
300            "assertion_results": [
301                {
302                    "id": "a1",
303                    "passed": true,
304                    "evidence": "quoted output",
305                    "grader": "transcript_check"
306                }
307            ],
308            "summary": { "passed": 1, "failed": 0, "total": 1, "pass_rate": 1.0 }
309        });
310        let r: Result<Value, _> =
311            validate_against_schema(SchemaName::Grading, &grading, "grading.json");
312        assert!(r.is_ok(), "grading should validate");
313    }
314}
eval_magic/validation/schema.rs

eval_magic/validation/
schema.rs
