eval_magic/validation/
schema.rs

1//! Schema embedding + the generic `validate_against_schema` entry point.
2//!
3//! The portable-artifact schemas are the single source of truth for each
4//! artifact's shape. They are embedded at compile time with `include_str!` (so
5//! the binary is self-contained, with no `schema/` directory to ship alongside)
6//! and compiled once into reusable `jsonschema` validators.
7
8use std::collections::HashMap;
9use std::sync::LazyLock;
10
11use jsonschema::Validator;
12use serde::de::DeserializeOwned;
13use serde_json::Value;
14
15use crate::validation::error::ValidationError;
16
17/// Names the portable-artifact schemas. `benchmark` and `judge-tasks` are
18/// first-class here, schema-gated like every other pipeline output.
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
20pub enum SchemaName {
21    RunRecord,
22    Evals,
23    Grading,
24    StrayWrites,
25    GuardDenials,
26    Benchmark,
27    JudgeTasks,
28    CommandCheck,
29    DiffScope,
30    HarnessDescriptor,
31    Conversation,
32}
33
34impl SchemaName {
35    /// Every schema, for building the validator cache.
36    const ALL: [SchemaName; 11] = [
37        SchemaName::RunRecord,
38        SchemaName::Evals,
39        SchemaName::Grading,
40        SchemaName::StrayWrites,
41        SchemaName::GuardDenials,
42        SchemaName::Benchmark,
43        SchemaName::JudgeTasks,
44        SchemaName::CommandCheck,
45        SchemaName::DiffScope,
46        SchemaName::HarnessDescriptor,
47        SchemaName::Conversation,
48    ];
49
50    /// The schema's kebab-case name, as used in error messages and the on-disk
51    /// `<name>.schema.json` filenames.
52    pub fn as_str(self) -> &'static str {
53        match self {
54            SchemaName::RunRecord => "run-record",
55            SchemaName::Evals => "evals",
56            SchemaName::Grading => "grading",
57            SchemaName::StrayWrites => "stray-writes",
58            SchemaName::GuardDenials => "guard-denials",
59            SchemaName::Benchmark => "benchmark",
60            SchemaName::JudgeTasks => "judge-tasks",
61            SchemaName::CommandCheck => "command-check",
62            SchemaName::DiffScope => "diff-scope",
63            SchemaName::HarnessDescriptor => "harness-descriptor",
64            SchemaName::Conversation => "conversation",
65        }
66    }
67
68    /// The embedded schema JSON source.
69    fn source(self) -> &'static str {
70        match self {
71            SchemaName::RunRecord => include_str!("../../schema/run-record.schema.json"),
72            SchemaName::Evals => include_str!("../../schema/evals.schema.json"),
73            SchemaName::Grading => include_str!("../../schema/grading.schema.json"),
74            SchemaName::StrayWrites => include_str!("../../schema/stray-writes.schema.json"),
75            SchemaName::GuardDenials => {
76                include_str!("../../schema/guard-denials.schema.json")
77            }
78            SchemaName::Benchmark => include_str!("../../schema/benchmark.schema.json"),
79            SchemaName::JudgeTasks => include_str!("../../schema/judge-tasks.schema.json"),
80            SchemaName::CommandCheck => {
81                include_str!("../../schema/command-check.schema.json")
82            }
83            SchemaName::DiffScope => include_str!("../../schema/diff-scope.schema.json"),
84            SchemaName::HarnessDescriptor => {
85                include_str!("../../schema/harness-descriptor.schema.json")
86            }
87            SchemaName::Conversation => include_str!("../../schema/conversation.schema.json"),
88        }
89    }
90}
91
92/// Compiled validators, built once on first use. The schemas are embedded and
93/// known-valid, so a failure here is a programmer error (a malformed bundled
94/// schema) and panics rather than being surfaced as a runtime validation error.
95static VALIDATORS: LazyLock<HashMap<SchemaName, Validator>> = LazyLock::new(|| {
96    SchemaName::ALL
97        .iter()
98        .map(|&name| {
99            let schema: Value = serde_json::from_str(name.source()).unwrap_or_else(|e| {
100                panic!("bundled {} schema is not valid JSON: {e}", name.as_str())
101            });
102            let validator = jsonschema::validator_for(&schema).unwrap_or_else(|e| {
103                panic!("bundled {} schema does not compile: {e}", name.as_str())
104            });
105            (name, validator)
106        })
107        .collect()
108});
109
110/// Validate `data` against the named schema. Returns it deserialized into `T` on
111/// success; on mismatch, returns a `source`-prefixed [`ValidationError`] listing
112/// every failure.
113///
114/// Deserializing into `T` makes the typed result honest: the schema gate and
115/// the Rust type agree, or the call fails.
116pub fn validate_against_schema<T: DeserializeOwned>(
117    name: SchemaName,
118    data: &Value,
119    source: &str,
120) -> Result<T, ValidationError> {
121    let validator = &VALIDATORS[&name];
122
123    if !validator.is_valid(data) {
124        let details = validator
125            .iter_errors(data)
126            .map(|e| {
127                let instance = e.instance_path().to_string();
128                let instance = if instance.is_empty() {
129                    "/".to_string()
130                } else {
131                    instance
132                };
133                format!("  {instance} {e}")
134            })
135            .collect::<Vec<_>>()
136            .join("\n");
137        return Err(ValidationError::SchemaMismatch {
138            path: source.to_string(),
139            schema: name.as_str().to_string(),
140            details,
141        });
142    }
143
144    serde_json::from_value(data.clone()).map_err(|e| ValidationError::Deserialize {
145        path: source.to_string(),
146        message: e.to_string(),
147    })
148}
149
150#[cfg(test)]
151mod tests {
152    use super::{SchemaName, validate_against_schema};
153    use serde_json::{Value, json};
154
155    /// The canonical valid run-record the cases below mutate.
156    fn valid_run_record() -> Value {
157        json!({
158            "eval_id": "e1",
159            "condition": "with_skill",
160            "skill_path": null,
161            "prompt": "do the thing",
162            "files": [],
163            "final_message": "done",
164            "tool_invocations": [],
165            "total_tokens": 100,
166            "duration_ms": 1000
167        })
168    }
169
170    #[test]
171    fn returns_data_when_it_matches_the_run_record_schema() {
172        let data = valid_run_record();
173        let out: Value =
174            validate_against_schema(SchemaName::RunRecord, &data, "/tmp/run.json").unwrap();
175        assert_eq!(out, data);
176    }
177
178    #[test]
179    fn accepts_an_empty_tool_invocations_array() {
180        let mut data = valid_run_record();
181        data["tool_invocations"] = json!([]);
182        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
183        assert!(r.is_ok());
184    }
185
186    #[test]
187    fn conversation_tool_results_use_the_portable_run_record_value_shape() {
188        let conversation = json!({
189            "status": "completed",
190            "delivered_followups": 0,
191            "events": [
192                {"type": "user_message", "ordinal": 0, "round": 1, "text": "Fix it."},
193                {
194                    "type": "tool_invocation",
195                    "ordinal": 1,
196                    "round": 1,
197                    "name": "Read",
198                    "result": ["not", "portable"]
199                },
200                {"type": "assistant_message", "ordinal": 2, "round": 1, "text": "Done."}
201            ]
202        });
203
204        let result: Result<Value, _> =
205            validate_against_schema(SchemaName::Conversation, &conversation, "conversation.json");
206
207        assert!(result.is_err());
208    }
209
210    #[test]
211    fn accepts_skill_path_null_on_the_without_skill_arm() {
212        let mut data = valid_run_record();
213        data["condition"] = json!("without_skill");
214        data["skill_path"] = Value::Null;
215        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
216        assert!(r.is_ok());
217    }
218
219    #[test]
220    fn source_prefixed_error_when_required_field_missing() {
221        let mut data = valid_run_record();
222        data.as_object_mut().unwrap().remove("eval_id");
223        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "/tmp/run.json")
224            .unwrap_err()
225            .to_string();
226        assert!(err.contains("/tmp/run.json"), "error was: {err}");
227    }
228
229    #[test]
230    fn requires_skill_path() {
231        let mut data = valid_run_record();
232        data.as_object_mut().unwrap().remove("skill_path");
233        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
234            .unwrap_err()
235            .to_string();
236        assert!(err.contains("skill_path"), "error was: {err}");
237    }
238
239    #[test]
240    fn requires_files() {
241        let mut data = valid_run_record();
242        data.as_object_mut().unwrap().remove("files");
243        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
244            .unwrap_err()
245            .to_string();
246        assert!(err.contains("files"), "error was: {err}");
247    }
248
249    #[test]
250    fn rejects_unknown_extra_property() {
251        let mut data = valid_run_record();
252        data["surprise"] = json!(true);
253        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
254        assert!(r.is_err());
255    }
256
257    #[test]
258    fn grading_schema_accepts_command_check_grader() {
259        let data = json!({
260            "assertion_results": [{
261                "id": "tests-pass",
262                "passed": true,
263                "evidence": "exit code matched",
264                "confidence": 1.0,
265                "grader": "command_check"
266            }],
267            "summary": {
268                "passed": 1,
269                "failed": 0,
270                "total": 1,
271                "pass_rate": 1.0
272            }
273        });
274        let result: Result<Value, _> =
275            validate_against_schema(SchemaName::Grading, &data, "grading.json");
276        assert!(
277            result.is_ok(),
278            "command_check grading should validate: {result:?}"
279        );
280    }
281
282    #[test]
283    fn validates_diff_scope_metrics_and_rejects_extra_fields() {
284        let metrics = json!({
285            "files_touched": 2,
286            "lines_added": 4,
287            "lines_removed": 1,
288            "hunks": 2
289        });
290        let valid: Result<Value, _> =
291            validate_against_schema(SchemaName::DiffScope, &metrics, "diff-scope.json");
292        assert!(valid.is_ok(), "{valid:?}");
293
294        let mut extra = metrics;
295        extra["paths"] = json!(["src/main.rs"]);
296        let invalid: Result<Value, _> =
297            validate_against_schema(SchemaName::DiffScope, &extra, "diff-scope.json");
298        assert!(invalid.is_err());
299    }
300
301    #[test]
302    fn validates_guard_denials_and_rejects_unexpected_payload_content() {
303        let report = serde_json::json!({
304            "generated": "2026-07-26T12:00:00.000Z",
305            "iteration": 1,
306            "total_denials": 1,
307            "tasks": [{
308                "eval_id": "e1",
309                "condition": "old_skill",
310                "run_index": 2,
311                "denial_count": 1,
312                "denials": [{
313                    "timestamp": "2026-07-26T11:59:00.000Z",
314                    "harness": "codex",
315                    "tool": "Bash",
316                    "reason": "outside redirect",
317                    "resolved_targets": ["/etc/out"],
318                    "input_keys": ["command"],
319                }],
320            }],
321        });
322        validate_against_schema::<serde_json::Value>(
323            SchemaName::GuardDenials,
324            &report,
325            "guard-denials.json",
326        )
327        .unwrap();
328
329        let mut extra = report;
330        extra["tasks"][0]["denials"][0]["command"] = serde_json::json!("secret");
331        assert!(
332            validate_against_schema::<serde_json::Value>(
333                SchemaName::GuardDenials,
334                &extra,
335                "guard-denials.json",
336            )
337            .is_err()
338        );
339    }
340
341    #[test]
342    fn tool_invocation_ordinal_must_be_an_integer() {
343        let mut data = valid_run_record();
344        data["tool_invocations"] = json!([{ "name": "Bash", "ordinal": "zero" }]);
345        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
346        assert!(r.is_err());
347    }
348
349    #[test]
350    fn validates_a_well_formed_benchmark() {
351        let benchmark = json!({
352            "generated": "2026-06-08T00:00:00.000Z",
353            "mode": "new-skill",
354            "conditions_compared": ["with_skill", "without_skill"],
355            "missing_gradings": 0,
356            "validity_warnings": [],
357            "run_summary": {
358                "with_skill": {
359                    "pass_rate": { "mean": 1.0, "stddev": 0.0, "n": 1 },
360                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
361                    "total_tokens": { "mean": 5000.0, "stddev": 0.0, "n": 1 },
362                    "skill_invocation_n": 0,
363                    "skill_invocation_rate": null
364                },
365                "without_skill": {
366                    "pass_rate": { "mean": 0.0, "stddev": 0.0, "n": 1 },
367                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
368                    "total_tokens": { "mean": 3000.0, "stddev": 0.0, "n": 1 }
369                }
370            },
371            "delta": { "direction": "with_skill - without_skill", "pass_rate": 1.0, "duration_ms": 0.0, "total_tokens": 2000.0 }
372        });
373        let r: Result<Value, _> =
374            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
375        assert!(r.is_ok(), "benchmark should validate: {r:?}");
376    }
377
378    #[test]
379    fn rejects_a_benchmark_missing_delta() {
380        let mut benchmark = json!({
381            "generated": "t", "mode": "new-skill",
382            "conditions_compared": ["a", "b"], "missing_gradings": 0,
383            "validity_warnings": [], "run_summary": {},
384            "delta": { "direction": "a - b", "pass_rate": 0.0, "duration_ms": 0.0, "total_tokens": 0.0 }
385        });
386        benchmark.as_object_mut().unwrap().remove("delta");
387        let r: Result<Value, _> =
388            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
389        assert!(r.is_err());
390    }
391
392    #[test]
393    fn validates_a_well_formed_judge_tasks_file() {
394        let tasks = json!({
395            "generated": "2026-06-08T00:00:00.000Z",
396            "total_tasks": 1,
397            "meta_tasks_injected": 1,
398            "skipped_transcript_checks": 0,
399            "tasks": [{
400                "eval_id": "e1", "condition": "with_skill", "assertion_id": "__skill_invoked",
401                "rubric": "did it apply the skill?", "model": null, "is_meta": true,
402                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
403                "response_path": "/w/judge-responses/__skill_invoked.json",
404                "dispatch_prompt_path": "/w/judge-prompts/__skill_invoked.txt"
405            }]
406        });
407        let r: Result<Value, _> =
408            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
409        assert!(r.is_ok(), "judge-tasks should validate: {r:?}");
410    }
411
412    #[test]
413    fn rejects_a_judge_task_with_an_inlined_dispatch_prompt() {
414        // dispatch_prompt is stripped before write; the schema forbids extras.
415        let tasks = json!({
416            "generated": "t", "total_tasks": 1, "meta_tasks_injected": 0,
417            "skipped_transcript_checks": 0,
418            "tasks": [{
419                "eval_id": "e1", "condition": "with_skill", "assertion_id": "a1",
420                "rubric": "r", "model": null, "is_meta": false,
421                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
422                "response_path": "/w/r.json", "dispatch_prompt_path": "/w/p.txt",
423                "dispatch_prompt": "SHOULD NOT BE HERE"
424            }]
425        });
426        let r: Result<Value, _> =
427            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
428        assert!(r.is_err());
429    }
430
431    #[test]
432    fn compiles_and_validates_the_grading_schema_too() {
433        let grading = json!({
434            "assertion_results": [
435                {
436                    "id": "a1",
437                    "passed": true,
438                    "evidence": "quoted output",
439                    "grader": "transcript_check"
440                }
441            ],
442            "summary": { "passed": 1, "failed": 0, "total": 1, "pass_rate": 1.0 }
443        });
444        let r: Result<Value, _> =
445            validate_against_schema(SchemaName::Grading, &grading, "grading.json");
446        assert!(r.is_ok(), "grading should validate");
447    }
448}
eval_magic/validation/schema.rs

eval_magic/validation/
schema.rs
