eval_magic/validation/

schema.rs

//! Schema embedding + the generic `validate_against_schema` entry point.
//!
//! The portable-artifact schemas are the single source of truth for each
//! artifact's shape. They are embedded at compile time with `include_str!` (so
//! the binary is self-contained, with no `schema/` directory to ship alongside)
//! and compiled once into reusable `jsonschema` validators.

use std::collections::HashMap;
use std::sync::LazyLock;

use jsonschema::Validator;
use serde::de::DeserializeOwned;
use serde_json::Value;

use crate::validation::error::ValidationError;

/// Names the portable-artifact schemas. `benchmark` and `judge-tasks` are
/// first-class here, schema-gated like every other pipeline output.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SchemaName {
    RunRecord,
    Evals,
    Grading,
    StrayWrites,
    Benchmark,
    JudgeTasks,
    CommandCheck,
    DiffScope,
    HarnessDescriptor,
    Conversation,
}

impl SchemaName {
    /// Every schema, for building the validator cache.
    const ALL: [SchemaName; 10] = [
        SchemaName::RunRecord,
        SchemaName::Evals,
        SchemaName::Grading,
        SchemaName::StrayWrites,
        SchemaName::Benchmark,
        SchemaName::JudgeTasks,
        SchemaName::CommandCheck,
        SchemaName::DiffScope,
        SchemaName::HarnessDescriptor,
        SchemaName::Conversation,
    ];

    /// The schema's kebab-case name, as used in error messages and the on-disk
    /// `<name>.schema.json` filenames.
    pub fn as_str(self) -> &'static str {
        match self {
            SchemaName::RunRecord => "run-record",
            SchemaName::Evals => "evals",
            SchemaName::Grading => "grading",
            SchemaName::StrayWrites => "stray-writes",
            SchemaName::Benchmark => "benchmark",
            SchemaName::JudgeTasks => "judge-tasks",
            SchemaName::CommandCheck => "command-check",
            SchemaName::DiffScope => "diff-scope",
            SchemaName::HarnessDescriptor => "harness-descriptor",
            SchemaName::Conversation => "conversation",
        }
    }

    /// The embedded schema JSON source.
    fn source(self) -> &'static str {
        match self {
            SchemaName::RunRecord => include_str!("../../schema/run-record.schema.json"),
            SchemaName::Evals => include_str!("../../schema/evals.schema.json"),
            SchemaName::Grading => include_str!("../../schema/grading.schema.json"),
            SchemaName::StrayWrites => include_str!("../../schema/stray-writes.schema.json"),
            SchemaName::Benchmark => include_str!("../../schema/benchmark.schema.json"),
            SchemaName::JudgeTasks => include_str!("../../schema/judge-tasks.schema.json"),
            SchemaName::CommandCheck => {
                include_str!("../../schema/command-check.schema.json")
            }
            SchemaName::DiffScope => include_str!("../../schema/diff-scope.schema.json"),
            SchemaName::HarnessDescriptor => {
                include_str!("../../schema/harness-descriptor.schema.json")
            }
            SchemaName::Conversation => include_str!("../../schema/conversation.schema.json"),
        }
    }
}

/// Compiled validators, built once on first use. The schemas are embedded and
/// known-valid, so a failure here is a programmer error (a malformed bundled
/// schema) and panics rather than being surfaced as a runtime validation error.
static VALIDATORS: LazyLock<HashMap<SchemaName, Validator>> = LazyLock::new(|| {
    SchemaName::ALL
        .iter()
        .map(|&name| {
            let schema: Value = serde_json::from_str(name.source()).unwrap_or_else(|e| {
                panic!("bundled {} schema is not valid JSON: {e}", name.as_str())
            });
            let validator = jsonschema::validator_for(&schema).unwrap_or_else(|e| {
                panic!("bundled {} schema does not compile: {e}", name.as_str())
            });
            (name, validator)
        })
        .collect()
});

/// Validate `data` against the named schema. Returns it deserialized into `T` on
/// success; on mismatch, returns a `source`-prefixed [`ValidationError`] listing
/// every failure.
///
/// Deserializing into `T` makes the typed result honest: the schema gate and
/// the Rust type agree, or the call fails.
pub fn validate_against_schema<T: DeserializeOwned>(
    name: SchemaName,
    data: &Value,
    source: &str,
) -> Result<T, ValidationError> {
    let validator = &VALIDATORS[&name];

    if !validator.is_valid(data) {
        let details = validator
            .iter_errors(data)
            .map(|e| {
                let instance = e.instance_path().to_string();
                let instance = if instance.is_empty() {
                    "/".to_string()
                } else {
                    instance
                };
                format!("  {instance} {e}")
            })
            .collect::<Vec<_>>()
            .join("\n");
        return Err(ValidationError::SchemaMismatch {
            path: source.to_string(),
            schema: name.as_str().to_string(),
            details,
        });
    }

    serde_json::from_value(data.clone()).map_err(|e| ValidationError::Deserialize {
        path: source.to_string(),
        message: e.to_string(),
    })
}

#[cfg(test)]
mod tests {
    use super::{SchemaName, validate_against_schema};
    use serde_json::{Value, json};

    /// The canonical valid run-record the cases below mutate.
    fn valid_run_record() -> Value {
        json!({
            "eval_id": "e1",
            "condition": "with_skill",
            "skill_path": null,
            "prompt": "do the thing",
            "files": [],
            "final_message": "done",
            "tool_invocations": [],
            "total_tokens": 100,
            "duration_ms": 1000
        })
    }

    #[test]
    fn returns_data_when_it_matches_the_run_record_schema() {
        let data = valid_run_record();
        let out: Value =
            validate_against_schema(SchemaName::RunRecord, &data, "/tmp/run.json").unwrap();
        assert_eq!(out, data);
    }

    #[test]
    fn accepts_an_empty_tool_invocations_array() {
        let mut data = valid_run_record();
        data["tool_invocations"] = json!([]);
        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
        assert!(r.is_ok());
    }

    #[test]
    fn conversation_tool_results_use_the_portable_run_record_value_shape() {
        let conversation = json!({
            "status": "completed",
            "delivered_followups": 0,
            "events": [
                {"type": "user_message", "ordinal": 0, "round": 1, "text": "Fix it."},
                {
                    "type": "tool_invocation",
                    "ordinal": 1,
                    "round": 1,
                    "name": "Read",
                    "result": ["not", "portable"]
                },
                {"type": "assistant_message", "ordinal": 2, "round": 1, "text": "Done."}
            ]
        });

        let result: Result<Value, _> =
            validate_against_schema(SchemaName::Conversation, &conversation, "conversation.json");

        assert!(result.is_err());
    }

    #[test]
    fn accepts_skill_path_null_on_the_without_skill_arm() {
        let mut data = valid_run_record();
        data["condition"] = json!("without_skill");
        data["skill_path"] = Value::Null;
        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
        assert!(r.is_ok());
    }

    #[test]
    fn source_prefixed_error_when_required_field_missing() {
        let mut data = valid_run_record();
        data.as_object_mut().unwrap().remove("eval_id");
        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "/tmp/run.json")
            .unwrap_err()
            .to_string();
        assert!(err.contains("/tmp/run.json"), "error was: {err}");
    }

    #[test]
    fn requires_skill_path() {
        let mut data = valid_run_record();
        data.as_object_mut().unwrap().remove("skill_path");
        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
            .unwrap_err()
            .to_string();
        assert!(err.contains("skill_path"), "error was: {err}");
    }

    #[test]
    fn requires_files() {
        let mut data = valid_run_record();
        data.as_object_mut().unwrap().remove("files");
        let err = validate_against_schema::<Value>(SchemaName::RunRecord, &data, "run.json")
            .unwrap_err()
            .to_string();
        assert!(err.contains("files"), "error was: {err}");
    }

    #[test]
    fn rejects_unknown_extra_property() {
        let mut data = valid_run_record();
        data["surprise"] = json!(true);
        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
        assert!(r.is_err());
    }

    #[test]
    fn grading_schema_accepts_command_check_grader() {
        let data = json!({
            "assertion_results": [{
                "id": "tests-pass",
                "passed": true,
                "evidence": "exit code matched",
                "confidence": 1.0,
                "grader": "command_check"
            }],
            "summary": {
                "passed": 1,
                "failed": 0,
                "total": 1,
                "pass_rate": 1.0
            }
        });
        let result: Result<Value, _> =
            validate_against_schema(SchemaName::Grading, &data, "grading.json");
        assert!(
            result.is_ok(),
            "command_check grading should validate: {result:?}"
        );
    }

    #[test]
    fn validates_diff_scope_metrics_and_rejects_extra_fields() {
        let metrics = json!({
            "files_touched": 2,
            "lines_added": 4,
            "lines_removed": 1,
            "hunks": 2
        });
        let valid: Result<Value, _> =
            validate_against_schema(SchemaName::DiffScope, &metrics, "diff-scope.json");
        assert!(valid.is_ok(), "{valid:?}");

        let mut extra = metrics;
        extra["paths"] = json!(["src/main.rs"]);
        let invalid: Result<Value, _> =
            validate_against_schema(SchemaName::DiffScope, &extra, "diff-scope.json");
        assert!(invalid.is_err());
    }

    #[test]
    fn tool_invocation_ordinal_must_be_an_integer() {
        let mut data = valid_run_record();
        data["tool_invocations"] = json!([{ "name": "Bash", "ordinal": "zero" }]);
        let r: Result<Value, _> = validate_against_schema(SchemaName::RunRecord, &data, "run.json");
        assert!(r.is_err());
    }

    #[test]
    fn validates_a_well_formed_benchmark() {
        let benchmark = json!({
            "generated": "2026-06-08T00:00:00.000Z",
            "mode": "new-skill",
            "conditions_compared": ["with_skill", "without_skill"],
            "missing_gradings": 0,
            "validity_warnings": [],
            "run_summary": {
                "with_skill": {
                    "pass_rate": { "mean": 1.0, "stddev": 0.0, "n": 1 },
                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
                    "total_tokens": { "mean": 5000.0, "stddev": 0.0, "n": 1 },
                    "skill_invocation_n": 0,
                    "skill_invocation_rate": null
                },
                "without_skill": {
                    "pass_rate": { "mean": 0.0, "stddev": 0.0, "n": 1 },
                    "duration_ms": { "mean": 1000.0, "stddev": 0.0, "n": 1 },
                    "total_tokens": { "mean": 3000.0, "stddev": 0.0, "n": 1 }
                }
            },
            "delta": { "direction": "with_skill - without_skill", "pass_rate": 1.0, "duration_ms": 0.0, "total_tokens": 2000.0 }
        });
        let r: Result<Value, _> =
            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
        assert!(r.is_ok(), "benchmark should validate: {r:?}");
    }

    #[test]
    fn rejects_a_benchmark_missing_delta() {
        let mut benchmark = json!({
            "generated": "t", "mode": "new-skill",
            "conditions_compared": ["a", "b"], "missing_gradings": 0,
            "validity_warnings": [], "run_summary": {},
            "delta": { "direction": "a - b", "pass_rate": 0.0, "duration_ms": 0.0, "total_tokens": 0.0 }
        });
        benchmark.as_object_mut().unwrap().remove("delta");
        let r: Result<Value, _> =
            validate_against_schema(SchemaName::Benchmark, &benchmark, "benchmark.json");
        assert!(r.is_err());
    }

    #[test]
    fn validates_a_well_formed_judge_tasks_file() {
        let tasks = json!({
            "generated": "2026-06-08T00:00:00.000Z",
            "total_tasks": 1,
            "meta_tasks_injected": 1,
            "skipped_transcript_checks": 0,
            "tasks": [{
                "eval_id": "e1", "condition": "with_skill", "assertion_id": "__skill_invoked",
                "rubric": "did it apply the skill?", "model": null, "is_meta": true,
                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
                "response_path": "/w/judge-responses/__skill_invoked.json",
                "dispatch_prompt_path": "/w/judge-prompts/__skill_invoked.txt"
            }]
        });
        let r: Result<Value, _> =
            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
        assert!(r.is_ok(), "judge-tasks should validate: {r:?}");
    }

    #[test]
    fn rejects_a_judge_task_with_an_inlined_dispatch_prompt() {
        // dispatch_prompt is stripped before write; the schema forbids extras.
        let tasks = json!({
            "generated": "t", "total_tasks": 1, "meta_tasks_injected": 0,
            "skipped_transcript_checks": 0,
            "tasks": [{
                "eval_id": "e1", "condition": "with_skill", "assertion_id": "a1",
                "rubric": "r", "model": null, "is_meta": false,
                "run_record_path": "/w/run.json", "outputs_dir": "/w/outputs",
                "response_path": "/w/r.json", "dispatch_prompt_path": "/w/p.txt",
                "dispatch_prompt": "SHOULD NOT BE HERE"
            }]
        });
        let r: Result<Value, _> =
            validate_against_schema(SchemaName::JudgeTasks, &tasks, "judge-tasks.json");
        assert!(r.is_err());
    }

    #[test]
    fn compiles_and_validates_the_grading_schema_too() {
        let grading = json!({
            "assertion_results": [
                {
                    "id": "a1",
                    "passed": true,
                    "evidence": "quoted output",
                    "grader": "transcript_check"
                }
            ],
            "summary": { "passed": 1, "failed": 0, "total": 1, "pass_rate": 1.0 }
        });
        let r: Result<Value, _> =
            validate_against_schema(SchemaName::Grading, &grading, "grading.json");
        assert!(r.is_ok(), "grading should validate");
    }
}