eval_magic/validation/
evals.rs

1//! High-level `evals.json` validation: structural schema check plus the
2//! hand-rolled constraints draft-07 can't express.
3
4use std::collections::HashSet;
5
6use serde_json::Value;
7
8use crate::core::EvalsConfig;
9use crate::validation::error::ValidationError;
10use crate::validation::schema::{SchemaName, validate_against_schema};
11
12/// Validate a parsed `evals.json`. Runs the structural schema check, then the
13/// supplemental duplicate-`id` guard (uniqueness by a sub-field isn't
14/// expressible in JSON Schema draft-07), returning the typed config on success.
15pub fn validate_evals_config(config: &Value, source: &str) -> Result<EvalsConfig, ValidationError> {
16    let validated: EvalsConfig = validate_against_schema(SchemaName::Evals, config, source)?;
17
18    let mut seen = HashSet::new();
19    for (index, ev) in validated.evals.iter().enumerate() {
20        if !seen.insert(ev.id.as_str()) {
21            return Err(ValidationError::DuplicateId {
22                path: source.to_string(),
23                index,
24                id: ev.id.clone(),
25            });
26        }
27    }
28
29    Ok(validated)
30}
31
32#[cfg(test)]
33mod tests {
34    use super::validate_evals_config;
35    use serde_json::{Value, json};
36
37    /// The minimal valid config the cases below mutate.
38    fn base() -> Value {
39        json!({
40            "skill_name": "demo",
41            "evals": [
42                {
43                    "id": "e1",
44                    "prompt": "do the thing",
45                    "expected_output": "the thing is done"
46                }
47            ]
48        })
49    }
50
51    #[test]
52    fn accepts_a_boolean_skill_should_trigger() {
53        let mut config = base();
54        config["evals"][0]["skill_should_trigger"] = json!(false);
55        let parsed = validate_evals_config(&config, "evals.json").unwrap();
56        assert_eq!(parsed.evals[0].skill_should_trigger, Some(false));
57    }
58
59    #[test]
60    fn accepts_evals_with_no_skill_should_trigger() {
61        let config = base();
62        let parsed = validate_evals_config(&config, "evals.json").unwrap();
63        assert_eq!(parsed.skill_name, "demo");
64        assert_eq!(parsed.evals[0].skill_should_trigger, None);
65    }
66
67    #[test]
68    fn rejects_a_non_boolean_skill_should_trigger() {
69        let mut config = base();
70        config["evals"][0]["skill_should_trigger"] = json!("false");
71        let err = validate_evals_config(&config, "evals.json")
72            .unwrap_err()
73            .to_string();
74        assert!(err.contains("skill_should_trigger"), "error was: {err}");
75    }
76
77    #[test]
78    fn rejects_a_non_kebab_case_id() {
79        let mut config = base();
80        config["evals"][0]["id"] = json!("Not Kebab");
81        assert!(validate_evals_config(&config, "evals.json").is_err());
82    }
83
84    #[test]
85    fn rejects_duplicate_eval_ids() {
86        let mut config = base();
87        let dup = config["evals"][0].clone();
88        config["evals"] = json!([dup.clone(), dup]);
89        let err = validate_evals_config(&config, "evals.json")
90            .unwrap_err()
91            .to_string();
92        assert!(err.contains("duplicate"), "error was: {err}");
93    }
94
95    #[test]
96    fn rejects_an_empty_evals_array() {
97        let mut config = base();
98        config["evals"] = json!([]);
99        assert!(validate_evals_config(&config, "evals.json").is_err());
100    }
101}
eval_magic/validation/evals.rs

eval_magic/validation/
evals.rs
