eval_magic/validation/
evals.rs

1//! High-level `evals.json` validation: structural schema check plus the
2//! hand-rolled constraints draft-07 can't express.
3
4use std::collections::HashSet;
5
6use serde_json::Value;
7
8use crate::core::EvalsConfig;
9use crate::validation::error::ValidationError;
10use crate::validation::schema::{SchemaName, validate_against_schema};
11
12/// Validate a parsed `evals.json`. Runs the structural schema check, then the
13/// supplemental duplicate-`id` guard (uniqueness by a sub-field isn't
14/// expressible in JSON Schema draft-07), returning the typed config on success.
15pub fn validate_evals_config(config: &Value, source: &str) -> Result<EvalsConfig, ValidationError> {
16    let validated: EvalsConfig = validate_against_schema(SchemaName::Evals, config, source)?;
17
18    let mut seen = HashSet::new();
19    for (index, ev) in validated.evals.iter().enumerate() {
20        if !seen.insert(ev.id.as_str()) {
21            return Err(ValidationError::DuplicateId {
22                path: source.to_string(),
23                index,
24                id: ev.id.clone(),
25            });
26        }
27    }
28
29    Ok(validated)
30}
31
32#[cfg(test)]
33mod tests {
34    use super::validate_evals_config;
35    use serde_json::{Value, json};
36
37    /// The minimal valid config the cases below mutate.
38    fn base() -> Value {
39        json!({
40            "skill_name": "demo",
41            "evals": [
42                {
43                    "id": "e1",
44                    "prompt": "do the thing",
45                    "expected_output": "the thing is done"
46                }
47            ]
48        })
49    }
50
51    #[test]
52    fn accepts_a_boolean_skill_should_trigger() {
53        let mut config = base();
54        config["evals"][0]["skill_should_trigger"] = json!(false);
55        let parsed = validate_evals_config(&config, "evals.json").unwrap();
56        assert_eq!(parsed.evals[0].skill_should_trigger, Some(false));
57    }
58
59    #[test]
60    fn accepts_evals_with_no_skill_should_trigger() {
61        let config = base();
62        let parsed = validate_evals_config(&config, "evals.json").unwrap();
63        assert_eq!(parsed.skill_name, "demo");
64        assert_eq!(parsed.evals[0].skill_should_trigger, None);
65    }
66
67    #[test]
68    fn rejects_a_non_boolean_skill_should_trigger() {
69        let mut config = base();
70        config["evals"][0]["skill_should_trigger"] = json!("false");
71        let err = validate_evals_config(&config, "evals.json")
72            .unwrap_err()
73            .to_string();
74        assert!(err.contains("skill_should_trigger"), "error was: {err}");
75    }
76
77    #[test]
78    fn accepts_isolation_isolated() {
79        let mut config = base();
80        config["evals"][0]["isolation"] = json!("isolated");
81        let parsed = validate_evals_config(&config, "evals.json").unwrap();
82        assert_eq!(
83            parsed.evals[0].isolation,
84            Some(crate::core::Isolation::Isolated)
85        );
86    }
87
88    #[test]
89    fn accepts_isolation_shared() {
90        let mut config = base();
91        config["evals"][0]["isolation"] = json!("shared");
92        let parsed = validate_evals_config(&config, "evals.json").unwrap();
93        assert_eq!(
94            parsed.evals[0].isolation,
95            Some(crate::core::Isolation::Shared)
96        );
97    }
98
99    #[test]
100    fn defaults_isolation_to_none_when_absent() {
101        let config = base();
102        let parsed = validate_evals_config(&config, "evals.json").unwrap();
103        assert_eq!(parsed.evals[0].isolation, None);
104    }
105
106    #[test]
107    fn rejects_an_unknown_isolation_value() {
108        let mut config = base();
109        config["evals"][0]["isolation"] = json!("sometimes");
110        let err = validate_evals_config(&config, "evals.json")
111            .unwrap_err()
112            .to_string();
113        assert!(err.contains("isolation"), "error was: {err}");
114    }
115
116    #[test]
117    fn rejects_a_non_kebab_case_id() {
118        let mut config = base();
119        config["evals"][0]["id"] = json!("Not Kebab");
120        assert!(validate_evals_config(&config, "evals.json").is_err());
121    }
122
123    #[test]
124    fn rejects_duplicate_eval_ids() {
125        let mut config = base();
126        let dup = config["evals"][0].clone();
127        config["evals"] = json!([dup.clone(), dup]);
128        let err = validate_evals_config(&config, "evals.json")
129            .unwrap_err()
130            .to_string();
131        assert!(err.contains("duplicate"), "error was: {err}");
132    }
133
134    #[test]
135    fn rejects_an_empty_evals_array() {
136        let mut config = base();
137        config["evals"] = json!([]);
138        assert!(validate_evals_config(&config, "evals.json").is_err());
139    }
140}
eval_magic/validation/evals.rs

eval_magic/validation/
evals.rs
