eval_magic/validation/
evals.rs

1//! High-level `evals.json` validation: structural schema check plus the
2//! hand-rolled constraints draft-07 can't express.
3
4use std::collections::HashSet;
5use std::path::{Component, Path, PathBuf};
6
7use regex::Regex;
8use serde_json::Value;
9
10use crate::core::{Assertion, DeliverWhen, EvalsConfig};
11use crate::validation::error::ValidationError;
12use crate::validation::schema::{SchemaName, validate_against_schema};
13
14/// Validate a parsed `evals.json`. Runs the structural schema check, then the
15/// supplemental duplicate-`id`, command environment, and held-out path guards,
16/// returning the typed config on success.
17pub fn validate_evals_config(config: &Value, source: &str) -> Result<EvalsConfig, ValidationError> {
18    let validated: EvalsConfig = validate_against_schema(SchemaName::Evals, config, source)?;
19
20    let mut seen = HashSet::new();
21    for (index, ev) in validated.evals.iter().enumerate() {
22        if !seen.insert(ev.id.as_str()) {
23            return Err(ValidationError::DuplicateId {
24                path: source.to_string(),
25                index,
26                id: ev.id.clone(),
27            });
28        }
29
30        for (turn_index, turn) in ev.turns.as_deref().unwrap_or(&[]).iter().enumerate() {
31            if turn.prompt.trim().is_empty() {
32                return Err(ValidationError::InvalidConfig {
33                    path: source.to_string(),
34                    message: format!(
35                        "eval '{}', turns[{turn_index}]: prompt must contain non-whitespace text",
36                        ev.id
37                    ),
38                });
39            }
40            if turn.deliver_when == DeliverWhen::Always && turn.agent_response_matches.is_some() {
41                return Err(ValidationError::InvalidConfig {
42                    path: source.to_string(),
43                    message: format!(
44                        "eval '{}', turns[{turn_index}]: agent_response_matches is only valid when deliver_when is 'agent_asks'",
45                        ev.id
46                    ),
47                });
48            }
49            if let Some(pattern) = &turn.agent_response_matches
50                && let Err(error) = Regex::new(pattern)
51            {
52                return Err(ValidationError::InvalidConfig {
53                    path: source.to_string(),
54                    message: format!(
55                        "eval '{}', turns[{turn_index}]: invalid agent_response_matches regex {pattern:?}: {error}",
56                        ev.id
57                    ),
58                });
59            }
60        }
61
62        let visible = ev.files.as_deref().unwrap_or(&[]);
63        for assertion in ev.assertions.as_deref().unwrap_or(&[]) {
64            if let Assertion::TranscriptCheck(check) = assertion {
65                if check.check == "assistant_message_matches" && ev.turns.is_none() {
66                    return Err(ValidationError::InvalidConfig {
67                        path: source.to_string(),
68                        message: format!(
69                            "eval '{}', transcript_check '{}': assistant_message_matches \
70                             requires a non-empty turns array",
71                            ev.id, check.id
72                        ),
73                    });
74                }
75                if let Some(pattern) = &check.pattern
76                    && let Err(error) = Regex::new(pattern)
77                {
78                    return Err(ValidationError::InvalidConfig {
79                        path: source.to_string(),
80                        message: format!(
81                            "eval '{}', transcript_check '{}': invalid pattern regex \
82                             {pattern:?}: {error}",
83                            ev.id, check.id
84                        ),
85                    });
86                }
87            }
88            let Assertion::CommandCheck(check) = assertion else {
89                continue;
90            };
91            for (name, value) in check.env.as_ref().into_iter().flatten() {
92                validate_environment_name(source, &ev.id, &check.id, "env", name)?;
93                validate_environment_value(source, &ev.id, &check.id, "env", name, value)?;
94            }
95            for (name, values) in check.matrix.as_ref().into_iter().flatten() {
96                validate_environment_name(source, &ev.id, &check.id, "matrix", name)?;
97                for value in values {
98                    validate_environment_value(source, &ev.id, &check.id, "matrix", name, value)?;
99                }
100            }
101            for setup in check.setup_files.as_deref().unwrap_or(&[]) {
102                let setup_path = normalize_relative(setup).map_err(|()| {
103                    ValidationError::InvalidConfig {
104                        path: source.to_string(),
105                        message: format!(
106                            "eval '{}', command_check '{}': setup_files path must be relative and stay within the task environment: {setup}",
107                            ev.id, check.id
108                        ),
109                    }
110                })?;
111                for fixture in visible {
112                    let Ok(fixture_path) = normalize_relative(fixture) else {
113                        continue;
114                    };
115                    if paths_overlap(&fixture_path, &setup_path) {
116                        return Err(ValidationError::InvalidConfig {
117                            path: source.to_string(),
118                            message: format!(
119                                "eval '{}', command_check '{}': visible fixture '{}' and setup_files path '{}' overlap; held-out setup paths must be disjoint from agent-visible files",
120                                ev.id, check.id, fixture, setup
121                            ),
122                        });
123                    }
124                }
125            }
126        }
127    }
128
129    Ok(validated)
130}
131
132fn validate_environment_name(
133    source: &str,
134    eval_id: &str,
135    check_id: &str,
136    field: &str,
137    name: &str,
138) -> Result<(), ValidationError> {
139    if name.is_empty() || name.contains('=') || name.contains('\0') {
140        return Err(ValidationError::InvalidConfig {
141            path: source.to_string(),
142            message: format!(
143                "eval '{eval_id}', command_check '{check_id}': {field} environment variable name must be non-empty and contain neither '=' nor NUL: {name:?}"
144            ),
145        });
146    }
147    Ok(())
148}
149
150fn validate_environment_value(
151    source: &str,
152    eval_id: &str,
153    check_id: &str,
154    field: &str,
155    name: &str,
156    value: &str,
157) -> Result<(), ValidationError> {
158    if value.contains('\0') {
159        return Err(ValidationError::InvalidConfig {
160            path: source.to_string(),
161            message: format!(
162                "eval '{eval_id}', command_check '{check_id}': {field} environment variable {name:?} value must not contain NUL"
163            ),
164        });
165    }
166    Ok(())
167}
168
169fn normalize_relative(value: &str) -> Result<PathBuf, ()> {
170    let mut normalized = PathBuf::new();
171    for component in Path::new(value).components() {
172        match component {
173            Component::Normal(part) => normalized.push(part),
174            Component::CurDir => {}
175            Component::ParentDir | Component::RootDir | Component::Prefix(_) => return Err(()),
176        }
177    }
178    Ok(normalized)
179}
180
181fn paths_overlap(left: &Path, right: &Path) -> bool {
182    left.starts_with(right) || right.starts_with(left)
183}
184
185#[cfg(test)]
186mod tests {
187    use super::validate_evals_config;
188    use serde_json::{Value, json};
189
190    /// The minimal valid config the cases below mutate.
191    fn base() -> Value {
192        json!({
193            "skill_name": "demo",
194            "evals": [
195                {
196                    "id": "e1",
197                    "prompt": "do the thing",
198                    "expected_output": "the thing is done"
199                }
200            ]
201        })
202    }
203
204    #[test]
205    fn accepts_a_boolean_skill_should_trigger() {
206        let mut config = base();
207        config["evals"][0]["skill_should_trigger"] = json!(false);
208        let parsed = validate_evals_config(&config, "evals.json").unwrap();
209        assert_eq!(parsed.evals[0].skill_should_trigger, Some(false));
210    }
211
212    #[test]
213    fn accepts_evals_with_no_skill_should_trigger() {
214        let config = base();
215        let parsed = validate_evals_config(&config, "evals.json").unwrap();
216        assert_eq!(parsed.skill_name, "demo");
217        assert_eq!(parsed.evals[0].skill_should_trigger, None);
218    }
219
220    #[test]
221    fn rejects_a_non_boolean_skill_should_trigger() {
222        let mut config = base();
223        config["evals"][0]["skill_should_trigger"] = json!("false");
224        let err = validate_evals_config(&config, "evals.json")
225            .unwrap_err()
226            .to_string();
227        assert!(err.contains("skill_should_trigger"), "error was: {err}");
228    }
229
230    #[test]
231    fn accepts_isolation_isolated() {
232        let mut config = base();
233        config["evals"][0]["isolation"] = json!("isolated");
234        let parsed = validate_evals_config(&config, "evals.json").unwrap();
235        assert_eq!(
236            parsed.evals[0].isolation,
237            Some(crate::core::Isolation::Isolated)
238        );
239    }
240
241    #[test]
242    fn accepts_isolation_shared() {
243        let mut config = base();
244        config["evals"][0]["isolation"] = json!("shared");
245        let parsed = validate_evals_config(&config, "evals.json").unwrap();
246        assert_eq!(
247            parsed.evals[0].isolation,
248            Some(crate::core::Isolation::Shared)
249        );
250    }
251
252    #[test]
253    fn defaults_isolation_to_none_when_absent() {
254        let config = base();
255        let parsed = validate_evals_config(&config, "evals.json").unwrap();
256        assert_eq!(parsed.evals[0].isolation, None);
257    }
258
259    #[test]
260    fn rejects_an_unknown_isolation_value() {
261        let mut config = base();
262        config["evals"][0]["isolation"] = json!("sometimes");
263        let err = validate_evals_config(&config, "evals.json")
264            .unwrap_err()
265            .to_string();
266        assert!(err.contains("isolation"), "error was: {err}");
267    }
268
269    #[test]
270    fn rejects_a_non_kebab_case_id() {
271        let mut config = base();
272        config["evals"][0]["id"] = json!("Not Kebab");
273        assert!(validate_evals_config(&config, "evals.json").is_err());
274    }
275
276    #[test]
277    fn rejects_duplicate_eval_ids() {
278        let mut config = base();
279        let dup = config["evals"][0].clone();
280        config["evals"] = json!([dup.clone(), dup]);
281        let err = validate_evals_config(&config, "evals.json")
282            .unwrap_err()
283            .to_string();
284        assert!(err.contains("duplicate"), "error was: {err}");
285    }
286
287    #[test]
288    fn rejects_an_empty_evals_array() {
289        let mut config = base();
290        config["evals"] = json!([]);
291        assert!(validate_evals_config(&config, "evals.json").is_err());
292    }
293
294    #[test]
295    fn accepts_ordered_scripted_follow_up_turns() {
296        let mut config = base();
297        config["evals"][0]["turns"] = json!([
298            {
299                "prompt": "Which users are affected?",
300                "deliver_when": "always"
301            },
302            {
303                "prompt": "They are all in US timezones and this is a date-only field.",
304                "deliver_when": "agent_asks",
305                "agent_response_matches": "(?i)(time ?zone|date-only)"
306            }
307        ]);
308
309        let parsed = validate_evals_config(&config, "evals.json").unwrap();
310        let turns = parsed.evals[0].turns.as_ref().unwrap();
311        assert_eq!(turns.len(), 2);
312        assert_eq!(turns[0].prompt, "Which users are affected?");
313        assert_eq!(turns[0].deliver_when, crate::core::DeliverWhen::Always);
314        assert_eq!(turns[1].deliver_when, crate::core::DeliverWhen::AgentAsks);
315        assert_eq!(
316            turns[1].agent_response_matches.as_deref(),
317            Some("(?i)(time ?zone|date-only)")
318        );
319    }
320
321    #[test]
322    fn rejects_assistant_message_check_without_scripted_turns() {
323        let mut config = base();
324        config["evals"][0]["assertions"] = json!([{
325            "id": "asked",
326            "type": "transcript_check",
327            "check": "assistant_message_matches",
328            "pattern": "timezone"
329        }]);
330        let err = validate_evals_config(&config, "evals.json")
331            .unwrap_err()
332            .to_string();
333        assert!(err.contains("assistant_message_matches"), "{err}");
334        assert!(err.contains("turns"), "{err}");
335    }
336
337    #[test]
338    fn rejects_invalid_scripted_turn_shapes() {
339        for (turn, expected) in [
340            (json!({"prompt": "", "deliver_when": "always"}), "prompt"),
341            (
342                json!({"prompt": "answer", "deliver_when": "sometimes"}),
343                "deliver_when",
344            ),
345            (
346                json!({
347                    "prompt": "answer",
348                    "deliver_when": "always",
349                    "agent_response_matches": "topic"
350                }),
351                "agent_response_matches",
352            ),
353            (
354                json!({
355                    "prompt": "answer",
356                    "deliver_when": "agent_asks",
357                    "agent_response_matches": "("
358                }),
359                "agent_response_matches",
360            ),
361        ] {
362            let mut config = base();
363            config["evals"][0]["turns"] = json!([turn]);
364            let error = validate_evals_config(&config, "evals.json")
365                .unwrap_err()
366                .to_string();
367            assert!(error.contains(expected), "{expected}: {error}");
368        }
369    }
370
371    #[test]
372    fn rejects_an_empty_scripted_turns_array() {
373        let mut config = base();
374        config["evals"][0]["turns"] = json!([]);
375        let error = validate_evals_config(&config, "evals.json")
376            .unwrap_err()
377            .to_string();
378        assert!(error.contains("turns"), "{error}");
379    }
380
381    #[test]
382    fn accepts_command_check_with_optional_setup_stdout_and_exit_code() {
383        let mut config = base();
384        config["evals"][0]["assertions"] = json!([
385            {
386                "id": "default-exit",
387                "type": "command_check",
388                "command": "cargo test"
389            },
390            {
391                "id": "full",
392                "type": "command_check",
393                "setup_files": ["holdout/test.rs"],
394                "command": "cargo test --test holdout",
395                "expect_exit_code": 2,
396                "expect_stdout": "2 tests passed"
397            }
398        ]);
399
400        let parsed = validate_evals_config(&config, "evals.json").unwrap();
401        let assertions = parsed.evals[0].assertions.as_ref().unwrap();
402        let crate::core::Assertion::CommandCheck(defaulted) = &assertions[0] else {
403            panic!("expected command_check");
404        };
405        assert_eq!(defaulted.expect_exit_code, 0);
406        let crate::core::Assertion::CommandCheck(full) = &assertions[1] else {
407            panic!("expected command_check");
408        };
409        assert_eq!(
410            full.setup_files.as_deref(),
411            Some(&["holdout/test.rs".into()][..])
412        );
413        assert_eq!(full.expect_exit_code, 2);
414        assert_eq!(full.expect_stdout.as_deref(), Some("2 tests passed"));
415    }
416
417    #[test]
418    fn accepts_command_check_environment_and_matrix() {
419        let mut config = base();
420        config["evals"][0]["assertions"] = json!([{
421            "id": "timezone-matrix",
422            "type": "command_check",
423            "command": "bun test",
424            "env": {
425                "CI": "1",
426                "EMPTY": "",
427                "TZ": "UTC"
428            },
429            "matrix": {
430                "LOCALE": ["en_US", "de_DE"],
431                "MODE": ["", "strict"],
432                "TZ": ["UTC", "America/Los_Angeles"]
433            }
434        }]);
435
436        let parsed = validate_evals_config(&config, "evals.json").unwrap();
437        let crate::core::Assertion::CommandCheck(check) =
438            &parsed.evals[0].assertions.as_ref().unwrap()[0]
439        else {
440            panic!("expected command_check");
441        };
442        assert_eq!(check.env.as_ref().unwrap()["CI"], "1");
443        assert_eq!(check.env.as_ref().unwrap()["EMPTY"], "");
444        assert_eq!(check.matrix.as_ref().unwrap()["MODE"][0], "");
445        assert_eq!(
446            check.matrix.as_ref().unwrap()["TZ"],
447            ["UTC", "America/Los_Angeles"]
448        );
449    }
450
451    #[test]
452    fn rejects_invalid_command_check_environment_names_and_nul_values() {
453        for (field, value) in [
454            ("env", json!({ "BAD=NAME": "value" })),
455            ("env", json!({ "GOOD_NAME": "bad\u{0}value" })),
456            ("matrix", json!({ "BAD=NAME": ["value"] })),
457            ("matrix", json!({ "GOOD_NAME": ["bad\u{0}value"] })),
458        ] {
459            let mut config = base();
460            config["evals"][0]["assertions"] = json!([{
461                "id": "environment",
462                "type": "command_check",
463                "command": "true",
464                field: value
465            }]);
466
467            let error = validate_evals_config(&config, "evals.json")
468                .unwrap_err()
469                .to_string();
470            assert!(error.contains(field), "{field}: {error}");
471            assert!(error.contains("environment"), "{field}: {error}");
472        }
473    }
474
475    #[test]
476    fn rejects_empty_or_duplicate_command_check_environment_collections() {
477        for (field, value) in [
478            ("env", json!({})),
479            ("matrix", json!({})),
480            ("matrix", json!({ "TZ": [] })),
481            ("matrix", json!({ "TZ": ["UTC", "UTC"] })),
482        ] {
483            let mut config = base();
484            config["evals"][0]["assertions"] = json!([{
485                "id": "environment",
486                "type": "command_check",
487                "command": "true",
488                field: value
489            }]);
490
491            let error = validate_evals_config(&config, "evals.json")
492                .unwrap_err()
493                .to_string();
494            assert!(error.contains(field), "{field}: {error}");
495        }
496    }
497
498    #[test]
499    fn rejects_non_string_command_check_environment_values() {
500        for (field, value) in [
501            ("env", json!({ "CI": 1 })),
502            ("env", json!({ "CI": null })),
503            ("matrix", json!({ "TZ": ["UTC", 1] })),
504            ("matrix", json!({ "TZ": "UTC" })),
505        ] {
506            let mut config = base();
507            config["evals"][0]["assertions"] = json!([{
508                "id": "environment",
509                "type": "command_check",
510                "command": "true",
511                field: value
512            }]);
513
514            let error = validate_evals_config(&config, "evals.json")
515                .unwrap_err()
516                .to_string();
517            assert!(error.contains(field), "{field}: {error}");
518        }
519    }
520
521    #[test]
522    fn accepts_diff_scope_with_either_or_both_thresholds() {
523        for assertion in [
524            json!({
525                "id": "files",
526                "type": "diff_scope",
527                "max_files_touched": 1
528            }),
529            json!({
530                "id": "lines",
531                "type": "diff_scope",
532                "max_lines_changed": 8
533            }),
534            json!({
535                "id": "both",
536                "type": "diff_scope",
537                "max_files_touched": 1,
538                "max_lines_changed": 8
539            }),
540        ] {
541            let mut config = base();
542            config["evals"][0]["assertions"] = json!([assertion]);
543            validate_evals_config(&config, "evals.json").unwrap();
544        }
545    }
546
547    #[test]
548    fn rejects_diff_scope_without_a_threshold() {
549        let mut config = base();
550        config["evals"][0]["assertions"] = json!([{
551            "id": "report-only",
552            "type": "diff_scope"
553        }]);
554        let error = validate_evals_config(&config, "evals.json")
555            .unwrap_err()
556            .to_string();
557        assert!(error.contains("diff_scope"), "{error}");
558    }
559
560    fn with_command_check(files: &[&str], setup_files: &[&str]) -> Value {
561        let mut config = base();
562        config["evals"][0]["files"] = json!(files);
563        config["evals"][0]["assertions"] = json!([{
564            "id": "held-out",
565            "type": "command_check",
566            "setup_files": setup_files,
567            "command": "test -f holdout/test.txt"
568        }]);
569        config
570    }
571
572    #[test]
573    fn rejects_exact_visible_and_setup_file_overlap() {
574        let config = with_command_check(&["holdout/test.txt"], &["holdout/test.txt"]);
575        let err = validate_evals_config(&config, "evals.json")
576            .unwrap_err()
577            .to_string();
578        assert!(err.contains("overlap"), "error was: {err}");
579        assert!(err.contains("holdout/test.txt"), "error was: {err}");
580    }
581
582    #[test]
583    fn rejects_visible_directory_ancestor_of_setup_file() {
584        let config = with_command_check(&["holdout"], &["holdout/test.txt"]);
585        let err = validate_evals_config(&config, "evals.json")
586            .unwrap_err()
587            .to_string();
588        assert!(err.contains("overlap"), "error was: {err}");
589        assert!(err.contains("holdout"), "error was: {err}");
590    }
591
592    #[test]
593    fn rejects_setup_directory_ancestor_of_visible_file() {
594        let config = with_command_check(&["src/main.rs"], &["src"]);
595        let err = validate_evals_config(&config, "evals.json")
596            .unwrap_err()
597            .to_string();
598        assert!(err.contains("overlap"), "error was: {err}");
599        assert!(err.contains("src"), "error was: {err}");
600    }
601
602    #[test]
603    fn rejects_absolute_and_escaping_setup_paths() {
604        for setup in ["/tmp/holdout.txt", "../holdout.txt", "holdout/../../escape"] {
605            let config = with_command_check(&["src/main.rs"], &[setup]);
606            let err = validate_evals_config(&config, "evals.json")
607                .unwrap_err()
608                .to_string();
609            assert!(err.contains("setup_files"), "{setup}: {err}");
610            assert!(err.contains("relative"), "{setup}: {err}");
611        }
612    }
613
614    #[test]
615    fn accepts_disjoint_visible_and_setup_paths() {
616        let config = with_command_check(&["src/main.rs"], &["holdout/test.txt"]);
617        validate_evals_config(&config, "evals.json").unwrap();
618    }
619}
eval_magic/validation/evals.rs

eval_magic/validation/
evals.rs
