vtcode_eval/
suite.rs

1use crate::task::EvalTask;
2use serde::{Deserialize, Serialize};
3
4#[derive(Debug, Clone, Serialize, Deserialize)]
5pub struct EvalSuite {
6    pub(crate) id: String,
7    pub name: String,
8    pub tasks: Vec<EvalTask>,
9    pub attempts: u32,
10}
11
12impl EvalSuite {
13    /// Validate suite invariants shared by CLI and programmatic callers.
14    ///
15    /// Serde intentionally still accepts `attempts: 0` so persisted fixtures
16    /// fail with this explicit error instead of a schema break.
17    pub fn validate(&self) -> anyhow::Result<()> {
18        anyhow::ensure!(self.attempts >= 1, "evaluation suite attempts must be at least one");
19        Ok(())
20    }
21}
22
23#[cfg(test)]
24mod tests {
25    use super::*;
26    use crate::task::EvalCategory;
27
28    #[test]
29    fn suite_round_trips_through_json() {
30        let suite = EvalSuite {
31            id: "s1".into(),
32            name: "demo".into(),
33            tasks: vec![crate::task::EvalTask {
34                id: "t1".into(),
35                name: "t1".into(),
36                category: EvalCategory::Capability,
37                prompt: "do it".into(),
38                verify_commands: vec!["cargo test".into()],
39                timeout_secs: Some(30),
40            }],
41            attempts: 3,
42        };
43        let json = serde_json::to_string(&suite).unwrap();
44        let back: EvalSuite = serde_json::from_str(&json).unwrap();
45        assert_eq!(back.id, "s1");
46        assert_eq!(back.attempts, 3);
47        assert_eq!(back.tasks.len(), 1);
48        assert_eq!(back.tasks[0].timeout_secs, Some(30));
49    }
50
51    #[test]
52    fn suite_rejects_zero_attempts_via_validation() {
53        // Serde itself allows 0; `validate()` is the shared guardrail called
54        // by both the CLI entrypoint and `run_suite_with_options`.
55        let suite: EvalSuite = serde_json::from_str(r#"{"id":"s","name":"n","tasks":[],"attempts":0}"#).unwrap();
56        assert_eq!(suite.attempts, 0);
57        assert!(suite.validate().is_err());
58        let valid: EvalSuite = serde_json::from_str(r#"{"id":"s","name":"n","tasks":[],"attempts":1}"#).unwrap();
59        assert!(valid.validate().is_ok());
60    }
61
62    #[test]
63    fn preview_budget_blocked_replan_regression_suite_is_env_verified() {
64        let json = include_str!("../evals/preview-budget-blocked-replan.json");
65        let suite: EvalSuite = serde_json::from_str(json).unwrap();
66        assert_eq!(suite.id, "preview-budget-blocked-replan");
67        assert_eq!(suite.attempts, 3);
68        assert_eq!(suite.tasks.len(), 3);
69        for task in &suite.tasks {
70            assert_eq!(task.category, EvalCategory::Regression);
71            assert!(!task.prompt.is_empty());
72            assert!(!task.verify_commands.is_empty());
73        }
74        let ids: Vec<&str> = suite.tasks.iter().map(|task| task.id.as_str()).collect();
75        assert_eq!(
76            ids,
77            vec![
78                "preview-budget-no-retry",
79                "blocked-handoff-resume",
80                "replan-continuation"
81            ]
82        );
83    }
84}
vtcode_eval/suite.rs

vtcode_eval/
suite.rs
