vtcode_eval/
suite.rs

1use crate::task::EvalTask;
2use serde::{Deserialize, Serialize};
3
4#[derive(Debug, Clone, Serialize, Deserialize)]
5pub struct EvalSuite {
6    pub(crate) id: String,
7    pub name: String,
8    pub tasks: Vec<EvalTask>,
9    pub attempts: u32,
10}
11
12#[cfg(test)]
13mod tests {
14    use super::*;
15    use crate::task::EvalCategory;
16
17    #[test]
18    fn suite_round_trips_through_json() {
19        let suite = EvalSuite {
20            id: "s1".into(),
21            name: "demo".into(),
22            tasks: vec![crate::task::EvalTask {
23                id: "t1".into(),
24                name: "t1".into(),
25                category: EvalCategory::Capability,
26                prompt: "do it".into(),
27                verify_commands: vec!["cargo test".into()],
28                timeout_secs: Some(30),
29            }],
30            attempts: 3,
31        };
32        let json = serde_json::to_string(&suite).unwrap();
33        let back: EvalSuite = serde_json::from_str(&json).unwrap();
34        assert_eq!(back.id, "s1");
35        assert_eq!(back.attempts, 3);
36        assert_eq!(back.tasks.len(), 1);
37        assert_eq!(back.tasks[0].timeout_secs, Some(30));
38    }
39
40    #[test]
41    fn suite_rejects_zero_attempts_via_validation() {
42        // The runner enforces attempts >= 1; serde itself allows 0, so the
43        // guardrail lives in the CLI entrypoint (see eval.rs M3).
44        let suite: EvalSuite = serde_json::from_str(r#"{"id":"s","name":"n","tasks":[],"attempts":0}"#).unwrap();
45        assert_eq!(suite.attempts, 0);
46    }
47}
vtcode_eval/suite.rs

vtcode_eval/
suite.rs
