vtcode_eval/
task.rs

1use serde::{Deserialize, Serialize};
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
4#[serde(rename_all = "snake_case")]
5pub enum EvalCategory {
6    #[default]
7    Capability,
8    Regression,
9}
10
11impl EvalCategory {
12    pub(crate) fn label(&self) -> &'static str {
13        match self {
14            Self::Capability => "Capability",
15            Self::Regression => "Regression",
16        }
17    }
18    pub fn as_str(&self) -> &'static str {
19        self.label()
20    }
21}
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
24#[serde(rename_all = "snake_case")]
25pub enum RunOutcome {
26    #[default]
27    Pass,
28    Fail,
29    Error,
30}
31
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct EvalTask {
34    pub id: String,
35    pub name: String,
36    pub category: EvalCategory,
37    pub prompt: String,
38    pub verify_commands: Vec<String>,
39    pub timeout_secs: Option<u64>,
40}
41
42#[derive(Debug, Clone, Serialize, Deserialize)]
43pub struct EvalRunResult {
44    pub task_id: String,
45    pub outcome: RunOutcome,
46    pub error_message: Option<String>,
47    pub duration_secs: f64,
48    pub attempt: u32,
49    pub cost_usd: Option<f64>,
50    pub transcript_path: Option<String>,
51}
vtcode_eval/task.rs

vtcode_eval/
task.rs
