vtcode-eval 0.141.2

Agent evaluation framework for VT Code: pass@k / pass^k metrics, capability and regression evals, and environment-based outcome verification.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum EvalCategory {
    #[default]
    Capability,
    Regression,
}

impl EvalCategory {
    pub(crate) fn label(&self) -> &'static str {
        match self {
            Self::Capability => "Capability",
            Self::Regression => "Regression",
        }
    }
    pub fn as_str(&self) -> &'static str {
        self.label()
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum RunOutcome {
    #[default]
    Pass,
    Fail,
    Error,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvalTask {
    pub id: String,
    pub name: String,
    pub category: EvalCategory,
    pub prompt: String,
    pub verify_commands: Vec<String>,
    pub timeout_secs: Option<u64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvalRunResult {
    pub task_id: String,
    pub outcome: RunOutcome,
    pub error_message: Option<String>,
    pub duration_secs: f64,
    pub attempt: u32,
    pub cost_usd: Option<f64>,
    pub transcript_path: Option<String>,
}