vtcode-eval 0.141.2

Agent evaluation framework for VT Code: pass@k / pass^k metrics, capability and regression evals, and environment-based outcome verification.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

pub trait EnvironmentProbe: Send + Sync {
    fn check(&self, workspace: &std::path::Path) -> bool;
}

pub struct CommandProbe {
    command: String,
    args: Vec<String>,
}
impl CommandProbe {
    pub fn new(command: String, args: Vec<String>) -> Self {
        Self { command, args }
    }
}
impl EnvironmentProbe for CommandProbe {
    fn check(&self, workspace: &std::path::Path) -> bool {
        use std::process::Command;
        Command::new(&self.command)
            .args(&self.args)
            .current_dir(workspace)
            .output()
            .map(|o| o.status.success())
            .unwrap_or(false)
    }
}

pub struct FileExistsProbe {
    path: std::path::PathBuf,
}
impl FileExistsProbe {
    pub fn new(path: std::path::PathBuf) -> Self {
        Self { path }
    }
}
impl EnvironmentProbe for FileExistsProbe {
    fn check(&self, _workspace: &std::path::Path) -> bool {
        self.path.exists()
    }
}

pub struct GitCleanProbe;
impl EnvironmentProbe for GitCleanProbe {
    fn check(&self, workspace: &std::path::Path) -> bool {
        use std::process::Command;
        Command::new("git")
            .args(["diff", "--quiet"])
            .current_dir(workspace)
            .output()
            .map(|o| o.status.success())
            .unwrap_or(false)
    }
}