vtcode_eval/
executor.rs

1//! Executor trait boundary and pure eval orchestration.
2//!
3//! The [`EvalExecutor`] trait isolates the harness orchestration from the
4//! concrete agent runner. This is the "interface guard rail": `run_suite`
5//! depends only on the trait, so it can be unit-tested with a fake executor
6//! and the production executor implementation can be swapped
7//! without touching the orchestration logic.
8
9use anyhow::Result;
10use async_trait::async_trait;
11
12use crate::task::EvalTask;
13use crate::{
14    EvalReport, EvalRunResult, EvalSuite, SuiteReport, aggregate_metrics, build_task_report,
15    compute_metric,
16};
17
18/// Executes a single eval task attempt and returns the verified outcome.
19///
20/// Implementations own the full "execute this task" semantics — running the
21/// agent and applying any environment verification (probes). Keeping this
22/// behind a trait decouples the orchestration in [`run_suite`] from the
23/// concrete runner, which makes the harness independently testable.
24#[async_trait]
25pub trait EvalExecutor: Send + Sync {
26    /// Run one attempt of `task` and return the outcome.
27    async fn execute_task(&self, task: &EvalTask) -> Result<EvalRunResult>;
28}
29
30/// Pure orchestration core: loop tasks × attempts through the executor,
31/// compute per-task metrics, and assemble the report.
32///
33/// This function performs no file I/O, no config reads, and no trust checks —
34/// those belong to the caller. It depends only on [`EvalExecutor`], which makes
35/// it fully unit-testable with an in-memory fake (see `executor::tests`).
36pub async fn run_suite(executor: &dyn EvalExecutor, suite: &EvalSuite) -> Result<EvalReport> {
37    let mut all_task_reports = Vec::new();
38
39    for task in &suite.tasks {
40        let mut run_results = Vec::new();
41        for _attempt in 1..=suite.attempts {
42            let result = executor.execute_task(task).await?;
43            run_results.push(result);
44        }
45        let metric = compute_metric(&task.id, &run_results);
46        all_task_reports.push(build_task_report(&task.id, &task.name, task.category, metric));
47    }
48
49    let all_metrics: Vec<_> = all_task_reports.iter().map(|r| r.metric.clone()).collect();
50    let cap_metrics: Vec<_> = all_task_reports
51        .iter()
52        .filter(|r| r.category == "Capability")
53        .map(|r| r.metric.clone())
54        .collect();
55    let reg_metrics: Vec<_> = all_task_reports
56        .iter()
57        .filter(|r| r.category == "Regression")
58        .map(|r| r.metric.clone())
59        .collect();
60
61    Ok(EvalReport {
62        generated_at: chrono::Utc::now().to_rfc3339(),
63        suites: vec![SuiteReport {
64            suite_id: suite.id.clone(),
65            suite_name: suite.name.clone(),
66            task_reports: all_task_reports,
67            aggregate: aggregate_metrics(&all_metrics),
68            capability_metrics: aggregate_metrics(&cap_metrics),
69            regression_metrics: aggregate_metrics(&reg_metrics),
70        }],
71    })
72}
73
74#[cfg(test)]
75mod tests {
76    use super::*;
77    use crate::task::{EvalCategory, EvalRunResult, EvalTask, RunOutcome};
78    use std::sync::atomic::{AtomicUsize, Ordering};
79
80    /// In-memory fake executor for isolating `run_suite`.
81    struct FakeExecutor {
82        outcomes: Vec<RunOutcome>,
83        calls: AtomicUsize,
84    }
85
86    #[async_trait]
87    impl EvalExecutor for FakeExecutor {
88        async fn execute_task(&self, _task: &EvalTask) -> Result<EvalRunResult> {
89            let i = self.calls.fetch_add(1, Ordering::SeqCst);
90            let outcome = self.outcomes[i % self.outcomes.len()];
91            Ok(EvalRunResult {
92                task_id: _task.id.clone(),
93                outcome,
94                error_message: None,
95                duration_secs: 0.0,
96                attempt: (i + 1) as u32,
97                cost_usd: None,
98                transcript_path: None,
99            })
100        }
101    }
102
103    fn suite(attempts: u32) -> EvalSuite {
104        EvalSuite {
105            id: "s1".into(),
106            name: "demo".into(),
107            tasks: vec![
108                EvalTask {
109                    id: "t1".into(),
110                    name: "t1".into(),
111                    category: EvalCategory::Capability,
112                    prompt: "p".into(),
113                    verify_commands: vec![],
114                    timeout_secs: None,
115                },
116                EvalTask {
117                    id: "t2".into(),
118                    name: "t2".into(),
119                    category: EvalCategory::Regression,
120                    prompt: "p".into(),
121                    verify_commands: vec![],
122                    timeout_secs: None,
123                },
124            ],
125            attempts,
126        }
127    }
128
129    #[tokio::test]
130    async fn run_suite_aggregates_capability_and_regression() {
131        // t1 gets [Pass, Fail]; t2 gets [Pass, Pass]
132        let exec = FakeExecutor {
133            outcomes: vec![
134                RunOutcome::Pass,
135                RunOutcome::Fail,
136                RunOutcome::Pass,
137                RunOutcome::Pass,
138            ],
139            calls: AtomicUsize::new(0),
140        };
141        let report = run_suite(&exec, &suite(2)).await.unwrap();
142        let s = &report.suites[0];
143        // t1: 1/2 pass, t2: 2/2 pass
144        assert_eq!(s.aggregate.passed_runs, 3);
145        assert_eq!(s.aggregate.total_runs, 4);
146        assert!((s.capability_metrics.pass_at_k - 0.5).abs() < 1e-9);
147        assert!((s.regression_metrics.pass_at_k - 1.0).abs() < 1e-9);
148        assert!(s.capability_metrics.pass_all_k < 1.0);
149        assert_eq!(s.regression_metrics.pass_all_k, 1.0);
150    }
151}
vtcode_eval/executor.rs

vtcode_eval/
executor.rs
