vtcode_eval/
executor.rs

1//! Executor trait boundary and pure eval orchestration.
2//!
3//! The [`EvalExecutor`] trait isolates the harness orchestration from the
4//! concrete agent runner. This is the "interface guard rail": `run_suite`
5//! depends only on the trait, so it can be unit-tested with a fake executor
6//! and the production executor implementation can be swapped
7//! without touching the orchestration logic.
8
9use anyhow::Result;
10use async_trait::async_trait;
11
12use crate::task::EvalTask;
13use crate::{
14    EvalReport, EvalRunResult, EvalSuite, SuiteReport, aggregate_metrics, build_task_report,
15    compute_metric,
16};
17
18/// Executes a single eval task attempt and returns the verified outcome.
19///
20/// Implementations own the full "execute this task" semantics — running the
21/// agent and applying any environment verification (probes). Keeping this
22/// behind a trait decouples the orchestration in [`run_suite`] from the
23/// concrete runner, which makes the harness independently testable.
24#[async_trait]
25pub trait EvalExecutor: Send + Sync {
26    /// Run one attempt of `task` and return the outcome.
27    async fn execute_task(&self, task: &EvalTask) -> Result<EvalRunResult>;
28}
29
30/// Pure orchestration core: loop tasks × attempts through the executor,
31/// compute per-task metrics, and assemble the report.
32///
33/// This function performs no file I/O, no config reads, and no trust checks —
34/// those belong to the caller. It depends only on [`EvalExecutor`], which makes
35/// it fully unit-testable with an in-memory fake (see `executor::tests`).
36pub async fn run_suite(executor: &dyn EvalExecutor, suite: &EvalSuite) -> Result<EvalReport> {
37    let mut all_task_reports = Vec::new();
38
39    for task in &suite.tasks {
40        let mut run_results = Vec::new();
41        for _attempt in 1..=suite.attempts {
42            let result = executor.execute_task(task).await?;
43            run_results.push(result);
44        }
45        let metric = compute_metric(&task.id, &run_results);
46        all_task_reports.push(build_task_report(
47            &task.id,
48            &task.name,
49            task.category,
50            metric,
51        ));
52    }
53
54    let all_metrics: Vec<_> = all_task_reports.iter().map(|r| r.metric.clone()).collect();
55    let cap_metrics: Vec<_> = all_task_reports
56        .iter()
57        .filter(|r| r.category == "Capability")
58        .map(|r| r.metric.clone())
59        .collect();
60    let reg_metrics: Vec<_> = all_task_reports
61        .iter()
62        .filter(|r| r.category == "Regression")
63        .map(|r| r.metric.clone())
64        .collect();
65
66    Ok(EvalReport {
67        generated_at: chrono::Utc::now().to_rfc3339(),
68        suites: vec![SuiteReport {
69            suite_id: suite.id.clone(),
70            suite_name: suite.name.clone(),
71            task_reports: all_task_reports,
72            aggregate: aggregate_metrics(&all_metrics),
73            capability_metrics: aggregate_metrics(&cap_metrics),
74            regression_metrics: aggregate_metrics(&reg_metrics),
75        }],
76    })
77}
78
79#[cfg(test)]
80mod tests {
81    use super::*;
82    use crate::task::{EvalCategory, EvalRunResult, EvalTask, RunOutcome};
83    use std::sync::atomic::{AtomicUsize, Ordering};
84
85    /// In-memory fake executor for isolating `run_suite`.
86    struct FakeExecutor {
87        outcomes: Vec<RunOutcome>,
88        calls: AtomicUsize,
89    }
90
91    #[async_trait]
92    impl EvalExecutor for FakeExecutor {
93        async fn execute_task(&self, _task: &EvalTask) -> Result<EvalRunResult> {
94            let i = self.calls.fetch_add(1, Ordering::SeqCst);
95            let outcome = self.outcomes[i % self.outcomes.len()];
96            Ok(EvalRunResult {
97                task_id: _task.id.clone(),
98                outcome,
99                error_message: None,
100                duration_secs: 0.0,
101                attempt: (i + 1) as u32,
102                cost_usd: None,
103                transcript_path: None,
104            })
105        }
106    }
107
108    fn suite(attempts: u32) -> EvalSuite {
109        EvalSuite {
110            id: "s1".into(),
111            name: "demo".into(),
112            tasks: vec![
113                EvalTask {
114                    id: "t1".into(),
115                    name: "t1".into(),
116                    category: EvalCategory::Capability,
117                    prompt: "p".into(),
118                    verify_commands: vec![],
119                    timeout_secs: None,
120                },
121                EvalTask {
122                    id: "t2".into(),
123                    name: "t2".into(),
124                    category: EvalCategory::Regression,
125                    prompt: "p".into(),
126                    verify_commands: vec![],
127                    timeout_secs: None,
128                },
129            ],
130            attempts,
131        }
132    }
133
134    #[tokio::test]
135    async fn run_suite_aggregates_capability_and_regression() {
136        // t1 gets [Pass, Fail]; t2 gets [Pass, Pass]
137        let exec = FakeExecutor {
138            outcomes: vec![
139                RunOutcome::Pass,
140                RunOutcome::Fail,
141                RunOutcome::Pass,
142                RunOutcome::Pass,
143            ],
144            calls: AtomicUsize::new(0),
145        };
146        let report = run_suite(&exec, &suite(2)).await.unwrap();
147        let s = &report.suites[0];
148        // t1: 1/2 pass, t2: 2/2 pass
149        assert_eq!(s.aggregate.passed_runs, 3);
150        assert_eq!(s.aggregate.total_runs, 4);
151        assert!((s.capability_metrics.pass_at_k - 0.5).abs() < 1e-9);
152        assert!((s.regression_metrics.pass_at_k - 1.0).abs() < 1e-9);
153        assert!(s.capability_metrics.pass_all_k < 1.0);
154        assert_eq!(s.regression_metrics.pass_all_k, 1.0);
155    }
156}
vtcode_eval/executor.rs

vtcode_eval/
executor.rs
