vtcode_eval/
executor.rs

1//! Executor trait boundary and pure eval orchestration.
2//!
3//! The [`EvalExecutor`] trait isolates the harness orchestration from the
4//! concrete agent runner. This is the "interface guard rail": `run_suite`
5//! depends only on the trait, so it can be unit-tested with a fake executor
6//! and the production executor implementation can be swapped
7//! without touching the orchestration logic.
8
9use anyhow::Result;
10use async_trait::async_trait;
11
12use crate::task::EvalTask;
13use crate::{EvalReport, EvalRunResult, EvalSuite, SuiteReport, aggregate_metrics, build_task_report, compute_metric};
14
15/// Executes a single eval task attempt and returns the verified outcome.
16///
17/// Implementations own the full "execute this task" semantics — running the
18/// agent and applying any environment verification (probes). Keeping this
19/// behind a trait decouples the orchestration in [`run_suite`] from the
20/// concrete runner, which makes the harness independently testable.
21#[async_trait]
22pub trait EvalExecutor: Send + Sync {
23    /// Run one attempt of `task` and return the outcome.
24    async fn execute_task(&self, task: &EvalTask) -> Result<EvalRunResult>;
25}
26
27/// Pure orchestration core: loop tasks × attempts through the executor,
28/// compute per-task metrics, and assemble the report.
29///
30/// This function performs no file I/O, no config reads, and no trust checks —
31/// those belong to the caller. It depends only on [`EvalExecutor`], which makes
32/// it fully unit-testable with an in-memory fake (see `executor::tests`).
33pub async fn run_suite(executor: &dyn EvalExecutor, suite: &EvalSuite) -> Result<EvalReport> {
34    let mut all_task_reports = Vec::new();
35
36    for task in &suite.tasks {
37        let mut run_results = Vec::new();
38        for _attempt in 1..=suite.attempts {
39            let result = executor.execute_task(task).await?;
40            run_results.push(result);
41        }
42        let metric = compute_metric(&task.id, &run_results);
43        all_task_reports.push(build_task_report(&task.id, &task.name, task.category, metric));
44    }
45
46    let all_metrics: Vec<_> = all_task_reports.iter().map(|r| r.metric.clone()).collect();
47    let cap_metrics: Vec<_> = all_task_reports
48        .iter()
49        .filter(|r| r.category == "Capability")
50        .map(|r| r.metric.clone())
51        .collect();
52    let reg_metrics: Vec<_> = all_task_reports
53        .iter()
54        .filter(|r| r.category == "Regression")
55        .map(|r| r.metric.clone())
56        .collect();
57
58    Ok(EvalReport {
59        generated_at: chrono::Utc::now().to_rfc3339(),
60        suites: vec![SuiteReport {
61            suite_id: suite.id.clone(),
62            suite_name: suite.name.clone(),
63            task_reports: all_task_reports,
64            aggregate: aggregate_metrics(&all_metrics),
65            capability_metrics: aggregate_metrics(&cap_metrics),
66            regression_metrics: aggregate_metrics(&reg_metrics),
67        }],
68    })
69}
70
71#[cfg(test)]
72mod tests {
73    use super::*;
74    use crate::task::{EvalCategory, EvalRunResult, EvalTask, RunOutcome};
75    use std::sync::atomic::{AtomicUsize, Ordering};
76
77    /// In-memory fake executor for isolating `run_suite`.
78    struct FakeExecutor {
79        outcomes: Vec<RunOutcome>,
80        calls: AtomicUsize,
81    }
82
83    #[async_trait]
84    impl EvalExecutor for FakeExecutor {
85        async fn execute_task(&self, _task: &EvalTask) -> Result<EvalRunResult> {
86            let i = self.calls.fetch_add(1, Ordering::SeqCst);
87            let outcome = self.outcomes[i % self.outcomes.len()];
88            Ok(EvalRunResult {
89                task_id: _task.id.clone(),
90                outcome,
91                error_message: None,
92                duration_secs: 0.0,
93                attempt: (i + 1) as u32,
94                cost_usd: None,
95                transcript_path: None,
96            })
97        }
98    }
99
100    fn suite(attempts: u32) -> EvalSuite {
101        EvalSuite {
102            id: "s1".into(),
103            name: "demo".into(),
104            tasks: vec![
105                EvalTask {
106                    id: "t1".into(),
107                    name: "t1".into(),
108                    category: EvalCategory::Capability,
109                    prompt: "p".into(),
110                    verify_commands: vec![],
111                    timeout_secs: None,
112                },
113                EvalTask {
114                    id: "t2".into(),
115                    name: "t2".into(),
116                    category: EvalCategory::Regression,
117                    prompt: "p".into(),
118                    verify_commands: vec![],
119                    timeout_secs: None,
120                },
121            ],
122            attempts,
123        }
124    }
125
126    #[tokio::test]
127    async fn run_suite_aggregates_capability_and_regression() {
128        // t1 gets [Pass, Fail]; t2 gets [Pass, Pass]
129        let exec = FakeExecutor {
130            outcomes: vec![RunOutcome::Pass, RunOutcome::Fail, RunOutcome::Pass, RunOutcome::Pass],
131            calls: AtomicUsize::new(0),
132        };
133        let report = run_suite(&exec, &suite(2)).await.unwrap();
134        let s = &report.suites[0];
135        // t1: 1/2 pass, t2: 2/2 pass
136        assert_eq!(s.aggregate.passed_runs, 3);
137        assert_eq!(s.aggregate.total_runs, 4);
138        assert!((s.capability_metrics.pass_at_k - 0.5).abs() < 1e-9);
139        assert!((s.regression_metrics.pass_at_k - 1.0).abs() < 1e-9);
140        assert!(s.capability_metrics.pass_all_k < 1.0);
141        assert_eq!(s.regression_metrics.pass_all_k, 1.0);
142    }
143}
vtcode_eval/executor.rs

vtcode_eval/
executor.rs
