aether_evals/spec/
runner.rs

1use super::ResolvedEvalSpec;
2use super::error::EvalFileError;
3use super::judge::JudgeRunner;
4use super::report::{EvalOutcome, JudgeSummary};
5use crate::agents::{DockerAgent, build_images};
6use crate::evals::TaskRun;
7use aether_core::events::AgentMessage;
8use futures::StreamExt;
9use futures::stream::iter;
10use llm::StreamingModelProvider;
11use llm::parser::ModelProviderParser;
12use std::num::NonZeroUsize;
13
14/// Whether the eval workspace is retained on disk after the run.
15#[derive(Debug, Clone, Copy)]
16pub enum WorkspaceRetention {
17    Discard,
18    Retain,
19}
20
21/// Run resolved eval specs. Agents and judge models are assembled before anything runs, so a
22/// broken case fails fast. Docker image builds are deduplicated and run concurrently before eval
23/// execution. Outcomes are returned in input order.
24pub async fn run_eval_specs(
25    specs: impl IntoIterator<Item = ResolvedEvalSpec>,
26    retention: WorkspaceRetention,
27    max_concurrency: NonZeroUsize,
28) -> Result<Vec<EvalOutcome>, EvalFileError> {
29    let mut prepared = Vec::new();
30    for spec in specs {
31        prepared.push(prepare_spec(spec).await?);
32    }
33
34    build_images(prepared.iter().filter_map(|(case, _, _)| case.docker.build.clone()), max_concurrency).await?;
35    let futures = prepared
36        .into_iter()
37        .map(|(case, agent, judge_llm)| async move { run_eval_spec(case, agent, judge_llm, retention, |_| {}).await });
38
39    let evals = iter(futures).buffered(max_concurrency.get()).collect().await;
40    Ok(evals)
41}
42
43pub async fn run_eval_spec_streaming<T: FnMut(&AgentMessage) + Send>(
44    spec: ResolvedEvalSpec,
45    retention: WorkspaceRetention,
46    on_message: T,
47) -> Result<EvalOutcome, EvalFileError> {
48    let (spec, agent, judge_llm) = prepare_spec(spec).await?;
49    build_images(spec.docker.build.clone(), NonZeroUsize::MIN).await?;
50    Ok(run_eval_spec(spec, agent, judge_llm, retention, on_message).await)
51}
52
53async fn run_eval_spec<T: FnMut(&AgentMessage) + Send>(
54    spec: ResolvedEvalSpec,
55    agent: DockerAgent,
56    judge_llm: Option<Box<dyn StreamingModelProvider>>,
57    retention: WorkspaceRetention,
58    on_message: T,
59) -> EvalOutcome {
60    let task = match spec.task() {
61        Ok(task) => task,
62        Err(error) => return EvalOutcome::setup_failed(spec.name(), error),
63    };
64
65    let run = match task.run_observed(&agent, on_message).await {
66        Ok(run) => run,
67        Err(error) => {
68            let (run, error) = error.into_parts();
69            return EvalOutcome::from_task_run(
70                spec.name(),
71                run,
72                vec![format!("agent run failed: {error}")],
73                None,
74                retention,
75            );
76        }
77    };
78
79    let mut failures = spec.expectations().evaluate(&run);
80    let judge = run_judge(&spec, &run, &mut failures, judge_llm.as_deref()).await;
81    EvalOutcome::from_task_run(spec.name(), run, failures, judge, retention)
82}
83
84async fn run_judge(
85    case: &ResolvedEvalSpec,
86    run: &TaskRun,
87    failures: &mut Vec<String>,
88    judge_llm: Option<&dyn StreamingModelProvider>,
89) -> Option<JudgeSummary> {
90    let Some(spec) = &case.judge else {
91        return None;
92    };
93    let llm = judge_llm.expect("judge model is parsed before any eval runs when a judge is configured");
94    let result = match JudgeRunner::from_eval_run(llm, run, case.expectations(), spec) {
95        Ok(judge) => judge.run().await,
96        Err(error) => Err(error),
97    };
98    match result {
99        Ok(summary) => {
100            failures.extend(summary.blocking_failures());
101            Some(summary)
102        }
103        Err(error) => {
104            failures.push(format!("judge failed: {error}"));
105            None
106        }
107    }
108}
109
110async fn prepare_spec(
111    spec: ResolvedEvalSpec,
112) -> Result<(ResolvedEvalSpec, DockerAgent, Option<Box<dyn StreamingModelProvider>>), EvalFileError> {
113    let agent = spec.agent();
114    let judge_llm = parse_judge_llm(&spec).await?;
115    Ok((spec, agent, judge_llm))
116}
117
118async fn parse_judge_llm(spec: &ResolvedEvalSpec) -> Result<Option<Box<dyn StreamingModelProvider>>, EvalFileError> {
119    let Some(judge) = &spec.judge else {
120        return Ok(None);
121    };
122    let (llm, _) = ModelProviderParser::default()
123        .parse(&judge.model)
124        .await
125        .map_err(|source| EvalFileError::JudgeModel { model: judge.model.clone(), source })?;
126    Ok(Some(llm))
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132    use crate::FakeAgent;
133    use crate::agents::{DockerImage, ImageBuildError};
134    use crate::spec::{EvalFileError, EvalSpec, EvalSpecLoadOptions, ResolvedEvalSpec};
135    use std::fs::{read_to_string, remove_dir_all};
136    use std::path::{Path, PathBuf};
137
138    fn eval_case(json: &str) -> ResolvedEvalSpec {
139        ResolvedEvalSpec::resolve(serde_json::from_value(eval_json_with_defaults(json)).unwrap(), ".").unwrap()
140    }
141
142    fn eval_case_in(json: &str, base_dir: &Path) -> ResolvedEvalSpec {
143        ResolvedEvalSpec::resolve(serde_json::from_value(eval_json_with_defaults(json)).unwrap(), base_dir).unwrap()
144    }
145
146    fn eval_json_with_defaults(json: &str) -> serde_json::Value {
147        let mut value: serde_json::Value = serde_json::from_str(json).unwrap();
148        if let serde_json::Value::Object(object) = &mut value {
149            object.entry("docker").or_insert_with(|| serde_json::json!({ "image": "sandbox:latest" }));
150            object.entry("agent").or_insert_with(|| serde_json::json!({ "command": ["agent"] }));
151        }
152        value
153    }
154
155    async fn run_case_with_agent(
156        case: ResolvedEvalSpec,
157        agent: &impl crate::Agent,
158        retention: WorkspaceRetention,
159    ) -> EvalOutcome {
160        let task = match case.task() {
161            Ok(task) => task,
162            Err(error) => return EvalOutcome::setup_failed(case.name(), error),
163        };
164        match task.run(agent).await {
165            Ok(run) => {
166                let failures = case.expectations().evaluate(&run);
167                EvalOutcome::from_task_run(case.name(), run, failures, None, retention)
168            }
169            Err(error) => {
170                let (run, error) = error.into_parts();
171                EvalOutcome::from_task_run(
172                    case.name(),
173                    run,
174                    vec![format!("agent run failed: {error}")],
175                    None,
176                    retention,
177                )
178            }
179        }
180    }
181
182    #[tokio::test]
183    async fn run_passes_when_all_expectations_met() {
184        let case = eval_case(
185            r#"{"name":"expectations","task":{"prompt":"do the thing","workspace":{"files":{"notes.txt":"beta\n"}}},"expect":{"toolCalls":{"bash":{"exactly":1}},"files":{"notes.txt":"beta\n"},"filesContain":{"notes.txt":"beta"}}}"#,
186        );
187
188        let outcome =
189            run_case_with_agent(case, &FakeAgent::with_tool_call("bash", "ok"), WorkspaceRetention::Discard).await;
190
191        assert!(outcome.passed, "failures: {:?}", outcome.failures);
192    }
193
194    #[tokio::test]
195    async fn run_reports_tool_calls_for_native_assertions() {
196        let case = eval_case(r#"{"name":"observations","task":{"prompt":"p"}}"#);
197
198        let outcome = run_case_with_agent(
199            case,
200            &FakeAgent::with_tool_call("weather__get_current", "ok"),
201            WorkspaceRetention::Discard,
202        )
203        .await;
204
205        assert_eq!(outcome.tool_calls.len(), 1);
206        assert_eq!(outcome.tool_calls[0].name, "weather__get_current");
207        assert_eq!(outcome.tool_calls[0].arguments, Some(serde_json::json!({})));
208        assert_eq!(outcome.tool_calls[0].raw_arguments, "{}");
209    }
210
211    #[tokio::test]
212    async fn run_keep_workspace_persists_directory_and_reports_path() {
213        let case = eval_case(r#"{"name":"keep","task":{"prompt":"p","workspace":{"files":{"notes.txt":"hi\n"}}}}"#);
214
215        let outcome =
216            run_case_with_agent(case, &FakeAgent::with_tool_call("bash", "ok"), WorkspaceRetention::Retain).await;
217
218        let workspace = outcome.retained_workspace.expect("workspace is reported when retained");
219        assert!(workspace.root_path.exists(), "retained workspace should outlive the run");
220        assert_eq!(workspace.root_path, workspace.path);
221        assert_eq!(read_to_string(workspace.path.join("notes.txt")).unwrap(), "hi\n");
222        remove_dir_all(&workspace.root_path).expect("caller cleans up the retained workspace");
223    }
224
225    #[tokio::test]
226    async fn run_reports_no_retained_workspace_when_not_retained() {
227        let case = eval_case(r#"{"name":"drop","task":{"prompt":"p","workspace":{"files":{"a.txt":"x"}}}}"#);
228
229        let outcome =
230            run_case_with_agent(case, &FakeAgent::with_tool_call("bash", "ok"), WorkspaceRetention::Discard).await;
231
232        assert!(outcome.retained_workspace.is_none());
233    }
234
235    #[tokio::test]
236    async fn run_reports_missing_tool_and_wrong_count() {
237        let case = eval_case(
238            r#"{"name":"counts","task":{"prompt":"p"},"expect":{"toolCalls":{"read":{"atLeast":1},"bash":{"exactly":2}}}}"#,
239        );
240
241        let outcome =
242            run_case_with_agent(case, &FakeAgent::with_tool_call("bash", "ok"), WorkspaceRetention::Discard).await;
243
244        assert!(!outcome.passed);
245        assert!(outcome.failures.iter().any(|f| f.contains("`read`")));
246        assert!(outcome.failures.iter().any(|f| f.contains("`bash`") && f.contains("2 time")));
247    }
248
249    #[tokio::test]
250    async fn run_reports_file_mismatch_and_missing_file() {
251        let case = eval_case(
252            r#"{"name":"files","task":{"prompt":"p","workspace":{"files":{"notes.txt":"alpha\n"}}},"expect":{"files":{"notes.txt":"beta\n"},"filesContain":{"missing.txt":"x"}}}"#,
253        );
254
255        let outcome =
256            run_case_with_agent(case, &FakeAgent::with_tool_call("bash", "ok"), WorkspaceRetention::Discard).await;
257
258        assert!(!outcome.passed);
259        assert!(outcome.failures.iter().any(|f| f.contains("`notes.txt`") && f.contains("mismatch")));
260        assert!(outcome.failures.iter().any(|f| f.contains("`missing.txt`") && f.contains("could not be read")));
261    }
262
263    #[tokio::test]
264    async fn run_records_fault_as_failed_eval_instead_of_aborting() {
265        let base_dir = tempfile::tempdir().unwrap();
266        let case = eval_case_in(
267            r#"{"name":"broken","task":{"prompt":"p","workspace":{"dir":"missing-dir"}}}"#,
268            base_dir.path(),
269        );
270
271        let outcome =
272            run_case_with_agent(case, &FakeAgent::with_tool_call("bash", "ok"), WorkspaceRetention::Discard).await;
273
274        assert!(!outcome.passed);
275        assert!(outcome.failures[0].contains("workspace setup failed"), "got: {:?}", outcome.failures);
276    }
277
278    #[test]
279    fn load_rejects_missing_docker_section() {
280        let error = serde_json::from_str::<EvalSpec>(r#"{"name":"c","task":{"prompt":"p"}}"#).unwrap_err();
281
282        assert!(error.to_string().contains("missing field `docker`"), "got: {error}");
283    }
284
285    #[test]
286    fn load_rejects_missing_dockerfile() {
287        let base_dir = tempfile::tempdir().unwrap();
288        let eval: EvalSpec = serde_json::from_str(
289            r#"{"docker":{"file":"Dockerfile"},"name":"x","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
290        )
291        .unwrap();
292
293        let error = ResolvedEvalSpec::resolve(eval, base_dir.path()).unwrap_err();
294
295        assert!(matches!(error, EvalFileError::DockerPath { .. }));
296    }
297
298    #[tokio::test]
299    async fn run_evals_rejects_conflicting_image_tags_before_running() {
300        let dir = tempfile::tempdir().unwrap();
301        for (subdir, name) in [("a", "one"), ("b", "two")] {
302            let eval_dir = dir.path().join(subdir);
303            std::fs::create_dir_all(&eval_dir).unwrap();
304            std::fs::write(eval_dir.join("Dockerfile"), "FROM scratch").unwrap();
305            std::fs::write(
306                eval_dir.join(format!("{name}.eval.json")),
307                format!(
308                    r#"{{"docker":{{"file":"Dockerfile","image":"same:tag"}},"name":"{name}","agent":{{"command":["agent"]}},"task":{{"prompt":"p"}}}}"#
309                ),
310            )
311            .unwrap();
312        }
313        let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions { paths: vec![dir.path().to_path_buf()], filter: None })
314            .unwrap();
315        let error =
316            run_eval_specs(cases, WorkspaceRetention::Discard, NonZeroUsize::new(2).unwrap()).await.unwrap_err();
317
318        assert!(matches!(error, EvalFileError::ImageBuild(ImageBuildError::ConflictingTag { .. })));
319    }
320
321    #[tokio::test]
322    async fn run_evals_rejects_invalid_judge_model_before_running() {
323        let dir = tempfile::tempdir().unwrap();
324        std::fs::write(
325            dir.path().join("bad-judge.eval.json"),
326            r#"{"docker":{"image":"sandbox:latest"},"name":"bad-judge","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"not-a-provider:nope","criteria":[{"id":"a","description":"ok"}]}}}"#,
327        )
328        .unwrap();
329
330        let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions { paths: vec![dir.path().to_path_buf()], filter: None })
331            .unwrap();
332        let error =
333            run_eval_specs(cases, WorkspaceRetention::Discard, NonZeroUsize::new(1).unwrap()).await.unwrap_err();
334
335        assert!(matches!(error, EvalFileError::JudgeModel { .. }), "got: {error:?}");
336    }
337
338    #[test]
339    fn load_reads_spec_and_returns_its_directory() {
340        let dir = tempfile::tempdir().unwrap();
341        let path = dir.path().join("example.eval.json");
342        std::fs::write(
343            &path,
344            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
345        )
346        .unwrap();
347
348        let case = ResolvedEvalSpec::load_file(&path).unwrap();
349
350        let docker = &case.docker;
351        assert_eq!(docker.image, DockerImage::new("sandbox", "latest"));
352        assert!(docker.build.is_none());
353        assert_eq!(case.name, "c");
354        assert_eq!(case.base_dir, dir.path().canonicalize().unwrap());
355    }
356
357    #[test]
358    fn load_resolves_dockerfile_build_relative_to_spec() {
359        let dir = tempfile::tempdir().unwrap();
360        std::fs::write(dir.path().join("Dockerfile"), "FROM scratch").unwrap();
361        let path = dir.path().join("example.eval.json");
362        std::fs::write(
363            &path,
364            r#"{"docker":{"file":"Dockerfile","image":"sandbox:dev"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
365        )
366        .unwrap();
367
368        let case = ResolvedEvalSpec::load_file(&path).unwrap();
369
370        let docker = &case.docker;
371        assert_eq!(docker.image, DockerImage::new("sandbox", "dev"));
372        let build = docker.build.as_ref().expect("expected a build request");
373        assert_eq!(build.dockerfile, dir.path().canonicalize().unwrap().join("Dockerfile"));
374        assert_eq!(build.context, dir.path().canonicalize().unwrap());
375        assert_eq!(build.tag, "sandbox:dev");
376    }
377
378    #[test]
379    fn load_distinguishes_missing_file_from_invalid_json() {
380        let missing = ResolvedEvalSpec::load_file("/nonexistent/example.eval.json").unwrap_err();
381        assert!(matches!(missing, EvalFileError::ReadEvalFile { .. }));
382
383        let dir = tempfile::tempdir().unwrap();
384        let path = dir.path().join("example.eval.json");
385        std::fs::write(&path, "not json").unwrap();
386        assert!(matches!(ResolvedEvalSpec::load_file(&path).unwrap_err(), EvalFileError::ParseEvalFile { .. }));
387    }
388
389    #[test]
390    fn load_resolves_shared_judge_file_relative_to_spec() {
391        let dir = tempfile::tempdir().unwrap();
392        std::fs::create_dir_all(dir.path().join("shared")).unwrap();
393        std::fs::write(
394            dir.path().join("shared/maintainer.judge.json"),
395            r#"{"model":"anthropic:claude-sonnet-4-5","instructions":"grade it","criteria":[{"id":"behavior","description":"works"}]}"#,
396        )
397        .unwrap();
398        let path = dir.path().join("nested/example.eval.json");
399        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
400        std::fs::write(
401            &path,
402            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":"../shared/maintainer.judge.json"}}"#,
403        )
404        .unwrap();
405
406        let case = ResolvedEvalSpec::load_file(&path).unwrap();
407
408        let judge = case.judge.as_ref().expect("expected a resolved judge");
409        assert_eq!(judge.model, "anthropic:claude-sonnet-4-5");
410        assert_eq!(judge.instructions.as_deref(), Some("grade it"));
411        assert_eq!(judge.criteria[0].id, "behavior");
412    }
413
414    #[test]
415    fn load_rejects_missing_or_invalid_shared_judge_file() {
416        let dir = tempfile::tempdir().unwrap();
417        let path = dir.path().join("example.eval.json");
418        std::fs::write(
419            &path,
420            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":"missing.judge.json"}}"#,
421        )
422        .unwrap();
423        assert!(matches!(ResolvedEvalSpec::load_file(&path).unwrap_err(), EvalFileError::ReadJudgeFile { .. }));
424
425        std::fs::write(dir.path().join("bad.judge.json"), r#"{"model":"m","criteria":[]}"#).unwrap();
426        std::fs::write(
427            &path,
428            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":"bad.judge.json"}}"#,
429        )
430        .unwrap();
431        let error = ResolvedEvalSpec::load_file(&path).unwrap_err();
432        assert!(matches!(&error, EvalFileError::InvalidInlineJudge { .. }));
433        assert!(error.to_string().contains("criteria must not be empty"), "got: {error}");
434    }
435
436    #[test]
437    fn discovers_and_loads_specs_recursively() {
438        let dir = tempfile::tempdir().unwrap();
439        write_named_spec(dir.path().join("smoke.eval.json"), "smoke");
440        write_named_spec(dir.path().join("nested/edit.eval.json"), "edit");
441        write_named_spec(dir.path().join("nested/settings.json"), "settings");
442
443        let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions { paths: vec![dir.path().to_path_buf()], filter: None })
444            .unwrap();
445        let names = cases.iter().map(|case| case.name.clone()).collect::<Vec<_>>();
446
447        assert_eq!(names, vec!["edit", "smoke"]);
448    }
449
450    #[test]
451    fn explicit_file_path_is_loaded_regardless_of_name() {
452        let dir = tempfile::tempdir().unwrap();
453        let path = dir.path().join("eval.json");
454        write_named_spec(&path, "explicit");
455
456        let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions { paths: vec![path], filter: None }).unwrap();
457
458        assert_eq!(cases.len(), 1);
459        assert_eq!(cases[0].name, "explicit");
460        assert_eq!(cases[0].base_dir, dir.path().canonicalize().unwrap());
461    }
462
463    #[test]
464    fn load_filters_by_eval_name() {
465        let dir = tempfile::tempdir().unwrap();
466        write_named_spec(dir.path().join("smoke.eval.json"), "smoke");
467        write_named_spec(dir.path().join("edit.eval.json"), "edit");
468
469        let cases = ResolvedEvalSpec::load(EvalSpecLoadOptions {
470            paths: vec![dir.path().to_path_buf()],
471            filter: Some("smoke".to_string()),
472        })
473        .unwrap();
474
475        assert_eq!(cases.len(), 1);
476        assert_eq!(cases[0].name, "smoke");
477    }
478
479    #[test]
480    fn load_reports_missing_filter_match() {
481        let dir = tempfile::tempdir().unwrap();
482        write_named_spec(dir.path().join("smoke.eval.json"), "smoke");
483
484        let error = ResolvedEvalSpec::load(EvalSpecLoadOptions {
485            paths: vec![dir.path().to_path_buf()],
486            filter: Some("missing".to_string()),
487        })
488        .unwrap_err();
489
490        assert!(matches!(error, EvalFileError::NoMatchingEval { .. }));
491    }
492
493    #[test]
494    fn discover_reports_missing_paths() {
495        let missing = ResolvedEvalSpec::load(EvalSpecLoadOptions {
496            paths: vec![PathBuf::from("/nonexistent/evals")],
497            filter: None,
498        })
499        .unwrap_err();
500
501        assert!(matches!(missing, EvalFileError::EvalPathNotFound { .. }));
502    }
503
504    fn write_named_spec(path: impl AsRef<Path>, name: &str) {
505        let path = path.as_ref();
506        if let Some(parent) = path.parent() {
507            std::fs::create_dir_all(parent).unwrap();
508        }
509        std::fs::write(
510            path,
511            format!(r#"{{"docker":{{"image":"sandbox:latest"}},"name":"{name}","agent":{{"command":["agent"]}},"task":{{"prompt":"p"}}}}"#),
512        )
513        .unwrap();
514    }
515}
aether_evals/spec/runner.rs

aether_evals/spec/
runner.rs
