aether_evals/spec/
runner.rs

1use super::error::EvalFileError;
2use super::report::EvalFilesReport;
3use super::{EvalCase, EvalSpec};
4use crate::agents::build_images;
5use futures::StreamExt;
6use llm::parser::ModelProviderParser;
7use std::collections::BTreeSet;
8use std::num::NonZeroUsize;
9use std::path::PathBuf;
10
11const DEFAULT_EVALS_DIR: &str = "evals";
12
13/// Request to discover and run a collection of declarative eval files.
14#[derive(Debug, Clone)]
15pub struct EvalRunOptions {
16    /// Eval files or directories to discover eval files under. Empty means `./evals`.
17    pub paths: Vec<PathBuf>,
18    /// When set, run only the eval with this name.
19    pub filter: Option<String>,
20    pub max_concurrency: NonZeroUsize,
21}
22
23/// Discover, validate, and run declarative eval files. Agents and judge models are assembled
24/// before anything runs, so a broken case fails fast. Docker image builds are deduplicated and
25/// run concurrently before eval execution. Outcomes are returned in discovery order.
26pub async fn run_eval_files(options: EvalRunOptions) -> Result<EvalFilesReport, EvalFileError> {
27    let paths = if options.paths.is_empty() { vec![PathBuf::from(DEFAULT_EVALS_DIR)] } else { options.paths };
28    let mut cases = EvalSpec::load_all(&paths)?;
29    if cases.is_empty() {
30        return Err(EvalFileError::NoEvalFilesFound { paths });
31    }
32
33    reject_duplicate_names(&cases)?;
34
35    if let Some(name) = &options.filter {
36        cases.retain(|case| &case.eval.name == name);
37        if cases.is_empty() {
38            return Err(EvalFileError::NoMatchingEval { name: name.clone() });
39        }
40    }
41
42    let parser = ModelProviderParser::default();
43    let mut ready = Vec::with_capacity(cases.len());
44    for case in cases {
45        let agent = case.agent()?;
46        let judge_llm = match &case.judge {
47            Some(judge) => {
48                let (llm, _) = parser
49                    .parse(&judge.model)
50                    .await
51                    .map_err(|source| EvalFileError::JudgeModel { model: judge.model.clone(), source })?;
52                Some(llm)
53            }
54            None => None,
55        };
56        ready.push((case, agent, judge_llm));
57    }
58
59    let builds = ready.iter().filter_map(|(case, ..)| case.docker.as_ref().and_then(|docker| docker.build.clone()));
60    build_images(builds, options.max_concurrency).await?;
61
62    let evals = futures::stream::iter(
63        ready
64            .into_iter()
65            .map(|(case, agent, judge_llm)| async move { case.run_with(&agent, judge_llm.as_deref()).await }),
66    )
67    .buffered(options.max_concurrency.get())
68    .collect()
69    .await;
70
71    Ok(EvalFilesReport { evals })
72}
73
74fn reject_duplicate_names(cases: &[EvalCase]) -> Result<(), EvalFileError> {
75    let mut names = BTreeSet::new();
76    for case in cases {
77        if !names.insert(case.eval.name.as_str()) {
78            return Err(EvalFileError::DuplicateEvalName { name: case.eval.name.clone() });
79        }
80    }
81    Ok(())
82}
83
84#[cfg(test)]
85mod tests {
86    use super::*;
87    use crate::FakeAgent;
88    use crate::agents::{DockerImage, ImageBuildError};
89    use crate::spec::{EvalCase, EvalFileError, SettingsRef};
90    use std::path::{Path, PathBuf};
91
92    fn eval_case(json: &str) -> EvalCase {
93        EvalCase::new(serde_json::from_str(json).unwrap(), ".").unwrap()
94    }
95
96    fn eval_case_in(json: &str, base_dir: &Path) -> EvalCase {
97        EvalCase::new(serde_json::from_str(json).unwrap(), base_dir).unwrap()
98    }
99
100    #[tokio::test]
101    async fn run_with_passes_when_all_expectations_met() {
102        let case = eval_case(
103            r#"{
104                "name": "expectations",
105                "prompt": "do the thing",
106                "workspace": { "files": { "notes.txt": "beta\n" } },
107                "expect": {
108                    "toolCalls": { "bash": { "exactly": 1 } },
109                    "files": { "notes.txt": "beta\n" },
110                    "filesContain": { "notes.txt": "beta" }
111                }
112            }"#,
113        );
114
115        let outcome = case.run_with(&FakeAgent::with_tool_call("bash", "ok"), None).await;
116
117        assert!(outcome.passed, "failures: {:?}", outcome.failures);
118    }
119
120    #[tokio::test]
121    async fn run_with_reports_missing_tool_and_wrong_count() {
122        let case = eval_case(
123            r#"{
124                "name": "counts",
125                "prompt": "p",
126                "expect": { "toolCalls": { "read": { "atLeast": 1 }, "bash": { "exactly": 2 } } }
127            }"#,
128        );
129
130        let outcome = case.run_with(&FakeAgent::with_tool_call("bash", "ok"), None).await;
131
132        assert!(!outcome.passed);
133        assert!(outcome.failures.iter().any(|f| f.contains("`read`")));
134        assert!(outcome.failures.iter().any(|f| f.contains("`bash`") && f.contains("2 time")));
135    }
136
137    #[tokio::test]
138    async fn run_with_reports_file_mismatch_and_missing_file() {
139        let case = eval_case(
140            r#"{
141                "name": "files",
142                "prompt": "p",
143                "workspace": { "files": { "notes.txt": "alpha\n" } },
144                "expect": { "files": { "notes.txt": "beta\n" }, "filesContain": { "missing.txt": "x" } }
145            }"#,
146        );
147
148        let outcome = case.run_with(&FakeAgent::with_tool_call("bash", "ok"), None).await;
149
150        assert!(!outcome.passed);
151        assert!(outcome.failures.iter().any(|f| f.contains("`notes.txt`") && f.contains("mismatch")));
152        assert!(outcome.failures.iter().any(|f| f.contains("`missing.txt`") && f.contains("could not be read")));
153    }
154
155    #[tokio::test]
156    async fn run_with_records_fault_as_failed_eval_instead_of_aborting() {
157        let base_dir = tempfile::tempdir().unwrap();
158        let case = eval_case_in(r#"{"name":"broken","prompt":"p","workspace":{"dir":"missing-dir"}}"#, base_dir.path());
159
160        let outcome = case.run_with(&FakeAgent::with_tool_call("bash", "ok"), None).await;
161
162        assert!(!outcome.passed);
163        assert!(outcome.failures[0].contains("workspace setup failed"), "got: {:?}", outcome.failures);
164    }
165
166    #[test]
167    fn agent_rejects_case_without_docker_section() {
168        let case = eval_case(r#"{"name":"c","prompt":"p"}"#);
169
170        let error = case.agent().map(|_| ()).unwrap_err();
171
172        assert!(matches!(error, EvalFileError::NoImage));
173    }
174
175    #[test]
176    fn load_rejects_missing_dockerfile() {
177        let base_dir = tempfile::tempdir().unwrap();
178        let eval = serde_json::from_str(r#"{"docker":{"file":"Dockerfile"},"name":"x","prompt":"p"}"#).unwrap();
179
180        let error = EvalCase::new(eval, base_dir.path()).unwrap_err();
181
182        assert!(matches!(error, EvalFileError::DockerPath { .. }));
183    }
184
185    #[tokio::test]
186    async fn run_evals_rejects_conflicting_image_tags_before_running() {
187        let dir = tempfile::tempdir().unwrap();
188        for (subdir, name) in [("a", "one"), ("b", "two")] {
189            let eval_dir = dir.path().join(subdir);
190            std::fs::create_dir_all(&eval_dir).unwrap();
191            std::fs::write(eval_dir.join("Dockerfile"), "FROM scratch").unwrap();
192            std::fs::write(
193                eval_dir.join(format!("{name}.eval.json")),
194                format!(r#"{{"docker":{{"file":"Dockerfile","image":"same:tag"}},"name":"{name}","prompt":"p"}}"#),
195            )
196            .unwrap();
197        }
198        let error = run_eval_files(EvalRunOptions {
199            paths: vec![dir.path().to_path_buf()],
200            filter: None,
201            max_concurrency: NonZeroUsize::new(2).unwrap(),
202        })
203        .await
204        .unwrap_err();
205
206        assert!(matches!(error, EvalFileError::ImageBuild(ImageBuildError::ConflictingTag { .. })));
207    }
208
209    #[tokio::test]
210    async fn run_evals_rejects_invalid_judge_model_before_running() {
211        let dir = tempfile::tempdir().unwrap();
212        std::fs::write(
213            dir.path().join("bad-judge.eval.json"),
214            r#"{"docker":{"image":"sandbox:latest"},"name":"bad-judge","prompt":"p","expect":{"judge":{"model":"not-a-provider:nope","criteria":[{"id":"a","description":"ok"}]}}}"#,
215        )
216        .unwrap();
217
218        let error = run_eval_files(EvalRunOptions {
219            paths: vec![dir.path().to_path_buf()],
220            filter: None,
221            max_concurrency: NonZeroUsize::new(1).unwrap(),
222        })
223        .await
224        .unwrap_err();
225
226        assert!(matches!(error, EvalFileError::JudgeModel { .. }), "got: {error:?}");
227    }
228
229    #[test]
230    fn load_reads_spec_and_returns_its_directory() {
231        let dir = tempfile::tempdir().unwrap();
232        let path = dir.path().join("example.eval.json");
233        std::fs::write(&path, r#"{"docker":{"image":"sandbox:latest"},"name":"c","prompt":"p"}"#).unwrap();
234
235        let case = EvalSpec::load(&path).unwrap();
236
237        let docker = case.docker.as_ref().expect("expected a resolved docker image");
238        assert_eq!(docker.image, DockerImage::new("sandbox", "latest"));
239        assert!(docker.build.is_none());
240        assert_eq!(case.eval.name, "c");
241        assert_eq!(case.base_dir, dir.path().canonicalize().unwrap());
242    }
243
244    #[test]
245    fn load_resolves_dockerfile_build_relative_to_spec() {
246        let dir = tempfile::tempdir().unwrap();
247        std::fs::write(dir.path().join("Dockerfile"), "FROM scratch").unwrap();
248        let path = dir.path().join("example.eval.json");
249        std::fs::write(&path, r#"{"docker":{"file":"Dockerfile","image":"sandbox:dev"},"name":"c","prompt":"p"}"#)
250            .unwrap();
251
252        let case = EvalSpec::load(&path).unwrap();
253
254        let docker = case.docker.as_ref().expect("expected a resolved docker image");
255        assert_eq!(docker.image, DockerImage::new("sandbox", "dev"));
256        let build = docker.build.as_ref().expect("expected a build request");
257        assert_eq!(build.dockerfile, dir.path().canonicalize().unwrap().join("Dockerfile"));
258        assert_eq!(build.context, dir.path().canonicalize().unwrap());
259        assert_eq!(build.tag, "sandbox:dev");
260    }
261
262    #[test]
263    fn load_distinguishes_missing_file_from_invalid_json() {
264        let missing = EvalSpec::load("/nonexistent/example.eval.json").unwrap_err();
265        assert!(matches!(missing, EvalFileError::ReadEvalFile { .. }));
266
267        let dir = tempfile::tempdir().unwrap();
268        let path = dir.path().join("example.eval.json");
269        std::fs::write(&path, "not json").unwrap();
270        assert!(matches!(EvalSpec::load(&path).unwrap_err(), EvalFileError::ParseEvalFile { .. }));
271    }
272
273    #[test]
274    fn load_resolves_shared_judge_file_relative_to_spec() {
275        let dir = tempfile::tempdir().unwrap();
276        std::fs::create_dir_all(dir.path().join("shared")).unwrap();
277        std::fs::write(
278            dir.path().join("shared/maintainer.judge.json"),
279            r#"{"model":"anthropic:claude-sonnet-4-5","instructions":"grade it","criteria":[{"id":"behavior","description":"works"}]}"#,
280        )
281        .unwrap();
282        let path = dir.path().join("nested/example.eval.json");
283        std::fs::create_dir_all(path.parent().unwrap()).unwrap();
284        std::fs::write(
285            &path,
286            r#"{"docker":{"image":"sandbox:latest"},"name":"c","prompt":"p","expect":{"judge":"../shared/maintainer.judge.json"}}"#,
287        )
288        .unwrap();
289
290        let case = EvalSpec::load(&path).unwrap();
291
292        let judge = case.judge.as_ref().expect("expected a resolved judge");
293        assert_eq!(judge.model, "anthropic:claude-sonnet-4-5");
294        assert_eq!(judge.instructions.as_deref(), Some("grade it"));
295        assert_eq!(judge.criteria[0].id, "behavior");
296    }
297
298    #[test]
299    fn load_rejects_missing_or_invalid_shared_judge_file() {
300        let dir = tempfile::tempdir().unwrap();
301        let path = dir.path().join("example.eval.json");
302        std::fs::write(
303            &path,
304            r#"{"docker":{"image":"sandbox:latest"},"name":"c","prompt":"p","expect":{"judge":"missing.judge.json"}}"#,
305        )
306        .unwrap();
307        assert!(matches!(EvalSpec::load(&path).unwrap_err(), EvalFileError::ReadJudgeFile { .. }));
308
309        std::fs::write(dir.path().join("bad.judge.json"), r#"{"model":"m","criteria":[]}"#).unwrap();
310        std::fs::write(
311            &path,
312            r#"{"docker":{"image":"sandbox:latest"},"name":"c","prompt":"p","expect":{"judge":"bad.judge.json"}}"#,
313        )
314        .unwrap();
315        let error = EvalSpec::load(&path).unwrap_err();
316        assert!(matches!(&error, EvalFileError::InvalidInlineJudge { .. }));
317        assert!(error.to_string().contains("criteria must not be empty"), "got: {error}");
318    }
319
320    #[test]
321    fn path_settings_resolve_aether_project_resources_from_project_root() {
322        let project = tempfile::tempdir().unwrap();
323        let settings_dir = project.path().join(".aether");
324        let eval_dir = project.path().join("packages/internal-evals/examples");
325        std::fs::create_dir_all(&settings_dir).unwrap();
326        std::fs::create_dir_all(&eval_dir).unwrap();
327        std::fs::write(settings_dir.join("SYSTEM.md"), "System prompt").unwrap();
328        std::fs::write(settings_dir.join("settings.json"), r#"{"prompts":[".aether/SYSTEM.md"],"agents":[]}"#).unwrap();
329
330        let settings = SettingsRef::Path(PathBuf::from("../../../.aether/settings.json")).resolve(&eval_dir).unwrap();
331
332        assert_eq!(settings.prompts, vec![aether_project::PromptSource::Text { text: "System prompt".to_string() }]);
333    }
334
335    #[test]
336    fn discovers_and_loads_specs_recursively() {
337        let dir = tempfile::tempdir().unwrap();
338        write_named_spec(dir.path().join("smoke.eval.json"), "smoke");
339        write_named_spec(dir.path().join("nested/edit.eval.json"), "edit");
340        write_named_spec(dir.path().join("nested/settings.json"), "settings");
341
342        let cases = EvalSpec::load_all(&[dir.path().to_path_buf()]).unwrap();
343        let names = cases.iter().map(|case| case.eval.name.clone()).collect::<Vec<_>>();
344
345        assert_eq!(names, vec!["edit", "smoke"]);
346    }
347
348    #[test]
349    fn explicit_file_path_is_loaded_regardless_of_name() {
350        let dir = tempfile::tempdir().unwrap();
351        let path = dir.path().join("eval.json");
352        write_named_spec(&path, "explicit");
353
354        let cases = EvalSpec::load_all(&[path]).unwrap();
355
356        assert_eq!(cases.len(), 1);
357        assert_eq!(cases[0].eval.name, "explicit");
358        assert_eq!(cases[0].base_dir, dir.path().canonicalize().unwrap());
359    }
360
361    #[test]
362    fn discover_reports_missing_paths() {
363        let missing = EvalSpec::load_all(&[PathBuf::from("/nonexistent/evals")]).unwrap_err();
364
365        assert!(matches!(missing, EvalFileError::EvalPathNotFound { .. }));
366    }
367
368    fn write_named_spec(path: impl AsRef<Path>, name: &str) {
369        let path = path.as_ref();
370        if let Some(parent) = path.parent() {
371            std::fs::create_dir_all(parent).unwrap();
372        }
373        std::fs::write(path, format!(r#"{{"docker":{{"image":"sandbox:latest"}},"name":"{name}","prompt":"p"}}"#))
374            .unwrap();
375    }
376}
aether_evals/spec/runner.rs

aether_evals/spec/
runner.rs
