aether_evals/spec/
mod.rs

1mod error;
2mod judge;
3mod report;
4mod runner;
5mod types;
6
7pub use error::EvalFileError;
8pub use report::{EvalFilesReport, EvalOutcome, EvalToolCall, JudgeCriterionSummary, JudgeSummary};
9pub use runner::{WorkspaceRetention, run_eval_specs};
10use serde_json::from_str;
11pub(crate) use types::ResolvedDocker;
12pub use types::{AgentSpec, DockerSpec, Expect, JudgeRef, JudgeSpec, TaskSpec, ToolCallExpectation, WorkspaceSpec};
13
14use schemars::{JsonSchema, Schema, schema_for};
15use serde::Deserialize;
16use std::{
17    collections::BTreeSet,
18    fs::read_to_string,
19    path::{Path, PathBuf},
20};
21
22use crate::{CONTAINER_AETHER_HOME, DockerAgent, default_eval_env_vars};
23
24const DEFAULT_EVALS_DIR: &str = "evals";
25
26/// A JSON serializable representation of a single eval.
27#[derive(Debug, Clone, Deserialize, JsonSchema)]
28#[serde(rename_all = "camelCase", deny_unknown_fields)]
29pub struct EvalSpec {
30    pub name: String,
31    /// Docker configuration (image, Dockerfile, build context).
32    pub docker: DockerSpec,
33
34    /// Command that emits newline-delimited `AgentMessage` JSON on stdout.
35    pub agent: AgentSpec,
36
37    pub task: TaskSpec,
38    #[serde(default)]
39    pub expect: Expect,
40}
41
42impl EvalSpec {
43    /// JSON schema for the eval spec (the input authored as JSON / generated as a TypeScript type).
44    pub fn schema() -> Schema {
45        schema_for!(Self)
46    }
47}
48
49/// Options for discovering and loading resolved eval specs.
50#[derive(Debug, Clone, Default)]
51pub struct EvalSpecLoadOptions {
52    /// Eval files or directories to discover eval files under. Empty means `./evals`.
53    pub paths: Vec<PathBuf>,
54    /// When set, load only the eval with this name.
55    pub filter: Option<String>,
56}
57
58/// An [`EvalSpec`] resolved against its base directory.
59#[derive(Debug)]
60pub struct ResolvedEvalSpec {
61    pub name: String,
62    pub base_dir: PathBuf,
63    pub(crate) docker: ResolvedDocker,
64    pub(crate) agent: AgentSpec,
65    pub(crate) task: TaskSpec,
66    pub(crate) expect: Expect,
67    pub(crate) judge: Option<JudgeSpec>,
68}
69
70impl ResolvedEvalSpec {
71    /// Discover, read, and parse eval files using the provided options. Directory discovery is
72    /// recursive and includes files ending in `.eval.json`. Explicit file paths are always treated
73    /// as eval files, regardless of filename.
74    pub fn load(options: EvalSpecLoadOptions) -> Result<Vec<Self>, EvalFileError> {
75        let paths = if options.paths.is_empty() { vec![PathBuf::from(DEFAULT_EVALS_DIR)] } else { options.paths };
76        let mut spec_paths = Vec::new();
77        for path in &paths {
78            discover_specs_at(path, &mut spec_paths)?;
79        }
80
81        spec_paths.sort();
82        spec_paths.dedup();
83        let mut cases = spec_paths.into_iter().map(Self::load_file).collect::<Result<Vec<_>, _>>()?;
84        if cases.is_empty() {
85            return Err(EvalFileError::NoEvalFilesFound { paths });
86        }
87
88        reject_duplicate_names(&cases)?;
89
90        if let Some(name) = &options.filter {
91            cases.retain(|case| &case.name == name);
92            if cases.is_empty() {
93                return Err(EvalFileError::NoMatchingEval { name: name.clone() });
94            }
95        }
96
97        Ok(cases)
98    }
99
100    /// Read and parse an eval file, resolving it against its base directory (the eval file's
101    /// parent).
102    pub fn load_file(path: impl AsRef<Path>) -> Result<Self, EvalFileError> {
103        let path = path.as_ref();
104        let path =
105            path.canonicalize().map_err(|source| EvalFileError::ReadEvalFile { path: path.to_path_buf(), source })?;
106
107        let content =
108            read_to_string(&path).map_err(|source| EvalFileError::ReadEvalFile { path: path.clone(), source })?;
109
110        let eval: EvalSpec =
111            from_str(&content).map_err(|source| EvalFileError::ParseEvalFile { path: path.clone(), source })?;
112
113        let base_dir = path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf();
114        Self::resolve(eval, base_dir)
115    }
116
117    /// Resolve a parsed eval file against `base_dir`, resolving the Docker image and reading
118    /// referenced judge files.
119    pub fn resolve(eval: EvalSpec, base_dir: impl Into<PathBuf>) -> Result<Self, EvalFileError> {
120        let EvalSpec { name, docker, agent, task, expect } = eval;
121        let base_dir = base_dir.into();
122        agent.validate().map_err(|message| EvalFileError::InvalidAgentCommand { message })?;
123
124        let docker = docker.resolve(&base_dir)?;
125        let judge = expect.judge.as_ref().map(|judge| judge.resolve(&base_dir)).transpose()?;
126
127        Ok(Self { name, base_dir, docker, agent, task, expect, judge })
128    }
129}
130
131impl ResolvedEvalSpec {
132    pub(crate) fn name(&self) -> &str {
133        &self.name
134    }
135
136    pub(crate) fn task(&self) -> Result<crate::Task, crate::WorkspaceError> {
137        self.task.build(&self.base_dir)
138    }
139
140    pub(crate) fn agent(&self) -> DockerAgent {
141        DockerAgent::new(self.docker.image.clone(), self.agent.command.clone())
142            .with_env_vars(default_eval_env_vars())
143            .with_env_vars(self.agent.env.clone())
144            .with_ephemeral_mount(CONTAINER_AETHER_HOME)
145    }
146
147    pub(crate) fn expectations(&self) -> &Expect {
148        &self.expect
149    }
150}
151
152fn discover_specs_at(path: &Path, spec_paths: &mut Vec<PathBuf>) -> Result<(), EvalFileError> {
153    if path.is_file() {
154        spec_paths.push(path.to_path_buf());
155        return Ok(());
156    }
157
158    if !path.is_dir() {
159        return Err(EvalFileError::EvalPathNotFound { path: path.to_path_buf() });
160    }
161
162    let mut entries = std::fs::read_dir(path)
163        .map_err(|source| EvalFileError::DiscoverEvalFiles { path: path.to_path_buf(), source })?
164        .collect::<Result<Vec<_>, _>>()
165        .map_err(|source| EvalFileError::DiscoverEvalFiles { path: path.to_path_buf(), source })?;
166
167    entries.sort_by_key(std::fs::DirEntry::path);
168    for entry in entries {
169        let path = entry.path();
170        if path.is_dir() {
171            discover_specs_at(&path, spec_paths)?;
172        } else if is_discovered_spec(&path) {
173            spec_paths.push(path);
174        }
175    }
176
177    Ok(())
178}
179
180fn is_discovered_spec(path: &Path) -> bool {
181    let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
182        return false;
183    };
184
185    file_name.ends_with(".eval.json")
186}
187
188fn reject_duplicate_names(cases: &[ResolvedEvalSpec]) -> Result<(), EvalFileError> {
189    let mut names = BTreeSet::new();
190    for case in cases {
191        if !names.insert(case.name.as_str()) {
192            return Err(EvalFileError::DuplicateEvalName { name: case.name.clone() });
193        }
194    }
195    Ok(())
196}
197
198#[cfg(test)]
199mod tests {
200    use super::*;
201
202    #[test]
203    fn parses_camel_case_spec_with_inline_files() {
204        let spec: EvalSpec = serde_json::from_str(
205            r#"{"docker":{"file":"Dockerfile","image":"sandbox:latest","context":"../../.."},"name":"edit","agent":{"command":["agent"]},"task":{"prompt":"do it","workspace":{"files":{"a.txt":"hi"}}},"expect":{"toolCalls":{"coding__read_file":{"atLeast":1},"coding__edit_file":{"exactly":1}},"files":{"a.txt":"bye"},"judge":{"model":"anthropic:claude-sonnet-4-5","instructions":"grade it","contextFiles":["a.txt"],"criteria":[{"id":"behavior","description":"did it work?"},{"id":"clarity","description":"explained it","blocking":false,"weight":0.5,"threshold":0.7}]}}}"#,
206        )
207        .unwrap();
208
209        assert!(matches!(
210            &spec.docker,
211            DockerSpec::Build { file, image: Some(image), context: Some(context) }
212                if file == Path::new("Dockerfile") && image == "sandbox:latest" && context == Path::new("../../..")
213        ));
214        assert_eq!(spec.name, "edit");
215        assert!(matches!(&spec.task.workspace, WorkspaceSpec::Files(files) if files["a.txt"] == "hi"));
216        assert!(matches!(spec.expect.tool_calls["coding__read_file"], ToolCallExpectation::AtLeast(1)));
217        assert!(matches!(spec.expect.tool_calls["coding__edit_file"], ToolCallExpectation::Exactly(1)));
218        let Some(JudgeRef::Inline(judge)) = &spec.expect.judge else {
219            panic!("expected an inline judge, got {:?}", spec.expect.judge);
220        };
221        assert_eq!(judge.instructions.as_deref(), Some("grade it"));
222        assert_eq!(judge.context_files, vec!["a.txt".to_string()]);
223        assert_eq!(judge.criteria[0].id, "behavior");
224        assert!(judge.criteria[0].blocking);
225        assert!((judge.criteria[0].weight - 1.0).abs() < f64::EPSILON);
226        assert!((judge.criteria[0].threshold - 1.0).abs() < f64::EPSILON);
227        assert!(!judge.criteria[1].blocking);
228        assert!((judge.criteria[1].weight - 0.5).abs() < f64::EPSILON);
229        assert!((judge.criteria[1].threshold - 0.7).abs() < f64::EPSILON);
230    }
231
232    #[test]
233    fn parses_git_and_dir_workspaces() {
234        let git: EvalSpec = serde_json::from_str(
235            r#"{"docker":{"image":"sandbox:latest"},"name":"g","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"git":{"url":"u","startCommit":"a","goldCommit":"b"}}}}"#,
236        )
237        .unwrap();
238        assert!(matches!(git.task.workspace, WorkspaceSpec::Git(_)));
239
240        let dir: EvalSpec = serde_json::from_str(
241            r#"{"docker":{"image":"sandbox:latest"},"name":"d","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"dir":"fixtures/x"}}}"#,
242        )
243        .unwrap();
244        assert!(matches!(dir.task.workspace, WorkspaceSpec::Dir(_)));
245    }
246
247    #[test]
248    fn rejects_invalid_docker_specs() {
249        for (json, expected) in [
250            (r#"{"docker":{},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#, "docker must specify"),
251            (
252                r#"{"docker":{"image":"sandbox:latest","context":"x"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
253                "`context` requires a Dockerfile `file`",
254            ),
255            (
256                r#"{"docker":{"dockerfile":"Dockerfile"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
257                "unknown field `dockerfile`",
258            ),
259        ] {
260            let error = serde_json::from_str::<EvalSpec>(json).unwrap_err().to_string();
261            assert!(error.contains(expected), "expected {expected:?} in {error:?}");
262        }
263    }
264
265    #[test]
266    fn rejects_invalid_tool_call_expectations() {
267        for json in [
268            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"toolCalls":{"bash":{}}}}"#,
269            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"toolCalls":{"bash":{"atLeast":1,"exactly":1}}}}"#,
270        ] {
271            assert!(serde_json::from_str::<EvalSpec>(json).is_err(), "expected rejection for {json}");
272        }
273
274        let unknown = serde_json::from_str::<EvalSpec>(
275            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"toolCalls":{"bash":{"minimum":1}}}}"#,
276        )
277        .unwrap_err()
278        .to_string();
279        assert!(unknown.contains("unknown variant `minimum`"), "got: {unknown}");
280    }
281
282    #[test]
283    fn rejects_judge_without_model() {
284        let result: Result<EvalSpec, _> = serde_json::from_str(
285            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"criteria":[{"id":"ok","description":"ok"}]}}}"#,
286        );
287
288        assert!(result.unwrap_err().to_string().contains("missing field `model`"));
289    }
290
291    #[test]
292    fn rejects_unknown_fields() {
293        for json in [
294            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"bogus":true}"#,
295            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"runtime":{"type":"acp","command":["a"]}}"#,
296            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"baseDir":"."}"#,
297            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"judge":null}"#,
298            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"type":"aether","command":["nope"]},"task":{"prompt":"p"}}"#,
299            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["node"],"modelName":"m"},"task":{"prompt":"p"}}"#,
300        ] {
301            let result: Result<EvalSpec, _> = serde_json::from_str(json);
302            assert!(result.is_err());
303        }
304    }
305
306    #[test]
307    fn rejects_workspace_with_more_than_one_source() {
308        let result: Result<EvalSpec, _> = serde_json::from_str(
309            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"files":{},"dir":"x"}}}"#,
310        );
311
312        assert!(result.is_err());
313    }
314
315    #[test]
316    fn rejects_workspace_with_unknown_field_by_name() {
317        let result: Result<EvalSpec, _> = serde_json::from_str(
318            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"dirr":"x"}}}"#,
319        );
320
321        assert!(result.unwrap_err().to_string().contains("unknown variant `dirr`"));
322    }
323
324    #[test]
325    fn rejects_git_workspace_with_unknown_field_by_name() {
326        let result: Result<EvalSpec, _> = serde_json::from_str(
327            r#"{"docker":{"image":"sandbox:latest"},"name":"g","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"git":{"url":"u","startCommit":"a","goldComit":"b"}}}}"#,
328        );
329
330        assert!(result.unwrap_err().to_string().contains("unknown field `goldComit`"));
331    }
332
333    #[test]
334    fn rejects_invalid_judge_criteria() {
335        for (json, expected) in [
336            (
337                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[]}}}"#,
338                "criteria must not be empty",
339            ),
340            (
341                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"","description":"ok"}]}}}"#,
342                "id must not be empty",
343            ),
344            (
345                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":"ok"},{"id":"a","description":"ok"}]}}}"#,
346                "duplicate judge criterion id `a`",
347            ),
348            (
349                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":""}]}}}"#,
350                "description must not be empty",
351            ),
352            (
353                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":"ok","weight":0.0}]}}}"#,
354                "weight must be positive and finite",
355            ),
356            (
357                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":"ok","threshold":1.5}]}}}"#,
358                "threshold must be between 0.0 and 1.0",
359            ),
360        ] {
361            let eval = serde_json::from_str::<EvalSpec>(json).unwrap();
362            let error = ResolvedEvalSpec::resolve(eval, ".").unwrap_err().to_string();
363            assert!(error.contains(expected), "expected {expected:?} in {error:?}");
364        }
365    }
366}
aether_evals/spec/mod.rs

aether_evals/spec/
mod.rs
