aether_evals/spec/
mod.rs

1mod error;
2mod judge;
3mod report;
4mod runner;
5mod types;
6
7pub use error::EvalFileError;
8pub use judge::{Judge, JudgeBuilder, JudgeContext, JudgeCriterionResponse, JudgeError, JudgeRubricResponse, judge};
9pub use report::{EvalFilesReport, EvalOutcome, EvalStreamEvent, EvalToolCall, JudgeCriterionSummary, JudgeSummary};
10pub use runner::{WorkspaceRetention, run_eval_spec_streaming, run_eval_specs};
11use serde_json::from_str;
12pub(crate) use types::ResolvedDocker;
13pub use types::{
14    AgentSpec, DockerSpec, Expect, JudgeCriterionSpec, JudgeRef, JudgeSpec, TaskSpec, ToolCallExpectation,
15    WorkspaceSpec,
16};
17
18use schemars::{JsonSchema, Schema, schema_for};
19use serde::Deserialize;
20use std::{
21    collections::BTreeSet,
22    fs::read_to_string,
23    path::{Path, PathBuf},
24};
25
26use crate::{CONTAINER_AETHER_HOME, DockerAgent, default_eval_env_vars};
27
28const DEFAULT_EVALS_DIR: &str = "evals";
29
30/// A JSON serializable representation of a single eval.
31#[derive(Debug, Clone, Deserialize, JsonSchema)]
32#[serde(rename_all = "camelCase", deny_unknown_fields)]
33pub struct EvalSpec {
34    pub name: String,
35    /// Docker configuration (image, Dockerfile, build context).
36    pub docker: DockerSpec,
37
38    /// Command that emits newline-delimited `AgentMessage` JSON on stdout.
39    pub agent: AgentSpec,
40
41    pub task: TaskSpec,
42    #[serde(default)]
43    pub expect: Expect,
44}
45
46impl EvalSpec {
47    /// JSON schema for the eval spec (the input authored as JSON / generated as a TypeScript type).
48    pub fn schema() -> Schema {
49        schema_for!(Self)
50    }
51}
52
53/// Options for discovering and loading resolved eval specs.
54#[derive(Debug, Clone, Default)]
55pub struct EvalSpecLoadOptions {
56    /// Eval files or directories to discover eval files under. Empty means `./evals`.
57    pub paths: Vec<PathBuf>,
58    /// When set, load only the eval with this name.
59    pub filter: Option<String>,
60}
61
62/// An [`EvalSpec`] resolved against its base directory.
63#[derive(Debug)]
64pub struct ResolvedEvalSpec {
65    pub name: String,
66    pub base_dir: PathBuf,
67    pub(crate) docker: ResolvedDocker,
68    pub(crate) agent: AgentSpec,
69    pub(crate) task: TaskSpec,
70    pub(crate) expect: Expect,
71    pub(crate) judge: Option<JudgeSpec>,
72}
73
74impl ResolvedEvalSpec {
75    /// Discover, read, and parse eval files using the provided options. Directory discovery is
76    /// recursive and includes files ending in `.eval.json`. Explicit file paths are always treated
77    /// as eval files, regardless of filename.
78    pub fn load(options: EvalSpecLoadOptions) -> Result<Vec<Self>, EvalFileError> {
79        let paths = if options.paths.is_empty() { vec![PathBuf::from(DEFAULT_EVALS_DIR)] } else { options.paths };
80        let mut spec_paths = Vec::new();
81        for path in &paths {
82            discover_specs_at(path, &mut spec_paths)?;
83        }
84
85        spec_paths.sort();
86        spec_paths.dedup();
87        let mut cases = spec_paths.into_iter().map(Self::load_file).collect::<Result<Vec<_>, _>>()?;
88        if cases.is_empty() {
89            return Err(EvalFileError::NoEvalFilesFound { paths });
90        }
91
92        reject_duplicate_names(&cases)?;
93
94        if let Some(name) = &options.filter {
95            cases.retain(|case| &case.name == name);
96            if cases.is_empty() {
97                return Err(EvalFileError::NoMatchingEval { name: name.clone() });
98            }
99        }
100
101        Ok(cases)
102    }
103
104    /// Read and parse an eval file, resolving it against its base directory (the eval file's
105    /// parent).
106    pub fn load_file(path: impl AsRef<Path>) -> Result<Self, EvalFileError> {
107        let path = path.as_ref();
108        let path =
109            path.canonicalize().map_err(|source| EvalFileError::ReadEvalFile { path: path.to_path_buf(), source })?;
110
111        let content =
112            read_to_string(&path).map_err(|source| EvalFileError::ReadEvalFile { path: path.clone(), source })?;
113
114        let eval: EvalSpec =
115            from_str(&content).map_err(|source| EvalFileError::ParseEvalFile { path: path.clone(), source })?;
116
117        let base_dir = path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf();
118        Self::resolve(eval, base_dir)
119    }
120
121    /// Resolve a parsed eval file against `base_dir`, resolving the Docker image and reading
122    /// referenced judge files.
123    pub fn resolve(eval: EvalSpec, base_dir: impl Into<PathBuf>) -> Result<Self, EvalFileError> {
124        let EvalSpec { name, docker, agent, task, expect } = eval;
125        let base_dir = base_dir.into();
126        agent.validate().map_err(|message| EvalFileError::InvalidAgentCommand { message })?;
127
128        let docker = docker.resolve(&base_dir)?;
129        let judge = expect.judge.as_ref().map(|judge| judge.resolve(&base_dir)).transpose()?;
130
131        Ok(Self { name, base_dir, docker, agent, task, expect, judge })
132    }
133}
134
135impl ResolvedEvalSpec {
136    pub(crate) fn name(&self) -> &str {
137        &self.name
138    }
139
140    pub(crate) fn task(&self) -> Result<crate::Task, crate::WorkspaceError> {
141        self.task.build(&self.base_dir)
142    }
143
144    pub(crate) fn agent(&self) -> DockerAgent {
145        DockerAgent::new(self.docker.image.clone(), self.agent.command.clone())
146            .with_env_vars(default_eval_env_vars())
147            .with_env_vars(self.agent.env.clone())
148            .with_ephemeral_mount(CONTAINER_AETHER_HOME)
149    }
150
151    pub(crate) fn expectations(&self) -> &Expect {
152        &self.expect
153    }
154}
155
156fn discover_specs_at(path: &Path, spec_paths: &mut Vec<PathBuf>) -> Result<(), EvalFileError> {
157    if path.is_file() {
158        spec_paths.push(path.to_path_buf());
159        return Ok(());
160    }
161
162    if !path.is_dir() {
163        return Err(EvalFileError::EvalPathNotFound { path: path.to_path_buf() });
164    }
165
166    let mut entries = std::fs::read_dir(path)
167        .map_err(|source| EvalFileError::DiscoverEvalFiles { path: path.to_path_buf(), source })?
168        .collect::<Result<Vec<_>, _>>()
169        .map_err(|source| EvalFileError::DiscoverEvalFiles { path: path.to_path_buf(), source })?;
170
171    entries.sort_by_key(std::fs::DirEntry::path);
172    for entry in entries {
173        let path = entry.path();
174        if path.is_dir() {
175            discover_specs_at(&path, spec_paths)?;
176        } else if is_discovered_spec(&path) {
177            spec_paths.push(path);
178        }
179    }
180
181    Ok(())
182}
183
184fn is_discovered_spec(path: &Path) -> bool {
185    let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else {
186        return false;
187    };
188
189    file_name.ends_with(".eval.json")
190}
191
192fn reject_duplicate_names(cases: &[ResolvedEvalSpec]) -> Result<(), EvalFileError> {
193    let mut names = BTreeSet::new();
194    for case in cases {
195        if !names.insert(case.name.as_str()) {
196            return Err(EvalFileError::DuplicateEvalName { name: case.name.clone() });
197        }
198    }
199    Ok(())
200}
201
202#[cfg(test)]
203mod tests {
204    use super::*;
205
206    #[test]
207    fn parses_camel_case_spec_with_inline_files() {
208        let spec: EvalSpec = serde_json::from_str(
209            r#"{"docker":{"file":"Dockerfile","image":"sandbox:latest","context":"../../.."},"name":"edit","agent":{"command":["agent"]},"task":{"prompt":"do it","workspace":{"files":{"a.txt":"hi"}}},"expect":{"toolCalls":{"coding__read_file":{"atLeast":1},"coding__edit_file":{"exactly":1}},"files":{"a.txt":"bye"},"judge":{"model":"anthropic:claude-sonnet-4-5","instructions":"grade it","contextFiles":["a.txt"],"criteria":[{"id":"behavior","description":"did it work?"},{"id":"clarity","description":"explained it","blocking":false,"weight":0.5,"threshold":0.7}]}}}"#,
210        )
211        .unwrap();
212
213        assert!(matches!(
214            &spec.docker,
215            DockerSpec::Build { file, image: Some(image), context: Some(context) }
216                if file == Path::new("Dockerfile") && image == "sandbox:latest" && context == Path::new("../../..")
217        ));
218        assert_eq!(spec.name, "edit");
219        assert!(matches!(&spec.task.workspace, WorkspaceSpec::Files(files) if files["a.txt"] == "hi"));
220        assert!(matches!(spec.expect.tool_calls["coding__read_file"], ToolCallExpectation::AtLeast(1)));
221        assert!(matches!(spec.expect.tool_calls["coding__edit_file"], ToolCallExpectation::Exactly(1)));
222        let Some(JudgeRef::Inline(judge)) = &spec.expect.judge else {
223            panic!("expected an inline judge, got {:?}", spec.expect.judge);
224        };
225        assert_eq!(judge.instructions.as_deref(), Some("grade it"));
226        assert_eq!(judge.context_files, vec!["a.txt".to_string()]);
227        assert_eq!(judge.criteria[0].id, "behavior");
228        assert!(judge.criteria[0].blocking);
229        assert!((judge.criteria[0].weight - 1.0).abs() < f64::EPSILON);
230        assert!((judge.criteria[0].threshold - 1.0).abs() < f64::EPSILON);
231        assert!(!judge.criteria[1].blocking);
232        assert!((judge.criteria[1].weight - 0.5).abs() < f64::EPSILON);
233        assert!((judge.criteria[1].threshold - 0.7).abs() < f64::EPSILON);
234    }
235
236    #[test]
237    fn parses_git_and_dir_workspaces() {
238        let git: EvalSpec = serde_json::from_str(
239            r#"{"docker":{"image":"sandbox:latest"},"name":"g","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"git":{"url":"u","startCommit":"a","goldCommit":"b"}}}}"#,
240        )
241        .unwrap();
242        assert!(matches!(git.task.workspace, WorkspaceSpec::Git(_)));
243
244        let dir: EvalSpec = serde_json::from_str(
245            r#"{"docker":{"image":"sandbox:latest"},"name":"d","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"dir":"fixtures/x"}}}"#,
246        )
247        .unwrap();
248        assert!(matches!(dir.task.workspace, WorkspaceSpec::Dir(_)));
249    }
250
251    #[test]
252    fn rejects_invalid_docker_specs() {
253        for (json, expected) in [
254            (r#"{"docker":{},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#, "docker must specify"),
255            (
256                r#"{"docker":{"image":"sandbox:latest","context":"x"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
257                "`context` requires a Dockerfile `file`",
258            ),
259            (
260                r#"{"docker":{"dockerfile":"Dockerfile"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"}}"#,
261                "unknown field `dockerfile`",
262            ),
263        ] {
264            let error = serde_json::from_str::<EvalSpec>(json).unwrap_err().to_string();
265            assert!(error.contains(expected), "expected {expected:?} in {error:?}");
266        }
267    }
268
269    #[test]
270    fn rejects_invalid_tool_call_expectations() {
271        for json in [
272            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"toolCalls":{"bash":{}}}}"#,
273            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"toolCalls":{"bash":{"atLeast":1,"exactly":1}}}}"#,
274        ] {
275            assert!(serde_json::from_str::<EvalSpec>(json).is_err(), "expected rejection for {json}");
276        }
277
278        let unknown = serde_json::from_str::<EvalSpec>(
279            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"toolCalls":{"bash":{"minimum":1}}}}"#,
280        )
281        .unwrap_err()
282        .to_string();
283        assert!(unknown.contains("unknown variant `minimum`"), "got: {unknown}");
284    }
285
286    #[test]
287    fn rejects_judge_without_model() {
288        let result: Result<EvalSpec, _> = serde_json::from_str(
289            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"criteria":[{"id":"ok","description":"ok"}]}}}"#,
290        );
291
292        assert!(result.unwrap_err().to_string().contains("missing field `model`"));
293    }
294
295    #[test]
296    fn rejects_unknown_fields() {
297        for json in [
298            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"bogus":true}"#,
299            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"runtime":{"type":"acp","command":["a"]}}"#,
300            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"baseDir":"."}"#,
301            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"judge":null}"#,
302            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"type":"aether","command":["nope"]},"task":{"prompt":"p"}}"#,
303            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["node"],"modelName":"m"},"task":{"prompt":"p"}}"#,
304        ] {
305            let result: Result<EvalSpec, _> = serde_json::from_str(json);
306            assert!(result.is_err());
307        }
308    }
309
310    #[test]
311    fn rejects_workspace_with_more_than_one_source() {
312        let result: Result<EvalSpec, _> = serde_json::from_str(
313            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"files":{},"dir":"x"}}}"#,
314        );
315
316        assert!(result.is_err());
317    }
318
319    #[test]
320    fn rejects_workspace_with_unknown_field_by_name() {
321        let result: Result<EvalSpec, _> = serde_json::from_str(
322            r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"dirr":"x"}}}"#,
323        );
324
325        assert!(result.unwrap_err().to_string().contains("unknown variant `dirr`"));
326    }
327
328    #[test]
329    fn rejects_git_workspace_with_unknown_field_by_name() {
330        let result: Result<EvalSpec, _> = serde_json::from_str(
331            r#"{"docker":{"image":"sandbox:latest"},"name":"g","agent":{"command":["agent"]},"task":{"prompt":"p","workspace":{"git":{"url":"u","startCommit":"a","goldComit":"b"}}}}"#,
332        );
333
334        assert!(result.unwrap_err().to_string().contains("unknown field `goldComit`"));
335    }
336
337    #[test]
338    fn rejects_invalid_judge_criteria() {
339        for (json, expected) in [
340            (
341                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[]}}}"#,
342                "criteria must not be empty",
343            ),
344            (
345                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"","description":"ok"}]}}}"#,
346                "id must not be empty",
347            ),
348            (
349                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":"ok"},{"id":"a","description":"ok"}]}}}"#,
350                "duplicate judge criterion id `a`",
351            ),
352            (
353                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":""}]}}}"#,
354                "description must not be empty",
355            ),
356            (
357                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":"ok","weight":0.0}]}}}"#,
358                "weight must be positive and finite",
359            ),
360            (
361                r#"{"docker":{"image":"sandbox:latest"},"name":"c","agent":{"command":["agent"]},"task":{"prompt":"p"},"expect":{"judge":{"model":"m","criteria":[{"id":"a","description":"ok","threshold":1.5}]}}}"#,
362                "threshold must be between 0.0 and 1.0",
363            ),
364        ] {
365            let eval = serde_json::from_str::<EvalSpec>(json).unwrap();
366            let error = ResolvedEvalSpec::resolve(eval, ".").unwrap_err().to_string();
367            assert!(error.contains(expected), "expected {expected:?} in {error:?}");
368        }
369    }
370}
aether_evals/spec/mod.rs

aether_evals/spec/
mod.rs
