aether_evals/spec/
types.rs

1use super::error::EvalFileError;
2use crate::agents::{DockerImage, ImageBuildRequest};
3use crate::{GitRepoSpec, Task, TaskRun, Workspace, WorkspaceError};
4use schemars::JsonSchema;
5use serde::Deserialize;
6use serde::de::DeserializeOwned;
7use std::collections::{BTreeMap, BTreeSet};
8use std::hash::{DefaultHasher, Hash, Hasher};
9use std::path::{Path, PathBuf};
10
11/// Docker image configuration for an eval sandbox: either a prebuilt `image`, or a Dockerfile
12/// `file` to build (optionally tagged with `image`).
13#[derive(Debug, Clone, Deserialize)]
14#[serde(try_from = "DockerSpecRepr")]
15pub enum DockerSpec {
16    /// Prebuilt sandbox image to run the eval in. The image must have `aether` on its `PATH`.
17    Prebuilt { image: String },
18    /// Dockerfile (relative to the eval file) to build into an image. When `image` is set, the
19    /// built image is tagged with it; otherwise the tag is derived from the Dockerfile and
20    /// context paths. `context` is relative to the eval file and defaults to its directory.
21    Build { file: PathBuf, image: Option<String>, context: Option<PathBuf> },
22}
23
24/// A [`DockerSpec`] resolved against the eval file's directory: the image to run in and, for
25/// Dockerfile-backed specs, the build that produces it.
26#[derive(Debug, Clone)]
27pub(crate) struct ResolvedDocker {
28    pub image: DockerImage,
29    pub build: Option<ImageBuildRequest>,
30}
31
32#[derive(Debug, Clone, Deserialize, JsonSchema)]
33#[serde(rename_all = "camelCase", deny_unknown_fields)]
34pub struct TaskSpec {
35    pub prompt: String,
36    #[serde(default)]
37    pub workspace: WorkspaceSpec,
38}
39
40/// The starting workspace for an eval. Omitted means an empty temporary directory.
41#[derive(Debug, Clone, Default, Deserialize, JsonSchema)]
42#[serde(rename_all = "camelCase")]
43pub enum WorkspaceSpec {
44    #[default]
45    Empty,
46    /// Inline files written into a fresh workspace, keyed by relative path.
47    Files(BTreeMap<String, String>),
48    /// A directory (relative to the eval file) copied into a fresh workspace.
49    Dir(PathBuf),
50    /// A git repository checked out at `startCommit`.
51    Git(GitRepoSpec),
52}
53
54/// An LLM-as-judge rubric, either inline or a path (relative to the eval file) to a shared JSON
55/// file holding a [`JudgeSpec`], so one rubric can be reused across evals.
56#[derive(Debug, Clone)]
57pub enum PathOrInline<T> {
58    Path(PathBuf),
59    Inline(T),
60}
61
62pub type JudgeRef = PathOrInline<JudgeSpec>;
63
64/// An LLM-as-judge rubric for an eval.
65#[derive(Debug, Clone, Deserialize, JsonSchema)]
66#[serde(rename_all = "camelCase", deny_unknown_fields)]
67pub struct JudgeSpec {
68    /// Model to use for this judge check (e.g. "anthropic:claude-sonnet-4-5").
69    pub model: String,
70
71    /// Optional high-level grading instructions for the LLM judge.
72    #[serde(default)]
73    pub instructions: Option<String>,
74
75    /// Workspace-relative files to include in the judge context in addition to deterministic
76    /// expectation files.
77    #[serde(default)]
78    pub context_files: Vec<String>,
79
80    /// Ordered rubric criteria to score on a normalized 0.0..=1.0 scale.
81    #[serde(default)]
82    pub criteria: Vec<JudgeCriterionSpec>,
83}
84
85#[derive(Debug, Clone, Deserialize, JsonSchema)]
86#[serde(rename_all = "camelCase", deny_unknown_fields)]
87pub struct JudgeCriterionSpec {
88    pub id: String,
89    pub description: String,
90    #[serde(default = "default_blocking")]
91    pub blocking: bool,
92    #[serde(default = "default_weight")]
93    pub weight: f64,
94    #[serde(default = "default_threshold")]
95    pub threshold: f64,
96}
97
98#[derive(Debug, Clone, Deserialize, JsonSchema)]
99#[serde(rename_all = "camelCase")]
100pub enum ToolCallExpectation {
101    AtLeast(usize),
102    Exactly(usize),
103}
104
105/// Expectations checked against the agent's run. All are optional; an empty `expect` passes as
106/// long as the agent runs to completion.
107#[derive(Debug, Clone, Default, Deserialize, JsonSchema)]
108#[serde(rename_all = "camelCase", deny_unknown_fields)]
109pub struct Expect {
110    /// Tool call count requirements by tool name.
111    #[serde(default)]
112    pub tool_calls: BTreeMap<String, ToolCallExpectation>,
113
114    /// Files whose full content must equal the given value, keyed by workspace-relative path.
115    #[serde(default)]
116    pub files: BTreeMap<String, String>,
117
118    /// Files that must contain the given substring, keyed by workspace-relative path.
119    #[serde(default)]
120    pub files_contain: BTreeMap<String, String>,
121
122    /// LLM-as-judge evaluation, inline or a path to a shared judge file.
123    #[serde(default)]
124    pub judge: Option<JudgeRef>,
125}
126
127#[derive(Debug, Clone, Deserialize, JsonSchema)]
128#[serde(rename_all = "camelCase", deny_unknown_fields)]
129pub struct AgentSpec {
130    pub command: Vec<String>,
131
132    #[serde(default)]
133    pub env: BTreeMap<String, String>,
134}
135
136impl AgentSpec {
137    pub(crate) fn validate(&self) -> Result<(), String> {
138        let Some(program) = self.command.first() else {
139            return Err("agent command must not be empty".to_string());
140        };
141        if program.trim().is_empty() {
142            return Err("agent command program must not be empty".to_string());
143        }
144        for key in self.env.keys() {
145            if key.trim().is_empty() || key.contains('=') {
146                return Err(format!("agent env contains invalid key `{key}`"));
147            }
148        }
149        Ok(())
150    }
151}
152
153impl DockerSpec {
154    /// Resolve the image to run in, and the build producing it for Dockerfile-backed specs, with
155    /// paths resolved relative to `base_dir`.
156    pub(crate) fn resolve(&self, base_dir: &Path) -> Result<ResolvedDocker, EvalFileError> {
157        match self {
158            Self::Prebuilt { image } => Ok(ResolvedDocker { image: DockerImage::parse(image)?, build: None }),
159            Self::Build { file, image, context } => {
160                let dockerfile = canonical_docker_path(base_dir.join(file))?;
161                let context = canonical_docker_path(
162                    context.as_ref().map_or_else(|| base_dir.to_path_buf(), |context| base_dir.join(context)),
163                )?;
164                let tag = image.clone().unwrap_or_else(|| derived_tag(&dockerfile, &context));
165                Ok(ResolvedDocker {
166                    image: DockerImage::parse(&tag)?,
167                    build: Some(ImageBuildRequest { dockerfile, context, tag }),
168                })
169            }
170        }
171    }
172}
173
174impl TaskSpec {
175    pub(crate) fn build(&self, base_dir: &Path) -> Result<Task, WorkspaceError> {
176        Ok(Task::new(self.prompt.clone(), self.workspace.build(base_dir)?))
177    }
178}
179
180impl WorkspaceSpec {
181    pub(crate) fn build(&self, base_dir: &Path) -> Result<Workspace, WorkspaceError> {
182        match self {
183            Self::Empty => Workspace::empty(),
184            Self::Files(files) => Workspace::from_files(files),
185            Self::Dir(dir) => Workspace::from_dir(base_dir.join(dir)),
186            Self::Git(git) => Workspace::from_git_repo(git.clone()),
187        }
188    }
189}
190
191impl Expect {
192    pub(crate) fn evaluate(&self, run: &TaskRun) -> Vec<String> {
193        let mut failures = Vec::new();
194
195        for (tool, expectation) in &self.tool_calls {
196            let actual = run.transcript().tool_call_count(tool);
197            if let Some(failure) = expectation.failure(tool, actual) {
198                failures.push(failure);
199            }
200        }
201
202        for (path, expected) in &self.files {
203            match read_workspace_file(run, path) {
204                Ok(actual) if &actual == expected => {}
205                Ok(actual) => failures
206                    .push(format!("file `{path}` content mismatch:\n  expected: {expected:?}\n  actual:   {actual:?}")),
207                Err(error) => failures.push(format!("file `{path}` could not be read: {error}")),
208            }
209        }
210
211        for (path, needle) in &self.files_contain {
212            match read_workspace_file(run, path) {
213                Ok(actual) if actual.contains(needle) => {}
214                Ok(_) => failures.push(format!("file `{path}` does not contain {needle:?}")),
215                Err(error) => failures.push(format!("file `{path}` could not be read: {error}")),
216            }
217        }
218
219        failures
220    }
221}
222
223fn read_workspace_file(run: &TaskRun, relative_path: &str) -> std::io::Result<String> {
224    std::fs::read_to_string(run.workspace().join(relative_path))
225}
226
227impl JudgeRef {
228    /// Resolve to a concrete judge spec, reading path references relative to `base_dir`.
229    pub fn resolve(&self, base_dir: &Path) -> Result<JudgeSpec, EvalFileError> {
230        match self {
231            JudgeRef::Inline(spec) => {
232                spec.validate().map_err(|message| EvalFileError::InvalidInlineJudge { message })?;
233                Ok(spec.clone())
234            }
235            JudgeRef::Path(path) => {
236                let path = base_dir.join(path);
237                let content = std::fs::read_to_string(&path)
238                    .map_err(|source| EvalFileError::ReadJudgeFile { path: path.clone(), source })?;
239                let spec: JudgeSpec = serde_json::from_str(&content)
240                    .map_err(|source| EvalFileError::ParseJudgeFile { path: path.clone(), source })?;
241                spec.validate().map_err(|message| EvalFileError::InvalidInlineJudge { message })?;
242                Ok(spec)
243            }
244        }
245    }
246}
247
248impl<'de, T> Deserialize<'de> for PathOrInline<T>
249where
250    T: DeserializeOwned,
251{
252    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
253        match serde_json::Value::deserialize(deserializer)? {
254            serde_json::Value::String(path) => Ok(Self::Path(PathBuf::from(path))),
255            value => serde_json::from_value(value).map(Self::Inline).map_err(serde::de::Error::custom),
256        }
257    }
258}
259
260impl JsonSchema for DockerSpec {
261    fn schema_name() -> std::borrow::Cow<'static, str> {
262        "DockerSpec".into()
263    }
264
265    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
266        DockerSpecRepr::json_schema(generator)
267    }
268}
269
270impl<T: JsonSchema> JsonSchema for PathOrInline<T> {
271    fn schema_name() -> std::borrow::Cow<'static, str> {
272        std::borrow::Cow::Owned(format!("PathOr{}", T::schema_name()))
273    }
274
275    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
276        let inline = generator.subschema_for::<T>().to_value();
277        schemars::Schema::try_from(serde_json::json!({
278            "oneOf": [{ "type": "string" }, inline],
279        }))
280        .expect("PathOrInline schema must be valid")
281    }
282}
283
284#[derive(Deserialize, JsonSchema)]
285#[serde(rename_all = "camelCase", deny_unknown_fields)]
286struct DockerSpecRepr {
287    #[serde(default)]
288    image: Option<String>,
289    #[serde(default)]
290    file: Option<PathBuf>,
291    #[serde(default)]
292    context: Option<PathBuf>,
293}
294
295impl TryFrom<DockerSpecRepr> for DockerSpec {
296    type Error = &'static str;
297
298    fn try_from(repr: DockerSpecRepr) -> Result<Self, Self::Error> {
299        match (repr.file, repr.image, repr.context) {
300            (Some(file), image, context) => Ok(Self::Build { file, image, context }),
301            (None, Some(image), None) => Ok(Self::Prebuilt { image }),
302            (None, Some(_), Some(_)) => Err("docker `context` requires a Dockerfile `file`"),
303            (None, None, _) => Err("docker must specify a prebuilt `image` and/or a Dockerfile `file`"),
304        }
305    }
306}
307
308impl ToolCallExpectation {
309    pub(crate) fn failure(&self, tool: &str, actual: usize) -> Option<String> {
310        match self {
311            Self::AtLeast(expected) if actual < *expected => Some(format!(
312                "expected tool `{tool}` to be called at least {expected} time(s), but was called {actual}"
313            )),
314            Self::Exactly(expected) if actual != *expected => {
315                Some(format!("expected tool `{tool}` to be called exactly {expected} time(s), but was called {actual}"))
316            }
317            _ => None,
318        }
319    }
320}
321
322impl JudgeSpec {
323    pub(crate) fn validate(&self) -> Result<(), String> {
324        if self.criteria.is_empty() {
325            return Err("judge criteria must not be empty".to_string());
326        }
327
328        let mut ids = BTreeSet::new();
329        for criterion in &self.criteria {
330            if criterion.id.trim().is_empty() {
331                return Err("judge criterion id must not be empty".to_string());
332            }
333            if !ids.insert(criterion.id.clone()) {
334                return Err(format!("duplicate judge criterion id `{}`", criterion.id));
335            }
336            if criterion.description.trim().is_empty() {
337                return Err(format!("judge criterion `{}` description must not be empty", criterion.id));
338            }
339            if !criterion.weight.is_finite() || criterion.weight <= 0.0 {
340                return Err(format!("judge criterion `{}` weight must be positive and finite", criterion.id));
341            }
342            if !criterion.threshold.is_finite() || !(0.0..=1.0).contains(&criterion.threshold) {
343                return Err(format!("judge criterion `{}` threshold must be between 0.0 and 1.0", criterion.id));
344            }
345        }
346
347        Ok(())
348    }
349}
350
351fn canonical_docker_path(path: PathBuf) -> Result<PathBuf, EvalFileError> {
352    path.canonicalize().map_err(|source| EvalFileError::DockerPath { path, source })
353}
354
355/// Tag for a built image whose eval file does not name one. Derived from the Dockerfile and
356/// context paths so distinct builds never collide on a shared default tag, while eval files
357/// sharing a Dockerfile and context share one build.
358fn derived_tag(dockerfile: &Path, context: &Path) -> String {
359    let mut hasher = DefaultHasher::new();
360    (dockerfile, context).hash(&mut hasher);
361    format!("aether-eval-sandbox:{:016x}", hasher.finish())
362}
363
364fn default_blocking() -> bool {
365    true
366}
367
368fn default_weight() -> f64 {
369    1.0
370}
371
372fn default_threshold() -> f64 {
373    1.0
374}
375
376#[cfg(test)]
377mod tests {
378    use super::*;
379
380    #[test]
381    fn agent_spec_parses_command_only_json() {
382        let agent: AgentSpec = serde_json::from_str(
383            r#"{"command":["node","/app/eval-agent.js"],"env":{"AETHER_EVAL_MODEL":"test:model"}}"#,
384        )
385        .unwrap();
386
387        assert_eq!(agent.command, vec!["node", "/app/eval-agent.js"]);
388        assert_eq!(agent.env["AETHER_EVAL_MODEL"], "test:model");
389    }
390
391    #[test]
392    fn agent_spec_rejects_empty_command() {
393        let agent: AgentSpec = serde_json::from_str(r#"{"command":[]}"#).unwrap();
394        let error = agent.validate().unwrap_err();
395        assert_eq!(error, "agent command must not be empty");
396    }
397
398    #[test]
399    fn agent_spec_rejects_blank_program() {
400        let agent: AgentSpec = serde_json::from_str(r#"{"command":["  "]}"#).unwrap();
401
402        let error = agent.validate().unwrap_err();
403        assert_eq!(error, "agent command program must not be empty");
404    }
405
406    #[test]
407    fn agent_spec_rejects_invalid_env_keys() {
408        for json in [r#"{"command":["agent"],"env":{"":"x"}}"#, r#"{"command":["agent"],"env":{"BAD=KEY":"x"}}"#] {
409            let agent: AgentSpec = serde_json::from_str(json).unwrap();
410            let error = agent.validate().unwrap_err();
411            assert!(error.contains("agent env contains invalid key"), "got: {error}");
412        }
413    }
414
415    #[test]
416    fn agent_spec_rejects_old_tagged_shapes() {
417        for json in [r#"{"type":"aether","settings":{}}"#, r#"{"type":"command","command":["agent"]}"#] {
418            let error = serde_json::from_str::<AgentSpec>(json).unwrap_err().to_string();
419
420            assert!(error.contains("unknown field `type`"), "got: {error}");
421        }
422    }
423
424    #[test]
425    fn judge_ref_parses_path_or_inline() {
426        let path: JudgeRef = serde_json::from_str(r#""shared/maintainer.judge.json""#).unwrap();
427        assert!(matches!(path, JudgeRef::Path(_)));
428
429        let inline: JudgeRef =
430            serde_json::from_str(r#"{"model":"m","criteria":[{"id":"a","description":"ok"}]}"#).unwrap();
431        assert!(matches!(inline, JudgeRef::Inline(_)));
432    }
433
434    #[test]
435    fn judge_ref_preserves_inline_validation_errors() {
436        let error = serde_json::from_str::<JudgeRef>(r#"{"criteria":[{"id":"a","description":"ok"}]}"#).unwrap_err();
437
438        assert!(error.to_string().contains("missing field `model`"), "got: {error}");
439    }
440
441    #[test]
442    fn derived_tags_differ_per_dockerfile_and_match_for_identical_builds() {
443        let tag_a = derived_tag(Path::new("/repo/a/Dockerfile"), Path::new("/repo"));
444        let tag_b = derived_tag(Path::new("/repo/b/Dockerfile"), Path::new("/repo"));
445
446        assert_eq!(tag_a, derived_tag(Path::new("/repo/a/Dockerfile"), Path::new("/repo")));
447        assert_ne!(tag_a, tag_b);
448    }
449}
aether_evals/spec/types.rs

aether_evals/spec/
types.rs
