aether_evals/spec/
types.rs

1use super::error::EvalFileError;
2use crate::agents::{DockerImage, ImageBuildRequest};
3use crate::{GitRepoSpec, Task, TaskRun, Workspace, WorkspaceError};
4use schemars::JsonSchema;
5use serde::Deserialize;
6use serde::de::DeserializeOwned;
7use std::collections::{BTreeMap, BTreeSet};
8use std::hash::{DefaultHasher, Hash, Hasher};
9use std::path::{Path, PathBuf};
10
11/// Docker image configuration for an eval sandbox: either a prebuilt `image`, or a Dockerfile
12/// `file` to build (optionally tagged with `image`).
13#[derive(Debug, Clone, Deserialize)]
14#[serde(try_from = "DockerSpecRepr")]
15pub enum DockerSpec {
16    /// Prebuilt sandbox image to run the eval in. The image must have `aether` on its `PATH`.
17    Prebuilt { image: String },
18    /// Dockerfile (relative to the eval file) to build into an image. When `image` is set, the
19    /// built image is tagged with it; otherwise the tag is derived from the Dockerfile and
20    /// context paths. `context` is relative to the eval file and defaults to its directory.
21    Build { file: PathBuf, image: Option<String>, context: Option<PathBuf> },
22}
23
24/// A [`DockerSpec`] resolved against the eval file's directory: the image to run in and, for
25/// Dockerfile-backed specs, the build that produces it.
26#[derive(Debug, Clone)]
27pub(crate) struct ResolvedDocker {
28    pub image: DockerImage,
29    pub build: Option<ImageBuildRequest>,
30}
31
32#[derive(Debug, Clone, Deserialize, JsonSchema)]
33#[serde(rename_all = "camelCase", deny_unknown_fields)]
34pub struct TaskSpec {
35    pub prompt: String,
36    #[serde(default)]
37    pub workspace: WorkspaceSpec,
38}
39
40/// The starting workspace for an eval. Omitted means an empty temporary directory.
41#[derive(Debug, Clone, Default, Deserialize, JsonSchema)]
42#[serde(rename_all = "camelCase")]
43pub enum WorkspaceSpec {
44    #[default]
45    Empty,
46    /// Inline files written into a fresh workspace, keyed by relative path.
47    Files(BTreeMap<String, String>),
48    /// A directory (relative to the eval file) copied into a fresh workspace.
49    Dir(PathBuf),
50    /// A git repository checked out at `startCommit`.
51    Git(GitRepoSpec),
52}
53
54/// An LLM-as-judge rubric, either inline or a path (relative to the eval file) to a shared JSON
55/// file holding a [`JudgeSpec`], so one rubric can be reused across evals.
56#[derive(Debug, Clone)]
57pub enum PathOrInline<T> {
58    Path(PathBuf),
59    Inline(T),
60}
61
62pub type JudgeRef = PathOrInline<JudgeSpec>;
63
64/// An LLM-as-judge rubric for an eval.
65#[derive(Debug, Clone, Deserialize, JsonSchema)]
66#[serde(rename_all = "camelCase", deny_unknown_fields)]
67pub struct JudgeSpec {
68    /// Model to use for this judge check (e.g. "anthropic:claude-sonnet-4-5").
69    pub model: String,
70
71    /// Optional high-level grading instructions for the LLM judge.
72    #[serde(default)]
73    pub instructions: Option<String>,
74
75    /// Workspace-relative files to include in the judge context in addition to deterministic
76    /// expectation files.
77    #[serde(default)]
78    pub context_files: Vec<String>,
79
80    /// Ordered rubric criteria to score on a normalized 0.0..=1.0 scale.
81    #[serde(default)]
82    pub criteria: Vec<JudgeCriterionSpec>,
83}
84
85#[derive(Debug, Clone, Deserialize, JsonSchema)]
86#[serde(rename_all = "camelCase", deny_unknown_fields)]
87pub struct JudgeCriterionSpec {
88    pub id: String,
89    pub description: String,
90    #[serde(default = "default_blocking")]
91    pub blocking: bool,
92    #[serde(default = "default_weight")]
93    pub weight: f64,
94    #[serde(default = "default_threshold")]
95    pub threshold: f64,
96}
97
98impl JudgeCriterionSpec {
99    pub fn new(id: impl Into<String>, description: impl Into<String>) -> Self {
100        Self {
101            id: id.into(),
102            description: description.into(),
103            blocking: default_blocking(),
104            weight: default_weight(),
105            threshold: default_threshold(),
106        }
107    }
108
109    pub fn blocking(mut self, blocking: bool) -> Self {
110        self.blocking = blocking;
111        self
112    }
113
114    pub fn weight(mut self, weight: f64) -> Self {
115        self.weight = weight;
116        self
117    }
118
119    pub fn threshold(mut self, threshold: f64) -> Self {
120        self.threshold = threshold;
121        self
122    }
123}
124#[derive(Debug, Clone, Deserialize, JsonSchema)]
125#[serde(rename_all = "camelCase")]
126pub enum ToolCallExpectation {
127    AtLeast(usize),
128    Exactly(usize),
129}
130
131/// Expectations checked against the agent's run. All are optional; an empty `expect` passes as
132/// long as the agent runs to completion.
133#[derive(Debug, Clone, Default, Deserialize, JsonSchema)]
134#[serde(rename_all = "camelCase", deny_unknown_fields)]
135pub struct Expect {
136    /// Tool call count requirements by tool name.
137    #[serde(default)]
138    pub tool_calls: BTreeMap<String, ToolCallExpectation>,
139
140    /// Files whose full content must equal the given value, keyed by workspace-relative path.
141    #[serde(default)]
142    pub files: BTreeMap<String, String>,
143
144    /// Files that must contain the given substring, keyed by workspace-relative path.
145    #[serde(default)]
146    pub files_contain: BTreeMap<String, String>,
147
148    /// LLM-as-judge evaluation, inline or a path to a shared judge file.
149    #[serde(default)]
150    pub judge: Option<JudgeRef>,
151}
152
153#[derive(Debug, Clone, Deserialize, JsonSchema)]
154#[serde(rename_all = "camelCase", deny_unknown_fields)]
155pub struct AgentSpec {
156    pub command: Vec<String>,
157
158    #[serde(default)]
159    pub env: BTreeMap<String, String>,
160}
161
162impl AgentSpec {
163    pub(crate) fn validate(&self) -> Result<(), String> {
164        let Some(program) = self.command.first() else {
165            return Err("agent command must not be empty".to_string());
166        };
167        if program.trim().is_empty() {
168            return Err("agent command program must not be empty".to_string());
169        }
170        for key in self.env.keys() {
171            if key.trim().is_empty() || key.contains('=') {
172                return Err(format!("agent env contains invalid key `{key}`"));
173            }
174        }
175        Ok(())
176    }
177}
178
179impl DockerSpec {
180    /// Resolve the image to run in, and the build producing it for Dockerfile-backed specs, with
181    /// paths resolved relative to `base_dir`.
182    pub(crate) fn resolve(&self, base_dir: &Path) -> Result<ResolvedDocker, EvalFileError> {
183        match self {
184            Self::Prebuilt { image } => Ok(ResolvedDocker { image: DockerImage::parse(image)?, build: None }),
185            Self::Build { file, image, context } => {
186                let dockerfile = canonical_docker_path(base_dir.join(file))?;
187                let context = canonical_docker_path(
188                    context.as_ref().map_or_else(|| base_dir.to_path_buf(), |context| base_dir.join(context)),
189                )?;
190                let tag = image.clone().unwrap_or_else(|| derived_tag(&dockerfile, &context));
191                Ok(ResolvedDocker {
192                    image: DockerImage::parse(&tag)?,
193                    build: Some(ImageBuildRequest { dockerfile, context, tag }),
194                })
195            }
196        }
197    }
198}
199
200impl TaskSpec {
201    pub(crate) fn build(&self, base_dir: &Path) -> Result<Task, WorkspaceError> {
202        Ok(Task::new(self.prompt.clone(), self.workspace.build(base_dir)?))
203    }
204}
205
206impl WorkspaceSpec {
207    pub(crate) fn build(&self, base_dir: &Path) -> Result<Workspace, WorkspaceError> {
208        match self {
209            Self::Empty => Workspace::empty(),
210            Self::Files(files) => Workspace::from_files(files),
211            Self::Dir(dir) => Workspace::from_dir(base_dir.join(dir)),
212            Self::Git(git) => Workspace::from_git_repo(git.clone()),
213        }
214    }
215}
216
217impl Expect {
218    pub(crate) fn evaluate(&self, run: &TaskRun) -> Vec<String> {
219        let mut failures = Vec::new();
220
221        for (tool, expectation) in &self.tool_calls {
222            let actual = run.transcript().tool_call_count(tool);
223            if let Some(failure) = expectation.failure(tool, actual) {
224                failures.push(failure);
225            }
226        }
227
228        for (path, expected) in &self.files {
229            match read_workspace_file(run, path) {
230                Ok(actual) if &actual == expected => {}
231                Ok(actual) => failures
232                    .push(format!("file `{path}` content mismatch:\n  expected: {expected:?}\n  actual:   {actual:?}")),
233                Err(error) => failures.push(format!("file `{path}` could not be read: {error}")),
234            }
235        }
236
237        for (path, needle) in &self.files_contain {
238            match read_workspace_file(run, path) {
239                Ok(actual) if actual.contains(needle) => {}
240                Ok(_) => failures.push(format!("file `{path}` does not contain {needle:?}")),
241                Err(error) => failures.push(format!("file `{path}` could not be read: {error}")),
242            }
243        }
244
245        failures
246    }
247}
248
249fn read_workspace_file(run: &TaskRun, relative_path: &str) -> std::io::Result<String> {
250    std::fs::read_to_string(run.workspace().join(relative_path))
251}
252
253impl JudgeRef {
254    /// Resolve to a concrete judge spec, reading path references relative to `base_dir`.
255    pub fn resolve(&self, base_dir: &Path) -> Result<JudgeSpec, EvalFileError> {
256        match self {
257            JudgeRef::Inline(spec) => {
258                spec.validate().map_err(|message| EvalFileError::InvalidInlineJudge { message })?;
259                Ok(spec.clone())
260            }
261            JudgeRef::Path(path) => {
262                let path = base_dir.join(path);
263                let content = std::fs::read_to_string(&path)
264                    .map_err(|source| EvalFileError::ReadJudgeFile { path: path.clone(), source })?;
265                let spec: JudgeSpec = serde_json::from_str(&content)
266                    .map_err(|source| EvalFileError::ParseJudgeFile { path: path.clone(), source })?;
267                spec.validate().map_err(|message| EvalFileError::InvalidInlineJudge { message })?;
268                Ok(spec)
269            }
270        }
271    }
272}
273
274impl<'de, T> Deserialize<'de> for PathOrInline<T>
275where
276    T: DeserializeOwned,
277{
278    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
279        match serde_json::Value::deserialize(deserializer)? {
280            serde_json::Value::String(path) => Ok(Self::Path(PathBuf::from(path))),
281            value => serde_json::from_value(value).map(Self::Inline).map_err(serde::de::Error::custom),
282        }
283    }
284}
285
286impl JsonSchema for DockerSpec {
287    fn schema_name() -> std::borrow::Cow<'static, str> {
288        "DockerSpec".into()
289    }
290
291    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
292        DockerSpecRepr::json_schema(generator)
293    }
294}
295
296impl<T: JsonSchema> JsonSchema for PathOrInline<T> {
297    fn schema_name() -> std::borrow::Cow<'static, str> {
298        std::borrow::Cow::Owned(format!("PathOr{}", T::schema_name()))
299    }
300
301    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
302        let inline = generator.subschema_for::<T>().to_value();
303        schemars::Schema::try_from(serde_json::json!({
304            "oneOf": [{ "type": "string" }, inline],
305        }))
306        .expect("PathOrInline schema must be valid")
307    }
308}
309
310#[derive(Deserialize, JsonSchema)]
311#[serde(rename_all = "camelCase", deny_unknown_fields)]
312struct DockerSpecRepr {
313    #[serde(default)]
314    image: Option<String>,
315    #[serde(default)]
316    file: Option<PathBuf>,
317    #[serde(default)]
318    context: Option<PathBuf>,
319}
320
321impl TryFrom<DockerSpecRepr> for DockerSpec {
322    type Error = &'static str;
323
324    fn try_from(repr: DockerSpecRepr) -> Result<Self, Self::Error> {
325        match (repr.file, repr.image, repr.context) {
326            (Some(file), image, context) => Ok(Self::Build { file, image, context }),
327            (None, Some(image), None) => Ok(Self::Prebuilt { image }),
328            (None, Some(_), Some(_)) => Err("docker `context` requires a Dockerfile `file`"),
329            (None, None, _) => Err("docker must specify a prebuilt `image` and/or a Dockerfile `file`"),
330        }
331    }
332}
333
334impl ToolCallExpectation {
335    pub(crate) fn failure(&self, tool: &str, actual: usize) -> Option<String> {
336        match self {
337            Self::AtLeast(expected) if actual < *expected => Some(format!(
338                "expected tool `{tool}` to be called at least {expected} time(s), but was called {actual}"
339            )),
340            Self::Exactly(expected) if actual != *expected => {
341                Some(format!("expected tool `{tool}` to be called exactly {expected} time(s), but was called {actual}"))
342            }
343            _ => None,
344        }
345    }
346}
347
348impl JudgeSpec {
349    pub(crate) fn validate(&self) -> Result<(), String> {
350        if self.criteria.is_empty() {
351            return Err("judge criteria must not be empty".to_string());
352        }
353
354        let mut ids = BTreeSet::new();
355        for criterion in &self.criteria {
356            if criterion.id.trim().is_empty() {
357                return Err("judge criterion id must not be empty".to_string());
358            }
359            if !ids.insert(criterion.id.clone()) {
360                return Err(format!("duplicate judge criterion id `{}`", criterion.id));
361            }
362            if criterion.description.trim().is_empty() {
363                return Err(format!("judge criterion `{}` description must not be empty", criterion.id));
364            }
365            if !criterion.weight.is_finite() || criterion.weight <= 0.0 {
366                return Err(format!("judge criterion `{}` weight must be positive and finite", criterion.id));
367            }
368            if !criterion.threshold.is_finite() || !(0.0..=1.0).contains(&criterion.threshold) {
369                return Err(format!("judge criterion `{}` threshold must be between 0.0 and 1.0", criterion.id));
370            }
371        }
372
373        Ok(())
374    }
375}
376
377fn canonical_docker_path(path: PathBuf) -> Result<PathBuf, EvalFileError> {
378    path.canonicalize().map_err(|source| EvalFileError::DockerPath { path, source })
379}
380
381/// Tag for a built image whose eval file does not name one. Derived from the Dockerfile and
382/// context paths so distinct builds never collide on a shared default tag, while eval files
383/// sharing a Dockerfile and context share one build.
384fn derived_tag(dockerfile: &Path, context: &Path) -> String {
385    let mut hasher = DefaultHasher::new();
386    (dockerfile, context).hash(&mut hasher);
387    format!("aether-eval-sandbox:{:016x}", hasher.finish())
388}
389
390fn default_blocking() -> bool {
391    true
392}
393
394fn default_weight() -> f64 {
395    1.0
396}
397
398fn default_threshold() -> f64 {
399    1.0
400}
401
402#[cfg(test)]
403mod tests {
404    use super::*;
405
406    #[test]
407    fn agent_spec_parses_command_only_json() {
408        let agent: AgentSpec = serde_json::from_str(
409            r#"{"command":["node","/app/eval-agent.js"],"env":{"AETHER_EVAL_MODEL":"test:model"}}"#,
410        )
411        .unwrap();
412
413        assert_eq!(agent.command, vec!["node", "/app/eval-agent.js"]);
414        assert_eq!(agent.env["AETHER_EVAL_MODEL"], "test:model");
415    }
416
417    #[test]
418    fn agent_spec_rejects_empty_command() {
419        let agent: AgentSpec = serde_json::from_str(r#"{"command":[]}"#).unwrap();
420        let error = agent.validate().unwrap_err();
421        assert_eq!(error, "agent command must not be empty");
422    }
423
424    #[test]
425    fn agent_spec_rejects_blank_program() {
426        let agent: AgentSpec = serde_json::from_str(r#"{"command":["  "]}"#).unwrap();
427
428        let error = agent.validate().unwrap_err();
429        assert_eq!(error, "agent command program must not be empty");
430    }
431
432    #[test]
433    fn agent_spec_rejects_invalid_env_keys() {
434        for json in [r#"{"command":["agent"],"env":{"":"x"}}"#, r#"{"command":["agent"],"env":{"BAD=KEY":"x"}}"#] {
435            let agent: AgentSpec = serde_json::from_str(json).unwrap();
436            let error = agent.validate().unwrap_err();
437            assert!(error.contains("agent env contains invalid key"), "got: {error}");
438        }
439    }
440
441    #[test]
442    fn agent_spec_rejects_old_tagged_shapes() {
443        for json in [r#"{"type":"aether","settings":{}}"#, r#"{"type":"command","command":["agent"]}"#] {
444            let error = serde_json::from_str::<AgentSpec>(json).unwrap_err().to_string();
445
446            assert!(error.contains("unknown field `type`"), "got: {error}");
447        }
448    }
449
450    #[test]
451    fn judge_ref_parses_path_or_inline() {
452        let path: JudgeRef = serde_json::from_str(r#""shared/maintainer.judge.json""#).unwrap();
453        assert!(matches!(path, JudgeRef::Path(_)));
454
455        let inline: JudgeRef =
456            serde_json::from_str(r#"{"model":"m","criteria":[{"id":"a","description":"ok"}]}"#).unwrap();
457        assert!(matches!(inline, JudgeRef::Inline(_)));
458    }
459
460    #[test]
461    fn judge_ref_preserves_inline_validation_errors() {
462        let error = serde_json::from_str::<JudgeRef>(r#"{"criteria":[{"id":"a","description":"ok"}]}"#).unwrap_err();
463
464        assert!(error.to_string().contains("missing field `model`"), "got: {error}");
465    }
466
467    #[test]
468    fn derived_tags_differ_per_dockerfile_and_match_for_identical_builds() {
469        let tag_a = derived_tag(Path::new("/repo/a/Dockerfile"), Path::new("/repo"));
470        let tag_b = derived_tag(Path::new("/repo/b/Dockerfile"), Path::new("/repo"));
471
472        assert_eq!(tag_a, derived_tag(Path::new("/repo/a/Dockerfile"), Path::new("/repo")));
473        assert_ne!(tag_a, tag_b);
474    }
475}
aether_evals/spec/types.rs

aether_evals/spec/
types.rs
