aether_evals/spec/
error.rs

1use crate::agents::{DockerImageParseError, ImageBuildError};
2use aether_project::SettingsError;
3use std::path::PathBuf;
4use thiserror::Error;
5
6/// Eval-file setup errors. Setup — loading, image resolution and builds, settings — happens
7/// before any eval runs and aborts the collection. Faults inside an individual eval (workspace
8/// setup, the agent run, the judge call) do not abort a collection run; they are recorded on
9/// that eval's [`crate::spec::EvalOutcome`] instead.
10#[derive(Debug, Error)]
11pub enum EvalFileError {
12    #[error("failed to read eval file '{}': {source}", path.display())]
13    ReadEvalFile {
14        path: PathBuf,
15        #[source]
16        source: std::io::Error,
17    },
18
19    #[error("failed to parse eval file '{}': {source}", path.display())]
20    ParseEvalFile {
21        path: PathBuf,
22        #[source]
23        source: serde_json::Error,
24    },
25
26    #[error("eval path '{}' does not exist", path.display())]
27    EvalPathNotFound { path: PathBuf },
28
29    #[error("failed to discover eval files under '{}': {source}", path.display())]
30    DiscoverEvalFiles {
31        path: PathBuf,
32        #[source]
33        source: std::io::Error,
34    },
35
36    #[error("no eval files found under {}", format_paths(paths))]
37    NoEvalFilesFound { paths: Vec<PathBuf> },
38
39    #[error("no eval named '{name}' in the selected eval file(s)")]
40    NoMatchingEval { name: String },
41
42    #[error("duplicate eval name '{name}' in selected eval files")]
43    DuplicateEvalName { name: String },
44
45    #[error("failed to build judge model '{model}': {source}")]
46    JudgeModel {
47        model: String,
48        #[source]
49        source: llm::LlmError,
50    },
51
52    #[error("eval file does not specify a Docker image (set `docker.image` and/or `docker.file`)")]
53    NoImage,
54
55    #[error("invalid image reference: {0}")]
56    Image(#[from] DockerImageParseError),
57
58    #[error("failed to resolve docker path '{}': {source}", path.display())]
59    DockerPath {
60        path: PathBuf,
61        #[source]
62        source: std::io::Error,
63    },
64
65    #[error(transparent)]
66    ImageBuild(#[from] ImageBuildError),
67
68    #[error("invalid settings: {0}")]
69    Settings(#[from] SettingsError),
70
71    #[error("invalid inline judge: {message}")]
72    InvalidInlineJudge { message: String },
73
74    #[error("failed to read judge file '{}': {source}", path.display())]
75    ReadJudgeFile {
76        path: PathBuf,
77        #[source]
78        source: std::io::Error,
79    },
80
81    #[error("failed to parse judge file '{}': {source}", path.display())]
82    ParseJudgeFile {
83        path: PathBuf,
84        #[source]
85        source: serde_json::Error,
86    },
87}
88
89fn format_paths(paths: &[PathBuf]) -> String {
90    paths.iter().map(|path| path.display().to_string()).collect::<Vec<_>>().join(", ")
91}
aether_evals/spec/error.rs

aether_evals/spec/
error.rs
