aether_evals/spec/
error.rs

1use crate::agents::{DockerImageParseError, ImageBuildError};
2use std::path::PathBuf;
3use thiserror::Error;
4
5/// Eval-file setup errors. Setup — loading, image resolution and builds, settings — happens
6/// before any eval runs and aborts the collection. Faults inside an individual eval (workspace
7/// setup, the agent run, the judge call) do not abort a collection run; they are recorded on
8/// that eval's [`crate::spec::EvalOutcome`] instead.
9#[derive(Debug, Error)]
10pub enum EvalFileError {
11    #[error("failed to read eval file '{}': {source}", path.display())]
12    ReadEvalFile {
13        path: PathBuf,
14        #[source]
15        source: std::io::Error,
16    },
17
18    #[error("failed to parse eval file '{}': {source}", path.display())]
19    ParseEvalFile {
20        path: PathBuf,
21        #[source]
22        source: serde_json::Error,
23    },
24
25    #[error("failed to parse eval spec: {source}")]
26    ParseSpec {
27        #[source]
28        source: serde_json::Error,
29    },
30
31    #[error("invalid agent selection: {message}")]
32    InvalidAgentCommand { message: String },
33
34    #[error("eval path '{}' does not exist", path.display())]
35    EvalPathNotFound { path: PathBuf },
36
37    #[error("failed to discover eval files under '{}': {source}", path.display())]
38    DiscoverEvalFiles {
39        path: PathBuf,
40        #[source]
41        source: std::io::Error,
42    },
43
44    #[error("no eval files found under {}", format_paths(paths))]
45    NoEvalFilesFound { paths: Vec<PathBuf> },
46
47    #[error("no eval named '{name}' in the selected eval file(s)")]
48    NoMatchingEval { name: String },
49
50    #[error("duplicate eval name '{name}' in selected eval files")]
51    DuplicateEvalName { name: String },
52
53    #[error("failed to build judge model '{model}': {source}")]
54    JudgeModel {
55        model: String,
56        #[source]
57        source: llm::LlmError,
58    },
59
60    #[error("invalid image reference: {0}")]
61    Image(#[from] DockerImageParseError),
62
63    #[error("failed to resolve docker path '{}': {source}", path.display())]
64    DockerPath {
65        path: PathBuf,
66        #[source]
67        source: std::io::Error,
68    },
69
70    #[error(transparent)]
71    ImageBuild(#[from] ImageBuildError),
72
73    #[error("invalid inline judge: {message}")]
74    InvalidInlineJudge { message: String },
75
76    #[error("failed to read judge file '{}': {source}", path.display())]
77    ReadJudgeFile {
78        path: PathBuf,
79        #[source]
80        source: std::io::Error,
81    },
82
83    #[error("failed to parse judge file '{}': {source}", path.display())]
84    ParseJudgeFile {
85        path: PathBuf,
86        #[source]
87        source: serde_json::Error,
88    },
89}
90
91fn format_paths(paths: &[PathBuf]) -> String {
92    paths.iter().map(|path| path.display().to_string()).collect::<Vec<_>>().join(", ")
93}
aether_evals/spec/error.rs

aether_evals/spec/
error.rs
