zeph_experiments/error.rs
1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Error types for the experiments module.
5
6/// Errors that can occur during experiment evaluation, benchmark loading, or persistence.
7///
8/// Most variants carry structured context (file paths, token counts, parameter names)
9/// so that callers can surface actionable diagnostics to the user.
10///
11/// # Examples
12///
13/// ```rust
14/// use zeph_experiments::EvalError;
15///
16/// let err = EvalError::BudgetExceeded { used: 1_500, budget: 1_000 };
17/// assert!(err.to_string().contains("1500"));
18///
19/// let err = EvalError::InvalidRadius { radius: -1.0 };
20/// assert!(err.to_string().contains("finite and positive"));
21/// ```
22#[non_exhaustive]
23#[derive(Debug, thiserror::Error)]
24pub enum EvalError {
25 /// The benchmark TOML file could not be opened or read.
26 #[error("failed to load benchmark file {0}: {1}")]
27 BenchmarkLoad(String, #[source] std::io::Error),
28
29 /// The benchmark TOML file could not be parsed.
30 #[error("failed to parse benchmark file {0}: {1}")]
31 BenchmarkParse(String, String),
32
33 /// [`BenchmarkSet::validate`] was called on an empty `cases` vec.
34 ///
35 /// [`BenchmarkSet::validate`]: crate::BenchmarkSet::validate
36 #[error("benchmark set is empty")]
37 EmptyBenchmarkSet,
38
39 /// The cumulative token budget for judge calls was exhausted.
40 ///
41 /// When this error is returned from [`Evaluator::evaluate`], the report will
42 /// have `is_partial = true` and only include cases scored before the budget was hit.
43 ///
44 /// [`Evaluator::evaluate`]: crate::Evaluator::evaluate
45 #[error("evaluation budget exceeded: used {used} of {budget} tokens")]
46 BudgetExceeded { used: u64, budget: u64 },
47
48 /// An LLM call failed (network, auth, timeout, or API error).
49 #[error("LLM error during evaluation: {0}")]
50 Llm(#[from] zeph_llm::LlmError),
51
52 /// The subject or judge LLM call did not complete within the configured timeout.
53 ///
54 /// The case is excluded from scores and counted in [`EvalReport::error_count`].
55 ///
56 /// [`EvalReport::error_count`]: crate::EvalReport::error_count
57 #[error("{role} LLM call timed out after {timeout_secs}s for case {case_index}")]
58 Timeout {
59 /// Which model timed out: `"subject"` or `"judge"`.
60 role: &'static str,
61 /// Configured timeout in seconds.
62 timeout_secs: u64,
63 /// Zero-based index of the benchmark case.
64 case_index: usize,
65 },
66
67 /// The judge model returned a non-finite or structurally invalid score.
68 #[error("judge output parse failed for case {case_index}: {detail}")]
69 JudgeParse {
70 /// Zero-based index of the benchmark case that produced the invalid output.
71 case_index: usize,
72 /// Description of the parse failure (e.g., `"non-finite score: NaN"`).
73 detail: String,
74 },
75
76 /// The internal tokio semaphore used for concurrency control was closed.
77 ///
78 /// This is an internal invariant violation and should never occur in normal usage.
79 #[error("semaphore acquire failed: {0}")]
80 Semaphore(String),
81
82 /// The benchmark file exceeds the 10 MiB size limit.
83 #[error("benchmark file exceeds size limit ({size} bytes > {limit} bytes): {path}")]
84 BenchmarkTooLarge {
85 /// Canonicalized path of the file.
86 path: String,
87 /// Actual file size in bytes.
88 size: u64,
89 /// Maximum allowed size in bytes (currently 10 MiB).
90 limit: u64,
91 },
92
93 /// The benchmark file's canonical path escaped the expected parent directory.
94 ///
95 /// This indicates a symlink traversal attack and is rejected before any file I/O.
96 #[error("benchmark file path escapes allowed directory: {0}")]
97 PathTraversal(String),
98
99 /// A parameter value was outside its declared `[min, max]` range.
100 #[error("parameter out of range: {kind} value {value} not in [{min}, {max}]")]
101 OutOfRange {
102 /// Parameter name (e.g., `"temperature"`).
103 kind: String,
104 /// The value that was rejected.
105 value: f64,
106 /// Minimum allowed value (inclusive).
107 min: f64,
108 /// Maximum allowed value (inclusive).
109 max: f64,
110 },
111
112 /// All variations in the generator's search space have been visited.
113 #[error("search space exhausted: all variations in {strategy} have been visited")]
114 SearchSpaceExhausted {
115 /// Name of the strategy that exhausted (e.g., `"grid"`).
116 strategy: &'static str,
117 },
118
119 /// The [`Neighborhood`] radius was not finite and positive.
120 ///
121 /// [`Neighborhood`]: crate::Neighborhood
122 #[error("invalid neighborhood radius {radius}: must be finite and positive")]
123 InvalidRadius {
124 /// The invalid radius value that was rejected.
125 radius: f64,
126 },
127
128 /// An experiment result could not be persisted to SQLite.
129 #[error("experiment storage error: {0}")]
130 Storage(String),
131
132 /// The `min` and `max` bounds of a [`ParameterRange`] are inverted or equal.
133 ///
134 /// [`ParameterRange`]: crate::ParameterRange
135 #[error("invalid parameter range: min ({min}) must be strictly less than max ({max})")]
136 InvalidRange {
137 /// The minimum bound that was rejected.
138 min: f64,
139 /// The maximum bound that was rejected.
140 max: f64,
141 },
142
143 /// The `default` value of a [`ParameterRange`] lies outside `[min, max]`.
144 ///
145 /// [`ParameterRange`]: crate::ParameterRange
146 #[error("parameter default ({default}) out of range [{min}, {max}]")]
147 DefaultOutOfRange {
148 /// The default value that was rejected.
149 default: f64,
150 /// Minimum allowed value (inclusive).
151 min: f64,
152 /// Maximum allowed value (inclusive).
153 max: f64,
154 },
155}