zeph_experiments/error.rs
1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Error types for the experiments module.
5
6/// Errors that can occur during experiment evaluation, benchmark loading, or persistence.
7///
8/// Most variants carry structured context (file paths, token counts, parameter names)
9/// so that callers can surface actionable diagnostics to the user.
10///
11/// # Examples
12///
13/// ```rust
14/// use zeph_experiments::EvalError;
15///
16/// let err = EvalError::BudgetExceeded { used: 1_500, budget: 1_000 };
17/// assert!(err.to_string().contains("1500"));
18///
19/// let err = EvalError::InvalidRadius { radius: -1.0 };
20/// assert!(err.to_string().contains("finite and positive"));
21/// ```
22#[derive(Debug, thiserror::Error)]
23pub enum EvalError {
24 /// The benchmark TOML file could not be opened or read.
25 #[error("failed to load benchmark file {0}: {1}")]
26 BenchmarkLoad(String, #[source] std::io::Error),
27
28 /// The benchmark TOML file could not be parsed.
29 #[error("failed to parse benchmark file {0}: {1}")]
30 BenchmarkParse(String, String),
31
32 /// [`BenchmarkSet::validate`] was called on an empty `cases` vec.
33 ///
34 /// [`BenchmarkSet::validate`]: crate::BenchmarkSet::validate
35 #[error("benchmark set is empty")]
36 EmptyBenchmarkSet,
37
38 /// The cumulative token budget for judge calls was exhausted.
39 ///
40 /// When this error is returned from [`Evaluator::evaluate`], the report will
41 /// have `is_partial = true` and only include cases scored before the budget was hit.
42 ///
43 /// [`Evaluator::evaluate`]: crate::Evaluator::evaluate
44 #[error("evaluation budget exceeded: used {used} of {budget} tokens")]
45 BudgetExceeded { used: u64, budget: u64 },
46
47 /// An LLM call failed (network, auth, timeout, or API error).
48 #[error("LLM error during evaluation: {0}")]
49 Llm(#[from] zeph_llm::LlmError),
50
51 /// The subject or judge LLM call did not complete within the configured timeout.
52 ///
53 /// The case is excluded from scores and counted in [`EvalReport::error_count`].
54 ///
55 /// [`EvalReport::error_count`]: crate::EvalReport::error_count
56 #[error("{role} LLM call timed out after {timeout_secs}s for case {case_index}")]
57 Timeout {
58 /// Which model timed out: `"subject"` or `"judge"`.
59 role: &'static str,
60 /// Configured timeout in seconds.
61 timeout_secs: u64,
62 /// Zero-based index of the benchmark case.
63 case_index: usize,
64 },
65
66 /// The judge model returned a non-finite or structurally invalid score.
67 #[error("judge output parse failed for case {case_index}: {detail}")]
68 JudgeParse {
69 /// Zero-based index of the benchmark case that produced the invalid output.
70 case_index: usize,
71 /// Description of the parse failure (e.g., `"non-finite score: NaN"`).
72 detail: String,
73 },
74
75 /// The internal tokio semaphore used for concurrency control was closed.
76 ///
77 /// This is an internal invariant violation and should never occur in normal usage.
78 #[error("semaphore acquire failed: {0}")]
79 Semaphore(String),
80
81 /// The benchmark file exceeds the 10 MiB size limit.
82 #[error("benchmark file exceeds size limit ({size} bytes > {limit} bytes): {path}")]
83 BenchmarkTooLarge {
84 /// Canonicalized path of the file.
85 path: String,
86 /// Actual file size in bytes.
87 size: u64,
88 /// Maximum allowed size in bytes (currently 10 MiB).
89 limit: u64,
90 },
91
92 /// The benchmark file's canonical path escaped the expected parent directory.
93 ///
94 /// This indicates a symlink traversal attack and is rejected before any file I/O.
95 #[error("benchmark file path escapes allowed directory: {0}")]
96 PathTraversal(String),
97
98 /// A parameter value was outside its declared `[min, max]` range.
99 #[error("parameter out of range: {kind} value {value} not in [{min}, {max}]")]
100 OutOfRange {
101 /// Parameter name (e.g., `"temperature"`).
102 kind: String,
103 /// The value that was rejected.
104 value: f64,
105 /// Minimum allowed value (inclusive).
106 min: f64,
107 /// Maximum allowed value (inclusive).
108 max: f64,
109 },
110
111 /// All variations in the generator's search space have been visited.
112 #[error("search space exhausted: all variations in {strategy} have been visited")]
113 SearchSpaceExhausted {
114 /// Name of the strategy that exhausted (e.g., `"grid"`).
115 strategy: &'static str,
116 },
117
118 /// The [`Neighborhood`] radius was not finite and positive.
119 ///
120 /// [`Neighborhood`]: crate::Neighborhood
121 #[error("invalid neighborhood radius {radius}: must be finite and positive")]
122 InvalidRadius {
123 /// The invalid radius value that was rejected.
124 radius: f64,
125 },
126
127 /// An experiment result could not be persisted to SQLite.
128 #[error("experiment storage error: {0}")]
129 Storage(String),
130
131 /// The `min` and `max` bounds of a [`ParameterRange`] are inverted or equal.
132 ///
133 /// [`ParameterRange`]: crate::ParameterRange
134 #[error("invalid parameter range: min ({min}) must be strictly less than max ({max})")]
135 InvalidRange {
136 /// The minimum bound that was rejected.
137 min: f64,
138 /// The maximum bound that was rejected.
139 max: f64,
140 },
141
142 /// The `default` value of a [`ParameterRange`] lies outside `[min, max]`.
143 ///
144 /// [`ParameterRange`]: crate::ParameterRange
145 #[error("parameter default ({default}) out of range [{min}, {max}]")]
146 DefaultOutOfRange {
147 /// The default value that was rejected.
148 default: f64,
149 /// Minimum allowed value (inclusive).
150 min: f64,
151 /// Maximum allowed value (inclusive).
152 max: f64,
153 },
154}