assay_core/errors/
mod.rs

1pub mod diagnostic;
2pub mod similarity;
3
4pub use diagnostic::Diagnostic;
5
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub enum RunErrorKind {
8    TraceNotFound,
9    MissingConfig,
10    ConfigParse,
11    InvalidArgs,
12    ProviderRateLimit,
13    ProviderTimeout,
14    ProviderServer,
15    Network,
16    JudgeUnavailable,
17    Other,
18}
19
20#[derive(Debug, Clone, PartialEq, Eq)]
21pub struct RunError {
22    pub kind: RunErrorKind,
23    pub message: String,
24    pub path: Option<String>,
25    pub status: Option<u16>,
26    pub provider: Option<String>,
27    pub detail: Option<String>,
28    /// True when kind was inferred from free-form message parsing.
29    pub legacy_classified: bool,
30}
31
32impl std::fmt::Display for RunError {
33    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
34        f.write_str(&self.message)
35    }
36}
37
38impl std::error::Error for RunError {}
39
40impl RunError {
41    pub fn new(kind: RunErrorKind, message: impl Into<String>) -> Self {
42        Self {
43            kind,
44            message: message.into(),
45            path: None,
46            status: None,
47            provider: None,
48            detail: None,
49            legacy_classified: false,
50        }
51    }
52
53    pub fn with_path(mut self, path: impl Into<String>) -> Self {
54        self.path = Some(path.into());
55        self
56    }
57
58    pub fn with_status(mut self, status: u16) -> Self {
59        self.status = Some(status);
60        self
61    }
62
63    pub fn with_provider(mut self, provider: impl Into<String>) -> Self {
64        self.provider = Some(provider.into());
65        self
66    }
67
68    pub fn with_detail(mut self, detail: impl Into<String>) -> Self {
69        self.detail = Some(detail.into());
70        self
71    }
72
73    pub fn trace_not_found(path: impl Into<String>, detail: impl Into<String>) -> Self {
74        let path = path.into();
75        let detail = detail.into();
76        Self::new(
77            RunErrorKind::TraceNotFound,
78            format!("Trace not found: {}", path),
79        )
80        .with_path(path)
81        .with_detail(detail)
82    }
83
84    pub fn missing_config(path: impl Into<String>, detail: impl Into<String>) -> Self {
85        let path = path.into();
86        let detail = detail.into();
87        Self::new(
88            RunErrorKind::MissingConfig,
89            format!("Config file not found: {}", path),
90        )
91        .with_path(path)
92        .with_detail(detail)
93    }
94
95    pub fn config_parse(path: Option<String>, detail: impl Into<String>) -> Self {
96        let detail = detail.into();
97        let mut err = Self::new(RunErrorKind::ConfigParse, detail.clone()).with_detail(detail);
98        if let Some(path) = path {
99            err = err.with_path(path);
100        }
101        err
102    }
103
104    pub fn invalid_args(detail: impl Into<String>) -> Self {
105        let detail = detail.into();
106        Self::new(RunErrorKind::InvalidArgs, detail.clone()).with_detail(detail)
107    }
108
109    pub fn provider_rate_limit(
110        status: u16,
111        provider: Option<String>,
112        detail: impl Into<String>,
113    ) -> Self {
114        let detail = detail.into();
115        let mut err = Self::new(RunErrorKind::ProviderRateLimit, detail.clone())
116            .with_status(status)
117            .with_detail(detail);
118        if let Some(provider) = provider {
119            err = err.with_provider(provider);
120        }
121        err
122    }
123
124    pub fn provider_timeout(provider: Option<String>, detail: impl Into<String>) -> Self {
125        let detail = detail.into();
126        let mut err = Self::new(RunErrorKind::ProviderTimeout, detail.clone()).with_detail(detail);
127        if let Some(provider) = provider {
128            err = err.with_provider(provider);
129        }
130        err
131    }
132
133    pub fn provider_server(
134        status: Option<u16>,
135        provider: Option<String>,
136        detail: impl Into<String>,
137    ) -> Self {
138        let detail = detail.into();
139        let mut err = Self::new(RunErrorKind::ProviderServer, detail.clone()).with_detail(detail);
140        if let Some(status) = status {
141            err = err.with_status(status);
142        }
143        if let Some(provider) = provider {
144            err = err.with_provider(provider);
145        }
146        err
147    }
148
149    pub fn network(provider: Option<String>, detail: impl Into<String>) -> Self {
150        let detail = detail.into();
151        let mut err = Self::new(RunErrorKind::Network, detail.clone()).with_detail(detail);
152        if let Some(provider) = provider {
153            err = err.with_provider(provider);
154        }
155        err
156    }
157
158    pub fn judge_unavailable(provider: Option<String>, detail: impl Into<String>) -> Self {
159        let detail = detail.into();
160        let mut err = Self::new(RunErrorKind::JudgeUnavailable, detail.clone()).with_detail(detail);
161        if let Some(provider) = provider {
162            err = err.with_provider(provider);
163        }
164        err
165    }
166
167    pub fn other(detail: impl Into<String>) -> Self {
168        let detail = detail.into();
169        Self::new(RunErrorKind::Other, detail.clone()).with_detail(detail)
170    }
171
172    pub fn classify_message(message: impl Into<String>) -> Self {
173        Self::legacy_classify_message(message)
174    }
175
176    pub fn legacy_classify_message(message: impl Into<String>) -> Self {
177        let message = message.into();
178        let msg = message.to_lowercase();
179        let has_not_found_signal = msg.contains("no such file")
180            || msg.contains("not found")
181            || msg.contains("cannot find")
182            || msg.contains("can't find")
183            || msg.contains("could not find")
184            || msg.contains("os error 2");
185
186        let kind = if msg.contains("trace not found")
187            || msg.contains("tracenotfound")
188            || msg.contains("failed to load trace")
189            || (msg.contains("failed to ingest trace") && has_not_found_signal)
190            || (msg.contains("trace") && has_not_found_signal)
191        {
192            RunErrorKind::TraceNotFound
193        } else if msg.contains("failed to ingest trace") {
194            RunErrorKind::ConfigParse
195        } else if msg.contains("no config found")
196            || msg.contains("config missing")
197            || msg.contains("config file not found")
198            || (msg.contains("failed to read config") && has_not_found_signal)
199        {
200            RunErrorKind::MissingConfig
201        } else if msg.contains("cannot use --")
202            || msg.contains("invalid argument")
203            || msg.contains("invalid args")
204        {
205            RunErrorKind::InvalidArgs
206        } else if msg.contains("config error")
207            || msg.contains("configerror")
208            || msg.contains("failed to parse yaml")
209            || msg.contains("unknown field")
210        {
211            RunErrorKind::ConfigParse
212        } else if msg.contains("rate limit") || msg.contains("429") {
213            RunErrorKind::ProviderRateLimit
214        } else if msg.contains("timeout") {
215            RunErrorKind::ProviderTimeout
216        } else if msg.contains("500")
217            || msg.contains("502")
218            || msg.contains("503")
219            || msg.contains("504")
220            || msg.contains("provider error")
221        {
222            RunErrorKind::ProviderServer
223        } else if msg.contains("network") || msg.contains("connection") || msg.contains("dns") {
224            RunErrorKind::Network
225        } else if msg.contains("judge unavailable")
226            || msg.contains("judge error")
227            || msg.contains("judge failed")
228        {
229            RunErrorKind::JudgeUnavailable
230        } else {
231            RunErrorKind::Other
232        };
233
234        let mut run_error = Self::new(kind, message);
235        run_error.legacy_classified = true;
236        run_error
237    }
238
239    pub fn from_anyhow(err: &anyhow::Error) -> Self {
240        Self::legacy_classify_message(err.to_string())
241    }
242}
243
244/// Maps selected anyhow errors to structured diagnostics with actionable fixes.
245pub fn try_map_error(err: &anyhow::Error) -> Option<Diagnostic> {
246    // Preserve existing typed diagnostics.
247    if let Some(diag) = err.downcast_ref::<Diagnostic>() {
248        return Some(diag.clone());
249    }
250
251    let msg = err.to_string();
252
253    // Best-effort mapping for embedding dimension mismatch messages.
254    if msg.contains("embedding dims mismatch") || msg.contains("dimension mismatch") {
255        return Some(
256            Diagnostic::new(
257                diagnostic::codes::E_EMB_DIMS,
258                "Embedding dimensions mismatch",
259            )
260            .with_context(serde_json::json!({ "raw_error": msg }))
261            .with_fix_step("Run: assay trace precompute-embeddings --trace <file> ...")
262            .with_fix_step(
263                "Ensure the same embedding model is used for baseline and candidate runs",
264            ),
265        );
266    }
267
268    // Baseline schema/version incompatibility.
269    if msg.contains("Baseline mismatch") || (msg.contains("baseline") && msg.contains("schema")) {
270        return Some(
271            Diagnostic::new(
272                diagnostic::codes::E_BASE_MISMATCH,
273                "Baseline incompatible with current run",
274            )
275            .with_context(serde_json::json!({ "raw_error": msg }))
276            .with_fix_step("Regenerate baseline on main branch: assay ci --export-baseline ...")
277            .with_fix_step("Check that your config suite name matches the baseline suite"),
278        );
279    }
280
281    None
282}
283
284use std::fmt::{Display, Formatter};
285
286#[derive(Debug)]
287pub struct ConfigError(pub String);
288
289impl Display for ConfigError {
290    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
291        write!(f, "ConfigError: {}", self.0)
292    }
293}
294impl std::error::Error for ConfigError {}
295
296#[cfg(test)]
297mod tests {
298    use super::{RunError, RunErrorKind};
299
300    #[test]
301    fn classify_message_maps_config_errors() {
302        assert_eq!(
303            RunError::classify_message("trace not found: traces/missing.jsonl").kind,
304            RunErrorKind::TraceNotFound
305        );
306        assert_eq!(
307            RunError::classify_message(
308                "ConfigError: failed to read config eval.yaml: No such file or directory (os error 2)"
309            )
310            .kind,
311            RunErrorKind::MissingConfig
312        );
313        assert_eq!(
314            RunError::classify_message("config error: unknown field `foo`").kind,
315            RunErrorKind::ConfigParse
316        );
317        assert_eq!(
318            RunError::classify_message("Failed to ingest trace: invalid JSON on line 1").kind,
319            RunErrorKind::ConfigParse
320        );
321    }
322
323    #[test]
324    fn classify_message_does_not_misclassify_ingest_errors_as_not_found() {
325        assert_ne!(
326            RunError::classify_message("Failed to ingest trace: unsupported schema_version").kind,
327            RunErrorKind::TraceNotFound
328        );
329    }
330
331    #[test]
332    fn classify_message_maps_infra_errors() {
333        assert_eq!(
334            RunError::classify_message("provider returned 429").kind,
335            RunErrorKind::ProviderRateLimit
336        );
337        assert_eq!(
338            RunError::classify_message("request timeout while calling provider").kind,
339            RunErrorKind::ProviderTimeout
340        );
341        assert_eq!(
342            RunError::classify_message("provider error: 503").kind,
343            RunErrorKind::ProviderServer
344        );
345        assert_eq!(
346            RunError::classify_message("network dns resolution failed").kind,
347            RunErrorKind::Network
348        );
349    }
350
351    #[test]
352    fn typed_constructors_capture_stable_fields() {
353        let trace = RunError::trace_not_found("traces/missing.jsonl", "os error 2");
354        assert_eq!(trace.kind, RunErrorKind::TraceNotFound);
355        assert_eq!(trace.path.as_deref(), Some("traces/missing.jsonl"));
356        assert_eq!(trace.detail.as_deref(), Some("os error 2"));
357        assert!(!trace.legacy_classified);
358
359        let provider = RunError::provider_server(
360            Some(503),
361            Some("openai".to_string()),
362            "provider unavailable",
363        );
364        assert_eq!(provider.kind, RunErrorKind::ProviderServer);
365        assert_eq!(provider.status, Some(503));
366        assert_eq!(provider.provider.as_deref(), Some("openai"));
367        assert!(!provider.legacy_classified);
368    }
369
370    #[test]
371    fn legacy_classification_is_explicitly_marked() {
372        let legacy = RunError::classify_message("provider returned 429");
373        assert!(legacy.legacy_classified);
374    }
375}
assay_core/errors/mod.rs

assay_core/errors/
mod.rs
