Skip to main content

sloop/
vendor_error.rs

1//! Data-driven classification of rejected agent requests. Catalogs describe
2//! evidence only; outcome and cooldown policy remain in the scheduler.
3
4use std::collections::HashSet;
5use std::fmt;
6
7use serde::{Deserialize, Serialize};
8
9const SCHEMA_VERSION: u32 = 1;
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
12#[serde(rename_all = "snake_case")]
13pub enum VendorErrorClass {
14    AuthenticationRequired,
15    InvalidConfiguration,
16    RateLimited,
17    UnknownRejection,
18}
19
20impl VendorErrorClass {
21    pub fn requires_cooldown(self) -> bool {
22        matches!(self, Self::RateLimited | Self::UnknownRejection)
23    }
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
27pub struct VendorErrorMatch {
28    pub class: VendorErrorClass,
29    pub vendor: String,
30    pub rule_id: String,
31    pub diagnostic: String,
32}
33
34impl VendorErrorMatch {
35    pub fn evidence_json(&self, cooldown_until_ms: Option<i64>) -> String {
36        serde_json::json!({
37            "class": self.class,
38            "vendor": self.vendor,
39            "rule_id": self.rule_id,
40            "diagnostic": self.diagnostic,
41            "cooldown_until_ms": cooldown_until_ms,
42        })
43        .to_string()
44    }
45}
46
47#[derive(Debug, Clone)]
48pub struct VendorErrorClassifier {
49    rules: Vec<Rule>,
50}
51
52impl VendorErrorClassifier {
53    pub fn built_in() -> Result<Self, CatalogError> {
54        Self::from_yaml(&[
55            ("codex", include_str!("codex/errors.yaml")),
56            ("opencode", include_str!("opencode/errors.yaml")),
57            ("claude", include_str!("claude/errors.yaml")),
58        ])
59    }
60
61    fn from_yaml(catalogs: &[(&str, &str)]) -> Result<Self, CatalogError> {
62        let mut rules = Vec::new();
63        let mut ids = HashSet::new();
64        for (expected_vendor, yaml) in catalogs {
65            let catalog: Catalog = serde_yaml::from_str(yaml).map_err(|error| {
66                CatalogError(format!("invalid {expected_vendor} error catalog: {error}"))
67            })?;
68            if catalog.version != SCHEMA_VERSION {
69                return Err(CatalogError(format!(
70                    "unsupported {} error catalog schema version {}; expected {SCHEMA_VERSION}",
71                    catalog.vendor, catalog.version
72                )));
73            }
74            if catalog.vendor != *expected_vendor {
75                return Err(CatalogError(format!(
76                    "error catalog vendor `{}` does not match `{expected_vendor}`",
77                    catalog.vendor
78                )));
79            }
80            for raw in catalog.rules {
81                if raw.id.trim().is_empty() {
82                    return Err(CatalogError(format!(
83                        "{expected_vendor} error catalog contains an empty rule ID"
84                    )));
85                }
86                if !ids.insert(raw.id.clone()) {
87                    return Err(CatalogError(format!(
88                        "duplicate vendor error rule ID `{}`",
89                        raw.id
90                    )));
91                }
92                if raw.diagnostic.trim().is_empty() {
93                    return Err(CatalogError(format!(
94                        "vendor error rule `{}` has an empty diagnostic",
95                        raw.id
96                    )));
97                }
98                if raw.conditions.message_signatures.is_empty() {
99                    return Err(CatalogError(format!(
100                        "vendor error rule `{}` has no message signatures",
101                        raw.id
102                    )));
103                }
104                if raw
105                    .conditions
106                    .message_signatures
107                    .iter()
108                    .any(|signature| signature.is_empty())
109                {
110                    return Err(CatalogError(format!(
111                        "vendor error rule `{}` has an empty message signature",
112                        raw.id
113                    )));
114                }
115                rules.push(Rule {
116                    vendor: catalog.vendor.clone(),
117                    id: raw.id,
118                    class: raw.class,
119                    diagnostic: raw.diagnostic,
120                    conditions: raw.conditions,
121                });
122            }
123        }
124        Ok(Self { rules })
125    }
126
127    pub fn classify(
128        &self,
129        exit_status: Option<i32>,
130        stdout: &[u8],
131        stderr: &[u8],
132    ) -> Option<VendorErrorMatch> {
133        let mut scanner = self.scanner(exit_status);
134        scanner.feed_stdout(stdout);
135        scanner.feed_stderr(stderr);
136        scanner.finish()
137    }
138
139    pub fn scanner(&self, exit_status: Option<i32>) -> VendorErrorScanner<'_> {
140        VendorErrorScanner {
141            classifier: self,
142            exit_status,
143            states: self
144                .rules
145                .iter()
146                .map(|rule| RuleStates {
147                    stdout: ConditionState::new(&rule.conditions),
148                    stderr: ConditionState::new(&rule.conditions),
149                })
150                .collect(),
151        }
152    }
153}
154
155pub struct VendorErrorScanner<'a> {
156    classifier: &'a VendorErrorClassifier,
157    exit_status: Option<i32>,
158    states: Vec<RuleStates>,
159}
160
161impl VendorErrorScanner<'_> {
162    pub fn feed_stdout(&mut self, bytes: &[u8]) {
163        self.feed(Stream::Stdout, bytes);
164    }
165
166    pub fn feed_stderr(&mut self, bytes: &[u8]) {
167        self.feed(Stream::Stderr, bytes);
168    }
169
170    fn feed(&mut self, stream: Stream, bytes: &[u8]) {
171        for (rule, states) in self.classifier.rules.iter().zip(&mut self.states) {
172            if rule.conditions.stream.is_none() || rule.conditions.stream == Some(stream) {
173                states.get_mut(stream).feed(&rule.conditions, bytes, false);
174            }
175        }
176    }
177
178    pub fn finish(mut self) -> Option<VendorErrorMatch> {
179        for (rule, states) in self.classifier.rules.iter().zip(&mut self.states) {
180            if !rule.matches_exit(self.exit_status) {
181                continue;
182            }
183            states.stdout.feed(&rule.conditions, &[], true);
184            states.stderr.feed(&rule.conditions, &[], true);
185            let matched = match rule.conditions.stream {
186                Some(Stream::Stdout) => states.stdout.matched(),
187                Some(Stream::Stderr) => states.stderr.matched(),
188                None => states.stdout.matched() || states.stderr.matched(),
189            };
190            if matched {
191                return Some(VendorErrorMatch {
192                    class: rule.class,
193                    vendor: rule.vendor.clone(),
194                    rule_id: rule.id.clone(),
195                    diagnostic: rule.diagnostic.clone(),
196                });
197            }
198        }
199        None
200    }
201}
202
203#[derive(Debug)]
204pub struct CatalogError(String);
205
206impl fmt::Display for CatalogError {
207    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
208        formatter.write_str(&self.0)
209    }
210}
211
212impl std::error::Error for CatalogError {}
213
214#[derive(Debug, Deserialize)]
215#[serde(deny_unknown_fields)]
216struct Catalog {
217    version: u32,
218    vendor: String,
219    rules: Vec<RawRule>,
220}
221
222#[derive(Debug, Deserialize)]
223#[serde(deny_unknown_fields)]
224struct RawRule {
225    id: String,
226    class: VendorErrorClass,
227    diagnostic: String,
228    #[serde(rename = "match")]
229    conditions: MatchConditions,
230}
231
232#[derive(Debug, Clone)]
233struct Rule {
234    vendor: String,
235    id: String,
236    class: VendorErrorClass,
237    diagnostic: String,
238    conditions: MatchConditions,
239}
240
241impl Rule {
242    fn matches_exit(&self, exit_status: Option<i32>) -> bool {
243        self.conditions
244            .exit_status
245            .is_none_or(|expected| exit_status == Some(expected))
246    }
247}
248
249#[derive(Debug, Clone, Deserialize)]
250#[serde(deny_unknown_fields)]
251struct MatchConditions {
252    #[serde(default)]
253    exit_status: Option<i32>,
254    #[serde(default)]
255    stream: Option<Stream>,
256    #[serde(default)]
257    status_code: Option<u16>,
258    #[serde(default)]
259    error_code: Option<String>,
260    message_signatures: Vec<String>,
261}
262
263impl MatchConditions {
264    fn overlap_len(&self) -> usize {
265        let signatures = self
266            .message_signatures
267            .iter()
268            .map(|signature| signature.len());
269        let status = self
270            .status_code
271            .into_iter()
272            .flat_map(status_code_patterns)
273            .map(|pattern| pattern.len());
274        let error = self
275            .error_code
276            .as_deref()
277            .into_iter()
278            .flat_map(error_code_patterns)
279            .map(|pattern| pattern.len());
280        signatures.chain(status).chain(error).max().unwrap_or(1)
281    }
282}
283
284#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
285#[serde(rename_all = "snake_case")]
286enum Stream {
287    Stdout,
288    Stderr,
289}
290
291struct RuleStates {
292    stdout: ConditionState,
293    stderr: ConditionState,
294}
295
296impl RuleStates {
297    fn get_mut(&mut self, stream: Stream) -> &mut ConditionState {
298        match stream {
299            Stream::Stdout => &mut self.stdout,
300            Stream::Stderr => &mut self.stderr,
301        }
302    }
303}
304
305struct ConditionState {
306    status_found: bool,
307    error_found: bool,
308    signatures_found: Vec<bool>,
309    tail: Vec<u8>,
310    overlap_len: usize,
311}
312
313impl ConditionState {
314    fn new(conditions: &MatchConditions) -> Self {
315        Self {
316            status_found: conditions.status_code.is_none(),
317            error_found: conditions.error_code.is_none(),
318            signatures_found: vec![false; conditions.message_signatures.len()],
319            tail: Vec::new(),
320            overlap_len: conditions.overlap_len(),
321        }
322    }
323
324    fn feed(&mut self, conditions: &MatchConditions, bytes: &[u8], final_chunk: bool) {
325        let mut window = std::mem::take(&mut self.tail);
326        window.extend_from_slice(bytes);
327        if !self.status_found
328            && let Some(code) = conditions.status_code
329        {
330            self.status_found = contains_status_code(&window, code, final_chunk);
331        }
332        if !self.error_found
333            && let Some(code) = conditions.error_code.as_deref()
334        {
335            self.error_found = contains_error_code(&window, code, final_chunk);
336        }
337        for (found, signature) in self
338            .signatures_found
339            .iter_mut()
340            .zip(&conditions.message_signatures)
341        {
342            *found |= contains(&window, signature.as_bytes());
343        }
344        // Keep one full pattern length because status/error codes also need
345        // the following byte to prove their token boundary.
346        let keep = self.overlap_len.min(window.len());
347        self.tail.extend_from_slice(&window[window.len() - keep..]);
348    }
349
350    fn matched(&self) -> bool {
351        self.status_found && self.error_found && self.signatures_found.iter().all(|found| *found)
352    }
353}
354
355fn contains(bytes: &[u8], needle: &[u8]) -> bool {
356    !needle.is_empty() && bytes.windows(needle.len()).any(|window| window == needle)
357}
358
359fn status_code_patterns(code: u16) -> Vec<String> {
360    let code = code.to_string();
361    vec![
362        format!("status {code}"),
363        format!("status: {code}"),
364        format!("status={code}"),
365        format!("\"status\":{code}"),
366        format!("\"status\": {code}"),
367        format!("\"status_code\":{code}"),
368        format!("\"statuscode\":{code}"),
369    ]
370}
371
372fn contains_status_code(bytes: &[u8], code: u16, allow_end: bool) -> bool {
373    let text = String::from_utf8_lossy(bytes).to_ascii_lowercase();
374    status_code_patterns(code).iter().any(|pattern| {
375        contains_with_boundary(&text, pattern, allow_end, |byte| byte.is_ascii_digit())
376    })
377}
378
379fn error_code_patterns(code: &str) -> Vec<String> {
380    let code = code.to_ascii_lowercase();
381    vec![
382        format!("code {code}"),
383        format!("code: {code}"),
384        format!("code={code}"),
385        format!("\"code\":\"{code}\""),
386        format!("\"code\": \"{code}\""),
387        format!("\"error_code\":\"{code}\""),
388    ]
389}
390
391fn contains_error_code(bytes: &[u8], code: &str, allow_end: bool) -> bool {
392    let text = String::from_utf8_lossy(bytes).to_ascii_lowercase();
393    error_code_patterns(code).iter().any(|pattern| {
394        contains_with_boundary(&text, pattern, allow_end, |byte| {
395            byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')
396        })
397    })
398}
399
400fn contains_with_boundary(
401    haystack: &str,
402    needle: &str,
403    allow_end: bool,
404    continues_value: impl Fn(u8) -> bool,
405) -> bool {
406    haystack.match_indices(needle).any(|(start, _)| {
407        haystack
408            .as_bytes()
409            .get(start + needle.len())
410            .map_or(allow_end, |byte| !continues_value(*byte))
411    })
412}
413
414#[cfg(test)]
415mod tests {
416    use super::*;
417
418    const VALID: &str = r#"
419version: 1
420vendor: test
421rules:
422  - id: test.rate-limit
423    class: rate_limited
424    diagnostic: Request rate limited
425    match:
426      exit_status: 1
427      stream: stderr
428      status_code: 429
429      error_code: rate_limit_exceeded
430      message_signatures: ["try again later"]
431"#;
432
433    #[test]
434    fn built_in_catalogs_validate_and_match_captured_vendor_fixtures() {
435        let classifier = VendorErrorClassifier::built_in().unwrap();
436        let cases = [
437            (
438                br#"Error: status 401 Missing bearer or basic authentication in header"#.as_slice(),
439                VendorErrorClass::AuthenticationRequired,
440                "codex.authentication.missing-header",
441            ),
442            (
443                br#"request failed: status: 400 model is not supported when using Codex with a ChatGPT account"#.as_slice(),
444                VendorErrorClass::InvalidConfiguration,
445                "codex.configuration.unsupported-chatgpt-model",
446            ),
447            (
448                br#"UnknownError: Unexpected server error. Check server logs for details."#.as_slice(),
449                VendorErrorClass::UnknownRejection,
450                "opencode.rejection.unexpected-server-error",
451            ),
452            (
453                br#"API Error: You've hit your limit; resets 12am (UTC)"#.as_slice(),
454                VendorErrorClass::RateLimited,
455                "claude.rate-limit.usage-limit",
456            ),
457        ];
458        for (stderr, class, id) in cases {
459            let matched = classifier.classify(Some(1), b"", stderr).unwrap();
460            assert_eq!(matched.class, class);
461            assert_eq!(matched.rule_id, id);
462        }
463    }
464
465    #[test]
466    fn schema_validation_rejects_unknown_classes_duplicate_ids_and_empty_signatures() {
467        let unknown = VALID.replace("rate_limited", "retry_someday");
468        assert!(VendorErrorClassifier::from_yaml(&[("test", &unknown)]).is_err());
469
470        let duplicate = format!("{VALID}\n{}", "");
471        assert!(
472            VendorErrorClassifier::from_yaml(&[("test", &duplicate), ("test", VALID)]).is_err()
473        );
474
475        let empty = VALID.replace("[\"try again later\"]", "[\"\"]");
476        let error = VendorErrorClassifier::from_yaml(&[("test", &empty)])
477            .unwrap_err()
478            .to_string();
479        assert!(error.contains("empty message signature"), "{error}");
480    }
481
482    #[test]
483    fn all_conditions_must_match_in_one_selected_stream() {
484        let classifier = VendorErrorClassifier::from_yaml(&[("test", VALID)]).unwrap();
485        let matching =
486            br#"{"status":429,"code":"rate_limit_exceeded","message":"try again later"}"#;
487        assert!(classifier.classify(Some(1), b"", matching).is_some());
488        assert!(classifier.classify(Some(0), b"", matching).is_none());
489        assert!(classifier.classify(Some(1), matching, b"").is_none());
490        assert!(
491            classifier
492                .classify(
493                    Some(1),
494                    br#"{"status":429,"code":"rate_limit_exceeded"}"#,
495                    b"try again later"
496                )
497                .is_none()
498        );
499
500        let mut scanner = classifier.scanner(Some(1));
501        for chunk in [
502            b"{\"sta".as_slice(),
503            b"tus\":429,\"code\":\"rate_",
504            b"limit_exceeded\",\"message\":\"try ag",
505            b"ain later\"}",
506        ] {
507            scanner.feed_stderr(chunk);
508        }
509        assert!(scanner.finish().is_some());
510    }
511
512    #[test]
513    fn duplicate_signatures_binary_bytes_and_near_matches_are_safe() {
514        let classifier = VendorErrorClassifier::from_yaml(&[("test", VALID)]).unwrap();
515        let mut bytes = vec![0xff, 0x00];
516        bytes.extend_from_slice(
517            br#"status 429 code rate_limit_exceeded try again later try again later"#,
518        );
519        assert!(classifier.classify(Some(1), b"", &bytes).is_some());
520        assert!(
521            classifier
522                .classify(
523                    Some(1),
524                    b"",
525                    b"status 429 code rate_limit_exceeded try again much later"
526                )
527                .is_none()
528        );
529        assert!(
530            classifier
531                .classify(Some(1), b"", b"plain number 429")
532                .is_none()
533        );
534        assert!(
535            classifier
536                .classify(
537                    Some(1),
538                    b"",
539                    b"status 4290 code rate_limit_exceeded try again later"
540                )
541                .is_none()
542        );
543    }
544}