yapi_ai/
errors.rs

1//! Classifying failed responses: context overflow, recoverable truncation, and
2//! transient errors worth retrying.
3//!
4//! Port of `packages/ai/src/utils/overflow.ts` and `retry.ts` in pi `v1.0.0`.
5
6use std::sync::OnceLock;
7
8use regex_lite::{Regex, RegexBuilder};
9use yapi_types::message::{AssistantMessage, StopReason};
10
11/// Error texts that mean the input exceeded the context window.
12const OVERFLOW_PATTERNS: &[&str] = &[
13    r"prompt (?:is )?too long",
14    r"prompt exceeds max length",
15    r"request_too_large",
16    r"input is too long for requested model",
17    r"exceeds the context window",
18    r"exceeds (?:the )?(?:model'?s )?maximum context length(?: of [\d,]+ tokens?|\s*\([\d,]+\))",
19    r"input token count.*exceeds the maximum",
20    r"maximum prompt length is \d+",
21    r"reduce the length of the messages",
22    r"maximum context length is \d+ tokens",
23    r"exceeds (?:the )?maximum allowed input length of [\d,]+ tokens?",
24    r"input \(\d+ tokens\) is longer than the model'?s context length \(\d+ tokens\)",
25    r"exceeds the limit of \d+",
26    r"exceeds the available context size",
27    r"greater than the context length",
28    r"context window exceeds limit",
29    r"exceeded model token limit",
30    r"too large for model with \d+ maximum context length",
31    r"prompt has [\d,]+ tokens?, but the configured context size is [\d,]+ tokens?",
32    r"model_context_window_exceeded",
33    r"prompt too long; exceeded (?:max )?context length",
34    r"range of input length should be",
35    r"context[_ ]length[_ ]exceeded",
36    r"too many tokens",
37    r"token limit exceeded",
38];
39
40/// Error texts that are never overflow, even when an overflow pattern matches.
41const NON_OVERFLOW_PATTERNS: &[&str] = &[
42    r"^(Throttling error|Service unavailable):",
43    r"rate limit",
44    r"too many requests",
45];
46
47const CEREBRAS_BODYLESS_OVERFLOW: &str = r"^4(?:00|13)\s*(?:status code)?\s*\(no body\)";
48
49/// Subscription and quota limits: they do not clear on retry.
50const NON_RETRYABLE: &[&str] = &[
51    "GoUsageLimitError",
52    "FreeUsageLimitError",
53    "Monthly usage limit reached",
54    "available balance",
55    "insufficient_quota",
56    "out of budget",
57    "quota exceeded",
58    "billing",
59    "subscription_sharing_usage_limit_exceeded",
60];
61
62/// Transient provider, transport and stream failures.
63const RETRYABLE: &[&str] = &[
64    "overloaded",
65    "currently experiencing high demand",
66    "rate.?limit",
67    "too many requests",
68    "429",
69    "500",
70    "502",
71    "503",
72    "504",
73    "520",
74    "524",
75    "service.?unavailable",
76    "server.?error",
77    "internal.?error",
78    "provider.?returned.?error",
79    "exceeded request buffer limit while retrying upstream",
80    "network.?error",
81    "connection.?error",
82    "connection.?refused",
83    "connection.?lost",
84    "other side closed",
85    "fetch failed",
86    "getaddrinfo",
87    "ENOTFOUND",
88    "EAI_AGAIN",
89    "upstream.?connect",
90    "reset before headers",
91    "socket hang up",
92    "socket connection was closed",
93    "timed? out",
94    "timeout",
95    "terminated",
96    "websocket.?closed",
97    "websocket.?error",
98    "ended without",
99    "stream ended before message_stop",
100    "stream ended before a terminal response event",
101    "http2 request did not get a response",
102    "retry delay",
103    "you can retry your request",
104    "try your request again",
105    "please retry your request",
106    "ResourceExhausted",
107    "subscription_sharing_usage_unavailable",
108    "subscription_sharing_user_unavailable",
109];
110
111/// Default cap on one agent-level retry delay.
112pub const DEFAULT_MAX_AGENT_RETRY_DELAY_MS: u64 = 60_000;
113
114fn insensitive(pattern: &str) -> Regex {
115    RegexBuilder::new(pattern)
116        .case_insensitive(true)
117        .build()
118        .expect("constant pattern; `patterns_compile` checks each one")
119}
120
121struct Patterns {
122    overflow: Vec<Regex>,
123    non_overflow: Vec<Regex>,
124    cerebras: Regex,
125    non_retryable: Regex,
126    retryable: Regex,
127}
128
129fn patterns() -> &'static Patterns {
130    static PATTERNS: OnceLock<Patterns> = OnceLock::new();
131    PATTERNS.get_or_init(|| Patterns {
132        overflow: OVERFLOW_PATTERNS.iter().map(|p| insensitive(p)).collect(),
133        non_overflow: NON_OVERFLOW_PATTERNS
134            .iter()
135            .map(|p| insensitive(p))
136            .collect(),
137        cerebras: insensitive(CEREBRAS_BODYLESS_OVERFLOW),
138        non_retryable: insensitive(&NON_RETRYABLE.join("|")),
139        retryable: insensitive(&RETRYABLE.join("|")),
140    })
141}
142
143/// Whether a response failed or was cut short because the input exceeded the
144/// context window: an overflow error text, input above `context_window` on a
145/// completed response, or a length stop with no output and a full window.
146/// A `context_window` of 0 checks the error text only.
147pub fn is_context_overflow(message: &AssistantMessage, context_window: u64) -> bool {
148    let patterns = patterns();
149    if message.stop_reason == StopReason::Error
150        && let Some(error) = message.error_message.as_deref().filter(|e| !e.is_empty())
151        && !patterns.non_overflow.iter().any(|p| p.is_match(error))
152        && (patterns.overflow.iter().any(|p| p.is_match(error))
153            || (message.provider == "cerebras" && patterns.cerebras.is_match(error)))
154    {
155        return true;
156    }
157    let input = message.usage.input + message.usage.cache_read;
158    if context_window > 0 && message.stop_reason == StopReason::Stop && input > context_window {
159        return true;
160    }
161    context_window > 0
162        && message.stop_reason == StopReason::Length
163        && message.usage.output == 0
164        && input as f64 >= context_window as f64 * 0.99
165}
166
167/// Whether a length stop ended below the intended output limit, so one
168/// compact-and-retry attempt may recover it. `desired_max_output` is the limit
169/// before any clamping to the context.
170pub fn is_recoverable_length(message: &AssistantMessage, desired_max_output: u64) -> bool {
171    message.stop_reason == StopReason::Length
172        && desired_max_output > 0
173        && message.usage.output < desired_max_output
174}
175
176/// Whether a failed response is transient and worth retrying.
177pub fn is_retryable_assistant_error(message: &AssistantMessage) -> bool {
178    let Some(error) = message
179        .error_message
180        .as_deref()
181        .filter(|error| message.stop_reason == StopReason::Error && !error.is_empty())
182    else {
183        return false;
184    };
185    let patterns = patterns();
186    !patterns.non_retryable.is_match(error) && patterns.retryable.is_match(error)
187}
188
189/// The backoff before retry `attempt` (1-based): `base * 2^(attempt-1)`, capped.
190pub fn retry_delay_ms(base_delay_ms: u64, max_delay_ms: Option<u64>, attempt: u32) -> u64 {
191    let factor = 1u64
192        .checked_shl(attempt.saturating_sub(1))
193        .unwrap_or(u64::MAX);
194    base_delay_ms
195        .saturating_mul(factor)
196        .min(max_delay_ms.unwrap_or(DEFAULT_MAX_AGENT_RETRY_DELAY_MS))
197}
198
199#[cfg(test)]
200mod tests {
201    use super::*;
202
203    fn failed(error: &str) -> AssistantMessage {
204        let mut message = crate::stream::new_output(
205            &serde_json::from_value(serde_json::json!({
206                "id": "m", "name": "m", "api": "openai-responses", "provider": "openai",
207                "baseUrl": "", "reasoning": false, "input": ["text"],
208                "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0},
209                "contextWindow": 100, "maxTokens": 10,
210            }))
211            .unwrap(),
212            0,
213        );
214        message.stop_reason = StopReason::Error;
215        message.error_message = Some(error.into());
216        message
217    }
218
219    #[test]
220    fn patterns_compile() {
221        for pattern in OVERFLOW_PATTERNS.iter().chain(NON_OVERFLOW_PATTERNS) {
222            RegexBuilder::new(pattern).build().unwrap();
223        }
224        RegexBuilder::new(CEREBRAS_BODYLESS_OVERFLOW)
225            .build()
226            .unwrap();
227    }
228
229    #[test]
230    fn classifies_errors() {
231        assert!(is_context_overflow(
232            &failed("prompt is too long: 213462 tokens > 200000 maximum"),
233            0
234        ));
235        assert!(!is_context_overflow(
236            &failed("ThrottlingException: Too many tokens, rate limit"),
237            0
238        ));
239        assert!(is_retryable_assistant_error(&failed(
240            "server_error: The server had an error"
241        )));
242        assert!(is_retryable_assistant_error(&failed(
243            "OpenAI API error (500): {}"
244        )));
245        assert!(!is_retryable_assistant_error(&failed(
246            "429 insufficient_quota"
247        )));
248        assert!(!is_retryable_assistant_error(&failed("400 bad request")));
249        assert_eq!(retry_delay_ms(2000, None, 3), 8000);
250        assert_eq!(retry_delay_ms(2000, None, 64), 60_000);
251    }
252}
yapi_ai/errors.rs

yapi_ai/
errors.rs
