rig_core/error/
retry.rs

1//! What a failed call's report says about retrying it, for a retry policy
2//! such as rig-ecs's: the wait asked for, and whether the request outgrew
3//! the model's context window.
4
5use std::time::{Duration, SystemTime};
6
7use super::{ErrorKind, ErrorReport};
8
9/// The code of a report that the request outgrew the context window.
10const CONTEXT_OVERFLOW: &str = "context_length_exceeded";
11
12impl ErrorReport {
13    /// A request error saying the request does not fit the model's context
14    /// window, which [`is_context_overflow`](Self::is_context_overflow)
15    /// recognizes.
16    pub fn context_overflow(message: impl Into<String>) -> Self {
17        Self::new(ErrorKind::Request, message).with_code(CONTEXT_OVERFLOW)
18    }
19
20    /// The wait the provider asked for, as of `now`: the `retry-after-ms`
21    /// header (OpenAI), else `retry-after` in seconds or as an HTTP date,
22    /// else a hint in the body: Gemini's `"retryDelay": "2s"` or OpenAI's
23    /// "try again in 1.5s".
24    pub fn retry_after(&self, now: SystemTime) -> Option<Duration> {
25        let header = |name: &str| {
26            self.provider_response_headers()
27                .and_then(|headers| headers.get(name))
28                .and_then(|value| value.to_str().ok())
29                .map(str::trim)
30        };
31        if let Some(millis) = header("retry-after-ms").and_then(|value| value.parse::<f64>().ok()) {
32            return seconds(millis / 1000.0);
33        }
34        if let Some(value) = header("retry-after") {
35            return match value.parse::<f64>() {
36                Ok(secs) => seconds(secs),
37                Err(_) => httpdate::parse_http_date(value)
38                    .ok()
39                    .map(|at| at.duration_since(now).unwrap_or_default()),
40            };
41        }
42        let body = self
43            .provider_response_body()
44            .unwrap_or(&self.message)
45            .to_ascii_lowercase();
46        ["\"retrydelay\"", "try again in "]
47            .iter()
48            .find_map(|marker| {
49                let after = body.get(body.find(marker)? + marker.len()..)?;
50                hinted_wait(after.trim_start_matches([' ', ':', '"']))
51            })
52    }
53
54    /// Whether this report says the request does not fit the model's
55    /// context window: the `context_length_exceeded` code, a 413, or the
56    /// provider saying so in its own words, less rate limits that happen to
57    /// mention tokens. A retryable failure is never an overflow, which fails
58    /// the same way every time (Bedrock's "too many tokens" throttling is
59    /// retryable).
60    pub fn is_context_overflow(&self) -> bool {
61        if self.code.as_deref() == Some(CONTEXT_OVERFLOW) {
62            return true;
63        }
64        if self.is_retryable() {
65            return false;
66        }
67        if self.http_status == Some(413) {
68            return true;
69        }
70        [Some(self.message.as_str()), self.provider_response_body()]
71            .into_iter()
72            .flatten()
73            .map(str::to_ascii_lowercase)
74            .any(|text| says_overflow(&text) && !says_rate_limit(&text))
75    }
76}
77
78/// The ways providers say the context is full, lower case.
79// The forms pi knows: https://github.com/badlogic/pi-mono, `packages/ai/src/utils/overflow.ts`.
80const OVERFLOW: &[&str] = &[
81    "prompt is too long",
82    "prompt too long",
83    "prompt exceeds max length",
84    "request_too_large",
85    "input is too long for requested model",
86    "exceeds the context window",
87    "maximum context length",
88    "maximum prompt length is",
89    "reduce the length of the messages",
90    "maximum allowed input length",
91    "than the model's context length",
92    "than the models context length",
93    "greater than the context length",
94    "exceeds the limit of ",
95    "exceeds the available context size",
96    "context window exceeds limit",
97    "exceeded model token limit",
98    "but the configured context size is",
99    "model_context_window_exceeded",
100    "exceeded context length",
101    "exceeded max context length",
102    "range of input length should be",
103    CONTEXT_OVERFLOW,
104    "context length exceeded",
105    "too many tokens",
106    "token limit exceeded",
107];
108
109/// Whether lower-case `text` says the context is full.
110fn says_overflow(text: &str) -> bool {
111    OVERFLOW.iter().any(|phrase| text.contains(phrase))
112        || (text.contains("input token count") && text.contains("exceeds the maximum"))
113}
114
115/// Whether lower-case `text` is a rate limit or an outage whose words also
116/// match [`OVERFLOW`].
117fn says_rate_limit(text: &str) -> bool {
118    text.starts_with("throttling error:")
119        || text.starts_with("service unavailable:")
120        || text.contains("rate limit")
121        || text.contains("too many requests")
122}
123
124/// `secs` as a duration, when it is a sane one.
125fn seconds(secs: f64) -> Option<Duration> {
126    Duration::try_from_secs_f64(secs.max(0.0)).ok()
127}
128
129/// The wait `text` starts with: a number and `ms` or `s`.
130fn hinted_wait(text: &str) -> Option<Duration> {
131    let end = text
132        .find(|c: char| !(c.is_ascii_digit() || c == '.'))
133        .unwrap_or(text.len());
134    let amount: f64 = text.get(..end)?.parse().ok()?;
135    let unit = text.get(end..)?.trim_start();
136    let unit_end = unit
137        .find(|c: char| !c.is_ascii_alphanumeric())
138        .unwrap_or(unit.len());
139    match unit.get(..unit_end)? {
140        "ms" => seconds(amount / 1000.0),
141        "s" => seconds(amount),
142        _ => None,
143    }
144}
145
146#[cfg(test)]
147mod tests;
rig_core/error/retry.rs

rig_core/error/
retry.rs
