Skip to main content

rig_core/error/
retry.rs

1//! What a failed call's report says about retrying it, for a retry policy:
2//! the wait asked for, and whether the request outgrew the model's context
3//! window.
4
5use std::time::{Duration, SystemTime};
6
7use super::ErrorReport;
8
9/// The code of a report that the request outgrew the context window.
10const CONTEXT_OVERFLOW: &str = "context_length_exceeded";
11
12impl ErrorReport {
13    /// The wait the provider asked for, as of `now`: the `retry-after-ms`
14    /// header (OpenAI), else `retry-after` in seconds or as an HTTP date,
15    /// else a hint in the body: Gemini's `"retryDelay": "2s"` or OpenAI's
16    /// "try again in 1.5s" or "try again in 6m0s".
17    pub fn retry_after(&self, now: SystemTime) -> Option<Duration> {
18        let header = |name: &str| {
19            self.provider_response_headers()
20                .and_then(|headers| headers.get(name))
21                .and_then(|value| value.to_str().ok())
22                .map(str::trim)
23        };
24        if let Some(millis) = header("retry-after-ms").and_then(|value| value.parse::<f64>().ok()) {
25            return seconds(millis / 1000.0);
26        }
27        if let Some(value) = header("retry-after") {
28            return match value.parse::<f64>() {
29                Ok(secs) => seconds(secs),
30                Err(_) => httpdate::parse_http_date(value)
31                    .ok()
32                    .map(|at| at.duration_since(now).unwrap_or_default()),
33            };
34        }
35        let body = self
36            .provider_response_body()
37            .unwrap_or(&self.message)
38            .to_ascii_lowercase();
39        ["\"retrydelay\"", "try again in "]
40            .iter()
41            .find_map(|marker| {
42                let after = body.get(body.find(marker)? + marker.len()..)?;
43                hinted_wait(after.trim_start_matches([' ', ':', '"']))
44            })
45    }
46
47    /// Whether this report says the request does not fit the model's
48    /// context window: the `context_length_exceeded` code, a 413, or the
49    /// provider saying so in its own words, less rate limits that happen to
50    /// mention tokens. A retryable failure is never an overflow, which fails
51    /// the same way every time (Bedrock's "too many tokens" throttling is
52    /// retryable).
53    pub fn is_context_overflow(&self) -> bool {
54        if self.code.as_deref() == Some(CONTEXT_OVERFLOW) {
55            return true;
56        }
57        if self.is_retryable() {
58            return false;
59        }
60        if self.http_status == Some(413) {
61            return true;
62        }
63        [Some(self.message.as_str()), self.provider_response_body()]
64            .into_iter()
65            .flatten()
66            .map(str::to_ascii_lowercase)
67            .any(|text| says_overflow(&text) && !says_rate_limit(&text))
68    }
69}
70
71/// The ways providers say the context is full, lower case.
72// The forms pi knows: https://github.com/badlogic/pi-mono, `packages/ai/src/utils/overflow.ts`.
73const OVERFLOW: &[&str] = &[
74    "prompt is too long",
75    "prompt too long",
76    "prompt exceeds max length",
77    "request_too_large",
78    "input is too long for requested model",
79    "exceeds the context window",
80    "maximum context length",
81    "maximum prompt length is",
82    "reduce the length of the messages",
83    "maximum allowed input length",
84    "than the model's context length",
85    "than the models context length",
86    "greater than the context length",
87    "exceeds the limit of ",
88    "exceeds the available context size",
89    "context window exceeds limit",
90    "exceeded model token limit",
91    "but the configured context size is",
92    "model_context_window_exceeded",
93    "exceeded context length",
94    "exceeded max context length",
95    "range of input length should be",
96    CONTEXT_OVERFLOW,
97    "context length exceeded",
98    "too many tokens",
99    "token limit exceeded",
100];
101
102/// Whether lower-case `text` says the context is full.
103fn says_overflow(text: &str) -> bool {
104    OVERFLOW.iter().any(|phrase| text.contains(phrase))
105        || (text.contains("input token count") && text.contains("exceeds the maximum"))
106}
107
108/// Whether lower-case `text` is a rate limit or an outage whose words also
109/// match [`OVERFLOW`].
110fn says_rate_limit(text: &str) -> bool {
111    text.starts_with("throttling error:")
112        || text.starts_with("service unavailable:")
113        || text.contains("rate limit")
114        || text.contains("too many requests")
115}
116
117/// `secs` as a duration, when it is a sane one.
118pub(super) fn seconds(secs: f64) -> Option<Duration> {
119    Duration::try_from_secs_f64(secs.max(0.0)).ok()
120}
121
122/// The wait `text` starts with, as Go writes a duration: numbers each
123/// followed by `h`, `m`, `s`, `ms`, `us`, `µs` or `ns` (`6m0s`, `1h2m3.5s`,
124/// `20ms`, `.5s`). A space may part the first number from its unit
125/// (`1.5 s`).
126pub(super) fn hinted_wait(text: &str) -> Option<Duration> {
127    let mut rest = text;
128    let mut total = None;
129    loop {
130        let digits = rest.strip_prefix('.').unwrap_or(rest);
131        if !digits.starts_with(|c: char| c.is_ascii_digit()) {
132            return seconds(total?);
133        }
134        let end = rest
135            .find(|c: char| !(c.is_ascii_digit() || c == '.'))
136            .unwrap_or(rest.len());
137        let amount: f64 = rest.get(..end)?.parse().ok()?;
138        let unit = rest.get(end..)?;
139        let unit = if total.is_none() {
140            unit.trim_start()
141        } else {
142            unit
143        };
144        // Go writes the micro sign; the Greek mu looks the same.
145        let (micro, unit) = match unit.strip_prefix(['\u{b5}', '\u{3bc}']) {
146            Some(unit) => (true, unit),
147            None => (false, unit),
148        };
149        let unit_end = unit
150            .find(|c: char| !c.is_ascii_alphabetic())
151            .unwrap_or(unit.len());
152        let scale = match (micro, unit.get(..unit_end)?) {
153            (false, "h") => 3600.0,
154            (false, "m") => 60.0,
155            (false, "s") => 1.0,
156            (false, "ms") => 0.001,
157            (false, "us") | (true, "s") => 1e-6,
158            (false, "ns") => 1e-9,
159            _ => return None,
160        };
161        total = Some(total.unwrap_or(0.0) + amount * scale);
162        rest = unit.get(unit_end..)?;
163    }
164}
165
166#[cfg(test)]
167mod tests;