Skip to main content

mutil_ai/
retry.rs

1use std::future::Future;
2use std::time::{Duration, SystemTime, UNIX_EPOCH};
3
4use reqwest::header::HeaderMap;
5use time::OffsetDateTime;
6use time::format_description::well_known::Rfc3339;
7
8use crate::error::{Error, Result};
9
10/// Parse the HTTP `Retry-After` header.
11///
12/// The header has two valid forms:
13///
14/// ```text
15/// Retry-After: 120
16/// Retry-After: Wed, 21 Oct 2026 07:28:00 GMT
17/// ```
18///
19/// The first form means "wait 120 seconds". The second means "wait until this
20/// date". If the date is already in the past, the returned duration is zero.
21pub fn parse_retry_after(value: &str) -> Option<Duration> {
22    parse_retry_after_at(value, SystemTime::now())
23}
24
25/// Deterministic version of [`parse_retry_after`] for tests and custom clocks.
26pub fn parse_retry_after_at(value: &str, now: SystemTime) -> Option<Duration> {
27    let value = value.trim();
28    if value.is_empty() {
29        return None;
30    }
31
32    if let Ok(seconds) = value.parse::<u64>() {
33        return Some(Duration::from_secs(seconds));
34    }
35
36    let retry_at = httpdate::parse_http_date(value).ok()?;
37    Some(
38        retry_at
39            .duration_since(now)
40            .unwrap_or(Duration::from_secs(0)),
41    )
42}
43
44/// Provider family used to decide which vendor-specific reset headers to read.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum RetryProvider {
47    /// Read only standard headers (`retry-after-ms`, `retry-after`).
48    Generic,
49    /// Additionally read OpenAI `x-ratelimit-reset-*` headers.
50    OpenAi,
51    /// Additionally read Anthropic `anthropic-ratelimit-*-reset` headers.
52    Anthropic,
53    /// Google-style provider; no vendor-specific headers are read yet.
54    Google,
55}
56
57/// Where a retry delay came from. This is useful for logs and debugging.
58#[derive(Debug, Clone, Copy, PartialEq, Eq)]
59pub enum RetrySource {
60    /// Non-standard `retry-after-ms` header, in milliseconds.
61    RetryAfterMs,
62    /// Standard `retry-after` header, in (possibly fractional) seconds.
63    RetryAfterSeconds,
64    /// Standard `retry-after` header, as an HTTP-date.
65    RetryAfterDate,
66    /// OpenAI `x-ratelimit-reset-requests`, in Go duration format.
67    OpenAiResetRequests,
68    /// OpenAI `x-ratelimit-reset-tokens`, in Go duration format.
69    OpenAiResetTokens,
70    /// Anthropic `anthropic-ratelimit-requests-reset`, as an RFC 3339 date.
71    AnthropicResetRequests,
72    /// Anthropic `anthropic-ratelimit-tokens-reset`, as an RFC 3339 date.
73    AnthropicResetTokens,
74    /// Anthropic `anthropic-ratelimit-input-tokens-reset`, as an RFC 3339 date.
75    AnthropicResetInputTokens,
76    /// Anthropic `anthropic-ratelimit-output-tokens-reset`, as an RFC 3339 date.
77    AnthropicResetOutputTokens,
78}
79
80/// A parsed server-directed retry instruction.
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub struct RetryDirective {
83    /// How long to wait before the next attempt. A past reset date yields
84    /// [`Duration::ZERO`].
85    pub delay: Duration,
86    /// Which header the delay was parsed from.
87    pub source: RetrySource,
88}
89
90const RETRY_AFTER_MS: &str = "retry-after-ms";
91const RETRY_AFTER: &str = "retry-after";
92const OPENAI_RESET_REQUESTS: &str = "x-ratelimit-reset-requests";
93const OPENAI_RESET_TOKENS: &str = "x-ratelimit-reset-tokens";
94const ANTHROPIC_RESET_REQUESTS: &str = "anthropic-ratelimit-requests-reset";
95const ANTHROPIC_RESET_TOKENS: &str = "anthropic-ratelimit-tokens-reset";
96const ANTHROPIC_RESET_INPUT_TOKENS: &str = "anthropic-ratelimit-input-tokens-reset";
97const ANTHROPIC_RESET_OUTPUT_TOKENS: &str = "anthropic-ratelimit-output-tokens-reset";
98
99/// Read all supported retry headers using the current time.
100///
101/// Vendor-specific headers are only consulted for the given [`RetryProvider`].
102/// Precedence: `retry-after-ms`, then `retry-after`, then the provider's
103/// reset headers.
104pub fn parse_retry_headers(provider: RetryProvider, headers: &HeaderMap) -> Option<RetryDirective> {
105    parse_retry_headers_at(provider, headers, SystemTime::now())
106}
107
108/// Deterministic version of [`parse_retry_headers`].
109pub fn parse_retry_headers_at(
110    provider: RetryProvider,
111    headers: &HeaderMap,
112    now: SystemTime,
113) -> Option<RetryDirective> {
114    // `retry-after-ms` is non-standard, but is supported by the official
115    // OpenAI, Anthropic and Google SDKs and is more precise.
116    if let Some(value) = header_value(headers, RETRY_AFTER_MS)
117        && let Some(delay) = parse_millis(value)
118    {
119        return Some(RetryDirective {
120            delay,
121            source: RetrySource::RetryAfterMs,
122        });
123    }
124
125    // Standard HTTP Retry-After: integer/float seconds or HTTP-date.
126    if let Some(value) = header_value(headers, RETRY_AFTER) {
127        if let Some(delay) = parse_seconds(value) {
128            return Some(RetryDirective {
129                delay,
130                source: RetrySource::RetryAfterSeconds,
131            });
132        }
133        if let Some(delay) = parse_retry_after_at(value, now) {
134            return Some(RetryDirective {
135                delay,
136                source: RetrySource::RetryAfterDate,
137            });
138        }
139    }
140
141    match provider {
142        RetryProvider::OpenAi => {
143            if let Some(value) = header_value(headers, OPENAI_RESET_REQUESTS)
144                && let Some(delay) = parse_go_duration(value)
145            {
146                return Some(RetryDirective {
147                    delay,
148                    source: RetrySource::OpenAiResetRequests,
149                });
150            }
151            if let Some(value) = header_value(headers, OPENAI_RESET_TOKENS)
152                && let Some(delay) = parse_go_duration(value)
153            {
154                return Some(RetryDirective {
155                    delay,
156                    source: RetrySource::OpenAiResetTokens,
157                });
158            }
159        }
160        RetryProvider::Anthropic => {
161            let headers_and_sources = [
162                (
163                    ANTHROPIC_RESET_REQUESTS,
164                    RetrySource::AnthropicResetRequests,
165                ),
166                (ANTHROPIC_RESET_TOKENS, RetrySource::AnthropicResetTokens),
167                (
168                    ANTHROPIC_RESET_INPUT_TOKENS,
169                    RetrySource::AnthropicResetInputTokens,
170                ),
171                (
172                    ANTHROPIC_RESET_OUTPUT_TOKENS,
173                    RetrySource::AnthropicResetOutputTokens,
174                ),
175            ];
176            for (header, source) in headers_and_sources {
177                if let Some(value) = header_value(headers, header)
178                    && let Some(delay) = parse_rfc3339_until(value, now)
179                {
180                    return Some(RetryDirective { delay, source });
181                }
182            }
183        }
184        RetryProvider::Google | RetryProvider::Generic => {}
185    }
186
187    None
188}
189
190fn header_value<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
191    headers.get(name)?.to_str().ok()
192}
193
194fn parse_millis(value: &str) -> Option<Duration> {
195    let milliseconds = value.trim().parse::<f64>().ok()?;
196    if !milliseconds.is_finite() || milliseconds < 0.0 {
197        return None;
198    }
199    Some(Duration::from_secs_f64(milliseconds / 1000.0))
200}
201
202fn parse_seconds(value: &str) -> Option<Duration> {
203    let seconds = value.trim().parse::<f64>().ok()?;
204    if !seconds.is_finite() || seconds < 0.0 {
205        return None;
206    }
207    Some(Duration::from_secs_f64(seconds))
208}
209
210/// Parse OpenAI's reset headers such as `1s`, `6m0s`, `1h`, `500ms`.
211///
212/// These headers are informational rate-limit headers; they are used only as
213/// a fallback when neither `retry-after-ms` nor `retry-after` is present.
214fn parse_go_duration(value: &str) -> Option<Duration> {
215    let mut rest = value.trim();
216    if rest.is_empty() {
217        return None;
218    }
219
220    let mut total = 0.0_f64;
221    let mut saw_value = false;
222
223    while !rest.is_empty() {
224        let number_len = rest
225            .find(|character: char| !character.is_ascii_digit() && character != '.')
226            .unwrap_or(rest.len());
227        if number_len == 0 {
228            return None;
229        }
230
231        let amount = rest[..number_len].parse::<f64>().ok()?;
232        rest = &rest[number_len..];
233        saw_value = true;
234
235        if rest.is_empty() {
236            // A bare number in a reset header is treated as seconds.
237            total += amount;
238            break;
239        }
240
241        let (unit, seconds) = if rest.starts_with("ns") {
242            ("ns", amount / 1_000_000_000.0)
243        } else if rest.starts_with("us") {
244            ("us", amount / 1_000_000.0)
245        } else if rest.starts_with("µs") {
246            ("µs", amount / 1_000_000.0)
247        } else if rest.starts_with("ms") {
248            ("ms", amount / 1_000.0)
249        } else if rest.starts_with('s') {
250            ("s", amount)
251        } else if rest.starts_with('m') {
252            ("m", amount * 60.0)
253        } else if rest.starts_with('h') {
254            ("h", amount * 3600.0)
255        } else {
256            return None;
257        };
258
259        total += seconds;
260        rest = &rest[unit.len()..];
261    }
262
263    if !saw_value || !total.is_finite() || total < 0.0 {
264        return None;
265    }
266    Some(Duration::from_secs_f64(total))
267}
268
269fn parse_rfc3339_until(value: &str, now: SystemTime) -> Option<Duration> {
270    let retry_at = OffsetDateTime::parse(value.trim(), &Rfc3339).ok()?;
271    let retry_nanos = retry_at.unix_timestamp_nanos();
272    let now_nanos = now.duration_since(UNIX_EPOCH).ok()?.as_nanos() as i128;
273    let delta = retry_nanos - now_nanos;
274
275    if delta <= 0 {
276        return Some(Duration::ZERO);
277    }
278
279    u64::try_from(delta)
280        .ok()
281        .map(Duration::from_nanos)
282        .or(Some(Duration::MAX))
283}
284
285/// Controls how transient errors are retried.
286#[derive(Debug, Clone, PartialEq)]
287pub struct RetryPolicy {
288    /// Total attempts, including the first request. `4` means 1 initial try
289    /// plus up to 3 retries.
290    pub max_attempts: u32,
291    /// Base delay for exponential backoff.
292    pub base_delay: Duration,
293    /// Maximum delay for computed backoff. A server-provided `Retry-After`
294    /// is handled separately by `max_retry_after`.
295    pub max_delay: Duration,
296    /// Fractional jitter around the computed delay. `0.2` means ±20%.
297    pub jitter_ratio: f32,
298    /// If `Retry-After` is larger than this, stop retrying instead of waiting
299    /// an unexpectedly long time.
300    pub max_retry_after: Duration,
301    /// When true, a request without an idempotency key is sent once but is not
302    /// automatically replayed.
303    pub require_idempotency_key: bool,
304}
305
306impl Default for RetryPolicy {
307    fn default() -> Self {
308        Self {
309            max_attempts: 4,
310            base_delay: Duration::from_millis(500),
311            max_delay: Duration::from_secs(8),
312            jitter_ratio: 0.2,
313            max_retry_after: Duration::from_secs(60),
314            require_idempotency_key: true,
315        }
316    }
317}
318
319impl RetryPolicy {
320    /// Creates a policy with the given attempt count and default timings.
321    pub fn new(max_attempts: u32) -> Self {
322        Self {
323            max_attempts,
324            ..Self::default()
325        }
326    }
327
328    /// Sets the total attempt count, including the first request.
329    pub fn max_attempts(mut self, max_attempts: u32) -> Self {
330        self.max_attempts = max_attempts;
331        self
332    }
333
334    /// Sets the base delay for exponential backoff.
335    pub fn base_delay(mut self, base_delay: Duration) -> Self {
336        self.base_delay = base_delay;
337        self
338    }
339
340    /// Sets the upper bound for computed backoff delays.
341    pub fn max_delay(mut self, max_delay: Duration) -> Self {
342        self.max_delay = max_delay;
343        self
344    }
345
346    /// Sets the fractional jitter around the computed delay.
347    pub fn jitter_ratio(mut self, jitter_ratio: f32) -> Self {
348        self.jitter_ratio = jitter_ratio;
349        self
350    }
351
352    /// Sets the largest accepted server-directed `Retry-After`.
353    pub fn max_retry_after(mut self, max_retry_after: Duration) -> Self {
354        self.max_retry_after = max_retry_after;
355        self
356    }
357
358    /// Toggles whether replaying requires an idempotency key.
359    pub fn require_idempotency_key(mut self, require_idempotency_key: bool) -> Self {
360        self.require_idempotency_key = require_idempotency_key;
361        self
362    }
363
364    /// Compute the wait before the next attempt.
365    ///
366    /// `retry_number` is zero-based: the first failure uses `0`.
367    /// `None` means "do not retry".
368    pub fn delay_for(&self, retry_number: u32, retry_after: Option<Duration>) -> Option<Duration> {
369        if let Some(retry_after) = retry_after {
370            if retry_after > self.max_retry_after {
371                return None;
372            }
373            return Some(retry_after);
374        }
375
376        let exponent = retry_number.min(20);
377        let multiplier = 2_u32.pow(exponent);
378        let base = self
379            .base_delay
380            .saturating_mul(multiplier)
381            .min(self.max_delay);
382        let jitter = self.jitter_ratio.clamp(0.0, 1.0);
383
384        if jitter == 0.0 || base.is_zero() {
385            return Some(base);
386        }
387
388        let unit = pseudo_random_unit(retry_number);
389        let factor = (1.0 - jitter) + (2.0 * jitter * unit);
390        Some(base.mul_f32(factor).min(self.max_delay))
391    }
392}
393
394/// Retry an async operation according to a [`RetryPolicy`].
395///
396/// The operation closure is called again for every attempt, so each request
397/// body and HTTP request can be rebuilt cleanly. Only retryable errors are
398/// replayed, and a server-directed delay larger than
399/// [`RetryPolicy::max_retry_after`] stops retrying immediately.
400///
401/// # Errors
402///
403/// Returns the operation's error once it succeeds, the error is not
404/// retryable, the allowed attempts are exhausted, or the delay would exceed
405/// the policy limit. Returns [`Error::InvalidRetryPolicy`] if
406/// `policy.max_attempts` is zero.
407pub async fn retry_async<F, Fut, T>(policy: &RetryPolicy, mut operation: F) -> Result<T>
408where
409    F: FnMut() -> Fut,
410    Fut: Future<Output = Result<T>>,
411{
412    if policy.max_attempts == 0 {
413        return Err(Error::InvalidRetryPolicy(
414            "max_attempts must be at least 1".to_string(),
415        ));
416    }
417
418    for attempt in 0..policy.max_attempts {
419        match operation().await {
420            Ok(value) => return Ok(value),
421            Err(error) => {
422                let Some(delay) = next_delay(policy, attempt, &error) else {
423                    return Err(error);
424                };
425                tokio::time::sleep(delay).await;
426            }
427        }
428    }
429
430    unreachable!("the loop returns on success or on the final error")
431}
432
433pub(crate) fn next_delay(policy: &RetryPolicy, attempt: u32, error: &Error) -> Option<Duration> {
434    let has_attempt_left = attempt + 1 < policy.max_attempts;
435    if !has_attempt_left || !error.is_retryable() {
436        return None;
437    }
438    policy.delay_for(attempt, error.retry_after())
439}
440
441fn pseudo_random_unit(seed: u32) -> f32 {
442    let time_seed = SystemTime::now()
443        .duration_since(UNIX_EPOCH)
444        .map(|duration| duration.subsec_nanos())
445        .unwrap_or(0);
446    let mut value = seed ^ time_seed;
447    value ^= value << 13;
448    value ^= value >> 17;
449    value ^= value << 5;
450    value as f32 / u32::MAX as f32
451}
452
453#[cfg(test)]
454mod tests {
455    use std::sync::Arc;
456    use std::sync::atomic::{AtomicUsize, Ordering};
457
458    use reqwest::header::HeaderValue;
459
460    use super::*;
461
462    fn api_error(status: u16, retry_after: Option<Duration>) -> Error {
463        Error::Api {
464            provider: "test",
465            status,
466            body: "test error".to_string(),
467            retry_after,
468            retry_source: None,
469            request_id: None,
470        }
471    }
472
473    #[test]
474    fn parses_delta_seconds() {
475        assert_eq!(parse_retry_after("120"), Some(Duration::from_secs(120)));
476        assert_eq!(parse_retry_after(" 5 "), Some(Duration::from_secs(5)));
477    }
478
479    #[test]
480    fn parses_http_date() {
481        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
482        let retry_at = now + Duration::from_secs(30);
483        let value = httpdate::fmt_http_date(retry_at);
484
485        assert_eq!(
486            parse_retry_after_at(&value, now),
487            Some(Duration::from_secs(30))
488        );
489    }
490
491    #[test]
492    fn past_http_date_becomes_zero() {
493        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
494        let retry_at = now - Duration::from_secs(30);
495        let value = httpdate::fmt_http_date(retry_at);
496
497        assert_eq!(parse_retry_after_at(&value, now), Some(Duration::ZERO));
498    }
499
500    #[test]
501    fn rejects_garbage() {
502        assert_eq!(parse_retry_after("not-a-date"), None);
503        assert_eq!(parse_retry_after(""), None);
504    }
505
506    #[test]
507    fn honors_server_retry_after() {
508        let policy = RetryPolicy::default();
509        assert_eq!(
510            policy.delay_for(0, Some(Duration::from_secs(3))),
511            Some(Duration::from_secs(3))
512        );
513    }
514
515    #[test]
516    fn refuses_too_long_retry_after() {
517        let policy = RetryPolicy::default().max_retry_after(Duration::from_secs(5));
518        assert_eq!(policy.delay_for(0, Some(Duration::from_secs(6))), None);
519    }
520
521    #[test]
522    fn prefers_retry_after_ms() {
523        let mut headers = HeaderMap::new();
524        headers.insert(RETRY_AFTER_MS, HeaderValue::from_static("1500"));
525        headers.insert(RETRY_AFTER, HeaderValue::from_static("10"));
526
527        let directive = parse_retry_headers(RetryProvider::Generic, &headers).unwrap();
528        assert_eq!(directive.delay, Duration::from_millis(1500));
529        assert_eq!(directive.source, RetrySource::RetryAfterMs);
530    }
531
532    #[test]
533    fn parses_openai_reset_duration_headers() {
534        let mut headers = HeaderMap::new();
535        headers.insert(OPENAI_RESET_REQUESTS, HeaderValue::from_static("6m0s"));
536        headers.insert(OPENAI_RESET_TOKENS, HeaderValue::from_static("1h"));
537
538        let directive = parse_retry_headers(RetryProvider::OpenAi, &headers).unwrap();
539        assert_eq!(directive.delay, Duration::from_secs(360));
540        assert_eq!(directive.source, RetrySource::OpenAiResetRequests);
541    }
542
543    #[test]
544    fn parses_anthropic_rfc3339_reset_headers() {
545        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
546        let mut headers = HeaderMap::new();
547        // 1970-01-01T00:16:40Z is exactly `now`; the reset is 45 seconds later.
548        headers.insert(
549            ANTHROPIC_RESET_REQUESTS,
550            HeaderValue::from_static("1970-01-01T00:17:25Z"),
551        );
552
553        let directive = parse_retry_headers_at(RetryProvider::Anthropic, &headers, now).unwrap();
554        assert_eq!(directive.delay, Duration::from_secs(45));
555        assert_eq!(directive.source, RetrySource::AnthropicResetRequests);
556    }
557
558    #[test]
559    fn computes_exponential_backoff_without_jitter() {
560        let policy = RetryPolicy::new(4)
561            .base_delay(Duration::from_secs(1))
562            .max_delay(Duration::from_secs(10))
563            .jitter_ratio(0.0);
564
565        assert_eq!(policy.delay_for(0, None), Some(Duration::from_secs(1)));
566        assert_eq!(policy.delay_for(1, None), Some(Duration::from_secs(2)));
567        assert_eq!(policy.delay_for(2, None), Some(Duration::from_secs(4)));
568        assert_eq!(policy.delay_for(3, None), Some(Duration::from_secs(8)));
569        assert_eq!(policy.delay_for(4, None), Some(Duration::from_secs(10)));
570    }
571
572    #[tokio::test]
573    async fn retries_until_success() {
574        let policy = RetryPolicy::new(3)
575            .base_delay(Duration::ZERO)
576            .max_delay(Duration::ZERO)
577            .jitter_ratio(0.0);
578        let attempts = Arc::new(AtomicUsize::new(0));
579
580        let result = retry_async(&policy, || {
581            let attempt = attempts.fetch_add(1, Ordering::SeqCst);
582            async move {
583                if attempt < 2 {
584                    Err(api_error(429, None))
585                } else {
586                    Ok(attempt)
587                }
588            }
589        })
590        .await
591        .unwrap();
592
593        assert_eq!(result, 2);
594        assert_eq!(attempts.load(Ordering::SeqCst), 3);
595    }
596
597    #[tokio::test]
598    async fn stops_on_non_retryable_error() {
599        let policy = RetryPolicy::new(3)
600            .base_delay(Duration::ZERO)
601            .jitter_ratio(0.0);
602        let attempts = Arc::new(AtomicUsize::new(0));
603
604        let result: Result<()> = retry_async(&policy, || {
605            attempts.fetch_add(1, Ordering::SeqCst);
606            async { Err(api_error(401, None)) }
607        })
608        .await;
609
610        assert!(result.is_err());
611        assert_eq!(attempts.load(Ordering::SeqCst), 1);
612    }
613
614    #[tokio::test]
615    async fn stops_when_retry_after_is_too_long() {
616        let policy = RetryPolicy::new(5)
617            .base_delay(Duration::ZERO)
618            .max_retry_after(Duration::from_secs(5));
619        let attempts = Arc::new(AtomicUsize::new(0));
620
621        let result: Result<()> = retry_async(&policy, || {
622            attempts.fetch_add(1, Ordering::SeqCst);
623            async { Err(api_error(429, Some(Duration::from_secs(60)))) }
624        })
625        .await;
626
627        assert!(result.is_err());
628        assert_eq!(attempts.load(Ordering::SeqCst), 1);
629    }
630}