use crate::error::CirrusError;
use std::time::Duration;
#[derive(Debug, Clone)]
pub struct RetryPolicy {
pub max_retries: u32,
pub base_delay: Duration,
pub max_delay: Duration,
pub jitter: bool,
pub retry_idempotent_5xx: bool,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_retries: 3,
base_delay: Duration::from_millis(100),
max_delay: Duration::from_secs(30),
jitter: true,
retry_idempotent_5xx: true,
}
}
}
impl RetryPolicy {
pub fn none() -> Self {
Self {
max_retries: 0,
..Self::default()
}
}
}
pub(crate) fn should_retry_status(
policy: &RetryPolicy,
method: &reqwest::Method,
status: u16,
attempt: u32,
) -> bool {
if attempt >= policy.max_retries {
return false;
}
match status {
429 | 503 => true,
500 | 502 | 504 if policy.retry_idempotent_5xx => is_idempotent(method),
_ => false,
}
}
pub(crate) fn should_retry_network(
policy: &RetryPolicy,
method: &reqwest::Method,
error: &CirrusError,
attempt: u32,
) -> bool {
if attempt >= policy.max_retries {
return false;
}
if !is_idempotent(method) {
return false;
}
matches!(error, CirrusError::Http(_))
}
fn is_idempotent(method: &reqwest::Method) -> bool {
matches!(
*method,
reqwest::Method::GET
| reqwest::Method::HEAD
| reqwest::Method::DELETE
| reqwest::Method::PUT
| reqwest::Method::OPTIONS
| reqwest::Method::TRACE
)
}
pub(crate) fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
let raw = headers.get(reqwest::header::RETRY_AFTER)?;
let s = raw.to_str().ok()?;
s.trim().parse::<u64>().ok().map(Duration::from_secs)
}
pub(crate) fn compute_delay(
policy: &RetryPolicy,
attempt: u32,
retry_after: Option<Duration>,
) -> Duration {
if let Some(hint) = retry_after {
let capped = hint.min(policy.max_delay);
tracing::warn!(
target: "cirrus::retry",
attempt = attempt + 1,
delay_ms = capped.as_millis() as u64,
source = "retry-after-header",
"scheduling request retry",
);
return capped;
}
let factor: u128 = 1u128.checked_shl(attempt).unwrap_or(u128::MAX);
let computed_ms = policy.base_delay.as_millis().saturating_mul(factor);
let max_ms = policy.max_delay.as_millis();
let capped_ms = computed_ms.min(max_ms);
let computed = Duration::from_millis(capped_ms.min(u64::MAX as u128) as u64);
let final_delay = if !policy.jitter {
computed
} else {
let max_ms = computed.as_millis() as u64;
if max_ms == 0 {
Duration::ZERO
} else {
let mut buf = [0u8; 8];
if getrandom::fill(&mut buf).is_err() {
computed
} else {
let r = u64::from_le_bytes(buf) % (max_ms + 1);
Duration::from_millis(r)
}
}
};
tracing::warn!(
target: "cirrus::retry",
attempt = attempt + 1,
delay_ms = final_delay.as_millis() as u64,
source = "exponential-backoff",
"scheduling request retry",
);
final_delay
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn default_policy_retries_three_times() {
let p = RetryPolicy::default();
assert_eq!(p.max_retries, 3);
assert!(p.jitter);
assert!(p.retry_idempotent_5xx);
}
#[test]
fn none_policy_disables_retry() {
let p = RetryPolicy::none();
assert!(!should_retry_status(&p, &reqwest::Method::GET, 429, 0));
assert!(!should_retry_status(&p, &reqwest::Method::GET, 503, 0));
}
#[test]
fn retries_429_and_503_for_any_method() {
let p = RetryPolicy::default();
for m in [
reqwest::Method::GET,
reqwest::Method::POST,
reqwest::Method::PATCH,
reqwest::Method::DELETE,
] {
assert!(should_retry_status(&p, &m, 429, 0), "429 retry for {m}");
assert!(should_retry_status(&p, &m, 503, 0), "503 retry for {m}");
}
}
#[test]
fn retries_5xx_only_for_idempotent_methods() {
let p = RetryPolicy::default();
for status in [500, 502, 504] {
assert!(should_retry_status(&p, &reqwest::Method::GET, status, 0));
assert!(should_retry_status(&p, &reqwest::Method::DELETE, status, 0));
assert!(should_retry_status(&p, &reqwest::Method::PUT, status, 0));
assert!(!should_retry_status(&p, &reqwest::Method::POST, status, 0));
assert!(!should_retry_status(&p, &reqwest::Method::PATCH, status, 0));
}
}
#[test]
fn does_not_retry_4xx_caller_errors() {
let p = RetryPolicy::default();
for status in [400, 401, 403, 404, 405, 422] {
assert!(
!should_retry_status(&p, &reqwest::Method::GET, status, 0),
"should not retry {status}"
);
}
}
#[test]
fn stops_retrying_at_max_retries() {
let p = RetryPolicy::default();
assert!(should_retry_status(&p, &reqwest::Method::GET, 429, 0));
assert!(should_retry_status(&p, &reqwest::Method::GET, 429, 2));
assert!(!should_retry_status(&p, &reqwest::Method::GET, 429, 3));
assert!(!should_retry_status(&p, &reqwest::Method::GET, 429, 99));
}
#[test]
fn retry_5xx_disabled_skips_other_5xx_but_keeps_429_503() {
let p = RetryPolicy {
retry_idempotent_5xx: false,
..RetryPolicy::default()
};
assert!(should_retry_status(&p, &reqwest::Method::GET, 429, 0));
assert!(should_retry_status(&p, &reqwest::Method::GET, 503, 0));
assert!(!should_retry_status(&p, &reqwest::Method::GET, 500, 0));
assert!(!should_retry_status(&p, &reqwest::Method::GET, 502, 0));
}
#[test]
fn parse_retry_after_handles_seconds() {
let mut h = reqwest::header::HeaderMap::new();
h.insert(
reqwest::header::RETRY_AFTER,
reqwest::header::HeaderValue::from_static("5"),
);
assert_eq!(parse_retry_after(&h), Some(Duration::from_secs(5)));
}
#[test]
fn parse_retry_after_returns_none_for_http_date_form() {
let mut h = reqwest::header::HeaderMap::new();
h.insert(
reqwest::header::RETRY_AFTER,
reqwest::header::HeaderValue::from_static("Wed, 21 Oct 2015 07:28:00 GMT"),
);
assert_eq!(parse_retry_after(&h), None);
}
#[test]
fn parse_retry_after_returns_none_when_absent() {
let h = reqwest::header::HeaderMap::new();
assert_eq!(parse_retry_after(&h), None);
}
#[test]
fn compute_delay_honors_retry_after_capped_at_max() {
let p = RetryPolicy {
max_delay: Duration::from_secs(10),
..RetryPolicy::default()
};
assert_eq!(
compute_delay(&p, 0, Some(Duration::from_secs(3))),
Duration::from_secs(3)
);
assert_eq!(
compute_delay(&p, 0, Some(Duration::from_secs(99))),
Duration::from_secs(10)
);
}
#[test]
fn compute_delay_caps_exponential_at_max_delay() {
let p = RetryPolicy {
base_delay: Duration::from_millis(100),
max_delay: Duration::from_secs(1),
jitter: false,
..RetryPolicy::default()
};
assert_eq!(compute_delay(&p, 0, None), Duration::from_millis(100));
assert_eq!(compute_delay(&p, 1, None), Duration::from_millis(200));
assert_eq!(compute_delay(&p, 2, None), Duration::from_millis(400));
assert_eq!(compute_delay(&p, 3, None), Duration::from_millis(800));
assert_eq!(compute_delay(&p, 4, None), Duration::from_secs(1));
assert_eq!(compute_delay(&p, 100, None), Duration::from_secs(1));
}
#[test]
fn compute_delay_jitter_stays_within_bounds() {
let p = RetryPolicy {
base_delay: Duration::from_millis(100),
max_delay: Duration::from_secs(60),
jitter: true,
..RetryPolicy::default()
};
for _ in 0..50 {
let d = compute_delay(&p, 2, None);
assert!(d <= Duration::from_millis(400));
}
}
#[test]
fn compute_delay_with_zero_base_returns_zero() {
let p = RetryPolicy {
base_delay: Duration::ZERO,
max_delay: Duration::ZERO,
jitter: true,
..RetryPolicy::default()
};
assert_eq!(compute_delay(&p, 0, None), Duration::ZERO);
assert_eq!(compute_delay(&p, 5, None), Duration::ZERO);
}
}