use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::time::{Duration, Instant, SystemTime};
use reqwest::StatusCode;
use reqwest::header::HeaderMap;
use crate::config::Config;
pub const DEFAULT_MAX_RETRIES: u32 = 3;
pub const DEFAULT_RETRY_BASE_MS: u64 = 200;
pub const DEFAULT_RETRY_MAX_DELAY_MS: u64 = 30_000;
pub const HARD_MAX_RETRIES: u32 = 10;
pub const HARD_MAX_DELAY_MS: u64 = 60_000;
pub const MIN_RETRY_BASE_MS: u64 = 50;
pub const HARD_MAX_ELAPSED_MS: u64 = 300_000;
pub const DEFAULT_RETRY_MAX_ELAPSED_MS: u64 = 30_000;
const RETRY_AFTER_SKEW_SECS: u64 = 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RetryConfig {
pub max_retries: u32,
pub base_ms: u64,
pub max_delay_ms: u64,
pub max_elapsed_ms: u64,
pub enabled: bool,
}
impl RetryConfig {
pub fn from_config(cfg: &Config) -> Self {
let max_retries = cfg.max_retries.min(HARD_MAX_RETRIES);
let base_ms = if cfg.retry_base_ms == 0 {
DEFAULT_RETRY_BASE_MS
} else {
cfg.retry_base_ms.max(MIN_RETRY_BASE_MS)
};
let max_delay_ms = cfg.retry_max_delay_ms.min(HARD_MAX_DELAY_MS).max(base_ms);
let derived = cfg.timeout_secs.saturating_mul(1000).max(1_000);
let max_elapsed_ms = if cfg.retry_max_elapsed_ms == 0 {
derived.min(HARD_MAX_ELAPSED_MS)
} else {
cfg.retry_max_elapsed_ms
.min(HARD_MAX_ELAPSED_MS)
.max(max_delay_ms)
};
let enabled = !cfg.disable_retry && max_retries > 0;
Self {
max_retries,
base_ms,
max_delay_ms,
max_elapsed_ms,
enabled,
}
}
pub fn max_attempts(self) -> u32 {
if !self.enabled {
1
} else {
self.max_retries.saturating_add(1)
}
}
pub fn may_retry(self, attempt: u32) -> bool {
self.enabled && attempt < self.max_attempts()
}
pub fn may_retry_within_budget(
self,
attempt: u32,
elapsed: Duration,
planned_wait: Duration,
) -> bool {
if !self.may_retry(attempt) {
return false;
}
let budget = Duration::from_millis(self.max_elapsed_ms);
if elapsed >= budget {
return false;
}
let remaining = budget.saturating_sub(elapsed);
planned_wait <= remaining
}
pub fn remaining_budget(self, elapsed: Duration) -> Duration {
Duration::from_millis(self.max_elapsed_ms).saturating_sub(elapsed)
}
pub fn delay_for_attempt(self, attempt: u32) -> Duration {
backoff_full_jitter(self.base_ms, attempt, self.max_delay_ms)
}
}
impl Default for RetryConfig {
fn default() -> Self {
Self {
max_retries: DEFAULT_MAX_RETRIES,
base_ms: DEFAULT_RETRY_BASE_MS,
max_delay_ms: DEFAULT_RETRY_MAX_DELAY_MS,
max_elapsed_ms: DEFAULT_RETRY_MAX_ELAPSED_MS,
enabled: true,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RetryClass {
Permanent,
Transient,
RateLimited {
retry_after: Option<Duration>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RetryKind {
Permanent,
TransientNetwork,
TransientServer,
RateLimited,
Timeout,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ErrorLayer {
Dns,
TcpConnect,
Tls,
HttpProtocol,
HttpStatus,
Application,
Unknown,
}
impl ErrorLayer {
pub fn as_str(self) -> &'static str {
match self {
Self::Dns => "dns",
Self::TcpConnect => "tcp_connect",
Self::Tls => "tls",
Self::HttpProtocol => "http_protocol",
Self::HttpStatus => "http_status",
Self::Application => "application",
Self::Unknown => "unknown",
}
}
}
impl RetryKind {
pub fn as_str(self) -> &'static str {
match self {
Self::Permanent => "permanent",
Self::TransientNetwork => "transient_network",
Self::TransientServer => "transient_server",
Self::RateLimited => "rate_limited",
Self::Timeout => "timeout",
}
}
pub fn is_retryable(self) -> bool {
!matches!(self, Self::Permanent)
}
}
pub fn classify_http_status(status: StatusCode, retry_after: Option<Duration>) -> RetryClass {
match status.as_u16() {
429 => RetryClass::RateLimited { retry_after },
408 | 500 | 502 | 504 => RetryClass::Transient,
503 => {
if retry_after.is_some() {
RetryClass::RateLimited { retry_after }
} else {
RetryClass::Transient
}
}
_ => RetryClass::Permanent,
}
}
pub(crate) fn source_chain_has_rustls(err: &(dyn std::error::Error + 'static)) -> bool {
let mut cur: Option<&(dyn std::error::Error + 'static)> = Some(err);
while let Some(e) = cur {
if e.downcast_ref::<rustls::Error>().is_some() {
return true;
}
cur = e.source();
}
false
}
pub fn classify_reqwest_error(err: &reqwest::Error) -> (crate::error::ErrorKind, ErrorLayer, bool) {
use crate::error::ErrorKind;
let tls = source_chain_has_rustls(err as &(dyn std::error::Error + 'static));
if err.is_timeout() {
let layer = if tls {
ErrorLayer::Tls
} else {
ErrorLayer::TcpConnect
};
return (ErrorKind::Timeout, layer, true);
}
if tls {
return (ErrorKind::Network, ErrorLayer::Tls, true);
}
if err.is_connect() {
return (ErrorKind::Network, ErrorLayer::TcpConnect, true);
}
if err.is_request() {
return (ErrorKind::Network, ErrorLayer::HttpProtocol, true);
}
if err.is_body() || err.is_decode() {
return (ErrorKind::Network, ErrorLayer::HttpProtocol, false);
}
(ErrorKind::Network, ErrorLayer::Unknown, false)
}
pub fn parse_retry_after(headers: &HeaderMap) -> Option<Duration> {
let raw = headers
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())?
.trim();
if raw.is_empty() {
return None;
}
if raw.bytes().all(|b| b.is_ascii_digit()) {
let secs: u64 = raw.parse().ok()?;
return Some(Duration::from_secs(secs.min(HARD_MAX_DELAY_MS / 1000)));
}
let when = httpdate::parse_http_date(raw).ok()?;
let now = SystemTime::now();
match when.duration_since(now) {
Ok(d) => Some(d.min(Duration::from_millis(HARD_MAX_DELAY_MS))),
Err(e) => {
if e.duration() <= Duration::from_secs(RETRY_AFTER_SKEW_SECS) {
Some(Duration::ZERO)
} else {
None
}
}
}
}
pub fn backoff_full_jitter(base_ms: u64, attempt: u32, max_delay_ms: u64) -> Duration {
let seed = mix_u64(attempt);
backoff_full_jitter_seeded(base_ms, attempt, max_delay_ms, seed)
}
pub fn backoff_full_jitter_seeded(
base_ms: u64,
attempt: u32,
max_delay_ms: u64,
seed: u64,
) -> Duration {
let base = base_ms.max(1);
let max_delay = max_delay_ms.max(base);
let exp = base.saturating_mul(1u64 << attempt.min(16));
let cap = exp.min(max_delay);
let pick = seed % (cap.saturating_add(1));
Duration::from_millis(pick)
}
pub fn politeness_delay(base: Duration) -> Duration {
let base_ms = base.as_millis() as u64;
if base_ms == 0 {
return Duration::ZERO;
}
let span = (base_ms / 5).max(1);
let extra = mix_u64(base_ms as u32) % (span.saturating_add(1));
Duration::from_millis(base_ms.saturating_add(extra))
}
pub fn wait_for_retry(
policy: RetryConfig,
attempt: u32,
retry_after: Option<Duration>,
) -> Duration {
if let Some(d) = retry_after {
return d.min(Duration::from_millis(HARD_MAX_DELAY_MS));
}
policy.delay_for_attempt(attempt)
}
fn mix_u64(attempt: u32) -> u64 {
let mut h = DefaultHasher::new();
Instant::now().hash(&mut h);
attempt.hash(&mut h);
std::thread::current().id().hash(&mut h);
let marker = &h as *const DefaultHasher as usize;
marker.hash(&mut h);
h.finish()
}
#[cfg(test)]
mod tests {
use super::*;
use reqwest::header::{HeaderValue, RETRY_AFTER};
#[test]
fn max_attempts_respects_kill_switch() {
let mut p = RetryConfig::default();
assert_eq!(p.max_attempts(), 4);
p.enabled = false;
assert_eq!(p.max_attempts(), 1);
assert!(!p.may_retry(1));
}
#[test]
fn from_config_disable_and_clamp() {
let cfg = Config {
disable_retry: true,
max_retries: 99,
retry_max_delay_ms: 999_999,
retry_base_ms: 10, retry_max_elapsed_ms: 0, timeout_secs: 12,
..Config::default()
};
let p = RetryConfig::from_config(&cfg);
assert!(!p.enabled);
assert_eq!(p.max_retries, HARD_MAX_RETRIES);
assert_eq!(p.max_delay_ms, HARD_MAX_DELAY_MS);
assert_eq!(p.base_ms, MIN_RETRY_BASE_MS);
assert_eq!(p.max_elapsed_ms, 12_000);
assert_eq!(p.max_attempts(), 1);
}
#[test]
fn from_config_explicit_max_elapsed() {
let cfg = Config {
retry_max_elapsed_ms: 5_000,
retry_base_ms: 200,
retry_max_delay_ms: 1_000,
..Config::default()
};
let p = RetryConfig::from_config(&cfg);
assert_eq!(p.max_elapsed_ms, 5_000);
}
#[test]
fn may_retry_within_budget_blocks_oversleep() {
let p = RetryConfig {
max_retries: 5,
base_ms: 100,
max_delay_ms: 1_000,
max_elapsed_ms: 500,
enabled: true,
};
assert!(p.may_retry_within_budget(1, Duration::from_millis(0), Duration::from_millis(400)));
assert!(!p.may_retry_within_budget(1, Duration::from_millis(0), Duration::from_millis(600)));
assert!(!p.may_retry_within_budget(1, Duration::from_millis(500), Duration::ZERO));
}
#[test]
fn backoff_respects_cap() {
let d = backoff_full_jitter(200, 20, 1_000);
assert!(d.as_millis() <= 1_000);
}
#[test]
fn backoff_seeded_is_deterministic() {
let a = backoff_full_jitter_seeded(200, 3, 10_000, 42);
let b = backoff_full_jitter_seeded(200, 3, 10_000, 42);
assert_eq!(a, b);
for seed in 0..64u64 {
for attempt in 0..8u32 {
let d = backoff_full_jitter_seeded(100, attempt, 500, seed);
assert!(d.as_millis() <= 500);
}
}
}
#[test]
fn politeness_delay_zero_stays_zero() {
assert_eq!(politeness_delay(Duration::ZERO), Duration::ZERO);
}
#[test]
fn politeness_delay_never_below_floor() {
let base = Duration::from_millis(1000);
for _ in 0..32 {
let d = politeness_delay(base);
assert!(d >= base, "delay {d:?} below floor {base:?}");
assert!(
d <= base + Duration::from_millis(200),
"delay {d:?} above floor+20%"
);
}
}
#[test]
fn classify_matrix() {
assert_eq!(
classify_http_status(StatusCode::OK, None),
RetryClass::Permanent
);
assert_eq!(
classify_http_status(StatusCode::NOT_FOUND, None),
RetryClass::Permanent
);
assert_eq!(
classify_http_status(StatusCode::BAD_REQUEST, None),
RetryClass::Permanent
);
assert_eq!(
classify_http_status(StatusCode::INTERNAL_SERVER_ERROR, None),
RetryClass::Transient
);
assert_eq!(
classify_http_status(StatusCode::REQUEST_TIMEOUT, None),
RetryClass::Transient
);
assert_eq!(
classify_http_status(StatusCode::TOO_MANY_REQUESTS, Some(Duration::from_secs(2))),
RetryClass::RateLimited {
retry_after: Some(Duration::from_secs(2))
}
);
assert_eq!(
classify_http_status(
StatusCode::SERVICE_UNAVAILABLE,
Some(Duration::from_secs(1))
),
RetryClass::RateLimited {
retry_after: Some(Duration::from_secs(1))
}
);
assert_eq!(
classify_http_status(StatusCode::SERVICE_UNAVAILABLE, None),
RetryClass::Transient
);
assert_eq!(
classify_http_status(StatusCode::UNAUTHORIZED, None),
RetryClass::Permanent
);
assert_eq!(
classify_http_status(StatusCode::FORBIDDEN, None),
RetryClass::Permanent
);
assert_eq!(
classify_http_status(StatusCode::UNPROCESSABLE_ENTITY, None),
RetryClass::Permanent
);
}
#[test]
fn parse_retry_after_seconds_and_http_date() {
let mut h = HeaderMap::new();
h.insert(RETRY_AFTER, HeaderValue::from_static("7"));
assert_eq!(parse_retry_after(&h), Some(Duration::from_secs(7)));
h.insert(RETRY_AFTER, HeaderValue::from_static(""));
assert_eq!(parse_retry_after(&h), None);
let future = SystemTime::now() + Duration::from_secs(2);
let http_date = httpdate::fmt_http_date(future);
h.insert(
RETRY_AFTER,
HeaderValue::from_str(&http_date).expect("http-date header"),
);
let d = parse_retry_after(&h).expect("HTTP-date Retry-After");
assert!(d <= Duration::from_secs(3), "got {d:?}");
assert!(d >= Duration::from_millis(500), "got {d:?}");
let past = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
let past_s = httpdate::fmt_http_date(past);
h.insert(
RETRY_AFTER,
HeaderValue::from_str(&past_s).expect("past http-date"),
);
assert_eq!(parse_retry_after(&h), None);
}
#[test]
fn wait_prefers_retry_after() {
let p = RetryConfig::default();
let w = wait_for_retry(p, 1, Some(Duration::from_secs(3)));
assert_eq!(w, Duration::from_secs(3));
}
#[test]
fn source_chain_detects_rustls_and_ignores_plain_io() {
let tls = rustls::Error::General("unit-test".into());
assert!(source_chain_has_rustls(&tls));
let io = std::io::Error::other("not tls");
assert!(!source_chain_has_rustls(&io));
assert_eq!(ErrorLayer::Tls.as_str(), "tls");
}
#[test]
fn retry_kind_labels() {
assert_eq!(RetryKind::Timeout.as_str(), "timeout");
assert!(RetryKind::TransientNetwork.is_retryable());
assert!(!RetryKind::Permanent.is_retryable());
assert_eq!(ErrorLayer::TcpConnect.as_str(), "tcp_connect");
}
}