use chrono::{DateTime, Utc};
use std::time::Duration;
#[derive(Debug, Clone)]
pub struct RetryPolicy {
pub max_retries: u32,
pub initial_backoff: Duration,
pub max_backoff: Duration,
pub backoff_multiplier: f64,
pub jitter_factor: f64,
pub exponential_backoff: bool,
pub enable_jitter: bool,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_retries: 5,
initial_backoff: Duration::from_secs(1),
max_backoff: Duration::from_secs(60),
backoff_multiplier: 2.0,
jitter_factor: 0.1,
exponential_backoff: true,
enable_jitter: true,
}
}
}
impl RetryPolicy {
pub fn standard() -> Self {
Self::default()
}
pub fn fast() -> Self {
Self {
max_retries: 5,
initial_backoff: Duration::from_millis(500),
max_backoff: Duration::from_secs(10),
backoff_multiplier: 1.5,
jitter_factor: 0.1,
exponential_backoff: true,
enable_jitter: true,
}
}
pub fn slow() -> Self {
Self {
max_retries: 5,
initial_backoff: Duration::from_secs(2),
max_backoff: Duration::from_secs(300), backoff_multiplier: 2.0,
jitter_factor: 0.2,
exponential_backoff: true,
enable_jitter: true,
}
}
pub fn calculate_backoff(&self, attempt: u32) -> Duration {
if !self.exponential_backoff {
return self.initial_backoff;
}
if attempt == 0 {
return self.initial_backoff;
}
let base_ms = self.initial_backoff.as_millis() as u64;
let max_ms = self.max_backoff.as_millis() as u64;
let spider_attempt = attempt - 1;
if self.enable_jitter {
crate::utils::backoff::backoff_delay(spider_attempt, base_ms, max_ms)
} else {
let exp = 2u64.saturating_pow(spider_attempt);
let cap_ms = base_ms.saturating_mul(exp).min(max_ms);
Duration::from_millis(cap_ms)
}
}
pub fn next_retry_time(&self, attempt: u32, base_time: DateTime<Utc>) -> DateTime<Utc> {
let backoff = self.calculate_backoff(attempt);
base_time + chrono::Duration::milliseconds(backoff.as_millis() as i64)
}
pub fn should_retry(&self, attempt: u32) -> bool {
attempt < self.max_retries
}
pub fn should_retry_with_error(&self, attempt: u32, error: &anyhow::Error) -> bool {
attempt < self.max_retries && is_retryable_error(error)
}
}
pub fn is_retryable_error(error: &anyhow::Error) -> bool {
let error_string = error.to_string().to_lowercase();
let retryable_patterns = [
"timeout",
"connection reset",
"connection refused",
"dns error",
"500 internal server error",
"502 bad gateway",
"503 service unavailable",
"504 gateway timeout",
"network is unreachable",
"broken pipe",
"too many connections",
"rate limit",
"error sending request",
"connection closed",
"timed out",
"operation timeout",
"connection error",
"temporarily unavailable",
];
retryable_patterns.iter().any(|&p| error_string.contains(p))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_calculate_backoff_exponential() {
let mut policy = RetryPolicy::standard();
policy.enable_jitter = false;
let backoff1 = policy.calculate_backoff(1);
assert_eq!(backoff1, Duration::from_secs(1));
let backoff2 = policy.calculate_backoff(2);
assert_eq!(backoff2, Duration::from_secs(2));
let backoff3 = policy.calculate_backoff(3);
assert_eq!(backoff3, Duration::from_secs(4)); }
#[test]
fn test_calculate_backoff_with_jitter() {
let mut policy = RetryPolicy::standard();
policy.enable_jitter = true;
let max_backoff = policy.max_backoff;
for _ in 0..1000 {
let backoff = policy.calculate_backoff(2);
assert!(
backoff <= max_backoff,
"backoff {:?} exceeds max_backoff {:?}",
backoff,
max_backoff
);
}
}
#[test]
fn test_calculate_backoff_full_jitter_higher_attempt_higher_mean() {
let mut policy = RetryPolicy::standard();
policy.enable_jitter = true;
let mean_low = sample_backoff_mean(&policy, 1, 500); let mean_high = sample_backoff_mean(&policy, 5, 500); assert!(
mean_high > mean_low * 5.0,
"expected mean_high (>5x mean_low), got high={}, low={}",
mean_high,
mean_low
);
}
#[test]
fn test_calculate_backoff_attempt_zero_returns_initial() {
let mut policy = RetryPolicy::standard();
policy.enable_jitter = true;
policy.initial_backoff = Duration::from_millis(800);
let backoff = policy.calculate_backoff(0);
assert_eq!(backoff, Duration::from_millis(800));
}
fn sample_backoff_mean(policy: &RetryPolicy, attempt: u32, n: usize) -> f64 {
let mut sum = 0u64;
for _ in 0..n {
sum += policy.calculate_backoff(attempt).as_millis() as u64;
}
sum as f64 / n as f64
}
#[test]
fn test_calculate_backoff_max_limit() {
let mut policy = RetryPolicy::standard();
policy.max_backoff = Duration::from_secs(5);
policy.enable_jitter = false;
let backoff = policy.calculate_backoff(10);
assert_eq!(backoff, Duration::from_secs(5)); }
#[test]
fn test_should_retry() {
let policy = RetryPolicy::standard();
assert!(policy.should_retry(0));
assert!(policy.should_retry(1));
assert!(policy.should_retry(2));
assert!(policy.should_retry(3));
assert!(policy.should_retry(4));
assert!(!policy.should_retry(5)); assert!(!policy.should_retry(6));
}
#[test]
fn test_is_retryable_error_reqwest_hyper_patterns() {
for msg in [
"error sending request for url (https://news.ycombinator.com/)",
"connection closed before message completed",
"operation timed out",
"temporarily unavailable",
] {
let err = anyhow::anyhow!(msg.to_string());
assert!(is_retryable_error(&err), "expected retryable: {msg}");
}
let permanent = anyhow::anyhow!("HTTP status client error (404 Not Found)");
assert!(!is_retryable_error(&permanent));
}
#[test]
fn test_should_retry_with_error_respects_max_retries() {
let policy = RetryPolicy {
max_retries: 2,
..Default::default()
};
let retryable = anyhow::anyhow!("error sending request for url (x)");
assert!(policy.should_retry_with_error(0, &retryable));
assert!(policy.should_retry_with_error(1, &retryable));
assert!(!policy.should_retry_with_error(2, &retryable));
}
#[test]
fn test_next_retry_time() {
use chrono::TimeZone;
let mut policy = RetryPolicy::standard();
policy.enable_jitter = false;
let base_time = Utc.with_ymd_and_hms(2024, 1, 1, 12, 0, 0).unwrap();
let next_retry = policy.next_retry_time(2, base_time);
let expected = base_time + chrono::Duration::seconds(2);
assert_eq!(next_retry, expected);
}
#[test]
fn test_calculate_backoff_non_exponential_returns_initial_backoff() {
let mut policy = RetryPolicy::standard();
policy.exponential_backoff = false;
policy.enable_jitter = false;
policy.initial_backoff = Duration::from_secs(3);
let backoff1 = policy.calculate_backoff(1);
let backoff5 = policy.calculate_backoff(5);
assert_eq!(backoff1, Duration::from_secs(3));
assert_eq!(backoff5, Duration::from_secs(3));
}
}