use crate::locked;
use getrandom::SysRng;
use rand_core::{Rng, UnwrapErr};
use std::future::Future;
use std::sync::Mutex;
use std::time::Duration;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RetryPolicy {
pub attempts: u32,
pub initial_delay: Duration,
pub max_delay: Duration,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct RetryFacts<'a> {
pub status: Option<u16>,
pub method: Option<&'a str>,
pub keyed: bool,
}
pub struct RetryRule {
pub name: &'static str,
retryable: fn(&RetryFacts<'_>) -> bool,
}
impl RetryRule {
pub fn retryable(&self, facts: &RetryFacts<'_>) -> bool {
(self.retryable)(facts)
}
}
const IDEMPOTENT_METHODS: [&str; 6] = ["GET", "HEAD", "PUT", "DELETE", "OPTIONS", "TRACE"];
pub const BOOT: RetryRule = RetryRule {
name: "boot",
retryable: |facts| matches!(facts.status, Some(429 | 503 | 504)),
};
pub const JOB: RetryRule = RetryRule {
name: "job",
retryable: |facts| matches!(facts.status, Some(status) if status == 408 || status == 429 || status >= 500),
};
pub const TRANSPORT: RetryRule = RetryRule {
name: "transport",
retryable: |facts| {
let Some(status) = facts.status else {
return false;
};
let repeatable = facts.keyed
|| facts.method.is_some_and(|method| {
IDEMPOTENT_METHODS
.iter()
.any(|m| m.eq_ignore_ascii_case(method))
});
repeatable && matches!(status, 408 | 413 | 429 | 500 | 502 | 503 | 504)
},
};
pub const REFRESH: RetryRule = RetryRule {
name: "refresh",
retryable: |facts| match facts.status {
None => true,
Some(status) => status == 408 || status == 429 || status >= 500,
},
};
pub const RETRY_RULES: [&RetryRule; 4] = [&BOOT, &JOB, &REFRESH, &TRANSPORT];
pub fn retry_after(header: Option<&str>) -> Option<Duration> {
let stated = header?.trim();
if stated.is_empty() || !stated.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
stated.parse::<u64>().ok().map(Duration::from_secs)
}
pub fn equal_jitter(cap: Duration, random: Option<&Mutex<dyn Rng + Send>>) -> Duration {
let drawn = match random {
Some(random) => locked(random).next_u64(),
None => UnwrapErr(SysRng).next_u64(),
};
let share = (drawn >> 11) as f64 / (1u64 << 53) as f64;
cap.div_f64(2.0) + cap.div_f64(2.0).mul_f64(share)
}
pub async fn retry_with_backoff<T, E, F>(
policy: RetryPolicy,
random: Option<&Mutex<dyn Rng + Send>>,
mut attempt: impl FnMut() -> F,
retryable: impl Fn(&E) -> bool,
stated_wait: impl Fn(&E) -> Option<Duration>,
) -> Result<T, E>
where
F: Future<Output = Result<T, E>>,
{
let mut cap = policy.initial_delay;
let mut made = 1;
loop {
let error = match attempt().await {
Ok(value) => return Ok(value),
Err(error) => error,
};
if made >= policy.attempts || !retryable(&error) {
return Err(error);
}
let wait = equal_jitter(cap, random).max(stated_wait(&error).unwrap_or_default());
tokio::time::sleep(wait).await;
cap = (cap * 2).min(policy.max_delay);
made += 1;
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::convert::Infallible;
struct Lowest;
impl rand_core::TryRng for Lowest {
type Error = Infallible;
fn try_next_u32(&mut self) -> Result<u32, Infallible> {
Ok(0)
}
fn try_next_u64(&mut self) -> Result<u64, Infallible> {
Ok(0)
}
fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Infallible> {
dst.fill(0);
Ok(())
}
}
#[test]
fn jitter_stays_between_half_the_cap_and_the_cap() {
let cap = Duration::from_millis(1000);
for _ in 0..1000 {
let wait = equal_jitter(cap, None);
assert!(wait >= cap / 2 && wait <= cap, "{wait:?}");
}
}
#[test]
fn a_jitter_drawn_lowest_from_the_callers_source_is_half_the_cap() {
let lowest = Mutex::new(Lowest);
let cap = Duration::from_millis(1000);
assert_eq!(equal_jitter(cap, Some(&lowest)), cap / 2);
}
#[tokio::test(start_paused = true)]
async fn each_wait_of_a_budget_is_drawn_from_the_callers_source() {
let lowest = Mutex::new(Lowest);
let policy = RetryPolicy {
attempts: 3,
initial_delay: Duration::from_millis(10),
max_delay: Duration::from_millis(20),
};
let started = tokio::time::Instant::now();
let outcome: Result<(), ()> = retry_with_backoff(
policy,
Some(&lowest),
|| async { Err(()) },
|_| true,
|_| None,
)
.await;
assert_eq!(outcome, Err(()));
assert_eq!(started.elapsed(), Duration::from_millis(5 + 10));
}
#[tokio::test(start_paused = true)]
async fn a_budget_is_spent_and_the_last_failure_returned() {
let policy = RetryPolicy {
attempts: 3,
initial_delay: Duration::from_millis(10),
max_delay: Duration::from_millis(20),
};
let mut made = 0;
let outcome: Result<(), u32> = retry_with_backoff(
policy,
None,
|| {
made += 1;
let n = made;
async move { Err(n) }
},
|_| true,
|_| None,
)
.await;
assert_eq!(outcome, Err(3));
}
#[tokio::test(start_paused = true)]
async fn a_failure_not_worth_retrying_ends_at_once() {
let policy = RetryPolicy {
attempts: 4,
initial_delay: Duration::from_millis(10),
max_delay: Duration::from_millis(20),
};
let mut made = 0;
let outcome: Result<(), u32> = retry_with_backoff(
policy,
None,
|| {
made += 1;
async { Err(400) }
},
|status| *status == 429,
|_| None,
)
.await;
assert_eq!(outcome, Err(400));
assert_eq!(made, 1);
}
#[tokio::test(start_paused = true)]
async fn a_stated_wait_is_a_floor_under_the_backoff() {
let policy = RetryPolicy {
attempts: 2,
initial_delay: Duration::from_millis(10),
max_delay: Duration::from_millis(10),
};
let started = tokio::time::Instant::now();
let mut made = 0;
let outcome: Result<u32, u32> = retry_with_backoff(
policy,
None,
|| {
made += 1;
let n = made;
async move { if n == 1 { Err(429) } else { Ok(n) } }
},
|_| true,
|_| Some(Duration::from_secs(5)),
)
.await;
assert_eq!(outcome, Ok(2));
assert!(started.elapsed() >= Duration::from_secs(5));
}
}