use super::clock::{Clock, StdClock};
use crate::backoff::{Backoff, ExponentialBackoff};
use crate::error::RetryError;
use crate::shared::{Decision, give_up, should_retry_after, trace_retry};
use std::time::Duration;
#[must_use = "a `Retry` does nothing until you `.call()` it"]
pub struct Retry<F, B, C, P> {
op: F,
backoff: B,
clock: C,
when: P,
max_elapsed: Option<Duration>,
}
pub fn retry<F, T, E>(op: F) -> Retry<F, ExponentialBackoff, StdClock, fn(&E) -> bool>
where
F: FnMut() -> Result<T, E>,
{
Retry {
op,
backoff: ExponentialBackoff::default(),
clock: StdClock,
when: (|_| true) as fn(&E) -> bool,
max_elapsed: None,
}
}
impl<F, B, C, P> Retry<F, B, C, P> {
pub fn backoff<B2>(self, backoff: B2) -> Retry<F, B2, C, P> {
Retry {
backoff,
op: self.op,
when: self.when,
clock: self.clock,
max_elapsed: self.max_elapsed,
}
}
pub fn clock<C2>(self, clock: C2) -> Retry<F, B, C2, P> {
Retry {
clock,
op: self.op,
backoff: self.backoff,
when: self.when,
max_elapsed: self.max_elapsed,
}
}
pub fn max_elapsed(mut self, budget: Duration) -> Self {
self.max_elapsed = Some(budget);
self
}
}
impl<F, T, E, B, C, P> Retry<F, B, C, P>
where
F: FnMut() -> Result<T, E>,
{
pub fn when<P2>(self, predicate: P2) -> Retry<F, B, C, P2>
where
P2: Fn(&E) -> bool,
{
Retry {
when: predicate,
op: self.op,
backoff: self.backoff,
clock: self.clock,
max_elapsed: self.max_elapsed,
}
}
}
impl<F, T, E, B, C, P> Retry<F, B, C, P>
where
F: FnMut() -> Result<T, E>,
B: Backoff,
C: Clock,
P: Fn(&E) -> bool,
E: std::fmt::Debug,
{
pub fn call(mut self) -> Result<T, RetryError<E>> {
let mut backoff = self.backoff;
let mut retries = 0u32;
let start = self.clock.now();
let elapsed = || self.clock.now().saturating_duration_since(start);
loop {
let err = match (self.op)() {
Ok(value) => return Ok(value),
Err(err) => err,
};
match should_retry_after(&err, &self.when, &mut backoff, self.max_elapsed, elapsed) {
Decision::Retry(delay) => {
retries += 1;
trace_retry(retries, &err, delay);
self.clock.sleep(delay);
}
Decision::Stop { reason, elapsed: m } => {
return Err(give_up(err, retries, reason, m, elapsed));
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backoff::ExponentialBackoffConfig;
use crate::error::StopReason;
use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
use std::sync::{Arc, Mutex};
use std::time::Instant;
#[derive(Clone)]
struct MockClock {
start: Instant,
elapsed: Arc<Mutex<Duration>>,
log: Arc<Mutex<Vec<Duration>>>,
now_calls: Arc<AtomicUsize>,
}
impl MockClock {
fn new() -> Self {
Self {
start: Instant::now(),
elapsed: Arc::new(Mutex::new(Duration::ZERO)),
log: Arc::new(Mutex::new(Vec::new())),
now_calls: Arc::new(AtomicUsize::new(0)),
}
}
fn slept(&self) -> Vec<Duration> {
self.log.lock().unwrap().clone()
}
fn now_calls(&self) -> usize {
self.now_calls.load(SeqCst)
}
}
impl Clock for MockClock {
fn now(&self) -> Instant {
self.now_calls.fetch_add(1, SeqCst);
self.start + *self.elapsed.lock().unwrap()
}
fn sleep(&self, dur: Duration) {
self.log.lock().unwrap().push(dur);
*self.elapsed.lock().unwrap() += dur;
}
}
fn backoff(max_retries: u32) -> ExponentialBackoff {
ExponentialBackoff::new(ExponentialBackoffConfig {
factor: 2,
base: Duration::from_secs(1),
max_retries,
max_delay: Duration::from_secs(100),
})
.unwrap()
}
fn secs(n: u64) -> Duration {
Duration::from_secs(n)
}
#[test]
fn succeeds_first_try() {
let clock = MockClock::new();
let result: Result<i32, RetryError<()>> = retry(|| Ok(42)).clock(clock.clone()).call();
assert_eq!(result.unwrap(), 42);
assert!(clock.slept().is_empty());
}
#[test]
fn retries_then_succeeds() {
let clock = MockClock::new();
let mut n = 0;
let result: Result<i32, RetryError<&str>> = retry(|| {
n += 1;
if n < 3 { Err("boom") } else { Ok(42) }
})
.backoff(backoff(5))
.clock(clock.clone())
.call();
assert_eq!(result.unwrap(), 42);
assert_eq!(n, 3); assert_eq!(clock.slept(), vec![secs(1), secs(2)]);
}
#[test]
fn stops_on_non_retryable() {
let clock = MockClock::new();
let mut calls = 0;
let result: Result<i32, RetryError<&str>> = retry(|| {
calls += 1;
Err("nope")
})
.backoff(backoff(5))
.clock(clock.clone())
.when(|_e| false)
.call();
let err = result.unwrap_err();
assert_eq!(*err.error(), "nope");
assert_eq!(err.stop_reason(), StopReason::NotRetryable);
assert_eq!(err.attempts(), 1); assert_eq!(err.elapsed(), Duration::ZERO);
assert_eq!(calls, 1); assert!(clock.slept().is_empty());
}
#[test]
fn exhausts_retries() {
let clock = MockClock::new();
let result: Result<i32, RetryError<&str>> = retry(|| Err("always"))
.backoff(backoff(3))
.clock(clock.clone())
.call();
let err = result.unwrap_err();
assert_eq!(*err.error(), "always");
assert_eq!(err.stop_reason(), StopReason::RetriesExhausted);
assert_eq!(err.attempts(), 4); assert_eq!(err.elapsed(), secs(7)); assert_eq!(clock.slept().len(), 3); }
#[test]
fn stops_on_time_budget() {
let clock = MockClock::new();
let result: Result<i32, RetryError<&str>> = retry(|| Err("slow"))
.backoff(backoff(100)) .clock(clock.clone())
.max_elapsed(secs(10))
.call();
let err = result.unwrap_err();
assert_eq!(*err.error(), "slow");
assert_eq!(err.stop_reason(), StopReason::MaxElapsed);
assert_eq!(err.attempts(), 4);
assert!(
err.elapsed() < secs(10),
"elapsed must stay under the budget"
);
assert_eq!(clock.slept(), vec![secs(1), secs(2), secs(4)]);
}
#[test]
fn drives_a_jittered_backoff() {
let clock = MockClock::new();
let out: Result<i32, RetryError<&str>> = retry(|| Err("boom"))
.backoff(backoff(3).jittered_with_seed(42))
.clock(&clock)
.call();
assert_eq!(*out.unwrap_err().error(), "boom");
let slept = clock.slept();
assert_eq!(slept.len(), 3);
for (d, cap) in slept.iter().zip([secs(1), secs(2), secs(4)]) {
assert!(*d <= cap, "jittered delay {d:?} exceeded {cap:?}");
}
}
#[test]
fn drives_a_decorrelated_backoff() {
use crate::backoff::{DecorrelatedBackoff, DecorrelatedBackoffConfig};
let clock = MockClock::new();
let out: Result<i32, RetryError<&str>> = retry(|| Err("boom"))
.backoff(
DecorrelatedBackoff::with_seed(
DecorrelatedBackoffConfig {
base: secs(1),
max_retries: 4,
max_delay: secs(20),
},
7,
)
.unwrap(),
)
.clock(&clock)
.call();
assert_eq!(*out.unwrap_err().error(), "boom");
let slept = clock.slept();
assert_eq!(slept.len(), 4);
assert!(
slept.iter().all(|d| *d >= secs(1) && *d <= secs(20)),
"delays escaped [base, max_delay]: {slept:?}"
);
}
#[test]
fn accepts_a_borrowed_or_shared_clock() {
let clock = MockClock::new();
let _: Result<i32, RetryError<&str>> =
retry(|| Err("x")).backoff(backoff(2)).clock(&clock).call();
assert_eq!(clock.slept(), vec![secs(1), secs(2)]);
let shared = std::sync::Arc::new(MockClock::new());
let _: Result<i32, RetryError<&str>> = retry(|| Err("x"))
.backoff(backoff(2))
.clock(Arc::clone(&shared))
.call();
assert_eq!(shared.slept(), vec![secs(1), secs(2)]);
}
#[test]
fn clock_reads_do_not_scale_with_attempts() {
for retries in [3, 30] {
let clock = MockClock::new();
let _: Result<i32, RetryError<&str>> = retry(|| Err("x"))
.backoff(backoff(retries))
.clock(clock.clone())
.call();
assert_eq!(clock.now_calls(), 2, "with {retries} retries and no budget");
}
let clock = MockClock::new();
let _: Result<i32, RetryError<&str>> = retry(|| Err("x"))
.backoff(backoff(3))
.clock(clock.clone())
.max_elapsed(secs(1000))
.call();
assert_eq!(clock.now_calls(), 5); }
#[test]
fn emits_a_tracing_event_per_retry() {
let events = crate::test_support::count_retry_events();
let clock = MockClock::new();
let mut n = 0;
let out: Result<i32, RetryError<&str>> = retry(|| {
n += 1;
if n < 3 { Err("boom") } else { Ok(42) }
})
.backoff(backoff(5))
.clock(clock)
.call();
assert_eq!(out.unwrap(), 42);
assert_eq!(events.get(), 2); }
}