use std::future::Future;
use std::time::Duration;
use crate::error::AxiamError;
use crate::telemetry::{Telemetry, TelemetryEvent};
pub const MAX_ATTEMPTS: u32 = 3;
pub const BASE_DELAY: Duration = Duration::from_millis(200);
pub const MAX_DELAY: Duration = Duration::from_secs(5);
pub(crate) trait Jitter: Send + Sync {
fn fraction(&self) -> f64;
}
pub(crate) struct ThreadRngJitter;
impl Jitter for ThreadRngJitter {
fn fraction(&self) -> f64 {
let mut buf = [0u8; 8];
if getrandom::fill(&mut buf).is_err() {
return 1.0;
}
(u64::from_le_bytes(buf) >> 11) as f64 / (1u64 << 53) as f64
}
}
pub(crate) trait Sleeper: Send + Sync {
fn sleep(&self, d: Duration) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send + '_>>;
}
pub(crate) struct TokioSleeper;
impl Sleeper for TokioSleeper {
fn sleep(&self, d: Duration) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send + '_>> {
Box::pin(tokio::time::sleep(d))
}
}
pub(crate) fn backoff_for(attempt: u32) -> Duration {
let shift = attempt.saturating_sub(1).min(32);
BASE_DELAY
.saturating_mul(1u32.checked_shl(shift).unwrap_or(u32::MAX))
.min(MAX_DELAY)
}
pub(crate) fn delay_for(
attempt: u32,
retry_after: Option<Duration>,
jitter: &dyn Jitter,
) -> Duration {
let backoff = backoff_for(attempt);
let fraction = jitter.fraction().clamp(0.0, 1.0);
let jittered = backoff.mul_f64(fraction);
match retry_after {
Some(hint) => jittered.max(hint),
None => jittered,
}
}
pub(crate) fn status_is_retryable(status: u16) -> bool {
status >= 500 || status == 408 || status == 429
}
pub(crate) fn is_retryable(err: &AxiamError) -> bool {
matches!(err, AxiamError::Network { .. })
}
pub(crate) struct Attempt {
pub(crate) err: AxiamError,
pub(crate) retry_after: Option<Duration>,
}
impl Attempt {
pub(crate) fn bare(err: AxiamError) -> Self {
Self {
err,
retry_after: None,
}
}
}
pub(crate) fn parse_retry_after(value: &str) -> Option<Duration> {
let secs: u64 = value.trim().parse().ok()?;
Some(Duration::from_secs(secs))
}
pub(crate) struct RetryRunner<'a> {
pub(crate) enabled: bool,
pub(crate) operation: &'static str,
pub(crate) telemetry: &'a Telemetry,
pub(crate) jitter: &'a dyn Jitter,
pub(crate) sleeper: &'a dyn Sleeper,
}
pub(crate) struct RetryBound<'a> {
pub(crate) deadline: crate::time::Instant,
pub(crate) now: &'a (dyn Fn() -> crate::time::Instant + Send + Sync),
pub(crate) sleeper: &'a dyn Sleeper,
}
pub(crate) struct Ended {
pub(crate) err: AxiamError,
pub(crate) at_deadline: bool,
pub(crate) retry_after: Option<Duration>,
}
impl RetryRunner<'_> {
pub(crate) async fn run<T, F, Fut>(&self, op: F) -> Result<T, AxiamError>
where
F: FnMut(u32) -> Fut,
Fut: Future<Output = Result<T, Attempt>>,
{
self.run_until(None, op).await.map_err(|ended| ended.err)
}
pub(crate) async fn run_until<T, F, Fut>(
&self,
bound: Option<&RetryBound<'_>>,
mut op: F,
) -> Result<T, Ended>
where
F: FnMut(u32) -> Fut,
Fut: Future<Output = Result<T, Attempt>>,
{
let cap = if self.enabled { MAX_ATTEMPTS } else { 1 };
let mut attempt = 1;
loop {
match op(attempt).await {
Ok(v) => return Ok(v),
Err(Attempt { err, retry_after }) => {
let last = attempt >= cap;
if last || !is_retryable(&err) {
return Err(Ended {
err,
at_deadline: false,
retry_after,
});
}
let mut delay = delay_for(attempt, retry_after, self.jitter);
let mut at_deadline = false;
if let Some(bound) = bound {
let left = bound.deadline.saturating_duration_since((bound.now)());
if delay >= left {
delay = left;
at_deadline = true;
}
}
self.telemetry.emit(TelemetryEvent::Retry {
operation: self.operation,
attempt,
delay,
reason: err.to_string(),
});
match bound {
Some(bound) => {
if !delay.is_zero() {
bound.sleeper.sleep(delay).await;
}
}
None => self.sleeper.sleep(delay).await,
}
if at_deadline {
return Err(Ended {
err,
at_deadline: true,
retry_after,
});
}
attempt += 1;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
struct Pinned(f64);
impl Jitter for Pinned {
fn fraction(&self) -> f64 {
self.0
}
}
#[test]
fn backoff_doubles_from_the_base_and_stops_at_the_cap() {
assert_eq!(backoff_for(1), Duration::from_millis(200));
assert_eq!(backoff_for(2), Duration::from_millis(400));
assert_eq!(backoff_for(20), MAX_DELAY);
assert_eq!(backoff_for(u32::MAX), MAX_DELAY);
}
#[test]
fn full_jitter_spans_zero_to_the_whole_backoff() {
assert_eq!(delay_for(1, None, &Pinned(0.0)), Duration::ZERO);
assert_eq!(delay_for(1, None, &Pinned(1.0)), Duration::from_millis(200));
assert_eq!(delay_for(2, None, &Pinned(1.0)), Duration::from_millis(400));
assert_eq!(delay_for(2, None, &Pinned(0.5)), Duration::from_millis(200));
}
#[test]
fn retry_after_raises_the_wait_but_never_lowers_it() {
assert_eq!(
delay_for(1, Some(Duration::from_secs(2)), &Pinned(1.0)),
Duration::from_secs(2)
);
assert_eq!(
delay_for(1, Some(Duration::ZERO), &Pinned(1.0)),
Duration::from_millis(200)
);
assert_eq!(
delay_for(1, Some(Duration::from_millis(50)), &Pinned(0.0)),
Duration::from_millis(50)
);
}
#[test]
fn a_jitter_fraction_outside_the_range_is_clamped() {
assert_eq!(delay_for(1, None, &Pinned(9.0)), Duration::from_millis(200));
assert_eq!(delay_for(1, None, &Pinned(-1.0)), Duration::ZERO);
}
#[test]
fn only_network_failures_are_retryable() {
assert!(is_retryable(&AxiamError::network("boom")));
assert!(!is_retryable(&AxiamError::auth("nope")));
assert!(!is_retryable(&AxiamError::authz("denied", None, None)));
}
#[test]
fn retry_after_parses_delta_seconds_and_ignores_the_date_form() {
assert_eq!(parse_retry_after("3"), Some(Duration::from_secs(3)));
assert_eq!(parse_retry_after(" 7 "), Some(Duration::from_secs(7)));
assert_eq!(parse_retry_after("0"), Some(Duration::ZERO));
assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
assert_eq!(parse_retry_after("garbage"), None);
assert_eq!(parse_retry_after("-5"), None);
assert_eq!(
parse_retry_after("86400"),
Some(Duration::from_secs(86_400))
);
}
#[derive(Default)]
struct RecordingSleeper {
delays: std::sync::Mutex<Vec<Duration>>,
}
impl Sleeper for RecordingSleeper {
fn sleep(&self, d: Duration) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send + '_>> {
self.delays.lock().unwrap().push(d);
Box::pin(std::future::ready(()))
}
}
fn runner<'a>(
enabled: bool,
telemetry: &'a Telemetry,
jitter: &'a dyn Jitter,
sleeper: &'a dyn Sleeper,
) -> RetryRunner<'a> {
RetryRunner {
enabled,
operation: "check_access",
telemetry,
jitter,
sleeper,
}
}
#[tokio::test]
async fn a_permanently_failing_operation_makes_exactly_three_attempts() {
let tel = Telemetry::default();
let sleeper = RecordingSleeper::default();
let calls = std::sync::atomic::AtomicU32::new(0);
let out: Result<(), _> = runner(true, &tel, &Pinned(1.0), &sleeper)
.run(|_| {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async { Err(Attempt::bare(AxiamError::network("down"))) }
})
.await;
assert!(out.is_err());
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 3);
assert_eq!(
*sleeper.delays.lock().unwrap(),
vec![Duration::from_millis(200), Duration::from_millis(400)]
);
}
#[tokio::test]
async fn a_decisive_failure_makes_exactly_one_attempt() {
for decisive in ["auth", "authz"] {
let tel = Telemetry::default();
let sleeper = RecordingSleeper::default();
let calls = std::sync::atomic::AtomicU32::new(0);
let out: Result<(), _> = runner(true, &tel, &Pinned(1.0), &sleeper)
.run(|_| {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async move {
Err(Attempt::bare(if decisive == "auth" {
AxiamError::auth("401")
} else {
AxiamError::authz("403", None, None)
}))
}
})
.await;
assert!(out.is_err(), "{decisive} must surface");
assert_eq!(
calls.load(std::sync::atomic::Ordering::SeqCst),
1,
"{decisive} must not be retried"
);
assert!(sleeper.delays.lock().unwrap().is_empty());
}
}
#[tokio::test]
async fn the_disable_switch_yields_exactly_one_attempt_even_on_a_503() {
let tel = Telemetry::default();
let sleeper = RecordingSleeper::default();
let calls = std::sync::atomic::AtomicU32::new(0);
let out: Result<(), _> = runner(false, &tel, &Pinned(1.0), &sleeper)
.run(|_| {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async { Err(Attempt::bare(AxiamError::network("503"))) }
})
.await;
assert!(out.is_err());
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
assert!(sleeper.delays.lock().unwrap().is_empty());
}
#[tokio::test]
async fn a_retry_that_then_succeeds_returns_the_value_and_emits_the_event() {
let events = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let sink_events = events.clone();
let tel = Telemetry::new(Some(std::sync::Arc::new(move |e: &TelemetryEvent| {
if let TelemetryEvent::Retry { attempt, delay, .. } = e {
sink_events
.lock()
.unwrap()
.push(format!("retry {attempt} {delay:?}"));
}
})));
let sleeper = RecordingSleeper::default();
let calls = std::sync::atomic::AtomicU32::new(0);
let out = runner(true, &tel, &Pinned(1.0), &sleeper)
.run(|_| {
let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async move {
if n == 0 {
Err(Attempt::bare(AxiamError::network("transient")))
} else {
Ok(42u32)
}
}
})
.await;
assert_eq!(out.unwrap(), 42);
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 2);
assert_eq!(events.lock().unwrap().len(), 1);
assert!(events.lock().unwrap()[0].starts_with("retry 1 "));
}
#[tokio::test]
async fn a_panicking_hook_does_not_fail_the_operation() {
let tel = Telemetry::new(Some(std::sync::Arc::new(|_: &TelemetryEvent| {
panic!("hook exploded");
})));
let sleeper = RecordingSleeper::default();
let calls = std::sync::atomic::AtomicU32::new(0);
let out = runner(true, &tel, &Pinned(0.0), &sleeper)
.run(|_| {
let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async move {
if n == 0 {
Err(Attempt::bare(AxiamError::network("transient")))
} else {
Ok(7u32)
}
}
})
.await;
assert_eq!(out.unwrap(), 7);
}
#[tokio::test]
async fn a_server_hint_lengthens_the_recorded_wait() {
let tel = Telemetry::default();
let sleeper = RecordingSleeper::default();
let out: Result<(), _> = runner(true, &tel, &Pinned(1.0), &sleeper)
.run(|_| async {
Err(Attempt {
err: AxiamError::network("429"),
retry_after: Some(Duration::from_secs(2)),
})
})
.await;
assert!(out.is_err());
assert_eq!(
*sleeper.delays.lock().unwrap(),
vec![Duration::from_secs(2), Duration::from_secs(2)]
);
}
struct Wound {
start: crate::time::Instant,
offset: std::sync::Mutex<Duration>,
waits: std::sync::Mutex<Vec<Duration>>,
}
impl Wound {
fn new() -> Self {
Self {
start: crate::time::Instant::now(),
offset: Default::default(),
waits: Default::default(),
}
}
fn now(&self) -> crate::time::Instant {
self.start + *self.offset.lock().unwrap()
}
}
impl Sleeper for Wound {
fn sleep(&self, d: Duration) -> std::pin::Pin<Box<dyn Future<Output = ()> + Send + '_>> {
*self.offset.lock().unwrap() += d;
self.waits.lock().unwrap().push(d);
Box::pin(std::future::ready(()))
}
}
#[tokio::test]
async fn a_bound_caps_the_wait_at_the_deadline_and_no_request_follows_it() {
let tel = Telemetry::default();
let unused = RecordingSleeper::default();
let clock = Wound::new();
let now = || clock.now();
let bound = RetryBound {
deadline: clock.start + Duration::from_secs(7),
now: &now,
sleeper: &clock,
};
let calls = std::sync::atomic::AtomicU32::new(0);
let out: Result<(), _> = runner(true, &tel, &Pinned(1.0), &unused)
.run_until(Some(&bound), |_| {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async {
Err(Attempt {
err: AxiamError::network("503"),
retry_after: Some(Duration::from_secs(3600)),
})
}
})
.await;
let ended = out.expect_err("the deadline ends the run");
assert!(ended.at_deadline);
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(*clock.waits.lock().unwrap(), vec![Duration::from_secs(7)]);
assert!(
unused.delays.lock().unwrap().is_empty(),
"the wait is served on the bound's clock, not the runner's"
);
}
#[tokio::test]
async fn a_bound_leaves_a_wait_that_ends_before_the_deadline_alone() {
let tel = Telemetry::default();
let unused = RecordingSleeper::default();
let clock = Wound::new();
let now = || clock.now();
let bound = RetryBound {
deadline: clock.start + Duration::from_secs(7),
now: &now,
sleeper: &clock,
};
let calls = std::sync::atomic::AtomicU32::new(0);
let out = runner(true, &tel, &Pinned(0.0), &unused)
.run_until(Some(&bound), |_| {
let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async move {
if n == 0 {
Err(Attempt {
err: AxiamError::network("503"),
retry_after: Some(Duration::from_secs(2)),
})
} else {
Ok(1u32)
}
}
})
.await;
assert_eq!(out.ok(), Some(1));
assert_eq!(*clock.waits.lock().unwrap(), vec![Duration::from_secs(2)]);
}
#[tokio::test]
async fn no_bound_is_the_plain_policy() {
let tel = Telemetry::default();
let sleeper = RecordingSleeper::default();
let out: Result<(), _> = runner(true, &tel, &Pinned(1.0), &sleeper)
.run_until(None, |_| async {
Err(Attempt {
err: AxiamError::network("503"),
retry_after: Some(Duration::from_secs(3600)),
})
})
.await;
assert!(!out.expect_err("exhausted").at_deadline);
assert_eq!(
*sleeper.delays.lock().unwrap(),
vec![Duration::from_secs(3600), Duration::from_secs(3600)],
"an unbounded Retry-After is a floor with no ceiling (§16.1)"
);
}
#[test]
fn the_production_jitter_source_stays_in_range() {
let j = ThreadRngJitter;
for _ in 0..1_000 {
let f = j.fraction();
assert!((0.0..=1.0).contains(&f), "fraction out of range: {f}");
}
}
}