use std::thread;
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use crate::backoff::ExponentialBackoff;
use crate::correctness::{check_predicate_true, CorrectnessError};
pub struct Retry;
impl Retry {
pub fn run<T, E, F>(max_attempts: u32, delay: Duration, mut f: F) -> Result<T, E>
where
F: FnMut() -> Result<T, E>,
{
assert!(max_attempts > 0, "max_attempts must be greater than 0");
for attempt in 0..max_attempts {
match f() {
Ok(value) => return Ok(value),
Err(e) => {
if attempt + 1 == max_attempts {
return Err(e);
}
thread::sleep(delay);
}
}
}
unreachable!()
}
pub fn with_backoff<T, E, F>(max_attempts: u32, initial_delay: Duration, mut f: F) -> Result<T, E>
where
F: FnMut() -> Result<T, E>,
{
assert!(max_attempts > 0, "max_attempts must be greater than 0");
let mut delay = initial_delay;
for attempt in 0..max_attempts {
match f() {
Ok(value) => return Ok(value),
Err(e) => {
if attempt + 1 == max_attempts {
return Err(e);
}
thread::sleep(delay);
delay = delay.checked_mul(2).unwrap_or(Duration::MAX);
}
}
}
unreachable!()
}
pub fn with_backoff_capped<T, E, F>(
max_attempts: u32,
initial_delay: Duration,
max_delay: Duration,
mut f: F,
) -> Result<T, E>
where
F: FnMut() -> Result<T, E>,
{
assert!(max_attempts > 0, "max_attempts must be greater than 0");
let mut delay = initial_delay.min(max_delay);
for attempt in 0..max_attempts {
match f() {
Ok(value) => return Ok(value),
Err(e) => {
if attempt + 1 == max_attempts {
return Err(e);
}
thread::sleep(delay);
delay = delay
.checked_mul(2)
.map(|d| d.min(max_delay))
.unwrap_or(max_delay);
}
}
}
unreachable!()
}
}
#[derive(Debug, Clone)]
pub struct RetryConfig {
pub max_attempts: u32,
pub initial_delay: Duration,
pub max_delay: Duration,
pub factor: f64,
pub jitter_ms: u64,
pub immediate_first: bool,
pub total_budget: Option<Duration>,
pub per_attempt_timeout: Option<Duration>,
}
impl RetryConfig {
pub fn defaults() -> Self {
Self {
max_attempts: 5,
initial_delay: Duration::from_millis(100),
max_delay: Duration::from_secs(10),
factor: 2.0,
jitter_ms: 25,
immediate_first: false,
total_budget: None,
per_attempt_timeout: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RetryError<E> {
Cancelled,
AttemptsExhausted {
attempts: u32,
last_error: Option<E>,
},
BudgetExhausted {
attempts: u32,
last_error: Option<E>,
},
}
impl<E: std::fmt::Display> std::fmt::Display for RetryError<E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Cancelled => write!(f, "retry cancelled"),
Self::AttemptsExhausted { attempts, last_error } => match last_error {
Some(e) => write!(f, "retry attempts exhausted ({attempts}): {e}"),
None => write!(f, "retry attempts exhausted ({attempts}): last attempt timed out"),
},
Self::BudgetExhausted { attempts, last_error } => match last_error {
Some(e) => write!(f, "retry budget exhausted ({attempts} attempts): {e}"),
None => write!(f, "retry budget exhausted ({attempts} attempts): last attempt timed out"),
},
}
}
}
impl<E: std::fmt::Display + std::fmt::Debug> std::error::Error for RetryError<E> {}
#[derive(Debug, Clone)]
pub struct RetryManager {
config: RetryConfig,
}
impl RetryManager {
pub fn new(config: RetryConfig) -> Result<Self, CorrectnessError> {
check_predicate_true(config.max_attempts >= 1, "max_attempts must be >= 1")?;
check_predicate_true(
config.per_attempt_timeout.map_or(true, |t| !t.is_zero()),
"per_attempt_timeout must be non-zero",
)?;
check_predicate_true(
config.total_budget.map_or(true, |t| !t.is_zero()),
"total_budget must be non-zero",
)?;
ExponentialBackoff::new(
config.initial_delay,
config.max_delay,
config.factor,
config.jitter_ms,
config.immediate_first,
)?;
Ok(Self { config })
}
pub async fn run<T, E, F, Fut>(&self, mut f: F) -> Result<T, RetryError<E>>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
{
self.run_impl(None, |_: &E| true, &mut f).await
}
pub async fn run_with_cancel<T, E, F, Fut>(
&self,
cancel: &CancellationToken,
mut f: F,
) -> Result<T, RetryError<E>>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
{
self.run_impl(Some(cancel), |_: &E| true, &mut f).await
}
pub async fn run_if<T, E, F, Fut, C>(&self, is_retryable: C, mut f: F) -> Result<T, RetryError<E>>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
C: Fn(&E) -> bool,
{
self.run_impl(None, is_retryable, &mut f).await
}
async fn run_impl<T, E, F, Fut, C>(
&self,
cancel: Option<&CancellationToken>,
is_retryable: C,
f: &mut F,
) -> Result<T, RetryError<E>>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
C: Fn(&E) -> bool,
{
let mut backoff = ExponentialBackoff::new(
self.config.initial_delay,
self.config.max_delay,
self.config.factor,
self.config.jitter_ms,
self.config.immediate_first,
)
.expect("RetryManager::new 已校验退避参数,此处不可失败");
let started = tokio::time::Instant::now();
let mut last_error: Option<E> = None;
let mut attempts: u32 = 0;
while attempts < self.config.max_attempts {
if let Some(c) = cancel {
if c.is_cancelled() {
return Err(RetryError::Cancelled);
}
}
if let Some(budget) = self.config.total_budget {
if started.elapsed() >= budget {
return Err(RetryError::BudgetExhausted {
attempts,
last_error,
});
}
}
attempts += 1;
let fut = f();
let result = match cancel {
Some(c) => {
let run_fut = maybe_timeout(self.config.per_attempt_timeout, fut);
tokio::select! {
biased;
_ = c.cancelled() => return Err(RetryError::Cancelled),
r = run_fut => r,
}
}
None => maybe_timeout(self.config.per_attempt_timeout, fut).await,
};
match result {
Some(Ok(value)) => return Ok(value),
Some(Err(e)) => {
if !is_retryable(&e) {
return Err(RetryError::AttemptsExhausted {
attempts,
last_error: Some(e),
});
}
last_error = Some(e);
}
None => last_error = None,
}
if attempts >= self.config.max_attempts {
break;
}
let delay = backoff.next_duration();
match cancel {
Some(c) => {
tokio::select! {
biased;
_ = c.cancelled() => return Err(RetryError::Cancelled),
() = tokio::time::sleep(delay) => {}
}
}
None => tokio::time::sleep(delay).await,
}
}
if let Some(budget) = self.config.total_budget {
if started.elapsed() >= budget {
return Err(RetryError::BudgetExhausted {
attempts,
last_error,
});
}
}
Err(RetryError::AttemptsExhausted {
attempts,
last_error,
})
}
}
async fn maybe_timeout<E, T>(
timeout: Option<Duration>,
fut: impl std::future::Future<Output = Result<T, E>>,
) -> Option<Result<T, E>> {
match timeout {
Some(t) => tokio::time::timeout(t, fut).await.ok(),
None => Some(fut.await),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
#[test]
fn test_run_success_first_try() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> = Retry::run(3, Duration::from_millis(1), move || {
counter_clone.fetch_add(1, Ordering::SeqCst);
Ok(42)
});
assert_eq!(result.unwrap(), 42);
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[test]
fn test_run_success_after_retries() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> = Retry::run(3, Duration::from_millis(1), move || {
let count = counter_clone.fetch_add(1, Ordering::SeqCst);
if count < 2 {
Err("not yet")
} else {
Ok(42)
}
});
assert_eq!(result.unwrap(), 42);
assert_eq!(counter.load(Ordering::SeqCst), 3);
}
#[test]
fn test_run_all_fail() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> = Retry::run(3, Duration::from_millis(1), move || {
counter_clone.fetch_add(1, Ordering::SeqCst);
Err("always fails")
});
assert_eq!(result.unwrap_err(), "always fails");
assert_eq!(counter.load(Ordering::SeqCst), 3);
}
#[test]
fn test_run_single_attempt() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> = Retry::run(1, Duration::from_millis(1), move || {
counter_clone.fetch_add(1, Ordering::SeqCst);
Err("fail")
});
assert_eq!(result.unwrap_err(), "fail");
assert_eq!(counter.load(Ordering::SeqCst), 1);
}
#[test]
fn test_with_backoff_success() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> =
Retry::with_backoff(3, Duration::from_millis(1), move || {
let count = counter_clone.fetch_add(1, Ordering::SeqCst);
if count < 1 {
Err("not yet")
} else {
Ok(42)
}
});
assert_eq!(result.unwrap(), 42);
assert_eq!(counter.load(Ordering::SeqCst), 2);
}
#[test]
fn test_with_backoff_all_fail() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> =
Retry::with_backoff(3, Duration::from_millis(1), move || {
counter_clone.fetch_add(1, Ordering::SeqCst);
Err("always fails")
});
assert_eq!(result.unwrap_err(), "always fails");
assert_eq!(counter.load(Ordering::SeqCst), 3);
}
#[test]
fn test_with_backoff_capped_success() {
let counter = Arc::new(AtomicU32::new(0));
let counter_clone = counter.clone();
let result: Result<i32, &str> = Retry::with_backoff_capped(
5,
Duration::from_millis(1),
Duration::from_millis(10),
move || {
let count = counter_clone.fetch_add(1, Ordering::SeqCst);
if count < 3 {
Err("not yet")
} else {
Ok(42)
}
},
);
assert_eq!(result.unwrap(), 42);
assert_eq!(counter.load(Ordering::SeqCst), 4);
}
#[test]
#[should_panic(expected = "max_attempts must be greater than 0")]
fn test_run_zero_attempts() {
let _ = Retry::run(0, Duration::from_millis(1), || Ok::<i32, &str>(42));
}
#[test]
#[should_panic(expected = "max_attempts must be greater than 0")]
fn test_with_backoff_zero_attempts() {
let _ = Retry::with_backoff(0, Duration::from_millis(1), || Ok::<i32, &str>(42));
}
#[test]
#[should_panic(expected = "max_attempts must be greater than 0")]
fn test_with_backoff_capped_zero_attempts() {
let _ = Retry::with_backoff_capped(
0,
Duration::from_millis(1),
Duration::from_millis(10),
|| Ok::<i32, &str>(42),
);
}
fn fast_cfg(max_attempts: u32) -> RetryConfig {
RetryConfig {
max_attempts,
initial_delay: Duration::from_millis(1),
max_delay: Duration::from_millis(5),
factor: 2.0,
jitter_ms: 0,
immediate_first: false,
total_budget: None,
per_attempt_timeout: None,
}
}
#[tokio::test]
async fn manager_success_first_try() {
let mgr = RetryManager::new(fast_cfg(3)).unwrap();
let n = mgr.run(|| async { Ok::<i32, &str>(1) }).await.unwrap();
assert_eq!(n, 1);
}
#[tokio::test]
async fn manager_success_after_retries() {
let mgr = RetryManager::new(fast_cfg(4)).unwrap();
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let n = mgr
.run(move || {
let c = c.clone();
async move {
let k = c.fetch_add(1, Ordering::SeqCst);
if k < 2 {
Err("no")
} else {
Ok(k)
}
}
})
.await
.unwrap();
assert_eq!(n, 2);
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn manager_attempts_exhausted() {
let mgr = RetryManager::new(fast_cfg(3)).unwrap();
let err = mgr
.run(|| async { Err::<i32, _>("boom") })
.await
.unwrap_err();
match err {
RetryError::AttemptsExhausted { attempts, last_error } => {
assert_eq!(attempts, 3);
assert_eq!(last_error, Some("boom"));
}
other => panic!("期望 AttemptsExhausted,实际 {other:?}"),
}
}
#[tokio::test]
async fn manager_config_validation() {
assert!(RetryManager::new(RetryConfig {
max_attempts: 0,
..fast_cfg(1)
})
.is_err());
assert!(RetryManager::new(RetryConfig {
initial_delay: Duration::ZERO,
..fast_cfg(1)
})
.is_err());
assert!(RetryManager::new(RetryConfig {
factor: 1000.0,
..fast_cfg(1)
})
.is_err());
}
#[tokio::test]
async fn manager_non_retryable_passthrough() {
let mgr = RetryManager::new(fast_cfg(5)).unwrap();
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let err = mgr
.run_if(
|e: &i32| *e != 7, move || {
let c = c.clone();
async move {
c.fetch_add(1, Ordering::SeqCst);
Err::<i32, _>(7)
}
},
)
.await
.unwrap_err();
match err {
RetryError::AttemptsExhausted { attempts, .. } => assert_eq!(attempts, 1),
other => panic!("期望 AttemptsExhausted,实际 {other:?}"),
}
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn manager_budget_exhausted() {
let mgr = RetryManager::new(RetryConfig {
max_attempts: 100,
initial_delay: Duration::from_secs(1),
max_delay: Duration::from_secs(1),
total_budget: Some(Duration::from_secs(3)),
..fast_cfg(1)
})
.unwrap();
let err = mgr
.run(|| async { Err::<i32, _>("boom") })
.await
.unwrap_err();
match err {
RetryError::BudgetExhausted { attempts, .. } => {
assert_eq!(attempts, 3);
}
other => panic!("期望 BudgetExhausted,实际 {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn manager_attempt_timeout_retries() {
let mgr = RetryManager::new(RetryConfig {
max_attempts: 2,
per_attempt_timeout: Some(Duration::from_millis(10)),
..fast_cfg(1)
})
.unwrap();
let err = mgr
.run(|| async {
tokio::time::sleep(Duration::from_secs(60)).await; Ok::<i32, &str>(1)
})
.await
.unwrap_err();
match err {
RetryError::AttemptsExhausted {
attempts,
last_error,
} => {
assert_eq!(attempts, 2);
assert!(last_error.is_none(), "超时无业务错误可携带");
}
other => panic!("期望 AttemptsExhausted,实际 {other:?}"),
}
}
#[tokio::test]
async fn manager_cancel_during_attempt() {
let mgr = RetryManager::new(fast_cfg(5)).unwrap();
let cancel = CancellationToken::new();
let c2 = cancel.clone();
let err = mgr
.run_with_cancel(&cancel, move || {
let c2 = c2.clone();
async move {
c2.cancel(); tokio::time::sleep(Duration::from_millis(10)).await;
Err::<i32, _>("boom")
}
})
.await
.unwrap_err();
assert!(matches!(err, RetryError::Cancelled));
}
#[tokio::test(start_paused = true)]
async fn manager_cancel_during_backoff() {
let mgr = RetryManager::new(RetryConfig {
initial_delay: Duration::from_millis(10),
max_delay: Duration::from_millis(10),
..fast_cfg(5)
})
.unwrap();
let cancel = CancellationToken::new();
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let c2 = cancel.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(2)).await;
c2.cancel();
});
let err = mgr
.run_with_cancel(&cancel, move || {
let c = c.clone();
async move {
c.fetch_add(1, Ordering::SeqCst);
Err::<i32, _>("boom")
}
})
.await
.unwrap_err();
assert!(matches!(err, RetryError::Cancelled));
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
}