use std::time::Duration;
use crate::error::{CortexError, CortexResult};
#[derive(Debug, Clone)]
pub enum RetryPolicy {
None,
Backoff {
max_retries: u32,
base_delay: Duration,
max_delay: Duration,
},
}
impl RetryPolicy {
#[must_use]
pub fn none() -> Self {
Self::None
}
#[must_use]
pub fn query() -> Self {
Self::Backoff {
max_retries: 3,
base_delay: Duration::from_millis(500),
max_delay: Duration::from_secs(10),
}
}
#[must_use]
pub fn idempotent() -> Self {
Self::Backoff {
max_retries: 2,
base_delay: Duration::from_secs(1),
max_delay: Duration::from_secs(15),
}
}
#[must_use]
pub fn stop() -> Self {
Self::Backoff {
max_retries: 2,
base_delay: Duration::from_secs(1),
max_delay: Duration::from_secs(15),
}
}
#[must_use]
pub fn custom(max_retries: u32, base_delay: Duration, max_delay: Duration) -> Self {
Self::Backoff {
max_retries,
base_delay,
max_delay,
}
}
}
pub async fn with_retry<F, Fut, T>(policy: &RetryPolicy, mut operation: F) -> CortexResult<T>
where
F: FnMut() -> Fut,
Fut: std::future::Future<Output = CortexResult<T>>,
{
match policy {
RetryPolicy::None => operation().await,
RetryPolicy::Backoff {
max_retries,
base_delay,
max_delay,
} => {
let mut delay = *base_delay;
for attempt in 0..=*max_retries {
match operation().await {
Ok(result) => return Ok(result),
Err(e) => {
if !e.is_retryable() {
return Err(e);
}
if attempt == *max_retries {
return Err(CortexError::RetriesExhausted {
attempts: attempt + 1,
last_error: Box::new(e),
});
}
tracing::warn!(
attempt = attempt + 1,
max = max_retries + 1,
error = %e,
delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
"Retrying after transient error"
);
tokio::time::sleep(delay).await;
delay = std::cmp::min(delay * 2, *max_delay);
}
}
}
operation().await
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, Ordering};
#[tokio::test]
async fn test_no_retry_succeeds() {
let result = with_retry(&RetryPolicy::none(), || async { Ok::<_, CortexError>(42) }).await;
assert_eq!(result.unwrap(), 42);
}
#[tokio::test]
async fn test_no_retry_fails_immediately() {
let result = with_retry(&RetryPolicy::none(), || async {
Err::<i32, _>(CortexError::Timeout { seconds: 5 })
})
.await;
assert!(matches!(result.unwrap_err(), CortexError::Timeout { .. }));
}
#[tokio::test]
async fn test_retry_succeeds_after_transient_failure() {
let attempts = AtomicU32::new(0);
let result = with_retry(
&RetryPolicy::custom(3, Duration::from_millis(1), Duration::from_millis(10)),
|| {
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
async move {
if attempt < 2 {
Err(CortexError::Timeout { seconds: 1 })
} else {
Ok(42)
}
}
},
)
.await;
assert_eq!(result.unwrap(), 42);
assert_eq!(attempts.load(Ordering::SeqCst), 3); }
#[tokio::test]
async fn test_retry_exhausted() {
let attempts = AtomicU32::new(0);
let result = with_retry(
&RetryPolicy::custom(2, Duration::from_millis(1), Duration::from_millis(10)),
|| {
attempts.fetch_add(1, Ordering::SeqCst);
async { Err::<i32, _>(CortexError::Timeout { seconds: 1 }) }
},
)
.await;
match result.unwrap_err() {
CortexError::RetriesExhausted { attempts: a, .. } => assert_eq!(a, 3),
other => panic!("Expected RetriesExhausted, got {other:?}"),
}
assert_eq!(attempts.load(Ordering::SeqCst), 3); }
#[tokio::test]
async fn test_retry_exhausted_preserves_last_error() {
let result = with_retry(
&RetryPolicy::custom(1, Duration::from_millis(1), Duration::from_millis(10)),
|| async { Err::<i32, _>(CortexError::Timeout { seconds: 5 }) },
)
.await;
let err = result.unwrap_err();
let CortexError::RetriesExhausted {
attempts,
last_error,
} = &err
else {
panic!("expected RetriesExhausted, got {err:?}");
};
assert_eq!(*attempts, 2); assert!(matches!(
last_error.as_ref(),
CortexError::Timeout { seconds: 5 }
));
assert!(last_error.is_retryable());
}
#[tokio::test]
async fn test_non_retryable_error_not_retried() {
let attempts = AtomicU32::new(0);
let result = with_retry(
&RetryPolicy::custom(3, Duration::from_millis(1), Duration::from_millis(10)),
|| {
attempts.fetch_add(1, Ordering::SeqCst);
async { Err::<i32, _>(CortexError::NoHeadsetFound) }
},
)
.await;
assert!(matches!(result.unwrap_err(), CortexError::NoHeadsetFound));
assert_eq!(attempts.load(Ordering::SeqCst), 1); }
#[tokio::test]
async fn test_backoff_policy_succeeds_on_first_try() {
let result = with_retry(&RetryPolicy::query(), || async { Ok::<_, CortexError>(99) }).await;
assert_eq!(result.unwrap(), 99);
}
#[test]
fn test_policy_constructor_defaults() {
match RetryPolicy::query() {
RetryPolicy::Backoff {
max_retries,
base_delay,
max_delay,
} => {
assert_eq!(max_retries, 3);
assert_eq!(base_delay, Duration::from_millis(500));
assert_eq!(max_delay, Duration::from_secs(10));
}
RetryPolicy::None => panic!("query policy should use backoff"),
}
match RetryPolicy::idempotent() {
RetryPolicy::Backoff {
max_retries,
base_delay,
max_delay,
} => {
assert_eq!(max_retries, 2);
assert_eq!(base_delay, Duration::from_secs(1));
assert_eq!(max_delay, Duration::from_secs(15));
}
RetryPolicy::None => panic!("idempotent policy should use backoff"),
}
match RetryPolicy::stop() {
RetryPolicy::Backoff {
max_retries,
base_delay,
max_delay,
} => {
assert_eq!(max_retries, 2);
assert_eq!(base_delay, Duration::from_secs(1));
assert_eq!(max_delay, Duration::from_secs(15));
}
RetryPolicy::None => panic!("stop policy should use backoff"),
}
}
#[tokio::test]
async fn test_backoff_delay_caps_at_max_delay() {
let attempts = AtomicU32::new(0);
let start = std::time::Instant::now();
let result = with_retry(
&RetryPolicy::custom(3, Duration::from_millis(1), Duration::from_millis(2)),
|| {
attempts.fetch_add(1, Ordering::SeqCst);
async { Err::<(), _>(CortexError::Timeout { seconds: 1 }) }
},
)
.await;
assert!(matches!(
result.unwrap_err(),
CortexError::RetriesExhausted { .. }
));
assert_eq!(attempts.load(Ordering::SeqCst), 4); assert!(
start.elapsed() >= Duration::from_millis(4),
"elapsed {:?} was too short for capped backoff",
start.elapsed()
);
}
}