use std::future::Future;
use std::time::Duration;
use taskfast_client::{Error, Result};
#[derive(Debug, Clone, Copy)]
pub struct BackoffOptions {
pub max_attempts: u32,
pub base_delay: Duration,
pub max_delay: Option<Duration>,
}
impl Default for BackoffOptions {
fn default() -> Self {
Self {
max_attempts: 5,
base_delay: Duration::from_millis(500),
max_delay: Some(Duration::from_secs(30)),
}
}
}
pub async fn with_backoff<T, F, Fut>(opts: BackoffOptions, mut op: F) -> Result<T>
where
F: FnMut(u32) -> Fut,
Fut: Future<Output = Result<T>>,
{
assert!(opts.max_attempts >= 1, "max_attempts must be at least 1");
let mut attempt: u32 = 1;
loop {
match op(attempt).await {
Ok(v) => return Ok(v),
Err(err) => {
if attempt >= opts.max_attempts {
return Err(err);
}
let delay = delay_for(&err, attempt, &opts);
tracing::debug!(attempt, ?delay, kind = err.kind(), "retrying after error");
tokio::time::sleep(delay).await;
attempt += 1;
}
}
}
}
fn delay_for(err: &Error, attempt: u32, opts: &BackoffOptions) -> Duration {
if let Error::RateLimited { retry_after } = err {
return *retry_after;
}
let shift = (attempt - 1).min(20);
let multiplier = 1u32 << shift;
let raw = opts.base_delay.saturating_mul(multiplier);
match opts.max_delay {
Some(cap) if raw > cap => cap,
_ => raw,
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
fn tiny_opts(max_attempts: u32) -> BackoffOptions {
BackoffOptions {
max_attempts,
base_delay: Duration::from_millis(10),
max_delay: Some(Duration::from_secs(1)),
}
}
#[tokio::test(start_paused = true)]
async fn succeeds_on_first_try_without_sleeping() {
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let out: Result<u32> = with_backoff(tiny_opts(3), move |_| {
let c = c.clone();
async move {
c.fetch_add(1, Ordering::SeqCst);
Ok(42u32)
}
})
.await;
assert_eq!(out.unwrap(), 42);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn retries_transient_errors_then_succeeds() {
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let out: Result<&'static str> = with_backoff(tiny_opts(5), move |attempt| {
let c = c.clone();
async move {
c.fetch_add(1, Ordering::SeqCst);
if attempt < 3 {
Err(Error::Server("boom".into()))
} else {
Ok("ok")
}
}
})
.await;
assert_eq!(out.unwrap(), "ok");
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test(start_paused = true)]
async fn returns_last_error_when_attempts_exhausted() {
let out: Result<()> = with_backoff(tiny_opts(2), |_| async move {
Err::<(), _>(Error::Server("persistent".into()))
})
.await;
match out {
Err(Error::Server(msg)) => assert_eq!(msg, "persistent"),
other => panic!("expected Server error, got {other:?}"),
}
}
#[tokio::test(start_paused = true)]
async fn rate_limited_honors_retry_after_exactly() {
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let out: Result<u8> = with_backoff(tiny_opts(3), move |_attempt| {
let c = c.clone();
async move {
let n = c.fetch_add(1, Ordering::SeqCst);
if n == 0 {
Err(Error::RateLimited {
retry_after: Duration::from_secs(7),
})
} else {
Ok(1u8)
}
}
})
.await;
assert_eq!(out.unwrap(), 1);
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test(start_paused = true)]
async fn non_retryable_errors_still_retry_at_agent_layer() {
let calls = Arc::new(AtomicU32::new(0));
let c = calls.clone();
let out: Result<()> = with_backoff(tiny_opts(3), move |_| {
let c = c.clone();
async move {
c.fetch_add(1, Ordering::SeqCst);
Err::<(), _>(Error::Validation {
code: "x".into(),
message: "y".into(),
})
}
})
.await;
assert!(out.is_err());
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[test]
fn delay_for_rate_limited_returns_retry_after_verbatim() {
let opts = BackoffOptions {
max_attempts: 5,
base_delay: Duration::from_secs(1),
max_delay: Some(Duration::from_secs(5)),
};
let err = Error::RateLimited {
retry_after: Duration::from_secs(12),
};
assert_eq!(delay_for(&err, 1, &opts), Duration::from_secs(12));
}
#[test]
fn delay_for_exponential_grows_then_caps() {
let opts = BackoffOptions {
max_attempts: 10,
base_delay: Duration::from_millis(100),
max_delay: Some(Duration::from_secs(1)),
};
let err = Error::Server("boom".into());
assert_eq!(delay_for(&err, 1, &opts), Duration::from_millis(100));
assert_eq!(delay_for(&err, 2, &opts), Duration::from_millis(200));
assert_eq!(delay_for(&err, 3, &opts), Duration::from_millis(400));
assert_eq!(delay_for(&err, 4, &opts), Duration::from_millis(800));
assert_eq!(delay_for(&err, 5, &opts), Duration::from_secs(1));
assert_eq!(delay_for(&err, 20, &opts), Duration::from_secs(1));
}
#[test]
fn delay_for_uncapped_is_uncapped() {
let opts = BackoffOptions {
max_attempts: 10,
base_delay: Duration::from_millis(100),
max_delay: None,
};
let err = Error::Server("unused".into());
assert_eq!(delay_for(&err, 5, &opts), Duration::from_millis(1600));
}
#[tokio::test(start_paused = true)]
async fn attempt_number_is_passed_to_closure() {
let seen = Arc::new(Mutex::new(Vec::<u32>::new()));
let s = seen.clone();
let _: Result<()> = with_backoff(tiny_opts(3), move |attempt| {
let s = s.clone();
async move {
s.lock().unwrap().push(attempt);
Err::<(), _>(Error::Server("x".into()))
}
})
.await;
assert_eq!(*seen.lock().unwrap(), vec![1, 2, 3]);
}
}