Skip to main content

taskfast_agent/
retry.rs

1// SPDX-License-Identifier: MIT
2//! Operation-level retry loops for business-layer flows (balance polling,
3//! faucet waits, bid submission races, etc.).
4//!
5//! # Not to be confused with [`taskfast_client::retry`]
6//!
7//! The client's retry is scoped to HTTP transport: it retries only
8//! `Server`/`Network`/`RateLimited` errors and exists to smooth over flaky
9//! endpoints. This one wraps **arbitrary** async operations and retries on
10//! **any** error. The policies differ in kind, not degree:
11//!
12//! - HTTP retry = "the request didn't land, try again"
13//! - Agent retry = "the goal wasn't met yet (e.g., balance still zero), try
14//!   again — regardless of why"
15//!
16//! # RateLimited special-case
17//!
18//! If the returned error is [`Error::RateLimited`] with a `retry_after`, the
19//! loop sleeps that exact duration instead of the next exponential step.
20//! This matches the semantic of the platform's 429 handler: the server told
21//! us *when* to come back, so we respect it.
22
23use std::future::Future;
24use std::time::Duration;
25
26use taskfast_client::{Error, Result};
27
28/// Knobs for [`with_backoff`]. Defaults tuned for the worker hot loop:
29/// 5 attempts over ~15 s of exponential wait (0.5 → 1 → 2 → 4 → 8 s) is
30/// usually enough for transient hiccups without wedging an event-handler
31/// on a genuine outage.
32#[derive(Debug, Clone, Copy)]
33pub struct BackoffOptions {
34    pub max_attempts: u32,
35    pub base_delay: Duration,
36    /// Upper bound on a single sleep. `None` = no cap. Cap applies to the
37    /// exponential branch only — `retry_after` from a server is honored
38    /// verbatim (the server is authoritative on its own quota).
39    pub max_delay: Option<Duration>,
40}
41
42impl Default for BackoffOptions {
43    fn default() -> Self {
44        Self {
45            max_attempts: 5,
46            base_delay: Duration::from_millis(500),
47            max_delay: Some(Duration::from_secs(30)),
48        }
49    }
50}
51
52/// Run `op` until it succeeds or `max_attempts` is exhausted.
53///
54/// - Success (`Ok(_)`): return immediately.
55/// - [`Error::RateLimited`]: sleep `retry_after`, then retry.
56/// - Any other error: sleep `base_delay * 2^(attempt-1)` (capped at
57///   `max_delay`), then retry.
58/// - Last attempt's error: returned as-is; loop does not swallow it.
59///
60/// `op` receives the 1-indexed attempt number, useful for logging or for
61/// slightly varying the request between tries (e.g., probing a different
62/// endpoint on attempt ≥ 2).
63pub async fn with_backoff<T, F, Fut>(opts: BackoffOptions, mut op: F) -> Result<T>
64where
65    F: FnMut(u32) -> Fut,
66    Fut: Future<Output = Result<T>>,
67{
68    assert!(opts.max_attempts >= 1, "max_attempts must be at least 1");
69    let mut attempt: u32 = 1;
70    loop {
71        match op(attempt).await {
72            Ok(v) => return Ok(v),
73            Err(err) => {
74                if attempt >= opts.max_attempts {
75                    return Err(err);
76                }
77                let delay = delay_for(&err, attempt, &opts);
78                // F6: log only the variant tag, never the inner message.
79                // If a server ever echoes the X-API-Key header into an
80                // error body, the full Display / Debug would carry that
81                // secret into every log sink downstream.
82                tracing::debug!(attempt, ?delay, kind = err.kind(), "retrying after error");
83                tokio::time::sleep(delay).await;
84                attempt += 1;
85            }
86        }
87    }
88}
89
90fn delay_for(err: &Error, attempt: u32, opts: &BackoffOptions) -> Duration {
91    if let Error::RateLimited { retry_after } = err {
92        return *retry_after;
93    }
94    // Exponential: base * 2^(attempt-1), guarding the shift against
95    // overflow on absurdly large attempt counts.
96    let shift = (attempt - 1).min(20);
97    let multiplier = 1u32 << shift;
98    let raw = opts.base_delay.saturating_mul(multiplier);
99    match opts.max_delay {
100        Some(cap) if raw > cap => cap,
101        _ => raw,
102    }
103}
104
105#[cfg(test)]
106mod tests {
107    use super::*;
108    use std::sync::atomic::{AtomicU32, Ordering};
109    use std::sync::{Arc, Mutex};
110
111    fn tiny_opts(max_attempts: u32) -> BackoffOptions {
112        BackoffOptions {
113            max_attempts,
114            base_delay: Duration::from_millis(10),
115            max_delay: Some(Duration::from_secs(1)),
116        }
117    }
118
119    #[tokio::test(start_paused = true)]
120    async fn succeeds_on_first_try_without_sleeping() {
121        let calls = Arc::new(AtomicU32::new(0));
122        let c = calls.clone();
123        let out: Result<u32> = with_backoff(tiny_opts(3), move |_| {
124            let c = c.clone();
125            async move {
126                c.fetch_add(1, Ordering::SeqCst);
127                Ok(42u32)
128            }
129        })
130        .await;
131        assert_eq!(out.unwrap(), 42);
132        assert_eq!(calls.load(Ordering::SeqCst), 1);
133    }
134
135    #[tokio::test(start_paused = true)]
136    async fn retries_transient_errors_then_succeeds() {
137        // Fail twice with Server, then succeed.
138        let calls = Arc::new(AtomicU32::new(0));
139        let c = calls.clone();
140        let out: Result<&'static str> = with_backoff(tiny_opts(5), move |attempt| {
141            let c = c.clone();
142            async move {
143                c.fetch_add(1, Ordering::SeqCst);
144                if attempt < 3 {
145                    Err(Error::Server("boom".into()))
146                } else {
147                    Ok("ok")
148                }
149            }
150        })
151        .await;
152        assert_eq!(out.unwrap(), "ok");
153        assert_eq!(calls.load(Ordering::SeqCst), 3);
154    }
155
156    #[tokio::test(start_paused = true)]
157    async fn returns_last_error_when_attempts_exhausted() {
158        let out: Result<()> = with_backoff(tiny_opts(2), |_| async move {
159            Err::<(), _>(Error::Server("persistent".into()))
160        })
161        .await;
162        match out {
163            Err(Error::Server(msg)) => assert_eq!(msg, "persistent"),
164            other => panic!("expected Server error, got {other:?}"),
165        }
166    }
167
168    #[tokio::test(start_paused = true)]
169    async fn rate_limited_honors_retry_after_exactly() {
170        // First attempt: RateLimited(retry_after=7s). Second attempt: ok.
171        // Paused runtime advances instantly when sleep is awaited, but we
172        // assert the attempts sequence (1 → sleep → 2) was followed.
173        let calls = Arc::new(AtomicU32::new(0));
174        let c = calls.clone();
175        let out: Result<u8> = with_backoff(tiny_opts(3), move |_attempt| {
176            let c = c.clone();
177            async move {
178                let n = c.fetch_add(1, Ordering::SeqCst);
179                if n == 0 {
180                    Err(Error::RateLimited {
181                        retry_after: Duration::from_secs(7),
182                    })
183                } else {
184                    Ok(1u8)
185                }
186            }
187        })
188        .await;
189        assert_eq!(out.unwrap(), 1);
190        assert_eq!(calls.load(Ordering::SeqCst), 2);
191    }
192
193    #[tokio::test(start_paused = true)]
194    async fn non_retryable_errors_still_retry_at_agent_layer() {
195        // Agent retry is "retry on anything" — unlike client retry, even
196        // Validation gets retried (until attempts exhaust). This is the
197        // caller's explicit choice: they wrapped the op because *they*
198        // decided the operation was idempotent + worth re-trying.
199        let calls = Arc::new(AtomicU32::new(0));
200        let c = calls.clone();
201        let out: Result<()> = with_backoff(tiny_opts(3), move |_| {
202            let c = c.clone();
203            async move {
204                c.fetch_add(1, Ordering::SeqCst);
205                Err::<(), _>(Error::Validation {
206                    code: "x".into(),
207                    message: "y".into(),
208                })
209            }
210        })
211        .await;
212        assert!(out.is_err());
213        assert_eq!(calls.load(Ordering::SeqCst), 3);
214    }
215
216    #[test]
217    fn delay_for_rate_limited_returns_retry_after_verbatim() {
218        let opts = BackoffOptions {
219            max_attempts: 5,
220            base_delay: Duration::from_secs(1),
221            max_delay: Some(Duration::from_secs(5)),
222        };
223        let err = Error::RateLimited {
224            retry_after: Duration::from_secs(12),
225        };
226        // max_delay cap does NOT apply to server-directed waits.
227        assert_eq!(delay_for(&err, 1, &opts), Duration::from_secs(12));
228    }
229
230    #[test]
231    fn delay_for_exponential_grows_then_caps() {
232        let opts = BackoffOptions {
233            max_attempts: 10,
234            base_delay: Duration::from_millis(100),
235            max_delay: Some(Duration::from_secs(1)),
236        };
237        let err = Error::Server("boom".into());
238        assert_eq!(delay_for(&err, 1, &opts), Duration::from_millis(100));
239        assert_eq!(delay_for(&err, 2, &opts), Duration::from_millis(200));
240        assert_eq!(delay_for(&err, 3, &opts), Duration::from_millis(400));
241        assert_eq!(delay_for(&err, 4, &opts), Duration::from_millis(800));
242        // attempt 5 would be 1600ms > 1s cap.
243        assert_eq!(delay_for(&err, 5, &opts), Duration::from_secs(1));
244        // attempt 20 still capped — overflow guarded.
245        assert_eq!(delay_for(&err, 20, &opts), Duration::from_secs(1));
246    }
247
248    #[test]
249    fn delay_for_uncapped_is_uncapped() {
250        let opts = BackoffOptions {
251            max_attempts: 10,
252            base_delay: Duration::from_millis(100),
253            max_delay: None,
254        };
255        // Any non-RateLimited error takes the exponential branch; Server
256        // keeps the test free of reqwest-error construction boilerplate.
257        let err = Error::Server("unused".into());
258        assert_eq!(delay_for(&err, 5, &opts), Duration::from_millis(1600));
259    }
260
261    #[tokio::test(start_paused = true)]
262    async fn attempt_number_is_passed_to_closure() {
263        let seen = Arc::new(Mutex::new(Vec::<u32>::new()));
264        let s = seen.clone();
265        let _: Result<()> = with_backoff(tiny_opts(3), move |attempt| {
266            let s = s.clone();
267            async move {
268                s.lock().unwrap().push(attempt);
269                Err::<(), _>(Error::Server("x".into()))
270            }
271        })
272        .await;
273        assert_eq!(*seen.lock().unwrap(), vec![1, 2, 3]);
274    }
275}