1use std::future::Future;
24use std::time::Duration;
25
26use taskfast_client::{Error, Result};
27
28#[derive(Debug, Clone, Copy)]
33pub struct BackoffOptions {
34 pub max_attempts: u32,
35 pub base_delay: Duration,
36 pub max_delay: Option<Duration>,
40}
41
42impl Default for BackoffOptions {
43 fn default() -> Self {
44 Self {
45 max_attempts: 5,
46 base_delay: Duration::from_millis(500),
47 max_delay: Some(Duration::from_secs(30)),
48 }
49 }
50}
51
52pub async fn with_backoff<T, F, Fut>(opts: BackoffOptions, mut op: F) -> Result<T>
64where
65 F: FnMut(u32) -> Fut,
66 Fut: Future<Output = Result<T>>,
67{
68 assert!(opts.max_attempts >= 1, "max_attempts must be at least 1");
69 let mut attempt: u32 = 1;
70 loop {
71 match op(attempt).await {
72 Ok(v) => return Ok(v),
73 Err(err) => {
74 if attempt >= opts.max_attempts {
75 return Err(err);
76 }
77 let delay = delay_for(&err, attempt, &opts);
78 tracing::debug!(attempt, ?delay, kind = err.kind(), "retrying after error");
83 tokio::time::sleep(delay).await;
84 attempt += 1;
85 }
86 }
87 }
88}
89
90fn delay_for(err: &Error, attempt: u32, opts: &BackoffOptions) -> Duration {
91 if let Error::RateLimited { retry_after } = err {
92 return *retry_after;
93 }
94 let shift = (attempt - 1).min(20);
97 let multiplier = 1u32 << shift;
98 let raw = opts.base_delay.saturating_mul(multiplier);
99 match opts.max_delay {
100 Some(cap) if raw > cap => cap,
101 _ => raw,
102 }
103}
104
105#[cfg(test)]
106mod tests {
107 use super::*;
108 use std::sync::atomic::{AtomicU32, Ordering};
109 use std::sync::{Arc, Mutex};
110
111 fn tiny_opts(max_attempts: u32) -> BackoffOptions {
112 BackoffOptions {
113 max_attempts,
114 base_delay: Duration::from_millis(10),
115 max_delay: Some(Duration::from_secs(1)),
116 }
117 }
118
119 #[tokio::test(start_paused = true)]
120 async fn succeeds_on_first_try_without_sleeping() {
121 let calls = Arc::new(AtomicU32::new(0));
122 let c = calls.clone();
123 let out: Result<u32> = with_backoff(tiny_opts(3), move |_| {
124 let c = c.clone();
125 async move {
126 c.fetch_add(1, Ordering::SeqCst);
127 Ok(42u32)
128 }
129 })
130 .await;
131 assert_eq!(out.unwrap(), 42);
132 assert_eq!(calls.load(Ordering::SeqCst), 1);
133 }
134
135 #[tokio::test(start_paused = true)]
136 async fn retries_transient_errors_then_succeeds() {
137 let calls = Arc::new(AtomicU32::new(0));
139 let c = calls.clone();
140 let out: Result<&'static str> = with_backoff(tiny_opts(5), move |attempt| {
141 let c = c.clone();
142 async move {
143 c.fetch_add(1, Ordering::SeqCst);
144 if attempt < 3 {
145 Err(Error::Server("boom".into()))
146 } else {
147 Ok("ok")
148 }
149 }
150 })
151 .await;
152 assert_eq!(out.unwrap(), "ok");
153 assert_eq!(calls.load(Ordering::SeqCst), 3);
154 }
155
156 #[tokio::test(start_paused = true)]
157 async fn returns_last_error_when_attempts_exhausted() {
158 let out: Result<()> = with_backoff(tiny_opts(2), |_| async move {
159 Err::<(), _>(Error::Server("persistent".into()))
160 })
161 .await;
162 match out {
163 Err(Error::Server(msg)) => assert_eq!(msg, "persistent"),
164 other => panic!("expected Server error, got {other:?}"),
165 }
166 }
167
168 #[tokio::test(start_paused = true)]
169 async fn rate_limited_honors_retry_after_exactly() {
170 let calls = Arc::new(AtomicU32::new(0));
174 let c = calls.clone();
175 let out: Result<u8> = with_backoff(tiny_opts(3), move |_attempt| {
176 let c = c.clone();
177 async move {
178 let n = c.fetch_add(1, Ordering::SeqCst);
179 if n == 0 {
180 Err(Error::RateLimited {
181 retry_after: Duration::from_secs(7),
182 })
183 } else {
184 Ok(1u8)
185 }
186 }
187 })
188 .await;
189 assert_eq!(out.unwrap(), 1);
190 assert_eq!(calls.load(Ordering::SeqCst), 2);
191 }
192
193 #[tokio::test(start_paused = true)]
194 async fn non_retryable_errors_still_retry_at_agent_layer() {
195 let calls = Arc::new(AtomicU32::new(0));
200 let c = calls.clone();
201 let out: Result<()> = with_backoff(tiny_opts(3), move |_| {
202 let c = c.clone();
203 async move {
204 c.fetch_add(1, Ordering::SeqCst);
205 Err::<(), _>(Error::Validation {
206 code: "x".into(),
207 message: "y".into(),
208 })
209 }
210 })
211 .await;
212 assert!(out.is_err());
213 assert_eq!(calls.load(Ordering::SeqCst), 3);
214 }
215
216 #[test]
217 fn delay_for_rate_limited_returns_retry_after_verbatim() {
218 let opts = BackoffOptions {
219 max_attempts: 5,
220 base_delay: Duration::from_secs(1),
221 max_delay: Some(Duration::from_secs(5)),
222 };
223 let err = Error::RateLimited {
224 retry_after: Duration::from_secs(12),
225 };
226 assert_eq!(delay_for(&err, 1, &opts), Duration::from_secs(12));
228 }
229
230 #[test]
231 fn delay_for_exponential_grows_then_caps() {
232 let opts = BackoffOptions {
233 max_attempts: 10,
234 base_delay: Duration::from_millis(100),
235 max_delay: Some(Duration::from_secs(1)),
236 };
237 let err = Error::Server("boom".into());
238 assert_eq!(delay_for(&err, 1, &opts), Duration::from_millis(100));
239 assert_eq!(delay_for(&err, 2, &opts), Duration::from_millis(200));
240 assert_eq!(delay_for(&err, 3, &opts), Duration::from_millis(400));
241 assert_eq!(delay_for(&err, 4, &opts), Duration::from_millis(800));
242 assert_eq!(delay_for(&err, 5, &opts), Duration::from_secs(1));
244 assert_eq!(delay_for(&err, 20, &opts), Duration::from_secs(1));
246 }
247
248 #[test]
249 fn delay_for_uncapped_is_uncapped() {
250 let opts = BackoffOptions {
251 max_attempts: 10,
252 base_delay: Duration::from_millis(100),
253 max_delay: None,
254 };
255 let err = Error::Server("unused".into());
258 assert_eq!(delay_for(&err, 5, &opts), Duration::from_millis(1600));
259 }
260
261 #[tokio::test(start_paused = true)]
262 async fn attempt_number_is_passed_to_closure() {
263 let seen = Arc::new(Mutex::new(Vec::<u32>::new()));
264 let s = seen.clone();
265 let _: Result<()> = with_backoff(tiny_opts(3), move |attempt| {
266 let s = s.clone();
267 async move {
268 s.lock().unwrap().push(attempt);
269 Err::<(), _>(Error::Server("x".into()))
270 }
271 })
272 .await;
273 assert_eq!(*seen.lock().unwrap(), vec![1, 2, 3]);
274 }
275}