1use faucet_core::FaucetError;
11use faucet_core::io::CallContext;
12use schemars::JsonSchema;
13use serde::{Deserialize, Serialize};
14use std::future::Future;
15use std::time::Duration;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum ErrorClass {
20 Throttle,
22 Transient,
24 Fatal,
26}
27
28pub fn classify_error(code: Option<&str>) -> ErrorClass {
31 match code {
32 None => ErrorClass::Transient,
33 Some(
34 "ProvisionedThroughputExceededException"
35 | "ThrottlingException"
36 | "Throttling"
37 | "RequestLimitExceeded"
38 | "LimitExceededException",
39 ) => ErrorClass::Throttle,
40 Some(
41 "InternalServerError"
42 | "InternalFailure"
43 | "ServiceUnavailable"
44 | "ServiceUnavailableException"
45 | "TransactionInProgressException",
46 ) => ErrorClass::Transient,
47 Some(_) => ErrorClass::Fatal,
48 }
49}
50
51pub fn sdk_error_parts<E, R>(
53 err: &aws_sdk_dynamodb::error::SdkError<E, R>,
54) -> (Option<String>, String)
55where
56 E: aws_sdk_dynamodb::error::ProvideErrorMetadata + std::error::Error + 'static,
57 R: std::fmt::Debug,
58{
59 use aws_sdk_dynamodb::error::ProvideErrorMetadata as _;
60 let code = err.code().map(str::to_string);
61 let message = aws_sdk_dynamodb::error::DisplayErrorContext(err).to_string();
62 (code, message)
63}
64
65#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
67#[serde(deny_unknown_fields)]
68pub struct RetryPolicy {
69 #[serde(default = "default_max_retries")]
72 pub max_retries: u32,
73 #[serde(default = "default_base_ms")]
76 pub initial_backoff_ms: u64,
77 #[serde(default = "default_max_ms")]
79 pub max_backoff_ms: u64,
80}
81
82fn default_max_retries() -> u32 {
83 8
84}
85fn default_base_ms() -> u64 {
86 100
87}
88fn default_max_ms() -> u64 {
89 10_000
90}
91
92impl Default for RetryPolicy {
93 fn default() -> Self {
94 Self {
95 max_retries: default_max_retries(),
96 initial_backoff_ms: default_base_ms(),
97 max_backoff_ms: default_max_ms(),
98 }
99 }
100}
101
102impl RetryPolicy {
103 pub fn delay(&self, attempt: u32) -> Duration {
105 let exp = self
106 .initial_backoff_ms
107 .saturating_mul(1u64 << attempt.min(20))
108 .min(self.max_backoff_ms.max(1));
109 faucet_core::retry::apply_jitter(Duration::from_millis(exp.max(1)))
110 }
111
112 pub async fn run<T, F, Fut>(&self, what: &str, op: F) -> Result<T, String>
116 where
117 F: FnMut() -> Fut,
118 Fut: Future<Output = Result<T, (Option<String>, String)>>,
119 {
120 let ctx = CallContext::new().without_timeout();
121 self.attempts(&ctx, "request", what, op)
122 .await
123 .map_err(|e| match e {
124 Ok(message) => message,
125 Err(spent) => format!("dynamodb: {what} failed: {spent}"),
126 })
127 }
128
129 pub async fn run_in<T, F, Fut>(
135 &self,
136 ctx: &CallContext,
137 call: &'static str,
138 what: &str,
139 fail: fn(String) -> FaucetError,
140 op: F,
141 ) -> Result<T, FaucetError>
142 where
143 F: FnMut() -> Fut,
144 Fut: Future<Output = Result<T, (Option<String>, String)>>,
145 {
146 self.attempts(ctx, call, what, op)
147 .await
148 .map_err(|e| e.map_or_else(|spent| spent, fail))
149 }
150
151 async fn attempts<T, F, Fut>(
154 &self,
155 ctx: &CallContext,
156 call: &'static str,
157 what: &str,
158 mut op: F,
159 ) -> Result<T, Result<String, FaucetError>>
160 where
161 F: FnMut() -> Fut,
162 Fut: Future<Output = Result<T, (Option<String>, String)>>,
163 {
164 let mut attempt = 0u32;
165 loop {
166 match ctx
167 .call(call, || {
168 let attempt = op();
169 async move { Ok(attempt.await) }
170 })
171 .await
172 {
173 Err(spent) => return Err(Err(spent)),
174 Ok(Ok(v)) => return Ok(v),
175 Ok(Err((code, message))) => {
176 let class = classify_error(code.as_deref());
177 if class == ErrorClass::Fatal || attempt >= self.max_retries {
178 return Err(Ok(format!(
179 "dynamodb: {what} failed after {} attempt(s): {message}",
180 attempt + 1
181 )));
182 }
183 let delay = self.delay(attempt);
184 tracing::debug!(what, ?class, attempt, delay_ms = delay.as_millis() as u64,
185 error = %message, "dynamodb: retrying");
186 ctx.wait(delay).await.map_err(Err)?;
187 attempt += 1;
188 }
189 }
190 }
191 }
192}
193
194#[cfg(test)]
195mod tests {
196 use super::*;
197 use std::sync::atomic::{AtomicU32, Ordering};
198
199 #[test]
200 fn classification() {
201 assert_eq!(classify_error(None), ErrorClass::Transient);
202 for c in [
203 "ProvisionedThroughputExceededException",
204 "ThrottlingException",
205 "Throttling",
206 "RequestLimitExceeded",
207 "LimitExceededException",
208 ] {
209 assert_eq!(classify_error(Some(c)), ErrorClass::Throttle, "{c}");
210 }
211 for c in [
212 "InternalServerError",
213 "InternalFailure",
214 "ServiceUnavailable",
215 "ServiceUnavailableException",
216 "TransactionInProgressException",
217 ] {
218 assert_eq!(classify_error(Some(c)), ErrorClass::Transient, "{c}");
219 }
220 assert_eq!(
221 classify_error(Some("ValidationException")),
222 ErrorClass::Fatal
223 );
224 }
225
226 #[test]
227 fn delay_doubles_and_caps() {
228 let p = RetryPolicy {
229 max_retries: 3,
230 initial_backoff_ms: 100,
231 max_backoff_ms: 400,
232 };
233 let d0 = p.delay(0);
234 assert!(d0 >= Duration::from_millis(50) && d0 < Duration::from_millis(150));
235 let d9 = p.delay(9);
236 assert!(d9 >= Duration::from_millis(200) && d9 < Duration::from_millis(600));
237 let zero = RetryPolicy {
238 max_retries: 0,
239 initial_backoff_ms: 0,
240 max_backoff_ms: 0,
241 };
242 assert!(!zero.delay(u32::MAX).is_zero());
243 assert_eq!(RetryPolicy::default().max_retries, 8);
244 }
245
246 fn fast(max_retries: u32) -> RetryPolicy {
247 RetryPolicy {
248 max_retries,
249 initial_backoff_ms: 1,
250 max_backoff_ms: 2,
251 }
252 }
253
254 #[tokio::test]
255 async fn run_retries_throttles_then_succeeds() {
256 let calls = AtomicU32::new(0);
257 let out = fast(3)
258 .run("scan", || async {
259 if calls.fetch_add(1, Ordering::SeqCst) < 2 {
260 Err((Some("ThrottlingException".to_string()), "slow".to_string()))
261 } else {
262 Ok(7)
263 }
264 })
265 .await
266 .unwrap();
267 assert_eq!(out, 7);
268 assert_eq!(calls.load(Ordering::SeqCst), 3);
269 }
270
271 #[tokio::test]
272 async fn run_stops_on_fatal_and_on_budget() {
273 let calls = AtomicU32::new(0);
274 let err = fast(5)
275 .run::<(), _, _>("scan", || async {
276 calls.fetch_add(1, Ordering::SeqCst);
277 Err((Some("ValidationException".to_string()), "bad".to_string()))
278 })
279 .await
280 .unwrap_err();
281 assert_eq!(calls.load(Ordering::SeqCst), 1);
282 assert!(
283 err.to_string().contains("scan failed after 1 attempt"),
284 "{err}"
285 );
286
287 let calls = AtomicU32::new(0);
288 let err = fast(2)
289 .run::<(), _, _>("query", || async {
290 calls.fetch_add(1, Ordering::SeqCst);
291 Err((None, "conn refused".to_string()))
292 })
293 .await
294 .unwrap_err();
295 assert_eq!(calls.load(Ordering::SeqCst), 3);
296 assert!(err.to_string().contains("conn refused"), "{err}");
297 }
298
299 #[tokio::test(start_paused = true)]
300 async fn run_in_makes_each_attempt_a_call_and_stops_at_the_deadline() {
301 let calls = AtomicU32::new(0);
302 let out = fast(3)
303 .run_in(
304 &CallContext::new(),
305 "scan",
306 "scan",
307 FaucetError::Source,
308 || async {
309 if calls.fetch_add(1, Ordering::SeqCst) < 1 {
310 Err((Some("ThrottlingException".to_string()), "slow".to_string()))
311 } else {
312 Ok(3)
313 }
314 },
315 )
316 .await
317 .unwrap();
318 assert_eq!(out, 3);
319
320 let err = fast(0)
321 .run_in::<(), _, _>(
322 &CallContext::new(),
323 "scan",
324 "scan",
325 FaucetError::Sink,
326 || async { Err((Some("ValidationException".to_string()), "bad".to_string())) },
327 )
328 .await
329 .unwrap_err();
330 assert!(matches!(err, FaucetError::Sink(m) if m.contains("bad")));
331
332 let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
333 let ctx = CallContext::new().with_deadline(deadline);
334 let err = fast(1)
335 .run_in::<(), _, _>(&ctx, "scan", "scan", FaucetError::Source, || {
336 std::future::pending()
337 })
338 .await
339 .unwrap_err();
340 assert!(matches!(err, FaucetError::Timeout { .. }), "{err:?}");
341
342 let err = RetryPolicy {
343 max_retries: 3,
344 initial_backoff_ms: 10_000,
345 max_backoff_ms: 10_000,
346 }
347 .run_in::<(), _, _>(&ctx, "scan", "scan", FaucetError::Source, || async {
348 Err((None, "down".to_string()))
349 })
350 .await
351 .unwrap_err();
352 assert!(matches!(err, FaucetError::Timeout { .. }), "{err:?}");
353
354 let cancel = faucet_core::CancellationToken::new();
355 cancel.cancel();
356 let err = fast(0)
357 .run("scan", || async { Ok::<_, (Option<String>, String)>(1) })
358 .await;
359 assert_eq!(err.unwrap(), 1);
360 let err = fast(0)
361 .run_in::<(), _, _>(
362 &CallContext::new().with_cancel(cancel),
363 "scan",
364 "scan",
365 FaucetError::Source,
366 || async { Ok(()) },
367 )
368 .await
369 .unwrap_err();
370 assert!(matches!(err, FaucetError::Cancelled { .. }), "{err:?}");
371 }
372
373 #[test]
374 fn policy_parses_with_defaults() {
375 let p: RetryPolicy = serde_json::from_value(serde_json::json!({"max_retries": 2})).unwrap();
376 assert_eq!(p.max_retries, 2);
377 assert_eq!(p.initial_backoff_ms, 100);
378 assert_eq!(p.max_backoff_ms, 10_000);
379 }
380}