Skip to main content

faucet_common_dynamodb/
retry.rs

1//! Throttle-aware retry shared by the DynamoDB source and sink.
2//!
3//! DynamoDB signals capacity pressure with typed error codes
4//! (`ProvisionedThroughputExceededException`, `ThrottlingException`,
5//! `RequestLimitExceeded`, Streams' `LimitExceededException`). Those, plus
6//! transient server/transport failures, are retried with jittered exponential
7//! backoff; anything else (validation, missing table, access denied) fails
8//! immediately.
9
10use faucet_core::FaucetError;
11use faucet_core::io::CallContext;
12use schemars::JsonSchema;
13use serde::{Deserialize, Serialize};
14use std::future::Future;
15use std::time::Duration;
16
17/// How a failed request should be treated.
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum ErrorClass {
20    /// Capacity throttling — back off and retry.
21    Throttle,
22    /// Transient server or transport failure — retry.
23    Transient,
24    /// Anything else — fail now.
25    Fatal,
26}
27
28/// Classify an AWS error code. `None` (no service error — a dispatch,
29/// timeout or transport failure) is transient.
30pub fn classify_error(code: Option<&str>) -> ErrorClass {
31    match code {
32        None => ErrorClass::Transient,
33        Some(
34            "ProvisionedThroughputExceededException"
35            | "ThrottlingException"
36            | "Throttling"
37            | "RequestLimitExceeded"
38            | "LimitExceededException",
39        ) => ErrorClass::Throttle,
40        Some(
41            "InternalServerError"
42            | "InternalFailure"
43            | "ServiceUnavailable"
44            | "ServiceUnavailableException"
45            | "TransactionInProgressException",
46        ) => ErrorClass::Transient,
47        Some(_) => ErrorClass::Fatal,
48    }
49}
50
51/// The code + human message of an SDK error.
52pub fn sdk_error_parts<E, R>(
53    err: &aws_sdk_dynamodb::error::SdkError<E, R>,
54) -> (Option<String>, String)
55where
56    E: aws_sdk_dynamodb::error::ProvideErrorMetadata + std::error::Error + 'static,
57    R: std::fmt::Debug,
58{
59    use aws_sdk_dynamodb::error::ProvideErrorMetadata as _;
60    let code = err.code().map(str::to_string);
61    let message = aws_sdk_dynamodb::error::DisplayErrorContext(err).to_string();
62    (code, message)
63}
64
65/// Retry/backoff settings exposed on both connector configs.
66#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
67#[serde(deny_unknown_fields)]
68pub struct RetryPolicy {
69    /// Retries after the first attempt for throttled or transient failures.
70    /// Default 8.
71    #[serde(default = "default_max_retries")]
72    pub max_retries: u32,
73    /// Initial backoff in milliseconds (doubled per attempt, jittered).
74    /// Default 100.
75    #[serde(default = "default_base_ms")]
76    pub initial_backoff_ms: u64,
77    /// Backoff ceiling in milliseconds. Default 10000.
78    #[serde(default = "default_max_ms")]
79    pub max_backoff_ms: u64,
80}
81
82fn default_max_retries() -> u32 {
83    8
84}
85fn default_base_ms() -> u64 {
86    100
87}
88fn default_max_ms() -> u64 {
89    10_000
90}
91
92impl Default for RetryPolicy {
93    fn default() -> Self {
94        Self {
95            max_retries: default_max_retries(),
96            initial_backoff_ms: default_base_ms(),
97            max_backoff_ms: default_max_ms(),
98        }
99    }
100}
101
102impl RetryPolicy {
103    /// Jittered delay before retry number `attempt` (0-based).
104    pub fn delay(&self, attempt: u32) -> Duration {
105        let exp = self
106            .initial_backoff_ms
107            .saturating_mul(1u64 << attempt.min(20))
108            .min(self.max_backoff_ms.max(1));
109        faucet_core::retry::apply_jitter(Duration::from_millis(exp.max(1)))
110    }
111
112    /// Run `op` until it succeeds, a fatal error occurs, or the retry budget is
113    /// spent. `op` returns the error's `(code, message)`; `what` names the call
114    /// in the final error message (the caller picks the `FaucetError` variant).
115    pub async fn run<T, F, Fut>(&self, what: &str, op: F) -> Result<T, String>
116    where
117        F: FnMut() -> Fut,
118        Fut: Future<Output = Result<T, (Option<String>, String)>>,
119    {
120        let ctx = CallContext::new().without_timeout();
121        self.attempts(&ctx, "request", what, op)
122            .await
123            .map_err(|e| match e {
124                Ok(message) => message,
125                Err(spent) => format!("dynamodb: {what} failed: {spent}"),
126            })
127    }
128
129    /// [`run`](Self::run) with every attempt made through `ctx` as one backend
130    /// call named `call` (#880): bounded by its timeout and deadline, stopped
131    /// by its cancel token, metered on its recorder; backoffs wait through it
132    /// too. A spent budget or a fatal error becomes `fail(message)`; the
133    /// context running out comes back as its own error.
134    pub async fn run_in<T, F, Fut>(
135        &self,
136        ctx: &CallContext,
137        call: &'static str,
138        what: &str,
139        fail: fn(String) -> FaucetError,
140        op: F,
141    ) -> Result<T, FaucetError>
142    where
143        F: FnMut() -> Fut,
144        Fut: Future<Output = Result<T, (Option<String>, String)>>,
145    {
146        self.attempts(ctx, call, what, op)
147            .await
148            .map_err(|e| e.map_or_else(|spent| spent, fail))
149    }
150
151    /// The retry loop. `Err(Ok(message))` is a fatal error or a spent budget;
152    /// `Err(Err(e))` is `ctx` running out.
153    async fn attempts<T, F, Fut>(
154        &self,
155        ctx: &CallContext,
156        call: &'static str,
157        what: &str,
158        mut op: F,
159    ) -> Result<T, Result<String, FaucetError>>
160    where
161        F: FnMut() -> Fut,
162        Fut: Future<Output = Result<T, (Option<String>, String)>>,
163    {
164        let mut attempt = 0u32;
165        loop {
166            match ctx
167                .call(call, || {
168                    let attempt = op();
169                    async move { Ok(attempt.await) }
170                })
171                .await
172            {
173                Err(spent) => return Err(Err(spent)),
174                Ok(Ok(v)) => return Ok(v),
175                Ok(Err((code, message))) => {
176                    let class = classify_error(code.as_deref());
177                    if class == ErrorClass::Fatal || attempt >= self.max_retries {
178                        return Err(Ok(format!(
179                            "dynamodb: {what} failed after {} attempt(s): {message}",
180                            attempt + 1
181                        )));
182                    }
183                    let delay = self.delay(attempt);
184                    tracing::debug!(what, ?class, attempt, delay_ms = delay.as_millis() as u64,
185                        error = %message, "dynamodb: retrying");
186                    ctx.wait(delay).await.map_err(Err)?;
187                    attempt += 1;
188                }
189            }
190        }
191    }
192}
193
194#[cfg(test)]
195mod tests {
196    use super::*;
197    use std::sync::atomic::{AtomicU32, Ordering};
198
199    #[test]
200    fn classification() {
201        assert_eq!(classify_error(None), ErrorClass::Transient);
202        for c in [
203            "ProvisionedThroughputExceededException",
204            "ThrottlingException",
205            "Throttling",
206            "RequestLimitExceeded",
207            "LimitExceededException",
208        ] {
209            assert_eq!(classify_error(Some(c)), ErrorClass::Throttle, "{c}");
210        }
211        for c in [
212            "InternalServerError",
213            "InternalFailure",
214            "ServiceUnavailable",
215            "ServiceUnavailableException",
216            "TransactionInProgressException",
217        ] {
218            assert_eq!(classify_error(Some(c)), ErrorClass::Transient, "{c}");
219        }
220        assert_eq!(
221            classify_error(Some("ValidationException")),
222            ErrorClass::Fatal
223        );
224    }
225
226    #[test]
227    fn delay_doubles_and_caps() {
228        let p = RetryPolicy {
229            max_retries: 3,
230            initial_backoff_ms: 100,
231            max_backoff_ms: 400,
232        };
233        let d0 = p.delay(0);
234        assert!(d0 >= Duration::from_millis(50) && d0 < Duration::from_millis(150));
235        let d9 = p.delay(9);
236        assert!(d9 >= Duration::from_millis(200) && d9 < Duration::from_millis(600));
237        let zero = RetryPolicy {
238            max_retries: 0,
239            initial_backoff_ms: 0,
240            max_backoff_ms: 0,
241        };
242        assert!(!zero.delay(u32::MAX).is_zero());
243        assert_eq!(RetryPolicy::default().max_retries, 8);
244    }
245
246    fn fast(max_retries: u32) -> RetryPolicy {
247        RetryPolicy {
248            max_retries,
249            initial_backoff_ms: 1,
250            max_backoff_ms: 2,
251        }
252    }
253
254    #[tokio::test]
255    async fn run_retries_throttles_then_succeeds() {
256        let calls = AtomicU32::new(0);
257        let out = fast(3)
258            .run("scan", || async {
259                if calls.fetch_add(1, Ordering::SeqCst) < 2 {
260                    Err((Some("ThrottlingException".to_string()), "slow".to_string()))
261                } else {
262                    Ok(7)
263                }
264            })
265            .await
266            .unwrap();
267        assert_eq!(out, 7);
268        assert_eq!(calls.load(Ordering::SeqCst), 3);
269    }
270
271    #[tokio::test]
272    async fn run_stops_on_fatal_and_on_budget() {
273        let calls = AtomicU32::new(0);
274        let err = fast(5)
275            .run::<(), _, _>("scan", || async {
276                calls.fetch_add(1, Ordering::SeqCst);
277                Err((Some("ValidationException".to_string()), "bad".to_string()))
278            })
279            .await
280            .unwrap_err();
281        assert_eq!(calls.load(Ordering::SeqCst), 1);
282        assert!(
283            err.to_string().contains("scan failed after 1 attempt"),
284            "{err}"
285        );
286
287        let calls = AtomicU32::new(0);
288        let err = fast(2)
289            .run::<(), _, _>("query", || async {
290                calls.fetch_add(1, Ordering::SeqCst);
291                Err((None, "conn refused".to_string()))
292            })
293            .await
294            .unwrap_err();
295        assert_eq!(calls.load(Ordering::SeqCst), 3);
296        assert!(err.to_string().contains("conn refused"), "{err}");
297    }
298
299    #[tokio::test(start_paused = true)]
300    async fn run_in_makes_each_attempt_a_call_and_stops_at_the_deadline() {
301        let calls = AtomicU32::new(0);
302        let out = fast(3)
303            .run_in(
304                &CallContext::new(),
305                "scan",
306                "scan",
307                FaucetError::Source,
308                || async {
309                    if calls.fetch_add(1, Ordering::SeqCst) < 1 {
310                        Err((Some("ThrottlingException".to_string()), "slow".to_string()))
311                    } else {
312                        Ok(3)
313                    }
314                },
315            )
316            .await
317            .unwrap();
318        assert_eq!(out, 3);
319
320        let err = fast(0)
321            .run_in::<(), _, _>(
322                &CallContext::new(),
323                "scan",
324                "scan",
325                FaucetError::Sink,
326                || async { Err((Some("ValidationException".to_string()), "bad".to_string())) },
327            )
328            .await
329            .unwrap_err();
330        assert!(matches!(err, FaucetError::Sink(m) if m.contains("bad")));
331
332        let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
333        let ctx = CallContext::new().with_deadline(deadline);
334        let err = fast(1)
335            .run_in::<(), _, _>(&ctx, "scan", "scan", FaucetError::Source, || {
336                std::future::pending()
337            })
338            .await
339            .unwrap_err();
340        assert!(matches!(err, FaucetError::Timeout { .. }), "{err:?}");
341
342        let err = RetryPolicy {
343            max_retries: 3,
344            initial_backoff_ms: 10_000,
345            max_backoff_ms: 10_000,
346        }
347        .run_in::<(), _, _>(&ctx, "scan", "scan", FaucetError::Source, || async {
348            Err((None, "down".to_string()))
349        })
350        .await
351        .unwrap_err();
352        assert!(matches!(err, FaucetError::Timeout { .. }), "{err:?}");
353
354        let cancel = faucet_core::CancellationToken::new();
355        cancel.cancel();
356        let err = fast(0)
357            .run("scan", || async { Ok::<_, (Option<String>, String)>(1) })
358            .await;
359        assert_eq!(err.unwrap(), 1);
360        let err = fast(0)
361            .run_in::<(), _, _>(
362                &CallContext::new().with_cancel(cancel),
363                "scan",
364                "scan",
365                FaucetError::Source,
366                || async { Ok(()) },
367            )
368            .await
369            .unwrap_err();
370        assert!(matches!(err, FaucetError::Cancelled { .. }), "{err:?}");
371    }
372
373    #[test]
374    fn policy_parses_with_defaults() {
375        let p: RetryPolicy = serde_json::from_value(serde_json::json!({"max_retries": 2})).unwrap();
376        assert_eq!(p.max_retries, 2);
377        assert_eq!(p.initial_backoff_ms, 100);
378        assert_eq!(p.max_backoff_ms, 10_000);
379    }
380}