aws_runtime/retries/
classifiers.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
7use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
8use aws_smithy_runtime_api::client::retries::classifiers::{
9    ClassifyRetry, RetryAction, RetryClassifierPriority, RetryReason,
10};
11use aws_smithy_types::error::metadata::ProvideErrorMetadata;
12use aws_smithy_types::retry::ErrorKind;
13use std::borrow::Cow;
14use std::error::Error as StdError;
15use std::marker::PhantomData;
16
17/// Parse the AWS-specific `x-amz-retry-after` response header (milliseconds) into
18/// a [`Duration`](std::time::Duration). Returns `None` when the header is absent
19/// or does not parse as an integer, in which case the SDK falls back to
20/// exponential backoff.
21fn retry_after_from_ctx(ctx: &InterceptorContext) -> Option<std::time::Duration> {
22    ctx.response()
23        .and_then(|res| res.headers().get("x-amz-retry-after"))
24        .and_then(|header| header.parse::<u64>().ok())
25        .map(std::time::Duration::from_millis)
26}
27
28/// AWS error codes that represent throttling errors.
29pub const THROTTLING_ERRORS: &[&str] = &[
30    "Throttling",
31    "ThrottlingException",
32    "ThrottledException",
33    "RequestThrottledException",
34    "TooManyRequestsException",
35    "ProvisionedThroughputExceededException",
36    "TransactionInProgressException",
37    "RequestLimitExceeded",
38    "BandwidthLimitExceeded",
39    "LimitExceededException",
40    "RequestThrottled",
41    "SlowDown",
42    "PriorRequestNotComplete",
43    "EC2ThrottledException",
44];
45
46/// AWS error codes that represent transient errors.
47pub const TRANSIENT_ERRORS: &[&str] = &["RequestTimeout", "RequestTimeoutException"];
48
49/// A retry classifier for determining if the response sent by an AWS service requires a retry.
50#[derive(Debug)]
51pub struct AwsErrorCodeClassifier<E> {
52    throttling_errors: Cow<'static, [&'static str]>,
53    transient_errors: Cow<'static, [&'static str]>,
54    _inner: PhantomData<E>,
55}
56
57impl<E> Default for AwsErrorCodeClassifier<E> {
58    fn default() -> Self {
59        Self {
60            throttling_errors: THROTTLING_ERRORS.into(),
61            transient_errors: TRANSIENT_ERRORS.into(),
62            _inner: PhantomData,
63        }
64    }
65}
66
67/// Builder for [`AwsErrorCodeClassifier`]
68#[derive(Debug)]
69pub struct AwsErrorCodeClassifierBuilder<E> {
70    throttling_errors: Option<Cow<'static, [&'static str]>>,
71    transient_errors: Option<Cow<'static, [&'static str]>>,
72    _inner: PhantomData<E>,
73}
74
75impl<E> AwsErrorCodeClassifierBuilder<E> {
76    /// Set `transient_errors` for the builder
77    pub fn transient_errors(
78        mut self,
79        transient_errors: impl Into<Cow<'static, [&'static str]>>,
80    ) -> Self {
81        self.transient_errors = Some(transient_errors.into());
82        self
83    }
84
85    /// Build a new [`AwsErrorCodeClassifier`]
86    pub fn build(self) -> AwsErrorCodeClassifier<E> {
87        AwsErrorCodeClassifier {
88            throttling_errors: self.throttling_errors.unwrap_or(THROTTLING_ERRORS.into()),
89            transient_errors: self.transient_errors.unwrap_or(TRANSIENT_ERRORS.into()),
90            _inner: self._inner,
91        }
92    }
93}
94
95impl<E> AwsErrorCodeClassifier<E> {
96    /// Create a new [`AwsErrorCodeClassifier`]
97    pub fn new() -> Self {
98        Self::default()
99    }
100
101    /// Return a builder that can create a new [`AwsErrorCodeClassifier`]
102    pub fn builder() -> AwsErrorCodeClassifierBuilder<E> {
103        AwsErrorCodeClassifierBuilder {
104            throttling_errors: None,
105            transient_errors: None,
106            _inner: PhantomData,
107        }
108    }
109}
110
111impl<E> ClassifyRetry for AwsErrorCodeClassifier<E>
112where
113    E: StdError + ProvideErrorMetadata + Send + Sync + 'static,
114{
115    fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction {
116        // Check for a result
117        let output_or_error = ctx.output_or_error();
118        // Check for an error
119        let error = match output_or_error {
120            Some(Ok(_)) | None => return RetryAction::NoActionIndicated,
121            Some(Err(err)) => err,
122        };
123
124        let retry_after = retry_after_from_ctx(ctx);
125
126        let error_code = OrchestratorError::as_operation_error(error)
127            .and_then(|err| err.downcast_ref::<E>())
128            .and_then(|err| err.code());
129
130        if let Some(error_code) = error_code {
131            if self.throttling_errors.contains(&error_code) {
132                return RetryAction::RetryIndicated(RetryReason::RetryableError {
133                    kind: ErrorKind::ThrottlingError,
134                    retry_after,
135                });
136            }
137            if self.transient_errors.contains(&error_code) {
138                return RetryAction::RetryIndicated(RetryReason::RetryableError {
139                    kind: ErrorKind::TransientError,
140                    retry_after,
141                });
142            }
143        };
144
145        RetryAction::NoActionIndicated
146    }
147
148    fn classify_retry_v2(&self, ctx: &InterceptorContext, previous: &RetryAction) -> RetryAction {
149        // First, run our own error-code-based classification. When it
150        // recognizes an AWS error code it already attaches x-amz-retry-after,
151        // so that result stands unchanged.
152        let own = self.classify_retry(ctx);
153        if own != RetryAction::NoActionIndicated {
154            return own;
155        }
156
157        // No modeled error code matched. Per the Retry Behavior 2.1 spec,
158        // x-amz-retry-after MUST be incorporated into the backoff for ANY
159        // retryable error -- including transient errors classified purely by
160        // HTTP status (e.g. a bare 500) by HttpStatusCodeClassifier, which
161        // cannot read this AWS-specific header. If an earlier-running
162        // classifier already marked the response retryable but without an
163        // explicit delay, tuck the server-directed delay into that verdict,
164        // preserving the upstream error kind. The header alone does not make a
165        // response retryable, so we only refine an already-retryable `previous`;
166        // an unparsable header leaves retry_after as None and the request falls
167        // back to exponential backoff.
168        if let RetryAction::RetryIndicated(RetryReason::RetryableError {
169            kind,
170            retry_after: None,
171        }) = previous
172        {
173            if let Some(retry_after) = retry_after_from_ctx(ctx) {
174                return RetryAction::RetryIndicated(RetryReason::RetryableError {
175                    kind: *kind,
176                    retry_after: Some(retry_after),
177                });
178            }
179        }
180
181        RetryAction::NoActionIndicated
182    }
183
184    fn name(&self) -> &'static str {
185        "AWS Error Code"
186    }
187
188    fn priority(&self) -> RetryClassifierPriority {
189        RetryClassifierPriority::run_before(
190            RetryClassifierPriority::modeled_as_retryable_classifier(),
191        )
192    }
193}
194
195#[cfg(test)]
196mod test {
197    use crate::retries::classifiers::AwsErrorCodeClassifier;
198    use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
199    use aws_smithy_runtime_api::client::interceptors::context::{Error, Input};
200    use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
201    use aws_smithy_runtime_api::client::retries::classifiers::{ClassifyRetry, RetryAction};
202    use aws_smithy_types::body::SdkBody;
203    use aws_smithy_types::error::metadata::ProvideErrorMetadata;
204    use aws_smithy_types::error::ErrorMetadata;
205    use aws_smithy_types::retry::ErrorKind;
206    use std::fmt;
207    use std::time::Duration;
208
209    #[derive(Debug)]
210    struct CodedError {
211        metadata: ErrorMetadata,
212    }
213
214    impl CodedError {
215        fn new(code: &'static str) -> Self {
216            Self {
217                metadata: ErrorMetadata::builder().code(code).build(),
218            }
219        }
220    }
221
222    impl fmt::Display for CodedError {
223        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
224            write!(f, "Coded Error")
225        }
226    }
227
228    impl std::error::Error for CodedError {}
229
230    impl ProvideErrorMetadata for CodedError {
231        fn meta(&self) -> &ErrorMetadata {
232            &self.metadata
233        }
234    }
235
236    #[test]
237    fn classify_by_error_code() {
238        let policy = AwsErrorCodeClassifier::<CodedError>::new();
239        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
240        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
241            CodedError::new("Throttling"),
242        ))));
243
244        assert_eq!(policy.classify_retry(&ctx), RetryAction::throttling_error());
245
246        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
247        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
248            CodedError::new("RequestTimeout"),
249        ))));
250        assert_eq!(policy.classify_retry(&ctx), RetryAction::transient_error())
251    }
252
253    #[test]
254    fn classify_generic() {
255        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
256        let err = ErrorMetadata::builder().code("SlowDown").build();
257        let test_response = http_1x::Response::new("OK").map(SdkBody::from);
258
259        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
260        ctx.set_response(test_response.try_into().unwrap());
261        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
262
263        assert_eq!(policy.classify_retry(&ctx), RetryAction::throttling_error());
264    }
265
266    #[test]
267    fn test_retry_after_header() {
268        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
269        let err = ErrorMetadata::builder().code("SlowDown").build();
270        let res = http_1x::Response::builder()
271            .header("x-amz-retry-after", "5000")
272            .body("retry later")
273            .unwrap()
274            .map(SdkBody::from);
275        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
276        ctx.set_response(res.try_into().unwrap());
277        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
278
279        assert_eq!(
280            policy.classify_retry(&ctx),
281            RetryAction::retryable_error_with_explicit_delay(
282                ErrorKind::ThrottlingError,
283                Duration::from_secs(5)
284            )
285        );
286    }
287
288    #[test]
289    fn test_invalid_retry_after_header_is_ignored() {
290        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
291        let err = ErrorMetadata::builder().code("SlowDown").build();
292        let res = http_1x::Response::builder()
293            .header("x-amz-retry-after", "invalid")
294            .body("retry later")
295            .unwrap()
296            .map(SdkBody::from);
297        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
298        ctx.set_response(res.try_into().unwrap());
299        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
300
301        // Invalid header ignored — retryable_error() has retry_after: None,
302        // so the retry strategy will fall back to exponential backoff.
303        assert_eq!(
304            policy.classify_retry(&ctx),
305            RetryAction::retryable_error(ErrorKind::ThrottlingError)
306        );
307    }
308
309    /// Build a context whose response optionally carries an `x-amz-retry-after`
310    /// header (whose value is a count of **milliseconds**) and whose error
311    /// optionally carries an error code (`None` models a bare response with no
312    /// identifiable AWS error code, e.g. an unparsable 5xx).
313    fn ctx_with(
314        retry_after_header_millis: Option<&str>,
315        error_code: Option<&'static str>,
316    ) -> InterceptorContext {
317        let mut builder = http_1x::Response::builder();
318        if let Some(millis) = retry_after_header_millis {
319            builder = builder.header("x-amz-retry-after", millis);
320        }
321        let res = builder.body("nope").unwrap().map(SdkBody::from);
322
323        let err = match error_code {
324            Some(code) => ErrorMetadata::builder().code(code).build(),
325            None => ErrorMetadata::builder().build(),
326        };
327
328        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
329        ctx.set_response(res.try_into().unwrap());
330        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
331        ctx
332    }
333
334    // The headline scenario: a bare 5xx with no identifiable AWS error code that
335    // an earlier classifier (`HttpStatusCodeClassifier`) marked transient.
336    // `classify_retry_v2` attaches the server-directed delay from
337    // `x-amz-retry-after`, which that earlier classifier cannot read. The header
338    // is a count of milliseconds, so 5000ms == 5s. The clamping
339    // to `[t_i, 5 + t_i]` happens later in the backoff strategy, so the
340    // classifier stores the raw parsed duration.
341    #[test]
342    fn classify_retry_v2_attaches_server_delay_on_bare_5xx() {
343        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
344        // No error code -> this classifier's own classification is `NoActionIndicated`.
345        let ctx = ctx_with(Some("5000"), None);
346
347        let previous = RetryAction::transient_error();
348        assert_eq!(
349            policy.classify_retry_v2(&ctx, &previous),
350            RetryAction::retryable_error_with_explicit_delay(
351                ErrorKind::TransientError,
352                Duration::from_secs(5)
353            ),
354        );
355    }
356
357    // Same refinement occurs when there is an error code that this classifier
358    // simply doesn't recognize ("InternalError" is not in its throttling/transient
359    // lists), so its own classification is still `NoActionIndicated`.
360    #[test]
361    fn classify_retry_v2_attaches_server_delay_on_unrecognized_error_code() {
362        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
363        let ctx = ctx_with(Some("5000"), Some("InternalError"));
364
365        let previous = RetryAction::transient_error();
366        assert_eq!(
367            policy.classify_retry_v2(&ctx, &previous),
368            RetryAction::retryable_error_with_explicit_delay(
369                ErrorKind::TransientError,
370                Duration::from_secs(5)
371            ),
372        );
373    }
374
375    // The `kind` of the previous (already-retryable) verdict must be preserved
376    // when the server-directed delay is tucked in.
377    #[test]
378    fn classify_retry_v2_preserves_previous_kind() {
379        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
380        let ctx = ctx_with(Some("2500"), None);
381
382        let previous = RetryAction::retryable_error(ErrorKind::ServerError);
383        assert_eq!(
384            policy.classify_retry_v2(&ctx, &previous),
385            RetryAction::retryable_error_with_explicit_delay(
386                ErrorKind::ServerError,
387                Duration::from_millis(2500)
388            ),
389        );
390    }
391
392    // When this classifier recognizes the error code itself, its own verdict
393    // (which already attaches `x-amz-retry-after`) stands unchanged, regardless
394    // of what an earlier classifier decided. `SlowDown` is a
395    // throttling error (not transient).
396    #[test]
397    fn classify_retry_v2_prefers_own_error_code_verdict() {
398        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
399        let ctx = ctx_with(Some("5000"), Some("SlowDown"));
400
401        // Even with a `previous` transient verdict, the modeled throttling code wins.
402        let previous = RetryAction::transient_error();
403        assert_eq!(
404            policy.classify_retry_v2(&ctx, &previous),
405            RetryAction::retryable_error_with_explicit_delay(
406                ErrorKind::ThrottlingError,
407                Duration::from_secs(5)
408            ),
409        );
410    }
411
412    // The header alone must not make a response retryable: if no earlier
413    // classifier marked the response retryable, `classify_retry_v2` indicates no
414    // action even when `x-amz-retry-after` is present.
415    #[test]
416    fn classify_retry_v2_ignores_header_without_retryable_previous() {
417        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
418        let ctx = ctx_with(Some("5000"), None);
419
420        assert_eq!(
421            policy.classify_retry_v2(&ctx, &RetryAction::NoActionIndicated),
422            RetryAction::NoActionIndicated,
423        );
424    }
425
426    // When the previous verdict already carries an explicit delay, the
427    // refinement branch (which only matches `retry_after: None`) is skipped and
428    // this classifier abstains by returning `NoActionIndicated`. Abstaining is
429    // precisely how it avoids overriding the existing delay: in
430    // `run_classifiers_on_ctx`, a `NoActionIndicated` result is a no-op that
431    // leaves the accumulated verdict (here, the `Some(1s)` delay) untouched.
432    // That end-to-end preservation is exercised in the aws-smithy-runtime
433    // `run_classifiers_on_ctx` test.
434    #[test]
435    fn classify_retry_v2_abstains_when_previous_has_explicit_delay() {
436        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
437        let ctx = ctx_with(Some("5000"), None);
438
439        let previous = RetryAction::retryable_error_with_explicit_delay(
440            ErrorKind::TransientError,
441            Duration::from_secs(1),
442        );
443        assert_eq!(
444            policy.classify_retry_v2(&ctx, &previous),
445            RetryAction::NoActionIndicated,
446        );
447    }
448
449    // Invalid values in `x-amz-retry-after` must be ignored (fall
450    // back to exponential backoff). With no header, or an unparsable one, an
451    // already-retryable `previous` is left as-is (this classifier returns
452    // `NoActionIndicated`).
453    #[test]
454    fn classify_retry_v2_falls_back_without_valid_header() {
455        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
456
457        let previous = RetryAction::transient_error();
458        // Absent header.
459        assert_eq!(
460            policy.classify_retry_v2(&ctx_with(None, None), &previous),
461            RetryAction::NoActionIndicated,
462        );
463        // Unparsable header.
464        assert_eq!(
465            policy.classify_retry_v2(&ctx_with(Some("invalid"), None), &previous),
466            RetryAction::NoActionIndicated,
467        );
468    }
469}
aws_runtime/retries/classifiers.rs

aws_runtime/retries/
classifiers.rs
