Skip to main content

aws_runtime/retries/
classifiers.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
7use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
8use aws_smithy_runtime_api::client::retries::classifiers::{
9    ClassifyRetry, RetryAction, RetryClassifierPriority, RetryReason,
10};
11use aws_smithy_types::error::metadata::ProvideErrorMetadata;
12use aws_smithy_types::retry::ErrorKind;
13use std::borrow::Cow;
14use std::error::Error as StdError;
15use std::marker::PhantomData;
16
17/// Parse the AWS-specific `x-amz-retry-after` response header (milliseconds) into
18/// a [`Duration`](std::time::Duration). Returns `None` when the header is absent
19/// or does not parse as an integer, in which case the SDK falls back to
20/// exponential backoff.
21fn retry_after_from_ctx(ctx: &InterceptorContext) -> Option<std::time::Duration> {
22    ctx.response()
23        .and_then(|res| res.headers().get("x-amz-retry-after"))
24        .and_then(|header| header.parse::<u64>().ok())
25        .map(std::time::Duration::from_millis)
26}
27
28/// AWS error codes that represent throttling errors.
29pub const THROTTLING_ERRORS: &[&str] = &[
30    "Throttling",
31    "ThrottlingException",
32    "ThrottledException",
33    "RequestThrottledException",
34    "TooManyRequestsException",
35    "ProvisionedThroughputExceededException",
36    "TransactionInProgressException",
37    "RequestLimitExceeded",
38    "BandwidthLimitExceeded",
39    "LimitExceededException",
40    "RequestThrottled",
41    "SlowDown",
42    "PriorRequestNotComplete",
43    "EC2ThrottledException",
44];
45
46/// AWS error codes that represent transient errors.
47pub const TRANSIENT_ERRORS: &[&str] = &["RequestTimeout", "RequestTimeoutException"];
48
49/// A retry classifier for determining if the response sent by an AWS service requires a retry.
50#[derive(Debug)]
51pub struct AwsErrorCodeClassifier<E> {
52    throttling_errors: Cow<'static, [&'static str]>,
53    transient_errors: Cow<'static, [&'static str]>,
54    _inner: PhantomData<E>,
55}
56
57impl<E> Default for AwsErrorCodeClassifier<E> {
58    fn default() -> Self {
59        Self {
60            throttling_errors: THROTTLING_ERRORS.into(),
61            transient_errors: TRANSIENT_ERRORS.into(),
62            _inner: PhantomData,
63        }
64    }
65}
66
67/// Builder for [`AwsErrorCodeClassifier`]
68#[derive(Debug)]
69pub struct AwsErrorCodeClassifierBuilder<E> {
70    throttling_errors: Option<Cow<'static, [&'static str]>>,
71    transient_errors: Option<Cow<'static, [&'static str]>>,
72    _inner: PhantomData<E>,
73}
74
75impl<E> AwsErrorCodeClassifierBuilder<E> {
76    /// Set `transient_errors` for the builder
77    pub fn transient_errors(
78        mut self,
79        transient_errors: impl Into<Cow<'static, [&'static str]>>,
80    ) -> Self {
81        self.transient_errors = Some(transient_errors.into());
82        self
83    }
84
85    /// Build a new [`AwsErrorCodeClassifier`]
86    pub fn build(self) -> AwsErrorCodeClassifier<E> {
87        AwsErrorCodeClassifier {
88            throttling_errors: self.throttling_errors.unwrap_or(THROTTLING_ERRORS.into()),
89            transient_errors: self.transient_errors.unwrap_or(TRANSIENT_ERRORS.into()),
90            _inner: self._inner,
91        }
92    }
93}
94
95impl<E> AwsErrorCodeClassifier<E> {
96    /// Create a new [`AwsErrorCodeClassifier`]
97    pub fn new() -> Self {
98        Self::default()
99    }
100
101    /// Return a builder that can create a new [`AwsErrorCodeClassifier`]
102    pub fn builder() -> AwsErrorCodeClassifierBuilder<E> {
103        AwsErrorCodeClassifierBuilder {
104            throttling_errors: None,
105            transient_errors: None,
106            _inner: PhantomData,
107        }
108    }
109}
110
111impl<E> ClassifyRetry for AwsErrorCodeClassifier<E>
112where
113    E: StdError + ProvideErrorMetadata + Send + Sync + 'static,
114{
115    fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction {
116        // Check for a result
117        let output_or_error = ctx.output_or_error();
118        // Check for an error
119        let error = match output_or_error {
120            Some(Ok(_)) | None => return RetryAction::NoActionIndicated,
121            Some(Err(err)) => err,
122        };
123
124        let retry_after = retry_after_from_ctx(ctx);
125
126        let error_code = OrchestratorError::as_operation_error(error)
127            .and_then(|err| err.downcast_ref::<E>())
128            .and_then(|err| err.code());
129
130        if let Some(error_code) = error_code {
131            if self.throttling_errors.contains(&error_code) {
132                return RetryAction::RetryIndicated(RetryReason::RetryableError {
133                    kind: ErrorKind::ThrottlingError,
134                    retry_after,
135                });
136            }
137            if self.transient_errors.contains(&error_code) {
138                return RetryAction::RetryIndicated(RetryReason::RetryableError {
139                    kind: ErrorKind::TransientError,
140                    retry_after,
141                });
142            }
143        };
144
145        // A bare HTTP 429 with no modeled error code is a throttling error. This check runs after
146        // the error-code checks above. The classifier runs before `ModeledAsRetryableClassifier`,
147        // so a modeled error's retryable trait still wins.
148        let is_too_many_requests = ctx
149            .response()
150            .map(|res| res.status().into())
151            .map(|status: u16| status == 429)
152            .unwrap_or(false);
153        if is_too_many_requests {
154            return RetryAction::RetryIndicated(RetryReason::RetryableError {
155                kind: ErrorKind::ThrottlingError,
156                retry_after,
157            });
158        }
159
160        RetryAction::NoActionIndicated
161    }
162
163    fn classify_retry_v2(&self, ctx: &InterceptorContext, previous: &RetryAction) -> RetryAction {
164        // First, run our own error-code-based classification. When it
165        // recognizes an AWS error code it already attaches x-amz-retry-after,
166        // so that result stands unchanged.
167        let own = self.classify_retry(ctx);
168        if own != RetryAction::NoActionIndicated {
169            return own;
170        }
171
172        // No modeled error code matched. Per the Retry Behavior 2.1 spec,
173        // x-amz-retry-after MUST be incorporated into the backoff for ANY
174        // retryable error -- including transient errors classified purely by
175        // HTTP status (e.g. a bare 500) by HttpStatusCodeClassifier, which
176        // cannot read this AWS-specific header. If an earlier-running
177        // classifier already marked the response retryable but without an
178        // explicit delay, tuck the server-directed delay into that verdict,
179        // preserving the upstream error kind. The header alone does not make a
180        // response retryable, so we only refine an already-retryable `previous`;
181        // an unparsable header leaves retry_after as None and the request falls
182        // back to exponential backoff.
183        if let RetryAction::RetryIndicated(RetryReason::RetryableError {
184            kind,
185            retry_after: None,
186        }) = previous
187        {
188            if let Some(retry_after) = retry_after_from_ctx(ctx) {
189                return RetryAction::RetryIndicated(RetryReason::RetryableError {
190                    kind: *kind,
191                    retry_after: Some(retry_after),
192                });
193            }
194        }
195
196        RetryAction::NoActionIndicated
197    }
198
199    fn name(&self) -> &'static str {
200        "AWS Error Code"
201    }
202
203    fn priority(&self) -> RetryClassifierPriority {
204        RetryClassifierPriority::run_before(
205            RetryClassifierPriority::modeled_as_retryable_classifier(),
206        )
207    }
208}
209
210#[cfg(test)]
211mod test {
212    use crate::retries::classifiers::AwsErrorCodeClassifier;
213    use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
214    use aws_smithy_runtime_api::client::interceptors::context::{Error, Input};
215    use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
216    use aws_smithy_runtime_api::client::retries::classifiers::{ClassifyRetry, RetryAction};
217    use aws_smithy_types::body::SdkBody;
218    use aws_smithy_types::error::metadata::ProvideErrorMetadata;
219    use aws_smithy_types::error::ErrorMetadata;
220    use aws_smithy_types::retry::ErrorKind;
221    use std::fmt;
222    use std::time::Duration;
223
224    #[derive(Debug)]
225    struct CodedError {
226        metadata: ErrorMetadata,
227    }
228
229    impl CodedError {
230        fn new(code: &'static str) -> Self {
231            Self {
232                metadata: ErrorMetadata::builder().code(code).build(),
233            }
234        }
235    }
236
237    impl fmt::Display for CodedError {
238        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
239            write!(f, "Coded Error")
240        }
241    }
242
243    impl std::error::Error for CodedError {}
244
245    impl ProvideErrorMetadata for CodedError {
246        fn meta(&self) -> &ErrorMetadata {
247            &self.metadata
248        }
249    }
250
251    #[test]
252    fn classify_by_error_code() {
253        let policy = AwsErrorCodeClassifier::<CodedError>::new();
254        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
255        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
256            CodedError::new("Throttling"),
257        ))));
258
259        assert_eq!(policy.classify_retry(&ctx), RetryAction::throttling_error());
260
261        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
262        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
263            CodedError::new("RequestTimeout"),
264        ))));
265        assert_eq!(policy.classify_retry(&ctx), RetryAction::transient_error())
266    }
267
268    #[test]
269    fn classify_generic() {
270        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
271        let err = ErrorMetadata::builder().code("SlowDown").build();
272        let test_response = http_1x::Response::new("OK").map(SdkBody::from);
273
274        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
275        ctx.set_response(test_response.try_into().unwrap());
276        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
277
278        assert_eq!(policy.classify_retry(&ctx), RetryAction::throttling_error());
279    }
280
281    #[test]
282    fn test_retry_after_header() {
283        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
284        let err = ErrorMetadata::builder().code("SlowDown").build();
285        let res = http_1x::Response::builder()
286            .header("x-amz-retry-after", "5000")
287            .body("retry later")
288            .unwrap()
289            .map(SdkBody::from);
290        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
291        ctx.set_response(res.try_into().unwrap());
292        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
293
294        assert_eq!(
295            policy.classify_retry(&ctx),
296            RetryAction::retryable_error_with_explicit_delay(
297                ErrorKind::ThrottlingError,
298                Duration::from_secs(5)
299            )
300        );
301    }
302
303    #[test]
304    fn test_invalid_retry_after_header_is_ignored() {
305        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
306        let err = ErrorMetadata::builder().code("SlowDown").build();
307        let res = http_1x::Response::builder()
308            .header("x-amz-retry-after", "invalid")
309            .body("retry later")
310            .unwrap()
311            .map(SdkBody::from);
312        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
313        ctx.set_response(res.try_into().unwrap());
314        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
315
316        // Invalid header ignored — retryable_error() has retry_after: None,
317        // so the retry strategy will fall back to exponential backoff.
318        assert_eq!(
319            policy.classify_retry(&ctx),
320            RetryAction::retryable_error(ErrorKind::ThrottlingError)
321        );
322    }
323
324    /// Build a context whose response optionally carries an `x-amz-retry-after`
325    /// header (whose value is a count of **milliseconds**) and whose error
326    /// optionally carries an error code (`None` models a bare response with no
327    /// identifiable AWS error code, e.g. an unparsable 5xx).
328    fn ctx_with(
329        retry_after_header_millis: Option<&str>,
330        error_code: Option<&'static str>,
331    ) -> InterceptorContext {
332        let mut builder = http_1x::Response::builder();
333        if let Some(millis) = retry_after_header_millis {
334            builder = builder.header("x-amz-retry-after", millis);
335        }
336        let res = builder.body("nope").unwrap().map(SdkBody::from);
337
338        let err = match error_code {
339            Some(code) => ErrorMetadata::builder().code(code).build(),
340            None => ErrorMetadata::builder().build(),
341        };
342
343        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
344        ctx.set_response(res.try_into().unwrap());
345        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
346        ctx
347    }
348
349    // The headline scenario: a bare 5xx with no identifiable AWS error code that
350    // an earlier classifier (`HttpStatusCodeClassifier`) marked transient.
351    // `classify_retry_v2` attaches the server-directed delay from
352    // `x-amz-retry-after`, which that earlier classifier cannot read. The header
353    // is a count of milliseconds, so 5000ms == 5s. The clamping
354    // to `[t_i, 5 + t_i]` happens later in the backoff strategy, so the
355    // classifier stores the raw parsed duration.
356    #[test]
357    fn classify_retry_v2_attaches_server_delay_on_bare_5xx() {
358        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
359        // No error code -> this classifier's own classification is `NoActionIndicated`.
360        let ctx = ctx_with(Some("5000"), None);
361
362        let previous = RetryAction::transient_error();
363        assert_eq!(
364            policy.classify_retry_v2(&ctx, &previous),
365            RetryAction::retryable_error_with_explicit_delay(
366                ErrorKind::TransientError,
367                Duration::from_secs(5)
368            ),
369        );
370    }
371
372    // Same refinement occurs when there is an error code that this classifier
373    // simply doesn't recognize ("InternalError" is not in its throttling/transient
374    // lists), so its own classification is still `NoActionIndicated`.
375    #[test]
376    fn classify_retry_v2_attaches_server_delay_on_unrecognized_error_code() {
377        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
378        let ctx = ctx_with(Some("5000"), Some("InternalError"));
379
380        let previous = RetryAction::transient_error();
381        assert_eq!(
382            policy.classify_retry_v2(&ctx, &previous),
383            RetryAction::retryable_error_with_explicit_delay(
384                ErrorKind::TransientError,
385                Duration::from_secs(5)
386            ),
387        );
388    }
389
390    // The `kind` of the previous (already-retryable) verdict must be preserved
391    // when the server-directed delay is tucked in.
392    #[test]
393    fn classify_retry_v2_preserves_previous_kind() {
394        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
395        let ctx = ctx_with(Some("2500"), None);
396
397        let previous = RetryAction::retryable_error(ErrorKind::ServerError);
398        assert_eq!(
399            policy.classify_retry_v2(&ctx, &previous),
400            RetryAction::retryable_error_with_explicit_delay(
401                ErrorKind::ServerError,
402                Duration::from_millis(2500)
403            ),
404        );
405    }
406
407    // When this classifier recognizes the error code itself, its own verdict
408    // (which already attaches `x-amz-retry-after`) stands unchanged, regardless
409    // of what an earlier classifier decided. `SlowDown` is a
410    // throttling error (not transient).
411    #[test]
412    fn classify_retry_v2_prefers_own_error_code_verdict() {
413        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
414        let ctx = ctx_with(Some("5000"), Some("SlowDown"));
415
416        // Even with a `previous` transient verdict, the modeled throttling code wins.
417        let previous = RetryAction::transient_error();
418        assert_eq!(
419            policy.classify_retry_v2(&ctx, &previous),
420            RetryAction::retryable_error_with_explicit_delay(
421                ErrorKind::ThrottlingError,
422                Duration::from_secs(5)
423            ),
424        );
425    }
426
427    // The header alone must not make a response retryable: if no earlier
428    // classifier marked the response retryable, `classify_retry_v2` indicates no
429    // action even when `x-amz-retry-after` is present.
430    #[test]
431    fn classify_retry_v2_ignores_header_without_retryable_previous() {
432        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
433        let ctx = ctx_with(Some("5000"), None);
434
435        assert_eq!(
436            policy.classify_retry_v2(&ctx, &RetryAction::NoActionIndicated),
437            RetryAction::NoActionIndicated,
438        );
439    }
440
441    // When the previous verdict already carries an explicit delay, the
442    // refinement branch (which only matches `retry_after: None`) is skipped and
443    // this classifier abstains by returning `NoActionIndicated`. Abstaining is
444    // precisely how it avoids overriding the existing delay: in
445    // `run_classifiers_on_ctx`, a `NoActionIndicated` result is a no-op that
446    // leaves the accumulated verdict (here, the `Some(1s)` delay) untouched.
447    // That end-to-end preservation is exercised in the aws-smithy-runtime
448    // `run_classifiers_on_ctx` test.
449    #[test]
450    fn classify_retry_v2_abstains_when_previous_has_explicit_delay() {
451        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
452        let ctx = ctx_with(Some("5000"), None);
453
454        let previous = RetryAction::retryable_error_with_explicit_delay(
455            ErrorKind::TransientError,
456            Duration::from_secs(1),
457        );
458        assert_eq!(
459            policy.classify_retry_v2(&ctx, &previous),
460            RetryAction::NoActionIndicated,
461        );
462    }
463
464    // Invalid values in `x-amz-retry-after` must be ignored (fall
465    // back to exponential backoff). With no header, or an unparsable one, an
466    // already-retryable `previous` is left as-is (this classifier returns
467    // `NoActionIndicated`).
468    #[test]
469    fn classify_retry_v2_falls_back_without_valid_header() {
470        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
471
472        let previous = RetryAction::transient_error();
473        // Absent header.
474        assert_eq!(
475            policy.classify_retry_v2(&ctx_with(None, None), &previous),
476            RetryAction::NoActionIndicated,
477        );
478        // Unparsable header.
479        assert_eq!(
480            policy.classify_retry_v2(&ctx_with(Some("invalid"), None), &previous),
481            RetryAction::NoActionIndicated,
482        );
483    }
484
485    // Build a context with the given HTTP status, an optional `x-amz-retry-after` (milliseconds)
486    // header, and an error carrying no identifiable AWS error code.
487    fn ctx_with_status(status: u16, retry_after_header_millis: Option<&str>) -> InterceptorContext {
488        let mut builder = http_1x::Response::builder().status(status);
489        if let Some(millis) = retry_after_header_millis {
490            builder = builder.header("x-amz-retry-after", millis);
491        }
492        let res = builder.body("nope").unwrap().map(SdkBody::from);
493        let mut ctx = InterceptorContext::new(Input::doesnt_matter());
494        ctx.set_response(res.try_into().unwrap());
495        ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
496            ErrorMetadata::builder().build(),
497        ))));
498        ctx
499    }
500
501    // A bare 429 with no modeled error code is a throttling error.
502    #[test]
503    fn bare_429_is_throttling() {
504        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
505        assert_eq!(
506            policy.classify_retry(&ctx_with_status(429, None)),
507            RetryAction::throttling_error()
508        );
509    }
510
511    // The 429 throttling verdict still honors x-amz-retry-after (5000ms == 5s).
512    #[test]
513    fn bare_429_carries_retry_after() {
514        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
515        assert_eq!(
516            policy.classify_retry(&ctx_with_status(429, Some("5000"))),
517            RetryAction::retryable_error_with_explicit_delay(
518                ErrorKind::ThrottlingError,
519                Duration::from_secs(5)
520            )
521        );
522    }
523
524    // A non-429 response with no error code is not retried by this classifier.
525    #[test]
526    fn non_429_without_code_is_no_action() {
527        let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
528        assert_eq!(
529            policy.classify_retry(&ctx_with_status(418, None)),
530            RetryAction::NoActionIndicated
531        );
532    }
533
534    // A modeled error's retryable trait wins over the bare-429 rule, because
535    // AwsErrorCodeClassifier runs before ModeledAsRetryableClassifier.
536    #[test]
537    fn modeled_retryable_wins_over_bare_429() {
538        use aws_smithy_runtime::client::retries::classifiers::{
539            run_classifiers_on_ctx, ModeledAsRetryableClassifier,
540        };
541        use aws_smithy_runtime_api::client::retries::classifiers::SharedRetryClassifier;
542        use aws_smithy_types::retry::ProvideErrorKind;
543
544        #[derive(Debug)]
545        struct TestError {
546            meta: ErrorMetadata,
547            retryable: Option<ErrorKind>,
548        }
549        impl fmt::Display for TestError {
550            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
551                write!(f, "test error")
552            }
553        }
554        impl std::error::Error for TestError {}
555        impl ProvideErrorMetadata for TestError {
556            fn meta(&self) -> &ErrorMetadata {
557                &self.meta
558            }
559        }
560        impl ProvideErrorKind for TestError {
561            fn retryable_error_kind(&self) -> Option<ErrorKind> {
562                self.retryable
563            }
564            fn code(&self) -> Option<&str> {
565                None
566            }
567        }
568
569        fn ctx_for(retryable: Option<ErrorKind>) -> InterceptorContext {
570            let res = http_1x::Response::builder()
571                .status(429)
572                .body("nope")
573                .unwrap()
574                .map(SdkBody::from);
575            let mut ctx = InterceptorContext::new(Input::doesnt_matter());
576            ctx.set_response(res.try_into().unwrap());
577            ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(TestError {
578                meta: ErrorMetadata::builder().build(),
579                retryable,
580            }))));
581            ctx
582        }
583
584        // Priority order: AwsErrorCodeClassifier runs before the modeled classifier.
585        let classifiers = || {
586            vec![
587                SharedRetryClassifier::new(AwsErrorCodeClassifier::<TestError>::new()),
588                SharedRetryClassifier::new(ModeledAsRetryableClassifier::<TestError>::new()),
589            ]
590        };
591
592        // Modeled retryable -> the trait wins over the 429 throttling rule.
593        assert_eq!(
594            run_classifiers_on_ctx(
595                classifiers().into_iter(),
596                &ctx_for(Some(ErrorKind::ServerError))
597            ),
598            RetryAction::retryable_error(ErrorKind::ServerError),
599        );
600
601        // Not modeled-retryable -> the bare-429 throttling rule stands.
602        assert_eq!(
603            run_classifiers_on_ctx(classifiers().into_iter(), &ctx_for(None)),
604            RetryAction::throttling_error(),
605        );
606    }
607}