1use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
7use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
8use aws_smithy_runtime_api::client::retries::classifiers::{
9 ClassifyRetry, RetryAction, RetryClassifierPriority, RetryReason,
10};
11use aws_smithy_types::error::metadata::ProvideErrorMetadata;
12use aws_smithy_types::retry::ErrorKind;
13use std::borrow::Cow;
14use std::error::Error as StdError;
15use std::marker::PhantomData;
16
17fn retry_after_from_ctx(ctx: &InterceptorContext) -> Option<std::time::Duration> {
22 ctx.response()
23 .and_then(|res| res.headers().get("x-amz-retry-after"))
24 .and_then(|header| header.parse::<u64>().ok())
25 .map(std::time::Duration::from_millis)
26}
27
28pub const THROTTLING_ERRORS: &[&str] = &[
30 "Throttling",
31 "ThrottlingException",
32 "ThrottledException",
33 "RequestThrottledException",
34 "TooManyRequestsException",
35 "ProvisionedThroughputExceededException",
36 "TransactionInProgressException",
37 "RequestLimitExceeded",
38 "BandwidthLimitExceeded",
39 "LimitExceededException",
40 "RequestThrottled",
41 "SlowDown",
42 "PriorRequestNotComplete",
43 "EC2ThrottledException",
44];
45
46pub const TRANSIENT_ERRORS: &[&str] = &["RequestTimeout", "RequestTimeoutException"];
48
49#[derive(Debug)]
51pub struct AwsErrorCodeClassifier<E> {
52 throttling_errors: Cow<'static, [&'static str]>,
53 transient_errors: Cow<'static, [&'static str]>,
54 _inner: PhantomData<E>,
55}
56
57impl<E> Default for AwsErrorCodeClassifier<E> {
58 fn default() -> Self {
59 Self {
60 throttling_errors: THROTTLING_ERRORS.into(),
61 transient_errors: TRANSIENT_ERRORS.into(),
62 _inner: PhantomData,
63 }
64 }
65}
66
67#[derive(Debug)]
69pub struct AwsErrorCodeClassifierBuilder<E> {
70 throttling_errors: Option<Cow<'static, [&'static str]>>,
71 transient_errors: Option<Cow<'static, [&'static str]>>,
72 _inner: PhantomData<E>,
73}
74
75impl<E> AwsErrorCodeClassifierBuilder<E> {
76 pub fn transient_errors(
78 mut self,
79 transient_errors: impl Into<Cow<'static, [&'static str]>>,
80 ) -> Self {
81 self.transient_errors = Some(transient_errors.into());
82 self
83 }
84
85 pub fn build(self) -> AwsErrorCodeClassifier<E> {
87 AwsErrorCodeClassifier {
88 throttling_errors: self.throttling_errors.unwrap_or(THROTTLING_ERRORS.into()),
89 transient_errors: self.transient_errors.unwrap_or(TRANSIENT_ERRORS.into()),
90 _inner: self._inner,
91 }
92 }
93}
94
95impl<E> AwsErrorCodeClassifier<E> {
96 pub fn new() -> Self {
98 Self::default()
99 }
100
101 pub fn builder() -> AwsErrorCodeClassifierBuilder<E> {
103 AwsErrorCodeClassifierBuilder {
104 throttling_errors: None,
105 transient_errors: None,
106 _inner: PhantomData,
107 }
108 }
109}
110
111impl<E> ClassifyRetry for AwsErrorCodeClassifier<E>
112where
113 E: StdError + ProvideErrorMetadata + Send + Sync + 'static,
114{
115 fn classify_retry(&self, ctx: &InterceptorContext) -> RetryAction {
116 let output_or_error = ctx.output_or_error();
118 let error = match output_or_error {
120 Some(Ok(_)) | None => return RetryAction::NoActionIndicated,
121 Some(Err(err)) => err,
122 };
123
124 let retry_after = retry_after_from_ctx(ctx);
125
126 let error_code = OrchestratorError::as_operation_error(error)
127 .and_then(|err| err.downcast_ref::<E>())
128 .and_then(|err| err.code());
129
130 if let Some(error_code) = error_code {
131 if self.throttling_errors.contains(&error_code) {
132 return RetryAction::RetryIndicated(RetryReason::RetryableError {
133 kind: ErrorKind::ThrottlingError,
134 retry_after,
135 });
136 }
137 if self.transient_errors.contains(&error_code) {
138 return RetryAction::RetryIndicated(RetryReason::RetryableError {
139 kind: ErrorKind::TransientError,
140 retry_after,
141 });
142 }
143 };
144
145 let is_too_many_requests = ctx
149 .response()
150 .map(|res| res.status().into())
151 .map(|status: u16| status == 429)
152 .unwrap_or(false);
153 if is_too_many_requests {
154 return RetryAction::RetryIndicated(RetryReason::RetryableError {
155 kind: ErrorKind::ThrottlingError,
156 retry_after,
157 });
158 }
159
160 RetryAction::NoActionIndicated
161 }
162
163 fn classify_retry_v2(&self, ctx: &InterceptorContext, previous: &RetryAction) -> RetryAction {
164 let own = self.classify_retry(ctx);
168 if own != RetryAction::NoActionIndicated {
169 return own;
170 }
171
172 if let RetryAction::RetryIndicated(RetryReason::RetryableError {
184 kind,
185 retry_after: None,
186 }) = previous
187 {
188 if let Some(retry_after) = retry_after_from_ctx(ctx) {
189 return RetryAction::RetryIndicated(RetryReason::RetryableError {
190 kind: *kind,
191 retry_after: Some(retry_after),
192 });
193 }
194 }
195
196 RetryAction::NoActionIndicated
197 }
198
199 fn name(&self) -> &'static str {
200 "AWS Error Code"
201 }
202
203 fn priority(&self) -> RetryClassifierPriority {
204 RetryClassifierPriority::run_before(
205 RetryClassifierPriority::modeled_as_retryable_classifier(),
206 )
207 }
208}
209
210#[cfg(test)]
211mod test {
212 use crate::retries::classifiers::AwsErrorCodeClassifier;
213 use aws_smithy_runtime_api::client::interceptors::context::InterceptorContext;
214 use aws_smithy_runtime_api::client::interceptors::context::{Error, Input};
215 use aws_smithy_runtime_api::client::orchestrator::OrchestratorError;
216 use aws_smithy_runtime_api::client::retries::classifiers::{ClassifyRetry, RetryAction};
217 use aws_smithy_types::body::SdkBody;
218 use aws_smithy_types::error::metadata::ProvideErrorMetadata;
219 use aws_smithy_types::error::ErrorMetadata;
220 use aws_smithy_types::retry::ErrorKind;
221 use std::fmt;
222 use std::time::Duration;
223
224 #[derive(Debug)]
225 struct CodedError {
226 metadata: ErrorMetadata,
227 }
228
229 impl CodedError {
230 fn new(code: &'static str) -> Self {
231 Self {
232 metadata: ErrorMetadata::builder().code(code).build(),
233 }
234 }
235 }
236
237 impl fmt::Display for CodedError {
238 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
239 write!(f, "Coded Error")
240 }
241 }
242
243 impl std::error::Error for CodedError {}
244
245 impl ProvideErrorMetadata for CodedError {
246 fn meta(&self) -> &ErrorMetadata {
247 &self.metadata
248 }
249 }
250
251 #[test]
252 fn classify_by_error_code() {
253 let policy = AwsErrorCodeClassifier::<CodedError>::new();
254 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
255 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
256 CodedError::new("Throttling"),
257 ))));
258
259 assert_eq!(policy.classify_retry(&ctx), RetryAction::throttling_error());
260
261 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
262 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
263 CodedError::new("RequestTimeout"),
264 ))));
265 assert_eq!(policy.classify_retry(&ctx), RetryAction::transient_error())
266 }
267
268 #[test]
269 fn classify_generic() {
270 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
271 let err = ErrorMetadata::builder().code("SlowDown").build();
272 let test_response = http_1x::Response::new("OK").map(SdkBody::from);
273
274 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
275 ctx.set_response(test_response.try_into().unwrap());
276 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
277
278 assert_eq!(policy.classify_retry(&ctx), RetryAction::throttling_error());
279 }
280
281 #[test]
282 fn test_retry_after_header() {
283 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
284 let err = ErrorMetadata::builder().code("SlowDown").build();
285 let res = http_1x::Response::builder()
286 .header("x-amz-retry-after", "5000")
287 .body("retry later")
288 .unwrap()
289 .map(SdkBody::from);
290 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
291 ctx.set_response(res.try_into().unwrap());
292 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
293
294 assert_eq!(
295 policy.classify_retry(&ctx),
296 RetryAction::retryable_error_with_explicit_delay(
297 ErrorKind::ThrottlingError,
298 Duration::from_secs(5)
299 )
300 );
301 }
302
303 #[test]
304 fn test_invalid_retry_after_header_is_ignored() {
305 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
306 let err = ErrorMetadata::builder().code("SlowDown").build();
307 let res = http_1x::Response::builder()
308 .header("x-amz-retry-after", "invalid")
309 .body("retry later")
310 .unwrap()
311 .map(SdkBody::from);
312 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
313 ctx.set_response(res.try_into().unwrap());
314 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
315
316 assert_eq!(
319 policy.classify_retry(&ctx),
320 RetryAction::retryable_error(ErrorKind::ThrottlingError)
321 );
322 }
323
324 fn ctx_with(
329 retry_after_header_millis: Option<&str>,
330 error_code: Option<&'static str>,
331 ) -> InterceptorContext {
332 let mut builder = http_1x::Response::builder();
333 if let Some(millis) = retry_after_header_millis {
334 builder = builder.header("x-amz-retry-after", millis);
335 }
336 let res = builder.body("nope").unwrap().map(SdkBody::from);
337
338 let err = match error_code {
339 Some(code) => ErrorMetadata::builder().code(code).build(),
340 None => ErrorMetadata::builder().build(),
341 };
342
343 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
344 ctx.set_response(res.try_into().unwrap());
345 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(err))));
346 ctx
347 }
348
349 #[test]
357 fn classify_retry_v2_attaches_server_delay_on_bare_5xx() {
358 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
359 let ctx = ctx_with(Some("5000"), None);
361
362 let previous = RetryAction::transient_error();
363 assert_eq!(
364 policy.classify_retry_v2(&ctx, &previous),
365 RetryAction::retryable_error_with_explicit_delay(
366 ErrorKind::TransientError,
367 Duration::from_secs(5)
368 ),
369 );
370 }
371
372 #[test]
376 fn classify_retry_v2_attaches_server_delay_on_unrecognized_error_code() {
377 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
378 let ctx = ctx_with(Some("5000"), Some("InternalError"));
379
380 let previous = RetryAction::transient_error();
381 assert_eq!(
382 policy.classify_retry_v2(&ctx, &previous),
383 RetryAction::retryable_error_with_explicit_delay(
384 ErrorKind::TransientError,
385 Duration::from_secs(5)
386 ),
387 );
388 }
389
390 #[test]
393 fn classify_retry_v2_preserves_previous_kind() {
394 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
395 let ctx = ctx_with(Some("2500"), None);
396
397 let previous = RetryAction::retryable_error(ErrorKind::ServerError);
398 assert_eq!(
399 policy.classify_retry_v2(&ctx, &previous),
400 RetryAction::retryable_error_with_explicit_delay(
401 ErrorKind::ServerError,
402 Duration::from_millis(2500)
403 ),
404 );
405 }
406
407 #[test]
412 fn classify_retry_v2_prefers_own_error_code_verdict() {
413 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
414 let ctx = ctx_with(Some("5000"), Some("SlowDown"));
415
416 let previous = RetryAction::transient_error();
418 assert_eq!(
419 policy.classify_retry_v2(&ctx, &previous),
420 RetryAction::retryable_error_with_explicit_delay(
421 ErrorKind::ThrottlingError,
422 Duration::from_secs(5)
423 ),
424 );
425 }
426
427 #[test]
431 fn classify_retry_v2_ignores_header_without_retryable_previous() {
432 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
433 let ctx = ctx_with(Some("5000"), None);
434
435 assert_eq!(
436 policy.classify_retry_v2(&ctx, &RetryAction::NoActionIndicated),
437 RetryAction::NoActionIndicated,
438 );
439 }
440
441 #[test]
450 fn classify_retry_v2_abstains_when_previous_has_explicit_delay() {
451 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
452 let ctx = ctx_with(Some("5000"), None);
453
454 let previous = RetryAction::retryable_error_with_explicit_delay(
455 ErrorKind::TransientError,
456 Duration::from_secs(1),
457 );
458 assert_eq!(
459 policy.classify_retry_v2(&ctx, &previous),
460 RetryAction::NoActionIndicated,
461 );
462 }
463
464 #[test]
469 fn classify_retry_v2_falls_back_without_valid_header() {
470 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
471
472 let previous = RetryAction::transient_error();
473 assert_eq!(
475 policy.classify_retry_v2(&ctx_with(None, None), &previous),
476 RetryAction::NoActionIndicated,
477 );
478 assert_eq!(
480 policy.classify_retry_v2(&ctx_with(Some("invalid"), None), &previous),
481 RetryAction::NoActionIndicated,
482 );
483 }
484
485 fn ctx_with_status(status: u16, retry_after_header_millis: Option<&str>) -> InterceptorContext {
488 let mut builder = http_1x::Response::builder().status(status);
489 if let Some(millis) = retry_after_header_millis {
490 builder = builder.header("x-amz-retry-after", millis);
491 }
492 let res = builder.body("nope").unwrap().map(SdkBody::from);
493 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
494 ctx.set_response(res.try_into().unwrap());
495 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(
496 ErrorMetadata::builder().build(),
497 ))));
498 ctx
499 }
500
501 #[test]
503 fn bare_429_is_throttling() {
504 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
505 assert_eq!(
506 policy.classify_retry(&ctx_with_status(429, None)),
507 RetryAction::throttling_error()
508 );
509 }
510
511 #[test]
513 fn bare_429_carries_retry_after() {
514 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
515 assert_eq!(
516 policy.classify_retry(&ctx_with_status(429, Some("5000"))),
517 RetryAction::retryable_error_with_explicit_delay(
518 ErrorKind::ThrottlingError,
519 Duration::from_secs(5)
520 )
521 );
522 }
523
524 #[test]
526 fn non_429_without_code_is_no_action() {
527 let policy = AwsErrorCodeClassifier::<ErrorMetadata>::new();
528 assert_eq!(
529 policy.classify_retry(&ctx_with_status(418, None)),
530 RetryAction::NoActionIndicated
531 );
532 }
533
534 #[test]
537 fn modeled_retryable_wins_over_bare_429() {
538 use aws_smithy_runtime::client::retries::classifiers::{
539 run_classifiers_on_ctx, ModeledAsRetryableClassifier,
540 };
541 use aws_smithy_runtime_api::client::retries::classifiers::SharedRetryClassifier;
542 use aws_smithy_types::retry::ProvideErrorKind;
543
544 #[derive(Debug)]
545 struct TestError {
546 meta: ErrorMetadata,
547 retryable: Option<ErrorKind>,
548 }
549 impl fmt::Display for TestError {
550 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
551 write!(f, "test error")
552 }
553 }
554 impl std::error::Error for TestError {}
555 impl ProvideErrorMetadata for TestError {
556 fn meta(&self) -> &ErrorMetadata {
557 &self.meta
558 }
559 }
560 impl ProvideErrorKind for TestError {
561 fn retryable_error_kind(&self) -> Option<ErrorKind> {
562 self.retryable
563 }
564 fn code(&self) -> Option<&str> {
565 None
566 }
567 }
568
569 fn ctx_for(retryable: Option<ErrorKind>) -> InterceptorContext {
570 let res = http_1x::Response::builder()
571 .status(429)
572 .body("nope")
573 .unwrap()
574 .map(SdkBody::from);
575 let mut ctx = InterceptorContext::new(Input::doesnt_matter());
576 ctx.set_response(res.try_into().unwrap());
577 ctx.set_output_or_error(Err(OrchestratorError::operation(Error::erase(TestError {
578 meta: ErrorMetadata::builder().build(),
579 retryable,
580 }))));
581 ctx
582 }
583
584 let classifiers = || {
586 vec![
587 SharedRetryClassifier::new(AwsErrorCodeClassifier::<TestError>::new()),
588 SharedRetryClassifier::new(ModeledAsRetryableClassifier::<TestError>::new()),
589 ]
590 };
591
592 assert_eq!(
594 run_classifiers_on_ctx(
595 classifiers().into_iter(),
596 &ctx_for(Some(ErrorKind::ServerError))
597 ),
598 RetryAction::retryable_error(ErrorKind::ServerError),
599 );
600
601 assert_eq!(
603 run_classifiers_on_ctx(classifiers().into_iter(), &ctx_for(None)),
604 RetryAction::throttling_error(),
605 );
606 }
607}