use std::future::Future;
use std::time::Duration;
use aws_sdk_s3::config::retry::{RetryPartition, TokenBucket};
use crate::error::{Error, ErrorKind};
pub(crate) use crate::metrics::latency::GuardError;
use crate::metrics::RequestMetrics;
const MAX_ATTEMPTS: u32 = 3;
const INITIAL_BACKOFF: Duration = Duration::from_millis(100);
const THROTTLE_INITIAL_BACKOFF: Duration = Duration::from_secs(1);
const MAX_BACKOFF: Duration = Duration::from_secs(5);
#[derive(Clone, Copy)]
pub(crate) struct Backoff {
initial: Duration,
max: Duration,
}
impl Backoff {
pub(crate) const fn transient() -> Self {
Self {
initial: INITIAL_BACKOFF,
max: MAX_BACKOFF,
}
}
pub(crate) const fn throttle() -> Self {
Self {
initial: THROTTLE_INITIAL_BACKOFF,
max: MAX_BACKOFF,
}
}
fn ceiling(&self, retry_index: u32) -> Duration {
let factor = 2u32.saturating_pow(retry_index);
self.initial.saturating_mul(factor).min(self.max)
}
pub(crate) fn delay(&self, retry_index: u32, rand_unit: f64) -> Duration {
self.ceiling(retry_index).mul_f64(rand_unit)
}
}
#[derive(Debug, PartialEq, Eq)]
pub(crate) enum RetryDecision {
Retry,
RetryThrottle,
NoRetry,
}
pub(crate) async fn retry<T, F, Fut>(
metrics: &mut RequestMetrics,
classify: impl Fn(&GuardError<Error>) -> RetryDecision,
mut build: F,
) -> Result<T, Error>
where
F: FnMut(bool) -> Fut,
Fut: Future<Output = Result<T, GuardError<Error>>>,
{
let transient = Backoff::transient();
let throttle = Backoff::throttle();
let mut attempt = 1u32;
let mut hedged = false;
loop {
match build(!hedged).await {
Ok(val) => return Ok(val),
Err(ge) => {
let is_hedge = matches!(ge, GuardError::DeadlineExceeded(_));
let free_hedge = is_hedge && !hedged;
let decision = classify(&ge);
let backoff = match decision {
RetryDecision::NoRetry => return Err(into_error(ge)),
_ if !free_hedge && attempt >= MAX_ATTEMPTS => {
metrics.record_retry_exhaustion();
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER,
attempts = attempt,
max_attempts = MAX_ATTEMPTS,
cause = %retry_cause(&ge),
"retry budget exhausted, returning last error"
);
return Err(into_error(ge));
}
RetryDecision::Retry => &transient,
RetryDecision::RetryThrottle => &throttle,
};
let delay = backoff.delay(attempt - 1, fastrand::f64());
if free_hedge {
metrics.record_hedge_reissue(delay);
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER,
backoff_ms = delay.as_millis() as u64,
"hedging request after latency deadline (free reissue)"
);
} else {
match decision {
RetryDecision::Retry => metrics.record_retry_reissue(delay),
RetryDecision::RetryThrottle => metrics.record_throttle_reissue(delay),
RetryDecision::NoRetry => unreachable!("terminal decisions return above"),
}
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER,
attempt,
max_attempts = MAX_ATTEMPTS,
throttled = matches!(decision, RetryDecision::RetryThrottle),
backoff_ms = delay.as_millis() as u64,
cause = %retry_cause(&ge),
"retrying operation"
);
}
tokio::time::sleep(delay).await;
if is_hedge {
hedged = true;
}
if !free_hedge {
attempt += 1;
}
}
}
}
}
fn into_error(ge: GuardError<Error>) -> Error {
match ge {
GuardError::DeadlineExceeded(deadline) => Error::new(
ErrorKind::IOError,
format!(
"request exceeded the {}ms latency deadline",
deadline.as_millis()
),
),
GuardError::Inner(e) => e,
}
}
fn retry_cause(ge: &GuardError<Error>) -> String {
match ge {
GuardError::DeadlineExceeded(dl) => format!("deadline exceeded ({}ms)", dl.as_millis()),
GuardError::Inner(e) => e.to_string(),
}
}
pub(crate) fn classify_upload_part_retry(ge: &GuardError<Error>) -> RetryDecision {
match ge {
GuardError::DeadlineExceeded(_) => RetryDecision::NoRetry,
GuardError::Inner(e) if e.is_transient_transport() => RetryDecision::Retry,
GuardError::Inner(e) if is_throttle(e) => RetryDecision::RetryThrottle,
GuardError::Inner(_) => RetryDecision::NoRetry,
}
}
fn is_throttle(e: &Error) -> bool {
matches!(
e.code(),
Some(
"SlowDown"
| "Throttling"
| "ThrottlingException"
| "RequestLimitExceeded"
| "BandwidthLimitExceeded"
)
)
}
pub(crate) fn classify_discovery_retry(ge: &GuardError<Error>) -> RetryDecision {
match ge {
GuardError::DeadlineExceeded(_) => RetryDecision::NoRetry,
GuardError::Inner(e) if e.is_transient_transport() => RetryDecision::Retry,
GuardError::Inner(e)
if is_throttle(e)
|| (e.operation_name() == Some("HeadObject")
&& matches!(e.http_status(), Some(429 | 503))) =>
{
RetryDecision::RetryThrottle
}
GuardError::Inner(_) => RetryDecision::NoRetry,
}
}
pub(crate) fn classify_body_retry(ge: &GuardError<Error>) -> RetryDecision {
let err = match ge {
GuardError::DeadlineExceeded(_) => return RetryDecision::Retry,
GuardError::Inner(e) => e,
};
match err.kind() {
ErrorKind::IntegrityError(_) => RetryDecision::NoRetry,
ErrorKind::ServiceError if is_throttle(err) => RetryDecision::RetryThrottle,
ErrorKind::ServiceError if err.is_transient_transport() => RetryDecision::Retry,
ErrorKind::IOError => RetryDecision::Retry,
_ => RetryDecision::NoRetry,
}
}
const RETRY_BUCKET_REFILL_RATE: f32 = 10.0;
const RETRY_BUCKET_CAPACITY: usize = 500;
pub(crate) fn bucket_retry_partition(bucket: &str) -> RetryPartition {
RetryPartition::custom(format!("s3-tm-{bucket}"))
.token_bucket(
TokenBucket::builder()
.capacity(RETRY_BUCKET_CAPACITY)
.refill_rate(RETRY_BUCKET_REFILL_RATE)
.build(),
)
.build()
}
const STALL_GRACE: Duration = Duration::from_secs(2);
pub(crate) fn tightened_ssp() -> aws_sdk_s3::config::StalledStreamProtectionConfig {
aws_sdk_s3::config::StalledStreamProtectionConfig::enabled()
.grace_period(STALL_GRACE)
.build()
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
fn classify_test(ge: &GuardError<Error>) -> RetryDecision {
match ge {
GuardError::DeadlineExceeded(_) => RetryDecision::Retry,
GuardError::Inner(e) if *e.kind() == ErrorKind::IOError => RetryDecision::Retry,
GuardError::Inner(_) => RetryDecision::NoRetry,
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn retries_then_succeeds() {
let attempts = AtomicUsize::new(0);
let mut metrics = RequestMetrics::default();
let result = retry(&mut metrics, classify_test, |_| {
let n = attempts.fetch_add(1, Ordering::Relaxed);
async move {
if n == 0 {
return Err(GuardError::Inner(Error::new(ErrorKind::IOError, "reset")));
}
Ok::<_, GuardError<Error>>(42)
}
})
.await;
assert_eq!(result.unwrap(), 42);
assert_eq!(attempts.load(Ordering::Relaxed), 2);
assert_eq!(metrics.retry_reissues, 1);
assert_eq!(metrics.throttle_reissues, 0);
assert_eq!(metrics.hedge_reissues, 0);
assert_eq!(metrics.retry_exhaustions, 0);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn deadline_exceeded_is_retried() {
let attempts = AtomicUsize::new(0);
let mut metrics = RequestMetrics::default();
let result: Result<(), _> = retry(&mut metrics, classify_test, |_| {
attempts.fetch_add(1, Ordering::Relaxed);
async { Err::<(), _>(GuardError::DeadlineExceeded(Duration::from_millis(200))) }
})
.await;
let err = result.unwrap_err();
assert_eq!(err.kind(), &ErrorKind::IOError);
assert_eq!(attempts.load(Ordering::Relaxed), MAX_ATTEMPTS as usize + 1);
assert_eq!(metrics.hedge_reissues, 1);
assert_eq!(metrics.retry_reissues, 2);
assert_eq!(metrics.retry_exhaustions, 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn hedge_allowed_only_until_first_deadline_exceedance() {
let flags = std::sync::Mutex::new(Vec::new());
let mut metrics = RequestMetrics::default();
let result: Result<(), _> = retry(&mut metrics, classify_test, |allow_hedge| {
flags.lock().unwrap().push(allow_hedge);
async { Err::<(), _>(GuardError::DeadlineExceeded(Duration::from_millis(200))) }
})
.await;
assert!(result.is_err());
assert_eq!(
*flags.lock().unwrap(),
vec![true, false, false, false],
"hedge allowed on the first call only, then cleared; the free hedge \
adds one iteration on top of MAX_ATTEMPTS"
);
assert_eq!(metrics.hedge_reissues, 1);
assert_eq!(metrics.retry_reissues, 2);
assert_eq!(metrics.retry_exhaustions, 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn non_deadline_failures_do_not_clear_the_hedge_flag() {
let flags = std::sync::Mutex::new(Vec::new());
let attempts = AtomicUsize::new(0);
let mut metrics = RequestMetrics::default();
let result: Result<(), _> = retry(&mut metrics, classify_test, |allow_hedge| {
flags.lock().unwrap().push(allow_hedge);
let n = attempts.fetch_add(1, Ordering::Relaxed);
async move {
if n < 2 {
return Err(GuardError::Inner(Error::new(ErrorKind::IOError, "reset")));
}
Ok::<(), GuardError<Error>>(())
}
})
.await;
assert!(result.is_ok());
assert_eq!(
*flags.lock().unwrap(),
vec![true, true, true],
"inner errors are not hedges; the hedge allowance stays open"
);
assert_eq!(metrics.retry_reissues, 2);
assert_eq!(metrics.hedge_reissues, 0);
assert_eq!(metrics.retry_exhaustions, 0);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn hedge_does_not_consume_a_transport_retry() {
let attempts = AtomicUsize::new(0);
let mut metrics = RequestMetrics::default();
let result = retry(&mut metrics, classify_test, |allow_hedge| {
let n = attempts.fetch_add(1, Ordering::Relaxed);
async move {
if n == 0 {
assert!(allow_hedge, "hedge must be allowed on the first call");
return Err(GuardError::DeadlineExceeded(Duration::from_millis(200)));
}
assert!(!allow_hedge, "hedge cleared after the first exceedance");
if (n as u32) < MAX_ATTEMPTS {
return Err(GuardError::Inner(Error::new(ErrorKind::IOError, "connect")));
}
Ok::<_, GuardError<Error>>(7)
}
})
.await;
assert_eq!(
result.unwrap(),
7,
"the full transport-retry budget survives a hedge"
);
assert_eq!(
attempts.load(Ordering::Relaxed),
MAX_ATTEMPTS as usize + 1,
"hedge is free; genuine failures keep the full MAX_ATTEMPTS"
);
assert_eq!(metrics.hedge_reissues, 1);
assert_eq!(metrics.retry_reissues, 2);
assert_eq!(metrics.retry_exhaustions, 0);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn exhausts_attempts_on_persistent_retryable_error() {
let attempts = AtomicUsize::new(0);
let mut metrics = RequestMetrics::default();
let result: Result<(), _> = retry(&mut metrics, classify_test, |_| {
attempts.fetch_add(1, Ordering::Relaxed);
async { Err::<(), _>(GuardError::Inner(Error::new(ErrorKind::IOError, "reset"))) }
})
.await;
assert!(result.is_err());
assert_eq!(attempts.load(Ordering::Relaxed), MAX_ATTEMPTS as usize);
assert_eq!(metrics.retry_reissues, 2);
assert_eq!(metrics.retry_exhaustions, 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn does_not_retry_terminal_inner_error() {
let attempts = AtomicUsize::new(0);
let mut metrics = RequestMetrics::default();
let result: Result<(), _> = retry(&mut metrics, classify_test, |_| {
attempts.fetch_add(1, Ordering::Relaxed);
async {
Err::<(), _>(GuardError::Inner(Error::new(
ErrorKind::RuntimeError,
"simulated SDK error",
)))
}
})
.await;
assert!(result.is_err());
assert_eq!(attempts.load(Ordering::Relaxed), 1);
assert_eq!(metrics, RequestMetrics::default());
}
#[cfg_attr(miri, ignore)]
#[tokio::test(start_paused = true)]
async fn throttle_decision_retries_and_uses_the_throttle_backoff() {
fn classify_throttle(ge: &GuardError<Error>) -> RetryDecision {
match ge {
GuardError::Inner(e) if is_throttle(e) => RetryDecision::RetryThrottle,
_ => RetryDecision::NoRetry,
}
}
fastrand::seed(2);
let attempts = AtomicUsize::new(0);
let start = tokio::time::Instant::now();
let mut metrics = RequestMetrics::default();
let result: Result<(), _> = retry(&mut metrics, classify_throttle, |_| {
attempts.fetch_add(1, Ordering::Relaxed);
async { Err::<(), _>(GuardError::Inner(Error::test_service_error("SlowDown"))) }
})
.await;
assert!(result.is_err());
assert_eq!(attempts.load(Ordering::Relaxed), MAX_ATTEMPTS as usize);
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_secs(1),
"elapsed {elapsed:?} is below the throttle base; the loop must use the \
throttle backoff (1s), not the transient backoff (max 300ms total)"
);
assert!(
elapsed <= Duration::from_secs(3),
"elapsed {elapsed:?} exceeds the throttle backoff ceiling sum"
);
assert_eq!(metrics.throttle_reissues, 2);
assert_eq!(metrics.retry_reissues, 0);
assert_eq!(metrics.retry_exhaustions, 1);
assert!(
metrics.backoff_duration.abs_diff(elapsed) <= Duration::from_millis(1),
"selected backoff {:?} must account for paused-clock elapsed {elapsed:?}",
metrics.backoff_duration
);
}
#[test]
fn backoff_ceiling_progression_doubles_then_caps() {
let b = Backoff::transient();
assert_eq!(b.delay(0, 1.0), Duration::from_millis(100));
assert_eq!(b.delay(1, 1.0), Duration::from_millis(200));
assert_eq!(b.delay(2, 1.0), Duration::from_millis(400));
assert_eq!(b.delay(3, 1.0), Duration::from_millis(800));
assert_eq!(b.delay(4, 1.0), Duration::from_millis(1600));
assert_eq!(b.delay(5, 1.0), Duration::from_millis(3200));
assert_eq!(b.delay(6, 1.0), Duration::from_secs(5));
assert_eq!(b.delay(7, 1.0), Duration::from_secs(5));
}
#[test]
fn backoff_large_index_saturates_without_overflow() {
let b = Backoff::transient();
assert_eq!(b.delay(u32::MAX, 1.0), Duration::from_secs(5));
}
#[test]
fn backoff_full_jitter_scales_linearly_within_ceiling() {
let b = Backoff::transient();
assert_eq!(b.delay(2, 0.0), Duration::ZERO);
assert_eq!(b.delay(2, 0.5), Duration::from_millis(200));
assert_eq!(b.delay(2, 0.25), Duration::from_millis(100));
}
#[test]
fn backoff_real_draws_stay_within_ceiling() {
let b = Backoff::transient();
for _ in 0..10_000 {
let d = b.delay(2, fastrand::f64());
assert!(
d <= Duration::from_millis(400),
"delay {d:?} exceeded ceiling"
);
}
}
#[test]
fn throttle_backoff_uses_the_1s_base_and_shares_the_cap() {
let b = Backoff::throttle();
assert_eq!(b.delay(0, 1.0), Duration::from_secs(1));
assert_eq!(b.delay(1, 1.0), Duration::from_secs(2));
assert_eq!(b.delay(2, 1.0), Duration::from_secs(4));
assert_eq!(b.delay(3, 1.0), Duration::from_secs(5));
assert_eq!(b.delay(0, 0.0), Duration::ZERO);
}
#[test]
fn upload_transient_transport_retries() {
let d = classify_upload_part_retry(&GuardError::Inner(Error::test_transient_transport()));
assert_eq!(d, RetryDecision::Retry);
}
#[test]
fn upload_non_throttle_service_error_is_terminal() {
let err = Error::test_service_error("AccessDenied");
assert_eq!(
classify_upload_part_retry(&GuardError::Inner(err)),
RetryDecision::NoRetry
);
}
#[test]
fn upload_throttle_retries_with_throttle_decision() {
let err = Error::test_service_error("SlowDown");
assert_eq!(
classify_upload_part_retry(&GuardError::Inner(err)),
RetryDecision::RetryThrottle
);
}
#[test]
fn upload_other_inner_error_is_terminal() {
let err = Error::new(ErrorKind::RuntimeError, "not a transport error");
assert_eq!(
classify_upload_part_retry(&GuardError::Inner(err)),
RetryDecision::NoRetry
);
}
#[test]
fn body_retry_deadline_exceeded_retries() {
let d = classify_body_retry(&GuardError::DeadlineExceeded(Duration::from_secs(1)));
assert_eq!(d, RetryDecision::Retry);
}
#[test]
fn body_retry_cancellation_does_not_retry() {
let err = Error::new(ErrorKind::OperationCancelled, "cancelled mid body");
assert_eq!(
classify_body_retry(&GuardError::Inner(err)),
RetryDecision::NoRetry
);
}
#[test]
fn body_retry_io_error_retries() {
let err = Error::new(ErrorKind::IOError, "connection reset by peer");
assert_eq!(
classify_body_retry(&GuardError::Inner(err)),
RetryDecision::Retry
);
}
#[test]
fn body_retry_transient_transport_retries() {
let d = classify_body_retry(&GuardError::Inner(Error::test_transient_transport()));
assert_eq!(d, RetryDecision::Retry);
}
#[test]
fn body_retry_throttle_retries_with_throttle_decision() {
let err = Error::test_service_error("SlowDown");
assert_eq!(
classify_body_retry(&GuardError::Inner(err)),
RetryDecision::RetryThrottle
);
}
#[test]
fn body_retry_non_throttle_service_error_is_terminal() {
let err = Error::test_service_error("AccessDenied");
assert_eq!(
classify_body_retry(&GuardError::Inner(err)),
RetryDecision::NoRetry
);
}
#[test]
fn discovery_head_retry_uses_status_without_an_error_body() {
use aws_sdk_s3::operation::head_object::HeadObjectError;
use aws_smithy_runtime_api::{
client::orchestrator::HttpResponse, client::result::SdkError, http::StatusCode,
};
use aws_smithy_types::{body::SdkBody, error::ErrorMetadata};
for status in [429, 503, 400, 403, 404, 412, 500] {
let sdk = SdkError::service_error(
HeadObjectError::generic(ErrorMetadata::builder().build()),
HttpResponse::new(StatusCode::try_from(status).unwrap(), SdkBody::empty()),
);
let error = crate::error::service_error(sdk);
assert_eq!(error.code(), None);
assert_eq!(error.http_status(), Some(status));
assert_eq!(
classify_discovery_retry(&GuardError::Inner(error)),
if matches!(status, 429 | 503) {
RetryDecision::RetryThrottle
} else {
RetryDecision::NoRetry
},
"HEAD status {status}",
);
}
}
#[test]
fn discovery_get_retry_does_not_treat_a_codeless_status_as_a_head_error() {
use aws_sdk_s3::operation::get_object::GetObjectError;
use aws_smithy_runtime_api::{
client::orchestrator::HttpResponse, client::result::SdkError, http::StatusCode,
};
use aws_smithy_types::{body::SdkBody, error::ErrorMetadata};
let sdk = SdkError::service_error(
GetObjectError::generic(ErrorMetadata::builder().build()),
HttpResponse::new(StatusCode::try_from(503).unwrap(), SdkBody::empty()),
);
assert_eq!(
classify_discovery_retry(&GuardError::Inner(crate::error::service_error(sdk))),
RetryDecision::NoRetry
);
}
#[test]
fn is_throttle_matches_the_known_codes() {
for code in [
"SlowDown",
"Throttling",
"ThrottlingException",
"RequestLimitExceeded",
"BandwidthLimitExceeded",
] {
assert!(
is_throttle(&Error::test_service_error(code)),
"{code} must be detected as a throttle"
);
}
}
#[test]
fn is_throttle_rejects_non_throttle_and_codeless() {
assert!(!is_throttle(&Error::test_service_error("AccessDenied")));
assert!(!is_throttle(&Error::new(ErrorKind::IOError, "reset")));
}
}