use std::time::Duration;
use crate::auth::TokenService;
use crate::error::TransferError;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetryPolicy {
pub max_attempts: u32,
pub base_delay: Duration,
pub max_duration: Duration,
pub jitter: bool,
pub honor_retry_after: bool,
pub retry_on_429: bool,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_attempts: 5,
base_delay: Duration::from_secs(3),
max_duration: Duration::from_secs(6 * 60),
jitter: true,
honor_retry_after: true,
retry_on_429: true,
}
}
}
impl RetryPolicy {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub const fn with_max_attempts(mut self, max_attempts: u32) -> Self {
self.max_attempts = max_attempts;
self
}
#[must_use]
pub const fn with_base_delay(mut self, base_delay: Duration) -> Self {
self.base_delay = base_delay;
self
}
#[must_use]
pub const fn with_max_duration(mut self, max_duration: Duration) -> Self {
self.max_duration = max_duration;
self
}
#[must_use]
pub const fn with_jitter(mut self, jitter: bool) -> Self {
self.jitter = jitter;
self
}
#[must_use]
pub const fn with_honor_retry_after(mut self, honor: bool) -> Self {
self.honor_retry_after = honor;
self
}
#[must_use]
pub const fn with_retry_on_429(mut self, retry: bool) -> Self {
self.retry_on_429 = retry;
self
}
#[must_use]
pub fn dedup() -> Self {
Self::default().with_retry_on_429(false)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RetryScope {
Read,
Write,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct RetryMarkers {
pub expected_404: bool,
pub expected_416: bool,
pub retry_on_429: bool,
pub retry_on_403: bool,
}
impl Default for RetryMarkers {
fn default() -> Self {
Self {
expected_404: false,
expected_416: false,
retry_on_429: true,
retry_on_403: false,
}
}
}
impl RetryMarkers {
pub(crate) const fn dedup() -> Self {
Self {
expected_404: true,
expected_416: false,
retry_on_429: false,
retry_on_403: false,
}
}
pub(crate) const fn reconstruction() -> Self {
Self {
expected_404: false,
expected_416: true,
retry_on_429: true,
retry_on_403: false,
}
}
pub(crate) const fn ranged_xorb() -> Self {
Self {
expected_404: false,
expected_416: true,
retry_on_429: true,
retry_on_403: true,
}
}
}
#[derive(Clone)]
pub struct RetryContext {
pub(crate) policy: RetryPolicy,
pub(crate) tokens: Option<TokenService>,
pub(crate) scope: RetryScope,
pub(crate) markers: RetryMarkers,
}
impl RetryContext {
pub(crate) async fn run<T, F, Fut>(
&self,
initial_token: String,
mut attempt: F,
) -> Result<T, TransferError>
where
F: FnMut(String) -> Fut,
Fut: std::future::Future<Output = Result<T, TransferError>>,
{
let max_requests = self.policy.max_attempts.saturating_add(1);
let mut token = initial_token;
let mut requests = 0u32;
let mut refreshed = false;
loop {
requests = requests.saturating_add(1);
let result = attempt(token.clone()).await;
match result {
Ok(value) => return Ok(value),
Err(error @ (TransferError::Unauthorized(_) | TransferError::Forbidden(_)))
if self.tokens.is_some() && !refreshed && self.refresh_allowed(&error) =>
{
token = self.refresh_token().await?;
refreshed = true;
}
Err(error)
if is_retryable(&error, self.markers.retry_on_429)
&& requests < max_requests =>
{
let delay = self.backoff_delay(requests.saturating_sub(1), &error);
tokio::time::sleep(delay).await;
}
Err(error) => return Err(error),
}
}
}
const fn refresh_allowed(&self, error: &TransferError) -> bool {
match error {
TransferError::Forbidden(_) => self.markers.retry_on_403,
TransferError::Unauthorized(_) => true,
TransferError::BadRequest(_)
| TransferError::NotFound(_)
| TransferError::RangeNotSatisfiable(_)
| TransferError::TooManyRequests { .. }
| TransferError::HttpStatus { .. }
| TransferError::Transport(_)
| TransferError::InvalidResponse(_)
| TransferError::MalformedMultipart(_)
| TransferError::TokenRefresh(_) => false,
}
}
async fn refresh_token(&self) -> Result<String, TransferError> {
let tokens = self
.tokens
.as_ref()
.ok_or_else(|| TransferError::TokenRefresh("no token service configured".to_owned()))?;
let scoped = match self.scope {
RetryScope::Read => {
tokens.invalidate_read();
tokens.read_token().await
}
RetryScope::Write => {
tokens.invalidate_write();
tokens.write_token().await
}
};
scoped
.map(|token| token.token)
.map_err(|error| TransferError::TokenRefresh(error.to_string()))
}
fn backoff_delay(&self, attempt: u32, error: &TransferError) -> Duration {
if self.policy.honor_retry_after
&& let Some(retry_after) = error.retry_after()
{
return retry_after.min(self.policy.max_duration);
}
let exponential = self
.policy
.base_delay
.saturating_mul(1u32 << attempt.min(16));
let capped = exponential.min(self.policy.max_duration);
if !self.policy.jitter {
return capped;
}
let jittered_nanos = capped
.as_nanos()
.saturating_mul(u128::from(jitter_per_mille(attempt)))
/ 1000;
let nanos = jittered_nanos.min(self.policy.max_duration.as_nanos());
let seconds = u64::try_from(nanos / 1_000_000_000).unwrap_or(u64::MAX);
let subsecond = u32::try_from(nanos % 1_000_000_000).unwrap_or(999_999_999);
Duration::new(seconds, subsecond)
}
}
fn is_retryable(error: &TransferError, retry_on_429: bool) -> bool {
match error {
TransferError::TooManyRequests { .. } => retry_on_429,
TransferError::HttpStatus { status, .. } => matches!(status, 408 | 500 | 503 | 504),
TransferError::Transport(reqwest_error) => {
reqwest_error.is_connect() || reqwest_error.is_timeout()
}
TransferError::BadRequest(_)
| TransferError::Unauthorized(_)
| TransferError::Forbidden(_)
| TransferError::NotFound(_)
| TransferError::RangeNotSatisfiable(_)
| TransferError::InvalidResponse(_)
| TransferError::MalformedMultipart(_)
| TransferError::TokenRefresh(_) => false,
}
}
fn jitter_per_mille(attempt: u32) -> u32 {
let x = u64::from(attempt)
.wrapping_mul(2_654_435_761)
.wrapping_add(0x9E37_79B9_7F4A_7C15);
u32::try_from(x % 1000).unwrap_or(0).saturating_add(500)
}
#[cfg(test)]
#[allow(clippy::float_arithmetic)] fn jitter_factor(attempt: u32) -> f64 {
f64::from(jitter_per_mille(attempt)) / 1000.0
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::time::Duration;
use crate::auth::TokenService;
use super::{RetryContext, RetryMarkers, RetryPolicy, RetryScope, is_retryable, jitter_factor};
use crate::error::TransferError;
fn transient_error() -> TransferError {
TransferError::HttpStatus {
status: 503,
message: String::new(),
retry_after: None,
}
}
#[test]
fn backoff_preserves_submillisecond_duration_and_growth_limit() {
for base in [Duration::from_micros(500), Duration::from_micros(1500)] {
let ctx = retry_context(
RetryPolicy::new()
.with_base_delay(base)
.with_max_duration(Duration::MAX)
.with_jitter(false),
None,
);
assert_eq!(ctx.backoff_delay(0, &transient_error()), base);
assert_eq!(
ctx.backoff_delay(1, &transient_error()),
base.saturating_mul(2)
);
assert_eq!(
ctx.backoff_delay(16, &transient_error()),
ctx.backoff_delay(u32::MAX, &transient_error())
);
}
}
#[test]
fn final_jittered_backoff_never_exceeds_configured_cap() {
let maximum = Duration::from_millis(200);
let ctx = retry_context(
RetryPolicy::new()
.with_base_delay(Duration::from_secs(10))
.with_max_duration(maximum),
None,
);
assert_eq!(ctx.backoff_delay(3, &transient_error()), maximum);
assert_eq!(
ctx.backoff_delay(0, &transient_error()),
Duration::from_millis(197)
);
for attempt in 0..200 {
assert!(ctx.backoff_delay(attempt, &transient_error()) <= maximum);
}
}
#[test]
fn huge_duration_caps_do_not_wrap_or_panic() {
for maximum in [
Duration::MAX,
Duration::from_millis(u64::MAX).saturating_add(Duration::from_millis(1)),
] {
let ctx = retry_context(
RetryPolicy::new()
.with_base_delay(Duration::from_millis(100))
.with_max_duration(maximum)
.with_jitter(false),
None,
);
assert_eq!(
ctx.backoff_delay(0, &transient_error()),
Duration::from_millis(100)
);
let ctx = retry_context(
RetryPolicy::new()
.with_base_delay(Duration::MAX)
.with_max_duration(maximum),
None,
);
for attempt in 0..20 {
assert!(ctx.backoff_delay(attempt, &transient_error()) <= maximum);
}
}
}
#[test]
fn nanosecond_rounding_zero_and_retry_after_controls() {
let error = transient_error();
let ctx = retry_context(
RetryPolicy::new()
.with_base_delay(Duration::from_nanos(1))
.with_max_duration(Duration::from_secs(1)),
None,
);
assert_eq!(ctx.backoff_delay(0, &error), Duration::ZERO);
assert_eq!(ctx.backoff_delay(3, &error), Duration::from_nanos(10));
for policy in [
RetryPolicy::new().with_base_delay(Duration::ZERO),
RetryPolicy::new().with_max_duration(Duration::ZERO),
] {
let ctx = retry_context(policy, None);
assert_eq!(ctx.backoff_delay(0, &error), Duration::ZERO);
}
let capped = Duration::from_micros(500);
let ctx = retry_context(RetryPolicy::new().with_max_duration(capped), None);
for seconds in [0, 1, u64::MAX] {
let error = TransferError::HttpStatus {
status: 503,
message: String::new(),
retry_after: Some(seconds),
};
assert_eq!(
ctx.backoff_delay(3, &error),
Duration::from_secs(seconds).min(capped)
);
}
let ctx = retry_context(
RetryPolicy::new()
.with_base_delay(Duration::from_millis(7))
.with_jitter(false)
.with_honor_retry_after(false),
None,
);
let error = TransferError::HttpStatus {
status: 503,
message: String::new(),
retry_after: Some(100),
};
assert_eq!(ctx.backoff_delay(0, &error), Duration::from_millis(7));
}
#[test]
fn default_policy_matches_reference_defaults() {
let policy = RetryPolicy::default();
assert_eq!(policy.max_attempts, 5);
assert_eq!(policy.base_delay, Duration::from_secs(3));
assert_eq!(policy.max_duration, Duration::from_secs(6 * 60));
assert!(policy.jitter);
assert!(policy.honor_retry_after);
assert!(policy.retry_on_429);
assert_eq!(policy.max_attempts.saturating_add(1), 6);
}
#[test]
fn classification_marks_retryable_and_non_retryable() {
let rt = |status| {
is_retryable(
&TransferError::HttpStatus {
status,
message: String::new(),
retry_after: None,
},
true,
)
};
assert!(rt(408));
assert!(rt(500));
assert!(rt(503));
assert!(rt(504));
assert!(!rt(400));
assert!(!rt(404));
assert!(!rt(416));
let _ = is_retryable(&TransferError::BadRequest(String::new()), true);
let _ = is_retryable(&TransferError::NotFound(String::new()), true);
assert!(!is_retryable(
&TransferError::RangeNotSatisfiable(String::new()),
true
));
assert!(is_retryable(
&TransferError::TooManyRequests {
message: String::new(),
retry_after: None
},
true
));
assert!(!is_retryable(
&TransferError::TooManyRequests {
message: String::new(),
retry_after: None
},
false
));
}
#[test]
fn retry_markers_default_and_dedup() {
let markers = RetryMarkers::default();
assert!(markers.retry_on_429);
assert!(!markers.retry_on_403);
let dedup = RetryMarkers::dedup();
assert!(!dedup.retry_on_429);
assert!(dedup.expected_404);
let recon = RetryMarkers::reconstruction();
assert!(recon.expected_416);
let xorb = RetryMarkers::ranged_xorb();
assert!(xorb.retry_on_403);
}
#[test]
fn jitter_is_bounded() {
for attempt in 0..200 {
let factor = jitter_factor(attempt);
assert!(
(0.5..1.5).contains(&factor),
"factor {factor} out of bounds"
);
}
}
#[test]
fn retry_scope_is_comparable() {
assert_eq!(RetryScope::Read, RetryScope::Read);
assert_ne!(RetryScope::Read, RetryScope::Write);
}
use crate::RepositoryId;
use crate::auth::Auth;
use serde_json::json;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{method, path},
};
async fn probe(server: &MockServer, token: &str) -> Result<(), TransferError> {
let response = reqwest::Client::new()
.get(format!("{}/probe", server.uri()))
.bearer_auth(token)
.send()
.await?;
if response.status().is_success() {
return Ok(());
}
let retry_after = response
.headers()
.get(reqwest::header::RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse::<u64>().ok());
let status = response.status();
let message = response.text().await.unwrap_or_default();
Err(match status.as_u16() {
400 => TransferError::BadRequest(message),
401 => TransferError::Unauthorized(message),
403 => TransferError::Forbidden(message),
404 => TransferError::NotFound(message),
416 => TransferError::RangeNotSatisfiable(message),
429 => TransferError::TooManyRequests {
message,
retry_after,
},
code => TransferError::HttpStatus {
status: code,
message,
retry_after,
},
})
}
fn retry_context(policy: RetryPolicy, tokens: Option<TokenService>) -> RetryContext {
RetryContext {
policy,
tokens,
scope: RetryScope::Read,
markers: RetryMarkers::default(),
}
}
async fn request_count(server: &MockServer) -> usize {
server.received_requests().await.unwrap_or_default().len()
}
fn write_token_service(server: &MockServer) -> TokenService {
let auth = Auth::new(
&server.uri(),
RepositoryId {
provider: "github".to_owned(),
owner: "team".to_owned(),
repo: "assets".to_owned(),
revision: "main".to_owned(),
},
)
.unwrap()
.with_token("server-token".to_owned());
auth.build().unwrap()
}
async fn mount_write_token(server: &MockServer, token: &str) {
Mock::given(method("GET"))
.and(path("/api/github/team/assets/xet-write-token/main"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"casUrl": server.uri(),
"exp": 4_000_000_000u64,
"accessToken": token,
})))
.mount(server)
.await;
}
#[tokio::test]
async fn retryable_status_retried_until_success() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(503).set_body_json(json!({"error": "x"})))
.up_to_n_times(3)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"ok"))
.mount(&server)
.await;
let ctx = retry_context(
RetryPolicy::new()
.with_max_attempts(5)
.with_base_delay(Duration::from_millis(1))
.with_jitter(false),
None,
);
let result = ctx
.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
})
.await;
assert!(result.is_ok());
assert_eq!(request_count(&server).await, 4);
}
#[tokio::test]
async fn max_attempts_semantics_yield_up_to_attempts_plus_one_requests() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(503).set_body_json(json!({"error": "x"})))
.mount(&server)
.await;
let ctx = retry_context(
RetryPolicy::new()
.with_max_attempts(5)
.with_base_delay(Duration::from_millis(1))
.with_jitter(false),
None,
);
let result = ctx
.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
})
.await;
assert!(result.is_err());
assert_eq!(request_count(&server).await, 6);
}
#[tokio::test]
async fn non_retryable_status_surfaces_immediately() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(400).set_body_json(json!({"error": "bad"})))
.mount(&server)
.await;
let ctx = retry_context(
RetryPolicy::new()
.with_max_attempts(5)
.with_base_delay(Duration::from_millis(1)),
None,
);
let result = ctx
.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
})
.await;
assert!(matches!(result, Err(TransferError::BadRequest(_))));
assert_eq!(request_count(&server).await, 1);
}
#[tokio::test]
async fn retry_after_header_short_circuits_backoff() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(
ResponseTemplate::new(429)
.insert_header("Retry-After", "0")
.set_body_json(json!({"error": "slow down"})),
)
.up_to_n_times(2)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"ok"))
.mount(&server)
.await;
let ctx = retry_context(
RetryPolicy::new()
.with_max_attempts(5)
.with_base_delay(Duration::from_secs(10))
.with_jitter(false),
None,
);
let result = tokio::time::timeout(
Duration::from_secs(5),
ctx.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
}),
)
.await
.expect("Retry-After was not honored; backoff took too long");
assert!(result.is_ok());
assert_eq!(request_count(&server).await, 3);
}
#[tokio::test]
async fn retry_on_429_disabled_surfaces_immediately() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(429).set_body_json(json!({"error": "overloaded"})))
.mount(&server)
.await;
let mut ctx = retry_context(
RetryPolicy::new()
.with_max_attempts(5)
.with_base_delay(Duration::from_millis(1)),
None,
);
ctx.markers.retry_on_429 = false;
let result = ctx
.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
})
.await;
assert!(matches!(result, Err(TransferError::TooManyRequests { .. })));
assert_eq!(request_count(&server).await, 1);
}
#[tokio::test]
async fn unauthorized_triggers_single_flight_token_refresh_and_one_retry() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(401).set_body_json(json!({"error": "expired"})))
.up_to_n_times(1)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"ok"))
.mount(&server)
.await;
mount_write_token(&server, "refreshed-token").await;
let ctx = RetryContext {
policy: RetryPolicy::new().with_max_attempts(5),
tokens: Some(write_token_service(&server)),
scope: RetryScope::Write,
markers: RetryMarkers::default(),
};
let result = ctx
.run("old-token".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
})
.await;
assert!(result.is_ok());
assert_eq!(request_count(&server).await, 3);
let token_hits = server
.received_requests()
.await
.unwrap_or_default()
.iter()
.filter(|request| request.url.path().ends_with("/xet-write-token/main"))
.count();
assert_eq!(token_hits, 1);
}
#[tokio::test]
async fn repeated_forbidden_after_refresh_is_surfaced_loop_guarded() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(403).set_body_json(json!({"error": "no scope"})))
.mount(&server)
.await;
mount_write_token(&server, "still-forbidden").await;
let ctx = RetryContext {
policy: RetryPolicy::new().with_max_attempts(5),
tokens: Some(write_token_service(&server)),
scope: RetryScope::Write,
markers: RetryMarkers {
retry_on_403: true,
..RetryMarkers::default()
},
};
let result = ctx
.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
})
.await;
assert!(matches!(result, Err(TransferError::Forbidden(_))));
assert_eq!(request_count(&server).await, 3);
}
#[tokio::test]
async fn jitter_produces_bounded_no_panic_sanity() {
let server = Arc::new(MockServer::start().await);
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(503).set_body_json(json!({"error": "x"})))
.up_to_n_times(2)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/probe"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"ok"))
.mount(&server)
.await;
let ctx = retry_context(
RetryPolicy::new()
.with_max_attempts(5)
.with_base_delay(Duration::from_millis(1))
.with_jitter(true),
None,
);
let result = tokio::time::timeout(
Duration::from_secs(5),
ctx.run("tok".to_owned(), |tok| {
let server = server.clone();
async move { probe(&server, &tok).await }
}),
)
.await
.expect("jittered backoff exceeded timeout");
assert!(result.is_ok());
}
}