use std::time::Duration;
use rand::Rng;
use tonic::{Code, Status};
pub(crate) const UNARY_MAX_ATTEMPTS: u32 = 5;
pub(crate) const UNARY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30);
pub(crate) const STREAM_MAX_CONSECUTIVE_OTHER_FAILURES: u32 = 10;
const UNARY_BASE: Duration = Duration::from_secs(5);
const UNARY_MAX: Duration = Duration::from_secs(80);
const STREAM_BASE: Duration = Duration::from_secs(1);
const STREAM_MAX: Duration = Duration::from_secs(30);
pub(crate) fn unary_backoff_cap(retry: u32) -> Duration {
exponential_cap(UNARY_BASE, UNARY_MAX, retry)
}
pub(crate) fn stream_backoff_cap(attempt: u32) -> Duration {
exponential_cap(STREAM_BASE, STREAM_MAX, attempt)
}
fn exponential_cap(base: Duration, max: Duration, exponent: u32) -> Duration {
base.saturating_mul(2u32.saturating_pow(exponent)).min(max)
}
pub(crate) fn full_jitter(cap: Duration, rng: &mut impl Rng) -> Duration {
let cap_nanos = u64::try_from(cap.as_nanos()).unwrap_or(u64::MAX);
Duration::from_nanos(rng.random_range(0..=cap_nanos))
}
pub(crate) fn is_unary_retryable(code: Code) -> bool {
matches!(
code,
Code::ResourceExhausted | Code::DeadlineExceeded | Code::Internal | Code::Unavailable
)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum StreamErrorClass {
Transient,
Fatal,
Other,
StopClean,
}
pub(crate) fn classify_stream_end(status: Option<&Status>) -> StreamErrorClass {
let Some(status) = status else {
return StreamErrorClass::Transient;
};
match status.code() {
Code::Unavailable | Code::Internal | Code::DeadlineExceeded | Code::ResourceExhausted => {
StreamErrorClass::Transient
}
Code::Unauthenticated
| Code::PermissionDenied
| Code::InvalidArgument
| Code::FailedPrecondition
| Code::NotFound
| Code::Unimplemented => StreamErrorClass::Fatal,
Code::Cancelled => StreamErrorClass::StopClean,
_ => StreamErrorClass::Other,
}
}
pub(crate) fn timed<T>(message: T) -> tonic::Request<T> {
let mut request = tonic::Request::new(message);
request.set_timeout(UNARY_ATTEMPT_TIMEOUT);
request
}
pub(crate) async fn unary<T, F, Fut>(mut call: F) -> Result<T, Status>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, Status>>,
{
let mut retry = 0;
loop {
let status = match tokio::time::timeout(UNARY_ATTEMPT_TIMEOUT, call()).await {
Ok(Ok(value)) => return Ok(value),
Ok(Err(status)) => status,
Err(_) => Status::deadline_exceeded("the attempt timed out"),
};
if !is_unary_retryable(status.code()) || retry + 1 >= UNARY_MAX_ATTEMPTS {
return Err(status);
}
let delay = full_jitter(unary_backoff_cap(retry), &mut rand::rng());
tracing::debug!(code = ?status.code(), retry, ?delay, "retrying gRPC call");
tokio::time::sleep(delay).await;
retry += 1;
}
}
#[cfg(test)]
mod tests {
use super::*;
use rand::SeedableRng;
use rand::rngs::StdRng;
use std::sync::atomic::{AtomicU32, Ordering};
#[test]
fn stream_backoff_cap_doubles_from_one_second_up_to_thirty() {
let caps: Vec<u64> = (0..7).map(|n| stream_backoff_cap(n).as_secs()).collect();
assert_eq!(caps, [1, 2, 4, 8, 16, 30, 30]);
assert_eq!(stream_backoff_cap(u32::MAX), Duration::from_secs(30));
}
#[test]
fn unary_backoff_cap_doubles_from_five_seconds_up_to_eighty() {
let caps: Vec<u64> = (0..6).map(|n| unary_backoff_cap(n).as_secs()).collect();
assert_eq!(caps, [5, 10, 20, 40, 80, 80]);
assert_eq!(unary_backoff_cap(u32::MAX), Duration::from_secs(80));
}
#[test]
fn full_jitter_stays_within_the_cap_and_actually_varies() {
let mut rng = StdRng::seed_from_u64(42);
for attempt in 0..8 {
let cap = stream_backoff_cap(attempt);
let delays: Vec<Duration> = (0..200).map(|_| full_jitter(cap, &mut rng)).collect();
assert!(delays.iter().all(|d| *d <= cap), "attempt {attempt}");
assert!(
delays.iter().any(|d| *d < cap / 2),
"attempt {attempt}: never low"
);
assert!(
delays.iter().any(|d| *d > cap / 2),
"attempt {attempt}: never high"
);
}
}
#[test]
fn full_jitter_is_reproducible_for_a_seed() {
let cap = Duration::from_secs(30);
let a = full_jitter(cap, &mut StdRng::seed_from_u64(7));
let b = full_jitter(cap, &mut StdRng::seed_from_u64(7));
assert_eq!(a, b);
assert_eq!(
full_jitter(Duration::ZERO, &mut StdRng::seed_from_u64(7)),
Duration::ZERO
);
}
#[test]
fn unary_retryable_codes() {
for code in [
Code::ResourceExhausted,
Code::DeadlineExceeded,
Code::Internal,
Code::Unavailable,
] {
assert!(is_unary_retryable(code), "{code:?}");
}
for code in [
Code::InvalidArgument,
Code::NotFound,
Code::Unauthenticated,
Code::Cancelled,
Code::Unknown,
] {
assert!(!is_unary_retryable(code), "{code:?}");
}
}
#[test]
fn stream_end_classification_follows_the_design_table() {
use StreamErrorClass::*;
assert_eq!(classify_stream_end(None), Transient, "EOF");
let table = [
(Code::Unavailable, Transient),
(Code::Internal, Transient),
(Code::DeadlineExceeded, Transient),
(Code::ResourceExhausted, Transient),
(Code::Unauthenticated, Fatal),
(Code::PermissionDenied, Fatal),
(Code::InvalidArgument, Fatal),
(Code::FailedPrecondition, Fatal),
(Code::NotFound, Fatal),
(Code::Unimplemented, Fatal),
(Code::Unknown, Other),
(Code::Cancelled, StopClean),
(Code::Aborted, Other),
(Code::AlreadyExists, Other),
(Code::OutOfRange, Other),
(Code::DataLoss, Other),
];
for (code, expected) in table {
let status = Status::new(code, "x");
assert_eq!(classify_stream_end(Some(&status)), expected, "{code:?}");
}
}
#[tokio::test(start_paused = true)]
async fn unary_returns_the_first_success_without_retrying() {
let calls = AtomicU32::new(0);
let result = unary(|| async {
calls.fetch_add(1, Ordering::SeqCst);
Ok::<_, Status>(7)
})
.await;
assert_eq!(result.unwrap(), 7);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn unary_retries_retryable_codes_until_success() {
let calls = AtomicU32::new(0);
let result = unary(|| async {
if calls.fetch_add(1, Ordering::SeqCst) < 2 {
Err(Status::unavailable("engine restarting"))
} else {
Ok("done")
}
})
.await;
assert_eq!(result.unwrap(), "done");
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test(start_paused = true)]
async fn unary_gives_up_after_five_attempts_with_the_last_status() {
let calls = AtomicU32::new(0);
let result: Result<(), Status> = unary(|| async {
let n = calls.fetch_add(1, Ordering::SeqCst);
Err(Status::unavailable(format!("attempt {n}")))
})
.await;
let status = result.unwrap_err();
assert_eq!(status.code(), Code::Unavailable);
assert_eq!(status.message(), "attempt 4");
assert_eq!(calls.load(Ordering::SeqCst), 5);
}
#[tokio::test(start_paused = true)]
async fn unary_does_not_retry_other_codes() {
let calls = AtomicU32::new(0);
let result: Result<(), Status> = unary(|| async {
calls.fetch_add(1, Ordering::SeqCst);
Err(Status::invalid_argument("bad"))
})
.await;
assert_eq!(result.unwrap_err().code(), Code::InvalidArgument);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn unary_times_out_an_attempt_after_thirty_seconds_and_retries() {
let calls = AtomicU32::new(0);
let started = tokio::time::Instant::now();
let result = unary(|| async {
if calls.fetch_add(1, Ordering::SeqCst) == 0 {
std::future::pending::<()>().await;
}
Ok::<_, Status>("second attempt")
})
.await;
assert_eq!(result.unwrap(), "second attempt");
assert_eq!(calls.load(Ordering::SeqCst), 2);
let elapsed = started.elapsed();
assert!(elapsed >= Duration::from_secs(30), "{elapsed:?}");
assert!(
elapsed <= Duration::from_secs(35),
"{elapsed:?}: 30s timeout + at most 5s backoff"
);
}
}