hatchet-client 0.1.0

Rust SDK for Hatchet: run workers and trigger tasks
Documentation
//! Retry policy: backoff bounds, full jitter, and gRPC status classification,
//! kept as pure functions, plus the unary retry loop built from them.
//!
//! A stream that ends with `Code::Cancelled` is treated as a clean stop
//! (matching the official Go SDK's `retry/stream.go`, which stops on
//! `Canceled`), not as an error to retry or surface.

use std::time::Duration;

use rand::Rng;
use tonic::{Code, Status};

/// A unary call is attempted at most this many times in total.
pub(crate) const UNARY_MAX_ATTEMPTS: u32 = 5;
/// Each unary attempt is abandoned after this long.
pub(crate) const UNARY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30);
/// A stream that keeps ending with unclassified errors is given up on after
/// this many consecutive failures.
///
/// A run's output stream counts one more thing towards the same limit (see
/// `runs::stream::run`): an `Internal` end that delivered nothing. `Internal`
/// is `StreamErrorClass::Transient` here and stays that way — a restarting
/// engine sends it — but a subscription poisoned by a chunk the engine cannot
/// marshal ends with exactly that code every time, and would otherwise be
/// reconnected for ever in silence.
pub(crate) const STREAM_MAX_CONSECUTIVE_OTHER_FAILURES: u32 = 10;

const UNARY_BASE: Duration = Duration::from_secs(5);
const UNARY_MAX: Duration = Duration::from_secs(80);
const STREAM_BASE: Duration = Duration::from_secs(1);
const STREAM_MAX: Duration = Duration::from_secs(30);

/// Upper bound of the delay before unary retry number `retry` (0-based).
pub(crate) fn unary_backoff_cap(retry: u32) -> Duration {
    exponential_cap(UNARY_BASE, UNARY_MAX, retry)
}

/// Upper bound of the delay before stream reconnect number `attempt` (0-based).
pub(crate) fn stream_backoff_cap(attempt: u32) -> Duration {
    exponential_cap(STREAM_BASE, STREAM_MAX, attempt)
}

fn exponential_cap(base: Duration, max: Duration, exponent: u32) -> Duration {
    base.saturating_mul(2u32.saturating_pow(exponent)).min(max)
}

/// Full jitter: a uniformly random delay in `[0, cap]`.
pub(crate) fn full_jitter(cap: Duration, rng: &mut impl Rng) -> Duration {
    let cap_nanos = u64::try_from(cap.as_nanos()).unwrap_or(u64::MAX);
    Duration::from_nanos(rng.random_range(0..=cap_nanos))
}

/// Whether a failed unary call is worth repeating.
pub(crate) fn is_unary_retryable(code: Code) -> bool {
    matches!(
        code,
        Code::ResourceExhausted | Code::DeadlineExceeded | Code::Internal | Code::Unavailable
    )
}

/// What to do after a server stream ends.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum StreamErrorClass {
    /// Reconnect, indefinitely.
    Transient,
    /// Reconnecting cannot help; stop and surface the error.
    Fatal,
    /// Reconnect, but give up after
    /// [`STREAM_MAX_CONSECUTIVE_OTHER_FAILURES`] in a row.
    Other,
    /// The stream was cancelled (by us or by the engine); stop without an error.
    StopClean,
}

/// Classifies how a stream ended; `None` means the server closed it cleanly.
pub(crate) fn classify_stream_end(status: Option<&Status>) -> StreamErrorClass {
    let Some(status) = status else {
        return StreamErrorClass::Transient;
    };
    match status.code() {
        Code::Unavailable | Code::Internal | Code::DeadlineExceeded | Code::ResourceExhausted => {
            StreamErrorClass::Transient
        }
        Code::Unauthenticated
        | Code::PermissionDenied
        | Code::InvalidArgument
        | Code::FailedPrecondition
        | Code::NotFound
        | Code::Unimplemented => StreamErrorClass::Fatal,
        Code::Cancelled => StreamErrorClass::StopClean,
        _ => StreamErrorClass::Other,
    }
}

/// Wraps a message in a request carrying the per-attempt deadline as a
/// `grpc-timeout` header.
///
/// Every call [`unary`] retries goes out this way: the local timeout alone
/// only stops *this* side waiting, and would leave the engine working on a
/// call nobody will read the answer to.
pub(crate) fn timed<T>(message: T) -> tonic::Request<T> {
    let mut request = tonic::Request::new(message);
    request.set_timeout(UNARY_ATTEMPT_TIMEOUT);
    request
}

/// Runs a unary call under the SDK's retry policy: up to
/// [`UNARY_MAX_ATTEMPTS`] attempts, [`UNARY_ATTEMPT_TIMEOUT`] each, full-jitter
/// backoff between them, retrying only [`is_unary_retryable`] codes.
pub(crate) async fn unary<T, F, Fut>(mut call: F) -> Result<T, Status>
where
    F: FnMut() -> Fut,
    Fut: Future<Output = Result<T, Status>>,
{
    let mut retry = 0;
    loop {
        let status = match tokio::time::timeout(UNARY_ATTEMPT_TIMEOUT, call()).await {
            Ok(Ok(value)) => return Ok(value),
            Ok(Err(status)) => status,
            Err(_) => Status::deadline_exceeded("the attempt timed out"),
        };
        if !is_unary_retryable(status.code()) || retry + 1 >= UNARY_MAX_ATTEMPTS {
            return Err(status);
        }
        let delay = full_jitter(unary_backoff_cap(retry), &mut rand::rng());
        tracing::debug!(code = ?status.code(), retry, ?delay, "retrying gRPC call");
        tokio::time::sleep(delay).await;
        retry += 1;
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rand::SeedableRng;
    use rand::rngs::StdRng;
    use std::sync::atomic::{AtomicU32, Ordering};

    #[test]
    fn stream_backoff_cap_doubles_from_one_second_up_to_thirty() {
        let caps: Vec<u64> = (0..7).map(|n| stream_backoff_cap(n).as_secs()).collect();
        assert_eq!(caps, [1, 2, 4, 8, 16, 30, 30]);
        assert_eq!(stream_backoff_cap(u32::MAX), Duration::from_secs(30));
    }

    #[test]
    fn unary_backoff_cap_doubles_from_five_seconds_up_to_eighty() {
        let caps: Vec<u64> = (0..6).map(|n| unary_backoff_cap(n).as_secs()).collect();
        assert_eq!(caps, [5, 10, 20, 40, 80, 80]);
        assert_eq!(unary_backoff_cap(u32::MAX), Duration::from_secs(80));
    }

    #[test]
    fn full_jitter_stays_within_the_cap_and_actually_varies() {
        let mut rng = StdRng::seed_from_u64(42);
        for attempt in 0..8 {
            let cap = stream_backoff_cap(attempt);
            let delays: Vec<Duration> = (0..200).map(|_| full_jitter(cap, &mut rng)).collect();
            assert!(delays.iter().all(|d| *d <= cap), "attempt {attempt}");
            assert!(
                delays.iter().any(|d| *d < cap / 2),
                "attempt {attempt}: never low"
            );
            assert!(
                delays.iter().any(|d| *d > cap / 2),
                "attempt {attempt}: never high"
            );
        }
    }

    #[test]
    fn full_jitter_is_reproducible_for_a_seed() {
        let cap = Duration::from_secs(30);
        let a = full_jitter(cap, &mut StdRng::seed_from_u64(7));
        let b = full_jitter(cap, &mut StdRng::seed_from_u64(7));
        assert_eq!(a, b);
        assert_eq!(
            full_jitter(Duration::ZERO, &mut StdRng::seed_from_u64(7)),
            Duration::ZERO
        );
    }

    #[test]
    fn unary_retryable_codes() {
        for code in [
            Code::ResourceExhausted,
            Code::DeadlineExceeded,
            Code::Internal,
            Code::Unavailable,
        ] {
            assert!(is_unary_retryable(code), "{code:?}");
        }
        for code in [
            Code::InvalidArgument,
            Code::NotFound,
            Code::Unauthenticated,
            Code::Cancelled,
            Code::Unknown,
        ] {
            assert!(!is_unary_retryable(code), "{code:?}");
        }
    }

    #[test]
    fn stream_end_classification_follows_the_design_table() {
        use StreamErrorClass::*;
        assert_eq!(classify_stream_end(None), Transient, "EOF");
        let table = [
            (Code::Unavailable, Transient),
            (Code::Internal, Transient),
            (Code::DeadlineExceeded, Transient),
            (Code::ResourceExhausted, Transient),
            (Code::Unauthenticated, Fatal),
            (Code::PermissionDenied, Fatal),
            (Code::InvalidArgument, Fatal),
            (Code::FailedPrecondition, Fatal),
            (Code::NotFound, Fatal),
            (Code::Unimplemented, Fatal),
            (Code::Unknown, Other),
            (Code::Cancelled, StopClean),
            (Code::Aborted, Other),
            (Code::AlreadyExists, Other),
            (Code::OutOfRange, Other),
            (Code::DataLoss, Other),
        ];
        for (code, expected) in table {
            let status = Status::new(code, "x");
            assert_eq!(classify_stream_end(Some(&status)), expected, "{code:?}");
        }
    }

    #[tokio::test(start_paused = true)]
    async fn unary_returns_the_first_success_without_retrying() {
        let calls = AtomicU32::new(0);
        let result = unary(|| async {
            calls.fetch_add(1, Ordering::SeqCst);
            Ok::<_, Status>(7)
        })
        .await;
        assert_eq!(result.unwrap(), 7);
        assert_eq!(calls.load(Ordering::SeqCst), 1);
    }

    #[tokio::test(start_paused = true)]
    async fn unary_retries_retryable_codes_until_success() {
        let calls = AtomicU32::new(0);
        let result = unary(|| async {
            if calls.fetch_add(1, Ordering::SeqCst) < 2 {
                Err(Status::unavailable("engine restarting"))
            } else {
                Ok("done")
            }
        })
        .await;
        assert_eq!(result.unwrap(), "done");
        assert_eq!(calls.load(Ordering::SeqCst), 3);
    }

    #[tokio::test(start_paused = true)]
    async fn unary_gives_up_after_five_attempts_with_the_last_status() {
        let calls = AtomicU32::new(0);
        let result: Result<(), Status> = unary(|| async {
            let n = calls.fetch_add(1, Ordering::SeqCst);
            Err(Status::unavailable(format!("attempt {n}")))
        })
        .await;
        let status = result.unwrap_err();
        assert_eq!(status.code(), Code::Unavailable);
        assert_eq!(status.message(), "attempt 4");
        assert_eq!(calls.load(Ordering::SeqCst), 5);
    }

    #[tokio::test(start_paused = true)]
    async fn unary_does_not_retry_other_codes() {
        let calls = AtomicU32::new(0);
        let result: Result<(), Status> = unary(|| async {
            calls.fetch_add(1, Ordering::SeqCst);
            Err(Status::invalid_argument("bad"))
        })
        .await;
        assert_eq!(result.unwrap_err().code(), Code::InvalidArgument);
        assert_eq!(calls.load(Ordering::SeqCst), 1);
    }

    #[tokio::test(start_paused = true)]
    async fn unary_times_out_an_attempt_after_thirty_seconds_and_retries() {
        let calls = AtomicU32::new(0);
        let started = tokio::time::Instant::now();
        let result = unary(|| async {
            if calls.fetch_add(1, Ordering::SeqCst) == 0 {
                std::future::pending::<()>().await;
            }
            Ok::<_, Status>("second attempt")
        })
        .await;
        assert_eq!(result.unwrap(), "second attempt");
        assert_eq!(calls.load(Ordering::SeqCst), 2);
        let elapsed = started.elapsed();
        assert!(elapsed >= Duration::from_secs(30), "{elapsed:?}");
        assert!(
            elapsed <= Duration::from_secs(35),
            "{elapsed:?}: 30s timeout + at most 5s backoff"
        );
    }
}