sail-rs 0.2.19

Official Rust SDK for Sail: create and drive sailboxes (sandboxed cloud VMs) with lifecycle, streaming exec, file transfer, and ingress.
Documentation
//! Retry policy for the Sail HTTP client.
//!
//! The SDK auto-retries transient failures that cannot have been observed by
//! the server in a meaningful way: synthesized 503s from transport errors,
//! upstream 502/503/504 envelopes, and 429s with a `Retry-After` hint. The
//! backend pairs this with Stripe-style `Idempotency-Key` dedupe for the
//! sailbox lifecycle endpoints so a retried POST cannot create duplicate
//! sailboxes or checkpoints.

use std::time::{Duration, SystemTime};

/// Knobs governing the auto-retry path.
///
/// `max_attempts` counts the initial request, so `max_attempts=3` means at
/// most two retries. The schedule is exponential with full jitter, capped at
/// `max_delay`. A server-sent `Retry-After` always wins over the computed
/// backoff (capped at `max_delay` so a hostile server cannot park the SDK
/// indefinitely).
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct RetryPolicy {
    /// Total attempts including the initial request; `1` disables retries.
    pub max_attempts: u32,
    /// Backoff base: the delay before the first retry, pre-jitter.
    pub base_delay: Duration,
    /// Upper bound on any single delay, including `Retry-After`.
    pub max_delay: Duration,
    /// Per-attempt growth factor for the exponential backoff schedule.
    pub multiplier: f64,
}

/// Default policy for idempotent reads and Idempotency-Key-deduped mutations:
/// up to two retries with full-jitter exponential backoff.
pub const DEFAULT_RETRY_POLICY: RetryPolicy = RetryPolicy {
    max_attempts: 3,
    base_delay: Duration::from_millis(500),
    max_delay: Duration::from_secs(8),
    multiplier: 2.0,
};

/// Single attempt (no retries) for non-idempotent mutations.
pub const NO_RETRY: RetryPolicy = RetryPolicy {
    max_attempts: 1,
    base_delay: Duration::from_millis(500),
    max_delay: Duration::from_secs(8),
    multiplier: 2.0,
};

/// HTTP statuses safe to retry under Idempotency-Key.
///
/// 502/503/504 retry unconditionally (transport-failure envelopes and
/// gateway transients, where the request may not have reached the backend).
/// 429 retries only when the server provides a valid `Retry-After`; a bare
/// 429 is surfaced as-is. 500 is intentionally excluded: the backend releases
/// the idempotency reservation on a 500, so a retried 500 on a mutation could
/// double-create the resource.
pub(crate) fn is_retryable_status(status_code: u16, headers: &[(String, String)]) -> bool {
    if matches!(status_code, 502..=504) {
        return true;
    }
    if status_code == 429 {
        return parse_retry_after(headers, /* now */ None).is_some();
    }
    false
}

/// Full-jitter exponential backoff for the given retry attempt.
///
/// `attempt` is 1-based: attempt 1 is the first retry (after the initial
/// request failed). Full jitter is a uniform sample in
/// `[0, min(max_delay, base * multiplier^(attempt-1))]`.
pub(crate) fn backoff_delay(attempt: u32, policy: &RetryPolicy) -> f64 {
    let exponent = attempt.saturating_sub(1);
    let base = policy.base_delay.as_secs_f64();
    let max = policy.max_delay.as_secs_f64();
    let capped = max.min(base * policy.multiplier.powi(exponent as i32));
    if capped <= 0.0 {
        return 0.0;
    }
    rand::random_range(0.0..=capped)
}

/// Parse a Retry-After header as seconds; None if absent or invalid.
///
/// Accepts both delta-seconds and HTTP-date forms (RFC 7231 ยง7.1.3). A
/// negative or non-finite delta-seconds is rejected as invalid (None) so it
/// cannot count as a valid 429 wait hint; a well-formed but already elapsed
/// HTTP-date yields 0 ("retry now"). HTTP-dates without the mandatory GMT
/// zone are rejected by the date parser.
pub(crate) fn parse_retry_after(
    headers: &[(String, String)],
    now: Option<SystemTime>,
) -> Option<f64> {
    let raw = header_value(headers, "Retry-After")?.trim();
    if raw.is_empty() {
        return None;
    }
    if let Ok(seconds) = raw.parse::<f64>() {
        if !seconds.is_finite() || seconds < 0.0 {
            return None;
        }
        return Some(seconds);
    }
    let target = httpdate::parse_http_date(raw).ok()?;
    let now = now.unwrap_or_else(SystemTime::now);
    match target.duration_since(now) {
        Ok(remaining) => Some(remaining.as_secs_f64()),
        Err(_) => Some(0.0),
    }
}

/// Pick the larger of Retry-After and computed backoff, capped at max_delay.
pub(crate) fn effective_delay(
    attempt: u32,
    policy: &RetryPolicy,
    headers: &[(String, String)],
) -> f64 {
    let mut delay = backoff_delay(attempt, policy);
    if let Some(hint) = parse_retry_after(headers, /* now */ None) {
        delay = delay.max(hint);
    }
    policy.max_delay.as_secs_f64().min(delay)
}

fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> {
    headers
        .iter()
        .find(|(key, _)| key.eq_ignore_ascii_case(name))
        .map(|(_, value)| value.as_str())
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::time::Duration;

    fn hdrs(value: &str) -> Vec<(String, String)> {
        vec![("Retry-After".to_string(), value.to_string())]
    }

    #[test]
    fn retryable_statuses() {
        assert!(is_retryable_status(502, &[]));
        assert!(is_retryable_status(503, &[]));
        assert!(is_retryable_status(504, &[]));
        assert!(!is_retryable_status(500, &[]));
        assert!(!is_retryable_status(429, &[]));
        assert!(is_retryable_status(429, &hdrs("2")));
        assert!(!is_retryable_status(429, &hdrs("-1")));
    }

    #[test]
    fn parse_retry_after_delta_seconds() {
        assert_eq!(parse_retry_after(&hdrs("2"), /* now */ None), Some(2.0));
        assert_eq!(parse_retry_after(&hdrs(" 1.5 "), /* now */ None), Some(1.5));
        assert_eq!(parse_retry_after(&hdrs("-3"), /* now */ None), None);
        assert_eq!(parse_retry_after(&hdrs("inf"), /* now */ None), None);
        assert_eq!(parse_retry_after(&hdrs("nan"), /* now */ None), None);
        assert_eq!(parse_retry_after(&hdrs(""), /* now */ None), None);
        assert_eq!(parse_retry_after(&[], /* now */ None), None);
    }

    #[test]
    fn parse_retry_after_http_date() {
        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(784_111_777);
        // 23 seconds after `now`.
        let value = "Sun, 06 Nov 1994 08:50:00 GMT";
        let parsed = parse_retry_after(&hdrs(value), Some(now)).unwrap();
        assert!((parsed - 23.0).abs() < 1.0, "parsed={parsed}");
        // Elapsed dates clamp to zero rather than failing.
        let past = "Sun, 06 Nov 1994 08:00:00 GMT";
        assert_eq!(parse_retry_after(&hdrs(past), Some(now)), Some(0.0));
        // Garbage and non-GMT forms are invalid.
        assert_eq!(parse_retry_after(&hdrs("tomorrow"), Some(now)), None);
    }

    #[test]
    fn backoff_within_bounds() {
        for attempt in 1..6 {
            let delay = backoff_delay(attempt, &DEFAULT_RETRY_POLICY);
            assert!(delay >= 0.0);
            assert!(delay <= DEFAULT_RETRY_POLICY.max_delay.as_secs_f64());
        }
    }

    #[test]
    fn effective_delay_prefers_hint_but_caps() {
        let delay = effective_delay(1, &DEFAULT_RETRY_POLICY, &hdrs("100"));
        assert_eq!(delay, DEFAULT_RETRY_POLICY.max_delay.as_secs_f64());
        let delay = effective_delay(1, &DEFAULT_RETRY_POLICY, &hdrs("3"));
        assert!(delay >= 3.0);
    }

    use proptest::prelude::*;

    proptest! {
        /// Full-jitter backoff is always non-negative and never exceeds the
        /// policy's max delay, for any attempt and any policy shape.
        #[test]
        fn backoff_stays_within_bounds(
            attempt in 0u32..32,
            base in 0.0f64..5.0,
            multiplier in 1.0f64..4.0,
            max in 0.0f64..30.0,
        ) {
            let policy = RetryPolicy { max_attempts: 3, base_delay: Duration::from_secs_f64(base), max_delay: Duration::from_secs_f64(max), multiplier };
            let delay = backoff_delay(attempt, &policy);
            prop_assert!(delay >= 0.0);
            prop_assert!(delay <= policy.max_delay.as_secs_f64());
        }

        /// The effective delay never exceeds max_delay and is at least the
        /// server's Retry-After hint (itself capped at max_delay). A hostile
        /// server can't park the SDK past the cap, nor can jitter undercut a
        /// valid hint.
        #[test]
        fn effective_delay_caps_and_honors_hint(
            attempt in 1u32..10,
            hint in 0.0f64..100.0,
            max in 0.0f64..50.0,
        ) {
            let policy = RetryPolicy { max_attempts: 3, base_delay: Duration::from_millis(500), max_delay: Duration::from_secs_f64(max), multiplier: 2.0 };
            let max_secs = policy.max_delay.as_secs_f64();
            let headers = hdrs(&format!("{hint}"));
            let delay = effective_delay(attempt, &policy, &headers);
            prop_assert!(delay <= max_secs);
            prop_assert!(delay >= hint.min(max_secs));
        }
    }
}