vantage_api_pool/resilient/
policy.rs

1//! Retry and back-off knobs.
2
3use std::time::Duration;
4
5/// The shortest sleep any retry mode may schedule. A policy configured with a
6/// zero base (or a zero ceiling) would otherwise spin.
7const MIN_BACKOFF: Duration = Duration::from_millis(1);
8
9/// Bounded retry: exponential from `base_backoff` (doubling per attempt),
10/// capped at `max_backoff`, with jitter added before each sleep.
11#[derive(Debug, Clone)]
12pub struct RetryPolicy {
13    pub max_retries: usize,
14    pub base_backoff: Duration,
15    pub max_backoff: Duration,
16}
17
18impl Default for RetryPolicy {
19    fn default() -> Self {
20        Self {
21            max_retries: 4,
22            base_backoff: Duration::from_millis(50),
23            max_backoff: Duration::from_secs(10),
24        }
25    }
26}
27
28impl RetryPolicy {
29    pub(crate) fn backoff(&self, attempt: usize) -> Duration {
30        exponential(self.base_backoff, self.max_backoff, attempt)
31    }
32}
33
34/// `base × 2^attempt`, capped at `max`. Both bounds are floored at
35/// [`MIN_BACKOFF`], so a policy configured with zeros still sleeps.
36fn exponential(base: Duration, max: Duration, attempt: usize) -> Duration {
37    let factor = 2u32.saturating_pow(attempt.min(31) as u32);
38    base.max(MIN_BACKOFF)
39        .saturating_mul(factor)
40        .min(max.max(MIN_BACKOFF))
41}
42
43/// Add up to +25% jitter so retrying clients don't synchronize.
44pub(crate) fn with_jitter(d: Duration) -> Duration {
45    let nanos = std::time::SystemTime::now()
46        .duration_since(std::time::UNIX_EPOCH)
47        .map(|t| t.subsec_nanos())
48        .unwrap_or(0);
49    let frac = (nanos % 250) as f64 / 1000.0;
50    d + d.mul_f64(frac)
51}
52
53/// How many times, and how long apart, a failed attempt is repeated.
54#[derive(Debug, Clone)]
55pub enum RetryMode {
56    /// One attempt. For work nobody is waiting on.
57    None,
58    /// The classic bounded retry.
59    Bounded(RetryPolicy),
60    /// Retry until the caller drops the future; exponential from `base`
61    /// capped at `max`. For work someone is waiting on.
62    UntilCancelled { base: Duration, max: Duration },
63}
64
65/// What an attempt does while the circuit breaker is open.
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67pub enum BreakerMode {
68    /// Return `ErrorKind::BreakerOpen` at once.
69    FailFast,
70    /// Sleep until the cooldown ends and take the half-open probe slot.
71    WaitForProbe,
72}
73
74/// The policy for one call.
75#[derive(Debug, Clone)]
76pub struct CallPolicy {
77    pub retry: RetryMode,
78    pub breaker: BreakerMode,
79}
80
81impl CallPolicy {
82    /// One attempt, fail fast. Polls, refreshes, hydration.
83    pub fn background() -> Self {
84        Self {
85            retry: RetryMode::None,
86            breaker: BreakerMode::FailFast,
87        }
88    }
89
90    /// Retry until cancelled (250 ms doubling to 10 s), wait for the probe.
91    /// Cold loads, uncached viewports, writes.
92    pub fn essential() -> Self {
93        Self {
94            retry: RetryMode::UntilCancelled {
95                base: Duration::from_millis(250),
96                max: Duration::from_secs(10),
97            },
98            breaker: BreakerMode::WaitForProbe,
99        }
100    }
101
102    /// Bounded retry, fail fast — what `execute` does.
103    pub fn bounded(policy: RetryPolicy) -> Self {
104        Self {
105            retry: RetryMode::Bounded(policy),
106            breaker: BreakerMode::FailFast,
107        }
108    }
109
110    pub fn wait_for_probe(mut self) -> Self {
111        self.breaker = BreakerMode::WaitForProbe;
112        self
113    }
114
115    /// The sleep before retry number `attempt` (0-based count of retries so
116    /// far), or `None` when this mode has no retry left.
117    pub(crate) fn next_backoff(&self, attempt: usize) -> Option<Duration> {
118        match &self.retry {
119            RetryMode::None => None,
120            RetryMode::Bounded(p) => (attempt < p.max_retries).then(|| p.backoff(attempt)),
121            RetryMode::UntilCancelled { base, max } => Some(exponential(*base, *max, attempt)),
122        }
123    }
124
125    /// The ceiling a server's `Retry-After` is clamped to under this mode. A
126    /// server asking for two minutes must not park a call whose own policy
127    /// promises to give up — or to retry sooner — well before that.
128    pub(crate) fn retry_after_cap(&self) -> Option<Duration> {
129        match &self.retry {
130            RetryMode::None => None,
131            RetryMode::Bounded(p) => Some(p.max_backoff.max(MIN_BACKOFF)),
132            RetryMode::UntilCancelled { max, .. } => Some((*max).max(MIN_BACKOFF)),
133        }
134    }
135}
136
137/// Statuses a retry may fix. Everything else in 4xx is final.
138pub(crate) fn is_retryable_status(status: u16) -> bool {
139    status == 408 || status == 429 || (500..600).contains(&status)
140}
141
142/// What one attempt's answer tells the circuit breaker. Retryability and
143/// health are separate questions: `429` is worth retrying but proves the API
144/// is up, and a `404` is not worth retrying but proves the same thing.
145#[derive(Debug, Clone, Copy, PartialEq, Eq)]
146pub(crate) enum Health {
147    /// A `2xx`: the API works. Closes the breaker and clears the failure run.
148    Healthy,
149    /// An answer a retry cannot fix. The API is reachable, so this closes an
150    /// open breaker, but it is not evidence of health: the failure run stands.
151    Reachable,
152    /// `408` / `429`: the server answered, but about its own load. Neither
153    /// opens nor closes the breaker.
154    Inconclusive,
155    /// `5xx`: counts toward opening the breaker.
156    Failing,
157}
158
159pub(crate) fn status_health(status: u16) -> Health {
160    match status {
161        200..=299 => Health::Healthy,
162        408 | 429 => Health::Inconclusive,
163        500..=599 => Health::Failing,
164        _ => Health::Reachable,
165    }
166}
167
168#[cfg(test)]
169mod tests {
170    use super::*;
171
172    #[test]
173    fn retryability_and_health_are_separate_questions() {
174        for status in [408, 429] {
175            assert!(is_retryable_status(status));
176            assert_eq!(status_health(status), Health::Inconclusive);
177        }
178        assert!(is_retryable_status(503));
179        assert_eq!(status_health(503), Health::Failing);
180        assert!(!is_retryable_status(404));
181        assert_eq!(status_health(404), Health::Reachable);
182        assert!(!is_retryable_status(200));
183        assert_eq!(status_health(200), Health::Healthy);
184    }
185
186    #[test]
187    fn a_zeroed_policy_still_sleeps() {
188        let p = CallPolicy {
189            retry: RetryMode::UntilCancelled {
190                base: Duration::ZERO,
191                max: Duration::ZERO,
192            },
193            breaker: BreakerMode::FailFast,
194        };
195        assert_eq!(p.next_backoff(0), Some(MIN_BACKOFF));
196        assert_eq!(p.next_backoff(9), Some(MIN_BACKOFF));
197
198        let bounded = CallPolicy::bounded(RetryPolicy {
199            max_retries: 2,
200            base_backoff: Duration::ZERO,
201            max_backoff: Duration::ZERO,
202        });
203        assert_eq!(bounded.next_backoff(0), Some(MIN_BACKOFF));
204        assert_eq!(bounded.next_backoff(2), None, "the budget still runs out");
205    }
206
207    #[test]
208    fn retry_after_is_capped_by_the_mode() {
209        assert_eq!(CallPolicy::background().retry_after_cap(), None);
210        assert_eq!(
211            CallPolicy::essential().retry_after_cap(),
212            Some(Duration::from_secs(10))
213        );
214        assert_eq!(
215            CallPolicy::bounded(RetryPolicy::default()).retry_after_cap(),
216            Some(Duration::from_secs(10))
217        );
218    }
219}
vantage_api_pool/resilient/policy.rs

vantage_api_pool/resilient/
policy.rs
