rainier_http_client/
retry.rs

1//! When to try again — [`Backoff`].
2//!
3//! Retrying is where an outbound client is most often wrong in both
4//! directions: not at all, so one dropped connection fails a job that would
5//! have worked; or everything, so a `422` is sent four times and a duplicate
6//! charge is created three of them.
7
8use std::time::Duration;
9
10/// How long to wait between attempts.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum Backoff {
13    /// The same wait every time.
14    Fixed(Duration),
15    /// Doubling: 100ms, 200ms, 400ms, …
16    ///
17    /// The default, and the right one for a dependency that is briefly
18    /// unavailable — retrying at full speed is what turns a blip into an
19    /// outage, because every caller does it at once.
20    Exponential {
21        /// The wait before the second attempt.
22        base: Duration,
23        /// The longest it will ever wait.
24        max: Duration,
25    },
26    /// No wait at all.
27    ///
28    /// For a test, and for the rare case where the failure is known to be
29    /// instantaneous and independent.
30    None,
31}
32
33impl Backoff {
34    /// Doubling from 100ms, capped at ten seconds.
35    pub fn exponential() -> Self {
36        Self::Exponential { base: Duration::from_millis(100), max: Duration::from_secs(10) }
37    }
38
39    /// The same wait every time.
40    pub fn fixed(wait: Duration) -> Self {
41        Self::Fixed(wait)
42    }
43
44    /// How long to wait before attempt `attempt`, counting the first as 1.
45    ///
46    /// Zero before the first, because nothing has failed yet.
47    pub fn wait_before(&self, attempt: u32) -> Duration {
48        if attempt <= 1 {
49            return Duration::ZERO;
50        }
51
52        match self {
53            Self::None => Duration::ZERO,
54            Self::Fixed(wait) => *wait,
55            Self::Exponential { base, max } => {
56                // `attempt - 2`, so the wait before the *second* attempt is
57                // the base rather than twice it.
58                let doublings = (attempt - 2).min(20);
59                let wait = base.saturating_mul(2u32.saturating_pow(doublings));
60                wait.min(*max)
61            }
62        }
63    }
64}
65
66impl Default for Backoff {
67    fn default() -> Self {
68        Self::exponential()
69    }
70}
71
72/// Whether a failed attempt is worth repeating.
73///
74/// The default policy, and the one worth understanding before overriding it:
75///
76/// | | Retried | Why |
77/// |---|---|---|
78/// | a transport failure | yes | a refused connection or a timeout is usually transient |
79/// | `408`, `425`, `429` | yes | the other end said "later" in as many words |
80/// | `5xx` | yes | the other end is having a bad time |
81/// | `4xx` otherwise | **no** | the request is wrong, and sending it again keeps it wrong |
82///
83/// The last row is the one that matters. Retrying a `422` four times does not
84/// fix the payload; retrying a `402` charges the card again if the other end
85/// is not idempotent.
86pub fn is_retryable(status: Option<u16>) -> bool {
87    match status {
88        // A transport failure — nothing came back at all.
89        None => true,
90        Some(408 | 425 | 429) => true,
91        Some(status) => status >= 500,
92    }
93}
94
95#[cfg(test)]
96mod tests {
97    use super::*;
98
99    #[test]
100    fn nothing_waits_before_the_first_attempt() {
101        for backoff in
102            [Backoff::exponential(), Backoff::fixed(Duration::from_secs(1)), Backoff::None]
103        {
104            assert_eq!(backoff.wait_before(1), Duration::ZERO, "{backoff:?}");
105            assert_eq!(backoff.wait_before(0), Duration::ZERO, "{backoff:?}");
106        }
107    }
108
109    #[test]
110    fn exponential_starts_at_the_base_and_doubles() {
111        let backoff =
112            Backoff::Exponential { base: Duration::from_millis(100), max: Duration::from_secs(60) };
113
114        assert_eq!(backoff.wait_before(2), Duration::from_millis(100));
115        assert_eq!(backoff.wait_before(3), Duration::from_millis(200));
116        assert_eq!(backoff.wait_before(4), Duration::from_millis(400));
117    }
118
119    #[test]
120    fn exponential_is_capped() {
121        // Without a cap, the twentieth attempt waits a day and a half.
122        let backoff =
123            Backoff::Exponential { base: Duration::from_millis(100), max: Duration::from_secs(2) };
124
125        assert_eq!(backoff.wait_before(20), Duration::from_secs(2));
126        assert_eq!(backoff.wait_before(1000), Duration::from_secs(2));
127    }
128
129    #[test]
130    fn fixed_is_the_same_every_time() {
131        let backoff = Backoff::fixed(Duration::from_millis(250));
132
133        assert_eq!(backoff.wait_before(2), Duration::from_millis(250));
134        assert_eq!(backoff.wait_before(9), Duration::from_millis(250));
135    }
136
137    #[test]
138    fn a_transport_failure_is_retried() {
139        assert!(is_retryable(None));
140    }
141
142    #[test]
143    fn the_server_saying_later_is_retried() {
144        for status in [408, 425, 429, 500, 502, 503, 504] {
145            assert!(is_retryable(Some(status)), "{status}");
146        }
147    }
148
149    #[test]
150    fn a_client_mistake_is_not_retried() {
151        // The row that matters. Retrying a 422 four times does not fix the
152        // payload, and retrying a 402 charges the card again.
153        for status in [400, 401, 402, 403, 404, 409, 422] {
154            assert!(!is_retryable(Some(status)), "{status}");
155        }
156    }
157
158    #[test]
159    fn success_is_not_retried() {
160        for status in [200, 201, 204, 301, 302] {
161            assert!(!is_retryable(Some(status)), "{status}");
162        }
163    }
164}
rainier_http_client/retry.rs

rainier_http_client/
retry.rs
