Skip to main content

jev_client/
retry.rs

1//! When to retry, and how long to wait first.
2
3use std::time::{Duration, SystemTime};
4
5use reqwest::header::{HeaderMap, RETRY_AFTER};
6
7/// Statuses that are never retried, whatever the policy says: the request itself is at fault, so
8/// sending it again can only fail the same way (and, for 401/403, hammer a rejected credential).
9const NEVER_RETRIED: [u16; 5] = [400, 401, 403, 404, 422];
10
11/// The non-standard header carrying a retry delay in milliseconds, honoured like the SDKs do.
12const RETRY_AFTER_MS: &str = "retry-after-ms";
13
14/// How failed requests are retried.
15///
16/// The defaults match the official TypeSafe SDKs: 2 retries, a 0.5 s first delay doubling up to
17/// 5 s, a quarter of each delay randomly subtracted, a 30 s timeout per attempt, and a server's
18/// `Retry-After` honoured up to 60 s. Retried by default are 408, 429, every 5xx (so 529 too),
19/// connection failures and timeouts.
20///
21/// ```
22/// use std::time::Duration;
23/// use jev_client::RetryPolicy;
24///
25/// let patient = RetryPolicy::default().with_max_retries(5).with_timeout(Duration::from_secs(60));
26/// assert_eq!(patient.max_retries, 5);
27///
28/// let never = RetryPolicy::none();
29/// assert_eq!(never.max_retries, 0);
30/// ```
31#[derive(Clone, Debug, PartialEq)]
32#[non_exhaustive]
33pub struct RetryPolicy {
34    /// Retries after the first attempt. `0` disables retrying.
35    pub max_retries: u32,
36    /// The delay before the first retry. Each later delay doubles, up to `backoff_max`.
37    pub backoff_initial: Duration,
38    /// The longest backoff delay.
39    pub backoff_max: Duration,
40    /// The fraction of each backoff delay that is randomly subtracted, from 0 to 1. Spreading
41    /// retries out stops many clients from retrying in step.
42    pub backoff_jitter: f64,
43    /// The time allowed for one attempt, from sending the request to reading the whole response.
44    pub timeout: Duration,
45    /// Whether a `Retry-After` or `retry-after-ms` response header sets the delay.
46    pub respect_retry_after: bool,
47    /// The longest server-requested delay that is honoured. A longer one falls back to backoff.
48    pub max_retry_after: Duration,
49    /// Whether a failure to connect, or a connection lost mid-response, is retried.
50    pub retry_connection_errors: bool,
51    /// Whether an attempt that timed out is retried.
52    pub retry_timeouts: bool,
53    /// The HTTP statuses that are retried. 400, 401, 403, 404 and 422 are never retried, even
54    /// when listed here.
55    pub retry_statuses: Vec<u16>,
56}
57
58impl Default for RetryPolicy {
59    fn default() -> Self {
60        Self {
61            max_retries: 2,
62            backoff_initial: Duration::from_millis(500),
63            backoff_max: Duration::from_secs(5),
64            backoff_jitter: 0.25,
65            timeout: Duration::from_secs(30),
66            respect_retry_after: true,
67            max_retry_after: Duration::from_secs(60),
68            retry_connection_errors: true,
69            retry_timeouts: true,
70            retry_statuses: [408, 429].into_iter().chain(500..=599).collect(),
71        }
72    }
73}
74
75impl RetryPolicy {
76    /// The default policy with retrying switched off. The per-attempt timeout still applies.
77    #[must_use]
78    pub fn none() -> Self {
79        Self::default().with_max_retries(0)
80    }
81
82    /// Sets the number of retries after the first attempt.
83    #[must_use]
84    pub const fn with_max_retries(mut self, max_retries: u32) -> Self {
85        self.max_retries = max_retries;
86        self
87    }
88
89    /// Sets the first backoff delay and the longest one.
90    #[must_use]
91    pub const fn with_backoff(mut self, initial: Duration, max: Duration) -> Self {
92        self.backoff_initial = initial;
93        self.backoff_max = max;
94        self
95    }
96
97    /// Sets the fraction of each backoff delay that is randomly subtracted. Values outside 0 to 1,
98    /// and NaN, are clamped into range when a delay is computed.
99    #[must_use]
100    pub const fn with_jitter(mut self, fraction: f64) -> Self {
101        self.backoff_jitter = fraction;
102        self
103    }
104
105    /// Sets the time allowed for one attempt.
106    #[must_use]
107    pub const fn with_timeout(mut self, timeout: Duration) -> Self {
108        self.timeout = timeout;
109        self
110    }
111
112    /// Returns `true` when a response with `status` should be retried under this policy.
113    #[must_use]
114    pub fn retries_status(&self, status: u16) -> bool {
115        !NEVER_RETRIED.contains(&status) && self.retry_statuses.contains(&status)
116    }
117
118    /// The backoff delay before retry number `retry` (0 for the first retry).
119    ///
120    /// `random` is a sample from 0 to 1 that decides how much of the jitter is applied; passing it
121    /// in keeps this function pure.
122    #[must_use]
123    pub fn backoff_delay(&self, retry: u32, random: f64) -> Duration {
124        let doubled = self
125            .backoff_initial
126            .saturating_mul(2_u32.saturating_pow(retry));
127        let capped = doubled.min(self.backoff_max);
128
129        let jitter = clamp_unit(self.backoff_jitter);
130        capped.mul_f64(1.0 - jitter * clamp_unit(random))
131    }
132
133    /// The delay before the next retry: the server's, when there is an acceptable one, and
134    /// otherwise backoff.
135    pub(crate) fn delay(
136        &self,
137        retry: u32,
138        server_delay: Option<Duration>,
139        random: f64,
140    ) -> Duration {
141        server_delay
142            .filter(|delay| self.respect_retry_after && *delay <= self.max_retry_after)
143            .unwrap_or_else(|| self.backoff_delay(retry, random))
144    }
145}
146
147/// Clamps to the range 0 to 1, reading NaN as 0.
148fn clamp_unit(value: f64) -> f64 {
149    if value.is_nan() {
150        0.0
151    } else {
152        value.clamp(0.0, 1.0)
153    }
154}
155
156/// The delay a server asked for, from `retry-after-ms` or `Retry-After`.
157///
158/// `Retry-After` may be a number of seconds or an HTTP date; a date is measured from `now`. A value
159/// that cannot be read, or that is negative, yields `None`.
160pub(crate) fn server_delay(headers: &HeaderMap, now: SystemTime) -> Option<Duration> {
161    let text = |name: &str| headers.get(name)?.to_str().ok().map(str::trim);
162
163    if let Some(milliseconds) = text(RETRY_AFTER_MS).and_then(|value| value.parse::<f64>().ok()) {
164        return duration_from_secs(milliseconds / 1000.0);
165    }
166
167    let value = text(RETRY_AFTER.as_str())?;
168    if let Ok(seconds) = value.parse::<f64>() {
169        return duration_from_secs(seconds);
170    }
171    let date = httpdate::parse_http_date(value).ok()?;
172    Some(date.duration_since(now).unwrap_or(Duration::ZERO))
173}
174
175fn duration_from_secs(seconds: f64) -> Option<Duration> {
176    Duration::try_from_secs_f64(seconds).ok()
177}
178
179#[cfg(test)]
180mod tests {
181    use std::time::{Duration, SystemTime};
182
183    use reqwest::header::{HeaderMap, HeaderValue};
184
185    use super::{RetryPolicy, server_delay};
186
187    fn headers(pairs: &[(&'static str, &'static str)]) -> HeaderMap {
188        pairs
189            .iter()
190            .map(|&(name, value)| (name.parse().unwrap(), HeaderValue::from_static(value)))
191            .collect()
192    }
193
194    #[test]
195    fn the_defaults_match_the_official_sdks() {
196        let policy = RetryPolicy::default();
197
198        assert_eq!(policy.max_retries, 2);
199        assert_eq!(policy.backoff_initial, Duration::from_millis(500));
200        assert_eq!(policy.backoff_max, Duration::from_secs(5));
201        assert_eq!(policy.backoff_jitter.to_bits(), 0.25_f64.to_bits());
202        assert_eq!(policy.timeout, Duration::from_secs(30));
203        assert_eq!(policy.max_retry_after, Duration::from_secs(60));
204        assert!(
205            policy.respect_retry_after && policy.retry_connection_errors && policy.retry_timeouts
206        );
207    }
208
209    #[test]
210    fn retries_408_429_and_every_5xx_including_529() {
211        let policy = RetryPolicy::default();
212
213        for status in [408, 429, 500, 502, 503, 504, 529, 599] {
214            assert!(policy.retries_status(status), "{status}");
215        }
216        for status in [200, 302, 400, 401, 403, 404, 409, 422, 499] {
217            assert!(!policy.retries_status(status), "{status}");
218        }
219    }
220
221    #[test]
222    fn request_errors_are_never_retried_even_when_the_policy_lists_them() {
223        let policy = RetryPolicy {
224            retry_statuses: vec![400, 401, 403, 404, 409, 422],
225            ..RetryPolicy::default()
226        };
227
228        for status in [400, 401, 403, 404, 422] {
229            assert!(!policy.retries_status(status), "{status}");
230        }
231        assert!(policy.retries_status(409));
232    }
233
234    #[test]
235    fn backoff_doubles_from_half_a_second_up_to_the_cap() {
236        let policy = RetryPolicy::default().with_jitter(0.0);
237
238        let delays: Vec<u128> = (0..6)
239            .map(|retry| policy.backoff_delay(retry, 0.5).as_millis())
240            .collect();
241
242        assert_eq!(delays, [500, 1000, 2000, 4000, 5000, 5000]);
243        assert_eq!(
244            policy.backoff_delay(u32::MAX, 0.5),
245            Duration::from_secs(5),
246            "no overflow"
247        );
248    }
249
250    #[test]
251    fn jitter_only_ever_subtracts_up_to_its_fraction() {
252        let policy = RetryPolicy::default();
253
254        assert_eq!(policy.backoff_delay(0, 0.0), Duration::from_millis(500));
255        assert_eq!(policy.backoff_delay(0, 1.0), Duration::from_millis(375));
256        assert_eq!(policy.backoff_delay(0, 0.5), Duration::from_micros(437_500));
257    }
258
259    #[test]
260    fn nonsense_jitter_and_random_values_are_clamped() {
261        let full = RetryPolicy::default().with_jitter(7.0);
262        let negative = RetryPolicy::default().with_jitter(-1.0);
263        let nan = RetryPolicy::default().with_jitter(f64::NAN);
264
265        assert_eq!(full.backoff_delay(0, 1.0), Duration::ZERO);
266        assert_eq!(negative.backoff_delay(0, 1.0), Duration::from_millis(500));
267        assert_eq!(nan.backoff_delay(0, 1.0), Duration::from_millis(500));
268        assert_eq!(
269            RetryPolicy::default().backoff_delay(0, f64::NAN),
270            Duration::from_millis(500)
271        );
272        assert_eq!(
273            RetryPolicy::default().backoff_delay(0, -3.0),
274            Duration::from_millis(500)
275        );
276    }
277
278    #[test]
279    fn reads_the_server_delay_in_seconds_milliseconds_or_as_a_date() {
280        let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000_000); // 2001-09-09T01:46:40Z
281
282        assert_eq!(
283            server_delay(&headers(&[("retry-after", "1")]), now),
284            Some(Duration::from_secs(1))
285        );
286        assert_eq!(
287            server_delay(&headers(&[("retry-after", " 2.5 ")]), now),
288            Some(Duration::from_millis(2500))
289        );
290        assert_eq!(
291            server_delay(&headers(&[("retry-after-ms", "250")]), now),
292            Some(Duration::from_millis(250))
293        );
294        assert_eq!(
295            server_delay(
296                &headers(&[("retry-after", "Sun, 09 Sep 2001 01:46:50 GMT")]),
297                now
298            ),
299            Some(Duration::from_secs(10))
300        );
301        assert_eq!(
302            server_delay(
303                &headers(&[("retry-after", "Sun, 09 Sep 2001 01:00:00 GMT")]),
304                now
305            ),
306            Some(Duration::ZERO),
307            "a date in the past means now"
308        );
309    }
310
311    #[test]
312    fn milliseconds_win_over_seconds_and_unreadable_values_are_ignored() {
313        let now = SystemTime::UNIX_EPOCH;
314
315        let both = headers(&[("retry-after", "30"), ("retry-after-ms", "100")]);
316        assert_eq!(server_delay(&both, now), Some(Duration::from_millis(100)));
317
318        for value in ["soon", "-1", "NaN", "inf", ""] {
319            assert_eq!(
320                server_delay(&headers(&[("retry-after", value)]), now),
321                None,
322                "{value:?}"
323            );
324        }
325        assert_eq!(server_delay(&HeaderMap::new(), now), None);
326    }
327
328    #[test]
329    fn an_acceptable_server_delay_replaces_backoff_and_an_excessive_one_does_not() {
330        let policy = RetryPolicy::default().with_jitter(0.0);
331
332        assert_eq!(
333            policy.delay(0, Some(Duration::from_secs(1)), 0.0),
334            Duration::from_secs(1)
335        );
336        assert_eq!(
337            policy.delay(0, Some(Duration::from_secs(60)), 0.0),
338            Duration::from_secs(60)
339        );
340        assert_eq!(
341            policy.delay(1, Some(Duration::from_secs(61)), 0.0),
342            Duration::from_secs(1)
343        );
344        assert_eq!(policy.delay(1, None, 0.0), Duration::from_secs(1));
345
346        let deaf = RetryPolicy {
347            respect_retry_after: false,
348            ..policy
349        };
350        assert_eq!(
351            deaf.delay(0, Some(Duration::from_secs(9)), 0.0),
352            Duration::from_millis(500)
353        );
354    }
355}