Skip to main content

thunderstore_engine/
retry.rs

1//! Bounded retry with exponential backoff for transient HTTP failures.
2//!
3//! Retries are applied only to idempotent network operations that failed for a
4//! *transient* reason (connection/timeout/body errors, `5xx`, or `429`) — never
5//! to a deterministic failure such as a `4xx` response or one of the crate's own
6//! terminal errors (a size-cap breach, a checksum mismatch, an unsafe filename).
7//! The whole download path is already retry-safe: archives stream to a temp file
8//! that is only renamed into place on success, so a retried attempt restarts
9//! cleanly.
10
11#![deny(missing_docs)]
12
13#[cfg(feature = "io")]
14use crate::error::{Error, Result};
15#[cfg(feature = "io")]
16use std::future::Future;
17#[cfg(feature = "io")]
18use std::sync::atomic::{AtomicU64, Ordering};
19use std::time::Duration;
20
21/// Policy for retrying transient network failures.
22///
23/// The delay before the retry following a (0-based) attempt is
24/// `base_delay * 2^attempt`, capped at `max_delay`, plus a little jitter.
25#[derive(Debug, Clone, Copy)]
26pub struct RetryPolicy {
27  /// Maximum number of retries *after* the initial attempt (so total attempts is
28  /// `max_retries + 1`). Zero disables retrying.
29  pub max_retries: u32,
30  /// The base backoff delay, doubled on each successive retry.
31  pub base_delay: Duration,
32  /// The ceiling on any single backoff delay.
33  pub max_delay: Duration,
34}
35
36impl Default for RetryPolicy {
37  fn default() -> Self {
38    Self {
39      max_retries: 3,
40      base_delay: Duration::from_millis(250),
41      max_delay: Duration::from_secs(10),
42    }
43  }
44}
45
46impl RetryPolicy {
47  /// A policy that performs no retries (a single attempt).
48  pub fn none() -> Self {
49    Self {
50      max_retries: 0,
51      ..Self::default()
52    }
53  }
54
55  /// The backoff delay before the retry following `attempt` (0-based).
56  #[cfg(feature = "io")]
57  fn backoff(&self, attempt: u32) -> Duration {
58    let factor = 1u32.checked_shl(attempt).unwrap_or(u32::MAX);
59    let capped = self.base_delay.saturating_mul(factor).min(self.max_delay);
60    let jitter_ceiling_ms = (capped.as_millis() / 4) as u64;
61
62    if jitter_ceiling_ms == 0 {
63      return capped;
64    }
65
66    capped + Duration::from_millis(jitter_source() % (jitter_ceiling_ms + 1))
67  }
68}
69
70/// A cheap, dependency-free varying value used only to desynchronize concurrent
71/// retries. Not cryptographic and not for any security purpose.
72#[cfg(feature = "io")]
73fn jitter_source() -> u64 {
74  static COUNTER: AtomicU64 = AtomicU64::new(0);
75
76  COUNTER.fetch_add(0x9E37_79B9_7F4A_7C15, Ordering::Relaxed)
77}
78
79/// Runs `op`, retrying per `policy` while `is_retryable` accepts the error.
80///
81/// Returns the first `Ok`, or the last `Err` once retries are exhausted or the
82/// error is non-retryable.
83///
84/// `op` is a plain `FnMut` returning a future (call sites pass `|| async { .. }`)
85/// rather than an `AsyncFnMut`. This is deliberate: an async closure's returned
86/// future carries a higher-ranked lifetime that the compiler cannot prove `Send`
87/// "for any lifetime", which would make every retried operation — and therefore
88/// the public `get_manifest`/`download_files` futures — `!Send`. A closure
89/// returning a single concrete future type has no such higher-ranked lifetime, so
90/// the futures stay `Send` and can be driven from a multi-threaded runtime.
91#[cfg(feature = "io")]
92pub(crate) async fn retry_with<T, F>(
93  policy: &RetryPolicy,
94  is_retryable: impl Fn(&Error) -> bool,
95  mut op: impl FnMut() -> F,
96) -> Result<T>
97where
98  F: Future<Output = Result<T>>,
99{
100  let mut attempt = 0u32;
101
102  loop {
103    match op().await {
104      Ok(value) => return Ok(value),
105      Err(err) => {
106        if attempt >= policy.max_retries || !is_retryable(&err) {
107          return Err(err);
108        }
109
110        let delay = policy.backoff(attempt);
111
112        tracing::warn!(
113          "transient network error ({}); retry {}/{}",
114          err,
115          attempt + 1,
116          policy.max_retries
117        );
118
119        if !delay.is_zero() {
120          tokio::time::sleep(delay).await;
121        }
122
123        attempt += 1;
124      }
125    }
126  }
127}
128
129/// [`retry_with`] using [`is_retryable_http`] as the classifier.
130#[cfg(feature = "io")]
131pub(crate) async fn retry_http<T, F>(policy: &RetryPolicy, op: impl FnMut() -> F) -> Result<T>
132where
133  F: Future<Output = Result<T>>,
134{
135  retry_with(policy, is_retryable_http, op).await
136}
137
138/// Whether an error is a transient HTTP failure worth retrying: a connection,
139/// timeout, request, or body error, or a `5xx`/`429` status. A `4xx` (other than
140/// `429`) and every non-HTTP error are treated as terminal.
141#[cfg(feature = "io")]
142pub(crate) fn is_retryable_http(err: &Error) -> bool {
143  match err {
144    Error::Http(e) => {
145      if let Some(status) = e.status() {
146        status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS
147      } else {
148        e.is_timeout() || e.is_connect() || e.is_request() || e.is_body()
149      }
150    }
151    _ => false,
152  }
153}
154
155#[cfg(all(test, feature = "io"))]
156mod tests {
157  use super::*;
158  use std::cell::Cell;
159
160  fn zero_delay(max_retries: u32) -> RetryPolicy {
161    RetryPolicy {
162      max_retries,
163      base_delay: Duration::ZERO,
164      max_delay: Duration::ZERO,
165    }
166  }
167
168  #[tokio::test]
169  async fn retries_until_success() {
170    let calls = Cell::new(0u32);
171
172    let result: Result<u32> = retry_with(
173      &zero_delay(3),
174      |_| true,
175      || async {
176        calls.set(calls.get() + 1);
177
178        if calls.get() < 3 {
179          Err(Error::Cache("transient".to_string()))
180        } else {
181          Ok(42)
182        }
183      },
184    )
185    .await;
186
187    assert_eq!(result.unwrap(), 42);
188    assert_eq!(calls.get(), 3);
189  }
190
191  #[tokio::test]
192  async fn gives_up_after_max_retries() {
193    let calls = Cell::new(0u32);
194
195    let result: Result<u32> = retry_with(
196      &zero_delay(2),
197      |_| true,
198      || async {
199        calls.set(calls.get() + 1);
200
201        Err(Error::Cache("always".to_string()))
202      },
203    )
204    .await;
205
206    assert!(result.is_err());
207    assert_eq!(calls.get(), 3); // initial attempt + 2 retries
208  }
209
210  #[tokio::test]
211  async fn does_not_retry_non_retryable() {
212    let calls = Cell::new(0u32);
213
214    let result: Result<u32> = retry_with(
215      &zero_delay(5),
216      |_| false,
217      || async {
218        calls.set(calls.get() + 1);
219
220        Err(Error::Cache("terminal".to_string()))
221      },
222    )
223    .await;
224
225    assert!(result.is_err());
226    assert_eq!(calls.get(), 1);
227  }
228
229  #[test]
230  fn crate_terminal_errors_are_not_retryable() {
231    assert!(!is_retryable_http(&Error::SizeLimitExceeded {
232      context: "download body".to_string(),
233      limit: 1,
234      actual: None,
235    }));
236    assert!(!is_retryable_http(&Error::Cache("bad cache".to_string())));
237    assert!(!is_retryable_http(&Error::ChecksumMismatch {
238      path: "x".to_string(),
239      expected: 1,
240      actual: 2,
241    }));
242  }
243}