1use std::collections::hash_map::DefaultHasher;
36use std::hash::{Hash, Hasher};
37use std::time::{Duration, Instant, SystemTime};
38
39use reqwest::StatusCode;
40use reqwest::header::HeaderMap;
41
42use crate::config::Config;
43
44pub const DEFAULT_MAX_RETRIES: u32 = 3;
46pub const DEFAULT_RETRY_BASE_MS: u64 = 200;
48pub const DEFAULT_RETRY_MAX_DELAY_MS: u64 = 30_000;
50pub const HARD_MAX_RETRIES: u32 = 10;
52pub const HARD_MAX_DELAY_MS: u64 = 60_000;
54pub const MIN_RETRY_BASE_MS: u64 = 50;
56pub const HARD_MAX_ELAPSED_MS: u64 = 300_000;
58pub const DEFAULT_RETRY_MAX_ELAPSED_MS: u64 = 30_000;
61const RETRY_AFTER_SKEW_SECS: u64 = 1;
63
64#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub struct RetryConfig {
70 pub max_retries: u32,
72 pub base_ms: u64,
74 pub max_delay_ms: u64,
76 pub max_elapsed_ms: u64,
78 pub enabled: bool,
80}
81
82impl RetryConfig {
83 pub fn from_config(cfg: &Config) -> Self {
88 let max_retries = cfg.max_retries.min(HARD_MAX_RETRIES);
89 let base_ms = if cfg.retry_base_ms == 0 {
90 DEFAULT_RETRY_BASE_MS
91 } else {
92 cfg.retry_base_ms.max(MIN_RETRY_BASE_MS)
93 };
94 let max_delay_ms = cfg.retry_max_delay_ms.min(HARD_MAX_DELAY_MS).max(base_ms);
95 let derived = cfg.timeout_secs.saturating_mul(1000).max(1_000);
98 let max_elapsed_ms = if cfg.retry_max_elapsed_ms == 0 {
99 derived.min(HARD_MAX_ELAPSED_MS)
100 } else {
101 cfg.retry_max_elapsed_ms
102 .min(HARD_MAX_ELAPSED_MS)
103 .max(max_delay_ms)
104 };
105 let enabled = !cfg.disable_retry && max_retries > 0;
106 Self {
107 max_retries,
108 base_ms,
109 max_delay_ms,
110 max_elapsed_ms,
111 enabled,
112 }
113 }
114
115 pub fn max_attempts(self) -> u32 {
117 if !self.enabled {
118 1
119 } else {
120 self.max_retries.saturating_add(1)
121 }
122 }
123
124 pub fn may_retry(self, attempt: u32) -> bool {
126 self.enabled && attempt < self.max_attempts()
127 }
128
129 pub fn may_retry_within_budget(
134 self,
135 attempt: u32,
136 elapsed: Duration,
137 planned_wait: Duration,
138 ) -> bool {
139 if !self.may_retry(attempt) {
140 return false;
141 }
142 let budget = Duration::from_millis(self.max_elapsed_ms);
143 if elapsed >= budget {
144 return false;
145 }
146 let remaining = budget.saturating_sub(elapsed);
147 planned_wait <= remaining
148 }
149
150 pub fn remaining_budget(self, elapsed: Duration) -> Duration {
152 Duration::from_millis(self.max_elapsed_ms).saturating_sub(elapsed)
153 }
154
155 pub fn delay_for_attempt(self, attempt: u32) -> Duration {
157 backoff_full_jitter(self.base_ms, attempt, self.max_delay_ms)
158 }
159}
160
161impl Default for RetryConfig {
162 fn default() -> Self {
163 Self {
164 max_retries: DEFAULT_MAX_RETRIES,
165 base_ms: DEFAULT_RETRY_BASE_MS,
166 max_delay_ms: DEFAULT_RETRY_MAX_DELAY_MS,
167 max_elapsed_ms: DEFAULT_RETRY_MAX_ELAPSED_MS,
168 enabled: true,
169 }
170 }
171}
172
173#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub enum RetryClass {
176 Permanent,
178 Transient,
180 RateLimited {
182 retry_after: Option<Duration>,
184 },
185}
186
187#[derive(Debug, Clone, Copy, PartialEq, Eq)]
189pub enum RetryKind {
190 Permanent,
192 TransientNetwork,
194 TransientServer,
196 RateLimited,
198 Timeout,
200}
201
202#[derive(Debug, Clone, Copy, PartialEq, Eq)]
208pub enum ErrorLayer {
209 Dns,
211 TcpConnect,
213 Tls,
215 HttpProtocol,
217 HttpStatus,
219 Application,
221 Unknown,
223}
224
225impl ErrorLayer {
226 pub fn as_str(self) -> &'static str {
228 match self {
229 Self::Dns => "dns",
230 Self::TcpConnect => "tcp_connect",
231 Self::Tls => "tls",
232 Self::HttpProtocol => "http_protocol",
233 Self::HttpStatus => "http_status",
234 Self::Application => "application",
235 Self::Unknown => "unknown",
236 }
237 }
238}
239
240impl RetryKind {
241 pub fn as_str(self) -> &'static str {
243 match self {
244 Self::Permanent => "permanent",
245 Self::TransientNetwork => "transient_network",
246 Self::TransientServer => "transient_server",
247 Self::RateLimited => "rate_limited",
248 Self::Timeout => "timeout",
249 }
250 }
251
252 pub fn is_retryable(self) -> bool {
254 !matches!(self, Self::Permanent)
255 }
256}
257
258pub fn classify_http_status(status: StatusCode, retry_after: Option<Duration>) -> RetryClass {
260 match status.as_u16() {
261 429 => RetryClass::RateLimited { retry_after },
262 408 | 500 | 502 | 504 => RetryClass::Transient,
264 503 => {
266 if retry_after.is_some() {
267 RetryClass::RateLimited { retry_after }
268 } else {
269 RetryClass::Transient
270 }
271 }
272 _ => RetryClass::Permanent,
274 }
275}
276
277pub(crate) fn source_chain_has_rustls(err: &(dyn std::error::Error + 'static)) -> bool {
283 let mut cur: Option<&(dyn std::error::Error + 'static)> = Some(err);
284 while let Some(e) = cur {
285 if e.downcast_ref::<rustls::Error>().is_some() {
286 return true;
287 }
288 cur = e.source();
289 }
290 false
291}
292
293pub fn classify_reqwest_error(err: &reqwest::Error) -> (crate::error::ErrorKind, ErrorLayer, bool) {
301 use crate::error::ErrorKind;
302
303 let tls = source_chain_has_rustls(err as &(dyn std::error::Error + 'static));
304
305 if err.is_timeout() {
306 let layer = if tls {
308 ErrorLayer::Tls
309 } else {
310 ErrorLayer::TcpConnect
311 };
312 return (ErrorKind::Timeout, layer, true);
313 }
314 if tls {
315 return (ErrorKind::Network, ErrorLayer::Tls, true);
317 }
318 if err.is_connect() {
319 return (ErrorKind::Network, ErrorLayer::TcpConnect, true);
321 }
322 if err.is_request() {
323 return (ErrorKind::Network, ErrorLayer::HttpProtocol, true);
325 }
326 if err.is_body() || err.is_decode() {
327 return (ErrorKind::Network, ErrorLayer::HttpProtocol, false);
330 }
331 (ErrorKind::Network, ErrorLayer::Unknown, false)
332}
333
334pub fn parse_retry_after(headers: &HeaderMap) -> Option<Duration> {
341 let raw = headers
342 .get(reqwest::header::RETRY_AFTER)
343 .and_then(|v| v.to_str().ok())?
344 .trim();
345 if raw.is_empty() {
346 return None;
347 }
348 if raw.bytes().all(|b| b.is_ascii_digit()) {
350 let secs: u64 = raw.parse().ok()?;
351 return Some(Duration::from_secs(secs.min(HARD_MAX_DELAY_MS / 1000)));
352 }
353 let when = httpdate::parse_http_date(raw).ok()?;
355 let now = SystemTime::now();
356 match when.duration_since(now) {
357 Ok(d) => Some(d.min(Duration::from_millis(HARD_MAX_DELAY_MS))),
358 Err(e) => {
359 if e.duration() <= Duration::from_secs(RETRY_AFTER_SKEW_SECS) {
361 Some(Duration::ZERO)
362 } else {
363 None
364 }
365 }
366 }
367}
368
369pub fn backoff_full_jitter(base_ms: u64, attempt: u32, max_delay_ms: u64) -> Duration {
375 let seed = mix_u64(attempt);
376 backoff_full_jitter_seeded(base_ms, attempt, max_delay_ms, seed)
377}
378
379pub fn backoff_full_jitter_seeded(
381 base_ms: u64,
382 attempt: u32,
383 max_delay_ms: u64,
384 seed: u64,
385) -> Duration {
386 let base = base_ms.max(1);
387 let max_delay = max_delay_ms.max(base);
388 let exp = base.saturating_mul(1u64 << attempt.min(16));
390 let cap = exp.min(max_delay);
391 let pick = seed % (cap.saturating_add(1));
392 Duration::from_millis(pick)
393}
394
395pub fn politeness_delay(base: Duration) -> Duration {
405 let base_ms = base.as_millis() as u64;
406 if base_ms == 0 {
407 return Duration::ZERO;
408 }
409 let span = (base_ms / 5).max(1);
410 let extra = mix_u64(base_ms as u32) % (span.saturating_add(1));
411 Duration::from_millis(base_ms.saturating_add(extra))
412}
413
414pub fn wait_for_retry(
422 policy: RetryConfig,
423 attempt: u32,
424 retry_after: Option<Duration>,
425) -> Duration {
426 if let Some(d) = retry_after {
427 return d.min(Duration::from_millis(HARD_MAX_DELAY_MS));
428 }
429 policy.delay_for_attempt(attempt)
430}
431
432fn mix_u64(attempt: u32) -> u64 {
433 let mut h = DefaultHasher::new();
434 Instant::now().hash(&mut h);
435 attempt.hash(&mut h);
436 std::thread::current().id().hash(&mut h);
437 let marker = &h as *const DefaultHasher as usize;
439 marker.hash(&mut h);
440 h.finish()
441}
442
443#[cfg(test)]
444mod tests {
445 use super::*;
446 use reqwest::header::{HeaderValue, RETRY_AFTER};
447
448 #[test]
449 fn max_attempts_respects_kill_switch() {
450 let mut p = RetryConfig::default();
451 assert_eq!(p.max_attempts(), 4);
452 p.enabled = false;
453 assert_eq!(p.max_attempts(), 1);
454 assert!(!p.may_retry(1));
455 }
456
457 #[test]
458 fn from_config_disable_and_clamp() {
459 let cfg = Config {
460 disable_retry: true,
461 max_retries: 99,
462 retry_max_delay_ms: 999_999,
463 retry_base_ms: 10, retry_max_elapsed_ms: 0, timeout_secs: 12,
466 ..Config::default()
467 };
468 let p = RetryConfig::from_config(&cfg);
469 assert!(!p.enabled);
470 assert_eq!(p.max_retries, HARD_MAX_RETRIES);
471 assert_eq!(p.max_delay_ms, HARD_MAX_DELAY_MS);
472 assert_eq!(p.base_ms, MIN_RETRY_BASE_MS);
473 assert_eq!(p.max_elapsed_ms, 12_000);
474 assert_eq!(p.max_attempts(), 1);
475 }
476
477 #[test]
478 fn from_config_explicit_max_elapsed() {
479 let cfg = Config {
480 retry_max_elapsed_ms: 5_000,
481 retry_base_ms: 200,
482 retry_max_delay_ms: 1_000,
483 ..Config::default()
484 };
485 let p = RetryConfig::from_config(&cfg);
486 assert_eq!(p.max_elapsed_ms, 5_000);
487 }
488
489 #[test]
490 fn may_retry_within_budget_blocks_oversleep() {
491 let p = RetryConfig {
492 max_retries: 5,
493 base_ms: 100,
494 max_delay_ms: 1_000,
495 max_elapsed_ms: 500,
496 enabled: true,
497 };
498 assert!(p.may_retry_within_budget(1, Duration::from_millis(0), Duration::from_millis(400)));
499 assert!(!p.may_retry_within_budget(1, Duration::from_millis(0), Duration::from_millis(600)));
500 assert!(!p.may_retry_within_budget(1, Duration::from_millis(500), Duration::ZERO));
501 }
502
503 #[test]
504 fn backoff_respects_cap() {
505 let d = backoff_full_jitter(200, 20, 1_000);
506 assert!(d.as_millis() <= 1_000);
507 }
508
509 #[test]
510 fn backoff_seeded_is_deterministic() {
511 let a = backoff_full_jitter_seeded(200, 3, 10_000, 42);
512 let b = backoff_full_jitter_seeded(200, 3, 10_000, 42);
513 assert_eq!(a, b);
514 for seed in 0..64u64 {
516 for attempt in 0..8u32 {
517 let d = backoff_full_jitter_seeded(100, attempt, 500, seed);
518 assert!(d.as_millis() <= 500);
519 }
520 }
521 }
522
523 #[test]
524 fn politeness_delay_zero_stays_zero() {
525 assert_eq!(politeness_delay(Duration::ZERO), Duration::ZERO);
526 }
527
528 #[test]
529 fn politeness_delay_never_below_floor() {
530 let base = Duration::from_millis(1000);
531 for _ in 0..32 {
532 let d = politeness_delay(base);
533 assert!(d >= base, "delay {d:?} below floor {base:?}");
534 assert!(
536 d <= base + Duration::from_millis(200),
537 "delay {d:?} above floor+20%"
538 );
539 }
540 }
541
542 #[test]
543 fn classify_matrix() {
544 assert_eq!(
545 classify_http_status(StatusCode::OK, None),
546 RetryClass::Permanent
547 );
548 assert_eq!(
549 classify_http_status(StatusCode::NOT_FOUND, None),
550 RetryClass::Permanent
551 );
552 assert_eq!(
553 classify_http_status(StatusCode::BAD_REQUEST, None),
554 RetryClass::Permanent
555 );
556 assert_eq!(
557 classify_http_status(StatusCode::INTERNAL_SERVER_ERROR, None),
558 RetryClass::Transient
559 );
560 assert_eq!(
561 classify_http_status(StatusCode::REQUEST_TIMEOUT, None),
562 RetryClass::Transient
563 );
564 assert_eq!(
565 classify_http_status(StatusCode::TOO_MANY_REQUESTS, Some(Duration::from_secs(2))),
566 RetryClass::RateLimited {
567 retry_after: Some(Duration::from_secs(2))
568 }
569 );
570 assert_eq!(
571 classify_http_status(
572 StatusCode::SERVICE_UNAVAILABLE,
573 Some(Duration::from_secs(1))
574 ),
575 RetryClass::RateLimited {
576 retry_after: Some(Duration::from_secs(1))
577 }
578 );
579 assert_eq!(
580 classify_http_status(StatusCode::SERVICE_UNAVAILABLE, None),
581 RetryClass::Transient
582 );
583 assert_eq!(
585 classify_http_status(StatusCode::UNAUTHORIZED, None),
586 RetryClass::Permanent
587 );
588 assert_eq!(
589 classify_http_status(StatusCode::FORBIDDEN, None),
590 RetryClass::Permanent
591 );
592 assert_eq!(
593 classify_http_status(StatusCode::UNPROCESSABLE_ENTITY, None),
594 RetryClass::Permanent
595 );
596 }
597
598 #[test]
599 fn parse_retry_after_seconds_and_http_date() {
600 let mut h = HeaderMap::new();
601 h.insert(RETRY_AFTER, HeaderValue::from_static("7"));
602 assert_eq!(parse_retry_after(&h), Some(Duration::from_secs(7)));
603
604 h.insert(RETRY_AFTER, HeaderValue::from_static(""));
605 assert_eq!(parse_retry_after(&h), None);
606
607 let future = SystemTime::now() + Duration::from_secs(2);
609 let http_date = httpdate::fmt_http_date(future);
610 h.insert(
611 RETRY_AFTER,
612 HeaderValue::from_str(&http_date).expect("http-date header"),
613 );
614 let d = parse_retry_after(&h).expect("HTTP-date Retry-After");
615 assert!(d <= Duration::from_secs(3), "got {d:?}");
616 assert!(d >= Duration::from_millis(500), "got {d:?}");
617
618 let past = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000);
620 let past_s = httpdate::fmt_http_date(past);
621 h.insert(
622 RETRY_AFTER,
623 HeaderValue::from_str(&past_s).expect("past http-date"),
624 );
625 assert_eq!(parse_retry_after(&h), None);
626 }
627
628 #[test]
629 fn wait_prefers_retry_after() {
630 let p = RetryConfig::default();
631 let w = wait_for_retry(p, 1, Some(Duration::from_secs(3)));
632 assert_eq!(w, Duration::from_secs(3));
633 }
634
635 #[test]
636 fn source_chain_detects_rustls_and_ignores_plain_io() {
637 let tls = rustls::Error::General("unit-test".into());
638 assert!(source_chain_has_rustls(&tls));
639 let io = std::io::Error::other("not tls");
640 assert!(!source_chain_has_rustls(&io));
641 assert_eq!(ErrorLayer::Tls.as_str(), "tls");
642 }
643
644 #[test]
645 fn retry_kind_labels() {
646 assert_eq!(RetryKind::Timeout.as_str(), "timeout");
647 assert!(RetryKind::TransientNetwork.is_retryable());
648 assert!(!RetryKind::Permanent.is_retryable());
649 assert_eq!(ErrorLayer::TcpConnect.as_str(), "tcp_connect");
650 }
651}