ironflow_core/
retry.rs

1//! Retry policy for transient failures with exponential backoff.
2//!
3//! The [`RetryPolicy`] struct configures how operations (HTTP requests, agent
4//! invocations) should be retried when they encounter transient errors. Retries
5//! use exponential backoff with optional jitter to avoid thundering-herd effects.
6//!
7//! # Retryable errors
8//!
9//! Not all errors are retried. Only *transient* failures are eligible:
10//!
11//! | Operation | Retried | Not retried |
12//! |-----------|---------|-------------|
13//! | **HTTP** | Transport errors (DNS, timeout, connection refused), 5xx, 429 | SSRF blocks, 4xx (except 429), response too large |
14//! | **Agent** | Process failures, timeouts, schema validation (CLI non-determinism) | Prompt too large |
15//!
16//! # Examples
17//!
18//! ```no_run
19//! use ironflow_core::operations::http::Http;
20//! use ironflow_core::retry::RetryPolicy;
21//!
22//! # async fn example() -> Result<(), ironflow_core::error::OperationError> {
23//! // Simple: retry up to 3 times with default backoff
24//! let output = Http::get("https://api.example.com/data")
25//!     .retry(3)
26//!     .await?;
27//!
28//! // Advanced: custom backoff settings
29//! let output = Http::get("https://api.example.com/data")
30//!     .retry_policy(
31//!         RetryPolicy::new(3)
32//!             .backoff(std::time::Duration::from_millis(500))
33//!             .max_backoff(std::time::Duration::from_secs(30))
34//!             .multiplier(3.0)
35//!     )
36//!     .await?;
37//! # Ok(())
38//! # }
39//! ```
40
41use std::time::Duration;
42
43use crate::error::{AgentError, OperationError};
44
45/// Default initial backoff between retries.
46const DEFAULT_INITIAL_BACKOFF: Duration = Duration::from_millis(200);
47
48/// Default maximum backoff between retries.
49const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
50
51/// Default backoff multiplier (doubles each retry).
52const DEFAULT_MULTIPLIER: f64 = 2.0;
53
54/// Retry policy with exponential backoff for transient failures.
55///
56/// Created via [`RetryPolicy::new`] with the maximum number of retries.
57/// All other parameters have sensible defaults and can be customised with
58/// builder methods.
59///
60/// # Defaults
61///
62/// | Parameter | Default |
63/// |-----------|---------|
64/// | `max_retries` | (required) |
65/// | `initial_backoff` | 200ms |
66/// | `max_backoff` | 30s |
67/// | `multiplier` | 2.0 |
68///
69/// # Examples
70///
71/// ```
72/// use std::time::Duration;
73/// use ironflow_core::retry::RetryPolicy;
74///
75/// let policy = RetryPolicy::new(3)
76///     .backoff(Duration::from_millis(100))
77///     .max_backoff(Duration::from_secs(10))
78///     .multiplier(3.0);
79/// ```
80#[derive(Debug, Clone)]
81pub struct RetryPolicy {
82    pub(crate) max_retries: u32,
83    pub(crate) initial_backoff: Duration,
84    pub(crate) max_backoff: Duration,
85    pub(crate) multiplier: f64,
86}
87
88impl RetryPolicy {
89    /// Create a new retry policy with the given maximum number of retries.
90    ///
91    /// The initial attempt is not counted as a retry, so `max_retries(3)` means
92    /// up to 4 total attempts (1 initial + 3 retries).
93    ///
94    /// # Panics
95    ///
96    /// Panics if `max_retries` is `0`.
97    ///
98    /// # Examples
99    ///
100    /// ```
101    /// use ironflow_core::retry::RetryPolicy;
102    ///
103    /// let policy = RetryPolicy::new(3);
104    /// assert_eq!(policy.max_retries(), 3);
105    /// ```
106    pub fn new(max_retries: u32) -> Self {
107        assert!(max_retries > 0, "max_retries must be greater than 0");
108        Self {
109            max_retries,
110            initial_backoff: DEFAULT_INITIAL_BACKOFF,
111            max_backoff: DEFAULT_MAX_BACKOFF,
112            multiplier: DEFAULT_MULTIPLIER,
113        }
114    }
115
116    /// Set the initial backoff duration before the first retry.
117    ///
118    /// Subsequent retries multiply this duration by the [`multiplier`](RetryPolicy::multiplier).
119    ///
120    /// # Panics
121    ///
122    /// Panics if `duration` is zero.
123    pub fn backoff(mut self, duration: Duration) -> Self {
124        assert!(!duration.is_zero(), "initial backoff must not be zero");
125        self.initial_backoff = duration;
126        self
127    }
128
129    /// Set the maximum backoff duration between retries.
130    ///
131    /// Even after many retries with exponential growth, the delay will never
132    /// exceed this value.
133    ///
134    /// # Panics
135    ///
136    /// Panics if `duration` is zero.
137    pub fn max_backoff(mut self, duration: Duration) -> Self {
138        assert!(!duration.is_zero(), "max backoff must not be zero");
139        self.max_backoff = duration;
140        self
141    }
142
143    /// Set the backoff multiplier applied after each retry.
144    ///
145    /// For example, with `multiplier(2.0)` and `backoff(200ms)`:
146    /// - Retry 1: 200ms
147    /// - Retry 2: 400ms
148    /// - Retry 3: 800ms
149    ///
150    /// # Panics
151    ///
152    /// Panics if `multiplier` is less than `1.0`, NaN, or infinity.
153    pub fn multiplier(mut self, multiplier: f64) -> Self {
154        assert!(
155            multiplier >= 1.0 && multiplier.is_finite(),
156            "multiplier must be >= 1.0 and finite, got {multiplier}"
157        );
158        self.multiplier = multiplier;
159        self
160    }
161
162    /// Return the maximum number of retries.
163    pub fn max_retries(&self) -> u32 {
164        self.max_retries
165    }
166
167    /// Compute the backoff duration for the given retry attempt (0-indexed).
168    ///
169    /// Returns the delay capped at [`max_backoff`](RetryPolicy::max_backoff).
170    pub(crate) fn delay_for_attempt(&self, attempt: u32) -> Duration {
171        let delay = self.initial_backoff.as_secs_f64() * self.multiplier.powi(attempt as i32);
172        let capped = delay.min(self.max_backoff.as_secs_f64());
173        Duration::from_secs_f64(capped)
174    }
175}
176
177/// Returns `true` if the given [`OperationError`] is transient and should be
178/// retried.
179///
180/// # Retryable errors
181///
182/// - [`OperationError::Http`] with no status (transport error) or status 5xx / 429
183/// - [`OperationError::Agent`] wrapping [`AgentError::ProcessFailed`] or [`AgentError::Timeout`]
184/// - [`OperationError::Timeout`] (operation-level timeout)
185///
186/// # Non-retryable errors
187///
188/// - [`OperationError::Http`] with 4xx status (except 429)
189/// - [`OperationError::Agent`] wrapping [`AgentError::PromptTooLarge`] or
190///   [`AgentError::BudgetExceeded`] (the budget is already spent, replaying it
191///   costs money and cannot succeed)
192/// - [`OperationError::Shell`]
193/// - [`OperationError::Deserialize`]
194pub fn is_retryable(error: &OperationError) -> bool {
195    match error {
196        OperationError::Http { status, .. } => match status {
197            None => true,
198            Some(code) => *code >= 500 || *code == 429,
199        },
200        OperationError::Agent(agent_err) => match agent_err {
201            AgentError::ProcessFailed { .. }
202            | AgentError::Timeout { .. }
203            | AgentError::SchemaValidation { .. }
204            | AgentError::RateLimited { .. } => true,
205            AgentError::HttpProvider { status_code, .. } => {
206                *status_code == 0 || *status_code >= 500
207            }
208            AgentError::PromptTooLarge { .. } | AgentError::BudgetExceeded { .. } => false,
209        },
210        OperationError::Timeout { .. } => true,
211        OperationError::Shell { .. } | OperationError::Deserialize { .. } => false,
212    }
213}
214
215/// Returns `true` if the given HTTP status code indicates a transient server
216/// error that should be retried (5xx or 429 Too Many Requests).
217pub(crate) fn is_retryable_status(status: u16) -> bool {
218    status >= 500 || status == 429
219}
220
221#[cfg(test)]
222mod tests {
223    use super::*;
224    use std::time::Duration;
225
226    // --- RetryPolicy builder ---
227
228    #[test]
229    fn new_creates_policy_with_defaults() {
230        let policy = RetryPolicy::new(3);
231        assert_eq!(policy.max_retries, 3);
232        assert_eq!(policy.initial_backoff, DEFAULT_INITIAL_BACKOFF);
233        assert_eq!(policy.max_backoff, DEFAULT_MAX_BACKOFF);
234        assert!((policy.multiplier - DEFAULT_MULTIPLIER).abs() < f64::EPSILON);
235    }
236
237    #[test]
238    #[should_panic(expected = "max_retries must be greater than 0")]
239    fn new_zero_retries_panics() {
240        let _ = RetryPolicy::new(0);
241    }
242
243    #[test]
244    fn backoff_sets_initial_backoff() {
245        let policy = RetryPolicy::new(1).backoff(Duration::from_secs(1));
246        assert_eq!(policy.initial_backoff, Duration::from_secs(1));
247    }
248
249    #[test]
250    #[should_panic(expected = "initial backoff must not be zero")]
251    fn backoff_zero_panics() {
252        let _ = RetryPolicy::new(1).backoff(Duration::ZERO);
253    }
254
255    #[test]
256    fn max_backoff_sets_cap() {
257        let policy = RetryPolicy::new(1).max_backoff(Duration::from_secs(60));
258        assert_eq!(policy.max_backoff, Duration::from_secs(60));
259    }
260
261    #[test]
262    #[should_panic(expected = "max backoff must not be zero")]
263    fn max_backoff_zero_panics() {
264        let _ = RetryPolicy::new(1).max_backoff(Duration::ZERO);
265    }
266
267    #[test]
268    fn multiplier_sets_value() {
269        let policy = RetryPolicy::new(1).multiplier(3.0);
270        assert!((policy.multiplier - 3.0).abs() < f64::EPSILON);
271    }
272
273    #[test]
274    #[should_panic(expected = "multiplier must be >= 1.0")]
275    fn multiplier_below_one_panics() {
276        let _ = RetryPolicy::new(1).multiplier(0.5);
277    }
278
279    #[test]
280    #[should_panic(expected = "multiplier must be >= 1.0 and finite")]
281    fn multiplier_nan_panics() {
282        let _ = RetryPolicy::new(1).multiplier(f64::NAN);
283    }
284
285    #[test]
286    #[should_panic(expected = "multiplier must be >= 1.0 and finite")]
287    fn multiplier_infinity_panics() {
288        let _ = RetryPolicy::new(1).multiplier(f64::INFINITY);
289    }
290
291    #[test]
292    fn max_retries_accessor() {
293        assert_eq!(RetryPolicy::new(5).max_retries(), 5);
294    }
295
296    // --- delay_for_attempt ---
297
298    #[test]
299    fn delay_for_attempt_zero_is_initial_backoff() {
300        let policy = RetryPolicy::new(3).backoff(Duration::from_millis(100));
301        let delay = policy.delay_for_attempt(0);
302        assert_eq!(delay, Duration::from_millis(100));
303    }
304
305    #[test]
306    fn delay_for_attempt_grows_exponentially() {
307        let policy = RetryPolicy::new(5)
308            .backoff(Duration::from_millis(100))
309            .multiplier(2.0);
310
311        assert_eq!(policy.delay_for_attempt(0), Duration::from_millis(100));
312        assert_eq!(policy.delay_for_attempt(1), Duration::from_millis(200));
313        assert_eq!(policy.delay_for_attempt(2), Duration::from_millis(400));
314        assert_eq!(policy.delay_for_attempt(3), Duration::from_millis(800));
315    }
316
317    #[test]
318    fn delay_for_attempt_capped_at_max_backoff() {
319        let policy = RetryPolicy::new(10)
320            .backoff(Duration::from_secs(1))
321            .max_backoff(Duration::from_secs(5))
322            .multiplier(10.0);
323
324        // attempt 0: 1s, attempt 1: 10s (capped to 5s)
325        assert_eq!(policy.delay_for_attempt(0), Duration::from_secs(1));
326        assert_eq!(policy.delay_for_attempt(1), Duration::from_secs(5));
327        assert_eq!(policy.delay_for_attempt(2), Duration::from_secs(5));
328    }
329
330    #[test]
331    fn delay_for_attempt_with_multiplier_one_is_constant() {
332        let policy = RetryPolicy::new(3)
333            .backoff(Duration::from_millis(500))
334            .multiplier(1.0);
335
336        assert_eq!(policy.delay_for_attempt(0), Duration::from_millis(500));
337        assert_eq!(policy.delay_for_attempt(1), Duration::from_millis(500));
338        assert_eq!(policy.delay_for_attempt(2), Duration::from_millis(500));
339    }
340
341    // --- is_retryable ---
342
343    #[test]
344    fn http_transport_error_is_retryable() {
345        let err = OperationError::Http {
346            status: None,
347            message: "connection refused".to_string(),
348        };
349        assert!(is_retryable(&err));
350    }
351
352    #[test]
353    fn http_500_is_retryable() {
354        let err = OperationError::Http {
355            status: Some(500),
356            message: "internal server error".to_string(),
357        };
358        assert!(is_retryable(&err));
359    }
360
361    #[test]
362    fn http_502_is_retryable() {
363        let err = OperationError::Http {
364            status: Some(502),
365            message: "bad gateway".to_string(),
366        };
367        assert!(is_retryable(&err));
368    }
369
370    #[test]
371    fn http_503_is_retryable() {
372        let err = OperationError::Http {
373            status: Some(503),
374            message: "service unavailable".to_string(),
375        };
376        assert!(is_retryable(&err));
377    }
378
379    #[test]
380    fn http_429_is_retryable() {
381        let err = OperationError::Http {
382            status: Some(429),
383            message: "too many requests".to_string(),
384        };
385        assert!(is_retryable(&err));
386    }
387
388    #[test]
389    fn http_400_is_not_retryable() {
390        let err = OperationError::Http {
391            status: Some(400),
392            message: "bad request".to_string(),
393        };
394        assert!(!is_retryable(&err));
395    }
396
397    #[test]
398    fn http_404_is_not_retryable() {
399        let err = OperationError::Http {
400            status: Some(404),
401            message: "not found".to_string(),
402        };
403        assert!(!is_retryable(&err));
404    }
405
406    #[test]
407    fn agent_process_failed_is_retryable() {
408        let err = OperationError::Agent(AgentError::ProcessFailed {
409            exit_code: 1,
410            stderr: "crash".to_string(),
411        });
412        assert!(is_retryable(&err));
413    }
414
415    #[test]
416    fn agent_timeout_is_retryable() {
417        let err = OperationError::Agent(AgentError::Timeout {
418            limit: Duration::from_secs(60),
419        });
420        assert!(is_retryable(&err));
421    }
422
423    #[test]
424    fn agent_prompt_too_large_is_not_retryable() {
425        let err = OperationError::Agent(AgentError::PromptTooLarge {
426            chars: 1_000_000,
427            estimated_tokens: 250_000,
428            model_limit: 200_000,
429        });
430        assert!(!is_retryable(&err));
431    }
432
433    #[test]
434    fn agent_budget_exceeded_is_not_retryable() {
435        let err = OperationError::Agent(AgentError::BudgetExceeded {
436            spent_usd: 0.30,
437            limit_usd: 0.25,
438            debug_messages: Vec::new(),
439            partial_usage: Box::default(),
440        });
441        assert!(!is_retryable(&err));
442    }
443
444    #[test]
445    fn agent_schema_validation_is_retryable() {
446        let err = OperationError::Agent(AgentError::SchemaValidation {
447            expected: "object".to_string(),
448            got: "string".to_string(),
449            debug_messages: Vec::new(),
450            partial_usage: Box::default(),
451            raw_response: None,
452        });
453        assert!(is_retryable(&err));
454    }
455
456    #[test]
457    fn operation_timeout_is_retryable() {
458        let err = OperationError::Timeout {
459            step: "fetch".to_string(),
460            limit: Duration::from_secs(30),
461        };
462        assert!(is_retryable(&err));
463    }
464
465    #[test]
466    fn shell_error_is_not_retryable() {
467        let err = OperationError::Shell {
468            exit_code: 1,
469            stderr: "fail".to_string(),
470        };
471        assert!(!is_retryable(&err));
472    }
473
474    #[test]
475    fn deserialize_error_is_not_retryable() {
476        let err = OperationError::Deserialize {
477            target_type: "MyStruct".to_string(),
478            reason: "missing field".to_string(),
479        };
480        assert!(!is_retryable(&err));
481    }
482
483    // --- is_retryable_status ---
484
485    #[test]
486    fn retryable_status_codes() {
487        assert!(is_retryable_status(500));
488        assert!(is_retryable_status(502));
489        assert!(is_retryable_status(503));
490        assert!(is_retryable_status(504));
491        assert!(is_retryable_status(429));
492    }
493
494    #[test]
495    fn non_retryable_status_codes() {
496        assert!(!is_retryable_status(200));
497        assert!(!is_retryable_status(201));
498        assert!(!is_retryable_status(301));
499        assert!(!is_retryable_status(400));
500        assert!(!is_retryable_status(401));
501        assert!(!is_retryable_status(403));
502        assert!(!is_retryable_status(404));
503        assert!(!is_retryable_status(422));
504        assert!(!is_retryable_status(428));
505    }
506
507    // --- builder chaining ---
508
509    #[test]
510    fn builder_chain_all_methods() {
511        let policy = RetryPolicy::new(5)
512            .backoff(Duration::from_millis(100))
513            .max_backoff(Duration::from_secs(10))
514            .multiplier(3.0);
515
516        assert_eq!(policy.max_retries, 5);
517        assert_eq!(policy.initial_backoff, Duration::from_millis(100));
518        assert_eq!(policy.max_backoff, Duration::from_secs(10));
519        assert!((policy.multiplier - 3.0).abs() < f64::EPSILON);
520    }
521
522    #[test]
523    fn clone_produces_independent_copy() {
524        let policy = RetryPolicy::new(3).backoff(Duration::from_millis(100));
525        let cloned = policy.clone();
526        assert_eq!(policy.max_retries, cloned.max_retries);
527        assert_eq!(policy.initial_backoff, cloned.initial_backoff);
528    }
529
530    #[test]
531    fn debug_does_not_panic() {
532        let policy = RetryPolicy::new(1);
533        let debug = format!("{:?}", policy);
534        assert!(debug.contains("RetryPolicy"));
535    }
536}
ironflow_core/retry.rs

ironflow_core/
retry.rs
