ironflow_core/
retry.rs

1//! Retry policy for transient failures with exponential backoff.
2//!
3//! The [`RetryPolicy`] struct configures how operations (HTTP requests, agent
4//! invocations) should be retried when they encounter transient errors. Retries
5//! use exponential backoff with optional jitter to avoid thundering-herd effects.
6//!
7//! # Retryable errors
8//!
9//! Not all errors are retried. Only *transient* failures are eligible:
10//!
11//! | Operation | Retried | Not retried |
12//! |-----------|---------|-------------|
13//! | **HTTP** | Transport errors (DNS, timeout, connection refused), 5xx, 429 | SSRF blocks, 4xx (except 429), response too large |
14//! | **Agent** | Process failures, timeouts, schema validation (CLI non-determinism) | Prompt too large |
15//!
16//! # Examples
17//!
18//! ```no_run
19//! use ironflow_core::operations::http::Http;
20//! use ironflow_core::retry::RetryPolicy;
21//!
22//! # async fn example() -> Result<(), ironflow_core::error::OperationError> {
23//! // Simple: retry up to 3 times with default backoff
24//! let output = Http::get("https://api.example.com/data")
25//!     .retry(3)
26//!     .await?;
27//!
28//! // Advanced: custom backoff settings
29//! let output = Http::get("https://api.example.com/data")
30//!     .retry_policy(
31//!         RetryPolicy::new(3)
32//!             .backoff(std::time::Duration::from_millis(500))
33//!             .max_backoff(std::time::Duration::from_secs(30))
34//!             .multiplier(3.0)
35//!     )
36//!     .await?;
37//! # Ok(())
38//! # }
39//! ```
40
41use std::time::Duration;
42
43use crate::error::{AgentError, OperationError};
44
45/// Default initial backoff between retries.
46const DEFAULT_INITIAL_BACKOFF: Duration = Duration::from_millis(200);
47
48/// Default maximum backoff between retries.
49const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
50
51/// Default backoff multiplier (doubles each retry).
52const DEFAULT_MULTIPLIER: f64 = 2.0;
53
54/// Retry policy with exponential backoff for transient failures.
55///
56/// Created via [`RetryPolicy::new`] with the maximum number of retries.
57/// All other parameters have sensible defaults and can be customised with
58/// builder methods.
59///
60/// # Defaults
61///
62/// | Parameter | Default |
63/// |-----------|---------|
64/// | `max_retries` | (required) |
65/// | `initial_backoff` | 200ms |
66/// | `max_backoff` | 30s |
67/// | `multiplier` | 2.0 |
68///
69/// # Examples
70///
71/// ```
72/// use std::time::Duration;
73/// use ironflow_core::retry::RetryPolicy;
74///
75/// let policy = RetryPolicy::new(3)
76///     .backoff(Duration::from_millis(100))
77///     .max_backoff(Duration::from_secs(10))
78///     .multiplier(3.0);
79/// ```
80#[derive(Debug, Clone)]
81pub struct RetryPolicy {
82    pub(crate) max_retries: u32,
83    pub(crate) initial_backoff: Duration,
84    pub(crate) max_backoff: Duration,
85    pub(crate) multiplier: f64,
86}
87
88impl RetryPolicy {
89    /// Create a new retry policy with the given maximum number of retries.
90    ///
91    /// The initial attempt is not counted as a retry, so `max_retries(3)` means
92    /// up to 4 total attempts (1 initial + 3 retries).
93    ///
94    /// # Panics
95    ///
96    /// Panics if `max_retries` is `0`.
97    ///
98    /// # Examples
99    ///
100    /// ```
101    /// use ironflow_core::retry::RetryPolicy;
102    ///
103    /// let policy = RetryPolicy::new(3);
104    /// assert_eq!(policy.max_retries(), 3);
105    /// ```
106    pub fn new(max_retries: u32) -> Self {
107        assert!(max_retries > 0, "max_retries must be greater than 0");
108        Self {
109            max_retries,
110            initial_backoff: DEFAULT_INITIAL_BACKOFF,
111            max_backoff: DEFAULT_MAX_BACKOFF,
112            multiplier: DEFAULT_MULTIPLIER,
113        }
114    }
115
116    /// Set the initial backoff duration before the first retry.
117    ///
118    /// Subsequent retries multiply this duration by the [`multiplier`](RetryPolicy::multiplier).
119    ///
120    /// # Panics
121    ///
122    /// Panics if `duration` is zero.
123    pub fn backoff(mut self, duration: Duration) -> Self {
124        assert!(!duration.is_zero(), "initial backoff must not be zero");
125        self.initial_backoff = duration;
126        self
127    }
128
129    /// Set the maximum backoff duration between retries.
130    ///
131    /// Even after many retries with exponential growth, the delay will never
132    /// exceed this value.
133    ///
134    /// # Panics
135    ///
136    /// Panics if `duration` is zero.
137    pub fn max_backoff(mut self, duration: Duration) -> Self {
138        assert!(!duration.is_zero(), "max backoff must not be zero");
139        self.max_backoff = duration;
140        self
141    }
142
143    /// Set the backoff multiplier applied after each retry.
144    ///
145    /// For example, with `multiplier(2.0)` and `backoff(200ms)`:
146    /// - Retry 1: 200ms
147    /// - Retry 2: 400ms
148    /// - Retry 3: 800ms
149    ///
150    /// # Panics
151    ///
152    /// Panics if `multiplier` is less than `1.0`, NaN, or infinity.
153    pub fn multiplier(mut self, multiplier: f64) -> Self {
154        assert!(
155            multiplier >= 1.0 && multiplier.is_finite(),
156            "multiplier must be >= 1.0 and finite, got {multiplier}"
157        );
158        self.multiplier = multiplier;
159        self
160    }
161
162    /// Return the maximum number of retries.
163    pub fn max_retries(&self) -> u32 {
164        self.max_retries
165    }
166
167    /// Compute the backoff duration for the given retry attempt (0-indexed).
168    ///
169    /// Returns the delay capped at [`max_backoff`](RetryPolicy::max_backoff).
170    pub(crate) fn delay_for_attempt(&self, attempt: u32) -> Duration {
171        let delay = self.initial_backoff.as_secs_f64() * self.multiplier.powi(attempt as i32);
172        let capped = delay.min(self.max_backoff.as_secs_f64());
173        Duration::from_secs_f64(capped)
174    }
175}
176
177/// Returns `true` if the given [`OperationError`] is transient and should be
178/// retried.
179///
180/// # Retryable errors
181///
182/// - [`OperationError::Http`] with no status (transport error) or status 5xx / 429
183/// - [`OperationError::Agent`] wrapping [`AgentError::ProcessFailed`] or [`AgentError::Timeout`]
184/// - [`OperationError::Timeout`] (operation-level timeout)
185///
186/// # Non-retryable errors
187///
188/// - [`OperationError::Http`] with 4xx status (except 429)
189/// - [`OperationError::Agent`] wrapping [`AgentError::PromptTooLarge`]
190/// - [`OperationError::Shell`]
191/// - [`OperationError::Deserialize`]
192pub fn is_retryable(error: &OperationError) -> bool {
193    match error {
194        OperationError::Http { status, .. } => match status {
195            None => true,
196            Some(code) => *code >= 500 || *code == 429,
197        },
198        OperationError::Agent(agent_err) => match agent_err {
199            AgentError::ProcessFailed { .. }
200            | AgentError::Timeout { .. }
201            | AgentError::SchemaValidation { .. }
202            | AgentError::RateLimited { .. } => true,
203            AgentError::HttpProvider { status_code, .. } => {
204                *status_code == 0 || *status_code >= 500
205            }
206            AgentError::PromptTooLarge { .. } => false,
207        },
208        OperationError::Timeout { .. } => true,
209        OperationError::Shell { .. } | OperationError::Deserialize { .. } => false,
210    }
211}
212
213/// Returns `true` if the given HTTP status code indicates a transient server
214/// error that should be retried (5xx or 429 Too Many Requests).
215pub(crate) fn is_retryable_status(status: u16) -> bool {
216    status >= 500 || status == 429
217}
218
219#[cfg(test)]
220mod tests {
221    use super::*;
222    use std::time::Duration;
223
224    // --- RetryPolicy builder ---
225
226    #[test]
227    fn new_creates_policy_with_defaults() {
228        let policy = RetryPolicy::new(3);
229        assert_eq!(policy.max_retries, 3);
230        assert_eq!(policy.initial_backoff, DEFAULT_INITIAL_BACKOFF);
231        assert_eq!(policy.max_backoff, DEFAULT_MAX_BACKOFF);
232        assert!((policy.multiplier - DEFAULT_MULTIPLIER).abs() < f64::EPSILON);
233    }
234
235    #[test]
236    #[should_panic(expected = "max_retries must be greater than 0")]
237    fn new_zero_retries_panics() {
238        let _ = RetryPolicy::new(0);
239    }
240
241    #[test]
242    fn backoff_sets_initial_backoff() {
243        let policy = RetryPolicy::new(1).backoff(Duration::from_secs(1));
244        assert_eq!(policy.initial_backoff, Duration::from_secs(1));
245    }
246
247    #[test]
248    #[should_panic(expected = "initial backoff must not be zero")]
249    fn backoff_zero_panics() {
250        let _ = RetryPolicy::new(1).backoff(Duration::ZERO);
251    }
252
253    #[test]
254    fn max_backoff_sets_cap() {
255        let policy = RetryPolicy::new(1).max_backoff(Duration::from_secs(60));
256        assert_eq!(policy.max_backoff, Duration::from_secs(60));
257    }
258
259    #[test]
260    #[should_panic(expected = "max backoff must not be zero")]
261    fn max_backoff_zero_panics() {
262        let _ = RetryPolicy::new(1).max_backoff(Duration::ZERO);
263    }
264
265    #[test]
266    fn multiplier_sets_value() {
267        let policy = RetryPolicy::new(1).multiplier(3.0);
268        assert!((policy.multiplier - 3.0).abs() < f64::EPSILON);
269    }
270
271    #[test]
272    #[should_panic(expected = "multiplier must be >= 1.0")]
273    fn multiplier_below_one_panics() {
274        let _ = RetryPolicy::new(1).multiplier(0.5);
275    }
276
277    #[test]
278    #[should_panic(expected = "multiplier must be >= 1.0 and finite")]
279    fn multiplier_nan_panics() {
280        let _ = RetryPolicy::new(1).multiplier(f64::NAN);
281    }
282
283    #[test]
284    #[should_panic(expected = "multiplier must be >= 1.0 and finite")]
285    fn multiplier_infinity_panics() {
286        let _ = RetryPolicy::new(1).multiplier(f64::INFINITY);
287    }
288
289    #[test]
290    fn max_retries_accessor() {
291        assert_eq!(RetryPolicy::new(5).max_retries(), 5);
292    }
293
294    // --- delay_for_attempt ---
295
296    #[test]
297    fn delay_for_attempt_zero_is_initial_backoff() {
298        let policy = RetryPolicy::new(3).backoff(Duration::from_millis(100));
299        let delay = policy.delay_for_attempt(0);
300        assert_eq!(delay, Duration::from_millis(100));
301    }
302
303    #[test]
304    fn delay_for_attempt_grows_exponentially() {
305        let policy = RetryPolicy::new(5)
306            .backoff(Duration::from_millis(100))
307            .multiplier(2.0);
308
309        assert_eq!(policy.delay_for_attempt(0), Duration::from_millis(100));
310        assert_eq!(policy.delay_for_attempt(1), Duration::from_millis(200));
311        assert_eq!(policy.delay_for_attempt(2), Duration::from_millis(400));
312        assert_eq!(policy.delay_for_attempt(3), Duration::from_millis(800));
313    }
314
315    #[test]
316    fn delay_for_attempt_capped_at_max_backoff() {
317        let policy = RetryPolicy::new(10)
318            .backoff(Duration::from_secs(1))
319            .max_backoff(Duration::from_secs(5))
320            .multiplier(10.0);
321
322        // attempt 0: 1s, attempt 1: 10s (capped to 5s)
323        assert_eq!(policy.delay_for_attempt(0), Duration::from_secs(1));
324        assert_eq!(policy.delay_for_attempt(1), Duration::from_secs(5));
325        assert_eq!(policy.delay_for_attempt(2), Duration::from_secs(5));
326    }
327
328    #[test]
329    fn delay_for_attempt_with_multiplier_one_is_constant() {
330        let policy = RetryPolicy::new(3)
331            .backoff(Duration::from_millis(500))
332            .multiplier(1.0);
333
334        assert_eq!(policy.delay_for_attempt(0), Duration::from_millis(500));
335        assert_eq!(policy.delay_for_attempt(1), Duration::from_millis(500));
336        assert_eq!(policy.delay_for_attempt(2), Duration::from_millis(500));
337    }
338
339    // --- is_retryable ---
340
341    #[test]
342    fn http_transport_error_is_retryable() {
343        let err = OperationError::Http {
344            status: None,
345            message: "connection refused".to_string(),
346        };
347        assert!(is_retryable(&err));
348    }
349
350    #[test]
351    fn http_500_is_retryable() {
352        let err = OperationError::Http {
353            status: Some(500),
354            message: "internal server error".to_string(),
355        };
356        assert!(is_retryable(&err));
357    }
358
359    #[test]
360    fn http_502_is_retryable() {
361        let err = OperationError::Http {
362            status: Some(502),
363            message: "bad gateway".to_string(),
364        };
365        assert!(is_retryable(&err));
366    }
367
368    #[test]
369    fn http_503_is_retryable() {
370        let err = OperationError::Http {
371            status: Some(503),
372            message: "service unavailable".to_string(),
373        };
374        assert!(is_retryable(&err));
375    }
376
377    #[test]
378    fn http_429_is_retryable() {
379        let err = OperationError::Http {
380            status: Some(429),
381            message: "too many requests".to_string(),
382        };
383        assert!(is_retryable(&err));
384    }
385
386    #[test]
387    fn http_400_is_not_retryable() {
388        let err = OperationError::Http {
389            status: Some(400),
390            message: "bad request".to_string(),
391        };
392        assert!(!is_retryable(&err));
393    }
394
395    #[test]
396    fn http_404_is_not_retryable() {
397        let err = OperationError::Http {
398            status: Some(404),
399            message: "not found".to_string(),
400        };
401        assert!(!is_retryable(&err));
402    }
403
404    #[test]
405    fn agent_process_failed_is_retryable() {
406        let err = OperationError::Agent(AgentError::ProcessFailed {
407            exit_code: 1,
408            stderr: "crash".to_string(),
409        });
410        assert!(is_retryable(&err));
411    }
412
413    #[test]
414    fn agent_timeout_is_retryable() {
415        let err = OperationError::Agent(AgentError::Timeout {
416            limit: Duration::from_secs(60),
417        });
418        assert!(is_retryable(&err));
419    }
420
421    #[test]
422    fn agent_prompt_too_large_is_not_retryable() {
423        let err = OperationError::Agent(AgentError::PromptTooLarge {
424            chars: 1_000_000,
425            estimated_tokens: 250_000,
426            model_limit: 200_000,
427        });
428        assert!(!is_retryable(&err));
429    }
430
431    #[test]
432    fn agent_schema_validation_is_retryable() {
433        let err = OperationError::Agent(AgentError::SchemaValidation {
434            expected: "object".to_string(),
435            got: "string".to_string(),
436            debug_messages: Vec::new(),
437            partial_usage: Box::default(),
438            raw_response: None,
439        });
440        assert!(is_retryable(&err));
441    }
442
443    #[test]
444    fn operation_timeout_is_retryable() {
445        let err = OperationError::Timeout {
446            step: "fetch".to_string(),
447            limit: Duration::from_secs(30),
448        };
449        assert!(is_retryable(&err));
450    }
451
452    #[test]
453    fn shell_error_is_not_retryable() {
454        let err = OperationError::Shell {
455            exit_code: 1,
456            stderr: "fail".to_string(),
457        };
458        assert!(!is_retryable(&err));
459    }
460
461    #[test]
462    fn deserialize_error_is_not_retryable() {
463        let err = OperationError::Deserialize {
464            target_type: "MyStruct".to_string(),
465            reason: "missing field".to_string(),
466        };
467        assert!(!is_retryable(&err));
468    }
469
470    // --- is_retryable_status ---
471
472    #[test]
473    fn retryable_status_codes() {
474        assert!(is_retryable_status(500));
475        assert!(is_retryable_status(502));
476        assert!(is_retryable_status(503));
477        assert!(is_retryable_status(504));
478        assert!(is_retryable_status(429));
479    }
480
481    #[test]
482    fn non_retryable_status_codes() {
483        assert!(!is_retryable_status(200));
484        assert!(!is_retryable_status(201));
485        assert!(!is_retryable_status(301));
486        assert!(!is_retryable_status(400));
487        assert!(!is_retryable_status(401));
488        assert!(!is_retryable_status(403));
489        assert!(!is_retryable_status(404));
490        assert!(!is_retryable_status(422));
491        assert!(!is_retryable_status(428));
492    }
493
494    // --- builder chaining ---
495
496    #[test]
497    fn builder_chain_all_methods() {
498        let policy = RetryPolicy::new(5)
499            .backoff(Duration::from_millis(100))
500            .max_backoff(Duration::from_secs(10))
501            .multiplier(3.0);
502
503        assert_eq!(policy.max_retries, 5);
504        assert_eq!(policy.initial_backoff, Duration::from_millis(100));
505        assert_eq!(policy.max_backoff, Duration::from_secs(10));
506        assert!((policy.multiplier - 3.0).abs() < f64::EPSILON);
507    }
508
509    #[test]
510    fn clone_produces_independent_copy() {
511        let policy = RetryPolicy::new(3).backoff(Duration::from_millis(100));
512        let cloned = policy.clone();
513        assert_eq!(policy.max_retries, cloned.max_retries);
514        assert_eq!(policy.initial_backoff, cloned.initial_backoff);
515    }
516
517    #[test]
518    fn debug_does_not_panic() {
519        let policy = RetryPolicy::new(1);
520        let debug = format!("{:?}", policy);
521        assert!(debug.contains("RetryPolicy"));
522    }
523}
ironflow_core/retry.rs

ironflow_core/
retry.rs
