Skip to main content

ftui_runtime/
retry.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Retry policies and timeout-enforced task helpers.
3//!
4//! Provides [`RetryPolicy`] for configurable retry-with-backoff and
5//! [`task_with_timeout`] / [`task_with_retry`] constructors that wrap
6//! [`Cmd::Task`](crate::Cmd) with deterministic lifecycle guarantees.
7//!
8//! # Migration rationale
9//!
10//! Source frameworks often have retry/timeout baked into effect middleware.
11//! These helpers give the migration code emitter explicit, testable primitives
12//! to target instead of ad-hoc retry loops.
13//!
14//! # Determinism
15//!
16//! Backoff delays use fixed formulas (no jitter/randomness) so that
17//! replay-based determinism tests can reproduce exact timing sequences.
18//!
19//! # Example
20//!
21//! ```
22//! use ftui_runtime::retry::{RetryPolicy, BackoffStrategy};
23//! use std::time::Duration;
24//!
25//! let policy = RetryPolicy::new(3, BackoffStrategy::Exponential {
26//!     base_ms: 100,
27//!     max_ms: 5000,
28//! });
29//!
30//! assert_eq!(policy.delay(0), Duration::from_millis(100));
31//! assert_eq!(policy.delay(1), Duration::from_millis(200));
32//! assert_eq!(policy.delay(2), Duration::from_millis(400));
33//! ```
34
35#![forbid(unsafe_code)]
36
37use std::sync::mpsc;
38
39use crate::cancellation::{CancellationSource, CancellationToken};
40use crate::program::{Cmd, TaskSpec};
41use web_time::Duration;
42
43const TASK_THREAD_JOIN_TIMEOUT: Duration = Duration::from_millis(250);
44const TASK_THREAD_JOIN_POLL: Duration = Duration::from_millis(5);
45
46fn duration_from_millis_saturating(millis: u128) -> Duration {
47    if millis >= Duration::MAX.as_millis() {
48        Duration::MAX
49    } else {
50        let seconds = millis / 1_000;
51        let subsecond_millis = millis % 1_000;
52        let Ok(seconds) = u64::try_from(seconds) else {
53            return Duration::MAX;
54        };
55        let Ok(nanos) = u32::try_from(subsecond_millis.saturating_mul(1_000_000)) else {
56            return Duration::MAX;
57        };
58        Duration::new(seconds, nanos)
59    }
60}
61
62fn add_millis_saturating(total: &mut u128, millis: u128) {
63    *total = total.saturating_add(millis).min(Duration::MAX.as_millis());
64}
65
66fn join_task_thread(handle: std::thread::JoinHandle<()>) {
67    let _ = handle.join();
68}
69
70/// Run `f` on a worker thread; `rx` delivers its result or its panic.
71///
72/// The worker is where the caller's code runs, so it gets what the executor
73/// gives a plain task: the panic is caught, with terminal cleanup hooks
74/// suppressed. The executor's suppression is per thread and does not reach
75/// this one. A panicking worker used to fire the terminal teardown mid-run,
76/// then read as a timeout, because its dropped sender looked like one to
77/// `recv_timeout`.
78fn spawn_worker<T, F>(
79    f: F,
80) -> (
81    std::thread::JoinHandle<()>,
82    mpsc::Receiver<std::thread::Result<T>>,
83)
84where
85    T: Send + 'static,
86    F: FnOnce() -> T + Send + 'static,
87{
88    let (tx, rx) = mpsc::channel();
89    let handle = std::thread::spawn(move || {
90        let result = ftui_core::with_panic_cleanup_suppressed(|| {
91            std::panic::catch_unwind(std::panic::AssertUnwindSafe(f))
92        });
93        let _ = tx.send(result);
94    });
95    (handle, rx)
96}
97
98fn join_task_thread_bounded(handle: std::thread::JoinHandle<()>, task_name: &'static str) {
99    let start = web_time::Instant::now();
100    while !handle.is_finished() {
101        if start.elapsed() >= TASK_THREAD_JOIN_TIMEOUT {
102            tracing::warn!(
103                task = task_name,
104                timeout_ms = TASK_THREAD_JOIN_TIMEOUT.as_millis() as u64,
105                "Timed-out worker thread did not exit within the cancellation join timeout; detaching"
106            );
107            return;
108        }
109        std::thread::sleep(TASK_THREAD_JOIN_POLL);
110    }
111    join_task_thread(handle);
112}
113
114/// Backoff strategy for retry delays.
115#[derive(Debug, Clone, PartialEq)]
116#[cfg_attr(
117    feature = "state-persistence",
118    derive(serde::Serialize, serde::Deserialize)
119)]
120pub enum BackoffStrategy {
121    /// Fixed delay between retries.
122    Fixed {
123        /// Delay in milliseconds.
124        delay_ms: u64,
125    },
126    /// Exponential backoff: `base_ms * 2^attempt`, capped at `max_ms`.
127    Exponential {
128        /// Base delay in milliseconds.
129        base_ms: u64,
130        /// Maximum delay cap in milliseconds.
131        max_ms: u64,
132    },
133    /// Linear backoff: `base_ms * (attempt + 1)`, capped at `max_ms`.
134    Linear {
135        /// Base delay in milliseconds.
136        base_ms: u64,
137        /// Maximum delay cap in milliseconds.
138        max_ms: u64,
139    },
140}
141
142/// A retry policy with configurable attempts and backoff.
143#[derive(Debug, Clone, PartialEq)]
144#[cfg_attr(
145    feature = "state-persistence",
146    derive(serde::Serialize, serde::Deserialize)
147)]
148pub struct RetryPolicy {
149    /// Maximum number of retry attempts (0 = no retries, just the initial attempt).
150    pub max_retries: u32,
151    /// Backoff strategy between retries.
152    pub backoff: BackoffStrategy,
153}
154
155impl RetryPolicy {
156    /// Create a new retry policy.
157    pub fn new(max_retries: u32, backoff: BackoffStrategy) -> Self {
158        Self {
159            max_retries,
160            backoff,
161        }
162    }
163
164    /// No retries — execute once.
165    pub fn no_retry() -> Self {
166        Self {
167            max_retries: 0,
168            backoff: BackoffStrategy::Fixed { delay_ms: 0 },
169        }
170    }
171
172    /// Compute the delay before the given attempt (0-indexed).
173    pub fn delay(&self, attempt: u32) -> Duration {
174        match &self.backoff {
175            BackoffStrategy::Fixed { delay_ms } => Duration::from_millis(*delay_ms),
176            BackoffStrategy::Exponential { base_ms, max_ms } => {
177                let multiplier = 1u64.checked_shl(attempt).unwrap_or(u64::MAX);
178                let delay = base_ms.saturating_mul(multiplier);
179                Duration::from_millis(delay.min(*max_ms))
180            }
181            BackoffStrategy::Linear { base_ms, max_ms } => {
182                let delay = base_ms.saturating_mul(u64::from(attempt) + 1);
183                Duration::from_millis(delay.min(*max_ms))
184            }
185        }
186    }
187
188    /// Total maximum delay across all retries (for timeout budgeting).
189    pub fn total_max_delay(&self) -> Duration {
190        let retry_count = u128::from(self.max_retries);
191        let max_duration_millis = Duration::MAX.as_millis();
192        match &self.backoff {
193            BackoffStrategy::Fixed { delay_ms } => {
194                duration_from_millis_saturating(u128::from(*delay_ms).saturating_mul(retry_count))
195            }
196            BackoffStrategy::Linear { base_ms, max_ms } => {
197                if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
198                    return Duration::ZERO;
199                }
200
201                let uncapped_terms = retry_count.min(u128::from(*max_ms / *base_ms));
202                let arithmetic_sum =
203                    uncapped_terms.saturating_mul(uncapped_terms.saturating_add(1)) / 2;
204                let mut total_millis = u128::from(*base_ms)
205                    .saturating_mul(arithmetic_sum)
206                    .min(max_duration_millis);
207
208                let capped_terms = retry_count.saturating_sub(uncapped_terms);
209                add_millis_saturating(
210                    &mut total_millis,
211                    u128::from(*max_ms).saturating_mul(capped_terms),
212                );
213                duration_from_millis_saturating(total_millis)
214            }
215            BackoffStrategy::Exponential { base_ms, max_ms } => {
216                if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
217                    return Duration::ZERO;
218                }
219
220                let mut total_millis = 0_u128;
221                let mut attempt = 0_u32;
222                while attempt < self.max_retries {
223                    let delay_millis = match 1_u64.checked_shl(attempt) {
224                        Some(multiplier) => base_ms.saturating_mul(multiplier).min(*max_ms),
225                        None => *max_ms,
226                    };
227                    add_millis_saturating(&mut total_millis, u128::from(delay_millis));
228                    attempt = attempt.saturating_add(1);
229
230                    if delay_millis == *max_ms {
231                        let remaining = u128::from(self.max_retries.saturating_sub(attempt));
232                        add_millis_saturating(
233                            &mut total_millis,
234                            u128::from(*max_ms).saturating_mul(remaining),
235                        );
236                        break;
237                    }
238                }
239                duration_from_millis_saturating(total_millis)
240            }
241        }
242    }
243}
244
245/// Create a [`Cmd::Task`] that enforces a cooperative timeout.
246///
247/// The worker closure receives a [`CancellationToken`] and must honor it for
248/// timely timeout teardown. On timeout, the runtime requests cancellation and
249/// returns `on_timeout`; any late worker result is discarded.
250pub fn task_with_timeout<M, F>(timeout: Duration, f: F, on_timeout: M) -> Cmd<M>
251where
252    M: Send + 'static,
253    F: FnOnce(CancellationToken) -> M + Send + 'static,
254{
255    Cmd::task(move || {
256        let source = CancellationSource::new();
257        let token = source.token();
258        let (handle, rx) = spawn_worker(move || f(token));
259        match rx.recv_timeout(timeout) {
260            Ok(result) => {
261                join_task_thread(handle);
262                // A panic is the task's, as it would be run inline.
263                result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
264            }
265            Err(_) => {
266                source.cancel();
267                join_task_thread_bounded(handle, "task_with_timeout");
268                on_timeout
269            }
270        }
271    })
272}
273
274/// Create a [`Cmd::Task`] with a named spec and cooperative timeout.
275pub fn task_with_timeout_named<M, F>(
276    name: impl Into<String>,
277    timeout: Duration,
278    f: F,
279    on_timeout: M,
280) -> Cmd<M>
281where
282    M: Send + 'static,
283    F: FnOnce(CancellationToken) -> M + Send + 'static,
284{
285    Cmd::task_with_spec(TaskSpec::default().with_name(name), move || {
286        let source = CancellationSource::new();
287        let token = source.token();
288        let (handle, rx) = spawn_worker(move || f(token));
289        match rx.recv_timeout(timeout) {
290            Ok(result) => {
291                join_task_thread(handle);
292                result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
293            }
294            Err(_) => {
295                source.cancel();
296                join_task_thread_bounded(handle, "task_with_timeout_named");
297                on_timeout
298            }
299        }
300    })
301}
302
303/// Create a [`Cmd::Task`] that retries on failure with the given policy.
304///
305/// The `f` closure returns `Result<M, String>`. On `Ok`, the message is
306/// returned immediately. On `Err`, the task retries according to the policy,
307/// sleeping between attempts. After all retries are exhausted, `on_exhaust`
308/// is called with the last error to produce a fallback message.
309pub fn task_with_retry<M, F>(policy: RetryPolicy, f: F, on_exhaust: fn(String) -> M) -> Cmd<M>
310where
311    M: Send + 'static,
312    F: Fn() -> Result<M, String> + Send + 'static,
313{
314    Cmd::task(move || {
315        let mut last_err = String::new();
316        for attempt in 0..=policy.max_retries {
317            match f() {
318                Ok(msg) => return msg,
319                Err(e) => {
320                    last_err = e;
321                    if attempt < policy.max_retries {
322                        std::thread::sleep(policy.delay(attempt));
323                    }
324                }
325            }
326        }
327        on_exhaust(last_err)
328    })
329}
330
331/// Create a [`Cmd::Task`] with both retry and timeout.
332///
333/// Each individual attempt is bounded by `per_attempt_timeout`. The worker
334/// receives a [`CancellationToken`] and must honor it for timely timeout
335/// teardown. The total number of attempts is governed by the retry policy.
336pub fn task_with_retry_and_timeout<M, F>(
337    policy: RetryPolicy,
338    per_attempt_timeout: Duration,
339    f: F,
340    on_exhaust: fn(String) -> M,
341) -> Cmd<M>
342where
343    M: Send + 'static,
344    F: Fn(CancellationToken) -> Result<M, String> + Send + Sync + 'static,
345{
346    Cmd::task(move || {
347        let f = std::sync::Arc::new(f);
348        let mut last_err = String::new();
349        for attempt in 0..=policy.max_retries {
350            let source = CancellationSource::new();
351            let token = source.token();
352            let f_clone = std::sync::Arc::clone(&f);
353            let (handle, rx) = spawn_worker(move || f_clone(token));
354            match rx.recv_timeout(per_attempt_timeout) {
355                Ok(Ok(Ok(msg))) => {
356                    join_task_thread(handle);
357                    return msg;
358                }
359                Ok(Ok(Err(e))) => {
360                    join_task_thread(handle);
361                    last_err = e;
362                }
363                // A panic is not a failed attempt to retry: it ends the task,
364                // as it does `task_with_retry`, which runs `f` inline.
365                Ok(Err(payload)) => {
366                    join_task_thread(handle);
367                    std::panic::resume_unwind(payload);
368                }
369                Err(_) => {
370                    source.cancel();
371                    join_task_thread_bounded(handle, "task_with_retry_and_timeout");
372                    last_err = "timeout".into();
373                }
374            }
375            if attempt < policy.max_retries {
376                std::thread::sleep(policy.delay(attempt));
377            }
378        }
379        on_exhaust(last_err)
380    })
381}
382
383#[cfg(test)]
384mod tests {
385    use super::*;
386
387    #[test]
388    fn fixed_backoff_constant_delay() {
389        let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
390        assert_eq!(policy.delay(0), Duration::from_millis(100));
391        assert_eq!(policy.delay(1), Duration::from_millis(100));
392        assert_eq!(policy.delay(2), Duration::from_millis(100));
393    }
394
395    #[test]
396    fn exponential_backoff_doubles() {
397        let policy = RetryPolicy::new(
398            5,
399            BackoffStrategy::Exponential {
400                base_ms: 100,
401                max_ms: 5000,
402            },
403        );
404        assert_eq!(policy.delay(0), Duration::from_millis(100));
405        assert_eq!(policy.delay(1), Duration::from_millis(200));
406        assert_eq!(policy.delay(2), Duration::from_millis(400));
407        assert_eq!(policy.delay(3), Duration::from_millis(800));
408    }
409
410    #[test]
411    fn exponential_backoff_caps_at_max() {
412        let policy = RetryPolicy::new(
413            5,
414            BackoffStrategy::Exponential {
415                base_ms: 1000,
416                max_ms: 3000,
417            },
418        );
419        assert_eq!(policy.delay(0), Duration::from_millis(1000));
420        assert_eq!(policy.delay(1), Duration::from_millis(2000));
421        assert_eq!(policy.delay(2), Duration::from_millis(3000)); // capped
422        assert_eq!(policy.delay(3), Duration::from_millis(3000)); // capped
423    }
424
425    #[test]
426    fn linear_backoff_increments() {
427        let policy = RetryPolicy::new(
428            4,
429            BackoffStrategy::Linear {
430                base_ms: 100,
431                max_ms: 500,
432            },
433        );
434        assert_eq!(policy.delay(0), Duration::from_millis(100));
435        assert_eq!(policy.delay(1), Duration::from_millis(200));
436        assert_eq!(policy.delay(2), Duration::from_millis(300));
437        assert_eq!(policy.delay(3), Duration::from_millis(400));
438        assert_eq!(policy.delay(4), Duration::from_millis(500)); // capped
439    }
440
441    #[test]
442    fn linear_backoff_caps_at_max() {
443        let policy = RetryPolicy::new(
444            4,
445            BackoffStrategy::Linear {
446                base_ms: 200,
447                max_ms: 500,
448            },
449        );
450        assert_eq!(policy.delay(2), Duration::from_millis(500)); // 200*3 = 600, capped at 500
451    }
452
453    #[test]
454    fn no_retry_policy() {
455        let policy = RetryPolicy::no_retry();
456        assert_eq!(policy.max_retries, 0);
457    }
458
459    #[test]
460    fn total_max_delay_fixed() {
461        let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
462        assert_eq!(policy.total_max_delay(), Duration::from_millis(300));
463    }
464
465    #[test]
466    fn total_max_delay_exponential() {
467        let policy = RetryPolicy::new(
468            3,
469            BackoffStrategy::Exponential {
470                base_ms: 100,
471                max_ms: 10000,
472            },
473        );
474        // Delays: 100 + 200 + 400 = 700
475        assert_eq!(policy.total_max_delay(), Duration::from_millis(700));
476    }
477
478    #[test]
479    fn total_max_delay_zero_retries() {
480        let policy = RetryPolicy::no_retry();
481        assert_eq!(policy.total_max_delay(), Duration::ZERO);
482    }
483
484    #[test]
485    fn total_max_delay_fixed_saturates_without_iterating() {
486        let policy = RetryPolicy::new(u32::MAX, BackoffStrategy::Fixed { delay_ms: u64::MAX });
487        assert_eq!(policy.total_max_delay(), Duration::MAX);
488    }
489
490    #[test]
491    fn total_max_delay_linear_handles_large_retry_counts() {
492        let policy = RetryPolicy::new(
493            u32::MAX,
494            BackoffStrategy::Linear {
495                base_ms: 1,
496                max_ms: 10,
497            },
498        );
499        assert_eq!(
500            policy.total_max_delay(),
501            Duration::from_millis(10_u64.saturating_mul(u64::from(u32::MAX)) - 45)
502        );
503    }
504
505    #[test]
506    fn total_max_delay_exponential_saturates_after_cap() {
507        let policy = RetryPolicy::new(
508            6,
509            BackoffStrategy::Exponential {
510                base_ms: 10,
511                max_ms: 35,
512            },
513        );
514        assert_eq!(
515            policy.total_max_delay(),
516            Duration::from_millis(10 + 20 + 35 * 4)
517        );
518    }
519
520    #[test]
521    fn total_max_delay_matches_delay_sequence_for_representative_policies() {
522        let policies = [
523            RetryPolicy::new(5, BackoffStrategy::Fixed { delay_ms: 7 }),
524            RetryPolicy::new(
525                6,
526                BackoffStrategy::Linear {
527                    base_ms: 3,
528                    max_ms: 10,
529                },
530            ),
531            RetryPolicy::new(
532                4,
533                BackoffStrategy::Linear {
534                    base_ms: 10,
535                    max_ms: 3,
536                },
537            ),
538            RetryPolicy::new(
539                6,
540                BackoffStrategy::Exponential {
541                    base_ms: 2,
542                    max_ms: 9,
543                },
544            ),
545        ];
546
547        for policy in policies {
548            let expected_millis = (0..policy.max_retries)
549                .map(|attempt| policy.delay(attempt).as_millis())
550                .sum::<u128>();
551            assert_eq!(policy.total_max_delay().as_millis(), expected_millis);
552        }
553    }
554
555    #[test]
556    fn exponential_backoff_overflow_saturates() {
557        let policy = RetryPolicy::new(
558            1,
559            BackoffStrategy::Exponential {
560                base_ms: u64::MAX / 2,
561                max_ms: u64::MAX,
562            },
563        );
564        // Should not panic on overflow
565        let _ = policy.delay(30);
566    }
567
568    #[test]
569    fn linear_backoff_overflow_saturates() {
570        let policy = RetryPolicy::new(
571            1,
572            BackoffStrategy::Linear {
573                base_ms: u64::MAX / 2,
574                max_ms: u64::MAX,
575            },
576        );
577        let _ = policy.delay(30);
578    }
579
580    #[test]
581    fn retry_policy_clone_eq() {
582        let policy = RetryPolicy::new(
583            3,
584            BackoffStrategy::Exponential {
585                base_ms: 100,
586                max_ms: 5000,
587            },
588        );
589        let cloned = policy.clone();
590        assert_eq!(policy, cloned);
591    }
592
593    #[test]
594    fn task_with_retry_succeeds_first_try() {
595        #[derive(Debug, PartialEq)]
596        enum Msg {
597            Ok(i32),
598            Err(String),
599        }
600
601        let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 1 });
602        let cmd = task_with_retry(policy, || Ok(Msg::Ok(42)), Msg::Err);
603
604        // Verify it produces a Task variant
605        assert_eq!(cmd.type_name(), "Task");
606    }
607
608    #[test]
609    fn task_with_timeout_produces_task() {
610        #[derive(Debug)]
611        #[allow(dead_code)]
612        enum Msg {
613            Result(i32),
614            Timeout,
615        }
616
617        let cmd = task_with_timeout(
618            Duration::from_secs(1),
619            |_token| Msg::Result(42),
620            Msg::Timeout,
621        );
622        assert_eq!(cmd.type_name(), "Task");
623    }
624
625    #[test]
626    fn task_with_timeout_requests_cancellation_on_timeout() {
627        use std::sync::Arc;
628        use std::sync::atomic::{AtomicBool, Ordering};
629
630        #[derive(Debug, PartialEq)]
631        enum Msg {
632            Finished,
633            Timeout,
634        }
635
636        let cancelled = Arc::new(AtomicBool::new(false));
637        let worker_exited = Arc::new(AtomicBool::new(false));
638        let cancelled_flag = Arc::clone(&cancelled);
639        let exited_flag = Arc::clone(&worker_exited);
640
641        let cmd = task_with_timeout(
642            Duration::from_millis(10),
643            move |token| {
644                cancelled_flag.store(token.wait_timeout(Duration::from_secs(1)), Ordering::SeqCst);
645                exited_flag.store(true, Ordering::SeqCst);
646                Msg::Finished
647            },
648            Msg::Timeout,
649        );
650
651        let result = match cmd {
652            Cmd::Task(_, task) => task(),
653            other => panic!("expected Task, got {other:?}"),
654        };
655
656        assert_eq!(result, Msg::Timeout);
657        std::thread::sleep(Duration::from_millis(50));
658        assert!(cancelled.load(Ordering::SeqCst));
659        assert!(worker_exited.load(Ordering::SeqCst));
660    }
661
662    #[cfg(not(panic = "abort"))]
663    #[test]
664    fn a_panicking_worker_is_the_tasks_panic_and_spares_the_terminal() {
665        use std::cell::Cell;
666        use std::sync::atomic::{AtomicUsize, Ordering};
667
668        // Only panics on a worker this test marked count, so panics in
669        // parallel tests cannot interfere.
670        thread_local! {
671            static MARKED_WORKER: Cell<bool> = const { Cell::new(false) };
672        }
673        static CLEANUPS: AtomicUsize = AtomicUsize::new(0);
674        fn count_cleanup() {
675            if MARKED_WORKER.with(Cell::get) {
676                CLEANUPS.fetch_add(1, Ordering::SeqCst);
677            }
678        }
679        ftui_core::session_teardown::install_chained_panic_hook(
680            "retry-worker-panic-test",
681            count_cleanup,
682        );
683
684        let cmd = task_with_timeout(
685            Duration::from_secs(5),
686            |_token| -> u8 {
687                MARKED_WORKER.with(|marked| marked.set(true));
688                panic!("worker bug")
689            },
690            0,
691        );
692        let Cmd::Task(_, task) = cmd else {
693            panic!("expected a Task");
694        };
695        let start = std::time::Instant::now();
696        let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(task))
697            .expect_err("the worker's panic should be the task's, not a timeout message");
698
699        assert_eq!(payload.downcast_ref::<&str>(), Some(&"worker bug"));
700        assert!(start.elapsed() < std::time::Duration::from_secs(5));
701        assert_eq!(
702            CLEANUPS.load(Ordering::SeqCst),
703            0,
704            "the worker's panic ran terminal cleanup"
705        );
706    }
707
708    #[cfg(not(panic = "abort"))]
709    #[test]
710    fn a_panic_is_not_a_failed_attempt_to_retry() {
711        use std::sync::Arc;
712        use std::sync::atomic::{AtomicUsize, Ordering};
713
714        let attempts = Arc::new(AtomicUsize::new(0));
715        let attempts_seen = Arc::clone(&attempts);
716        let cmd = task_with_retry_and_timeout(
717            RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 0 }),
718            Duration::from_secs(5),
719            move |_token| -> Result<u8, String> {
720                attempts_seen.fetch_add(1, Ordering::SeqCst);
721                panic!("worker bug")
722            },
723            |_err| 0,
724        );
725        let Cmd::Task(_, task) = cmd else {
726            panic!("expected a Task");
727        };
728
729        assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(task)).is_err());
730        assert_eq!(attempts.load(Ordering::SeqCst), 1, "a panic was retried");
731    }
732
733    #[test]
734    fn task_with_retry_and_timeout_cancels_each_timed_out_attempt() {
735        use std::sync::Arc;
736        use std::sync::atomic::{AtomicUsize, Ordering};
737
738        #[derive(Debug, PartialEq)]
739        enum Msg {
740            Exhausted(String),
741        }
742
743        fn on_exhaust(err: String) -> Msg {
744            Msg::Exhausted(err)
745        }
746
747        let attempts = Arc::new(AtomicUsize::new(0));
748        let cancelled = Arc::new(AtomicUsize::new(0));
749        let attempts_flag = Arc::clone(&attempts);
750        let cancelled_flag = Arc::clone(&cancelled);
751        let policy = RetryPolicy::new(1, BackoffStrategy::Fixed { delay_ms: 0 });
752
753        let cmd = task_with_retry_and_timeout(
754            policy,
755            Duration::from_millis(10),
756            move |token| {
757                attempts_flag.fetch_add(1, Ordering::SeqCst);
758                if token.wait_timeout(Duration::from_secs(1)) {
759                    cancelled_flag.fetch_add(1, Ordering::SeqCst);
760                }
761                Err("cancelled".to_owned())
762            },
763            on_exhaust,
764        );
765
766        let result = match cmd {
767            Cmd::Task(_, task) => task(),
768            other => panic!("expected Task, got {other:?}"),
769        };
770
771        assert_eq!(result, Msg::Exhausted("timeout".to_owned()));
772        std::thread::sleep(Duration::from_millis(50));
773        assert_eq!(attempts.load(Ordering::SeqCst), 2);
774        assert_eq!(cancelled.load(Ordering::SeqCst), 2);
775    }
776
777    #[test]
778    fn backoff_strategy_variants_debug() {
779        let fixed = BackoffStrategy::Fixed { delay_ms: 100 };
780        let exp = BackoffStrategy::Exponential {
781            base_ms: 100,
782            max_ms: 5000,
783        };
784        let linear = BackoffStrategy::Linear {
785            base_ms: 100,
786            max_ms: 500,
787        };
788        // Just verify Debug doesn't panic
789        let _ = format!("{fixed:?}");
790        let _ = format!("{exp:?}");
791        let _ = format!("{linear:?}");
792    }
793}