Skip to main content

ftui_runtime/
retry.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Retry policies and timeout-enforced task helpers.
3//!
4//! Provides [`RetryPolicy`] for configurable retry-with-backoff and
5//! [`task_with_timeout`] / [`task_with_retry`] constructors that wrap
6//! [`Cmd::Task`](crate::Cmd) with deterministic lifecycle guarantees.
7//!
8//! # Migration rationale
9//!
10//! Source frameworks often have retry/timeout baked into effect middleware.
11//! These helpers give the migration code emitter explicit, testable primitives
12//! to target instead of ad-hoc retry loops.
13//!
14//! # Determinism
15//!
16//! Backoff delays use fixed formulas (no jitter/randomness) so that
17//! replay-based determinism tests can reproduce exact timing sequences.
18//!
19//! # Example
20//!
21//! ```
22//! use ftui_runtime::retry::{RetryPolicy, BackoffStrategy};
23//! use std::time::Duration;
24//!
25//! let policy = RetryPolicy::new(3, BackoffStrategy::Exponential {
26//!     base_ms: 100,
27//!     max_ms: 5000,
28//! });
29//!
30//! assert_eq!(policy.delay(0), Duration::from_millis(100));
31//! assert_eq!(policy.delay(1), Duration::from_millis(200));
32//! assert_eq!(policy.delay(2), Duration::from_millis(400));
33//! ```
34
35#![forbid(unsafe_code)]
36
37use std::sync::mpsc;
38
39use crate::cancellation::{CancellationSource, CancellationToken};
40use crate::program::{Cmd, TaskSpec};
41use web_time::Duration;
42
43const TASK_THREAD_JOIN_TIMEOUT: Duration = Duration::from_millis(250);
44const TASK_THREAD_JOIN_POLL: Duration = Duration::from_millis(5);
45
46fn duration_from_millis_saturating(millis: u128) -> Duration {
47    if millis >= Duration::MAX.as_millis() {
48        Duration::MAX
49    } else {
50        let seconds = millis / 1_000;
51        let subsecond_millis = millis % 1_000;
52        let Ok(seconds) = u64::try_from(seconds) else {
53            return Duration::MAX;
54        };
55        let Ok(nanos) = u32::try_from(subsecond_millis.saturating_mul(1_000_000)) else {
56            return Duration::MAX;
57        };
58        Duration::new(seconds, nanos)
59    }
60}
61
62fn add_millis_saturating(total: &mut u128, millis: u128) {
63    *total = total.saturating_add(millis).min(Duration::MAX.as_millis());
64}
65
66fn join_task_thread(handle: std::thread::JoinHandle<()>) {
67    let _ = handle.join();
68}
69
70/// Run `f` on a worker thread; `rx` delivers its result or its panic.
71///
72/// The worker is where the caller's code runs, so it gets what the executor
73/// gives a plain task: the panic is caught, with terminal cleanup hooks
74/// suppressed. The executor's suppression is per thread and does not reach
75/// this one. A panicking worker used to fire the terminal teardown mid-run,
76/// then read as a timeout, because its dropped sender looked like one to
77/// `recv_timeout`.
78fn spawn_worker<T, F>(
79    f: F,
80) -> (
81    std::thread::JoinHandle<()>,
82    mpsc::Receiver<std::thread::Result<T>>,
83)
84where
85    T: Send + 'static,
86    F: FnOnce() -> T + Send + 'static,
87{
88    let (tx, rx) = mpsc::channel();
89    let handle = std::thread::spawn(move || {
90        let result = ftui_core::with_panic_cleanup_suppressed(|| {
91            std::panic::catch_unwind(std::panic::AssertUnwindSafe(f))
92        });
93        let _ = tx.send(result);
94    });
95    (handle, rx)
96}
97
98fn join_task_thread_bounded(handle: std::thread::JoinHandle<()>, task_name: &'static str) {
99    let start = web_time::Instant::now();
100    while !handle.is_finished() {
101        if start.elapsed() >= TASK_THREAD_JOIN_TIMEOUT {
102            tracing::warn!(
103                target: crate::telemetry_schema::TARGET_EFFECT,
104                task = task_name,
105                timeout_ms = TASK_THREAD_JOIN_TIMEOUT.as_millis() as u64,
106                "Timed-out worker thread did not exit within the cancellation join timeout; detaching"
107            );
108            return;
109        }
110        std::thread::sleep(TASK_THREAD_JOIN_POLL);
111    }
112    join_task_thread(handle);
113}
114
115/// Backoff strategy for retry delays.
116#[derive(Debug, Clone, PartialEq)]
117#[cfg_attr(
118    feature = "state-persistence",
119    derive(serde::Serialize, serde::Deserialize)
120)]
121pub enum BackoffStrategy {
122    /// Fixed delay between retries.
123    Fixed {
124        /// Delay in milliseconds.
125        delay_ms: u64,
126    },
127    /// Exponential backoff: `base_ms * 2^attempt`, capped at `max_ms`.
128    Exponential {
129        /// Base delay in milliseconds.
130        base_ms: u64,
131        /// Maximum delay cap in milliseconds.
132        max_ms: u64,
133    },
134    /// Linear backoff: `base_ms * (attempt + 1)`, capped at `max_ms`.
135    Linear {
136        /// Base delay in milliseconds.
137        base_ms: u64,
138        /// Maximum delay cap in milliseconds.
139        max_ms: u64,
140    },
141}
142
143/// A retry policy with configurable attempts and backoff.
144#[derive(Debug, Clone, PartialEq)]
145#[cfg_attr(
146    feature = "state-persistence",
147    derive(serde::Serialize, serde::Deserialize)
148)]
149pub struct RetryPolicy {
150    /// Maximum number of retry attempts (0 = no retries, just the initial attempt).
151    pub max_retries: u32,
152    /// Backoff strategy between retries.
153    pub backoff: BackoffStrategy,
154}
155
156impl RetryPolicy {
157    /// Create a new retry policy.
158    pub fn new(max_retries: u32, backoff: BackoffStrategy) -> Self {
159        Self {
160            max_retries,
161            backoff,
162        }
163    }
164
165    /// No retries — execute once.
166    pub fn no_retry() -> Self {
167        Self {
168            max_retries: 0,
169            backoff: BackoffStrategy::Fixed { delay_ms: 0 },
170        }
171    }
172
173    /// Compute the delay before the given attempt (0-indexed).
174    pub fn delay(&self, attempt: u32) -> Duration {
175        match &self.backoff {
176            BackoffStrategy::Fixed { delay_ms } => Duration::from_millis(*delay_ms),
177            BackoffStrategy::Exponential { base_ms, max_ms } => {
178                let multiplier = 1u64.checked_shl(attempt).unwrap_or(u64::MAX);
179                let delay = base_ms.saturating_mul(multiplier);
180                Duration::from_millis(delay.min(*max_ms))
181            }
182            BackoffStrategy::Linear { base_ms, max_ms } => {
183                let delay = base_ms.saturating_mul(u64::from(attempt) + 1);
184                Duration::from_millis(delay.min(*max_ms))
185            }
186        }
187    }
188
189    /// Total maximum delay across all retries (for timeout budgeting).
190    pub fn total_max_delay(&self) -> Duration {
191        let retry_count = u128::from(self.max_retries);
192        let max_duration_millis = Duration::MAX.as_millis();
193        match &self.backoff {
194            BackoffStrategy::Fixed { delay_ms } => {
195                duration_from_millis_saturating(u128::from(*delay_ms).saturating_mul(retry_count))
196            }
197            BackoffStrategy::Linear { base_ms, max_ms } => {
198                if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
199                    return Duration::ZERO;
200                }
201
202                let uncapped_terms = retry_count.min(u128::from(*max_ms / *base_ms));
203                let arithmetic_sum =
204                    uncapped_terms.saturating_mul(uncapped_terms.saturating_add(1)) / 2;
205                let mut total_millis = u128::from(*base_ms)
206                    .saturating_mul(arithmetic_sum)
207                    .min(max_duration_millis);
208
209                let capped_terms = retry_count.saturating_sub(uncapped_terms);
210                add_millis_saturating(
211                    &mut total_millis,
212                    u128::from(*max_ms).saturating_mul(capped_terms),
213                );
214                duration_from_millis_saturating(total_millis)
215            }
216            BackoffStrategy::Exponential { base_ms, max_ms } => {
217                if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
218                    return Duration::ZERO;
219                }
220
221                let mut total_millis = 0_u128;
222                let mut attempt = 0_u32;
223                while attempt < self.max_retries {
224                    let delay_millis = match 1_u64.checked_shl(attempt) {
225                        Some(multiplier) => base_ms.saturating_mul(multiplier).min(*max_ms),
226                        None => *max_ms,
227                    };
228                    add_millis_saturating(&mut total_millis, u128::from(delay_millis));
229                    attempt = attempt.saturating_add(1);
230
231                    if delay_millis == *max_ms {
232                        let remaining = u128::from(self.max_retries.saturating_sub(attempt));
233                        add_millis_saturating(
234                            &mut total_millis,
235                            u128::from(*max_ms).saturating_mul(remaining),
236                        );
237                        break;
238                    }
239                }
240                duration_from_millis_saturating(total_millis)
241            }
242        }
243    }
244}
245
246/// Create a [`Cmd::Task`] that enforces a cooperative timeout.
247///
248/// The worker closure receives a [`CancellationToken`] and must honor it for
249/// timely timeout teardown. On timeout, the runtime requests cancellation and
250/// returns `on_timeout`; any late worker result is discarded.
251pub fn task_with_timeout<M, F>(timeout: Duration, f: F, on_timeout: M) -> Cmd<M>
252where
253    M: Send + 'static,
254    F: FnOnce(CancellationToken) -> M + Send + 'static,
255{
256    Cmd::task(move || {
257        let source = CancellationSource::new();
258        let token = source.token();
259        let (handle, rx) = spawn_worker(move || f(token));
260        match rx.recv_timeout(timeout) {
261            Ok(result) => {
262                join_task_thread(handle);
263                // A panic is the task's, as it would be run inline.
264                result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
265            }
266            Err(_) => {
267                source.cancel();
268                join_task_thread_bounded(handle, "task_with_timeout");
269                on_timeout
270            }
271        }
272    })
273}
274
275/// Create a [`Cmd::Task`] with a named spec and cooperative timeout.
276pub fn task_with_timeout_named<M, F>(
277    name: impl Into<String>,
278    timeout: Duration,
279    f: F,
280    on_timeout: M,
281) -> Cmd<M>
282where
283    M: Send + 'static,
284    F: FnOnce(CancellationToken) -> M + Send + 'static,
285{
286    Cmd::task_with_spec(TaskSpec::default().with_name(name), move || {
287        let source = CancellationSource::new();
288        let token = source.token();
289        let (handle, rx) = spawn_worker(move || f(token));
290        match rx.recv_timeout(timeout) {
291            Ok(result) => {
292                join_task_thread(handle);
293                result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
294            }
295            Err(_) => {
296                source.cancel();
297                join_task_thread_bounded(handle, "task_with_timeout_named");
298                on_timeout
299            }
300        }
301    })
302}
303
304/// Create a [`Cmd::Task`] that retries on failure with the given policy.
305///
306/// The `f` closure returns `Result<M, String>`. On `Ok`, the message is
307/// returned immediately. On `Err`, the task retries according to the policy,
308/// sleeping between attempts. After all retries are exhausted, `on_exhaust`
309/// is called with the last error to produce a fallback message.
310pub fn task_with_retry<M, F>(policy: RetryPolicy, f: F, on_exhaust: fn(String) -> M) -> Cmd<M>
311where
312    M: Send + 'static,
313    F: Fn() -> Result<M, String> + Send + 'static,
314{
315    Cmd::task(move || {
316        let mut last_err = String::new();
317        for attempt in 0..=policy.max_retries {
318            match f() {
319                Ok(msg) => return msg,
320                Err(e) => {
321                    last_err = e;
322                    if attempt < policy.max_retries {
323                        std::thread::sleep(policy.delay(attempt));
324                    }
325                }
326            }
327        }
328        on_exhaust(last_err)
329    })
330}
331
332/// Create a [`Cmd::Task`] with both retry and timeout.
333///
334/// Each individual attempt is bounded by `per_attempt_timeout`. The worker
335/// receives a [`CancellationToken`] and must honor it for timely timeout
336/// teardown. The total number of attempts is governed by the retry policy.
337pub fn task_with_retry_and_timeout<M, F>(
338    policy: RetryPolicy,
339    per_attempt_timeout: Duration,
340    f: F,
341    on_exhaust: fn(String) -> M,
342) -> Cmd<M>
343where
344    M: Send + 'static,
345    F: Fn(CancellationToken) -> Result<M, String> + Send + Sync + 'static,
346{
347    Cmd::task(move || {
348        let f = std::sync::Arc::new(f);
349        let mut last_err = String::new();
350        for attempt in 0..=policy.max_retries {
351            let source = CancellationSource::new();
352            let token = source.token();
353            let f_clone = std::sync::Arc::clone(&f);
354            let (handle, rx) = spawn_worker(move || f_clone(token));
355            match rx.recv_timeout(per_attempt_timeout) {
356                Ok(Ok(Ok(msg))) => {
357                    join_task_thread(handle);
358                    return msg;
359                }
360                Ok(Ok(Err(e))) => {
361                    join_task_thread(handle);
362                    last_err = e;
363                }
364                // A panic is not a failed attempt to retry: it ends the task,
365                // as it does `task_with_retry`, which runs `f` inline.
366                Ok(Err(payload)) => {
367                    join_task_thread(handle);
368                    std::panic::resume_unwind(payload);
369                }
370                Err(_) => {
371                    source.cancel();
372                    join_task_thread_bounded(handle, "task_with_retry_and_timeout");
373                    last_err = "timeout".into();
374                }
375            }
376            if attempt < policy.max_retries {
377                std::thread::sleep(policy.delay(attempt));
378            }
379        }
380        on_exhaust(last_err)
381    })
382}
383
384#[cfg(test)]
385mod tests {
386    use super::*;
387
388    #[test]
389    fn fixed_backoff_constant_delay() {
390        let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
391        assert_eq!(policy.delay(0), Duration::from_millis(100));
392        assert_eq!(policy.delay(1), Duration::from_millis(100));
393        assert_eq!(policy.delay(2), Duration::from_millis(100));
394    }
395
396    #[test]
397    fn exponential_backoff_doubles() {
398        let policy = RetryPolicy::new(
399            5,
400            BackoffStrategy::Exponential {
401                base_ms: 100,
402                max_ms: 5000,
403            },
404        );
405        assert_eq!(policy.delay(0), Duration::from_millis(100));
406        assert_eq!(policy.delay(1), Duration::from_millis(200));
407        assert_eq!(policy.delay(2), Duration::from_millis(400));
408        assert_eq!(policy.delay(3), Duration::from_millis(800));
409    }
410
411    #[test]
412    fn exponential_backoff_caps_at_max() {
413        let policy = RetryPolicy::new(
414            5,
415            BackoffStrategy::Exponential {
416                base_ms: 1000,
417                max_ms: 3000,
418            },
419        );
420        assert_eq!(policy.delay(0), Duration::from_millis(1000));
421        assert_eq!(policy.delay(1), Duration::from_millis(2000));
422        assert_eq!(policy.delay(2), Duration::from_millis(3000)); // capped
423        assert_eq!(policy.delay(3), Duration::from_millis(3000)); // capped
424    }
425
426    #[test]
427    fn linear_backoff_increments() {
428        let policy = RetryPolicy::new(
429            4,
430            BackoffStrategy::Linear {
431                base_ms: 100,
432                max_ms: 500,
433            },
434        );
435        assert_eq!(policy.delay(0), Duration::from_millis(100));
436        assert_eq!(policy.delay(1), Duration::from_millis(200));
437        assert_eq!(policy.delay(2), Duration::from_millis(300));
438        assert_eq!(policy.delay(3), Duration::from_millis(400));
439        assert_eq!(policy.delay(4), Duration::from_millis(500)); // capped
440    }
441
442    #[test]
443    fn linear_backoff_caps_at_max() {
444        let policy = RetryPolicy::new(
445            4,
446            BackoffStrategy::Linear {
447                base_ms: 200,
448                max_ms: 500,
449            },
450        );
451        assert_eq!(policy.delay(2), Duration::from_millis(500)); // 200*3 = 600, capped at 500
452    }
453
454    #[test]
455    fn no_retry_policy() {
456        let policy = RetryPolicy::no_retry();
457        assert_eq!(policy.max_retries, 0);
458    }
459
460    #[test]
461    fn total_max_delay_fixed() {
462        let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
463        assert_eq!(policy.total_max_delay(), Duration::from_millis(300));
464    }
465
466    #[test]
467    fn total_max_delay_exponential() {
468        let policy = RetryPolicy::new(
469            3,
470            BackoffStrategy::Exponential {
471                base_ms: 100,
472                max_ms: 10000,
473            },
474        );
475        // Delays: 100 + 200 + 400 = 700
476        assert_eq!(policy.total_max_delay(), Duration::from_millis(700));
477    }
478
479    #[test]
480    fn total_max_delay_zero_retries() {
481        let policy = RetryPolicy::no_retry();
482        assert_eq!(policy.total_max_delay(), Duration::ZERO);
483    }
484
485    #[test]
486    fn total_max_delay_fixed_saturates_without_iterating() {
487        let policy = RetryPolicy::new(u32::MAX, BackoffStrategy::Fixed { delay_ms: u64::MAX });
488        assert_eq!(policy.total_max_delay(), Duration::MAX);
489    }
490
491    #[test]
492    fn total_max_delay_linear_handles_large_retry_counts() {
493        let policy = RetryPolicy::new(
494            u32::MAX,
495            BackoffStrategy::Linear {
496                base_ms: 1,
497                max_ms: 10,
498            },
499        );
500        assert_eq!(
501            policy.total_max_delay(),
502            Duration::from_millis(10_u64.saturating_mul(u64::from(u32::MAX)) - 45)
503        );
504    }
505
506    #[test]
507    fn total_max_delay_exponential_saturates_after_cap() {
508        let policy = RetryPolicy::new(
509            6,
510            BackoffStrategy::Exponential {
511                base_ms: 10,
512                max_ms: 35,
513            },
514        );
515        assert_eq!(
516            policy.total_max_delay(),
517            Duration::from_millis(10 + 20 + 35 * 4)
518        );
519    }
520
521    #[test]
522    fn total_max_delay_matches_delay_sequence_for_representative_policies() {
523        let policies = [
524            RetryPolicy::new(5, BackoffStrategy::Fixed { delay_ms: 7 }),
525            RetryPolicy::new(
526                6,
527                BackoffStrategy::Linear {
528                    base_ms: 3,
529                    max_ms: 10,
530                },
531            ),
532            RetryPolicy::new(
533                4,
534                BackoffStrategy::Linear {
535                    base_ms: 10,
536                    max_ms: 3,
537                },
538            ),
539            RetryPolicy::new(
540                6,
541                BackoffStrategy::Exponential {
542                    base_ms: 2,
543                    max_ms: 9,
544                },
545            ),
546        ];
547
548        for policy in policies {
549            let expected_millis = (0..policy.max_retries)
550                .map(|attempt| policy.delay(attempt).as_millis())
551                .sum::<u128>();
552            assert_eq!(policy.total_max_delay().as_millis(), expected_millis);
553        }
554    }
555
556    #[test]
557    fn exponential_backoff_overflow_saturates() {
558        let policy = RetryPolicy::new(
559            1,
560            BackoffStrategy::Exponential {
561                base_ms: u64::MAX / 2,
562                max_ms: u64::MAX,
563            },
564        );
565        // Should not panic on overflow
566        let _ = policy.delay(30);
567    }
568
569    #[test]
570    fn linear_backoff_overflow_saturates() {
571        let policy = RetryPolicy::new(
572            1,
573            BackoffStrategy::Linear {
574                base_ms: u64::MAX / 2,
575                max_ms: u64::MAX,
576            },
577        );
578        let _ = policy.delay(30);
579    }
580
581    #[test]
582    fn retry_policy_clone_eq() {
583        let policy = RetryPolicy::new(
584            3,
585            BackoffStrategy::Exponential {
586                base_ms: 100,
587                max_ms: 5000,
588            },
589        );
590        let cloned = policy.clone();
591        assert_eq!(policy, cloned);
592    }
593
594    #[test]
595    fn task_with_retry_succeeds_first_try() {
596        #[derive(Debug, PartialEq)]
597        enum Msg {
598            Ok(i32),
599            Err(String),
600        }
601
602        let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 1 });
603        let cmd = task_with_retry(policy, || Ok(Msg::Ok(42)), Msg::Err);
604
605        // Verify it produces a Task variant
606        assert_eq!(cmd.type_name(), "Task");
607    }
608
609    #[test]
610    fn task_with_timeout_produces_task() {
611        #[derive(Debug)]
612        #[allow(dead_code)]
613        enum Msg {
614            Result(i32),
615            Timeout,
616        }
617
618        let cmd = task_with_timeout(
619            Duration::from_secs(1),
620            |_token| Msg::Result(42),
621            Msg::Timeout,
622        );
623        assert_eq!(cmd.type_name(), "Task");
624    }
625
626    #[test]
627    fn task_with_timeout_requests_cancellation_on_timeout() {
628        use std::sync::Arc;
629        use std::sync::atomic::{AtomicBool, Ordering};
630
631        #[derive(Debug, PartialEq)]
632        enum Msg {
633            Finished,
634            Timeout,
635        }
636
637        let cancelled = Arc::new(AtomicBool::new(false));
638        let worker_exited = Arc::new(AtomicBool::new(false));
639        let cancelled_flag = Arc::clone(&cancelled);
640        let exited_flag = Arc::clone(&worker_exited);
641
642        let cmd = task_with_timeout(
643            Duration::from_millis(10),
644            move |token| {
645                cancelled_flag.store(token.wait_timeout(Duration::from_secs(1)), Ordering::SeqCst);
646                exited_flag.store(true, Ordering::SeqCst);
647                Msg::Finished
648            },
649            Msg::Timeout,
650        );
651
652        let result = match cmd {
653            Cmd::Task(_, task) => task(),
654            other => panic!("expected Task, got {other:?}"),
655        };
656
657        assert_eq!(result, Msg::Timeout);
658        std::thread::sleep(Duration::from_millis(50));
659        assert!(cancelled.load(Ordering::SeqCst));
660        assert!(worker_exited.load(Ordering::SeqCst));
661    }
662
663    #[cfg(not(panic = "abort"))]
664    #[test]
665    fn a_panicking_worker_is_the_tasks_panic_and_spares_the_terminal() {
666        use std::cell::Cell;
667        use std::sync::atomic::{AtomicUsize, Ordering};
668
669        // Only panics on a worker this test marked count, so panics in
670        // parallel tests cannot interfere.
671        thread_local! {
672            static MARKED_WORKER: Cell<bool> = const { Cell::new(false) };
673        }
674        static CLEANUPS: AtomicUsize = AtomicUsize::new(0);
675        fn count_cleanup() {
676            if MARKED_WORKER.with(Cell::get) {
677                CLEANUPS.fetch_add(1, Ordering::SeqCst);
678            }
679        }
680        ftui_core::session_teardown::install_chained_panic_hook(
681            "retry-worker-panic-test",
682            count_cleanup,
683            || false,
684        );
685
686        let cmd = task_with_timeout(
687            Duration::from_secs(5),
688            |_token| -> u8 {
689                MARKED_WORKER.with(|marked| marked.set(true));
690                panic!("worker bug")
691            },
692            0,
693        );
694        let Cmd::Task(_, task) = cmd else {
695            panic!("expected a Task");
696        };
697        let start = std::time::Instant::now();
698        let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(task))
699            .expect_err("the worker's panic should be the task's, not a timeout message");
700
701        assert_eq!(payload.downcast_ref::<&str>(), Some(&"worker bug"));
702        assert!(start.elapsed() < std::time::Duration::from_secs(5));
703        assert_eq!(
704            CLEANUPS.load(Ordering::SeqCst),
705            0,
706            "the worker's panic ran terminal cleanup"
707        );
708    }
709
710    #[cfg(not(panic = "abort"))]
711    #[test]
712    fn a_panic_is_not_a_failed_attempt_to_retry() {
713        use std::sync::Arc;
714        use std::sync::atomic::{AtomicUsize, Ordering};
715
716        let attempts = Arc::new(AtomicUsize::new(0));
717        let attempts_seen = Arc::clone(&attempts);
718        let cmd = task_with_retry_and_timeout(
719            RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 0 }),
720            Duration::from_secs(5),
721            move |_token| -> Result<u8, String> {
722                attempts_seen.fetch_add(1, Ordering::SeqCst);
723                panic!("worker bug")
724            },
725            |_err| 0,
726        );
727        let Cmd::Task(_, task) = cmd else {
728            panic!("expected a Task");
729        };
730
731        assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(task)).is_err());
732        assert_eq!(attempts.load(Ordering::SeqCst), 1, "a panic was retried");
733    }
734
735    #[test]
736    fn task_with_retry_and_timeout_cancels_each_timed_out_attempt() {
737        use std::sync::Arc;
738        use std::sync::atomic::{AtomicUsize, Ordering};
739
740        #[derive(Debug, PartialEq)]
741        enum Msg {
742            Exhausted(String),
743        }
744
745        fn on_exhaust(err: String) -> Msg {
746            Msg::Exhausted(err)
747        }
748
749        let attempts = Arc::new(AtomicUsize::new(0));
750        let cancelled = Arc::new(AtomicUsize::new(0));
751        let attempts_flag = Arc::clone(&attempts);
752        let cancelled_flag = Arc::clone(&cancelled);
753        let policy = RetryPolicy::new(1, BackoffStrategy::Fixed { delay_ms: 0 });
754
755        let cmd = task_with_retry_and_timeout(
756            policy,
757            Duration::from_millis(10),
758            move |token| {
759                attempts_flag.fetch_add(1, Ordering::SeqCst);
760                if token.wait_timeout(Duration::from_secs(1)) {
761                    cancelled_flag.fetch_add(1, Ordering::SeqCst);
762                }
763                Err("cancelled".to_owned())
764            },
765            on_exhaust,
766        );
767
768        let result = match cmd {
769            Cmd::Task(_, task) => task(),
770            other => panic!("expected Task, got {other:?}"),
771        };
772
773        assert_eq!(result, Msg::Exhausted("timeout".to_owned()));
774        std::thread::sleep(Duration::from_millis(50));
775        assert_eq!(attempts.load(Ordering::SeqCst), 2);
776        assert_eq!(cancelled.load(Ordering::SeqCst), 2);
777    }
778
779    #[test]
780    fn backoff_strategy_variants_debug() {
781        let fixed = BackoffStrategy::Fixed { delay_ms: 100 };
782        let exp = BackoffStrategy::Exponential {
783            base_ms: 100,
784            max_ms: 5000,
785        };
786        let linear = BackoffStrategy::Linear {
787            base_ms: 100,
788            max_ms: 500,
789        };
790        // Just verify Debug doesn't panic
791        let _ = format!("{fixed:?}");
792        let _ = format!("{exp:?}");
793        let _ = format!("{linear:?}");
794    }
795}