1#![forbid(unsafe_code)]
36
37use std::sync::mpsc;
38
39use crate::cancellation::{CancellationSource, CancellationToken};
40use crate::program::{Cmd, TaskSpec};
41use web_time::Duration;
42
43const TASK_THREAD_JOIN_TIMEOUT: Duration = Duration::from_millis(250);
44const TASK_THREAD_JOIN_POLL: Duration = Duration::from_millis(5);
45
46fn duration_from_millis_saturating(millis: u128) -> Duration {
47 if millis >= Duration::MAX.as_millis() {
48 Duration::MAX
49 } else {
50 let seconds = millis / 1_000;
51 let subsecond_millis = millis % 1_000;
52 let Ok(seconds) = u64::try_from(seconds) else {
53 return Duration::MAX;
54 };
55 let Ok(nanos) = u32::try_from(subsecond_millis.saturating_mul(1_000_000)) else {
56 return Duration::MAX;
57 };
58 Duration::new(seconds, nanos)
59 }
60}
61
62fn add_millis_saturating(total: &mut u128, millis: u128) {
63 *total = total.saturating_add(millis).min(Duration::MAX.as_millis());
64}
65
66fn join_task_thread(handle: std::thread::JoinHandle<()>) {
67 let _ = handle.join();
68}
69
70fn spawn_worker<T, F>(
79 f: F,
80) -> (
81 std::thread::JoinHandle<()>,
82 mpsc::Receiver<std::thread::Result<T>>,
83)
84where
85 T: Send + 'static,
86 F: FnOnce() -> T + Send + 'static,
87{
88 let (tx, rx) = mpsc::channel();
89 let handle = std::thread::spawn(move || {
90 let result = ftui_core::with_panic_cleanup_suppressed(|| {
91 std::panic::catch_unwind(std::panic::AssertUnwindSafe(f))
92 });
93 let _ = tx.send(result);
94 });
95 (handle, rx)
96}
97
98fn join_task_thread_bounded(handle: std::thread::JoinHandle<()>, task_name: &'static str) {
99 let start = web_time::Instant::now();
100 while !handle.is_finished() {
101 if start.elapsed() >= TASK_THREAD_JOIN_TIMEOUT {
102 tracing::warn!(
103 task = task_name,
104 timeout_ms = TASK_THREAD_JOIN_TIMEOUT.as_millis() as u64,
105 "Timed-out worker thread did not exit within the cancellation join timeout; detaching"
106 );
107 return;
108 }
109 std::thread::sleep(TASK_THREAD_JOIN_POLL);
110 }
111 join_task_thread(handle);
112}
113
114#[derive(Debug, Clone, PartialEq)]
116#[cfg_attr(
117 feature = "state-persistence",
118 derive(serde::Serialize, serde::Deserialize)
119)]
120pub enum BackoffStrategy {
121 Fixed {
123 delay_ms: u64,
125 },
126 Exponential {
128 base_ms: u64,
130 max_ms: u64,
132 },
133 Linear {
135 base_ms: u64,
137 max_ms: u64,
139 },
140}
141
142#[derive(Debug, Clone, PartialEq)]
144#[cfg_attr(
145 feature = "state-persistence",
146 derive(serde::Serialize, serde::Deserialize)
147)]
148pub struct RetryPolicy {
149 pub max_retries: u32,
151 pub backoff: BackoffStrategy,
153}
154
155impl RetryPolicy {
156 pub fn new(max_retries: u32, backoff: BackoffStrategy) -> Self {
158 Self {
159 max_retries,
160 backoff,
161 }
162 }
163
164 pub fn no_retry() -> Self {
166 Self {
167 max_retries: 0,
168 backoff: BackoffStrategy::Fixed { delay_ms: 0 },
169 }
170 }
171
172 pub fn delay(&self, attempt: u32) -> Duration {
174 match &self.backoff {
175 BackoffStrategy::Fixed { delay_ms } => Duration::from_millis(*delay_ms),
176 BackoffStrategy::Exponential { base_ms, max_ms } => {
177 let multiplier = 1u64.checked_shl(attempt).unwrap_or(u64::MAX);
178 let delay = base_ms.saturating_mul(multiplier);
179 Duration::from_millis(delay.min(*max_ms))
180 }
181 BackoffStrategy::Linear { base_ms, max_ms } => {
182 let delay = base_ms.saturating_mul(u64::from(attempt) + 1);
183 Duration::from_millis(delay.min(*max_ms))
184 }
185 }
186 }
187
188 pub fn total_max_delay(&self) -> Duration {
190 let retry_count = u128::from(self.max_retries);
191 let max_duration_millis = Duration::MAX.as_millis();
192 match &self.backoff {
193 BackoffStrategy::Fixed { delay_ms } => {
194 duration_from_millis_saturating(u128::from(*delay_ms).saturating_mul(retry_count))
195 }
196 BackoffStrategy::Linear { base_ms, max_ms } => {
197 if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
198 return Duration::ZERO;
199 }
200
201 let uncapped_terms = retry_count.min(u128::from(*max_ms / *base_ms));
202 let arithmetic_sum =
203 uncapped_terms.saturating_mul(uncapped_terms.saturating_add(1)) / 2;
204 let mut total_millis = u128::from(*base_ms)
205 .saturating_mul(arithmetic_sum)
206 .min(max_duration_millis);
207
208 let capped_terms = retry_count.saturating_sub(uncapped_terms);
209 add_millis_saturating(
210 &mut total_millis,
211 u128::from(*max_ms).saturating_mul(capped_terms),
212 );
213 duration_from_millis_saturating(total_millis)
214 }
215 BackoffStrategy::Exponential { base_ms, max_ms } => {
216 if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
217 return Duration::ZERO;
218 }
219
220 let mut total_millis = 0_u128;
221 let mut attempt = 0_u32;
222 while attempt < self.max_retries {
223 let delay_millis = match 1_u64.checked_shl(attempt) {
224 Some(multiplier) => base_ms.saturating_mul(multiplier).min(*max_ms),
225 None => *max_ms,
226 };
227 add_millis_saturating(&mut total_millis, u128::from(delay_millis));
228 attempt = attempt.saturating_add(1);
229
230 if delay_millis == *max_ms {
231 let remaining = u128::from(self.max_retries.saturating_sub(attempt));
232 add_millis_saturating(
233 &mut total_millis,
234 u128::from(*max_ms).saturating_mul(remaining),
235 );
236 break;
237 }
238 }
239 duration_from_millis_saturating(total_millis)
240 }
241 }
242 }
243}
244
245pub fn task_with_timeout<M, F>(timeout: Duration, f: F, on_timeout: M) -> Cmd<M>
251where
252 M: Send + 'static,
253 F: FnOnce(CancellationToken) -> M + Send + 'static,
254{
255 Cmd::task(move || {
256 let source = CancellationSource::new();
257 let token = source.token();
258 let (handle, rx) = spawn_worker(move || f(token));
259 match rx.recv_timeout(timeout) {
260 Ok(result) => {
261 join_task_thread(handle);
262 result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
264 }
265 Err(_) => {
266 source.cancel();
267 join_task_thread_bounded(handle, "task_with_timeout");
268 on_timeout
269 }
270 }
271 })
272}
273
274pub fn task_with_timeout_named<M, F>(
276 name: impl Into<String>,
277 timeout: Duration,
278 f: F,
279 on_timeout: M,
280) -> Cmd<M>
281where
282 M: Send + 'static,
283 F: FnOnce(CancellationToken) -> M + Send + 'static,
284{
285 Cmd::task_with_spec(TaskSpec::default().with_name(name), move || {
286 let source = CancellationSource::new();
287 let token = source.token();
288 let (handle, rx) = spawn_worker(move || f(token));
289 match rx.recv_timeout(timeout) {
290 Ok(result) => {
291 join_task_thread(handle);
292 result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
293 }
294 Err(_) => {
295 source.cancel();
296 join_task_thread_bounded(handle, "task_with_timeout_named");
297 on_timeout
298 }
299 }
300 })
301}
302
303pub fn task_with_retry<M, F>(policy: RetryPolicy, f: F, on_exhaust: fn(String) -> M) -> Cmd<M>
310where
311 M: Send + 'static,
312 F: Fn() -> Result<M, String> + Send + 'static,
313{
314 Cmd::task(move || {
315 let mut last_err = String::new();
316 for attempt in 0..=policy.max_retries {
317 match f() {
318 Ok(msg) => return msg,
319 Err(e) => {
320 last_err = e;
321 if attempt < policy.max_retries {
322 std::thread::sleep(policy.delay(attempt));
323 }
324 }
325 }
326 }
327 on_exhaust(last_err)
328 })
329}
330
331pub fn task_with_retry_and_timeout<M, F>(
337 policy: RetryPolicy,
338 per_attempt_timeout: Duration,
339 f: F,
340 on_exhaust: fn(String) -> M,
341) -> Cmd<M>
342where
343 M: Send + 'static,
344 F: Fn(CancellationToken) -> Result<M, String> + Send + Sync + 'static,
345{
346 Cmd::task(move || {
347 let f = std::sync::Arc::new(f);
348 let mut last_err = String::new();
349 for attempt in 0..=policy.max_retries {
350 let source = CancellationSource::new();
351 let token = source.token();
352 let f_clone = std::sync::Arc::clone(&f);
353 let (handle, rx) = spawn_worker(move || f_clone(token));
354 match rx.recv_timeout(per_attempt_timeout) {
355 Ok(Ok(Ok(msg))) => {
356 join_task_thread(handle);
357 return msg;
358 }
359 Ok(Ok(Err(e))) => {
360 join_task_thread(handle);
361 last_err = e;
362 }
363 Ok(Err(payload)) => {
366 join_task_thread(handle);
367 std::panic::resume_unwind(payload);
368 }
369 Err(_) => {
370 source.cancel();
371 join_task_thread_bounded(handle, "task_with_retry_and_timeout");
372 last_err = "timeout".into();
373 }
374 }
375 if attempt < policy.max_retries {
376 std::thread::sleep(policy.delay(attempt));
377 }
378 }
379 on_exhaust(last_err)
380 })
381}
382
383#[cfg(test)]
384mod tests {
385 use super::*;
386
387 #[test]
388 fn fixed_backoff_constant_delay() {
389 let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
390 assert_eq!(policy.delay(0), Duration::from_millis(100));
391 assert_eq!(policy.delay(1), Duration::from_millis(100));
392 assert_eq!(policy.delay(2), Duration::from_millis(100));
393 }
394
395 #[test]
396 fn exponential_backoff_doubles() {
397 let policy = RetryPolicy::new(
398 5,
399 BackoffStrategy::Exponential {
400 base_ms: 100,
401 max_ms: 5000,
402 },
403 );
404 assert_eq!(policy.delay(0), Duration::from_millis(100));
405 assert_eq!(policy.delay(1), Duration::from_millis(200));
406 assert_eq!(policy.delay(2), Duration::from_millis(400));
407 assert_eq!(policy.delay(3), Duration::from_millis(800));
408 }
409
410 #[test]
411 fn exponential_backoff_caps_at_max() {
412 let policy = RetryPolicy::new(
413 5,
414 BackoffStrategy::Exponential {
415 base_ms: 1000,
416 max_ms: 3000,
417 },
418 );
419 assert_eq!(policy.delay(0), Duration::from_millis(1000));
420 assert_eq!(policy.delay(1), Duration::from_millis(2000));
421 assert_eq!(policy.delay(2), Duration::from_millis(3000)); assert_eq!(policy.delay(3), Duration::from_millis(3000)); }
424
425 #[test]
426 fn linear_backoff_increments() {
427 let policy = RetryPolicy::new(
428 4,
429 BackoffStrategy::Linear {
430 base_ms: 100,
431 max_ms: 500,
432 },
433 );
434 assert_eq!(policy.delay(0), Duration::from_millis(100));
435 assert_eq!(policy.delay(1), Duration::from_millis(200));
436 assert_eq!(policy.delay(2), Duration::from_millis(300));
437 assert_eq!(policy.delay(3), Duration::from_millis(400));
438 assert_eq!(policy.delay(4), Duration::from_millis(500)); }
440
441 #[test]
442 fn linear_backoff_caps_at_max() {
443 let policy = RetryPolicy::new(
444 4,
445 BackoffStrategy::Linear {
446 base_ms: 200,
447 max_ms: 500,
448 },
449 );
450 assert_eq!(policy.delay(2), Duration::from_millis(500)); }
452
453 #[test]
454 fn no_retry_policy() {
455 let policy = RetryPolicy::no_retry();
456 assert_eq!(policy.max_retries, 0);
457 }
458
459 #[test]
460 fn total_max_delay_fixed() {
461 let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
462 assert_eq!(policy.total_max_delay(), Duration::from_millis(300));
463 }
464
465 #[test]
466 fn total_max_delay_exponential() {
467 let policy = RetryPolicy::new(
468 3,
469 BackoffStrategy::Exponential {
470 base_ms: 100,
471 max_ms: 10000,
472 },
473 );
474 assert_eq!(policy.total_max_delay(), Duration::from_millis(700));
476 }
477
478 #[test]
479 fn total_max_delay_zero_retries() {
480 let policy = RetryPolicy::no_retry();
481 assert_eq!(policy.total_max_delay(), Duration::ZERO);
482 }
483
484 #[test]
485 fn total_max_delay_fixed_saturates_without_iterating() {
486 let policy = RetryPolicy::new(u32::MAX, BackoffStrategy::Fixed { delay_ms: u64::MAX });
487 assert_eq!(policy.total_max_delay(), Duration::MAX);
488 }
489
490 #[test]
491 fn total_max_delay_linear_handles_large_retry_counts() {
492 let policy = RetryPolicy::new(
493 u32::MAX,
494 BackoffStrategy::Linear {
495 base_ms: 1,
496 max_ms: 10,
497 },
498 );
499 assert_eq!(
500 policy.total_max_delay(),
501 Duration::from_millis(10_u64.saturating_mul(u64::from(u32::MAX)) - 45)
502 );
503 }
504
505 #[test]
506 fn total_max_delay_exponential_saturates_after_cap() {
507 let policy = RetryPolicy::new(
508 6,
509 BackoffStrategy::Exponential {
510 base_ms: 10,
511 max_ms: 35,
512 },
513 );
514 assert_eq!(
515 policy.total_max_delay(),
516 Duration::from_millis(10 + 20 + 35 * 4)
517 );
518 }
519
520 #[test]
521 fn total_max_delay_matches_delay_sequence_for_representative_policies() {
522 let policies = [
523 RetryPolicy::new(5, BackoffStrategy::Fixed { delay_ms: 7 }),
524 RetryPolicy::new(
525 6,
526 BackoffStrategy::Linear {
527 base_ms: 3,
528 max_ms: 10,
529 },
530 ),
531 RetryPolicy::new(
532 4,
533 BackoffStrategy::Linear {
534 base_ms: 10,
535 max_ms: 3,
536 },
537 ),
538 RetryPolicy::new(
539 6,
540 BackoffStrategy::Exponential {
541 base_ms: 2,
542 max_ms: 9,
543 },
544 ),
545 ];
546
547 for policy in policies {
548 let expected_millis = (0..policy.max_retries)
549 .map(|attempt| policy.delay(attempt).as_millis())
550 .sum::<u128>();
551 assert_eq!(policy.total_max_delay().as_millis(), expected_millis);
552 }
553 }
554
555 #[test]
556 fn exponential_backoff_overflow_saturates() {
557 let policy = RetryPolicy::new(
558 1,
559 BackoffStrategy::Exponential {
560 base_ms: u64::MAX / 2,
561 max_ms: u64::MAX,
562 },
563 );
564 let _ = policy.delay(30);
566 }
567
568 #[test]
569 fn linear_backoff_overflow_saturates() {
570 let policy = RetryPolicy::new(
571 1,
572 BackoffStrategy::Linear {
573 base_ms: u64::MAX / 2,
574 max_ms: u64::MAX,
575 },
576 );
577 let _ = policy.delay(30);
578 }
579
580 #[test]
581 fn retry_policy_clone_eq() {
582 let policy = RetryPolicy::new(
583 3,
584 BackoffStrategy::Exponential {
585 base_ms: 100,
586 max_ms: 5000,
587 },
588 );
589 let cloned = policy.clone();
590 assert_eq!(policy, cloned);
591 }
592
593 #[test]
594 fn task_with_retry_succeeds_first_try() {
595 #[derive(Debug, PartialEq)]
596 enum Msg {
597 Ok(i32),
598 Err(String),
599 }
600
601 let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 1 });
602 let cmd = task_with_retry(policy, || Ok(Msg::Ok(42)), Msg::Err);
603
604 assert_eq!(cmd.type_name(), "Task");
606 }
607
608 #[test]
609 fn task_with_timeout_produces_task() {
610 #[derive(Debug)]
611 #[allow(dead_code)]
612 enum Msg {
613 Result(i32),
614 Timeout,
615 }
616
617 let cmd = task_with_timeout(
618 Duration::from_secs(1),
619 |_token| Msg::Result(42),
620 Msg::Timeout,
621 );
622 assert_eq!(cmd.type_name(), "Task");
623 }
624
625 #[test]
626 fn task_with_timeout_requests_cancellation_on_timeout() {
627 use std::sync::Arc;
628 use std::sync::atomic::{AtomicBool, Ordering};
629
630 #[derive(Debug, PartialEq)]
631 enum Msg {
632 Finished,
633 Timeout,
634 }
635
636 let cancelled = Arc::new(AtomicBool::new(false));
637 let worker_exited = Arc::new(AtomicBool::new(false));
638 let cancelled_flag = Arc::clone(&cancelled);
639 let exited_flag = Arc::clone(&worker_exited);
640
641 let cmd = task_with_timeout(
642 Duration::from_millis(10),
643 move |token| {
644 cancelled_flag.store(token.wait_timeout(Duration::from_secs(1)), Ordering::SeqCst);
645 exited_flag.store(true, Ordering::SeqCst);
646 Msg::Finished
647 },
648 Msg::Timeout,
649 );
650
651 let result = match cmd {
652 Cmd::Task(_, task) => task(),
653 other => panic!("expected Task, got {other:?}"),
654 };
655
656 assert_eq!(result, Msg::Timeout);
657 std::thread::sleep(Duration::from_millis(50));
658 assert!(cancelled.load(Ordering::SeqCst));
659 assert!(worker_exited.load(Ordering::SeqCst));
660 }
661
662 #[cfg(not(panic = "abort"))]
663 #[test]
664 fn a_panicking_worker_is_the_tasks_panic_and_spares_the_terminal() {
665 use std::cell::Cell;
666 use std::sync::atomic::{AtomicUsize, Ordering};
667
668 thread_local! {
671 static MARKED_WORKER: Cell<bool> = const { Cell::new(false) };
672 }
673 static CLEANUPS: AtomicUsize = AtomicUsize::new(0);
674 fn count_cleanup() {
675 if MARKED_WORKER.with(Cell::get) {
676 CLEANUPS.fetch_add(1, Ordering::SeqCst);
677 }
678 }
679 ftui_core::session_teardown::install_chained_panic_hook(
680 "retry-worker-panic-test",
681 count_cleanup,
682 );
683
684 let cmd = task_with_timeout(
685 Duration::from_secs(5),
686 |_token| -> u8 {
687 MARKED_WORKER.with(|marked| marked.set(true));
688 panic!("worker bug")
689 },
690 0,
691 );
692 let Cmd::Task(_, task) = cmd else {
693 panic!("expected a Task");
694 };
695 let start = std::time::Instant::now();
696 let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(task))
697 .expect_err("the worker's panic should be the task's, not a timeout message");
698
699 assert_eq!(payload.downcast_ref::<&str>(), Some(&"worker bug"));
700 assert!(start.elapsed() < std::time::Duration::from_secs(5));
701 assert_eq!(
702 CLEANUPS.load(Ordering::SeqCst),
703 0,
704 "the worker's panic ran terminal cleanup"
705 );
706 }
707
708 #[cfg(not(panic = "abort"))]
709 #[test]
710 fn a_panic_is_not_a_failed_attempt_to_retry() {
711 use std::sync::Arc;
712 use std::sync::atomic::{AtomicUsize, Ordering};
713
714 let attempts = Arc::new(AtomicUsize::new(0));
715 let attempts_seen = Arc::clone(&attempts);
716 let cmd = task_with_retry_and_timeout(
717 RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 0 }),
718 Duration::from_secs(5),
719 move |_token| -> Result<u8, String> {
720 attempts_seen.fetch_add(1, Ordering::SeqCst);
721 panic!("worker bug")
722 },
723 |_err| 0,
724 );
725 let Cmd::Task(_, task) = cmd else {
726 panic!("expected a Task");
727 };
728
729 assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(task)).is_err());
730 assert_eq!(attempts.load(Ordering::SeqCst), 1, "a panic was retried");
731 }
732
733 #[test]
734 fn task_with_retry_and_timeout_cancels_each_timed_out_attempt() {
735 use std::sync::Arc;
736 use std::sync::atomic::{AtomicUsize, Ordering};
737
738 #[derive(Debug, PartialEq)]
739 enum Msg {
740 Exhausted(String),
741 }
742
743 fn on_exhaust(err: String) -> Msg {
744 Msg::Exhausted(err)
745 }
746
747 let attempts = Arc::new(AtomicUsize::new(0));
748 let cancelled = Arc::new(AtomicUsize::new(0));
749 let attempts_flag = Arc::clone(&attempts);
750 let cancelled_flag = Arc::clone(&cancelled);
751 let policy = RetryPolicy::new(1, BackoffStrategy::Fixed { delay_ms: 0 });
752
753 let cmd = task_with_retry_and_timeout(
754 policy,
755 Duration::from_millis(10),
756 move |token| {
757 attempts_flag.fetch_add(1, Ordering::SeqCst);
758 if token.wait_timeout(Duration::from_secs(1)) {
759 cancelled_flag.fetch_add(1, Ordering::SeqCst);
760 }
761 Err("cancelled".to_owned())
762 },
763 on_exhaust,
764 );
765
766 let result = match cmd {
767 Cmd::Task(_, task) => task(),
768 other => panic!("expected Task, got {other:?}"),
769 };
770
771 assert_eq!(result, Msg::Exhausted("timeout".to_owned()));
772 std::thread::sleep(Duration::from_millis(50));
773 assert_eq!(attempts.load(Ordering::SeqCst), 2);
774 assert_eq!(cancelled.load(Ordering::SeqCst), 2);
775 }
776
777 #[test]
778 fn backoff_strategy_variants_debug() {
779 let fixed = BackoffStrategy::Fixed { delay_ms: 100 };
780 let exp = BackoffStrategy::Exponential {
781 base_ms: 100,
782 max_ms: 5000,
783 };
784 let linear = BackoffStrategy::Linear {
785 base_ms: 100,
786 max_ms: 500,
787 };
788 let _ = format!("{fixed:?}");
790 let _ = format!("{exp:?}");
791 let _ = format!("{linear:?}");
792 }
793}