1#![forbid(unsafe_code)]
36
37use std::sync::mpsc;
38
39use crate::cancellation::{CancellationSource, CancellationToken};
40use crate::program::{Cmd, TaskSpec};
41use web_time::Duration;
42
43const TASK_THREAD_JOIN_TIMEOUT: Duration = Duration::from_millis(250);
44const TASK_THREAD_JOIN_POLL: Duration = Duration::from_millis(5);
45
46fn duration_from_millis_saturating(millis: u128) -> Duration {
47 if millis >= Duration::MAX.as_millis() {
48 Duration::MAX
49 } else {
50 let seconds = millis / 1_000;
51 let subsecond_millis = millis % 1_000;
52 let Ok(seconds) = u64::try_from(seconds) else {
53 return Duration::MAX;
54 };
55 let Ok(nanos) = u32::try_from(subsecond_millis.saturating_mul(1_000_000)) else {
56 return Duration::MAX;
57 };
58 Duration::new(seconds, nanos)
59 }
60}
61
62fn add_millis_saturating(total: &mut u128, millis: u128) {
63 *total = total.saturating_add(millis).min(Duration::MAX.as_millis());
64}
65
66fn join_task_thread(handle: std::thread::JoinHandle<()>) {
67 let _ = handle.join();
68}
69
70fn spawn_worker<T, F>(
79 f: F,
80) -> (
81 std::thread::JoinHandle<()>,
82 mpsc::Receiver<std::thread::Result<T>>,
83)
84where
85 T: Send + 'static,
86 F: FnOnce() -> T + Send + 'static,
87{
88 let (tx, rx) = mpsc::channel();
89 let handle = std::thread::spawn(move || {
90 let result = ftui_core::with_panic_cleanup_suppressed(|| {
91 std::panic::catch_unwind(std::panic::AssertUnwindSafe(f))
92 });
93 let _ = tx.send(result);
94 });
95 (handle, rx)
96}
97
98fn join_task_thread_bounded(handle: std::thread::JoinHandle<()>, task_name: &'static str) {
99 let start = web_time::Instant::now();
100 while !handle.is_finished() {
101 if start.elapsed() >= TASK_THREAD_JOIN_TIMEOUT {
102 tracing::warn!(
103 target: crate::telemetry_schema::TARGET_EFFECT,
104 task = task_name,
105 timeout_ms = TASK_THREAD_JOIN_TIMEOUT.as_millis() as u64,
106 "Timed-out worker thread did not exit within the cancellation join timeout; detaching"
107 );
108 return;
109 }
110 std::thread::sleep(TASK_THREAD_JOIN_POLL);
111 }
112 join_task_thread(handle);
113}
114
115#[derive(Debug, Clone, PartialEq)]
117#[cfg_attr(
118 feature = "state-persistence",
119 derive(serde::Serialize, serde::Deserialize)
120)]
121pub enum BackoffStrategy {
122 Fixed {
124 delay_ms: u64,
126 },
127 Exponential {
129 base_ms: u64,
131 max_ms: u64,
133 },
134 Linear {
136 base_ms: u64,
138 max_ms: u64,
140 },
141}
142
143#[derive(Debug, Clone, PartialEq)]
145#[cfg_attr(
146 feature = "state-persistence",
147 derive(serde::Serialize, serde::Deserialize)
148)]
149pub struct RetryPolicy {
150 pub max_retries: u32,
152 pub backoff: BackoffStrategy,
154}
155
156impl RetryPolicy {
157 pub fn new(max_retries: u32, backoff: BackoffStrategy) -> Self {
159 Self {
160 max_retries,
161 backoff,
162 }
163 }
164
165 pub fn no_retry() -> Self {
167 Self {
168 max_retries: 0,
169 backoff: BackoffStrategy::Fixed { delay_ms: 0 },
170 }
171 }
172
173 pub fn delay(&self, attempt: u32) -> Duration {
175 match &self.backoff {
176 BackoffStrategy::Fixed { delay_ms } => Duration::from_millis(*delay_ms),
177 BackoffStrategy::Exponential { base_ms, max_ms } => {
178 let multiplier = 1u64.checked_shl(attempt).unwrap_or(u64::MAX);
179 let delay = base_ms.saturating_mul(multiplier);
180 Duration::from_millis(delay.min(*max_ms))
181 }
182 BackoffStrategy::Linear { base_ms, max_ms } => {
183 let delay = base_ms.saturating_mul(u64::from(attempt) + 1);
184 Duration::from_millis(delay.min(*max_ms))
185 }
186 }
187 }
188
189 pub fn total_max_delay(&self) -> Duration {
191 let retry_count = u128::from(self.max_retries);
192 let max_duration_millis = Duration::MAX.as_millis();
193 match &self.backoff {
194 BackoffStrategy::Fixed { delay_ms } => {
195 duration_from_millis_saturating(u128::from(*delay_ms).saturating_mul(retry_count))
196 }
197 BackoffStrategy::Linear { base_ms, max_ms } => {
198 if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
199 return Duration::ZERO;
200 }
201
202 let uncapped_terms = retry_count.min(u128::from(*max_ms / *base_ms));
203 let arithmetic_sum =
204 uncapped_terms.saturating_mul(uncapped_terms.saturating_add(1)) / 2;
205 let mut total_millis = u128::from(*base_ms)
206 .saturating_mul(arithmetic_sum)
207 .min(max_duration_millis);
208
209 let capped_terms = retry_count.saturating_sub(uncapped_terms);
210 add_millis_saturating(
211 &mut total_millis,
212 u128::from(*max_ms).saturating_mul(capped_terms),
213 );
214 duration_from_millis_saturating(total_millis)
215 }
216 BackoffStrategy::Exponential { base_ms, max_ms } => {
217 if self.max_retries == 0 || *base_ms == 0 || *max_ms == 0 {
218 return Duration::ZERO;
219 }
220
221 let mut total_millis = 0_u128;
222 let mut attempt = 0_u32;
223 while attempt < self.max_retries {
224 let delay_millis = match 1_u64.checked_shl(attempt) {
225 Some(multiplier) => base_ms.saturating_mul(multiplier).min(*max_ms),
226 None => *max_ms,
227 };
228 add_millis_saturating(&mut total_millis, u128::from(delay_millis));
229 attempt = attempt.saturating_add(1);
230
231 if delay_millis == *max_ms {
232 let remaining = u128::from(self.max_retries.saturating_sub(attempt));
233 add_millis_saturating(
234 &mut total_millis,
235 u128::from(*max_ms).saturating_mul(remaining),
236 );
237 break;
238 }
239 }
240 duration_from_millis_saturating(total_millis)
241 }
242 }
243 }
244}
245
246pub fn task_with_timeout<M, F>(timeout: Duration, f: F, on_timeout: M) -> Cmd<M>
252where
253 M: Send + 'static,
254 F: FnOnce(CancellationToken) -> M + Send + 'static,
255{
256 Cmd::task(move || {
257 let source = CancellationSource::new();
258 let token = source.token();
259 let (handle, rx) = spawn_worker(move || f(token));
260 match rx.recv_timeout(timeout) {
261 Ok(result) => {
262 join_task_thread(handle);
263 result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
265 }
266 Err(_) => {
267 source.cancel();
268 join_task_thread_bounded(handle, "task_with_timeout");
269 on_timeout
270 }
271 }
272 })
273}
274
275pub fn task_with_timeout_named<M, F>(
277 name: impl Into<String>,
278 timeout: Duration,
279 f: F,
280 on_timeout: M,
281) -> Cmd<M>
282where
283 M: Send + 'static,
284 F: FnOnce(CancellationToken) -> M + Send + 'static,
285{
286 Cmd::task_with_spec(TaskSpec::default().with_name(name), move || {
287 let source = CancellationSource::new();
288 let token = source.token();
289 let (handle, rx) = spawn_worker(move || f(token));
290 match rx.recv_timeout(timeout) {
291 Ok(result) => {
292 join_task_thread(handle);
293 result.unwrap_or_else(|payload| std::panic::resume_unwind(payload))
294 }
295 Err(_) => {
296 source.cancel();
297 join_task_thread_bounded(handle, "task_with_timeout_named");
298 on_timeout
299 }
300 }
301 })
302}
303
304pub fn task_with_retry<M, F>(policy: RetryPolicy, f: F, on_exhaust: fn(String) -> M) -> Cmd<M>
311where
312 M: Send + 'static,
313 F: Fn() -> Result<M, String> + Send + 'static,
314{
315 Cmd::task(move || {
316 let mut last_err = String::new();
317 for attempt in 0..=policy.max_retries {
318 match f() {
319 Ok(msg) => return msg,
320 Err(e) => {
321 last_err = e;
322 if attempt < policy.max_retries {
323 std::thread::sleep(policy.delay(attempt));
324 }
325 }
326 }
327 }
328 on_exhaust(last_err)
329 })
330}
331
332pub fn task_with_retry_and_timeout<M, F>(
338 policy: RetryPolicy,
339 per_attempt_timeout: Duration,
340 f: F,
341 on_exhaust: fn(String) -> M,
342) -> Cmd<M>
343where
344 M: Send + 'static,
345 F: Fn(CancellationToken) -> Result<M, String> + Send + Sync + 'static,
346{
347 Cmd::task(move || {
348 let f = std::sync::Arc::new(f);
349 let mut last_err = String::new();
350 for attempt in 0..=policy.max_retries {
351 let source = CancellationSource::new();
352 let token = source.token();
353 let f_clone = std::sync::Arc::clone(&f);
354 let (handle, rx) = spawn_worker(move || f_clone(token));
355 match rx.recv_timeout(per_attempt_timeout) {
356 Ok(Ok(Ok(msg))) => {
357 join_task_thread(handle);
358 return msg;
359 }
360 Ok(Ok(Err(e))) => {
361 join_task_thread(handle);
362 last_err = e;
363 }
364 Ok(Err(payload)) => {
367 join_task_thread(handle);
368 std::panic::resume_unwind(payload);
369 }
370 Err(_) => {
371 source.cancel();
372 join_task_thread_bounded(handle, "task_with_retry_and_timeout");
373 last_err = "timeout".into();
374 }
375 }
376 if attempt < policy.max_retries {
377 std::thread::sleep(policy.delay(attempt));
378 }
379 }
380 on_exhaust(last_err)
381 })
382}
383
384#[cfg(test)]
385mod tests {
386 use super::*;
387
388 #[test]
389 fn fixed_backoff_constant_delay() {
390 let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
391 assert_eq!(policy.delay(0), Duration::from_millis(100));
392 assert_eq!(policy.delay(1), Duration::from_millis(100));
393 assert_eq!(policy.delay(2), Duration::from_millis(100));
394 }
395
396 #[test]
397 fn exponential_backoff_doubles() {
398 let policy = RetryPolicy::new(
399 5,
400 BackoffStrategy::Exponential {
401 base_ms: 100,
402 max_ms: 5000,
403 },
404 );
405 assert_eq!(policy.delay(0), Duration::from_millis(100));
406 assert_eq!(policy.delay(1), Duration::from_millis(200));
407 assert_eq!(policy.delay(2), Duration::from_millis(400));
408 assert_eq!(policy.delay(3), Duration::from_millis(800));
409 }
410
411 #[test]
412 fn exponential_backoff_caps_at_max() {
413 let policy = RetryPolicy::new(
414 5,
415 BackoffStrategy::Exponential {
416 base_ms: 1000,
417 max_ms: 3000,
418 },
419 );
420 assert_eq!(policy.delay(0), Duration::from_millis(1000));
421 assert_eq!(policy.delay(1), Duration::from_millis(2000));
422 assert_eq!(policy.delay(2), Duration::from_millis(3000)); assert_eq!(policy.delay(3), Duration::from_millis(3000)); }
425
426 #[test]
427 fn linear_backoff_increments() {
428 let policy = RetryPolicy::new(
429 4,
430 BackoffStrategy::Linear {
431 base_ms: 100,
432 max_ms: 500,
433 },
434 );
435 assert_eq!(policy.delay(0), Duration::from_millis(100));
436 assert_eq!(policy.delay(1), Duration::from_millis(200));
437 assert_eq!(policy.delay(2), Duration::from_millis(300));
438 assert_eq!(policy.delay(3), Duration::from_millis(400));
439 assert_eq!(policy.delay(4), Duration::from_millis(500)); }
441
442 #[test]
443 fn linear_backoff_caps_at_max() {
444 let policy = RetryPolicy::new(
445 4,
446 BackoffStrategy::Linear {
447 base_ms: 200,
448 max_ms: 500,
449 },
450 );
451 assert_eq!(policy.delay(2), Duration::from_millis(500)); }
453
454 #[test]
455 fn no_retry_policy() {
456 let policy = RetryPolicy::no_retry();
457 assert_eq!(policy.max_retries, 0);
458 }
459
460 #[test]
461 fn total_max_delay_fixed() {
462 let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 100 });
463 assert_eq!(policy.total_max_delay(), Duration::from_millis(300));
464 }
465
466 #[test]
467 fn total_max_delay_exponential() {
468 let policy = RetryPolicy::new(
469 3,
470 BackoffStrategy::Exponential {
471 base_ms: 100,
472 max_ms: 10000,
473 },
474 );
475 assert_eq!(policy.total_max_delay(), Duration::from_millis(700));
477 }
478
479 #[test]
480 fn total_max_delay_zero_retries() {
481 let policy = RetryPolicy::no_retry();
482 assert_eq!(policy.total_max_delay(), Duration::ZERO);
483 }
484
485 #[test]
486 fn total_max_delay_fixed_saturates_without_iterating() {
487 let policy = RetryPolicy::new(u32::MAX, BackoffStrategy::Fixed { delay_ms: u64::MAX });
488 assert_eq!(policy.total_max_delay(), Duration::MAX);
489 }
490
491 #[test]
492 fn total_max_delay_linear_handles_large_retry_counts() {
493 let policy = RetryPolicy::new(
494 u32::MAX,
495 BackoffStrategy::Linear {
496 base_ms: 1,
497 max_ms: 10,
498 },
499 );
500 assert_eq!(
501 policy.total_max_delay(),
502 Duration::from_millis(10_u64.saturating_mul(u64::from(u32::MAX)) - 45)
503 );
504 }
505
506 #[test]
507 fn total_max_delay_exponential_saturates_after_cap() {
508 let policy = RetryPolicy::new(
509 6,
510 BackoffStrategy::Exponential {
511 base_ms: 10,
512 max_ms: 35,
513 },
514 );
515 assert_eq!(
516 policy.total_max_delay(),
517 Duration::from_millis(10 + 20 + 35 * 4)
518 );
519 }
520
521 #[test]
522 fn total_max_delay_matches_delay_sequence_for_representative_policies() {
523 let policies = [
524 RetryPolicy::new(5, BackoffStrategy::Fixed { delay_ms: 7 }),
525 RetryPolicy::new(
526 6,
527 BackoffStrategy::Linear {
528 base_ms: 3,
529 max_ms: 10,
530 },
531 ),
532 RetryPolicy::new(
533 4,
534 BackoffStrategy::Linear {
535 base_ms: 10,
536 max_ms: 3,
537 },
538 ),
539 RetryPolicy::new(
540 6,
541 BackoffStrategy::Exponential {
542 base_ms: 2,
543 max_ms: 9,
544 },
545 ),
546 ];
547
548 for policy in policies {
549 let expected_millis = (0..policy.max_retries)
550 .map(|attempt| policy.delay(attempt).as_millis())
551 .sum::<u128>();
552 assert_eq!(policy.total_max_delay().as_millis(), expected_millis);
553 }
554 }
555
556 #[test]
557 fn exponential_backoff_overflow_saturates() {
558 let policy = RetryPolicy::new(
559 1,
560 BackoffStrategy::Exponential {
561 base_ms: u64::MAX / 2,
562 max_ms: u64::MAX,
563 },
564 );
565 let _ = policy.delay(30);
567 }
568
569 #[test]
570 fn linear_backoff_overflow_saturates() {
571 let policy = RetryPolicy::new(
572 1,
573 BackoffStrategy::Linear {
574 base_ms: u64::MAX / 2,
575 max_ms: u64::MAX,
576 },
577 );
578 let _ = policy.delay(30);
579 }
580
581 #[test]
582 fn retry_policy_clone_eq() {
583 let policy = RetryPolicy::new(
584 3,
585 BackoffStrategy::Exponential {
586 base_ms: 100,
587 max_ms: 5000,
588 },
589 );
590 let cloned = policy.clone();
591 assert_eq!(policy, cloned);
592 }
593
594 #[test]
595 fn task_with_retry_succeeds_first_try() {
596 #[derive(Debug, PartialEq)]
597 enum Msg {
598 Ok(i32),
599 Err(String),
600 }
601
602 let policy = RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 1 });
603 let cmd = task_with_retry(policy, || Ok(Msg::Ok(42)), Msg::Err);
604
605 assert_eq!(cmd.type_name(), "Task");
607 }
608
609 #[test]
610 fn task_with_timeout_produces_task() {
611 #[derive(Debug)]
612 #[allow(dead_code)]
613 enum Msg {
614 Result(i32),
615 Timeout,
616 }
617
618 let cmd = task_with_timeout(
619 Duration::from_secs(1),
620 |_token| Msg::Result(42),
621 Msg::Timeout,
622 );
623 assert_eq!(cmd.type_name(), "Task");
624 }
625
626 #[test]
627 fn task_with_timeout_requests_cancellation_on_timeout() {
628 use std::sync::Arc;
629 use std::sync::atomic::{AtomicBool, Ordering};
630
631 #[derive(Debug, PartialEq)]
632 enum Msg {
633 Finished,
634 Timeout,
635 }
636
637 let cancelled = Arc::new(AtomicBool::new(false));
638 let worker_exited = Arc::new(AtomicBool::new(false));
639 let cancelled_flag = Arc::clone(&cancelled);
640 let exited_flag = Arc::clone(&worker_exited);
641
642 let cmd = task_with_timeout(
643 Duration::from_millis(10),
644 move |token| {
645 cancelled_flag.store(token.wait_timeout(Duration::from_secs(1)), Ordering::SeqCst);
646 exited_flag.store(true, Ordering::SeqCst);
647 Msg::Finished
648 },
649 Msg::Timeout,
650 );
651
652 let result = match cmd {
653 Cmd::Task(_, task) => task(),
654 other => panic!("expected Task, got {other:?}"),
655 };
656
657 assert_eq!(result, Msg::Timeout);
658 std::thread::sleep(Duration::from_millis(50));
659 assert!(cancelled.load(Ordering::SeqCst));
660 assert!(worker_exited.load(Ordering::SeqCst));
661 }
662
663 #[cfg(not(panic = "abort"))]
664 #[test]
665 fn a_panicking_worker_is_the_tasks_panic_and_spares_the_terminal() {
666 use std::cell::Cell;
667 use std::sync::atomic::{AtomicUsize, Ordering};
668
669 thread_local! {
672 static MARKED_WORKER: Cell<bool> = const { Cell::new(false) };
673 }
674 static CLEANUPS: AtomicUsize = AtomicUsize::new(0);
675 fn count_cleanup() {
676 if MARKED_WORKER.with(Cell::get) {
677 CLEANUPS.fetch_add(1, Ordering::SeqCst);
678 }
679 }
680 ftui_core::session_teardown::install_chained_panic_hook(
681 "retry-worker-panic-test",
682 count_cleanup,
683 || false,
684 );
685
686 let cmd = task_with_timeout(
687 Duration::from_secs(5),
688 |_token| -> u8 {
689 MARKED_WORKER.with(|marked| marked.set(true));
690 panic!("worker bug")
691 },
692 0,
693 );
694 let Cmd::Task(_, task) = cmd else {
695 panic!("expected a Task");
696 };
697 let start = std::time::Instant::now();
698 let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(task))
699 .expect_err("the worker's panic should be the task's, not a timeout message");
700
701 assert_eq!(payload.downcast_ref::<&str>(), Some(&"worker bug"));
702 assert!(start.elapsed() < std::time::Duration::from_secs(5));
703 assert_eq!(
704 CLEANUPS.load(Ordering::SeqCst),
705 0,
706 "the worker's panic ran terminal cleanup"
707 );
708 }
709
710 #[cfg(not(panic = "abort"))]
711 #[test]
712 fn a_panic_is_not_a_failed_attempt_to_retry() {
713 use std::sync::Arc;
714 use std::sync::atomic::{AtomicUsize, Ordering};
715
716 let attempts = Arc::new(AtomicUsize::new(0));
717 let attempts_seen = Arc::clone(&attempts);
718 let cmd = task_with_retry_and_timeout(
719 RetryPolicy::new(3, BackoffStrategy::Fixed { delay_ms: 0 }),
720 Duration::from_secs(5),
721 move |_token| -> Result<u8, String> {
722 attempts_seen.fetch_add(1, Ordering::SeqCst);
723 panic!("worker bug")
724 },
725 |_err| 0,
726 );
727 let Cmd::Task(_, task) = cmd else {
728 panic!("expected a Task");
729 };
730
731 assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(task)).is_err());
732 assert_eq!(attempts.load(Ordering::SeqCst), 1, "a panic was retried");
733 }
734
735 #[test]
736 fn task_with_retry_and_timeout_cancels_each_timed_out_attempt() {
737 use std::sync::Arc;
738 use std::sync::atomic::{AtomicUsize, Ordering};
739
740 #[derive(Debug, PartialEq)]
741 enum Msg {
742 Exhausted(String),
743 }
744
745 fn on_exhaust(err: String) -> Msg {
746 Msg::Exhausted(err)
747 }
748
749 let attempts = Arc::new(AtomicUsize::new(0));
750 let cancelled = Arc::new(AtomicUsize::new(0));
751 let attempts_flag = Arc::clone(&attempts);
752 let cancelled_flag = Arc::clone(&cancelled);
753 let policy = RetryPolicy::new(1, BackoffStrategy::Fixed { delay_ms: 0 });
754
755 let cmd = task_with_retry_and_timeout(
756 policy,
757 Duration::from_millis(10),
758 move |token| {
759 attempts_flag.fetch_add(1, Ordering::SeqCst);
760 if token.wait_timeout(Duration::from_secs(1)) {
761 cancelled_flag.fetch_add(1, Ordering::SeqCst);
762 }
763 Err("cancelled".to_owned())
764 },
765 on_exhaust,
766 );
767
768 let result = match cmd {
769 Cmd::Task(_, task) => task(),
770 other => panic!("expected Task, got {other:?}"),
771 };
772
773 assert_eq!(result, Msg::Exhausted("timeout".to_owned()));
774 std::thread::sleep(Duration::from_millis(50));
775 assert_eq!(attempts.load(Ordering::SeqCst), 2);
776 assert_eq!(cancelled.load(Ordering::SeqCst), 2);
777 }
778
779 #[test]
780 fn backoff_strategy_variants_debug() {
781 let fixed = BackoffStrategy::Fixed { delay_ms: 100 };
782 let exp = BackoffStrategy::Exponential {
783 base_ms: 100,
784 max_ms: 5000,
785 };
786 let linear = BackoffStrategy::Linear {
787 base_ms: 100,
788 max_ms: 500,
789 };
790 let _ = format!("{fixed:?}");
792 let _ = format!("{exp:?}");
793 let _ = format!("{linear:?}");
794 }
795}