par_term_scripting/
restart.rs

1//! Restart supervision for script subprocesses.
2//!
3//! `ScriptConfig::restart_policy` / `restart_delay_ms` are accepted by the
4//! config layer but the actual supervision happens in the frontend event loop,
5//! which is the only place that can re-register a terminal observer and re-spawn
6//! against a config index. This module owns the *decision* — a pure state
7//! machine with no I/O — so it stays deterministic and unit-testable.
8//!
9//! The orchestrator drives one [`ScriptRestartState`] per script slot:
10//!
11//! 1. On a fresh process start, call [`ScriptRestartState::on_started`].
12//! 2. Each frame, poll the process; if it has exited, call [`ScriptRestartState::on_exit`]
13//!    (only when not already pending) to get the first reaction.
14//! 3. While a restart is pending, call [`ScriptRestartState::poll`] to learn when
15//!    the delay has elapsed.
16
17use std::time::{Duration, Instant};
18
19use par_term_config::RestartPolicy;
20
21/// What the supervisor wants the orchestrator to do this frame for one slot.
22#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub enum RestartAction {
24    /// No transition observed and nothing pending — leave the slot alone.
25    Idle,
26    /// Tear the slot down: the policy says stop, or the attempt cap was hit.
27    Stop,
28    /// A pending restart's delay has elapsed — re-spawn now.
29    Restart,
30    /// A restart is scheduled but the delay has not yet elapsed.
31    Wait,
32}
33
34/// Maximum number of consecutive restart attempts before the supervisor gives up.
35///
36/// Consecutive failures are counted only within the [`RESTART_GRACE`] window: a
37/// process that runs past grace resets the counter, so a slow leak (e.g. crashes
38/// once an hour) never exhausts the budget. Only a tight crash-loop does.
39pub const MAX_RESTART_ATTEMPTS: u32 = 5;
40
41/// A process that survives this long before exiting resets the failure counter.
42pub const RESTART_GRACE: Duration = Duration::from_secs(5);
43
44/// Pure per-slot restart supervisor.
45#[derive(Debug, Clone)]
46pub struct ScriptRestartState {
47    policy: RestartPolicy,
48    delay: Duration,
49    /// When the current process was (re)started; `None` until the first start.
50    started_at: Option<Instant>,
51    /// When a pending restart should fire; `None` when the slot is idle.
52    deadline: Option<Instant>,
53    /// Consecutive restarts that re-exited inside the grace window.
54    consecutive_failures: u32,
55}
56
57impl ScriptRestartState {
58    /// Build a supervisor for a slot configured with `policy` and `delay_ms`.
59    pub fn new(policy: RestartPolicy, delay_ms: u64) -> Self {
60        Self {
61            policy,
62            delay: Duration::from_millis(delay_ms),
63            started_at: None,
64            deadline: None,
65            consecutive_failures: 0,
66        }
67    }
68
69    /// Apply a config edit without losing pending timing beyond the current cycle.
70    pub fn reconfigure(&mut self, policy: RestartPolicy, delay_ms: u64) {
71        self.policy = policy;
72        self.delay = Duration::from_millis(delay_ms);
73    }
74
75    /// Record that a (re)start just happened. Begins a fresh grace window.
76    pub fn on_started(&mut self, now: Instant) {
77        self.started_at = Some(now);
78        self.deadline = None;
79    }
80
81    /// React to a process exit. Only call when [`Self::pending`] is false, so
82    /// this fires once per exit rather than every sticky `Exited` poll.
83    pub fn on_exit(&mut self, now: Instant, success: bool) -> RestartAction {
84        // Reset the failure counter if the process ran past the grace window —
85        // a slow leak must not exhaust the crash-loop budget.
86        if let Some(start) = self.started_at
87            && now.duration_since(start) >= RESTART_GRACE
88        {
89            self.consecutive_failures = 0;
90        }
91        let should_restart = match self.policy {
92            RestartPolicy::Never => false,
93            RestartPolicy::Always => true,
94            RestartPolicy::OnFailure => !success,
95        };
96        if !should_restart {
97            self.deadline = None;
98            return RestartAction::Stop;
99        }
100        self.schedule(now)
101    }
102
103    /// Check whether a pending restart's deadline has fired. Call each frame
104    /// while [`Self::pending`].
105    pub fn poll(&mut self, now: Instant) -> RestartAction {
106        match self.deadline {
107            Some(deadline) if now >= deadline => {
108                self.deadline = None;
109                RestartAction::Restart
110            }
111            Some(_) => RestartAction::Wait,
112            None => RestartAction::Idle,
113        }
114    }
115
116    /// A restart is awaiting its delay window.
117    pub fn pending(&self) -> bool {
118        self.deadline.is_some()
119    }
120
121    /// Re-arm after a restart *attempt* failed to spawn (distinct from a real
122    /// exit — no process ever ran). Counts toward the attempt cap.
123    pub fn reschedule(&mut self, now: Instant) -> RestartAction {
124        self.schedule(now)
125    }
126
127    /// Consecutive failures accrued inside the grace window (for UI/debug).
128    pub fn consecutive_failures(&self) -> u32 {
129        self.consecutive_failures
130    }
131
132    fn schedule(&mut self, now: Instant) -> RestartAction {
133        if self.consecutive_failures >= MAX_RESTART_ATTEMPTS {
134            self.deadline = None;
135            return RestartAction::Stop;
136        }
137        self.consecutive_failures = self.consecutive_failures.saturating_add(1);
138        self.deadline = Some(now + self.delay);
139        RestartAction::Wait
140    }
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146
147    fn anchor() -> Instant {
148        Instant::now()
149    }
150
151    #[test]
152    fn never_policy_stops_on_clean_exit() {
153        let t0 = anchor();
154        let mut s = ScriptRestartState::new(RestartPolicy::Never, 0);
155        s.on_started(t0);
156        assert_eq!(s.on_exit(t0, true), RestartAction::Stop);
157        assert!(!s.pending());
158    }
159
160    #[test]
161    fn never_policy_stops_on_failure_too() {
162        let t0 = anchor();
163        let mut s = ScriptRestartState::new(RestartPolicy::Never, 0);
164        s.on_started(t0);
165        assert_eq!(s.on_exit(t0, false), RestartAction::Stop);
166    }
167
168    #[test]
169    fn always_policy_waits_for_delay_then_restarts() {
170        let t0 = anchor();
171        let mut s = ScriptRestartState::new(RestartPolicy::Always, 100);
172        s.on_started(t0);
173        assert_eq!(s.on_exit(t0, true), RestartAction::Wait);
174        assert!(s.pending());
175        assert_eq!(s.poll(t0 + Duration::from_millis(99)), RestartAction::Wait);
176        assert_eq!(
177            s.poll(t0 + Duration::from_millis(100)),
178            RestartAction::Restart
179        );
180        assert!(!s.pending());
181    }
182
183    #[test]
184    fn always_policy_restarts_immediately_when_delay_zero() {
185        let t0 = anchor();
186        let mut s = ScriptRestartState::new(RestartPolicy::Always, 0);
187        s.on_started(t0);
188        assert_eq!(s.on_exit(t0, true), RestartAction::Wait);
189        assert_eq!(s.poll(t0), RestartAction::Restart);
190    }
191
192    #[test]
193    fn on_failure_stops_on_clean_exit() {
194        let t0 = anchor();
195        let mut s = ScriptRestartState::new(RestartPolicy::OnFailure, 100);
196        s.on_started(t0);
197        assert_eq!(s.on_exit(t0, true), RestartAction::Stop);
198        assert!(!s.pending());
199    }
200
201    #[test]
202    fn on_failure_restarts_on_failure_after_delay() {
203        let t0 = anchor();
204        let mut s = ScriptRestartState::new(RestartPolicy::OnFailure, 50);
205        s.on_started(t0);
206        assert_eq!(s.on_exit(t0, false), RestartAction::Wait);
207        assert_eq!(s.poll(t0 + Duration::from_millis(49)), RestartAction::Wait);
208        assert_eq!(
209            s.poll(t0 + Duration::from_millis(50)),
210            RestartAction::Restart
211        );
212    }
213
214    #[test]
215    fn idle_when_nothing_has_happened() {
216        let mut s = ScriptRestartState::new(RestartPolicy::Always, 100);
217        s.on_started(anchor());
218        assert_eq!(s.poll(anchor()), RestartAction::Idle);
219        assert!(!s.pending());
220    }
221
222    #[test]
223    fn gives_up_after_max_attempts_within_grace_window() {
224        let t0 = anchor();
225        let mut s = ScriptRestartState::new(RestartPolicy::Always, 10);
226        for _ in 0..MAX_RESTART_ATTEMPTS {
227            s.on_started(t0);
228            assert_eq!(s.on_exit(t0, false), RestartAction::Wait);
229            assert_eq!(
230                s.poll(t0 + Duration::from_millis(10)),
231                RestartAction::Restart
232            );
233        }
234        s.on_started(t0);
235        assert_eq!(s.on_exit(t0, false), RestartAction::Stop);
236    }
237
238    #[test]
239    fn long_lived_process_resets_failure_counter() {
240        let t0 = anchor();
241        let mut s = ScriptRestartState::new(RestartPolicy::Always, 10);
242        s.on_started(t0);
243        s.on_exit(t0, false);
244        assert_eq!(s.consecutive_failures(), 1);
245        s.poll(t0 + Duration::from_millis(10));
246        let restart_at = t0 + Duration::from_millis(20);
247        s.on_started(restart_at);
248        let after_grace = restart_at + RESTART_GRACE + Duration::from_millis(1);
249        s.on_exit(after_grace, false);
250        assert_eq!(s.consecutive_failures(), 1);
251    }
252
253    #[test]
254    fn reschedule_after_spawn_failure_counts_toward_cap() {
255        let t0 = anchor();
256        let mut s = ScriptRestartState::new(RestartPolicy::Always, 10);
257        s.on_started(t0);
258        s.on_exit(t0, false); // failures = 1
259        s.poll(t0 + Duration::from_millis(10)); // fire restart
260        assert_eq!(
261            s.reschedule(t0 + Duration::from_millis(11)),
262            RestartAction::Wait
263        );
264        assert_eq!(s.consecutive_failures(), 2);
265    }
266
267    #[test]
268    fn reconfigure_swaps_policy_and_delay() {
269        let t0 = anchor();
270        let mut s = ScriptRestartState::new(RestartPolicy::Never, 0);
271        s.reconfigure(RestartPolicy::Always, 200);
272        s.on_started(t0);
273        assert_eq!(s.on_exit(t0, true), RestartAction::Wait);
274        assert_eq!(
275            s.poll(t0 + Duration::from_millis(200)),
276            RestartAction::Restart
277        );
278    }
279}
par_term_scripting/restart.rs

par_term_scripting/
restart.rs
