par_term_scripting/
restart.rs

1//! Restart supervision for script subprocesses.
2//!
3//! `ScriptConfig::restart_policy` / `restart_delay_ms` are accepted by the
4//! config layer but the actual supervision happens in the frontend event loop,
5//! which is the only place that can re-register a terminal observer and re-spawn
6//! against a config index. This module owns the *decision* — a pure state
7//! machine with no I/O — so it stays deterministic and unit-testable.
8//!
9//! The orchestrator drives one [`ScriptRestartState`] per script slot:
10//!
11//! 1. On a fresh process start, call [`ScriptRestartState::on_started`].
12//! 2. Each frame, poll the process; if it has exited, call [`ScriptRestartState::on_exit`]
13//!    (only when not already pending) to get the first reaction.
14//! 3. While a restart is pending, call [`ScriptRestartState::poll`] to learn when
15//!    the delay has elapsed.
16
17use std::time::{Duration, Instant};
18
19use par_term_config::RestartPolicy;
20
21/// What the supervisor wants the orchestrator to do this frame for one slot.
22#[derive(Debug, Clone, Copy, PartialEq, Eq)]
23pub enum RestartAction {
24    /// No transition observed and nothing pending — leave the slot alone.
25    Idle,
26    /// Tear the slot down: the policy says stop, or the attempt cap was hit.
27    Stop,
28    /// A pending restart's delay has elapsed — re-spawn now.
29    Restart,
30    /// A restart is scheduled but the delay has not yet elapsed.
31    Wait,
32}
33
34/// Maximum number of consecutive restart attempts before the supervisor gives up.
35///
36/// Consecutive failures are counted only within the [`RESTART_GRACE`] window: a
37/// process that runs past grace resets the counter, so a slow leak (e.g. crashes
38/// once an hour) never exhausts the budget. Only a tight crash-loop does.
39pub const MAX_RESTART_ATTEMPTS: u32 = 5;
40
41/// A process that survives this long before exiting resets the failure counter.
42pub const RESTART_GRACE: Duration = Duration::from_secs(5);
43
44/// Pure per-slot restart supervisor.
45#[derive(Debug, Clone)]
46pub struct ScriptRestartState {
47    policy: RestartPolicy,
48    delay: Duration,
49    /// When the current process was (re)started; `None` until the first start.
50    started_at: Option<Instant>,
51    /// When a pending restart should fire; `None` when the slot is idle.
52    deadline: Option<Instant>,
53    /// Consecutive restarts that re-exited inside the grace window.
54    consecutive_failures: u32,
55}
56
57impl ScriptRestartState {
58    /// Build a supervisor for a slot configured with `policy` and `delay_ms`.
59    pub fn new(policy: RestartPolicy, delay_ms: u64) -> Self {
60        Self {
61            policy,
62            delay: Duration::from_millis(delay_ms),
63            started_at: None,
64            deadline: None,
65            consecutive_failures: 0,
66        }
67    }
68
69    /// Apply a config edit without losing pending timing beyond the current cycle.
70    pub fn reconfigure(&mut self, policy: RestartPolicy, delay_ms: u64) {
71        self.policy = policy;
72        self.delay = Duration::from_millis(delay_ms);
73    }
74
75    /// Record that a (re)start just happened. Begins a fresh grace window.
76    pub fn on_started(&mut self, now: Instant) {
77        self.started_at = Some(now);
78        self.deadline = None;
79    }
80
81    /// React to a process exit. Only call when [`Self::pending`] is false, so
82    /// this fires once per exit rather than every sticky `Exited` poll.
83    pub fn on_exit(&mut self, now: Instant, success: bool) -> RestartAction {
84        // Reset the failure counter if the process ran past the grace window —
85        // a slow leak must not exhaust the crash-loop budget.
86        if let Some(start) = self.started_at
87            && now.duration_since(start) >= RESTART_GRACE
88        {
89            self.consecutive_failures = 0;
90        }
91        let should_restart = match self.policy {
92            RestartPolicy::Never => false,
93            RestartPolicy::Always => true,
94            RestartPolicy::OnFailure => !success,
95        };
96        if !should_restart {
97            self.deadline = None;
98            return RestartAction::Stop;
99        }
100        self.schedule(now)
101    }
102
103    /// Check whether a pending restart's deadline has fired. Call each frame
104    /// while [`Self::pending`].
105    pub fn poll(&mut self, now: Instant) -> RestartAction {
106        match self.deadline {
107            Some(deadline) if now >= deadline => {
108                self.deadline = None;
109                RestartAction::Restart
110            }
111            Some(_) => RestartAction::Wait,
112            None => RestartAction::Idle,
113        }
114    }
115
116    /// A restart is awaiting its delay window.
117    pub fn pending(&self) -> bool {
118        self.deadline.is_some()
119    }
120
121    /// Re-arm after a restart *attempt* failed to spawn (distinct from a real
122    /// exit — no process ever ran). Counts toward the attempt cap.
123    pub fn reschedule(&mut self, now: Instant) -> RestartAction {
124        self.schedule(now)
125    }
126
127    /// Consecutive failures accrued inside the grace window (for UI/debug).
128    pub fn consecutive_failures(&self) -> u32 {
129        self.consecutive_failures
130    }
131
132    /// The configured policy (for diagnostics surfaced on a crash-cap).
133    pub fn policy(&self) -> RestartPolicy {
134        self.policy
135    }
136
137    fn schedule(&mut self, now: Instant) -> RestartAction {
138        if self.consecutive_failures >= MAX_RESTART_ATTEMPTS {
139            self.deadline = None;
140            return RestartAction::Stop;
141        }
142        self.consecutive_failures = self.consecutive_failures.saturating_add(1);
143        self.deadline = Some(now + self.delay);
144        RestartAction::Wait
145    }
146}
147
148#[cfg(test)]
149mod tests {
150    use super::*;
151
152    fn anchor() -> Instant {
153        Instant::now()
154    }
155
156    #[test]
157    fn never_policy_stops_on_clean_exit() {
158        let t0 = anchor();
159        let mut s = ScriptRestartState::new(RestartPolicy::Never, 0);
160        s.on_started(t0);
161        assert_eq!(s.on_exit(t0, true), RestartAction::Stop);
162        assert!(!s.pending());
163    }
164
165    #[test]
166    fn never_policy_stops_on_failure_too() {
167        let t0 = anchor();
168        let mut s = ScriptRestartState::new(RestartPolicy::Never, 0);
169        s.on_started(t0);
170        assert_eq!(s.on_exit(t0, false), RestartAction::Stop);
171    }
172
173    #[test]
174    fn always_policy_waits_for_delay_then_restarts() {
175        let t0 = anchor();
176        let mut s = ScriptRestartState::new(RestartPolicy::Always, 100);
177        s.on_started(t0);
178        assert_eq!(s.on_exit(t0, true), RestartAction::Wait);
179        assert!(s.pending());
180        assert_eq!(s.poll(t0 + Duration::from_millis(99)), RestartAction::Wait);
181        assert_eq!(
182            s.poll(t0 + Duration::from_millis(100)),
183            RestartAction::Restart
184        );
185        assert!(!s.pending());
186    }
187
188    #[test]
189    fn always_policy_restarts_immediately_when_delay_zero() {
190        let t0 = anchor();
191        let mut s = ScriptRestartState::new(RestartPolicy::Always, 0);
192        s.on_started(t0);
193        assert_eq!(s.on_exit(t0, true), RestartAction::Wait);
194        assert_eq!(s.poll(t0), RestartAction::Restart);
195    }
196
197    #[test]
198    fn on_failure_stops_on_clean_exit() {
199        let t0 = anchor();
200        let mut s = ScriptRestartState::new(RestartPolicy::OnFailure, 100);
201        s.on_started(t0);
202        assert_eq!(s.on_exit(t0, true), RestartAction::Stop);
203        assert!(!s.pending());
204    }
205
206    #[test]
207    fn on_failure_restarts_on_failure_after_delay() {
208        let t0 = anchor();
209        let mut s = ScriptRestartState::new(RestartPolicy::OnFailure, 50);
210        s.on_started(t0);
211        assert_eq!(s.on_exit(t0, false), RestartAction::Wait);
212        assert_eq!(s.poll(t0 + Duration::from_millis(49)), RestartAction::Wait);
213        assert_eq!(
214            s.poll(t0 + Duration::from_millis(50)),
215            RestartAction::Restart
216        );
217    }
218
219    #[test]
220    fn idle_when_nothing_has_happened() {
221        let mut s = ScriptRestartState::new(RestartPolicy::Always, 100);
222        s.on_started(anchor());
223        assert_eq!(s.poll(anchor()), RestartAction::Idle);
224        assert!(!s.pending());
225    }
226
227    #[test]
228    fn gives_up_after_max_attempts_within_grace_window() {
229        let t0 = anchor();
230        let mut s = ScriptRestartState::new(RestartPolicy::Always, 10);
231        for _ in 0..MAX_RESTART_ATTEMPTS {
232            s.on_started(t0);
233            assert_eq!(s.on_exit(t0, false), RestartAction::Wait);
234            assert_eq!(
235                s.poll(t0 + Duration::from_millis(10)),
236                RestartAction::Restart
237            );
238        }
239        s.on_started(t0);
240        assert_eq!(s.on_exit(t0, false), RestartAction::Stop);
241    }
242
243    #[test]
244    fn long_lived_process_resets_failure_counter() {
245        let t0 = anchor();
246        let mut s = ScriptRestartState::new(RestartPolicy::Always, 10);
247        s.on_started(t0);
248        s.on_exit(t0, false);
249        assert_eq!(s.consecutive_failures(), 1);
250        s.poll(t0 + Duration::from_millis(10));
251        let restart_at = t0 + Duration::from_millis(20);
252        s.on_started(restart_at);
253        let after_grace = restart_at + RESTART_GRACE + Duration::from_millis(1);
254        s.on_exit(after_grace, false);
255        assert_eq!(s.consecutive_failures(), 1);
256    }
257
258    #[test]
259    fn reschedule_after_spawn_failure_counts_toward_cap() {
260        let t0 = anchor();
261        let mut s = ScriptRestartState::new(RestartPolicy::Always, 10);
262        s.on_started(t0);
263        s.on_exit(t0, false); // failures = 1
264        s.poll(t0 + Duration::from_millis(10)); // fire restart
265        assert_eq!(
266            s.reschedule(t0 + Duration::from_millis(11)),
267            RestartAction::Wait
268        );
269        assert_eq!(s.consecutive_failures(), 2);
270    }
271
272    #[test]
273    fn reconfigure_swaps_policy_and_delay() {
274        let t0 = anchor();
275        let mut s = ScriptRestartState::new(RestartPolicy::Never, 0);
276        s.reconfigure(RestartPolicy::Always, 200);
277        s.on_started(t0);
278        assert_eq!(s.on_exit(t0, true), RestartAction::Wait);
279        assert_eq!(
280            s.poll(t0 + Duration::from_millis(200)),
281            RestartAction::Restart
282        );
283    }
284}
par_term_scripting/restart.rs

par_term_scripting/
restart.rs
