aion_server/worker/supervisor/
policy.rs

1//! The supervision policy, and the refusal owed when there is none.
2//!
3//! Every number here is the operator's (ADR-001). Nothing in this module
4//! invents a backoff, a ceiling, or a grace period: an unconfigured server
5//! supervises nothing and says exactly which keys would change that.
6
7use std::num::NonZeroU32;
8use std::time::Duration;
9
10/// The operator-configured restart discipline for managed workers.
11#[derive(Clone, Copy, Debug, Eq, PartialEq)]
12pub struct SupervisionPolicy {
13    /// Delay before the FIRST restart in a window.
14    pub restart_backoff_initial: Duration,
15    /// Ceiling the geometric backoff is clamped to.
16    pub restart_backoff_max: Duration,
17    /// Geometric growth factor applied per restart already made in the window.
18    pub restart_backoff_multiplier: NonZeroU32,
19    /// Sliding window over which restarts are counted.
20    pub restart_window: Duration,
21    /// Restarts permitted within one window before the instance is failed.
22    pub max_restarts_per_window: NonZeroU32,
23    /// Grace given to each of `SIGTERM` and `SIGKILL` when stopping a group.
24    pub stop_grace: Duration,
25}
26
27impl SupervisionPolicy {
28    /// The delay owed before a restart, given how many restarts already
29    /// happened inside the current window.
30    ///
31    /// `restarts_in_window` counts the restarts already made, so the first
32    /// restart of a window (`0` prior) waits exactly
33    /// `restart_backoff_initial`. Growth is geometric and saturating, and the
34    /// result is always clamped to `restart_backoff_max`.
35    #[must_use]
36    pub fn backoff_after(&self, restarts_in_window: u32) -> Duration {
37        let multiplier = self.restart_backoff_multiplier.get();
38        let mut delay = self.restart_backoff_initial;
39        for _ in 0..restarts_in_window {
40            delay = delay.saturating_mul(multiplier);
41            if delay >= self.restart_backoff_max {
42                return self.restart_backoff_max;
43            }
44        }
45        delay.min(self.restart_backoff_max)
46    }
47}
48
49/// The exact remedy an operator is owed when supervision is not configured.
50///
51/// Held as one constant because it is spoken by three surfaces (the start
52/// refusal, the status report, and the boot warning): a remedy that drifts
53/// between them names keys that may no longer exist.
54pub const UNCOMMISSIONED_REMEDY: &str = "managed-worker supervision is not configured on this \
55     server: add a `[worker_supervision]` section naming \
56     `restart_backoff_initial_ms`, `restart_backoff_max_ms`, \
57     `restart_backoff_multiplier`, `restart_window_ms`, \
58     `max_restarts_per_window`, and `stop_grace_ms` (there are no defaults — \
59     the restart discipline is an operator decision), then restart the server";
60
61#[cfg(test)]
62mod tests {
63    use std::num::NonZeroU32;
64    use std::time::Duration;
65
66    use super::SupervisionPolicy;
67
68    fn policy(initial_ms: u64, max_ms: u64, multiplier: u32) -> Option<SupervisionPolicy> {
69        Some(SupervisionPolicy {
70            restart_backoff_initial: Duration::from_millis(initial_ms),
71            restart_backoff_max: Duration::from_millis(max_ms),
72            restart_backoff_multiplier: NonZeroU32::new(multiplier)?,
73            restart_window: Duration::from_secs(60),
74            max_restarts_per_window: NonZeroU32::new(3)?,
75            stop_grace: Duration::from_secs(1),
76        })
77    }
78
79    #[test]
80    fn first_restart_waits_exactly_the_configured_initial_backoff() -> Result<(), &'static str> {
81        let policy = policy(100, 10_000, 2).ok_or("policy must build")?;
82        assert_eq!(policy.backoff_after(0), Duration::from_millis(100));
83        Ok(())
84    }
85
86    #[test]
87    fn backoff_grows_geometrically_then_clamps_to_the_ceiling() -> Result<(), &'static str> {
88        let policy = policy(100, 400, 2).ok_or("policy must build")?;
89        assert_eq!(policy.backoff_after(1), Duration::from_millis(200));
90        assert_eq!(policy.backoff_after(2), Duration::from_millis(400));
91        assert_eq!(policy.backoff_after(3), Duration::from_millis(400));
92        assert_eq!(policy.backoff_after(64), Duration::from_millis(400));
93        Ok(())
94    }
95
96    /// A multiplier of one is a flat retry cadence, not a growing one, and must
97    /// not be mistaken for "no backoff".
98    #[test]
99    fn a_multiplier_of_one_keeps_the_initial_delay_forever() -> Result<(), &'static str> {
100        let policy = policy(250, 10_000, 1).ok_or("policy must build")?;
101        assert_eq!(policy.backoff_after(0), Duration::from_millis(250));
102        assert_eq!(policy.backoff_after(9), Duration::from_millis(250));
103        Ok(())
104    }
105
106    /// The growth is saturating: a large multiplier over many restarts must
107    /// clamp rather than overflow the duration arithmetic.
108    #[test]
109    fn extreme_growth_saturates_into_the_ceiling() -> Result<(), &'static str> {
110        let policy = policy(1_000, 30_000, u32::MAX).ok_or("policy must build")?;
111        assert_eq!(policy.backoff_after(16), Duration::from_secs(30));
112        Ok(())
113    }
114}
aion_server/worker/supervisor/policy.rs

aion_server/worker/supervisor/
policy.rs
