aion_server/config/
worker_supervision.rs

1//! The `[worker_supervision]` section: the operator's restart discipline.
2//!
3//! There is no default policy and there will not be one. A backoff, a restart
4//! budget, and a kill grace are operational decisions with real consequences —
5//! a guessed backoff turns a dependency outage into a thundering herd, and a
6//! guessed grace turns a slow shutdown into a data-losing `SIGKILL`. So the
7//! section is all-or-nothing: write every key and the server supervises;
8//! write none and it supervises nothing, loudly (ADR-001).
9//!
10//! A PARTIAL section is the one thing that is neither: it is refused at load
11//! with every missing key named, because silently defaulting the rest is
12//! exactly the guess this rule exists to prevent.
13
14use std::num::NonZeroU32;
15use std::time::Duration;
16
17use serde::Deserialize;
18
19use crate::error::ServerError;
20use crate::worker::supervisor::SupervisionPolicy;
21
22use super::config_error;
23
24/// Managed-worker supervision settings from `[worker_supervision]`.
25#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
26#[serde(default, deny_unknown_fields)]
27pub struct WorkerSupervisionConfig {
28    /// Delay before the first restart in a window, milliseconds.
29    pub restart_backoff_initial_ms: Option<u64>,
30    /// Ceiling the geometric backoff is clamped to, milliseconds.
31    pub restart_backoff_max_ms: Option<u64>,
32    /// Geometric growth factor per restart already made in the window.
33    pub restart_backoff_multiplier: Option<u32>,
34    /// Sliding window over which restarts are counted, milliseconds.
35    pub restart_window_ms: Option<u64>,
36    /// Restarts permitted within one window before an instance is failed.
37    pub max_restarts_per_window: Option<u32>,
38    /// Grace given to each of `SIGTERM` and `SIGKILL` when stopping a managed
39    /// worker's process group, milliseconds.
40    pub stop_grace_ms: Option<u64>,
41}
42
43/// Every key the section needs, in the order an operator writes them.
44const REQUIRED_KEYS: &[&str] = &[
45    "restart_backoff_initial_ms",
46    "restart_backoff_max_ms",
47    "restart_backoff_multiplier",
48    "restart_window_ms",
49    "max_restarts_per_window",
50    "stop_grace_ms",
51];
52
53impl WorkerSupervisionConfig {
54    /// Which required keys this section is missing.
55    fn missing(self) -> Vec<&'static str> {
56        let present = [
57            self.restart_backoff_initial_ms.is_some(),
58            self.restart_backoff_max_ms.is_some(),
59            self.restart_backoff_multiplier.is_some(),
60            self.restart_window_ms.is_some(),
61            self.max_restarts_per_window.is_some(),
62            self.stop_grace_ms.is_some(),
63        ];
64        REQUIRED_KEYS
65            .iter()
66            .zip(present)
67            .filter_map(|(key, present)| (!present).then_some(*key))
68            .collect()
69    }
70
71    /// Resolve the section into a policy.
72    ///
73    /// `Ok(None)` is the honest reading of an ENTIRELY absent section: the
74    /// operator has not commissioned supervision, and the managed-worker
75    /// surface refuses with the remedy rather than inventing numbers.
76    ///
77    /// # Errors
78    ///
79    /// Returns [`ServerError::Config`] naming every missing key for a PARTIAL
80    /// section, and naming the offending key for a value that cannot express a
81    /// working discipline (a zero interval, a zero budget, a shrinking
82    /// backoff, or a ceiling below the initial delay).
83    pub fn resolve(self) -> Result<Option<SupervisionPolicy>, ServerError> {
84        let missing = self.missing();
85        if missing.len() == REQUIRED_KEYS.len() {
86            return Ok(None);
87        }
88        if !missing.is_empty() {
89            return config_error(format!(
90                "[worker_supervision] is incomplete: it is missing {} — the section has no \
91                 defaults, so write every key or remove the section entirely",
92                missing.join(", ")
93            ));
94        }
95
96        let initial = Self::required_duration(self.restart_backoff_initial_ms, REQUIRED_KEYS[0])?;
97        let max = Self::required_duration(self.restart_backoff_max_ms, REQUIRED_KEYS[1])?;
98        let multiplier = Self::required_factor(self.restart_backoff_multiplier, REQUIRED_KEYS[2])?;
99        let window = Self::required_duration(self.restart_window_ms, REQUIRED_KEYS[3])?;
100        let budget = Self::required_factor(self.max_restarts_per_window, REQUIRED_KEYS[4])?;
101        let grace = Self::required_duration(self.stop_grace_ms, REQUIRED_KEYS[5])?;
102
103        if max < initial {
104            return config_error(format!(
105                "worker_supervision.restart_backoff_max_ms ({}) must be at least \
106                 restart_backoff_initial_ms ({})",
107                max.as_millis(),
108                initial.as_millis()
109            ));
110        }
111
112        Ok(Some(SupervisionPolicy {
113            restart_backoff_initial: initial,
114            restart_backoff_max: max,
115            restart_backoff_multiplier: multiplier,
116            restart_window: window,
117            max_restarts_per_window: budget,
118            stop_grace: grace,
119        }))
120    }
121
122    fn required_duration(value: Option<u64>, key: &'static str) -> Result<Duration, ServerError> {
123        match value {
124            Some(millis) if millis > 0 => Ok(Duration::from_millis(millis)),
125            _ => config_error(format!(
126                "worker_supervision.{key} must be greater than zero milliseconds"
127            )),
128        }
129    }
130
131    fn required_factor(value: Option<u32>, key: &'static str) -> Result<NonZeroU32, ServerError> {
132        value.and_then(NonZeroU32::new).map_or_else(
133            || {
134                config_error(format!(
135                    "worker_supervision.{key} must be greater than zero"
136                ))
137            },
138            Ok,
139        )
140    }
141}
142
143#[cfg(test)]
144mod tests {
145    use super::{REQUIRED_KEYS, WorkerSupervisionConfig};
146
147    fn complete() -> WorkerSupervisionConfig {
148        WorkerSupervisionConfig {
149            restart_backoff_initial_ms: Some(100),
150            restart_backoff_max_ms: Some(5_000),
151            restart_backoff_multiplier: Some(2),
152            restart_window_ms: Some(60_000),
153            max_restarts_per_window: Some(5),
154            stop_grace_ms: Some(2_000),
155        }
156    }
157
158    #[test]
159    fn an_absent_section_commissions_nothing_and_is_not_an_error()
160    -> Result<(), Box<dyn std::error::Error>> {
161        assert_eq!(WorkerSupervisionConfig::default().resolve()?, None);
162        Ok(())
163    }
164
165    #[test]
166    fn a_complete_section_resolves_every_value() -> Result<(), Box<dyn std::error::Error>> {
167        let policy = complete()
168            .resolve()?
169            .ok_or("a complete section must resolve")?;
170        assert_eq!(policy.restart_backoff_initial.as_millis(), 100);
171        assert_eq!(policy.restart_backoff_max.as_millis(), 5_000);
172        assert_eq!(policy.restart_backoff_multiplier.get(), 2);
173        assert_eq!(policy.restart_window.as_millis(), 60_000);
174        assert_eq!(policy.max_restarts_per_window.get(), 5);
175        assert_eq!(policy.stop_grace.as_millis(), 2_000);
176        Ok(())
177    }
178
179    /// A partial section is the dangerous case: defaulting the rest would
180    /// silently choose a restart discipline nobody wrote. Every missing key
181    /// must be named, one refusal, so the operator fixes it in one pass.
182    #[test]
183    fn a_partial_section_is_refused_and_names_every_missing_key()
184    -> Result<(), Box<dyn std::error::Error>> {
185        let partial = WorkerSupervisionConfig {
186            restart_backoff_initial_ms: Some(100),
187            ..WorkerSupervisionConfig::default()
188        };
189        let error = partial
190            .resolve()
191            .err()
192            .ok_or("a partial section must be refused")?
193            .to_string();
194        for key in &REQUIRED_KEYS[1..] {
195            assert!(
196                error.contains(key),
197                "the refusal must name `{key}`: {error}"
198            );
199        }
200        assert!(
201            !error.contains(REQUIRED_KEYS[0]),
202            "the refusal must not name a key that IS present: {error}"
203        );
204        Ok(())
205    }
206
207    #[test]
208    fn a_zero_interval_is_refused_by_name() -> Result<(), Box<dyn std::error::Error>> {
209        for (mutate, key) in [
210            (
211                (|config: &mut WorkerSupervisionConfig| config.restart_backoff_initial_ms = Some(0))
212                    as fn(&mut WorkerSupervisionConfig),
213                "restart_backoff_initial_ms",
214            ),
215            (
216                |config: &mut WorkerSupervisionConfig| config.restart_window_ms = Some(0),
217                "restart_window_ms",
218            ),
219            (
220                |config: &mut WorkerSupervisionConfig| config.stop_grace_ms = Some(0),
221                "stop_grace_ms",
222            ),
223            (
224                |config: &mut WorkerSupervisionConfig| config.restart_backoff_multiplier = Some(0),
225                "restart_backoff_multiplier",
226            ),
227            (
228                |config: &mut WorkerSupervisionConfig| config.max_restarts_per_window = Some(0),
229                "max_restarts_per_window",
230            ),
231        ] {
232            let mut config = complete();
233            mutate(&mut config);
234            let error = config
235                .resolve()
236                .err()
237                .ok_or("a zero value must be refused")?
238                .to_string();
239            assert!(
240                error.contains(key),
241                "the refusal must name `{key}`: {error}"
242            );
243        }
244        Ok(())
245    }
246
247    #[test]
248    fn a_ceiling_below_the_initial_backoff_is_refused() -> Result<(), Box<dyn std::error::Error>> {
249        let mut config = complete();
250        config.restart_backoff_max_ms = Some(10);
251        let error = config
252            .resolve()
253            .err()
254            .ok_or("a ceiling below the initial delay must be refused")?
255            .to_string();
256        assert!(error.contains("restart_backoff_max_ms"));
257        assert!(error.contains("restart_backoff_initial_ms"));
258        Ok(())
259    }
260}
aion_server/config/worker_supervision.rs

aion_server/config/
worker_supervision.rs
