chio_supervisor/
config.rs

1//! Supervisor policy: restart budget, backoff schedule, and the per-iteration
2//! outcome the worker body reports back to the loop.
3
4use crate::time::as_millis_u64;
5use std::time::Duration;
6
7/// Restart and trip policy for one supervised worker.
8#[derive(Debug, Clone)]
9pub struct SupervisorConfig {
10    /// Stable identifier used in log lines, thread names, and health reasons.
11    pub name: &'static str,
12    /// A tripped flag on a TCB-critical surface fails evaluations closed.
13    pub tcb_critical: bool,
14    /// Consecutive restarts before the flag trips to `Degraded`.
15    pub trip_after: u32,
16    /// Consecutive restarts before terminal `Failed`, after which the supervisor
17    /// stops respawning and leaves the flag set for surfaces to read.
18    pub max_restarts: u32,
19    /// First backoff delay; subsequent delays double up to `max_backoff`.
20    pub base_backoff: Duration,
21    /// Ceiling on the exponential backoff delay.
22    pub max_backoff: Duration,
23}
24
25impl SupervisorConfig {
26    /// A conventional policy: trip to degraded after a few consecutive failures,
27    /// give up after a larger budget, exponential backoff bounded at a minute.
28    #[must_use]
29    pub fn new(name: &'static str, tcb_critical: bool) -> Self {
30        Self {
31            name,
32            tcb_critical,
33            trip_after: 3,
34            max_restarts: 10,
35            base_backoff: Duration::from_millis(100),
36            max_backoff: Duration::from_secs(60),
37        }
38    }
39}
40
41/// Result of one worker-body invocation, reported back to the supervisor loop.
42pub enum SupervisedOutcome {
43    /// The worker observed the shutdown signal and exited cleanly. Do not restart.
44    Shutdown,
45    /// One iteration completed successfully (a one-shot tick such as a single cluster
46    /// sync round). Loop again without recording a failure and without backing off.
47    /// The worker is expected to have already stamped its own healthy heartbeat via
48    /// [`crate::HealthFlag::record_ok`], so a healthy tick is never misclassified as a
49    /// restart-worthy failure.
50    Continue,
51    /// The worker returned or failed and should be restarted with backoff.
52    Restart,
53}
54
55/// Exponential backoff for the `count`-th consecutive restart:
56/// `min(max_backoff, base_backoff * 2^(count - 1))`, computed with saturating
57/// arithmetic so a large `count` can never overflow or panic.
58#[must_use]
59pub(crate) fn backoff_delay(config: &SupervisorConfig, count: u32) -> Duration {
60    let shift = count.saturating_sub(1);
61    let factor = 1u64.checked_shl(shift).unwrap_or(u64::MAX);
62    let base_ms = as_millis_u64(config.base_backoff);
63    let uncapped = base_ms.saturating_mul(factor);
64    let capped = uncapped.min(as_millis_u64(config.max_backoff));
65    Duration::from_millis(capped)
66}
67
68#[cfg(all(test, not(loom)))]
69mod tests {
70    use super::*;
71
72    fn config() -> SupervisorConfig {
73        SupervisorConfig {
74            name: "test",
75            tcb_critical: false,
76            trip_after: 3,
77            max_restarts: 5,
78            base_backoff: Duration::from_millis(100),
79            max_backoff: Duration::from_secs(10),
80        }
81    }
82
83    #[test]
84    fn backoff_doubles_from_base() {
85        let cfg = config();
86        assert_eq!(backoff_delay(&cfg, 1), Duration::from_millis(100));
87        assert_eq!(backoff_delay(&cfg, 2), Duration::from_millis(200));
88        assert_eq!(backoff_delay(&cfg, 3), Duration::from_millis(400));
89    }
90
91    #[test]
92    fn backoff_saturates_at_max_and_never_overflows() {
93        let cfg = config();
94        assert_eq!(backoff_delay(&cfg, 100), cfg.max_backoff);
95        // A count large enough to overflow the shift must still saturate cleanly.
96        assert_eq!(backoff_delay(&cfg, u32::MAX), cfg.max_backoff);
97    }
98
99    #[test]
100    fn default_policy_is_sane() {
101        let cfg = SupervisorConfig::new("writer", true);
102        assert!(cfg.tcb_critical);
103        assert!(cfg.trip_after <= cfg.max_restarts);
104        assert!(cfg.base_backoff <= cfg.max_backoff);
105    }
106}
chio_supervisor/config.rs

chio_supervisor/
config.rs
