runner_manager_platform/wsl/
recovery.rs

1//! Fail-closed decision model for managed-WSL recovery.
2//!
3//! An unreachable guest is not evidence that it is idle.  This module keeps
4//! the destructive decision separate from probing so every missing or stale
5//! fact has one conservative answer and can be property-tested without WSL.
6
7use std::time::Duration;
8
9/// Evidence collected by the Windows-side supervisor for one distribution.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct RecoveryEvidence {
12    pub consecutive_probe_failures: u8,
13    pub failure_span: Option<Duration>,
14    pub failure_is_recoverable: bool,
15    pub drain_generation: u64,
16    pub acknowledged_generation: Option<u64>,
17    pub heartbeat_age: Option<Duration>,
18    pub local_active_attempts: Option<u32>,
19    pub consecutive_zero_attempt_heartbeats: u8,
20    pub managed_busy_runners: Option<u32>,
21    pub managed_online_registrations: Option<u32>,
22    pub consecutive_zero_inventory_reads: u8,
23    pub unmanaged_runner_services: Option<u32>,
24    pub task_is_product_owned: bool,
25    pub distribution_is_wsl2: bool,
26    pub inventory_authorized: bool,
27    pub recovery_fence_held: bool,
28    pub circuit_open: bool,
29}
30
31/// The only outcomes the watchdog may act on.
32#[derive(Debug, Clone, PartialEq, Eq)]
33pub enum RecoveryDecision {
34    Observe,
35    RequestDrain,
36    TerminateNamed,
37    Blocked(&'static str),
38}
39
40pub const FAILURE_THRESHOLD: u8 = 3;
41pub const MINIMUM_FAILURE_SPAN: Duration = Duration::from_secs(5 * 60);
42pub const MAX_HEARTBEAT_AGE: Duration = Duration::from_secs(30);
43
44/// Decide without optimistic defaults. `TerminateNamed` means every required
45/// proof is present, current and mutually consistent; `None` always blocks.
46#[must_use]
47pub fn decide(evidence: &RecoveryEvidence) -> RecoveryDecision {
48    if evidence.circuit_open {
49        return RecoveryDecision::Blocked("the recovery circuit is open");
50    }
51    if evidence.consecutive_probe_failures < FAILURE_THRESHOLD {
52        return RecoveryDecision::Observe;
53    }
54    if !evidence.failure_is_recoverable {
55        return RecoveryDecision::Blocked("the failure is not a recoverable WSL transport failure");
56    }
57    let Some(failure_span) = evidence.failure_span else {
58        return RecoveryDecision::Blocked("the probe failure window is unknown");
59    };
60    if failure_span < MINIMUM_FAILURE_SPAN {
61        return RecoveryDecision::Observe;
62    }
63    if !evidence.task_is_product_owned {
64        return RecoveryDecision::Blocked("the lifecycle task is not product-owned");
65    }
66    if !evidence.distribution_is_wsl2 {
67        return RecoveryDecision::Blocked("the distribution is not verified as WSL2");
68    }
69    if !evidence.inventory_authorized {
70        return RecoveryDecision::Blocked("GitHub runner inventory is not authorized");
71    }
72    if !evidence.recovery_fence_held {
73        return RecoveryDecision::Blocked("the recovery fence is not held");
74    }
75    let Some(unmanaged) = evidence.unmanaged_runner_services else {
76        return RecoveryDecision::Blocked("the unmanaged-runner audit is unknown");
77    };
78    if unmanaged != 0 {
79        return RecoveryDecision::Blocked("an unmanaged runner service exists");
80    }
81    let Some(age) = evidence.heartbeat_age else {
82        return RecoveryDecision::Blocked("the guest heartbeat is missing");
83    };
84    if age > MAX_HEARTBEAT_AGE {
85        return RecoveryDecision::Blocked("the guest heartbeat is stale");
86    }
87    if evidence.acknowledged_generation != Some(evidence.drain_generation) {
88        return RecoveryDecision::RequestDrain;
89    }
90    match evidence.local_active_attempts {
91        Some(0) => {}
92        Some(_) => return RecoveryDecision::Blocked("the guest owns an active attempt"),
93        None => return RecoveryDecision::Blocked("the guest attempt count is unknown"),
94    }
95    if evidence.consecutive_zero_attempt_heartbeats < 2 {
96        return RecoveryDecision::Blocked("idle guest state has not been confirmed twice");
97    }
98    match (
99        evidence.managed_busy_runners,
100        evidence.managed_online_registrations,
101    ) {
102        (Some(0), Some(0)) if evidence.consecutive_zero_inventory_reads >= 2 => {
103            RecoveryDecision::TerminateNamed
104        }
105        (Some(0), Some(0)) => {
106            RecoveryDecision::Blocked("empty GitHub inventory has not been confirmed twice")
107        }
108        (Some(busy), _) if busy != 0 => {
109            RecoveryDecision::Blocked("GitHub reports a busy managed runner")
110        }
111        (None, _) | (_, None) => {
112            RecoveryDecision::Blocked("the GitHub runner inventory is unknown")
113        }
114        _ => RecoveryDecision::Blocked("GitHub reports an online managed runner"),
115    }
116}
117
118#[cfg(test)]
119mod tests {
120    use super::*;
121
122    fn idle() -> RecoveryEvidence {
123        RecoveryEvidence {
124            consecutive_probe_failures: 3,
125            failure_span: Some(Duration::from_secs(5 * 60)),
126            failure_is_recoverable: true,
127            drain_generation: 7,
128            acknowledged_generation: Some(7),
129            heartbeat_age: Some(Duration::from_secs(2)),
130            local_active_attempts: Some(0),
131            consecutive_zero_attempt_heartbeats: 2,
132            managed_busy_runners: Some(0),
133            managed_online_registrations: Some(0),
134            consecutive_zero_inventory_reads: 2,
135            unmanaged_runner_services: Some(0),
136            task_is_product_owned: true,
137            distribution_is_wsl2: true,
138            inventory_authorized: true,
139            recovery_fence_held: true,
140            circuit_open: false,
141        }
142    }
143
144    #[test]
145    fn only_a_complete_idle_proof_allows_named_termination() {
146        assert_eq!(decide(&idle()), RecoveryDecision::TerminateNamed);
147    }
148
149    #[test]
150    fn every_unknown_safety_fact_blocks() {
151        let mutations: [fn(&mut RecoveryEvidence); 6] = [
152            |e| e.failure_span = None,
153            |e| e.heartbeat_age = None,
154            |e| e.local_active_attempts = None,
155            |e| e.managed_busy_runners = None,
156            |e| e.managed_online_registrations = None,
157            |e| e.unmanaged_runner_services = None,
158        ];
159        for mutate in mutations {
160            let mut evidence = idle();
161            mutate(&mut evidence);
162            assert!(matches!(decide(&evidence), RecoveryDecision::Blocked(_)));
163        }
164    }
165
166    #[test]
167    fn work_or_an_unmanaged_runner_blocks_recovery() {
168        for mutate in [
169            |e: &mut RecoveryEvidence| e.local_active_attempts = Some(1),
170            |e: &mut RecoveryEvidence| e.managed_busy_runners = Some(1),
171            |e: &mut RecoveryEvidence| e.unmanaged_runner_services = Some(1),
172        ] {
173            let mut evidence = idle();
174            mutate(&mut evidence);
175            assert!(matches!(decide(&evidence), RecoveryDecision::Blocked(_)));
176        }
177    }
178
179    #[test]
180    fn a_matching_drain_ack_is_mandatory() {
181        let mut evidence = idle();
182        evidence.acknowledged_generation = Some(6);
183        assert_eq!(decide(&evidence), RecoveryDecision::RequestDrain);
184    }
185
186    #[test]
187    fn recovery_requires_a_classified_five_minute_failure_and_two_idle_observations() {
188        for mutate in [
189            |e: &mut RecoveryEvidence| e.failure_is_recoverable = false,
190            |e: &mut RecoveryEvidence| e.failure_span = Some(Duration::from_secs(299)),
191            |e: &mut RecoveryEvidence| e.consecutive_zero_attempt_heartbeats = 1,
192            |e: &mut RecoveryEvidence| e.consecutive_zero_inventory_reads = 1,
193            |e: &mut RecoveryEvidence| e.recovery_fence_held = false,
194        ] {
195            let mut evidence = idle();
196            mutate(&mut evidence);
197            assert_ne!(decide(&evidence), RecoveryDecision::TerminateNamed);
198        }
199    }
200}
runner_manager_platform/wsl/recovery.rs

runner_manager_platform/wsl/
recovery.rs
