runner_manager_platform/wsl/
recovery.rs

1//! Fail-closed decision model for managed-WSL recovery.
2//!
3//! An unreachable guest is not evidence that it is idle.  This module keeps
4//! the destructive decision separate from probing so every missing or stale
5//! fact has one conservative answer and can be property-tested without WSL.
6
7use std::time::Duration;
8
9/// Evidence collected by the Windows-side supervisor for one distribution.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct RecoveryEvidence {
12    pub consecutive_probe_failures: u8,
13    pub failure_span: Option<Duration>,
14    pub failure_is_recoverable: bool,
15    pub drain_generation: u64,
16    pub acknowledged_generation: Option<u64>,
17    pub heartbeat_age: Option<Duration>,
18    pub local_active_attempts: Option<u32>,
19    pub local_busy_attempts: Option<u32>,
20    pub consecutive_zero_attempt_heartbeats: u8,
21    pub consecutive_no_busy_heartbeats: u8,
22    pub managed_busy_runners: Option<u32>,
23    pub managed_online_registrations: Option<u32>,
24    pub consecutive_zero_inventory_reads: u8,
25    pub unmanaged_runner_services: Option<u32>,
26    pub task_is_product_owned: bool,
27    pub distribution_is_wsl2: bool,
28    pub inventory_authorized: bool,
29    pub recovery_fence_held: bool,
30    pub circuit_open: bool,
31}
32
33/// The only outcomes the watchdog may act on.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub enum RecoveryDecision {
36    Observe,
37    RequestDrain,
38    TerminateNamed,
39    Blocked(&'static str),
40}
41
42pub const FAILURE_THRESHOLD: u8 = 3;
43pub const MINIMUM_FAILURE_SPAN: Duration = Duration::from_secs(5 * 60);
44pub const MAX_HEARTBEAT_AGE: Duration = Duration::from_secs(30);
45
46/// Decide without optimistic defaults. `TerminateNamed` means every required
47/// proof is present, current and mutually consistent; `None` always blocks.
48#[must_use]
49pub fn decide(evidence: &RecoveryEvidence) -> RecoveryDecision {
50    if evidence.circuit_open {
51        return RecoveryDecision::Blocked("the recovery circuit is open");
52    }
53    if evidence.consecutive_probe_failures < FAILURE_THRESHOLD {
54        return RecoveryDecision::Observe;
55    }
56    if !evidence.failure_is_recoverable {
57        return RecoveryDecision::Blocked("the failure is not a recoverable WSL transport failure");
58    }
59    let Some(failure_span) = evidence.failure_span else {
60        return RecoveryDecision::Blocked("the probe failure window is unknown");
61    };
62    if failure_span < MINIMUM_FAILURE_SPAN {
63        return RecoveryDecision::Observe;
64    }
65    if !evidence.task_is_product_owned {
66        return RecoveryDecision::Blocked("the lifecycle task is not product-owned");
67    }
68    if !evidence.distribution_is_wsl2 {
69        return RecoveryDecision::Blocked("the distribution is not verified as WSL2");
70    }
71    if !evidence.inventory_authorized {
72        return RecoveryDecision::Blocked("GitHub runner inventory is not authorized");
73    }
74    if !evidence.recovery_fence_held {
75        return RecoveryDecision::Blocked("the recovery fence is not held");
76    }
77    let Some(unmanaged) = evidence.unmanaged_runner_services else {
78        return RecoveryDecision::Blocked("the unmanaged-runner audit is unknown");
79    };
80    if unmanaged != 0 {
81        return RecoveryDecision::Blocked("an unmanaged runner service exists");
82    }
83    let Some(age) = evidence.heartbeat_age else {
84        return RecoveryDecision::Blocked("the guest heartbeat is missing");
85    };
86    if age > MAX_HEARTBEAT_AGE {
87        return RecoveryDecision::Blocked("the guest heartbeat is stale");
88    }
89    if evidence.acknowledged_generation != Some(evidence.drain_generation) {
90        return RecoveryDecision::RequestDrain;
91    }
92    let active = match evidence.local_active_attempts {
93        Some(active) => active,
94        None => return RecoveryDecision::Blocked("the guest attempt count is unknown"),
95    };
96    match evidence.local_busy_attempts {
97        Some(0) => {}
98        Some(_) => return RecoveryDecision::Blocked("the guest owns a busy attempt"),
99        None => return RecoveryDecision::Blocked("the guest busy-attempt count is unknown"),
100    }
101    if active == 0 {
102        if evidence.consecutive_zero_attempt_heartbeats < 2 {
103            return RecoveryDecision::Blocked("idle guest state has not been confirmed twice");
104        }
105    } else if evidence.consecutive_no_busy_heartbeats < 2 {
106        return RecoveryDecision::Blocked(
107            "the guest's active but non-busy attempts have not been confirmed twice",
108        );
109    }
110    match (
111        evidence.managed_busy_runners,
112        evidence.managed_online_registrations,
113    ) {
114        (Some(0), Some(0)) if evidence.consecutive_zero_inventory_reads >= 2 => {
115            RecoveryDecision::TerminateNamed
116        }
117        (Some(0), Some(0)) => {
118            RecoveryDecision::Blocked("empty GitHub inventory has not been confirmed twice")
119        }
120        (Some(busy), _) if busy != 0 => {
121            RecoveryDecision::Blocked("GitHub reports a busy managed runner")
122        }
123        (None, _) | (_, None) => {
124            RecoveryDecision::Blocked("the GitHub runner inventory is unknown")
125        }
126        _ => RecoveryDecision::Blocked("GitHub reports an online managed runner"),
127    }
128}
129
130#[cfg(test)]
131mod tests {
132    use super::*;
133
134    fn idle() -> RecoveryEvidence {
135        RecoveryEvidence {
136            consecutive_probe_failures: 3,
137            failure_span: Some(Duration::from_secs(5 * 60)),
138            failure_is_recoverable: true,
139            drain_generation: 7,
140            acknowledged_generation: Some(7),
141            heartbeat_age: Some(Duration::from_secs(2)),
142            local_active_attempts: Some(0),
143            local_busy_attempts: Some(0),
144            consecutive_zero_attempt_heartbeats: 2,
145            consecutive_no_busy_heartbeats: 2,
146            managed_busy_runners: Some(0),
147            managed_online_registrations: Some(0),
148            consecutive_zero_inventory_reads: 2,
149            unmanaged_runner_services: Some(0),
150            task_is_product_owned: true,
151            distribution_is_wsl2: true,
152            inventory_authorized: true,
153            recovery_fence_held: true,
154            circuit_open: false,
155        }
156    }
157
158    #[test]
159    fn only_a_complete_idle_proof_allows_named_termination() {
160        assert_eq!(decide(&idle()), RecoveryDecision::TerminateNamed);
161    }
162
163    #[test]
164    fn every_unknown_safety_fact_blocks() {
165        let mutations: [fn(&mut RecoveryEvidence); 7] = [
166            |e| e.failure_span = None,
167            |e| e.heartbeat_age = None,
168            |e| e.local_active_attempts = None,
169            |e| e.local_busy_attempts = None,
170            |e| e.managed_busy_runners = None,
171            |e| e.managed_online_registrations = None,
172            |e| e.unmanaged_runner_services = None,
173        ];
174        for mutate in mutations {
175            let mut evidence = idle();
176            mutate(&mut evidence);
177            assert!(matches!(decide(&evidence), RecoveryDecision::Blocked(_)));
178        }
179    }
180
181    #[test]
182    fn busy_work_or_an_unmanaged_runner_blocks_recovery() {
183        for mutate in [
184            |e: &mut RecoveryEvidence| {
185                e.local_active_attempts = Some(1);
186                e.local_busy_attempts = Some(1);
187            },
188            |e: &mut RecoveryEvidence| e.managed_busy_runners = Some(1),
189            |e: &mut RecoveryEvidence| e.unmanaged_runner_services = Some(1),
190        ] {
191            let mut evidence = idle();
192            mutate(&mut evidence);
193            assert!(matches!(decide(&evidence), RecoveryDecision::Blocked(_)));
194        }
195    }
196
197    #[test]
198    fn a_matching_drain_ack_is_mandatory() {
199        let mut evidence = idle();
200        evidence.acknowledged_generation = Some(6);
201        assert_eq!(decide(&evidence), RecoveryDecision::RequestDrain);
202    }
203
204    #[test]
205    fn recovery_requires_a_classified_five_minute_failure_and_two_idle_observations() {
206        for mutate in [
207            |e: &mut RecoveryEvidence| e.failure_is_recoverable = false,
208            |e: &mut RecoveryEvidence| e.failure_span = Some(Duration::from_secs(299)),
209            |e: &mut RecoveryEvidence| e.consecutive_zero_attempt_heartbeats = 1,
210            |e: &mut RecoveryEvidence| e.consecutive_zero_inventory_reads = 1,
211            |e: &mut RecoveryEvidence| e.recovery_fence_held = false,
212        ] {
213            let mut evidence = idle();
214            mutate(&mut evidence);
215            assert_ne!(decide(&evidence), RecoveryDecision::TerminateNamed);
216        }
217    }
218
219    #[test]
220    fn stale_non_busy_attempts_do_not_deadlock_a_fully_drained_host() {
221        let mut evidence = idle();
222        evidence.local_active_attempts = Some(4);
223        evidence.local_busy_attempts = Some(0);
224        evidence.consecutive_zero_attempt_heartbeats = 0;
225        evidence.consecutive_no_busy_heartbeats = 2;
226        assert_eq!(decide(&evidence), RecoveryDecision::TerminateNamed);
227    }
228
229    #[test]
230    fn even_complete_cloud_evidence_never_overrides_a_guest_busy_attempt() {
231        let mut evidence = idle();
232        evidence.local_active_attempts = Some(1);
233        evidence.local_busy_attempts = Some(1);
234        assert!(matches!(decide(&evidence), RecoveryDecision::Blocked(_)));
235    }
236}
runner_manager_platform/wsl/recovery.rs

runner_manager_platform/wsl/
recovery.rs
