Skip to main content

stateknot_testkit/
fault.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4use serde::{Deserialize, Serialize};
5use std::collections::BTreeSet;
6use thiserror::Error;
7
8const MAX_INVARIANT_CHECKS: u16 = 64;
9
10/// Stable fault case understood by qualification report schema version one.
11#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
12#[serde(rename_all = "snake_case")]
13pub enum FaultCaseId {
14    /// The primary database becomes unavailable and later recovers.
15    DatabaseUnavailable,
16    /// One host dependency readiness gate becomes unavailable and later recovers.
17    HostDependencyUnavailable,
18    /// A Worker exits while accepted work remains recoverable.
19    WorkerLoss,
20    /// A replacement host becomes ready before the old host drains.
21    HostRollingReplacement,
22    /// Online identity verification becomes unavailable and later recovers.
23    VerifierUnavailable,
24    /// The protected operations policy expires and is replaced.
25    OperationsPolicyExpiry,
26}
27
28impl FaultCaseId {
29    /// Every fault required by the version-one release profile.
30    pub const ALL: [Self; 6] = [
31        Self::DatabaseUnavailable,
32        Self::HostDependencyUnavailable,
33        Self::WorkerLoss,
34        Self::HostRollingReplacement,
35        Self::VerifierUnavailable,
36        Self::OperationsPolicyExpiry,
37    ];
38}
39
40/// Invalid fault-plan or transition data.
41#[derive(Clone, Debug, Eq, Error, PartialEq)]
42pub enum FaultError {
43    /// The plan is empty or exceeds the closed matrix.
44    #[error("fault plan must contain between one and six cases")]
45    InvalidPlanSize,
46    /// A case occurs more than once in a plan or report.
47    #[error("fault case {0:?} is duplicated")]
48    DuplicateCase(FaultCaseId),
49    /// The invariant-check requirement is outside its bound.
50    #[error("fault case {0:?} has an invalid invariant-check requirement")]
51    InvalidInvariantRequirement(FaultCaseId),
52    /// A transition references a case outside the plan.
53    #[error("fault case {0:?} was not planned")]
54    UnplannedCase(FaultCaseId),
55    /// A transition count overflowed.
56    #[error("fault case {0:?} transition count overflowed")]
57    CountOverflow(FaultCaseId),
58    /// Injection, recovery or invariant checks occurred in an impossible order.
59    #[error("fault case {0:?} has an invalid transition order")]
60    InvalidTransition(FaultCaseId),
61}
62
63/// One planned fault and the number of invariants that must be checked.
64#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
65#[serde(deny_unknown_fields)]
66pub struct FaultRequirement {
67    case: FaultCaseId,
68    required_invariant_checks: u16,
69}
70
71impl FaultRequirement {
72    /// Creates a requirement with a bounded non-zero invariant count.
73    pub fn new(case: FaultCaseId, required_invariant_checks: u16) -> Result<Self, FaultError> {
74        if required_invariant_checks == 0 || required_invariant_checks > MAX_INVARIANT_CHECKS {
75            return Err(FaultError::InvalidInvariantRequirement(case));
76        }
77        Ok(Self {
78            case,
79            required_invariant_checks,
80        })
81    }
82
83    /// Returns the stable fault identifier.
84    pub const fn case(&self) -> FaultCaseId {
85        self.case
86    }
87
88    /// Returns the required number of invariant checks.
89    pub const fn required_invariant_checks(&self) -> u16 {
90        self.required_invariant_checks
91    }
92
93    pub(crate) fn validate(&self) -> Result<(), FaultError> {
94        if self.required_invariant_checks == 0
95            || self.required_invariant_checks > MAX_INVARIANT_CHECKS
96        {
97            return Err(FaultError::InvalidInvariantRequirement(self.case));
98        }
99        Ok(())
100    }
101}
102
103/// Closed, deterministically ordered fault plan.
104#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
105#[serde(deny_unknown_fields)]
106pub struct FaultPlan {
107    requirements: Vec<FaultRequirement>,
108}
109
110impl FaultPlan {
111    /// Creates a non-empty plan with unique stable fault identifiers.
112    pub fn new(mut requirements: Vec<FaultRequirement>) -> Result<Self, FaultError> {
113        requirements.sort_by_key(FaultRequirement::case);
114        let plan = Self { requirements };
115        plan.validate()?;
116        Ok(plan)
117    }
118
119    /// Returns the reduced CI plan. It is never sufficient for release qualification.
120    pub fn ci_default() -> Self {
121        Self::new(vec![
122            FaultRequirement::new(FaultCaseId::HostDependencyUnavailable, 2)
123                .expect("constant requirement is valid"),
124            FaultRequirement::new(FaultCaseId::HostRollingReplacement, 3)
125                .expect("constant requirement is valid"),
126        ])
127        .expect("constant plan is valid")
128    }
129
130    /// Returns the complete version-one release fault matrix.
131    pub fn release_default() -> Self {
132        Self::new(
133            FaultCaseId::ALL
134                .into_iter()
135                .map(|case| FaultRequirement::new(case, 1).expect("constant requirement is valid"))
136                .collect(),
137        )
138        .expect("constant plan is valid")
139    }
140
141    /// Returns the sorted requirements.
142    pub fn requirements(&self) -> &[FaultRequirement] {
143        &self.requirements
144    }
145
146    pub(crate) fn validate(&self) -> Result<(), FaultError> {
147        if self.requirements.is_empty() || self.requirements.len() > FaultCaseId::ALL.len() {
148            return Err(FaultError::InvalidPlanSize);
149        }
150        let mut seen = BTreeSet::new();
151        for requirement in &self.requirements {
152            requirement.validate()?;
153            if !seen.insert(requirement.case) {
154                return Err(FaultError::DuplicateCase(requirement.case));
155            }
156        }
157        Ok(())
158    }
159
160    pub(crate) fn is_release_complete(&self) -> bool {
161        self.requirements.len() == FaultCaseId::ALL.len()
162            && FaultCaseId::ALL
163                .iter()
164                .all(|case| self.requirements.iter().any(|entry| entry.case == *case))
165    }
166}
167
168/// Executed counts for one planned fault.
169#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
170#[serde(deny_unknown_fields)]
171pub struct FaultResult {
172    case: FaultCaseId,
173    required_invariant_checks: u16,
174    injections: u32,
175    recoveries: u32,
176    invariant_checks: u32,
177}
178
179impl FaultResult {
180    pub(crate) fn planned(requirement: &FaultRequirement) -> Self {
181        Self {
182            case: requirement.case,
183            required_invariant_checks: requirement.required_invariant_checks,
184            injections: 0,
185            recoveries: 0,
186            invariant_checks: 0,
187        }
188    }
189
190    /// Returns the stable case identifier.
191    pub const fn case(&self) -> FaultCaseId {
192        self.case
193    }
194
195    /// Returns whether injection, recovery and every required invariant were observed.
196    pub const fn is_complete(&self) -> bool {
197        self.injections > 0
198            && self.recoveries > 0
199            && self.invariant_checks >= self.required_invariant_checks as u32
200    }
201
202    pub(crate) fn injected(&mut self) -> Result<(), FaultError> {
203        if self.injections != self.recoveries {
204            return Err(FaultError::InvalidTransition(self.case));
205        }
206        self.injections = self
207            .injections
208            .checked_add(1)
209            .ok_or(FaultError::CountOverflow(self.case))?;
210        Ok(())
211    }
212
213    pub(crate) fn recovered(&mut self) -> Result<(), FaultError> {
214        if self.recoveries >= self.injections {
215            return Err(FaultError::InvalidTransition(self.case));
216        }
217        self.recoveries = self
218            .recoveries
219            .checked_add(1)
220            .ok_or(FaultError::CountOverflow(self.case))?;
221        Ok(())
222    }
223
224    pub(crate) fn invariant_checked(&mut self) -> Result<(), FaultError> {
225        if self.injections == 0 {
226            return Err(FaultError::InvalidTransition(self.case));
227        }
228        self.invariant_checks = self
229            .invariant_checks
230            .checked_add(1)
231            .ok_or(FaultError::CountOverflow(self.case))?;
232        Ok(())
233    }
234
235    pub(crate) fn validate(&self) -> Result<(), FaultError> {
236        FaultRequirement::new(self.case, self.required_invariant_checks)?;
237        if self.recoveries > self.injections || (self.injections == 0 && self.invariant_checks != 0)
238        {
239            return Err(FaultError::InvalidTransition(self.case));
240        }
241        Ok(())
242    }
243}
244
245#[cfg(test)]
246mod tests {
247    use super::*;
248
249    #[test]
250    fn plans_are_closed_unique_and_sorted() {
251        let plan = FaultPlan::new(vec![
252            FaultRequirement::new(FaultCaseId::WorkerLoss, 1).unwrap(),
253            FaultRequirement::new(FaultCaseId::DatabaseUnavailable, 2).unwrap(),
254        ])
255        .unwrap();
256        assert_eq!(
257            plan.requirements()[0].case(),
258            FaultCaseId::DatabaseUnavailable
259        );
260        assert!(FaultPlan::new(vec![]).is_err());
261        assert!(
262            FaultPlan::new(vec![
263                FaultRequirement::new(FaultCaseId::WorkerLoss, 1).unwrap(),
264                FaultRequirement::new(FaultCaseId::WorkerLoss, 2).unwrap(),
265            ])
266            .is_err()
267        );
268        assert!(FaultPlan::release_default().is_release_complete());
269        assert!(!FaultPlan::ci_default().is_release_complete());
270    }
271
272    #[test]
273    fn fault_transitions_are_ordered() {
274        let requirement = FaultRequirement::new(FaultCaseId::WorkerLoss, 1).unwrap();
275        let mut result = FaultResult::planned(&requirement);
276        assert_eq!(
277            result.recovered(),
278            Err(FaultError::InvalidTransition(FaultCaseId::WorkerLoss))
279        );
280        assert_eq!(
281            result.invariant_checked(),
282            Err(FaultError::InvalidTransition(FaultCaseId::WorkerLoss))
283        );
284        result.injected().unwrap();
285        assert_eq!(
286            result.injected(),
287            Err(FaultError::InvalidTransition(FaultCaseId::WorkerLoss))
288        );
289        result.invariant_checked().unwrap();
290        result.recovered().unwrap();
291        assert!(result.is_complete());
292    }
293}