moonpool_sim/chaos/
buggify.rs

1//! Deterministic fault injection following FoundationDB's buggify approach.
2//!
3//! Each buggify location is randomly activated once per simulation run.
4//! Active locations fire probabilistically on each call.
5
6use crate::sim::rng::sim_random;
7use std::cell::RefCell;
8use std::collections::HashMap;
9
10thread_local! {
11    static STATE: RefCell<State> = RefCell::new(State::default());
12}
13
14#[derive(Default)]
15struct State {
16    enabled: bool,
17    active_locations: HashMap<String, bool>,
18    activation_prob: f64,
19}
20
21/// Initialize buggify for simulation run
22pub fn buggify_init(activation_prob: f64, _firing_prob: f64) {
23    STATE.with(|state| {
24        let mut state = state.borrow_mut();
25        state.enabled = true;
26        state.active_locations.clear();
27        state.activation_prob = activation_prob;
28    });
29}
30
31/// Reset/disable buggify
32pub fn buggify_reset() {
33    STATE.with(|state| {
34        let mut state = state.borrow_mut();
35        state.enabled = false;
36        state.active_locations.clear();
37        state.activation_prob = 0.0;
38    });
39}
40
41/// Internal buggify implementation
42pub fn buggify_internal(prob: f64, location: &'static str) -> bool {
43    STATE.with(|state| {
44        let mut state = state.borrow_mut();
45
46        if !state.enabled || prob <= 0.0 {
47            return false;
48        }
49
50        let location_str = location.to_string();
51        let activation_prob = state.activation_prob;
52
53        // Decide activation on first encounter
54        let is_active = *state
55            .active_locations
56            .entry(location_str)
57            .or_insert_with(|| sim_random::<f64>() < activation_prob);
58
59        // If active, fire probabilistically
60        is_active && sim_random::<f64>() < prob
61    })
62}
63
64/// Buggify with 25% probability
65#[macro_export]
66macro_rules! buggify {
67    () => {
68        $crate::chaos::buggify::buggify_internal(0.25, concat!(file!(), ":", line!()))
69    };
70}
71
72/// Buggify with custom probability
73#[macro_export]
74macro_rules! buggify_with_prob {
75    ($prob:expr) => {
76        $crate::chaos::buggify::buggify_internal($prob as f64, concat!(file!(), ":", line!()))
77    };
78}
79
80#[cfg(test)]
81mod tests {
82    use super::*;
83    use crate::sim::rng::{reset_sim_rng, set_sim_seed};
84
85    #[test]
86    fn test_disabled_by_default() {
87        buggify_reset();
88        for _ in 0..10 {
89            assert!(!buggify_internal(1.0, "test"));
90        }
91    }
92
93    #[test]
94    fn test_activation_consistency() {
95        set_sim_seed(12345);
96        buggify_init(0.5, 1.0);
97
98        let location = "test_location";
99        let first = buggify_internal(1.0, location);
100        let second = buggify_internal(1.0, location);
101
102        // Activation decision should be consistent
103        assert_eq!(first, second);
104        buggify_reset();
105    }
106
107    #[test]
108    fn test_deterministic() {
109        const SEED: u64 = 54321;
110        let mut results1 = Vec::new();
111        let mut results2 = Vec::new();
112
113        for run in 0..2 {
114            set_sim_seed(SEED);
115            buggify_init(0.5, 0.5);
116
117            let results = if run == 0 {
118                &mut results1
119            } else {
120                &mut results2
121            };
122
123            for i in 0..5 {
124                let location = format!("loc_{}", i);
125                results.push(buggify_internal(0.5, Box::leak(location.into_boxed_str())));
126            }
127
128            buggify_reset();
129            reset_sim_rng();
130        }
131
132        assert_eq!(results1, results2);
133    }
134}
moonpool_sim/chaos/buggify.rs

moonpool_sim/chaos/
buggify.rs
