use std::io;
use std::ops::Range;
use std::time::Duration;
use crate::util::DetRng;
#[derive(Debug, Clone)]
pub struct ChaosConfig {
pub seed: u64,
pub cancel_probability: f64,
pub delay_probability: f64,
pub delay_range: Range<Duration>,
pub io_error_probability: f64,
pub io_error_kinds: Vec<io::ErrorKind>,
pub wakeup_storm_probability: f64,
pub wakeup_storm_count: Range<usize>,
pub budget_exhaust_probability: f64,
}
impl Default for ChaosConfig {
fn default() -> Self {
Self::off()
}
}
impl ChaosConfig {
#[must_use]
pub const fn new(seed: u64) -> Self {
Self {
seed,
cancel_probability: 0.0,
delay_probability: 0.0,
delay_range: Duration::ZERO..Duration::ZERO,
io_error_probability: 0.0,
io_error_kinds: Vec::new(),
wakeup_storm_probability: 0.0,
wakeup_storm_count: 1..5,
budget_exhaust_probability: 0.0,
}
}
#[must_use]
pub const fn off() -> Self {
Self::new(0)
}
#[inline]
#[must_use]
pub fn light() -> Self {
Self {
seed: 0,
cancel_probability: 0.01,
delay_probability: 0.05,
delay_range: Duration::ZERO..Duration::from_millis(10),
io_error_probability: 0.02,
io_error_kinds: vec![
io::ErrorKind::ConnectionReset,
io::ErrorKind::TimedOut,
io::ErrorKind::WouldBlock,
],
wakeup_storm_probability: 0.01,
wakeup_storm_count: 1..5,
budget_exhaust_probability: 0.005,
}
}
#[inline]
#[must_use]
pub fn heavy() -> Self {
Self {
seed: 0,
cancel_probability: 0.10,
delay_probability: 0.20,
delay_range: Duration::ZERO..Duration::from_millis(100),
io_error_probability: 0.15,
io_error_kinds: vec![
io::ErrorKind::ConnectionReset,
io::ErrorKind::ConnectionRefused,
io::ErrorKind::ConnectionAborted,
io::ErrorKind::TimedOut,
io::ErrorKind::WouldBlock,
io::ErrorKind::BrokenPipe,
io::ErrorKind::NotConnected,
],
wakeup_storm_probability: 0.05,
wakeup_storm_count: 1..20,
budget_exhaust_probability: 0.05,
}
}
#[inline]
#[must_use]
pub const fn with_seed(mut self, seed: u64) -> Self {
self.seed = seed;
self
}
#[inline]
#[must_use]
pub fn with_cancel_probability(mut self, probability: f64) -> Self {
assert!(
(0.0..=1.0).contains(&probability),
"probability must be in [0.0, 1.0], got {probability}"
);
self.cancel_probability = probability;
self
}
#[inline]
#[must_use]
pub fn with_delay_probability(mut self, probability: f64) -> Self {
assert!(
(0.0..=1.0).contains(&probability),
"probability must be in [0.0, 1.0], got {probability}"
);
self.delay_probability = probability;
self
}
#[inline]
#[must_use]
pub fn with_delay_range(mut self, range: Range<Duration>) -> Self {
self.delay_range = range;
self
}
#[inline]
#[must_use]
pub fn with_io_error_probability(mut self, probability: f64) -> Self {
assert!(
(0.0..=1.0).contains(&probability),
"probability must be in [0.0, 1.0], got {probability}"
);
self.io_error_probability = probability;
self
}
#[inline]
#[must_use]
pub fn with_io_error_kinds(mut self, kinds: Vec<io::ErrorKind>) -> Self {
self.io_error_kinds = kinds;
self
}
#[inline]
#[must_use]
pub fn with_wakeup_storm_probability(mut self, probability: f64) -> Self {
assert!(
(0.0..=1.0).contains(&probability),
"probability must be in [0.0, 1.0], got {probability}"
);
self.wakeup_storm_probability = probability;
self
}
#[inline]
#[must_use]
pub fn with_wakeup_storm_count(mut self, range: Range<usize>) -> Self {
self.wakeup_storm_count = range;
self
}
#[inline]
#[must_use]
pub fn with_budget_exhaust_probability(mut self, probability: f64) -> Self {
assert!(
(0.0..=1.0).contains(&probability),
"probability must be in [0.0, 1.0], got {probability}"
);
self.budget_exhaust_probability = probability;
self
}
#[inline]
#[must_use]
pub fn is_enabled(&self) -> bool {
self.cancel_probability > 0.0
|| (self.delay_probability > 0.0 && delay_range_can_emit_nonzero(&self.delay_range))
|| (self.io_error_probability > 0.0 && !self.io_error_kinds.is_empty())
|| (self.wakeup_storm_probability > 0.0
&& wakeup_range_can_emit_positive(&self.wakeup_storm_count))
|| self.budget_exhaust_probability > 0.0
}
#[must_use]
pub fn summary(&self) -> String {
let mut parts = Vec::new();
if self.cancel_probability > 0.0 {
parts.push(format!("cancel:{:.1}%", self.cancel_probability * 100.0));
}
if self.delay_probability > 0.0 && delay_range_can_emit_nonzero(&self.delay_range) {
parts.push(format!("delay:{:.1}%", self.delay_probability * 100.0));
}
if self.io_error_probability > 0.0 && !self.io_error_kinds.is_empty() {
parts.push(format!("io_err:{:.1}%", self.io_error_probability * 100.0));
}
if self.wakeup_storm_probability > 0.0
&& wakeup_range_can_emit_positive(&self.wakeup_storm_count)
{
parts.push(format!(
"wakeup:{:.1}%",
self.wakeup_storm_probability * 100.0
));
}
if self.budget_exhaust_probability > 0.0 {
parts.push(format!(
"budget:{:.1}%",
self.budget_exhaust_probability * 100.0
));
}
if parts.is_empty() {
"off".to_string()
} else {
parts.join(",")
}
}
#[must_use]
pub fn rng(&self) -> ChaosRng {
ChaosRng::from_config(self)
}
}
#[derive(Debug, Clone)]
pub struct ChaosRng {
inner: DetRng,
}
impl ChaosRng {
#[must_use]
pub fn new(seed: u64) -> Self {
Self {
inner: DetRng::new(seed),
}
}
#[must_use]
pub fn from_config(config: &ChaosConfig) -> Self {
Self::new(config.seed)
}
#[must_use]
#[allow(clippy::cast_precision_loss)]
pub fn next_f64(&mut self) -> f64 {
let bits = self.inner.next_u64() >> 11;
bits as f64 * (1.0 / (1u64 << 53) as f64)
}
#[must_use]
pub fn next_u64(&mut self) -> u64 {
self.inner.next_u64()
}
#[must_use]
pub fn should_inject(&mut self, probability: f64) -> bool {
if probability <= 0.0 {
return false;
}
if probability >= 1.0 {
return true;
}
self.next_f64() < probability
}
#[must_use]
pub fn should_inject_cancel(&mut self, config: &ChaosConfig) -> bool {
self.should_inject(config.cancel_probability)
}
#[must_use]
pub fn should_inject_delay(&mut self, config: &ChaosConfig) -> bool {
if !delay_range_can_emit_nonzero(&config.delay_range) {
return false;
}
self.should_inject(config.delay_probability)
}
#[must_use]
pub fn next_delay(&mut self, config: &ChaosConfig) -> Duration {
let range = &config.delay_range;
let start_nanos = range.start.as_nanos();
let end_nanos = range.end.as_nanos();
if end_nanos <= start_nanos {
return Duration::ZERO;
}
let min_nanos = if start_nanos == 0 && end_nanos > 1 {
1
} else {
start_nanos
};
if end_nanos <= min_nanos {
return nanos_to_duration_saturating(min_nanos);
}
let delta = end_nanos - min_nanos;
let rand = (u128::from(self.inner.next_u64()) << 64) | u128::from(self.inner.next_u64());
let offset = rand % delta;
nanos_to_duration_saturating(min_nanos + offset)
}
#[must_use]
pub fn should_inject_io_error(&mut self, config: &ChaosConfig) -> bool {
if config.io_error_kinds.is_empty() {
return false;
}
self.should_inject(config.io_error_probability)
}
#[must_use]
pub fn next_io_error_kind(&mut self, config: &ChaosConfig) -> Option<io::ErrorKind> {
if config.io_error_kinds.is_empty() {
return None;
}
let idx = self.inner.next_usize(config.io_error_kinds.len());
Some(config.io_error_kinds[idx])
}
#[must_use]
pub fn next_io_error(&mut self, config: &ChaosConfig) -> Option<io::Error> {
self.next_io_error_kind(config)
.map(|kind| io::Error::new(kind, "chaos-injected I/O error"))
}
#[must_use]
pub fn should_inject_wakeup_storm(&mut self, config: &ChaosConfig) -> bool {
if !wakeup_range_can_emit_positive(&config.wakeup_storm_count) {
return false;
}
self.should_inject(config.wakeup_storm_probability)
}
#[must_use]
pub fn next_wakeup_count(&mut self, config: &ChaosConfig) -> usize {
let range = &config.wakeup_storm_count;
if range.end <= range.start {
return 0;
}
let min_count = if range.start == 0 && range.end > 1 {
1
} else {
range.start
};
if range.end <= min_count {
return min_count;
}
let delta = range.end - min_count;
min_count + self.inner.next_usize(delta)
}
#[must_use]
pub fn should_inject_budget_exhaust(&mut self, config: &ChaosConfig) -> bool {
self.should_inject(config.budget_exhaust_probability)
}
pub fn skip(&mut self, count: usize) {
for _ in 0..count {
let _ = self.inner.next_u64();
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum InjectionPoint {
SchedulerPoll,
TaskPoll,
ReactorPoll,
WakerInvoke,
BudgetCheck,
TimerFire,
SyncAcquire,
ChannelSend,
ChannelRecv,
}
impl InjectionPoint {
#[must_use]
pub fn applicable_chaos(&self) -> &'static [ChaosType] {
match self {
Self::TaskPoll => &[
ChaosType::Cancel,
ChaosType::Delay,
ChaosType::BudgetExhaust,
],
Self::ReactorPoll => &[ChaosType::IoError, ChaosType::Delay],
Self::WakerInvoke => &[ChaosType::WakeupStorm, ChaosType::Delay],
Self::BudgetCheck => &[ChaosType::BudgetExhaust],
Self::TimerFire => &[ChaosType::Delay],
Self::SchedulerPoll | Self::SyncAcquire | Self::ChannelSend | Self::ChannelRecv => {
&[ChaosType::Cancel, ChaosType::Delay]
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ChaosType {
Cancel,
Delay,
IoError,
WakeupStorm,
BudgetExhaust,
}
#[derive(Debug, Clone, Default)]
pub struct ChaosStats {
pub cancellations: u64,
pub delays: u64,
pub total_delay: Duration,
pub io_errors: u64,
pub wakeup_storms: u64,
pub spurious_wakeups: u64,
pub budget_exhaustions: u64,
pub decision_points: u64,
}
impl ChaosStats {
#[must_use]
pub const fn new() -> Self {
Self {
cancellations: 0,
delays: 0,
total_delay: Duration::ZERO,
io_errors: 0,
wakeup_storms: 0,
spurious_wakeups: 0,
budget_exhaustions: 0,
decision_points: 0,
}
}
pub fn record_cancel(&mut self) {
self.cancellations += 1;
self.decision_points += 1;
}
pub fn record_delay(&mut self, delay: Duration) {
self.delays += 1;
self.total_delay += delay;
self.decision_points += 1;
}
pub fn record_io_error(&mut self) {
self.io_errors += 1;
self.decision_points += 1;
}
pub fn record_wakeup_storm(&mut self, count: u64) {
self.wakeup_storms += 1;
self.spurious_wakeups += count;
self.decision_points += 1;
}
pub fn record_budget_exhaust(&mut self) {
self.budget_exhaustions += 1;
self.decision_points += 1;
}
pub fn record_pre_poll_outcomes(
&mut self,
cancel: bool,
delay: Option<Duration>,
budget_exhaust: bool,
) {
if cancel {
self.cancellations += 1;
}
if let Some(delay) = delay {
self.delays += 1;
self.total_delay += delay;
}
if budget_exhaust {
self.budget_exhaustions += 1;
}
self.decision_points += 1;
}
pub fn record_no_injection(&mut self) {
self.decision_points += 1;
}
pub fn merge(&mut self, other: &Self) {
self.cancellations = self.cancellations.saturating_add(other.cancellations);
self.delays = self.delays.saturating_add(other.delays);
self.total_delay = self.total_delay.saturating_add(other.total_delay);
self.io_errors = self.io_errors.saturating_add(other.io_errors);
self.wakeup_storms = self.wakeup_storms.saturating_add(other.wakeup_storms);
self.spurious_wakeups = self.spurious_wakeups.saturating_add(other.spurious_wakeups);
self.budget_exhaustions = self
.budget_exhaustions
.saturating_add(other.budget_exhaustions);
self.decision_points = self.decision_points.saturating_add(other.decision_points);
}
#[must_use]
#[allow(clippy::cast_precision_loss)]
pub fn injection_rate(&self) -> f64 {
if self.decision_points == 0 {
return 0.0;
}
let injections = self
.cancellations
.saturating_add(self.delays)
.saturating_add(self.io_errors)
.saturating_add(self.wakeup_storms)
.saturating_add(self.budget_exhaustions);
if self.decision_points == 0 {
return 0.0;
}
injections as f64 / self.decision_points as f64
}
}
impl std::fmt::Display for ChaosStats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"ChaosStats {{ decisions: {}, cancels: {}, delays: {} ({:?}), io_errors: {}, \
wakeup_storms: {} ({} wakeups), budget_exhausts: {}, rate: {:.2}% }}",
self.decision_points,
self.cancellations,
self.delays,
self.total_delay,
self.io_errors,
self.wakeup_storms,
self.spurious_wakeups,
self.budget_exhaustions,
self.injection_rate() * 100.0
)
}
}
fn nanos_to_duration_saturating(nanos: u128) -> Duration {
const NANOS_PER_SEC: u128 = 1_000_000_000;
let secs = nanos / NANOS_PER_SEC;
let subsec = (nanos % NANOS_PER_SEC) as u32;
if secs > u128::from(u64::MAX) {
Duration::MAX
} else {
Duration::new(secs as u64, subsec)
}
}
fn delay_range_can_emit_nonzero(range: &Range<Duration>) -> bool {
let start_nanos = range.start.as_nanos();
let end_nanos = range.end.as_nanos();
end_nanos > start_nanos && (start_nanos > 0 || end_nanos > 1)
}
fn wakeup_range_can_emit_positive(range: &Range<usize>) -> bool {
range.end > range.start && (range.start > 0 || range.end > 1)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn config_off_has_no_chaos() {
let config = ChaosConfig::off();
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
}
#[test]
fn config_light_has_chaos() {
let config = ChaosConfig::light();
assert!(config.is_enabled());
assert!(config.cancel_probability > 0.0);
assert!(config.delay_probability > 0.0);
}
#[test]
fn config_heavy_has_higher_probabilities() {
let light = ChaosConfig::light();
let heavy = ChaosConfig::heavy();
assert!(heavy.cancel_probability > light.cancel_probability);
assert!(heavy.delay_probability > light.delay_probability);
assert!(heavy.io_error_probability > light.io_error_probability);
}
#[test]
fn config_builder_pattern() {
let config = ChaosConfig::new(42)
.with_cancel_probability(0.1)
.with_delay_probability(0.2)
.with_delay_range(Duration::from_millis(5)..Duration::from_millis(50))
.with_io_error_probability(0.05)
.with_io_error_kinds(vec![io::ErrorKind::ConnectionReset])
.with_wakeup_storm_probability(0.01)
.with_wakeup_storm_count(1..10)
.with_budget_exhaust_probability(0.02);
assert_eq!(config.seed, 42);
assert!((config.cancel_probability - 0.1).abs() < f64::EPSILON);
assert!((config.delay_probability - 0.2).abs() < f64::EPSILON);
assert_eq!(config.delay_range.start, Duration::from_millis(5));
assert_eq!(config.delay_range.end, Duration::from_millis(50));
assert!((config.io_error_probability - 0.05).abs() < f64::EPSILON);
assert_eq!(config.io_error_kinds.len(), 1);
assert!((config.wakeup_storm_probability - 0.01).abs() < f64::EPSILON);
assert_eq!(config.wakeup_storm_count, 1..10);
assert!((config.budget_exhaust_probability - 0.02).abs() < f64::EPSILON);
}
#[test]
#[should_panic(expected = "probability must be in [0.0, 1.0]")]
fn config_rejects_invalid_probability() {
let _ = ChaosConfig::new(42).with_cancel_probability(1.5);
}
#[test]
fn config_summary() {
let config = ChaosConfig::new(42)
.with_cancel_probability(0.1)
.with_io_error_probability(0.05)
.with_io_error_kinds(vec![std::io::ErrorKind::ConnectionReset]);
let summary = config.summary();
assert!(summary.contains("cancel:10.0%"));
assert!(summary.contains("io_err:5.0%"));
}
#[test]
fn rng_deterministic() {
let config = ChaosConfig::new(42).with_cancel_probability(0.5);
let mut rng1 = ChaosRng::from_config(&config);
let mut rng2 = ChaosRng::from_config(&config);
for _ in 0..100 {
assert_eq!(
rng1.should_inject_cancel(&config),
rng2.should_inject_cancel(&config)
);
}
}
#[test]
fn rng_f64_range() {
let mut rng = ChaosRng::new(42);
for _ in 0..1000 {
let val = rng.next_f64();
assert!((0.0..1.0).contains(&val), "f64 out of range: {val}");
}
}
#[test]
fn rng_should_inject_bounds() {
let mut rng = ChaosRng::new(42);
for _ in 0..100 {
assert!(!rng.should_inject(0.0));
}
for _ in 0..100 {
assert!(rng.should_inject(1.0));
}
}
#[test]
fn rng_delay_generation() {
let config = ChaosConfig::new(42)
.with_delay_range(Duration::from_millis(10)..Duration::from_millis(100));
let mut rng = config.rng();
for _ in 0..100 {
let delay = rng.next_delay(&config);
assert!(delay >= Duration::from_millis(10));
assert!(delay < Duration::from_millis(100));
}
}
#[test]
fn delay_probability_without_nonzero_delay_range_is_effectively_disabled() {
let config = ChaosConfig::new(42)
.with_delay_probability(1.0)
.with_delay_range(Duration::ZERO..Duration::ZERO);
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_delay(&config),
"delay chaos without a nonzero delay range must never inject"
);
assert_eq!(rng.next_delay(&config), Duration::ZERO);
}
}
#[test]
fn delay_probability_with_reversed_range_is_effectively_disabled() {
let config = ChaosConfig::new(42)
.with_delay_probability(1.0)
.with_delay_range(Duration::from_millis(5)..Duration::from_millis(1));
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_delay(&config),
"delay chaos with a reversed range must never inject"
);
assert_eq!(rng.next_delay(&config), Duration::ZERO);
}
}
#[test]
fn delay_probability_with_empty_positive_range_is_effectively_disabled() {
let config = ChaosConfig::new(42)
.with_delay_probability(1.0)
.with_delay_range(Duration::from_millis(5)..Duration::from_millis(5));
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_delay(&config),
"delay chaos with an empty range must never inject"
);
assert_eq!(rng.next_delay(&config), Duration::ZERO);
}
}
#[test]
fn rng_delay_generation_excludes_zero_when_positive_delays_are_possible() {
let config = ChaosConfig::new(42).with_delay_range(Duration::ZERO..Duration::from_nanos(3));
let mut rng = config.rng();
for _ in 0..64 {
let delay = rng.next_delay(&config);
assert!(
delay >= Duration::from_nanos(1),
"delay {delay:?} should exclude zero when positive delays are possible"
);
assert!(
delay < Duration::from_nanos(3),
"delay {delay:?} should stay within configured range"
);
}
}
#[test]
fn rng_delay_generation_handles_large_duration_ranges() {
let start = Duration::from_secs(40_000_000_000);
let end = start + Duration::from_secs(100);
let config = ChaosConfig::new(42).with_delay_range(start..end);
let mut rng = config.rng();
for _ in 0..100 {
let delay = rng.next_delay(&config);
assert!(
delay >= start,
"delay {delay:?} should be >= start {start:?}"
);
assert!(delay < end, "delay {delay:?} should be < end {end:?}");
}
}
#[test]
fn rng_io_error_kind() {
let config = ChaosConfig::new(42).with_io_error_kinds(vec![
io::ErrorKind::ConnectionReset,
io::ErrorKind::TimedOut,
]);
let mut rng = config.rng();
for _ in 0..100 {
let kind = rng.next_io_error_kind(&config).unwrap();
assert!(
kind == io::ErrorKind::ConnectionReset || kind == io::ErrorKind::TimedOut,
"Unexpected error kind: {kind:?}"
);
}
}
#[test]
fn io_error_probability_without_kinds_is_effectively_disabled() {
let config = ChaosConfig::new(42).with_io_error_probability(1.0);
assert!(
!config.is_enabled(),
"io-error chaos without error kinds should not report enabled"
);
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_io_error(&config),
"io-error chaos without kinds must never inject"
);
assert!(
rng.next_io_error_kind(&config).is_none(),
"io-error chaos without kinds must not fabricate an error kind"
);
}
}
#[test]
fn rng_wakeup_count() {
let config = ChaosConfig::new(42).with_wakeup_storm_count(5..15);
let mut rng = config.rng();
for _ in 0..100 {
let count = rng.next_wakeup_count(&config);
assert!((5..15).contains(&count), "Count out of range: {count}");
}
}
#[test]
fn wakeup_probability_without_positive_count_is_effectively_disabled() {
let config = ChaosConfig::new(42)
.with_wakeup_storm_probability(1.0)
.with_wakeup_storm_count(0..1);
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_wakeup_storm(&config),
"wakeup storms without positive wake counts must never inject"
);
assert_eq!(rng.next_wakeup_count(&config), 0);
}
}
#[test]
#[allow(clippy::reversed_empty_ranges)]
fn wakeup_probability_with_reversed_count_range_is_effectively_disabled() {
let config = ChaosConfig::new(42)
.with_wakeup_storm_probability(1.0)
.with_wakeup_storm_count(5..1);
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_wakeup_storm(&config),
"wakeup storms with a reversed count range must never inject"
);
assert_eq!(rng.next_wakeup_count(&config), 0);
}
}
#[test]
fn wakeup_probability_with_empty_positive_count_range_is_effectively_disabled() {
let config = ChaosConfig::new(42)
.with_wakeup_storm_probability(1.0)
.with_wakeup_storm_count(5..5);
assert!(!config.is_enabled());
assert_eq!(config.summary(), "off");
let mut rng = config.rng();
for _ in 0..32 {
assert!(
!rng.should_inject_wakeup_storm(&config),
"wakeup storms with an empty count range must never inject"
);
assert_eq!(rng.next_wakeup_count(&config), 0);
}
}
#[test]
fn rng_wakeup_count_excludes_zero_when_positive_counts_are_possible() {
let config = ChaosConfig::new(42).with_wakeup_storm_count(0..3);
let mut rng = config.rng();
for _ in 0..64 {
let count = rng.next_wakeup_count(&config);
assert!(count > 0, "wakeup storm count should exclude zero");
assert!(count < 3, "count should stay within configured range");
}
}
#[test]
fn injection_point_applicable_chaos() {
let applicable = InjectionPoint::TaskPoll.applicable_chaos();
assert!(applicable.contains(&ChaosType::Cancel));
assert!(applicable.contains(&ChaosType::Delay));
assert!(applicable.contains(&ChaosType::BudgetExhaust));
assert!(!applicable.contains(&ChaosType::IoError));
let applicable = InjectionPoint::ReactorPoll.applicable_chaos();
assert!(applicable.contains(&ChaosType::IoError));
assert!(applicable.contains(&ChaosType::Delay));
assert!(!applicable.contains(&ChaosType::Cancel));
}
#[test]
fn stats_tracking() {
let mut stats = ChaosStats::new();
stats.record_cancel();
stats.record_delay(Duration::from_millis(10));
stats.record_io_error();
stats.record_wakeup_storm(5);
stats.record_budget_exhaust();
stats.record_no_injection();
stats.record_no_injection();
assert_eq!(stats.cancellations, 1);
assert_eq!(stats.delays, 1);
assert_eq!(stats.total_delay, Duration::from_millis(10));
assert_eq!(stats.io_errors, 1);
assert_eq!(stats.wakeup_storms, 1);
assert_eq!(stats.spurious_wakeups, 5);
assert_eq!(stats.budget_exhaustions, 1);
assert_eq!(stats.decision_points, 7);
let rate = stats.injection_rate();
assert!((rate - 5.0 / 7.0).abs() < 0.001);
}
#[test]
fn stats_merge() {
let mut stats1 = ChaosStats::new();
stats1.record_cancel();
stats1.record_cancel();
let mut stats2 = ChaosStats::new();
stats2.record_io_error();
stats2.record_delay(Duration::from_millis(5));
stats1.merge(&stats2);
assert_eq!(stats1.cancellations, 2);
assert_eq!(stats1.io_errors, 1);
assert_eq!(stats1.delays, 1);
assert_eq!(stats1.decision_points, 4);
}
#[test]
fn pre_poll_outcomes_count_as_one_decision_point() {
let mut stats = ChaosStats::new();
stats.record_pre_poll_outcomes(true, Some(Duration::from_millis(2)), true);
assert_eq!(stats.cancellations, 1);
assert_eq!(stats.delays, 1);
assert_eq!(stats.total_delay, Duration::from_millis(2));
assert_eq!(stats.budget_exhaustions, 1);
assert_eq!(stats.decision_points, 1);
}
#[test]
fn stats_display() {
let mut stats = ChaosStats::new();
stats.record_cancel();
stats.record_delay(Duration::from_millis(10));
let display = format!("{stats}");
assert!(display.contains("cancels: 1"));
assert!(display.contains("delays: 1"));
}
#[test]
fn injection_point_debug_clone_copy_eq_hash() {
use std::collections::HashSet;
let all = [
InjectionPoint::SchedulerPoll,
InjectionPoint::TaskPoll,
InjectionPoint::ReactorPoll,
InjectionPoint::WakerInvoke,
InjectionPoint::BudgetCheck,
InjectionPoint::TimerFire,
InjectionPoint::SyncAcquire,
InjectionPoint::ChannelSend,
InjectionPoint::ChannelRecv,
];
let mut set = HashSet::new();
for ip in &all {
let copied = *ip;
let cloned = *ip;
assert_eq!(copied, cloned);
assert!(!format!("{ip:?}").is_empty());
set.insert(*ip);
}
assert_eq!(set.len(), 9);
}
#[test]
fn chaos_type_debug_clone_copy_eq_hash() {
use std::collections::HashSet;
let all = [
ChaosType::Cancel,
ChaosType::Delay,
ChaosType::IoError,
ChaosType::WakeupStorm,
ChaosType::BudgetExhaust,
];
let mut set = HashSet::new();
for ct in &all {
let copied = *ct;
let cloned = *ct;
assert_eq!(copied, cloned);
set.insert(*ct);
}
assert_eq!(set.len(), 5);
assert_ne!(ChaosType::Cancel, ChaosType::Delay);
}
#[test]
fn chaos_stats_debug_clone_default() {
let def = ChaosStats::default();
assert_eq!(def.cancellations, 0);
assert_eq!(def.delays, 0);
assert_eq!(def.io_errors, 0);
let dbg = format!("{def:?}");
assert!(dbg.contains("ChaosStats"), "{dbg}");
let cloned = def;
assert_eq!(cloned.cancellations, 0);
}
}