use serde::{Deserialize, Serialize};
use std::sync::atomic::{AtomicU8, AtomicU32, Ordering};
#[derive(
Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
)]
pub enum DialLevel {
#[default]
L0,
L1,
L2,
L3,
L4,
L5,
}
impl DialLevel {
pub fn capacity_ratio(self) -> f64 {
match self {
Self::L0 => 1.0,
Self::L1 => 0.8,
Self::L2 => 0.6,
Self::L3 => 0.4,
Self::L4 => 0.2,
Self::L5 => 0.0,
}
}
pub fn escalate(self) -> Option<Self> {
match self {
Self::L0 => Some(Self::L1),
Self::L1 => Some(Self::L2),
Self::L2 => Some(Self::L3),
Self::L3 => Some(Self::L4),
Self::L4 => Some(Self::L5),
Self::L5 => None,
}
}
pub fn deescalate(self) -> Option<Self> {
match self {
Self::L0 => None,
Self::L1 => Some(Self::L0),
Self::L2 => Some(Self::L1),
Self::L3 => Some(Self::L2),
Self::L4 => Some(Self::L3),
Self::L5 => Some(Self::L4),
}
}
pub fn as_usize(self) -> usize {
match self {
Self::L0 => 0,
Self::L1 => 1,
Self::L2 => 2,
Self::L3 => 3,
Self::L4 => 4,
Self::L5 => 5,
}
}
fn from_u8(v: u8) -> Self {
match v {
0 => Self::L0,
1 => Self::L1,
2 => Self::L2,
3 => Self::L3,
4 => Self::L4,
_ => Self::L5,
}
}
}
impl std::fmt::Display for DialLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::L0 => write!(f, "L0"),
Self::L1 => write!(f, "L1"),
Self::L2 => write!(f, "L2"),
Self::L3 => write!(f, "L3"),
Self::L4 => write!(f, "L4"),
Self::L5 => write!(f, "L5"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DialThresholds {
pub error_rate: [f64; 5],
pub latency_ms: [f64; 5],
pub resource_usage: [f64; 5],
}
impl Default for DialThresholds {
fn default() -> Self {
Self {
error_rate: [0.05, 0.10, 0.20, 0.30, 0.50],
latency_ms: [100.0, 200.0, 500.0, 1000.0, 2000.0],
resource_usage: [0.70, 0.80, 0.90, 0.95, 0.99],
}
}
}
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
pub struct SystemMetrics {
pub error_rate: f64,
pub latency_ms: f64,
pub resource_usage: f64,
}
impl SystemMetrics {
pub fn is_healthy(&self, thresholds: &DialThresholds) -> bool {
self.error_rate < thresholds.error_rate[0]
&& self.latency_ms < thresholds.latency_ms[0]
&& self.resource_usage < thresholds.resource_usage[0]
}
}
pub struct CapacityDial {
thresholds: DialThresholds,
recovery_consecutive_healthy: u32,
current_level: AtomicU8,
consecutive_healthy: AtomicU32,
}
impl CapacityDial {
pub fn new(thresholds: DialThresholds, recovery_consecutive_healthy: u32) -> Self {
assert!(
recovery_consecutive_healthy >= 1,
"CapacityDial: recovery_consecutive_healthy must be >= 1; \
0 would de-escalate on every healthy evaluation (anti-oscillation broken)"
);
Self {
thresholds,
recovery_consecutive_healthy,
current_level: AtomicU8::new(0),
consecutive_healthy: AtomicU32::new(0),
}
}
pub fn level(&self) -> DialLevel {
DialLevel::from_u8(self.current_level.load(Ordering::Relaxed))
}
pub fn capacity_ratio(&self) -> f64 {
self.level().capacity_ratio()
}
fn target_level(&self, metrics: &SystemMetrics) -> DialLevel {
let t = &self.thresholds;
let mut max_level = DialLevel::L0;
for (i, &threshold) in t.error_rate.iter().enumerate() {
if metrics.error_rate >= threshold {
max_level = max_level.max(level_from_index(i));
}
}
for (i, &threshold) in t.latency_ms.iter().enumerate() {
if metrics.latency_ms >= threshold {
max_level = max_level.max(level_from_index(i));
}
}
for (i, &threshold) in t.resource_usage.iter().enumerate() {
if metrics.resource_usage >= threshold {
max_level = max_level.max(level_from_index(i));
}
}
max_level
}
pub fn evaluate(&self, metrics: &SystemMetrics) -> DialLevel {
let target = self.target_level(metrics);
let current = self.level();
if target > current {
self.current_level
.fetch_max(target as u8, Ordering::Relaxed);
self.consecutive_healthy.store(0, Ordering::Relaxed);
return self.level();
}
if metrics.is_healthy(&self.thresholds) {
let healthy = self.consecutive_healthy.fetch_add(1, Ordering::Relaxed) + 1;
if healthy >= self.recovery_consecutive_healthy {
let cur = self.level();
if let Some(lower) = cur.deescalate() {
let _ = self.current_level.compare_exchange(
cur as u8,
lower as u8,
Ordering::Relaxed,
Ordering::Relaxed,
);
}
self.consecutive_healthy.store(0, Ordering::Relaxed);
}
} else {
self.consecutive_healthy.store(0, Ordering::Relaxed);
}
self.level()
}
pub fn force_recover(&self) -> DialLevel {
let cur = self.level();
if let Some(lower) = cur.deescalate() {
let _ = self.current_level.compare_exchange(
cur as u8,
lower as u8,
Ordering::Relaxed,
Ordering::Relaxed,
);
}
self.consecutive_healthy.store(0, Ordering::Relaxed);
self.level()
}
pub fn reset(&self) {
self.current_level.store(0, Ordering::Relaxed);
self.consecutive_healthy.store(0, Ordering::Relaxed);
}
}
impl Default for CapacityDial {
fn default() -> Self {
Self::new(DialThresholds::default(), 3)
}
}
fn level_from_index(i: usize) -> DialLevel {
match i {
0 => DialLevel::L1,
1 => DialLevel::L2,
2 => DialLevel::L3,
3 => DialLevel::L4,
_ => DialLevel::L5,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn level_capacity_ratios() {
assert_eq!(DialLevel::L0.capacity_ratio(), 1.0);
assert_eq!(DialLevel::L1.capacity_ratio(), 0.8);
assert_eq!(DialLevel::L2.capacity_ratio(), 0.6);
assert_eq!(DialLevel::L3.capacity_ratio(), 0.4);
assert_eq!(DialLevel::L4.capacity_ratio(), 0.2);
assert_eq!(DialLevel::L5.capacity_ratio(), 0.0);
}
#[test]
fn level_escalate() {
assert_eq!(DialLevel::L0.escalate(), Some(DialLevel::L1));
assert_eq!(DialLevel::L1.escalate(), Some(DialLevel::L2));
assert_eq!(DialLevel::L4.escalate(), Some(DialLevel::L5));
assert_eq!(DialLevel::L5.escalate(), None);
}
#[test]
fn level_deescalate() {
assert_eq!(DialLevel::L5.deescalate(), Some(DialLevel::L4));
assert_eq!(DialLevel::L1.deescalate(), Some(DialLevel::L0));
assert_eq!(DialLevel::L0.deescalate(), None);
}
#[test]
fn level_as_usize() {
assert_eq!(DialLevel::L0.as_usize(), 0);
assert_eq!(DialLevel::L5.as_usize(), 5);
}
#[test]
fn level_display() {
assert_eq!(DialLevel::L0.to_string(), "L0");
assert_eq!(DialLevel::L5.to_string(), "L5");
}
#[test]
fn level_default_is_l0() {
assert_eq!(DialLevel::default(), DialLevel::L0);
}
#[test]
fn level_serde_roundtrip() {
let json = serde_json::to_string(&DialLevel::L3).unwrap();
let back: DialLevel = serde_json::from_str(&json).unwrap();
assert_eq!(back, DialLevel::L3);
}
#[test]
fn level_ordering() {
assert!(DialLevel::L0 < DialLevel::L1);
assert!(DialLevel::L4 < DialLevel::L5);
assert!(DialLevel::L2 < DialLevel::L4);
}
#[test]
fn metrics_healthy_when_below_all_thresholds() {
let t = DialThresholds::default();
let m = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.5,
};
assert!(m.is_healthy(&t));
}
#[test]
fn metrics_unhealthy_when_error_rate_at_threshold() {
let t = DialThresholds::default();
let m = SystemMetrics {
error_rate: 0.05,
latency_ms: 50.0,
resource_usage: 0.5,
};
assert!(
!m.is_healthy(&t),
"error_rate >= threshold[0] is not healthy"
);
}
#[test]
fn metrics_unhealthy_when_latency_high() {
let t = DialThresholds::default();
let m = SystemMetrics {
error_rate: 0.01,
latency_ms: 150.0,
resource_usage: 0.5,
};
assert!(!m.is_healthy(&t));
}
#[test]
#[should_panic(expected = "recovery_consecutive_healthy must be >= 1")]
fn dial_zero_recovery_panics() {
let _ = CapacityDial::new(DialThresholds::default(), 0);
}
#[test]
fn dial_starts_at_l0() {
let dial = CapacityDial::default();
assert_eq!(dial.level(), DialLevel::L0);
assert_eq!(dial.capacity_ratio(), 1.0);
}
#[test]
fn evaluate_escalates_on_high_error_rate() {
let dial = CapacityDial::default();
let metrics = SystemMetrics {
error_rate: 0.25,
latency_ms: 50.0,
resource_usage: 0.5,
};
let level = dial.evaluate(&metrics);
assert_eq!(level, DialLevel::L3);
assert!((dial.capacity_ratio() - 0.4).abs() < 1e-9);
}
#[test]
fn evaluate_escalates_on_high_latency() {
let dial = CapacityDial::default();
let metrics = SystemMetrics {
error_rate: 0.01,
latency_ms: 600.0,
resource_usage: 0.5,
};
let level = dial.evaluate(&metrics);
assert_eq!(level, DialLevel::L3);
}
#[test]
fn evaluate_escalates_on_high_resource_usage() {
let dial = CapacityDial::default();
let metrics = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.92,
};
let level = dial.evaluate(&metrics);
assert_eq!(level, DialLevel::L3);
}
#[test]
fn evaluate_takes_max_level_across_metrics() {
let dial = CapacityDial::default();
let metrics = SystemMetrics {
error_rate: 0.15,
latency_ms: 1200.0,
resource_usage: 0.5,
};
let level = dial.evaluate(&metrics);
assert_eq!(level, DialLevel::L4);
}
#[test]
fn evaluate_escalates_to_l5_on_total_failure() {
let dial = CapacityDial::default();
let metrics = SystemMetrics {
error_rate: 0.6,
latency_ms: 3000.0,
resource_usage: 0.99,
};
let level = dial.evaluate(&metrics);
assert_eq!(level, DialLevel::L5);
assert_eq!(dial.capacity_ratio(), 0.0);
}
#[test]
fn evaluate_no_change_when_metrics_moderate_and_already_at_target() {
let dial = CapacityDial::default();
let m1 = SystemMetrics {
error_rate: 0.15,
latency_ms: 50.0,
resource_usage: 0.5,
};
dial.evaluate(&m1);
assert_eq!(dial.level(), DialLevel::L2);
let m2 = SystemMetrics {
error_rate: 0.12,
latency_ms: 50.0,
resource_usage: 0.5,
};
let level = dial.evaluate(&m2);
assert_eq!(level, DialLevel::L2);
}
#[test]
fn evaluate_recovers_after_consecutive_healthy() {
let dial = CapacityDial::default();
let bad = SystemMetrics {
error_rate: 0.25,
latency_ms: 600.0,
resource_usage: 0.92,
};
dial.evaluate(&bad);
assert_eq!(dial.level(), DialLevel::L3);
let healthy = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.5,
};
dial.evaluate(&healthy);
assert_eq!(dial.level(), DialLevel::L3, "1st healthy: no recovery yet");
dial.evaluate(&healthy);
assert_eq!(dial.level(), DialLevel::L3, "2nd healthy: no recovery yet");
dial.evaluate(&healthy);
assert_eq!(dial.level(), DialLevel::L2, "3rd healthy: recover to L2");
}
#[test]
fn evaluate_recovery_resets_on_unhealthy() {
let dial = CapacityDial::default();
dial.evaluate(&SystemMetrics {
error_rate: 0.15,
latency_ms: 50.0,
resource_usage: 0.5,
});
assert_eq!(dial.level(), DialLevel::L2);
let healthy = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.5,
};
dial.evaluate(&healthy);
dial.evaluate(&healthy);
let moderate = SystemMetrics {
error_rate: 0.12,
latency_ms: 50.0,
resource_usage: 0.5,
};
dial.evaluate(&moderate);
assert_eq!(dial.level(), DialLevel::L2, "no recovery");
dial.evaluate(&healthy);
dial.evaluate(&healthy);
assert_eq!(dial.level(), DialLevel::L2, "only 2 healthy after reset");
dial.evaluate(&healthy);
assert_eq!(
dial.level(),
DialLevel::L1,
"3rd healthy after reset: recover"
);
}
#[test]
fn evaluate_escalation_resets_healthy_counter() {
let dial = CapacityDial::default();
dial.evaluate(&SystemMetrics {
error_rate: 0.15,
latency_ms: 50.0,
resource_usage: 0.5,
});
let healthy = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.5,
};
dial.evaluate(&healthy);
dial.evaluate(&healthy);
dial.evaluate(&SystemMetrics {
error_rate: 0.35,
latency_ms: 1200.0,
resource_usage: 0.5,
});
assert_eq!(dial.level(), DialLevel::L4);
dial.evaluate(&healthy);
dial.evaluate(&healthy);
assert_eq!(dial.level(), DialLevel::L4, "escalation reset counter");
dial.evaluate(&healthy);
assert_eq!(dial.level(), DialLevel::L3, "3rd healthy: recover to L3");
}
#[test]
fn force_recover_deescalates_one_level() {
let dial = CapacityDial::default();
dial.evaluate(&SystemMetrics {
error_rate: 0.6,
latency_ms: 3000.0,
resource_usage: 0.99,
});
assert_eq!(dial.level(), DialLevel::L5);
dial.force_recover();
assert_eq!(dial.level(), DialLevel::L4);
dial.force_recover();
assert_eq!(dial.level(), DialLevel::L3);
}
#[test]
fn force_recover_at_l0_no_change() {
let dial = CapacityDial::default();
dial.force_recover();
assert_eq!(dial.level(), DialLevel::L0);
}
#[test]
fn reset_returns_to_l0() {
let dial = CapacityDial::default();
dial.evaluate(&SystemMetrics {
error_rate: 0.6,
latency_ms: 3000.0,
resource_usage: 0.99,
});
assert_eq!(dial.level(), DialLevel::L5);
dial.reset();
assert_eq!(dial.level(), DialLevel::L0);
assert_eq!(dial.capacity_ratio(), 1.0);
}
#[test]
fn thresholds_default_values() {
let t = DialThresholds::default();
assert_eq!(t.error_rate, [0.05, 0.10, 0.20, 0.30, 0.50]);
assert_eq!(t.latency_ms, [100.0, 200.0, 500.0, 1000.0, 2000.0]);
assert_eq!(t.resource_usage, [0.70, 0.80, 0.90, 0.95, 0.99]);
}
#[test]
fn thresholds_serde_roundtrip() {
let t = DialThresholds::default();
let json = serde_json::to_string(&t).unwrap();
let back: DialThresholds = serde_json::from_str(&json).unwrap();
assert_eq!(back.error_rate, t.error_rate);
assert_eq!(back.latency_ms, t.latency_ms);
assert_eq!(back.resource_usage, t.resource_usage);
}
#[test]
fn metrics_serde_roundtrip() {
let m = SystemMetrics {
error_rate: 0.15,
latency_ms: 250.0,
resource_usage: 0.85,
};
let json = serde_json::to_string(&m).unwrap();
let back: SystemMetrics = serde_json::from_str(&json).unwrap();
assert!((back.error_rate - 0.15).abs() < 1e-9);
assert!((back.latency_ms - 250.0).abs() < 1e-9);
assert!((back.resource_usage - 0.85).abs() < 1e-9);
}
#[test]
fn full_degradation_and_recovery_cycle() {
let dial = CapacityDial::default();
for _ in 0..5 {
dial.evaluate(&SystemMetrics {
error_rate: 0.6,
latency_ms: 3000.0,
resource_usage: 0.99,
});
}
assert_eq!(dial.level(), DialLevel::L5);
let healthy = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.5,
};
let expected_sequence = [
DialLevel::L4,
DialLevel::L3,
DialLevel::L2,
DialLevel::L1,
DialLevel::L0,
];
for (i, &expected) in expected_sequence.iter().enumerate() {
for _ in 0..3 {
dial.evaluate(&healthy);
}
assert_eq!(
dial.level(),
expected,
"recovery step {}: expected {:?}",
i,
expected
);
}
}
#[test]
fn concurrent_evaluation_state_stays_coherent() {
let dial = std::sync::Arc::new(CapacityDial::default());
let bad = SystemMetrics {
error_rate: 0.6,
latency_ms: 3000.0,
resource_usage: 0.99,
};
let healthy = SystemMetrics {
error_rate: 0.01,
latency_ms: 50.0,
resource_usage: 0.5,
};
std::thread::scope(|s| {
for _ in 0..4 {
let dial = dial.clone();
s.spawn(move || {
for _ in 0..200 {
dial.evaluate(&bad);
}
});
}
for _ in 0..4 {
let dial = dial.clone();
s.spawn(move || {
for _ in 0..200 {
dial.evaluate(&healthy);
}
});
}
});
let start = dial.level().as_usize();
let healthy_owned = healthy;
let mut evaluations = 0usize;
let mut prev = dial.level().as_usize();
loop {
let level = dial.evaluate(&healthy_owned);
evaluations += 1;
let now = level.as_usize();
assert!(
now <= prev,
"the tier must not climb back under healthy input: {prev} -> {now}"
);
prev = now;
if level == DialLevel::L0 {
break;
}
assert!(
evaluations <= 3 * (start + 1) + 8,
"recovery stepping out of control: still at {level} after {evaluations} evaluations"
);
}
assert_eq!(dial.capacity_ratio(), 1.0);
}
}