use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use parking_lot::RwLock;
use crate::error::Error;
use crate::isolation::IsolationToken;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CircuitHealth {
Healthy,
Degraded,
Unhealthy,
Dead,
}
impl CircuitHealth {
pub fn is_usable(&self) -> bool {
matches!(self, Self::Healthy | Self::Degraded)
}
pub fn should_rotate(&self) -> bool {
matches!(self, Self::Degraded | Self::Unhealthy | Self::Dead)
}
}
#[derive(Debug)]
pub struct CircuitMetrics {
pub requests: AtomicU64,
pub successes: AtomicU64,
pub failures: AtomicU64,
pub total_latency_ms: AtomicU64,
pub consecutive_failures: AtomicU32,
pub last_success: RwLock<Option<Instant>>,
pub last_failure: RwLock<Option<Instant>>,
pub created_at: Instant,
}
impl Default for CircuitMetrics {
fn default() -> Self {
Self::new()
}
}
impl CircuitMetrics {
pub fn new() -> Self {
Self {
requests: AtomicU64::new(0),
successes: AtomicU64::new(0),
failures: AtomicU64::new(0),
total_latency_ms: AtomicU64::new(0),
consecutive_failures: AtomicU32::new(0),
last_success: RwLock::new(None),
last_failure: RwLock::new(None),
created_at: Instant::now(),
}
}
pub fn record_success(&self, latency: Duration) {
self.requests.fetch_add(1, Ordering::Relaxed);
self.successes.fetch_add(1, Ordering::Relaxed);
self.total_latency_ms
.fetch_add(latency.as_millis() as u64, Ordering::Relaxed);
self.consecutive_failures.store(0, Ordering::Relaxed);
*self.last_success.write() = Some(Instant::now());
}
pub fn record_failure(&self) {
self.requests.fetch_add(1, Ordering::Relaxed);
self.failures.fetch_add(1, Ordering::Relaxed);
self.consecutive_failures.fetch_add(1, Ordering::Relaxed);
*self.last_failure.write() = Some(Instant::now());
}
pub fn success_rate(&self) -> f64 {
let total = self.requests.load(Ordering::Relaxed);
if total == 0 {
return 1.0;
}
let successes = self.successes.load(Ordering::Relaxed);
successes as f64 / total as f64
}
pub fn avg_latency(&self) -> Duration {
let successes = self.successes.load(Ordering::Relaxed);
if successes == 0 {
return Duration::ZERO;
}
let total_ms = self.total_latency_ms.load(Ordering::Relaxed);
Duration::from_millis(total_ms / successes)
}
pub fn health(&self) -> CircuitHealth {
let consecutive = self.consecutive_failures.load(Ordering::Relaxed);
let rate = self.success_rate();
if consecutive >= 5 {
return CircuitHealth::Dead;
}
if consecutive >= 3 || rate < 0.5 {
return CircuitHealth::Unhealthy;
}
if consecutive >= 1 || rate < 0.8 {
return CircuitHealth::Degraded;
}
CircuitHealth::Healthy
}
pub fn age(&self) -> Duration {
self.created_at.elapsed()
}
}
#[derive(Debug, Clone)]
pub struct RotationConfig {
pub max_consecutive_failures: u32,
pub min_success_rate: f64,
pub max_circuit_age: Duration,
pub rotation_cooldown: Duration,
pub rotate_on_timeout: bool,
pub rotate_on_connection_error: bool,
}
impl Default for RotationConfig {
fn default() -> Self {
Self {
max_consecutive_failures: 3,
min_success_rate: 0.7,
max_circuit_age: Duration::from_secs(600), rotation_cooldown: Duration::from_secs(30),
rotate_on_timeout: true,
rotate_on_connection_error: true,
}
}
}
impl RotationConfig {
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_max_failures(mut self, max: u32) -> Self {
self.max_consecutive_failures = max;
self
}
#[must_use]
pub fn with_min_success_rate(mut self, rate: f64) -> Self {
self.min_success_rate = rate.clamp(0.0, 1.0);
self
}
#[must_use]
pub fn with_max_age(mut self, age: Duration) -> Self {
self.max_circuit_age = age;
self
}
pub fn should_rotate_on_error(&self, error: &Error) -> bool {
match error {
Error::Timeout { .. } => self.rotate_on_timeout,
Error::Connection { .. } => self.rotate_on_connection_error,
Error::Http { .. } => false, _ => false,
}
}
}
#[derive(Debug)]
pub struct CircuitRotator {
config: RotationConfig,
metrics: Arc<RwLock<HashMap<IsolationToken, Arc<CircuitMetrics>>>>,
last_rotation: Arc<RwLock<HashMap<IsolationToken, Instant>>>,
}
impl Default for CircuitRotator {
fn default() -> Self {
Self::new(RotationConfig::default())
}
}
impl CircuitRotator {
pub fn new(config: RotationConfig) -> Self {
Self {
config,
metrics: Arc::new(RwLock::new(HashMap::new())),
last_rotation: Arc::new(RwLock::new(HashMap::new())),
}
}
pub fn get_metrics(&self, token: &IsolationToken) -> Arc<CircuitMetrics> {
let mut metrics = self.metrics.write();
metrics
.entry(*token)
.or_insert_with(|| Arc::new(CircuitMetrics::new()))
.clone()
}
pub fn record_success(&self, token: &IsolationToken, latency: Duration) {
self.get_metrics(token).record_success(latency);
}
pub fn record_failure(&self, token: &IsolationToken) {
self.get_metrics(token).record_failure();
}
pub fn should_rotate(&self, token: &IsolationToken) -> bool {
let metrics = self.get_metrics(token);
{
let last_rotation = self.last_rotation.read();
if let Some(&last) = last_rotation.get(token) {
if last.elapsed() < self.config.rotation_cooldown {
return false;
}
}
}
if metrics.age() > self.config.max_circuit_age {
return true;
}
let health = metrics.health();
if !health.is_usable() {
return true;
}
let consecutive = metrics.consecutive_failures.load(Ordering::Relaxed);
if consecutive >= self.config.max_consecutive_failures {
return true;
}
let requests = metrics.requests.load(Ordering::Relaxed);
if requests >= 10 && metrics.success_rate() < self.config.min_success_rate {
return true;
}
false
}
pub fn should_rotate_on_error(&self, token: &IsolationToken, error: &Error) -> bool {
if !self.config.should_rotate_on_error(error) {
return false;
}
self.record_failure(token);
self.should_rotate(token)
}
pub fn mark_rotated(&self, token: &IsolationToken) {
self.last_rotation.write().insert(*token, Instant::now());
self.metrics.write().remove(token);
}
pub fn get_health(&self, token: &IsolationToken) -> CircuitHealth {
self.get_metrics(token).health()
}
pub fn stats(&self) -> RotationStats {
let metrics = self.metrics.read();
let mut total_circuits = 0;
let mut healthy = 0;
let mut degraded = 0;
let mut unhealthy = 0;
let mut dead = 0;
for circuit_metrics in metrics.values() {
total_circuits += 1;
match circuit_metrics.health() {
CircuitHealth::Healthy => healthy += 1,
CircuitHealth::Degraded => degraded += 1,
CircuitHealth::Unhealthy => unhealthy += 1,
CircuitHealth::Dead => dead += 1,
}
}
RotationStats {
total_circuits,
healthy,
degraded,
unhealthy,
dead,
}
}
pub fn cleanup(&self, max_age: Duration) {
let mut metrics = self.metrics.write();
metrics.retain(|_, m| m.age() < max_age);
let mut last_rotation = self.last_rotation.write();
last_rotation.retain(|_, &mut t| t.elapsed() < max_age);
}
}
#[derive(Debug, Clone)]
pub struct RotationStats {
pub total_circuits: usize,
pub healthy: usize,
pub degraded: usize,
pub unhealthy: usize,
pub dead: usize,
}
impl RotationStats {
pub fn health_percentage(&self) -> f64 {
if self.total_circuits == 0 {
return 100.0;
}
(self.healthy as f64 / self.total_circuits as f64) * 100.0
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RotationDecision {
Keep,
Rotate {
reason: RotationReason,
},
RotateNow {
reason: RotationReason,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RotationReason {
ConsecutiveFailures,
LowSuccessRate,
AgeLimit,
Error,
Manual,
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
#[test]
fn test_circuit_metrics() {
let metrics = CircuitMetrics::new();
metrics.record_success(Duration::from_millis(100));
metrics.record_success(Duration::from_millis(200));
metrics.record_failure();
assert_eq!(metrics.requests.load(Ordering::Relaxed), 3);
assert_eq!(metrics.successes.load(Ordering::Relaxed), 2);
assert_eq!(metrics.failures.load(Ordering::Relaxed), 1);
assert!(metrics.success_rate() > 0.6 && metrics.success_rate() < 0.7);
}
#[test]
fn test_circuit_health() {
let metrics = CircuitMetrics::new();
assert_eq!(metrics.health(), CircuitHealth::Healthy);
metrics.record_success(Duration::from_millis(100));
metrics.record_success(Duration::from_millis(100));
metrics.record_success(Duration::from_millis(100));
metrics.record_success(Duration::from_millis(100));
assert_eq!(metrics.health(), CircuitHealth::Healthy);
metrics.record_failure();
assert_eq!(metrics.health(), CircuitHealth::Degraded);
metrics.record_failure();
metrics.record_failure();
assert_eq!(metrics.health(), CircuitHealth::Unhealthy);
metrics.record_failure();
metrics.record_failure();
assert_eq!(metrics.health(), CircuitHealth::Dead);
}
#[test]
fn test_circuit_rotator() {
let rotator = CircuitRotator::default();
let token = IsolationToken::new();
assert!(!rotator.should_rotate(&token));
for _ in 0..3 {
rotator.record_failure(&token);
}
assert!(rotator.should_rotate(&token));
}
#[test]
fn test_rotation_config() {
let config = RotationConfig::new()
.with_max_failures(5)
.with_min_success_rate(0.9)
.with_max_age(Duration::from_secs(300));
assert_eq!(config.max_consecutive_failures, 5);
assert!((config.min_success_rate - 0.9).abs() < f64::EPSILON);
assert_eq!(config.max_circuit_age, Duration::from_secs(300));
}
#[test]
fn test_rotation_stats() {
let rotator = CircuitRotator::default();
let token1 = IsolationToken::new();
rotator.get_metrics(&token1);
let token2 = IsolationToken::from_raw(2);
rotator.record_failure(&token2);
let stats = rotator.stats();
assert_eq!(stats.total_circuits, 2);
assert!(stats.healthy >= 1);
assert!(stats.healthy + stats.degraded + stats.unhealthy + stats.dead == 2);
}
}