use crate::clock::{Clock, SystemClock};
use crate::constants::{
DEFAULT_MANUAL_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
DEFAULT_MANUAL_CIRCUIT_BREAKER_HALF_OPEN_INTERVAL_SECS,
DEFAULT_MANUAL_CIRCUIT_BREAKER_HALF_OPEN_MAX_SUCCESSES,
DEFAULT_MANUAL_CIRCUIT_BREAKER_MIN_REQUESTS,
};
use crate::error::CircuitState;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
#[derive(Debug, Clone)]
pub struct ManualCircuitBreakerConfig {
pub failure_threshold: f64,
pub min_requests: u64,
pub half_open_interval: Duration,
pub half_open_max_successes: u64,
}
impl Default for ManualCircuitBreakerConfig {
fn default() -> Self {
Self {
failure_threshold: DEFAULT_MANUAL_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
min_requests: DEFAULT_MANUAL_CIRCUIT_BREAKER_MIN_REQUESTS,
half_open_interval: Duration::from_secs(
DEFAULT_MANUAL_CIRCUIT_BREAKER_HALF_OPEN_INTERVAL_SECS,
),
half_open_max_successes: DEFAULT_MANUAL_CIRCUIT_BREAKER_HALF_OPEN_MAX_SUCCESSES,
}
}
}
struct ManualCircuitBreakerInner {
config: ManualCircuitBreakerConfig,
state: Mutex<CircuitState>,
opened_at: Mutex<Option<Instant>>,
failures: AtomicU64,
successes: AtomicU64,
total: AtomicU64,
half_open_successes: AtomicU64,
}
#[derive(Clone)]
pub struct ManualCircuitBreaker {
inner: Arc<ManualCircuitBreakerInner>,
clock: Arc<dyn Clock>,
}
impl ManualCircuitBreaker {
pub fn new(config: ManualCircuitBreakerConfig) -> Self {
Self::with_clock(config, Arc::new(SystemClock))
}
pub fn with_clock(config: ManualCircuitBreakerConfig, clock: Arc<dyn Clock>) -> Self {
Self {
inner: Arc::new(ManualCircuitBreakerInner {
config,
state: Mutex::new(CircuitState::Closed),
opened_at: Mutex::new(None),
failures: AtomicU64::new(0),
successes: AtomicU64::new(0),
total: AtomicU64::new(0),
half_open_successes: AtomicU64::new(0),
}),
clock,
}
}
pub async fn state(&self) -> CircuitState {
let mut state = self.inner.state.lock().await;
self.maybe_transition(&mut state).await;
*state
}
pub async fn admit(&self) -> bool {
self.state().await != CircuitState::Open
}
pub async fn record_success(&self) {
self.inner.successes.fetch_add(1, Ordering::Relaxed);
self.inner.total.fetch_add(1, Ordering::Relaxed);
let mut state = self.inner.state.lock().await;
if *state == CircuitState::HalfOpen {
let count = self
.inner
.half_open_successes
.fetch_add(1, Ordering::Relaxed)
+ 1;
if count >= self.inner.config.half_open_max_successes {
self.reset_counters();
*state = CircuitState::Closed;
}
}
}
pub async fn record_failure(&self) {
self.inner.failures.fetch_add(1, Ordering::Relaxed);
self.inner.total.fetch_add(1, Ordering::Relaxed);
let mut state = self.inner.state.lock().await;
if *state == CircuitState::HalfOpen {
self.open(&mut state).await;
} else if *state == CircuitState::Closed {
self.maybe_trip(&mut state).await;
}
}
async fn maybe_transition(&self, state: &mut CircuitState) {
if *state == CircuitState::Open {
let opened_at = self.inner.opened_at.lock().await;
if let Some(instant) = *opened_at
&& self.clock.now().duration_since(instant) >= self.inner.config.half_open_interval
{
self.inner.half_open_successes.store(0, Ordering::Relaxed);
*state = CircuitState::HalfOpen;
}
}
}
async fn maybe_trip(&self, state: &mut CircuitState) {
let total = self.inner.total.load(Ordering::Relaxed);
if total < self.inner.config.min_requests {
return;
}
let failures = self.inner.failures.load(Ordering::Relaxed);
let ratio = failures as f64 / total as f64;
if ratio >= self.inner.config.failure_threshold {
self.open(state).await;
}
}
async fn open(&self, state: &mut CircuitState) {
*state = CircuitState::Open;
*self.inner.opened_at.lock().await = Some(self.clock.now());
}
fn reset_counters(&self) {
self.inner.failures.store(0, Ordering::Relaxed);
self.inner.successes.store(0, Ordering::Relaxed);
self.inner.total.store(0, Ordering::Relaxed);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::clock::MockClock;
use std::time::Duration;
fn mock_breaker(config: ManualCircuitBreakerConfig) -> (ManualCircuitBreaker, Arc<MockClock>) {
let mock = Arc::new(MockClock::new());
let breaker = ManualCircuitBreaker::with_clock(config, mock.clone() as Arc<dyn Clock>);
(breaker, mock)
}
#[test]
fn default_config() {
let config = ManualCircuitBreakerConfig::default();
assert!((config.failure_threshold - 0.5).abs() < f64::EPSILON);
assert_eq!(config.min_requests, 10);
assert_eq!(config.half_open_interval, Duration::from_secs(30));
assert_eq!(config.half_open_max_successes, 3);
}
#[tokio::test]
async fn starts_closed() {
let (cb, _clock) = mock_breaker(ManualCircuitBreakerConfig::default());
assert_eq!(cb.state().await, CircuitState::Closed);
}
#[tokio::test]
async fn trips_on_threshold() {
let config = ManualCircuitBreakerConfig {
min_requests: 5,
failure_threshold: 0.5,
..Default::default()
};
let (cb, _clock) = mock_breaker(config);
for _ in 0..5 {
cb.record_failure().await;
}
assert_eq!(cb.state().await, CircuitState::Open);
}
#[tokio::test]
async fn trips_exactly_at_ratio_boundary() {
let config = ManualCircuitBreakerConfig {
min_requests: 4,
failure_threshold: 0.5,
..Default::default()
};
let (cb, _clock) = mock_breaker(config);
for _ in 0..2 {
cb.record_success().await;
}
for _ in 0..2 {
cb.record_failure().await;
}
assert_eq!(cb.state().await, CircuitState::Open);
}
#[tokio::test]
async fn stays_closed_below_threshold() {
let config = ManualCircuitBreakerConfig {
min_requests: 10,
failure_threshold: 0.5,
..Default::default()
};
let (cb, _clock) = mock_breaker(config);
for _ in 0..6 {
cb.record_success().await;
}
for _ in 0..3 {
cb.record_failure().await;
}
assert_eq!(cb.state().await, CircuitState::Closed);
}
#[tokio::test]
async fn below_min_requests_never_trips() {
let config = ManualCircuitBreakerConfig {
min_requests: 5,
failure_threshold: 0.5,
..Default::default()
};
let (cb, _clock) = mock_breaker(config);
for _ in 0..4 {
cb.record_failure().await;
}
assert_eq!(cb.state().await, CircuitState::Closed);
}
#[tokio::test]
async fn recovers_through_half_open_after_cooldown() {
let config = ManualCircuitBreakerConfig {
min_requests: 2,
failure_threshold: 0.5,
..Default::default()
};
let interval = config.half_open_interval;
let (cb, clock) = mock_breaker(config);
for _ in 0..2 {
cb.record_failure().await;
}
assert_eq!(cb.state().await, CircuitState::Open);
clock.advance(interval + Duration::from_millis(50));
assert_eq!(cb.state().await, CircuitState::HalfOpen);
for _ in 0..3 {
cb.record_success().await;
}
assert_eq!(cb.state().await, CircuitState::Closed);
}
#[tokio::test]
async fn half_open_probe_failure_reopens() {
let config = ManualCircuitBreakerConfig {
min_requests: 2,
failure_threshold: 0.5,
..Default::default()
};
let interval = config.half_open_interval;
let (cb, clock) = mock_breaker(config);
for _ in 0..2 {
cb.record_failure().await;
}
assert_eq!(cb.state().await, CircuitState::Open);
clock.advance(interval + Duration::from_millis(50));
assert_eq!(cb.state().await, CircuitState::HalfOpen);
cb.record_failure().await;
assert_eq!(cb.state().await, CircuitState::Open);
clock.advance(interval + Duration::from_millis(50));
assert_eq!(cb.state().await, CircuitState::HalfOpen);
}
#[tokio::test]
async fn admit_gates_while_open() {
let config = ManualCircuitBreakerConfig {
min_requests: 2,
failure_threshold: 0.5,
..Default::default()
};
let interval = config.half_open_interval;
let (cb, clock) = mock_breaker(config);
assert!(cb.admit().await, "Closed state must admit");
for _ in 0..2 {
cb.record_failure().await;
}
assert!(
!cb.admit().await,
"must reject before the Open cooldown expires"
);
clock.advance(interval + Duration::from_millis(50));
assert!(
cb.admit().await,
"after cooldown into HalfOpen, admit the probe"
);
}
#[tokio::test]
async fn clone_shares_state_and_reports_from_spawned_task() {
let config = ManualCircuitBreakerConfig {
min_requests: 3,
failure_threshold: 0.5,
half_open_interval: Duration::from_secs(3600),
..Default::default()
};
let (cb, _clock) = mock_breaker(config);
let cloned = cb.clone();
let handle = tokio::spawn(async move {
for _ in 0..3 {
cloned.record_failure().await;
}
});
handle.await.unwrap();
assert_eq!(cb.state().await, CircuitState::Open);
}
}