use crate::clock::{Clock, SystemClock};
use crate::constants::{
DEFAULT_CIRCUIT_BREAKER_FAILURE_THRESHOLD, DEFAULT_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS,
DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_DURATION_MILLIS,
DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE_THRESHOLD, DEFAULT_CIRCUIT_BREAKER_SUCCESS_THRESHOLD,
DEFAULT_CIRCUIT_BREAKER_TIMEOUT_SECS,
};
use crate::error::{CircuitBreakerStats, CircuitCallError, CircuitState, LimiteronError};
use crate::i18n::t;
use log::{info, trace, warn};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
pub trait FailureClassifier<E>: Send + Sync + std::fmt::Debug {
fn is_counted_as_failure(&self, error: &E) -> bool;
}
#[derive(Debug)]
pub struct DefaultFailureClassifier;
impl FailureClassifier<LimiteronError> for DefaultFailureClassifier {
fn is_counted_as_failure(&self, error: &LimiteronError) -> bool {
match error {
LimiteronError::StorageError(storage_err) => storage_err.is_transient(),
LimiteronError::LimitError(_) | LimiteronError::CircuitBreakerError(_) => false,
LimiteronError::ValidationError(_) => false,
_ => true,
}
}
}
pub struct CircuitBreakerConfig<E = LimiteronError> {
pub failure_threshold: u64,
pub success_threshold: u64,
pub timeout: Duration,
pub half_open_max_calls: u64,
pub half_open_max_duration: Duration,
pub slow_call_duration_threshold: Duration,
pub slow_call_rate_threshold: f64,
pub error_classifier: Arc<dyn FailureClassifier<E>>,
}
impl<E> std::fmt::Debug for CircuitBreakerConfig<E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CircuitBreakerConfig")
.field("failure_threshold", &self.failure_threshold)
.field("success_threshold", &self.success_threshold)
.field("timeout", &self.timeout)
.field("half_open_max_calls", &self.half_open_max_calls)
.field("half_open_max_duration", &self.half_open_max_duration)
.field(
"slow_call_duration_threshold",
&self.slow_call_duration_threshold,
)
.field("slow_call_rate_threshold", &self.slow_call_rate_threshold)
.field("error_classifier", &self.error_classifier)
.finish()
}
}
impl<E> Clone for CircuitBreakerConfig<E> {
fn clone(&self) -> Self {
Self {
failure_threshold: self.failure_threshold,
success_threshold: self.success_threshold,
timeout: self.timeout,
half_open_max_calls: self.half_open_max_calls,
half_open_max_duration: self.half_open_max_duration,
slow_call_duration_threshold: self.slow_call_duration_threshold,
slow_call_rate_threshold: self.slow_call_rate_threshold,
error_classifier: Arc::clone(&self.error_classifier),
}
}
}
impl<E> CircuitBreakerConfig<E> {
pub fn with_error_classifier(error_classifier: Arc<dyn FailureClassifier<E>>) -> Self {
Self {
failure_threshold: DEFAULT_CIRCUIT_BREAKER_FAILURE_THRESHOLD,
success_threshold: DEFAULT_CIRCUIT_BREAKER_SUCCESS_THRESHOLD,
timeout: Duration::from_secs(DEFAULT_CIRCUIT_BREAKER_TIMEOUT_SECS),
half_open_max_calls: DEFAULT_CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS,
half_open_max_duration: Duration::from_secs(30),
slow_call_duration_threshold: Duration::from_millis(
DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_DURATION_MILLIS,
),
slow_call_rate_threshold: DEFAULT_CIRCUIT_BREAKER_SLOW_CALL_RATE_THRESHOLD,
error_classifier,
}
}
pub fn half_open_max_calls(mut self, max_calls: u64) -> Self {
self.half_open_max_calls = max_calls;
self
}
pub fn slow_call_duration_threshold(mut self, threshold: Duration) -> Self {
self.slow_call_duration_threshold = threshold;
self
}
pub fn slow_call_rate_threshold(mut self, threshold: f64) -> Self {
self.slow_call_rate_threshold = threshold;
self
}
pub fn error_classifier(mut self, classifier: Arc<dyn FailureClassifier<E>>) -> Self {
self.error_classifier = classifier;
self
}
}
impl Default for CircuitBreakerConfig<LimiteronError> {
fn default() -> Self {
Self::with_error_classifier(Arc::new(DefaultFailureClassifier))
}
}
impl CircuitBreakerConfig<LimiteronError> {
pub fn new(failure_threshold: u64, success_threshold: u64, timeout: Duration) -> Self {
Self {
failure_threshold,
success_threshold,
timeout,
..Self::default()
}
}
}
pub struct CircuitBreaker<E = LimiteronError> {
state: Arc<RwLock<CircuitState>>,
failure_count: Arc<AtomicU64>,
success_count: Arc<AtomicU64>,
total_calls: Arc<AtomicU64>,
slow_call_count: Arc<AtomicU64>,
last_failure_time: Arc<RwLock<Option<Instant>>>,
half_open_entered_at: Arc<RwLock<Option<Instant>>>,
last_state_change: Arc<RwLock<Option<Instant>>>,
last_failure_time_utc: Arc<RwLock<Option<chrono::DateTime<chrono::Utc>>>>,
last_state_change_utc: Arc<RwLock<Option<chrono::DateTime<chrono::Utc>>>>,
half_open_calls: Arc<AtomicU64>,
config: CircuitBreakerConfig<E>,
clock: Arc<dyn Clock>,
#[cfg(feature = "event-system")]
event_emitter: Option<Arc<crate::events::EventEmitter>>,
}
#[derive(Debug, Clone)]
pub struct CircuitBreakerBuilder {
config: CircuitBreakerConfig<LimiteronError>,
}
impl CircuitBreakerBuilder {
pub fn new() -> Self {
Self {
config: CircuitBreakerConfig::default(),
}
}
pub fn failure_threshold(mut self, failure_threshold: u64) -> Self {
self.config.failure_threshold = failure_threshold;
self
}
pub fn success_threshold(mut self, success_threshold: u64) -> Self {
self.config.success_threshold = success_threshold;
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.config.timeout = timeout;
self
}
pub fn half_open_max_duration(mut self, duration: Duration) -> Self {
self.config.half_open_max_duration = duration;
self
}
pub fn half_open_max_calls(mut self, max_calls: u64) -> Self {
self.config.half_open_max_calls = max_calls;
self
}
pub fn slow_call_duration_threshold(mut self, threshold: Duration) -> Self {
self.config.slow_call_duration_threshold = threshold;
self
}
pub fn slow_call_rate_threshold(mut self, threshold: f64) -> Self {
self.config.slow_call_rate_threshold = threshold;
self
}
pub fn error_classifier(
mut self,
classifier: Arc<dyn FailureClassifier<LimiteronError>>,
) -> Self {
self.config.error_classifier = classifier;
self
}
pub fn build(&self) -> CircuitBreaker {
CircuitBreaker::with_dependencies(self.config.clone())
}
}
impl Default for CircuitBreakerBuilder {
fn default() -> Self {
Self::new()
}
}
impl<E> CircuitBreaker<E> {
pub fn with_dependencies(config: CircuitBreakerConfig<E>) -> Self {
Self::with_clock(config, Arc::new(SystemClock))
}
pub fn with_clock(config: CircuitBreakerConfig<E>, clock: Arc<dyn Clock>) -> Self {
info!(
"{}",
t(
"circuit-created",
&[
("failure_threshold", config.failure_threshold.to_string(),),
("success_threshold", config.success_threshold.to_string(),),
("timeout", format!("{:?}", config.timeout)),
],
)
);
Self {
state: Arc::new(RwLock::new(CircuitState::Closed)),
failure_count: Arc::new(AtomicU64::new(0)),
success_count: Arc::new(AtomicU64::new(0)),
total_calls: Arc::new(AtomicU64::new(0)),
slow_call_count: Arc::new(AtomicU64::new(0)),
last_failure_time: Arc::new(RwLock::new(None)),
last_state_change: Arc::new(RwLock::new(Some(clock.now()))),
last_failure_time_utc: Arc::new(RwLock::new(None)),
last_state_change_utc: Arc::new(RwLock::new(Some(chrono::Utc::now()))),
half_open_calls: Arc::new(AtomicU64::new(0)),
half_open_entered_at: Arc::new(RwLock::new(None)),
config,
clock,
#[cfg(feature = "event-system")]
event_emitter: None,
}
}
pub fn new(config: CircuitBreakerConfig<E>) -> Self {
Self::with_dependencies(config)
}
}
impl CircuitBreaker<LimiteronError> {
pub fn builder() -> CircuitBreakerBuilder {
CircuitBreakerBuilder::new()
}
}
impl Default for CircuitBreaker<LimiteronError> {
fn default() -> Self {
Self::with_dependencies(CircuitBreakerConfig::default())
}
}
impl<E> CircuitBreaker<E> {
pub async fn execute<F, Fut, T>(&self, operation: F) -> Result<T, CircuitCallError<E>>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
{
let state = self.state.read().await;
let mut half_open_probe = false;
match *state {
CircuitState::Open => {
let last_failure = self.last_failure_time.read().await;
if let Some(last_failure) = *last_failure {
if self.clock.now().duration_since(last_failure) >= self.config.timeout {
drop(state);
if self.transition_to_half_open_if_open().await {
half_open_probe = true;
} else if *self.state.read().await == CircuitState::HalfOpen {
half_open_probe = true;
}
} else {
drop(state);
warn!("{}", t("circuit-open-rejecting", &[]));
return Err(CircuitCallError::Open);
}
} else {
drop(state);
warn!("{}", t("circuit-open-rejecting", &[]));
return Err(CircuitCallError::Open);
}
}
CircuitState::HalfOpen => {
half_open_probe = true;
drop(state);
}
CircuitState::Closed => {
drop(state);
}
}
if half_open_probe {
loop {
let calls = self.half_open_calls.load(Ordering::Relaxed);
if calls >= self.config.half_open_max_calls {
let escaped = {
let entered = self.half_open_entered_at.read().await;
matches!(
*entered,
Some(t)
if self.clock.now().duration_since(t)
>= self.config.half_open_max_duration
)
};
if escaped {
self.transition_to(CircuitState::Open).await;
warn!("{}", t("circuit-half-open-escape-timeout", &[]));
return Err(CircuitCallError::Open);
}
warn!("{}", t("circuit-half-open-limit-reached", &[]));
return Err(CircuitCallError::Open);
}
match self.half_open_calls.compare_exchange(
calls,
calls + 1,
Ordering::AcqRel,
Ordering::Relaxed,
) {
Ok(_) => break,
Err(_) => std::hint::spin_loop(), }
}
}
self.total_calls.fetch_add(1, Ordering::Relaxed);
let start_time = self.clock.now();
let result = operation().await;
let elapsed = start_time.elapsed();
self.record_call_duration(elapsed).await;
match result {
Ok(value) => {
self.on_success().await;
Ok(value)
}
Err(e) => {
self.on_failure_probe_aware(&e, half_open_probe).await;
Err(CircuitCallError::Inner(e))
}
}
}
async fn on_success(&self) {
let state = self.state.read().await;
match *state {
CircuitState::Closed => {
self.failure_count.store(0, Ordering::Relaxed);
self.success_count.fetch_add(1, Ordering::Relaxed);
trace!("operation succeeded (closed state)");
}
CircuitState::HalfOpen => {
let success_count = self.success_count.fetch_add(1, Ordering::Relaxed) + 1;
if success_count >= self.config.success_threshold {
drop(state);
self.transition_to_closed_if_half_open().await;
} else {
trace!(
"operation succeeded (half-open state): {}/{}",
success_count, self.config.success_threshold
);
}
}
CircuitState::Open => {
warn!("{}", t("circuit-success-while-open", &[]));
}
}
}
async fn on_failure_probe_aware(&self, error: &E, was_probe: bool) {
if !self.config.error_classifier.is_counted_as_failure(error) {
trace!("error not counted as failure by error classifier");
return;
}
let state = self.state.read().await;
match *state {
CircuitState::Closed => {
if was_probe {
drop(state);
warn!("{}", t("circuit-half-open-probe-failed", &[]));
self.transition_to_open().await;
return;
}
let failure_count = self.failure_count.fetch_add(1, Ordering::Relaxed) + 1;
*self.last_failure_time.write().await = Some(self.clock.now());
*self.last_failure_time_utc.write().await = Some(chrono::Utc::now());
if failure_count >= self.config.failure_threshold {
drop(state);
self.transition_to_open().await;
} else {
trace!(
"operation failed (closed state): {}/{}",
failure_count, self.config.failure_threshold
);
}
}
CircuitState::HalfOpen => {
drop(state);
self.transition_to_open().await;
}
CircuitState::Open => {
warn!("{}", t("circuit-failure-while-open", &[]));
}
}
}
}
impl<E> CircuitBreaker<E> {
async fn record_call_duration(&self, elapsed: Duration) {
if elapsed >= self.config.slow_call_duration_threshold {
let slow_calls = self.slow_call_count.fetch_add(1, Ordering::Relaxed) + 1;
let total_calls = self.total_calls.load(Ordering::Relaxed);
trace!(
"slow call detected: elapsed={:?}, threshold={:?}, slow_calls={}/{}",
elapsed, self.config.slow_call_duration_threshold, slow_calls, total_calls
);
self.check_slow_call_rate(slow_calls, total_calls).await;
}
}
async fn check_slow_call_rate(&self, slow_calls: u64, total_calls: u64) {
if total_calls == 0 {
return;
}
let slow_call_rate = slow_calls as f64 / total_calls as f64;
if slow_call_rate >= self.config.slow_call_rate_threshold {
let state = self.state.read().await;
if *state == CircuitState::Closed {
drop(state);
warn!(
"{}",
t(
"circuit-slow-call-rate-exceeded",
&[
("rate", format!("{:.2}", slow_call_rate * 100.0)),
(
"threshold",
format!("{:.2}", self.config.slow_call_rate_threshold * 100.0)
),
],
)
);
self.transition_to_open().await;
}
}
}
async fn transition_to(&self, new_state: CircuitState) {
let old_state = *self.state.read().await;
if old_state == new_state {
return; }
*self.state.write().await = new_state;
self.finalize_transition(old_state, new_state).await;
}
async fn transition_to_closed_if_half_open(&self) -> bool {
let old_state = {
let mut state = self.state.write().await;
if *state != CircuitState::HalfOpen {
return false;
}
let old = *state;
*state = CircuitState::Closed;
old
};
self.finalize_transition(old_state, CircuitState::Closed)
.await;
true
}
async fn finalize_transition(&self, old_state: CircuitState, new_state: CircuitState) {
*self.last_state_change.write().await = Some(self.clock.now());
*self.last_state_change_utc.write().await = Some(chrono::Utc::now());
match new_state {
CircuitState::Open => {
self.success_count.store(0, Ordering::Relaxed);
self.half_open_calls.store(0, Ordering::Relaxed);
*self.last_failure_time.write().await = Some(self.clock.now());
*self.half_open_entered_at.write().await = None;
warn!(
"{}",
t(
"circuit-state-changed-open",
&[
("old_state", format!("{:?}", old_state)),
(
"failure_count",
self.failure_count.load(Ordering::Relaxed).to_string()
),
],
)
);
}
CircuitState::HalfOpen => {
self.success_count.store(0, Ordering::Relaxed);
let mut entered = self.half_open_entered_at.write().await;
*entered = Some(self.clock.now());
self.half_open_calls.store(0, Ordering::Relaxed);
info!(
"{}",
t(
"circuit-state-changed-half-open",
&[("old_state", format!("{:?}", old_state))],
)
);
}
CircuitState::Closed => {
self.failure_count.store(0, Ordering::Relaxed);
self.success_count.store(0, Ordering::Relaxed);
self.half_open_calls.store(0, Ordering::Relaxed);
self.slow_call_count.store(0, Ordering::Relaxed);
*self.half_open_entered_at.write().await = None;
self.total_calls.store(0, Ordering::Relaxed);
info!(
"{}",
t(
"circuit-state-changed-closed",
&[("old_state", format!("{:?}", old_state))],
)
);
}
}
#[cfg(feature = "event-system")]
{
if let Some(ref emitter) = self.event_emitter {
let old_state_str = format!("{:?}", old_state);
let new_state_str = format!("{:?}", new_state);
let event =
crate::events::Event::new(crate::events::EventType::CircuitStateChanged {
from: old_state_str,
to: new_state_str,
});
if let Err(e) = emitter.emit(event).await {
log::error!("Failed to emit circuit state change event: {}", e);
}
}
}
}
async fn transition_to_open(&self) {
self.transition_to(CircuitState::Open).await;
}
async fn transition_to_half_open_if_open(&self) -> bool {
let old_state = {
let mut state = self.state.write().await;
if *state != CircuitState::Open {
return false;
}
let old = *state;
*state = CircuitState::HalfOpen;
old
};
self.finalize_transition(old_state, CircuitState::HalfOpen)
.await;
true
}
async fn is_state(&self, target_state: CircuitState) -> bool {
let state = self.state.read().await;
*state == target_state
}
pub async fn is_open(&self) -> bool {
self.is_state(CircuitState::Open).await
}
pub async fn is_half_open(&self) -> bool {
self.is_state(CircuitState::HalfOpen).await
}
pub async fn is_closed(&self) -> bool {
self.is_state(CircuitState::Closed).await
}
pub async fn get_state(&self) -> CircuitState {
*self.state.read().await
}
pub async fn reset(&self) {
info!("{}", t("circuit-reset", &[]));
*self.state.write().await = CircuitState::Closed;
self.failure_count.store(0, Ordering::Relaxed);
self.success_count.store(0, Ordering::Relaxed);
self.total_calls.store(0, Ordering::Relaxed);
self.slow_call_count.store(0, Ordering::Relaxed);
*self.last_failure_time.write().await = None;
*self.last_failure_time_utc.write().await = None;
*self.last_state_change.write().await = Some(self.clock.now());
*self.last_state_change_utc.write().await = Some(chrono::Utc::now());
self.half_open_calls.store(0, Ordering::Relaxed);
}
pub async fn get_stats(&self) -> CircuitBreakerStats {
let state = *self.state.read().await;
let last_failure = self.last_failure_time_utc.read().await;
let last_state_change = self.last_state_change_utc.read().await;
CircuitBreakerStats {
state,
failure_count: self.failure_count.load(Ordering::Relaxed),
success_count: self.success_count.load(Ordering::Relaxed),
total_calls: self.total_calls.load(Ordering::Relaxed),
last_failure_time: *last_failure,
last_state_change: *last_state_change,
}
}
pub fn config(&self) -> &CircuitBreakerConfig<E> {
&self.config
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_circuit_breaker_config_default() {
let config = CircuitBreakerConfig::default();
assert_eq!(config.failure_threshold, 5);
assert_eq!(config.success_threshold, 3);
assert_eq!(config.timeout, Duration::from_secs(30));
assert_eq!(config.half_open_max_calls, 3);
}
#[test]
fn test_circuit_breaker_config_new() {
let config = CircuitBreakerConfig::new(10, 3, Duration::from_secs(120));
assert_eq!(config.failure_threshold, 10);
assert_eq!(config.success_threshold, 3);
assert_eq!(config.timeout, Duration::from_secs(120));
}
#[test]
fn test_circuit_breaker_config_builder() {
let config =
CircuitBreakerConfig::new(5, 2, Duration::from_secs(60)).half_open_max_calls(5);
assert_eq!(config.half_open_max_calls, 5);
}
#[tokio::test]
async fn test_circuit_breaker_initial_state() {
let breaker = CircuitBreaker::default();
assert!(breaker.is_closed().await);
assert!(!breaker.is_open().await);
assert!(!breaker.is_half_open().await);
let stats = breaker.get_stats().await;
assert_eq!(stats.state, CircuitState::Closed);
assert_eq!(stats.failure_count, 0);
assert_eq!(stats.success_count, 0);
assert_eq!(stats.total_calls, 0);
}
#[tokio::test]
async fn test_circuit_breaker_success() {
let breaker = CircuitBreaker::default();
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
let stats = breaker.get_stats().await;
assert_eq!(stats.success_count, 1);
assert_eq!(stats.failure_count, 0);
assert_eq!(stats.total_calls, 1);
assert!(breaker.is_closed().await);
}
#[tokio::test]
async fn test_circuit_breaker_failure() {
let config = CircuitBreakerConfig::new(3, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
assert!(result.is_err());
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 1);
assert!(breaker.is_closed().await);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
assert!(result.is_err());
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 2);
assert!(breaker.is_closed().await);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
assert!(result.is_err());
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 3);
assert!(breaker.is_open().await);
}
#[tokio::test]
async fn test_circuit_breaker_open_rejects_requests() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_err());
let call_err = result.unwrap_err();
assert!(call_err.is_open(), "got: {call_err:?}");
assert_eq!(call_err.to_string(), "circuit breaker is open");
}
#[tokio::test]
async fn test_circuit_breaker_half_open_recovery() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_millis(100));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(breaker.is_half_open().await);
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(breaker.is_closed().await);
}
#[tokio::test]
async fn test_circuit_breaker_half_open_failure() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_millis(100));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(breaker.is_half_open().await);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
assert!(result.is_err());
assert!(breaker.is_open().await);
}
#[tokio::test]
async fn test_circuit_breaker_reset() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
breaker.reset().await;
assert!(breaker.is_closed().await);
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 0);
assert_eq!(stats.success_count, 0);
assert_eq!(stats.total_calls, 0);
}
#[tokio::test]
async fn test_circuit_breaker_get_state() {
let breaker = CircuitBreaker::default();
assert_eq!(breaker.get_state().await, CircuitState::Closed);
}
#[tokio::test]
async fn test_circuit_breaker_get_stats() {
let breaker = CircuitBreaker::default();
let _ = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
let stats = breaker.get_stats().await;
assert_eq!(stats.state, CircuitState::Closed);
assert_eq!(stats.success_count, 1);
assert_eq!(stats.total_calls, 1);
assert!(stats.last_state_change.is_some());
}
#[tokio::test]
async fn test_circuit_breaker_half_open_max_calls() {
let config =
CircuitBreakerConfig::new(2, 3, Duration::from_millis(100)).half_open_max_calls(2);
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("test error".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_err());
let call_err = result.unwrap_err();
assert!(call_err.is_open(), "got: {call_err:?}");
}
#[tokio::test]
async fn test_circuit_breaker_config() {
let config = CircuitBreakerConfig::new(10, 5, Duration::from_secs(30));
let breaker = CircuitBreaker::new(config);
let breaker_config = breaker.config();
assert_eq!(breaker_config.failure_threshold, 10);
assert_eq!(breaker_config.success_threshold, 5);
assert_eq!(breaker_config.timeout, Duration::from_secs(30));
}
#[tokio::test]
async fn test_state_transition_closed_to_open() {
let config = CircuitBreakerConfig::new(3, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
assert!(
breaker.is_closed().await,
"the initial state should be Closed"
);
assert_eq!(breaker.get_state().await, CircuitState::Closed);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("error 1".to_string()))
})
.await;
assert!(result.is_err());
assert!(
breaker.is_closed().await,
"should still be Closed after the first failure"
);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("error 2".to_string()))
})
.await;
assert!(result.is_err());
assert!(
breaker.is_closed().await,
"should still be Closed after the second failure"
);
let result = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("error 3".to_string()))
})
.await;
assert!(result.is_err());
assert!(
breaker.is_open().await,
"should transition to Open after the third failure"
);
assert_eq!(breaker.get_state().await, CircuitState::Open);
}
#[tokio::test]
async fn test_state_transition_open_to_half_open() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_millis(100));
let breaker = CircuitBreaker::new(config);
for i in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError(format!("error {}", i)))
})
.await;
}
assert!(breaker.is_open().await, "should be in the Open state");
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_err());
assert!(breaker.is_open().await, "must stay Open before the timeout");
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(
breaker.is_half_open().await,
"should transition to HalfOpen after the timeout"
);
}
#[tokio::test]
async fn test_state_transition_half_open_to_closed() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_millis(100));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("error".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(breaker.is_half_open().await);
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(
breaker.is_closed().await,
"should recover to Closed after the success count reaches the threshold"
);
}
#[tokio::test]
async fn test_state_transition_full_cycle() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_millis(100));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("error".to_string()))
})
.await;
}
assert!(
breaker.is_open().await,
"phase 1: should be in the Open state"
);
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(
breaker.is_half_open().await,
"phase 2: should be in the HalfOpen state"
);
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_ok());
assert!(
breaker.is_closed().await,
"phase 3: should recover to the Closed state"
);
}
#[tokio::test]
async fn test_builder_pattern() {
let breaker = CircuitBreaker::builder()
.failure_threshold(10)
.success_threshold(5)
.timeout(Duration::from_secs(30))
.half_open_max_calls(4)
.build();
let config = breaker.config();
assert_eq!(config.failure_threshold, 10);
assert_eq!(config.success_threshold, 5);
assert_eq!(config.timeout, Duration::from_secs(30));
assert_eq!(config.half_open_max_calls, 4);
assert!(breaker.is_closed().await);
}
#[tokio::test]
async fn test_slow_call_duration_threshold_config() {
let config = CircuitBreakerConfig::default()
.slow_call_duration_threshold(Duration::from_millis(100));
assert_eq!(
config.slow_call_duration_threshold,
Duration::from_millis(100)
);
}
#[tokio::test]
async fn test_slow_call_rate_threshold_config() {
let config = CircuitBreakerConfig::default().slow_call_rate_threshold(0.8);
assert_eq!(config.slow_call_rate_threshold, 0.8);
}
#[test]
fn test_default_error_classifier_storage_timeout() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::StorageError(crate::error::StorageError::TimeoutError(
"timeout".into(),
));
assert!(classifier.is_counted_as_failure(&error));
}
#[test]
fn test_default_error_classifier_connection_error() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::StorageError(crate::error::StorageError::ConnectionError(
"connection".into(),
));
assert!(classifier.is_counted_as_failure(&error));
}
#[test]
fn test_default_error_classifier_limit_error() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::LimitError("rate limited".into());
assert!(!classifier.is_counted_as_failure(&error));
}
#[test]
fn test_default_error_classifier_validation_error() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::ValidationError("invalid input".into());
assert!(!classifier.is_counted_as_failure(&error));
}
#[test]
fn test_default_error_classifier_circuit_breaker_error() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::CircuitBreakerError("circuit open".into());
assert!(!classifier.is_counted_as_failure(&error));
}
#[test]
fn test_default_error_classifier_other_errors() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::Other("unknown error".into());
assert!(classifier.is_counted_as_failure(&error));
}
#[tokio::test]
async fn test_custom_error_classifier() {
#[derive(Debug)]
struct CustomClassifier;
impl FailureClassifier<LimiteronError> for CustomClassifier {
fn is_counted_as_failure(&self, error: &LimiteronError) -> bool {
matches!(error, LimiteronError::StorageError(_))
}
}
let config = CircuitBreakerConfig {
error_classifier: Arc::new(CustomClassifier),
failure_threshold: 2,
..Default::default()
};
let breaker = CircuitBreaker::new(config);
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::ValidationError("test".into()))
})
.await;
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 0);
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::StorageError(
crate::error::StorageError::TimeoutError("timeout".into()),
))
})
.await;
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 1);
}
#[tokio::test]
async fn test_builder_with_slow_call_config() {
let breaker = CircuitBreaker::builder()
.slow_call_duration_threshold(Duration::from_millis(200))
.slow_call_rate_threshold(0.6)
.build();
let config = breaker.config();
assert_eq!(
config.slow_call_duration_threshold,
Duration::from_millis(200)
);
assert_eq!(config.slow_call_rate_threshold, 0.6);
}
#[test]
fn test_circuit_breaker_builder_default() {
let builder = CircuitBreakerBuilder::default();
assert_eq!(builder.config.failure_threshold, 5);
assert_eq!(builder.config.success_threshold, 3);
}
#[test]
fn test_config_error_classifier_builder() {
let classifier: Arc<dyn FailureClassifier<LimiteronError>> =
Arc::new(DefaultFailureClassifier);
let config = CircuitBreakerConfig::default().error_classifier(classifier);
assert_eq!(config.failure_threshold, 5);
}
#[test]
fn test_config_all_builder_methods() {
let classifier: Arc<dyn FailureClassifier<LimiteronError>> =
Arc::new(DefaultFailureClassifier);
let config = CircuitBreakerConfig::new(10, 5, Duration::from_secs(30))
.half_open_max_calls(4)
.slow_call_duration_threshold(Duration::from_millis(100))
.slow_call_rate_threshold(0.7)
.error_classifier(classifier);
assert_eq!(config.failure_threshold, 10);
assert_eq!(config.success_threshold, 5);
assert_eq!(config.half_open_max_calls, 4);
assert_eq!(
config.slow_call_duration_threshold,
Duration::from_millis(100)
);
assert!((config.slow_call_rate_threshold - 0.7).abs() < f64::EPSILON);
}
#[tokio::test]
async fn test_slow_call_rate_triggers_open() {
let config = CircuitBreakerConfig {
slow_call_duration_threshold: Duration::ZERO,
slow_call_rate_threshold: 0.5,
failure_threshold: 100,
..Default::default()
};
let breaker = CircuitBreaker::new(config);
let _ = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(
breaker.is_open().await,
"Slow call rate should trigger Open state"
);
}
#[tokio::test]
async fn test_half_open_probe_stall_escape() {
let config = CircuitBreakerConfig {
slow_call_duration_threshold: Duration::from_secs(3600),
half_open_max_calls: 1,
half_open_max_duration: Duration::from_millis(100),
..CircuitBreakerConfig::new(1, 2, Duration::from_millis(50))
};
let breaker = Arc::new(CircuitBreaker::new(config));
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("boom".into()))
})
.await;
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(60)).await;
let probe_breaker = breaker.clone();
let probe = tokio::spawn(async move {
let _ = probe_breaker
.execute(|| async {
tokio::time::sleep(Duration::from_millis(500)).await;
Ok::<(), LimiteronError>(())
})
.await;
});
tokio::time::sleep(Duration::from_millis(50)).await; probe.abort(); let _ = probe.await;
let r = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(
r.is_err(),
"should reject once the probe quota is cancelled and filled"
);
tokio::time::sleep(Duration::from_millis(120)).await;
let r = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(r.is_err(), "the escaping call must be rejected");
assert!(
breaker.is_open().await,
"a lingering timeout must force back to Open"
);
tokio::time::sleep(Duration::from_millis(60)).await;
let r = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(
r.is_ok(),
"after the escape restarts cooldown it must re-enter the half-open probe"
);
}
#[tokio::test]
async fn test_slow_call_rate_open_recovers_via_half_open() {
let config = CircuitBreakerConfig {
slow_call_duration_threshold: Duration::ZERO,
slow_call_rate_threshold: 0.5,
timeout: Duration::from_millis(100),
..Default::default()
};
let breaker = CircuitBreaker::new(config);
let _ = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(150)).await;
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(
result.is_ok(),
"the slow-call circuit must enter the half-open probe after its cooldown (recovery channel deadlock regression)"
);
}
#[tokio::test]
async fn test_slow_call_rate_below_threshold_stays_closed() {
let config = CircuitBreakerConfig {
slow_call_duration_threshold: Duration::from_secs(60),
slow_call_rate_threshold: 0.5,
..Default::default()
};
let breaker = CircuitBreaker::new(config);
for _ in 0..5 {
let _ = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
}
assert!(breaker.is_closed().await);
}
#[tokio::test]
async fn test_on_success_in_open_state_logs_warning() {
let config = CircuitBreakerConfig::new(1, 1, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
let _ = breaker
.execute(|| async { Err::<(), LimiteronError>(LimiteronError::BanError("e".into())) })
.await;
assert!(breaker.is_open().await);
let result = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_circuit_breaker_with_clock() {
use crate::clock::MockClock;
let mock_clock = Arc::new(MockClock::new());
let clock: Arc<dyn Clock> = mock_clock.clone();
let config = CircuitBreakerConfig::new(2, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::with_clock(config, clock);
assert!(breaker.is_closed().await);
let stats = breaker.get_stats().await;
assert_eq!(stats.state, CircuitState::Closed);
}
#[tokio::test]
async fn test_circuit_breaker_builder_error_classifier() {
let classifier: Arc<dyn FailureClassifier<LimiteronError>> =
Arc::new(DefaultFailureClassifier);
let breaker = CircuitBreaker::builder()
.failure_threshold(3)
.error_classifier(classifier)
.build();
assert_eq!(breaker.config().failure_threshold, 3);
}
#[tokio::test]
async fn test_get_stats_after_failure() {
let config = CircuitBreakerConfig::new(5, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
let _ = breaker
.execute(|| async { Err::<(), LimiteronError>(LimiteronError::BanError("e".into())) })
.await;
let stats = breaker.get_stats().await;
assert_eq!(stats.failure_count, 1);
assert_eq!(stats.total_calls, 1);
assert!(stats.last_failure_time.is_some());
}
#[test]
fn test_default_error_classifier_storage_not_transient() {
let classifier = DefaultFailureClassifier;
let error = LimiteronError::StorageError(crate::error::StorageError::NotFound("nf".into()));
assert!(!classifier.is_counted_as_failure(&error));
}
#[tokio::test]
async fn test_on_failure_when_open() {
let config = CircuitBreakerConfig::new(2, 2, Duration::from_secs(60));
let breaker = CircuitBreaker::new(config);
for _ in 0..2 {
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("e".to_string()))
})
.await;
}
assert!(breaker.is_open().await);
let error = LimiteronError::BanError("open-state failure".to_string());
breaker.on_failure_probe_aware(&error, false).await;
assert!(breaker.is_open().await);
}
#[tokio::test]
async fn test_half_open_admission_cas_no_overshoot() {
let config =
CircuitBreakerConfig::new(1, 100, Duration::from_millis(50)).half_open_max_calls(3);
let breaker = Arc::new(CircuitBreaker::new(config));
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("boom".to_string()))
})
.await;
assert!(breaker.is_open().await);
tokio::time::sleep(Duration::from_millis(80)).await;
let tasks = 16usize;
let barrier = Arc::new(tokio::sync::Barrier::new(tasks));
let allowed = Arc::new(std::sync::atomic::AtomicU64::new(0));
let mut handles = Vec::with_capacity(tasks);
for _ in 0..tasks {
let breaker = Arc::clone(&breaker);
let barrier = Arc::clone(&barrier);
let allowed = Arc::clone(&allowed);
handles.push(tokio::spawn(async move {
barrier.wait().await;
let result = breaker
.execute(|| async {
tokio::time::sleep(Duration::from_millis(10)).await;
Ok::<(), LimiteronError>(())
})
.await;
if result.is_ok() {
allowed.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}));
}
for h in handles {
h.await.unwrap();
}
assert_eq!(
allowed.load(std::sync::atomic::Ordering::SeqCst),
3,
"half-open admission must be limited precisely to half_open_max_calls"
);
}
#[tokio::test]
async fn test_total_calls_excludes_rejected_requests() {
let config = CircuitBreakerConfig::new(1, 1, Duration::from_secs(3600));
let breaker = CircuitBreaker::new(config);
let _ = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert_eq!(breaker.get_stats().await.total_calls, 1);
let _ = breaker
.execute(|| async {
Err::<(), LimiteronError>(LimiteronError::BanError("boom".to_string()))
})
.await;
assert!(breaker.is_open().await);
let total_after_open = breaker.get_stats().await.total_calls;
let rejected = breaker
.execute(|| async { Ok::<(), LimiteronError>(()) })
.await;
assert!(rejected.is_err());
assert_eq!(
breaker.get_stats().await.total_calls,
total_after_open,
"calls rejected by the circuit must not count into total_calls"
);
}
#[tokio::test]
async fn test_transition_to_closed_only_from_half_open() {
let breaker = CircuitBreaker::new(CircuitBreakerConfig::new(2, 1, Duration::from_secs(60)));
*breaker.state.write().await = CircuitState::Open;
assert!(breaker.is_open().await);
let transitioned = breaker.transition_to_closed_if_half_open().await;
assert!(
!transitioned,
"Open must not transition to Closed via the conditional close"
);
assert!(breaker.is_open().await, "the state must stay Open");
}
#[derive(Debug, Clone, PartialEq)]
struct ApiError {
transient: bool,
message: String,
}
#[derive(Debug)]
struct ApiErrorClassifier;
impl FailureClassifier<ApiError> for ApiErrorClassifier {
fn is_counted_as_failure(&self, error: &ApiError) -> bool {
error.transient
}
}
fn api_error_breaker() -> CircuitBreaker<ApiError> {
CircuitBreaker::with_dependencies(
CircuitBreakerConfig::with_error_classifier(Arc::new(ApiErrorClassifier))
.half_open_max_calls(2),
)
}
#[tokio::test]
async fn test_generic_error_inner_passthrough() {
let breaker = api_error_breaker();
let result = breaker
.execute(|| async {
Err::<(), ApiError>(ApiError {
transient: false,
message: "bad request".to_string(),
})
})
.await;
match result {
Err(CircuitCallError::Inner(e)) => {
assert_eq!(
e,
ApiError {
transient: false,
message: "bad request".to_string(),
},
"a failure from the call itself must pass through as-is"
);
}
other => panic!("the call failure should pass through Inner, got: {other:?}"),
}
assert_eq!(breaker.get_stats().await.failure_count, 0);
assert!(breaker.is_closed().await);
}
#[tokio::test]
async fn test_generic_error_classifier_counts_failures() {
let breaker = api_error_breaker();
for i in 0..5 {
let _ = breaker
.execute(|| async {
Err::<(), ApiError>(ApiError {
transient: true,
message: format!("boom {i}"),
})
})
.await;
}
assert_eq!(breaker.get_stats().await.failure_count, 5);
assert!(breaker.is_open().await);
}
#[tokio::test]
async fn test_generic_error_open_rejection_is_explicit() {
let breaker = api_error_breaker();
let executed = Arc::new(AtomicU64::new(0));
for _ in 0..5 {
let _ = breaker
.execute(|| async {
Err::<(), ApiError>(ApiError {
transient: true,
message: "boom".to_string(),
})
})
.await;
}
assert!(breaker.is_open().await);
let counter = Arc::clone(&executed);
let result = breaker
.execute(move || {
counter.fetch_add(1, Ordering::Relaxed);
async { Ok::<(), ApiError>(()) }
})
.await;
match result {
Err(call_err) => {
assert!(
call_err.is_open(),
"an open circuit should return the Open variant"
);
assert_eq!(
call_err.into_inner(),
None,
"Open carries no pass-through error"
);
}
Ok(()) => panic!("an open circuit must reject"),
}
assert_eq!(
executed.load(Ordering::Relaxed),
0,
"the closure must not run while the circuit is open"
);
}
#[tokio::test]
async fn test_generic_error_without_debug_usable() {
struct OpaqueError(u32);
#[derive(Debug)]
struct OpaqueClassifier;
impl FailureClassifier<OpaqueError> for OpaqueClassifier {
fn is_counted_as_failure(&self, _error: &OpaqueError) -> bool {
false
}
}
let breaker = CircuitBreaker::with_dependencies(
CircuitBreakerConfig::with_error_classifier(Arc::new(OpaqueClassifier)),
);
let result = breaker
.execute(|| async { Err::<(), OpaqueError>(OpaqueError(1)) })
.await;
match result {
Err(CircuitCallError::Inner(e)) => assert_eq!(e.0, 1),
Err(CircuitCallError::Open) => panic!("an unopened circuit must not reject"),
Ok(()) => panic!("the call should fail and pass through"),
}
assert_eq!(breaker.get_stats().await.failure_count, 0);
assert!(breaker.is_closed().await);
}
}