use crate::Error;
use std::sync::atomic::{AtomicU32, AtomicU64, AtomicU8, Ordering};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
fn base_instant() -> Instant {
static BASE: OnceLock<Instant> = OnceLock::new();
*BASE.get_or_init(Instant::now)
}
fn now_millis() -> u64 {
base_instant().elapsed().as_millis() as u64
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
#[non_exhaustive]
pub enum CircuitState {
Closed = 0,
Open = 1,
HalfOpen = 2,
}
impl CircuitState {
fn from_u8(v: u8) -> Self {
match v {
1 => CircuitState::Open,
2 => CircuitState::HalfOpen,
_ => CircuitState::Closed,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum Admission {
Normal,
Probe,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct BreakerConfig {
pub failure_threshold: u32,
pub cooldown: Duration,
pub name: Option<String>,
}
impl Default for BreakerConfig {
fn default() -> Self {
Self {
failure_threshold: 3,
cooldown: Duration::from_secs(30),
name: None,
}
}
}
impl BreakerConfig {
pub fn with_failure_threshold(mut self, failure_threshold: u32) -> Self {
self.failure_threshold = failure_threshold;
self
}
pub fn with_cooldown(mut self, cooldown: Duration) -> Self {
self.cooldown = cooldown;
self
}
pub fn with_name(mut self, name: impl Into<String>) -> Self {
self.name = Some(name.into());
self
}
pub fn validate(&self) -> Result<(), Error> {
if self.failure_threshold < 1 {
return Err(Error::InvalidBreakerConfig {
reason: "failure_threshold must be >= 1, got 0".into(),
upstream: None,
});
}
if self.cooldown < Duration::from_millis(1) {
return Err(Error::InvalidBreakerConfig {
reason: format!("cooldown must be >= 1ms, got {:?}", self.cooldown),
upstream: None,
});
}
Ok(())
}
}
pub struct Breaker {
name: Option<String>,
gauges: bool,
state: AtomicU8,
consecutive_failures: AtomicU32,
opened_at_millis: AtomicU64,
rearmed_at_millis: AtomicU64,
failure_threshold: AtomicU32,
cooldown_millis: AtomicU64,
}
impl std::fmt::Debug for Breaker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Breaker")
.field("name", &self.name)
.field("state", &self.state())
.field(
"failure_threshold",
&self.failure_threshold.load(Ordering::Relaxed),
)
.field(
"cooldown",
&Duration::from_millis(self.cooldown_millis.load(Ordering::Relaxed)),
)
.finish()
}
}
impl Breaker {
pub fn new(config: BreakerConfig) -> Result<Self, Error> {
config.validate()?;
Ok(Self::unchecked(
config.name,
false,
config.cooldown,
config.failure_threshold,
))
}
pub(crate) fn for_upstream(name: String, config: &BreakerConfig) -> Result<Self, Error> {
config.validate().map_err(|e| match e {
Error::InvalidBreakerConfig { reason, .. } => Error::InvalidBreakerConfig {
reason,
upstream: Some(name.clone()),
},
e => e,
})?;
Ok(Self::unchecked(
Some(name),
true,
config.cooldown,
config.failure_threshold,
))
}
pub(crate) fn unchecked(
name: Option<String>,
gauges: bool,
cooldown: Duration,
failure_threshold: u32,
) -> Self {
let b = Self {
name,
gauges,
state: AtomicU8::new(CircuitState::Closed as u8),
consecutive_failures: AtomicU32::new(0),
opened_at_millis: AtomicU64::new(0),
rearmed_at_millis: AtomicU64::new(u64::MAX),
failure_threshold: AtomicU32::new(0),
cooldown_millis: AtomicU64::new(0),
};
b.reconfigure(cooldown, failure_threshold);
b
}
pub(crate) fn reconfigure(&self, cooldown: Duration, failure_threshold: u32) {
self.cooldown_millis
.store(cooldown.as_millis() as u64, Ordering::Relaxed);
self.failure_threshold
.store(failure_threshold, Ordering::Relaxed);
}
pub fn state(&self) -> CircuitState {
CircuitState::from_u8(self.state.load(Ordering::Acquire))
}
pub fn try_acquire(&self) -> Option<Admission> {
match self.state() {
CircuitState::Closed => Some(Admission::Normal),
CircuitState::Open => {
if !self.claim_probe_slot() {
return None;
}
if self.transition(CircuitState::Open, CircuitState::HalfOpen) {
Some(Admission::Probe)
} else {
Some(Admission::Normal)
}
}
CircuitState::HalfOpen => self.claim_probe_slot().then_some(Admission::Probe),
}
}
fn claim_probe_slot(&self) -> bool {
let opened_at = self.opened_at_millis.load(Ordering::Acquire);
let now = now_millis();
if now.saturating_sub(opened_at) < self.cooldown_millis.load(Ordering::Relaxed) {
return false;
}
self.opened_at_millis
.compare_exchange(opened_at, now, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
}
pub fn release(&self, admission: Admission) {
if admission != Admission::Probe {
return;
}
let cooldown = self.cooldown_millis.load(Ordering::Relaxed);
let now = now_millis();
let stamp = self.opened_at_millis.load(Ordering::Acquire);
if self.state() != CircuitState::HalfOpen {
return;
}
let last = self.rearmed_at_millis.load(Ordering::Acquire);
if last != u64::MAX && now.saturating_sub(last) < cooldown {
return;
}
if self
.rearmed_at_millis
.compare_exchange(last, now, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return;
}
let _ = self.opened_at_millis.compare_exchange(
stamp,
now.saturating_sub(cooldown),
Ordering::AcqRel,
Ordering::Acquire,
);
}
pub fn record_success(&self, admission: Admission) {
match admission {
Admission::Probe => {
if self.state() == CircuitState::Closed {
return;
}
self.consecutive_failures.store(0, Ordering::Relaxed);
let prev = CircuitState::from_u8(
self.state
.swap(CircuitState::Closed as u8, Ordering::AcqRel),
);
if prev != CircuitState::Closed {
self.log_transition(prev, CircuitState::Closed);
self.set_gauges();
}
}
Admission::Normal => {
if self.state() == CircuitState::Closed {
self.consecutive_failures.store(0, Ordering::Relaxed);
}
}
}
}
pub fn record_failure(&self, admission: Admission) {
match (admission, self.state()) {
(Admission::Probe, CircuitState::HalfOpen) => {
self.opened_at_millis.store(now_millis(), Ordering::Release);
self.transition(CircuitState::HalfOpen, CircuitState::Open);
}
(Admission::Probe, CircuitState::Open) => {}
(_, CircuitState::Closed) => {
let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
if failures < self.failure_threshold.load(Ordering::Relaxed) {
return;
}
self.opened_at_millis.store(now_millis(), Ordering::Release);
self.transition(CircuitState::Closed, CircuitState::Open);
}
(Admission::Normal, _) => {}
}
}
fn transition(&self, from: CircuitState, to: CircuitState) -> bool {
let ok = self
.state
.compare_exchange(from as u8, to as u8, Ordering::AcqRel, Ordering::Acquire)
.is_ok();
if ok {
self.log_transition(from, to);
self.set_gauges();
}
ok
}
fn log_transition(&self, from: CircuitState, to: CircuitState) {
let name = self.name.as_deref().unwrap_or("unnamed");
if to == CircuitState::Open {
tracing::warn!(upstream = %name, ?from, ?to, "circuit breaker state change");
} else {
tracing::info!(upstream = %name, ?from, ?to, "circuit breaker state change");
}
}
pub(crate) fn set_gauges(&self) {
let (true, Some(name)) = (self.gauges, &self.name) else {
return;
};
let state = self.state();
metrics::gauge!("ferryman_circuit_state", "upstream" => name.clone())
.set(state as u8 as f64);
let alive = if state == CircuitState::Closed {
1.0
} else {
0.0
};
metrics::gauge!("ferryman_upstream_alive", "upstream" => name.clone()).set(alive);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::thread;
const N: Admission = Admission::Normal;
const P: Admission = Admission::Probe;
fn breaker(cooldown_ms: u64, threshold: u32) -> Breaker {
Breaker::unchecked(
Some("test:1".to_string()),
true,
Duration::from_millis(cooldown_ms),
threshold,
)
}
fn half_open(b: &Breaker) {
b.record_failure(N);
thread::sleep(Duration::from_millis(250));
assert_eq!(b.try_acquire(), Some(P));
assert_eq!(b.state(), CircuitState::HalfOpen);
}
#[test]
fn closed_always_acquires() {
let b = breaker(1_000, 3);
assert_eq!(b.state(), CircuitState::Closed);
assert_eq!(b.try_acquire(), Some(N));
}
#[test]
fn opens_after_threshold_failures() {
let b = breaker(1_000, 3);
b.record_failure(N);
b.record_failure(N);
assert_eq!(b.state(), CircuitState::Closed);
b.record_failure(N);
assert_eq!(b.state(), CircuitState::Open);
assert_eq!(b.try_acquire(), None);
}
#[test]
fn success_resets_failure_count() {
let b = breaker(1_000, 2);
b.record_failure(N);
b.record_success(N);
b.record_failure(N);
assert_eq!(b.state(), CircuitState::Closed);
}
#[test]
fn half_open_single_probe() {
let b = Arc::new(breaker(200, 1));
b.record_failure(N);
assert_eq!(b.state(), CircuitState::Open);
thread::sleep(Duration::from_millis(250));
let handles: Vec<_> = (0..8)
.map(|_| {
let b = b.clone();
thread::spawn(move || b.try_acquire())
})
.collect();
let winners = handles
.into_iter()
.filter_map(|h| h.join().unwrap())
.count();
assert_eq!(winners, 1, "exactly one caller should win the probe");
assert_eq!(b.state(), CircuitState::HalfOpen);
}
#[test]
fn probe_failure_reopens() {
let b = breaker(200, 1);
half_open(&b);
b.record_failure(P);
assert_eq!(b.state(), CircuitState::Open);
assert_eq!(b.try_acquire(), None);
}
#[test]
fn probe_failure_while_open_does_not_restamp() {
let b = breaker(200, 1);
b.record_failure(N);
for _ in 0..6 {
thread::sleep(Duration::from_millis(50));
b.record_failure(P);
}
assert_eq!(b.state(), CircuitState::Open);
assert_eq!(b.try_acquire(), Some(P), "cooldown must have elapsed");
b.record_success(P);
assert_eq!(b.state(), CircuitState::Closed);
}
#[test]
fn release_probe_rearms_slot_immediately() {
let b = breaker(10_000, 1);
b.record_failure(N);
b.reconfigure(Duration::from_millis(200), 1);
thread::sleep(Duration::from_millis(250));
assert_eq!(b.try_acquire(), Some(P));
assert_eq!(b.try_acquire(), None, "probe is out");
b.release(P);
assert_eq!(b.state(), CircuitState::HalfOpen, "state unchanged");
assert_eq!(b.try_acquire(), Some(P), "slot re-armed at once");
assert_eq!(b.try_acquire(), None);
}
#[test]
fn release_rearms_at_most_once_per_cooldown() {
let b = breaker(200, 1);
half_open(&b); let mut admitted = 1;
for _ in 0..1000 {
b.release(P);
if b.try_acquire().is_some() {
admitted += 1;
}
}
assert!(admitted <= 2, "probe flood: {admitted}");
assert_eq!(b.state(), CircuitState::HalfOpen);
thread::sleep(Duration::from_millis(250));
let _ = b.try_acquire();
b.release(P);
assert_eq!(b.try_acquire(), Some(P), "re-armed after a cooldown");
}
#[test]
fn release_normal_and_stale_release_are_noops() {
thread::sleep(Duration::from_millis(250));
let b = breaker(200, 2);
b.record_failure(N);
b.release(N);
b.release(P); assert_eq!(b.state(), CircuitState::Closed);
b.record_failure(N); assert_eq!(b.state(), CircuitState::Open);
b.release(N);
b.release(P); assert_eq!(b.try_acquire(), None);
thread::sleep(Duration::from_millis(250));
assert_eq!(b.try_acquire(), Some(P));
b.release(N); assert_eq!(b.try_acquire(), None);
}
#[test]
fn probe_success_closes() {
let b = breaker(200, 1);
half_open(&b);
b.record_success(P);
assert_eq!(b.state(), CircuitState::Closed);
assert_eq!(b.try_acquire(), Some(N));
}
#[test]
fn health_probe_success_closes_open_circuit() {
let b = breaker(10_000, 1);
b.record_failure(N);
assert_eq!(b.state(), CircuitState::Open);
b.record_success(P);
assert_eq!(b.state(), CircuitState::Closed);
}
#[test]
fn health_success_does_not_reset_request_failures() {
let b = breaker(10_000, 2);
b.record_failure(N);
b.record_success(P);
b.record_failure(N);
assert_eq!(b.state(), CircuitState::Open);
}
#[test]
fn late_normal_results_are_ignored() {
let b = breaker(200, 1);
b.record_failure(N);
b.record_success(N);
assert_eq!(b.state(), CircuitState::Open);
thread::sleep(Duration::from_millis(250));
assert_eq!(b.try_acquire(), Some(P));
b.record_failure(N);
assert_eq!(b.state(), CircuitState::HalfOpen);
}
#[test]
fn stuck_half_open_probe_recovers() {
let b = breaker(200, 1);
half_open(&b);
thread::sleep(Duration::from_millis(250));
assert_eq!(b.try_acquire(), Some(P));
assert_eq!(b.state(), CircuitState::HalfOpen);
}
#[test]
fn reconfigure_changes_cooldown() {
let b = breaker(10_000, 1);
b.record_failure(N);
assert_eq!(b.try_acquire(), None);
b.reconfigure(Duration::from_millis(0), 1);
assert_eq!(b.try_acquire(), Some(P));
}
}