use std::sync::Mutex;
use std::time::{Duration, Instant};
use tracing::{debug, info, warn};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FallbackState {
Primary,
Fallback,
Recovering,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FailureKind {
Transient,
RateLimit,
}
#[derive(Debug, Clone)]
pub(crate) struct FallbackEntry {
name: String,
available: bool,
attempt_count: usize,
max_fail_count: usize,
}
impl FallbackEntry {
pub(crate) fn new(name: impl Into<String>) -> Self {
Self {
name: name.into(),
available: true,
attempt_count: 0,
max_fail_count: 2,
}
}
#[must_use]
pub(crate) fn with_max_fail_count(mut self, max_fail_count: usize) -> Self {
let new_max = max_fail_count.max(1);
if self.failed() && self.attempt_count < new_max {
self.attempt_count = new_max;
}
self.max_fail_count = new_max;
self
}
pub(crate) fn failed(&self) -> bool {
!self.available || self.attempt_count >= self.max_fail_count
}
pub(crate) fn set_available(&mut self, available: bool) {
self.available = available;
}
pub(crate) fn record_attempt(&mut self) {
self.attempt_count = self.attempt_count.saturating_add(1);
}
pub(crate) fn clear_attempts(&mut self) {
self.attempt_count = 0;
}
}
#[derive(Debug, Clone)]
pub struct FallbackConfig {
pub trip_threshold: usize,
pub recovery_timeout: Duration,
pub recovery_successes_needed: usize,
pub max_fail_count: usize,
}
impl Default for FallbackConfig {
fn default() -> Self {
Self {
trip_threshold: 3,
recovery_timeout: Duration::from_mins(1),
recovery_successes_needed: 2,
max_fail_count: 2,
}
}
}
struct BreakerState {
state: FallbackState,
consecutive_failures: usize,
primary_success_count: usize,
fallback_activated: bool,
original_model: Option<String>,
fallback_models: Vec<FallbackEntry>,
active_fallback: Option<String>,
fallback_switched_at: Option<Instant>,
}
impl Default for BreakerState {
fn default() -> Self {
Self {
state: FallbackState::Primary,
consecutive_failures: 0,
primary_success_count: 0,
fallback_activated: false,
original_model: None,
fallback_models: Vec::new(),
active_fallback: None,
fallback_switched_at: None,
}
}
}
pub struct FallbackManager {
config: FallbackConfig,
state: Mutex<BreakerState>,
}
impl FallbackManager {
fn lock_state(
&self,
) -> Result<std::sync::MutexGuard<'_, BreakerState>, crate::error::LoopError> {
self.state
.lock()
.map_err(crate::error::from_poison("fallback"))
}
#[must_use]
pub fn new(fallback_threshold: usize, primary_resume_threshold: usize) -> Self {
Self {
config: FallbackConfig {
trip_threshold: fallback_threshold,
recovery_successes_needed: primary_resume_threshold,
..FallbackConfig::default()
},
state: Mutex::new(BreakerState::default()),
}
}
#[must_use]
pub fn with_config(mut self, config: FallbackConfig) -> Self {
self.config = config;
self
}
#[must_use]
pub fn config(&self) -> &FallbackConfig {
&self.config
}
#[must_use]
pub fn new_with_config(config: FallbackConfig) -> Self {
Self {
config,
state: Mutex::new(BreakerState::default()),
}
}
pub fn for_model(primary_model: impl Into<String>) -> Result<Self, crate::error::LoopError> {
let mgr = Self::new(3, 2);
let mut state = mgr.lock_state()?;
state.original_model = Some(primary_model.into());
drop(state);
Ok(mgr)
}
pub fn state(&self) -> Result<FallbackState, crate::error::LoopError> {
Ok(self.lock_state()?.state)
}
pub fn is_using_fallback(&self) -> Result<bool, crate::error::LoopError> {
Ok(matches!(self.state()?, FallbackState::Fallback))
}
pub fn is_fallback_active(&self) -> Result<bool, crate::error::LoopError> {
Ok(self.lock_state()?.fallback_activated)
}
pub fn consecutive_failures(&self) -> Result<usize, crate::error::LoopError> {
Ok(self.lock_state()?.consecutive_failures)
}
pub fn original_model(&self) -> Result<Option<String>, crate::error::LoopError> {
Ok(self.lock_state()?.original_model.clone())
}
pub fn set_original_model(&self, model: String) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
state.original_model = Some(model);
Ok(())
}
pub fn fallback_switched_at(&self) -> Result<Option<Instant>, crate::error::LoopError> {
Ok(self.lock_state()?.fallback_switched_at)
}
pub fn active_model(&self) -> Result<Option<String>, crate::error::LoopError> {
let state = self.lock_state()?;
Ok(match state.state {
FallbackState::Primary | FallbackState::Recovering => state.original_model.clone(),
FallbackState::Fallback => match &state.active_fallback {
Some(model) => Some(model.clone()),
None if state.fallback_models.is_empty() => state.original_model.clone(),
None => None,
},
})
}
pub fn set_fallback_model(
&self,
model: impl Into<String>,
) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
state.fallback_models.clear();
state
.fallback_models
.push(FallbackEntry::new(model).with_max_fail_count(self.config.max_fail_count));
Self::recompute_active_fallback_impl(&mut state);
Ok(())
}
pub fn fallback_model(&self) -> Result<Option<String>, crate::error::LoopError> {
Ok(self.lock_state()?.active_fallback.clone())
}
pub fn fallback_models(&self) -> Result<Vec<String>, crate::error::LoopError> {
Ok(self
.lock_state()?
.fallback_models
.iter()
.map(|e| e.name.clone())
.collect())
}
pub fn add_fallback_model(
&self,
model: impl Into<String>,
) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
state
.fallback_models
.push(FallbackEntry::new(model).with_max_fail_count(self.config.max_fail_count));
Self::recompute_active_fallback_impl(&mut state);
Ok(())
}
pub fn insert_fallback_model(
&self,
index: usize,
model: impl Into<String>,
) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
let entry = FallbackEntry::new(model).with_max_fail_count(self.config.max_fail_count);
if index >= state.fallback_models.len() {
state.fallback_models.push(entry);
} else {
state.fallback_models.insert(index, entry);
}
Self::recompute_active_fallback_impl(&mut state);
Ok(())
}
pub fn remove_fallback_model(&self, model: &str) -> Result<bool, crate::error::LoopError> {
let mut removed = false;
let mut state = self.lock_state()?;
if let Some(pos) = state.fallback_models.iter().position(|x| x.name == model) {
state.fallback_models.remove(pos);
removed = true;
}
if removed {
Self::recompute_active_fallback_impl(&mut state);
}
Ok(removed)
}
pub fn set_fallback_models(&self, models: Vec<String>) -> Result<(), crate::error::LoopError> {
let max_fc = self.config.max_fail_count;
let mut state = self.lock_state()?;
state.fallback_models = models
.into_iter()
.map(|name| FallbackEntry::new(name).with_max_fail_count(max_fc))
.collect();
Self::recompute_active_fallback_impl(&mut state);
Ok(())
}
pub fn mark_fallback_failed(&self, model: &str) -> Result<bool, crate::error::LoopError> {
let mut found = false;
let mut state = self.lock_state()?;
if let Some(entry) = state.fallback_models.iter_mut().find(|e| e.name == model) {
entry.record_attempt();
found = true;
}
if found {
Self::recompute_active_fallback_impl(&mut state);
}
Ok(found)
}
pub fn clear_fallback_failed(&self, model: &str) -> Result<bool, crate::error::LoopError> {
let mut found = false;
let mut state = self.lock_state()?;
if let Some(entry) = state.fallback_models.iter_mut().find(|e| e.name == model) {
entry.clear_attempts();
found = true;
}
if found {
Self::recompute_active_fallback_impl(&mut state);
}
Ok(found)
}
pub fn clear_all_fallback_failed(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
for entry in &mut state.fallback_models {
entry.clear_attempts();
}
Self::recompute_active_fallback_impl(&mut state);
Ok(())
}
pub fn failed_fallbacks(&self) -> Result<Vec<String>, crate::error::LoopError> {
Ok(self
.lock_state()?
.fallback_models
.iter()
.filter(|e| e.failed())
.map(|e| e.name.clone())
.collect())
}
pub fn available_fallbacks(&self) -> Result<Vec<String>, crate::error::LoopError> {
Ok(self
.lock_state()?
.fallback_models
.iter()
.filter(|e| !e.failed())
.map(|e| e.name.clone())
.collect())
}
pub fn set_fallback_available(
&self,
model: &str,
available: bool,
) -> Result<bool, crate::error::LoopError> {
let mut found = false;
let mut state = self.lock_state()?;
if let Some(entry) = state.fallback_models.iter_mut().find(|e| e.name == model) {
entry.set_available(available);
found = true;
}
if found {
Self::recompute_active_fallback_impl(&mut state);
}
Ok(found)
}
pub fn record_failure(&self, kind: FailureKind) -> Result<bool, crate::error::LoopError> {
let mut state = self.lock_state()?;
let tripped = match state.state {
FallbackState::Primary => {
state.consecutive_failures = state.consecutive_failures.saturating_add(1);
if state.consecutive_failures >= self.config.trip_threshold {
warn!(
consecutive_failures = state.consecutive_failures,
threshold = self.config.trip_threshold,
"Fallback threshold reached"
);
Self::transition_to_fallback_impl(&mut state);
true
} else {
false
}
}
FallbackState::Fallback => {
let fb_name = state
.active_fallback
.clone()
.unwrap_or_else(|| "unknown".to_string());
warn!(
"Fallback model \"{fb_name}\" also experiencing failures; the agent loop marks it failed automatically (direct callers of record_failure must call mark_fallback_failed themselves)"
);
false
}
FallbackState::Recovering => match kind {
FailureKind::RateLimit => {
warn!(
"Primary model rate-limited during recovery test, re-tripping to fallback"
);
Self::transition_to_fallback_impl(&mut state);
false
}
FailureKind::Transient => {
state.primary_success_count = 0;
warn!(
consecutive_failures = state.consecutive_failures,
"Transient failure recorded during recovery; success streak reset, staying half-open"
);
false
}
},
};
Ok(tripped)
}
pub fn reset_failure_counter(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
state.consecutive_failures = 0;
Ok(())
}
pub fn record_success(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
match state.state {
FallbackState::Primary => {
state.consecutive_failures = 0;
}
FallbackState::Fallback => {}
FallbackState::Recovering => {
state.primary_success_count = state.primary_success_count.saturating_add(1);
debug!(
successes = state.primary_success_count,
threshold = self.config.recovery_successes_needed,
"Primary model success during recovery test"
);
if state.primary_success_count >= self.config.recovery_successes_needed {
Self::transition_to_primary_impl(&mut state);
}
}
}
Ok(())
}
pub fn should_try_resume_primary(
&self,
min_fallback_duration: Duration,
) -> Result<bool, crate::error::LoopError> {
let state = self.lock_state()?;
Ok(Self::should_try_resume_primary_impl(
&state,
min_fallback_duration,
))
}
fn should_try_resume_primary_impl(
state: &BreakerState,
min_fallback_duration: Duration,
) -> bool {
if state.state != FallbackState::Fallback {
return false;
}
state
.fallback_switched_at
.is_some_and(|switched_at| switched_at.elapsed() >= min_fallback_duration)
}
pub fn transition_to_fallback(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
Self::transition_to_fallback_impl(&mut state);
Ok(())
}
fn transition_to_fallback_impl(state: &mut BreakerState) {
state.state = FallbackState::Fallback;
state.fallback_activated = true;
state.fallback_switched_at = Some(Instant::now());
state.primary_success_count = 0;
state.consecutive_failures = 0;
info!("Circuit breaker: transitioned to Fallback state");
}
pub fn transition_to_recovering(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
Self::transition_to_recovering_impl(&mut state);
Ok(())
}
fn transition_to_recovering_impl(state: &mut BreakerState) {
state.state = FallbackState::Recovering;
state.primary_success_count = 0;
info!("Circuit breaker: transitioned to Recovering state (testing primary)");
}
pub fn transition_to_primary(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
Self::transition_to_primary_impl(&mut state);
Ok(())
}
fn transition_to_primary_impl(state: &mut BreakerState) {
state.state = FallbackState::Primary;
state.fallback_switched_at = None;
state.primary_success_count = 0;
state.consecutive_failures = 0;
state.fallback_activated = false;
for entry in &mut state.fallback_models {
entry.clear_attempts();
}
Self::recompute_active_fallback_impl(state);
info!("Circuit breaker: transitioned to Primary state (primary model recovered)");
}
pub fn reset(&self) -> Result<(), crate::error::LoopError> {
let mut state = self.lock_state()?;
Self::transition_to_primary_impl(&mut state);
Ok(())
}
fn recompute_active_fallback_impl(state: &mut BreakerState) {
state.active_fallback = state
.fallback_models
.iter()
.find(|e| !e.failed())
.map(|e| e.name.clone());
}
}
impl Default for FallbackManager {
fn default() -> Self {
Self::new(3, 2)
}
}
#[cfg(test)]
mod tests {
fn poison(mgr: &std::sync::Arc<FallbackManager>) {
let m = std::sync::Arc::clone(mgr);
assert!(
std::thread::spawn(move || {
let _guard = m.state.lock().expect("lock before poison");
panic!("poison the breaker state");
})
.join()
.is_err(),
"the poisoning thread must panic"
);
}
#[test]
fn original_model_returns_poison_error_after_poison() {
let mgr = std::sync::Arc::new(FallbackManager::for_model("p").unwrap());
poison(&mgr);
match mgr.original_model() {
Err(crate::error::LoopError::LockPoisoned { what }) => {
assert_eq!(what, "fallback");
}
other => panic!("poison must propagate, not fail open: {other:?}"),
}
}
#[test]
fn set_original_model_propagates_poison() {
let mgr = std::sync::Arc::new(FallbackManager::new(3, 2));
poison(&mgr);
assert!(
mgr.set_original_model("x".to_string()).is_err(),
"a poisoned breaker must not silently accept writes"
);
}
#[test]
fn active_model_propagates_poison() {
let mgr = std::sync::Arc::new(FallbackManager::for_model("p").unwrap());
poison(&mgr);
assert!(
matches!(
mgr.active_model(),
Err(crate::error::LoopError::LockPoisoned { .. })
),
"model selection must not read desynchronised state"
);
}
#[test]
fn record_failure_propagates_when_poisoned() {
let mgr = std::sync::Arc::new(FallbackManager::new(3, 2));
poison(&mgr);
assert!(mgr.record_failure(FailureKind::Transient).is_err());
}
#[test]
fn state_returns_poison_error_not_primary() {
let mgr = std::sync::Arc::new(FallbackManager::new(3, 2));
poison(&mgr);
assert!(matches!(
mgr.state(),
Err(crate::error::LoopError::LockPoisoned { .. })
));
}
use super::*;
#[test]
fn test_initial_state() {
let mgr = FallbackManager::new(3, 2);
assert_eq!(mgr.state().unwrap(), FallbackState::Primary);
assert!(!mgr.is_using_fallback().unwrap());
assert!(!mgr.is_fallback_active().unwrap());
assert_eq!(mgr.consecutive_failures().unwrap(), 0);
}
#[test]
fn test_failure_threshold() {
let mgr = FallbackManager::new(3, 2);
assert!(!mgr.record_failure(FailureKind::Transient).unwrap()); assert!(!mgr.record_failure(FailureKind::Transient).unwrap()); assert!(mgr.record_failure(FailureKind::Transient).unwrap()); }
#[test]
fn test_model_failure_triggers_fallback() {
let mgr = FallbackManager::new(3, 2);
assert!(!mgr.record_failure(FailureKind::Transient).unwrap()); assert!(!mgr.record_failure(FailureKind::Transient).unwrap()); assert!(mgr.record_failure(FailureKind::Transient).unwrap()); assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
}
#[test]
fn test_recovery() {
let mgr = FallbackManager::new(3, 2);
for _ in 0..3 {
mgr.record_failure(FailureKind::Transient).unwrap();
}
assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
mgr.transition_to_recovering().unwrap();
assert_eq!(mgr.state().unwrap(), FallbackState::Recovering);
mgr.record_success().unwrap(); mgr.record_success().unwrap(); assert_eq!(mgr.state().unwrap(), FallbackState::Primary);
}
#[test]
fn test_recovery_failure_goes_back_to_fallback() {
let mgr = FallbackManager::new(3, 2);
for _ in 0..3 {
mgr.record_failure(FailureKind::Transient).unwrap();
}
mgr.transition_to_recovering().unwrap();
mgr.record_failure(FailureKind::RateLimit).unwrap(); assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
}
#[test]
fn test_should_try_resume_primary() {
let mgr = FallbackManager::new(3, 2);
assert!(
!mgr.should_try_resume_primary(Duration::from_secs(10))
.unwrap()
);
for _ in 0..3 {
mgr.record_failure(FailureKind::Transient).unwrap();
}
assert!(
!mgr.should_try_resume_primary(Duration::from_hours(1))
.unwrap()
);
assert!(
mgr.should_try_resume_primary(Duration::from_secs(0))
.unwrap()
);
}
#[test]
fn test_reset() {
let mgr = FallbackManager::new(3, 2);
for _ in 0..3 {
mgr.record_failure(FailureKind::Transient).unwrap();
}
assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
mgr.reset().unwrap();
assert_eq!(mgr.state().unwrap(), FallbackState::Primary);
assert!(!mgr.is_fallback_active().unwrap());
assert_eq!(mgr.consecutive_failures().unwrap(), 0);
}
#[test]
fn test_api_failure_does_not_retrip() {
let mgr = FallbackManager::new(3, 2);
for _ in 0..3 {
mgr.record_failure(FailureKind::Transient).unwrap();
}
mgr.transition_to_fallback().unwrap();
assert!(!mgr.record_failure(FailureKind::Transient).unwrap());
}
#[test]
fn test_record_success_resets_on_primary() {
let mgr = FallbackManager::new(3, 2);
mgr.record_failure(FailureKind::Transient).unwrap();
mgr.record_failure(FailureKind::Transient).unwrap();
assert_eq!(mgr.consecutive_failures().unwrap(), 2);
mgr.record_success().unwrap();
assert_eq!(mgr.consecutive_failures().unwrap(), 0);
}
#[test]
fn test_for_model() {
let mgr = FallbackManager::for_model("llm-70b").unwrap();
assert_eq!(mgr.original_model().unwrap(), Some("llm-70b".to_string()));
assert_eq!(mgr.state().unwrap(), FallbackState::Primary);
}
#[test]
fn test_concurrent_access() {
use std::sync::Arc;
use std::thread;
let mgr = Arc::new(FallbackManager::new(3, 2));
let mut handles = Vec::new();
for _ in 0..10 {
let mgr = Arc::clone(&mgr);
handles.push(thread::spawn(move || {
mgr.record_failure(FailureKind::Transient).unwrap();
mgr.record_success().unwrap();
mgr.state().unwrap();
mgr.consecutive_failures().unwrap();
}));
}
for h in handles {
h.join().unwrap();
}
}
#[test]
fn test_consolidated_mutex_fields_are_consistent() {
let mgr = FallbackManager::for_model("primary-model").unwrap();
mgr.add_fallback_model("fallback-model").unwrap();
assert_eq!(
mgr.active_model().unwrap(),
Some("primary-model".to_string())
);
assert!(mgr.fallback_switched_at().unwrap().is_none());
mgr.transition_to_fallback().unwrap();
assert_eq!(
mgr.active_model().unwrap(),
Some("fallback-model".to_string())
);
assert!(
mgr.fallback_switched_at().unwrap().is_some(),
"switch time should be set after transition"
);
}
#[test]
fn test_consolidated_mutex_clears_fields_together() {
let mgr = FallbackManager::for_model("primary-model").unwrap();
mgr.add_fallback_model("fallback-model").unwrap();
mgr.transition_to_fallback().unwrap();
assert_eq!(
mgr.active_model().unwrap(),
Some("fallback-model".to_string())
);
assert!(mgr.fallback_switched_at().unwrap().is_some());
mgr.transition_to_primary().unwrap();
assert!(
mgr.fallback_switched_at().unwrap().is_none(),
"switch time should be cleared after transition to primary"
);
}
#[test]
fn test_consolidated_mutex_reset_clears_all() {
let mgr = FallbackManager::for_model("primary-model").unwrap();
mgr.add_fallback_model("fallback-model").unwrap();
mgr.record_failure(FailureKind::Transient).unwrap();
assert!(
mgr.consecutive_failures().unwrap() > 0,
"counter is dirty before the trip"
);
mgr.transition_to_fallback().unwrap();
assert_eq!(mgr.consecutive_failures().unwrap(), 0);
assert!(mgr.fallback_switched_at().unwrap().is_some());
mgr.reset().unwrap();
assert_eq!(mgr.consecutive_failures().unwrap(), 0);
assert!(mgr.fallback_switched_at().unwrap().is_none());
}
#[test]
fn configured_max_fail_count_advances_the_chain_after_that_many_failures() {
let mgr = FallbackManager::for_model("primary")
.unwrap()
.with_config(FallbackConfig {
trip_threshold: 1,
recovery_successes_needed: 1,
max_fail_count: 3,
..FallbackConfig::default()
});
mgr.add_fallback_model("fb-1").unwrap();
mgr.add_fallback_model("fb-2").unwrap();
assert!(
mgr.record_failure(FailureKind::Transient).unwrap(),
"threshold 1 trips on the first primary failure"
);
assert_eq!(mgr.active_model().unwrap().as_deref(), Some("fb-1"));
mgr.mark_fallback_failed("fb-1").unwrap();
mgr.mark_fallback_failed("fb-1").unwrap();
assert_eq!(
mgr.active_model().unwrap().as_deref(),
Some("fb-1"),
"two failures stay on fb-1 — its configured budget is three"
);
mgr.mark_fallback_failed("fb-1").unwrap();
assert_eq!(
mgr.active_model().unwrap().as_deref(),
Some("fb-2"),
"the third failure exhausts fb-1's configured budget, not the \
default of two"
);
}
#[test]
fn zero_trip_threshold_trips_on_the_first_failure() {
let mgr = FallbackManager::for_model("primary")
.unwrap()
.with_config(FallbackConfig {
trip_threshold: 0,
recovery_successes_needed: 1,
max_fail_count: 2,
..FallbackConfig::default()
});
mgr.add_fallback_model("fb-1").unwrap();
assert_eq!(mgr.state().unwrap(), FallbackState::Primary);
assert!(
mgr.record_failure(FailureKind::Transient).unwrap(),
"a zero threshold trips on the very first failure"
);
assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
assert_eq!(mgr.active_model().unwrap().as_deref(), Some("fb-1"));
}
#[test]
fn bulk_chain_apis_roundtrip() {
let mgr = FallbackManager::for_model("primary").unwrap();
mgr.set_fallback_models(vec!["a".to_string(), "b".to_string(), "c".to_string()])
.unwrap();
assert_eq!(
mgr.fallback_models().unwrap(),
vec!["a".to_string(), "b".to_string(), "c".to_string()],
"set_fallback_models replaces the whole chain"
);
mgr.insert_fallback_model(1, "d").unwrap();
assert_eq!(
mgr.fallback_models().unwrap(),
vec![
"a".to_string(),
"d".to_string(),
"b".to_string(),
"c".to_string()
],
"insert_fallback_model splices at the index"
);
mgr.insert_fallback_model(usize::MAX, "e").unwrap();
assert_eq!(
mgr.fallback_models().unwrap().last().map(String::as_str),
Some("e"),
"an out-of-range index appends"
);
assert!(mgr.remove_fallback_model("b").unwrap());
assert!(!mgr.remove_fallback_model("b").unwrap());
assert_eq!(
mgr.fallback_models().unwrap(),
vec![
"a".to_string(),
"d".to_string(),
"c".to_string(),
"e".to_string()
]
);
}
#[test]
fn half_open_probe_targets_the_original_model() {
let mgr = FallbackManager::for_model("primary")
.unwrap()
.with_config(FallbackConfig {
trip_threshold: 1,
recovery_successes_needed: 1,
max_fail_count: 2,
..FallbackConfig::default()
});
mgr.set_fallback_models(vec!["fb-1".to_string()]).unwrap();
assert!(mgr.record_failure(FailureKind::Transient).unwrap());
assert_eq!(mgr.active_model().unwrap().as_deref(), Some("fb-1"));
mgr.transition_to_recovering().unwrap();
assert_eq!(
mgr.active_model().unwrap().as_deref(),
Some("primary"),
"the half-open state probes the PRIMARY — a configured, healthy \
fallback must not capture the recovery probe"
);
}
#[test]
fn tripped_breaker_with_empty_chain_keeps_serving_the_primary() {
let mgr = FallbackManager::for_model("primary")
.unwrap()
.with_config(FallbackConfig {
trip_threshold: 1,
recovery_successes_needed: 1,
max_fail_count: 2,
..FallbackConfig::default()
});
assert!(
mgr.record_failure(FailureKind::Transient).unwrap(),
"the breaker trips with no fallbacks configured"
);
assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
assert_eq!(
mgr.active_model().unwrap().as_deref(),
Some("primary"),
"an empty chain keeps serving the primary — the trip state is \
bookkeeping, not an outage"
);
mgr.transition_to_recovering().unwrap();
mgr.record_success().unwrap();
assert_eq!(
mgr.state().unwrap(),
FallbackState::Primary,
"and the normal resume path recovers it"
);
}
#[test]
fn with_max_fail_count_no_padding_when_not_failed() {
let entry = FallbackEntry::new("model-a").with_max_fail_count(5);
assert!(!entry.failed());
assert_eq!(entry.attempt_count, 0);
assert_eq!(entry.max_fail_count, 5);
}
#[test]
fn with_max_fail_count_pads_already_failed_entry() {
let mut entry = FallbackEntry::new("model-b");
entry.record_attempt();
entry.record_attempt();
assert!(entry.failed());
assert_eq!(entry.attempt_count, 2);
let entry = entry.with_max_fail_count(5);
assert_eq!(entry.max_fail_count, 5);
assert_eq!(entry.attempt_count, 5);
assert!(entry.failed());
}
#[test]
fn with_max_fail_count_pads_exactly_to_new_threshold() {
let mut entry = FallbackEntry::new("model-c");
entry.record_attempt();
entry.record_attempt();
assert!(entry.failed());
let entry = entry.with_max_fail_count(3);
assert_eq!(entry.attempt_count, 3);
assert!(entry.failed());
}
#[test]
fn with_max_fail_count_no_padding_when_lowering() {
let mut entry = FallbackEntry::new("model-d");
entry.record_attempt();
entry.record_attempt();
assert!(entry.failed());
let entry = entry.with_max_fail_count(1);
assert_eq!(entry.max_fail_count, 1);
assert_eq!(entry.attempt_count, 2);
assert!(entry.failed());
}
#[test]
fn with_max_fail_count_clamps_to_minimum_one() {
let entry = FallbackEntry::new("model-e").with_max_fail_count(0);
assert_eq!(entry.max_fail_count, 1);
}
#[test]
fn consecutive_failures_resets_on_trip() {
let mgr = FallbackManager::new(3, 2);
assert!(!mgr.record_failure(FailureKind::Transient).unwrap());
assert!(!mgr.record_failure(FailureKind::Transient).unwrap());
assert!(mgr.record_failure(FailureKind::Transient).unwrap());
assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
assert_eq!(
mgr.consecutive_failures().unwrap(),
0,
"field doc: reset to 0 on trip"
);
}
#[test]
fn transient_failure_during_recovery_breaks_success_streak() {
let mgr = FallbackManager::new(3, 2);
for _ in 0..3 {
mgr.record_failure(FailureKind::Transient).unwrap();
}
mgr.transition_to_recovering().unwrap();
mgr.record_success().unwrap();
mgr.record_failure(FailureKind::Transient).unwrap();
mgr.record_success().unwrap();
assert_eq!(
mgr.state().unwrap(),
FallbackState::Recovering,
"field doc: only consecutive successes accumulate toward recovery"
);
}
#[test]
fn active_model_does_not_return_primary_when_chain_is_exhausted() {
let mgr = FallbackManager::for_model("primary").unwrap();
mgr.add_fallback_model("fb1").unwrap();
mgr.add_fallback_model("fb2").unwrap();
mgr.mark_fallback_failed("fb1").unwrap();
mgr.mark_fallback_failed("fb1").unwrap();
mgr.mark_fallback_failed("fb2").unwrap();
mgr.mark_fallback_failed("fb2").unwrap();
mgr.transition_to_fallback().unwrap();
assert_eq!(mgr.state().unwrap(), FallbackState::Fallback);
assert_ne!(
mgr.active_model().unwrap().as_deref(),
Some("primary"),
"a dedicated fallback is configured; the exhausted chain must not silently route back to the failed primary"
);
}
}