use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use breaker_machines::{CircuitBreaker, CircuitBuilder};
use chrono_machines::{BackoffStrategy, ExponentialBackoff};
use rand::rng;
use tokio::sync::{mpsc, Mutex};
use crate::error::Result;
use crate::bridge::Bridge;
#[derive(Debug)]
pub enum WardenEvent {
JailFailed { name: String },
JailHealthFailed { name: String },
JailStarted { name: String },
JailStopped { name: String },
Shutdown,
}
struct RestartState {
attempts: u8,
backoff: ExponentialBackoff,
breaker: CircuitBreaker,
}
impl RestartState {
fn new(name: &str) -> Self {
Self {
attempts: 0,
backoff: ExponentialBackoff::new()
.base_delay_ms(1000) .max_delay_ms(60000) .multiplier(2.0) .max_attempts(10) .jitter_factor(0.5), breaker: CircuitBuilder::new(format!("warden_{}", name))
.failure_threshold(5)
.success_threshold(2)
.half_open_timeout_secs(300.0) .build(),
}
}
fn reset(&mut self) {
self.attempts = 0;
self.breaker.record_success(0.0);
}
fn record_failure(&mut self) {
self.attempts = self.attempts.saturating_add(1);
self.breaker.record_failure(0.0);
}
fn next_delay(&self) -> Option<Duration> {
let mut rng = rng();
self.backoff.delay(self.attempts, &mut rng)
.map(Duration::from_millis)
}
fn should_retry(&self) -> bool {
self.breaker.is_closed() && self.backoff.should_retry(self.attempts)
}
}
pub struct Warden {
rx: mpsc::Receiver<WardenEvent>,
tx: mpsc::Sender<WardenEvent>,
restart_states: HashMap<String, RestartState>,
bridge: Arc<Mutex<Bridge>>,
}
impl Warden {
pub fn new(bridge: Arc<Mutex<Bridge>>) -> Self {
let (tx, rx) = mpsc::channel(100);
Self {
rx,
tx,
restart_states: HashMap::new(),
bridge,
}
}
pub fn sender(&self) -> mpsc::Sender<WardenEvent> {
self.tx.clone()
}
pub async fn run(&mut self) {
println!("Warden: Starting jail supervisor");
while let Some(event) = self.rx.recv().await {
match event {
WardenEvent::JailFailed { name } => {
println!("Warden: Jail '{}' failed, initiating restart", name);
self.handle_failure(&name).await;
}
WardenEvent::JailHealthFailed { name } => {
println!("Warden: Jail '{}' health check failed, initiating restart", name);
self.handle_failure(&name).await;
}
WardenEvent::JailStarted { name } => {
println!("Warden: Jail '{}' started successfully", name);
if let Some(state) = self.restart_states.get_mut(&name) {
state.reset();
}
}
WardenEvent::JailStopped { name } => {
println!("Warden: Jail '{}' stopped intentionally", name);
self.restart_states.remove(&name);
}
WardenEvent::Shutdown => {
println!("Warden: Shutting down");
break;
}
}
}
println!("Warden: Supervisor stopped");
}
async fn handle_failure(&mut self, name: &str) {
let state = self
.restart_states
.entry(name.to_string())
.or_insert_with(|| RestartState::new(name));
if !state.should_retry() {
eprintln!(
"Warden: Not restarting jail '{}' (circuit breaker open or max attempts reached)",
name
);
return;
}
let delay = match state.next_delay() {
Some(d) => d,
None => {
eprintln!("Warden: Max retries reached for jail '{}'", name);
return;
}
};
state.record_failure();
println!(
"Warden: Restarting jail '{}' in {:?} (attempt {})",
name, delay, state.attempts
);
tokio::time::sleep(delay).await;
let result = {
let mut br = self.bridge.lock().await;
br.restart_jail(name)
};
match result {
Ok(_) => {
println!("Warden: Jail '{}' restarted successfully", name);
if let Some(state) = self.restart_states.get_mut(name) {
state.reset();
}
}
Err(e) => {
eprintln!("Warden: Failed to restart jail '{}': {}", name, e);
}
}
}
pub async fn request_shutdown(sender: &mpsc::Sender<WardenEvent>) {
let _ = sender.send(WardenEvent::Shutdown).await;
}
}
#[derive(Clone)]
pub struct WardenHandle {
sender: mpsc::Sender<WardenEvent>,
}
impl WardenHandle {
pub fn new(warden: &Warden) -> Self {
Self {
sender: warden.sender(),
}
}
pub fn notify_failure_blocking(&self, name: &str) -> Result<()> {
self.sender
.blocking_send(WardenEvent::JailFailed {
name: name.to_string(),
})
.map_err(|_| crate::error::Error::Io(std::io::Error::other("Warden channel closed")))
}
pub fn notify_health_failure_blocking(&self, name: &str) -> Result<()> {
self.sender
.blocking_send(WardenEvent::JailHealthFailed {
name: name.to_string(),
})
.map_err(|_| crate::error::Error::Io(std::io::Error::other("Warden channel closed")))
}
pub fn notify_started_blocking(&self, name: &str) -> Result<()> {
self.sender
.blocking_send(WardenEvent::JailStarted {
name: name.to_string(),
})
.map_err(|_| crate::error::Error::Io(std::io::Error::other("Warden channel closed")))
}
pub fn notify_stopped_blocking(&self, name: &str) -> Result<()> {
self.sender
.blocking_send(WardenEvent::JailStopped {
name: name.to_string(),
})
.map_err(|_| crate::error::Error::Io(std::io::Error::other("Warden channel closed")))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_restart_state_backoff() {
let state = RestartState::new("test_jail");
assert!(state.should_retry());
let delay = state.next_delay();
assert!(delay.is_some());
}
#[test]
fn test_restart_state_reset() {
let mut state = RestartState::new("test_jail");
state.attempts = 5;
state.reset();
assert_eq!(state.attempts, 0);
}
}