use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex as StdMutex};
use std::time::{Duration, Instant};
use breaker_machines::{CircuitBreaker, CircuitBuilder};
use chrono_machines::{BackoffStrategy, ExponentialBackoff};
use rand::rng;
use state_machines::runtime::{Clock, Runner};
use tokio::sync::{Mutex, mpsc};
use crate::bridge::Bridge;
use crate::error::Result;
use crate::jail::kqueue::{JailEvent, JailEventSource, JailIdent};
use crate::sys::OsVersion;
mod supervisor;
use supervisor::{DynamicSupervisor, SupervisorCtx, SupervisorEvent, SupervisorState};
const KQUEUE_DRAIN_INTERVAL: Duration = Duration::from_millis(250);
const SUPERVISION_STEPS: usize = 32;
const SUPERVISION_MAILBOX: usize = 8;
#[derive(Debug)]
pub enum WardenEvent {
JailFailed { name: String },
JailHealthFailed { name: String },
JailStarted { name: String },
JailStopped { name: String, jid: i32 },
JailHealthy { name: String },
QosTimeout { name: String },
RegisterJail {
name: String,
jid: i32,
descriptor_fd: Option<i32>,
},
Shutdown,
}
#[derive(Clone, Default)]
struct IntentionalStops {
jids: Arc<StdMutex<HashSet<i32>>>,
}
impl IntentionalStops {
fn lock_jids(&self) -> std::sync::MutexGuard<'_, HashSet<i32>> {
self.jids
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn mark(&self, jid: i32) {
self.lock_jids().insert(jid);
}
fn clear(&self, jid: i32) {
self.lock_jids().remove(&jid);
}
fn contains(&self, jid: i32) -> bool {
self.lock_jids().contains(&jid)
}
}
const fn monitor_ident(jid: i32, descriptor_fd: Option<i32>) -> JailIdent {
match descriptor_fd {
Some(fd) => JailIdent::Descriptor(fd),
None => JailIdent::Jid(jid),
}
}
fn has_descriptor_ident(monitored: &HashMap<JailIdent, MonitoredJail>, name: &str) -> bool {
monitored
.iter()
.any(|(ident, jail)| jail.name == name && matches!(ident, JailIdent::Descriptor(_)))
}
fn idents_for(
monitored: &HashMap<JailIdent, MonitoredJail>,
name: &str,
only_jid: Option<i32>,
) -> Vec<JailIdent> {
monitored
.iter()
.filter(|(_, jail)| jail.name == name && only_jid.is_none_or(|wanted| jail.jid == wanted))
.map(|(ident, _)| *ident)
.collect()
}
#[derive(Clone)]
struct MonitoredJail {
name: String,
jid: i32,
}
struct RestartState {
attempts: u8,
backoff: ExponentialBackoff,
breaker: CircuitBreaker,
}
impl RestartState {
fn new(name: &str) -> Self {
Self {
attempts: 0,
backoff: ExponentialBackoff::new()
.base_delay_ms(1000)
.max_delay_ms(60000)
.multiplier(2.0)
.max_attempts(10)
.jitter_factor(0.5),
breaker: CircuitBuilder::new(format!("warden_{name}"))
.failure_threshold(5)
.success_threshold(2)
.half_open_timeout_secs(300.0)
.build(),
}
}
fn reset(&mut self) {
self.attempts = 0;
self.breaker.record_success(0.0);
}
fn record_failure(&mut self) {
self.attempts = self.attempts.saturating_add(1);
self.breaker.record_failure(0.0);
}
fn next_delay(&self) -> Option<Duration> {
let mut rng = rng();
self.backoff
.delay(self.attempts, &mut rng)
.map(Duration::from_millis)
}
fn should_retry(&self) -> bool {
self.breaker.is_closed() && self.backoff.should_retry(self.attempts)
}
}
struct MillisClock {
origin: Instant,
}
impl MillisClock {
fn new() -> Self {
Self {
origin: Instant::now(),
}
}
}
impl Clock for MillisClock {
fn now(&self) -> u64 {
u64::try_from(self.origin.elapsed().as_millis()).unwrap_or(u64::MAX)
}
}
struct Supervision {
runner: Runner<DynamicSupervisor>,
policy: RestartState,
}
impl Supervision {
fn new(
name: &str,
bridge: Arc<Mutex<Bridge>>,
tx: mpsc::Sender<WardenEvent>,
clock: &MillisClock,
) -> Self {
let machine = DynamicSupervisor::new(SupervisorCtx::new(name, bridge, tx));
let mut runner = Runner::new(machine, SUPERVISION_MAILBOX);
if let Err(e) = runner.start(clock) {
eprintln!("Warden: Failed to start supervision of jail '{name}': {e:?}");
}
Self {
runner,
policy: RestartState::new(name),
}
}
fn state(&self) -> SupervisorState {
self.runner.machine().current_state()
}
fn arm_backoff(&mut self, name: &str, clock: &MillisClock) {
if self.state() != SupervisorState::Backoff || self.runner.next_deadline().is_some() {
return;
}
let delay = if self.policy.should_retry() {
self.policy.next_delay()
} else {
None
};
let Some(delay) = delay else {
eprintln!(
"Warden: Not restarting jail '{name}' (circuit breaker open or max attempts reached)"
);
self.enqueue(name, SupervisorEvent::Exhausted);
return;
};
self.policy.record_failure();
println!(
"Warden: Restarting jail '{}' in {:?} (attempt {})",
name, delay, self.policy.attempts
);
let ticks = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX);
if let Err(e) = self
.runner
.schedule_after(clock, ticks, SupervisorEvent::Retry)
{
eprintln!("Warden: Failed to schedule restart of jail '{name}': {e:?}");
}
}
fn settle(&mut self) {
if self.state() == SupervisorState::Healthy && self.policy.attempts > 0 {
self.policy.reset();
}
}
fn enqueue(&self, name: &str, event: SupervisorEvent) {
if let Err(e) = self.runner.enqueue(event) {
eprintln!("Warden: Supervision mailbox rejected event for '{name}': {e:?}");
}
}
}
pub struct Warden {
rx: mpsc::Receiver<WardenEvent>,
tx: mpsc::Sender<WardenEvent>,
supervisors: HashMap<String, Supervision>,
clock: MillisClock,
bridge: Arc<Mutex<Bridge>>,
kqueue: Option<JailEventSource>,
monitored: HashMap<JailIdent, MonitoredJail>,
intentional_stops: IntentionalStops,
}
impl Warden {
pub fn new(bridge: Arc<Mutex<Bridge>>) -> Self {
let (tx, rx) = mpsc::channel(100);
let kqueue = if OsVersion::detect_kernel().is_ok_and(|v| v.supports_jail_kqueue()) {
match JailEventSource::new() {
Ok(source) => {
println!("Warden: Using kqueue jail event monitoring");
Some(source)
}
Err(e) => {
eprintln!(
"Warden: Failed to create kqueue source: {e}, falling back to polling"
);
None
}
}
} else {
None
};
let intentional_stops = IntentionalStops::default();
Self {
rx,
tx,
supervisors: HashMap::new(),
clock: MillisClock::new(),
bridge,
kqueue,
monitored: HashMap::new(),
intentional_stops,
}
}
pub fn sender(&self) -> mpsc::Sender<WardenEvent> {
self.tx.clone()
}
pub fn register_jail_direct(&mut self, name: &str, jid: i32, descriptor_fd: Option<i32>) {
self.register_jail_monitoring(name, jid, descriptor_fd);
}
fn register_jail_monitoring(&mut self, name: &str, jid: i32, descriptor_fd: Option<i32>) {
let ident = monitor_ident(jid, descriptor_fd);
if descriptor_fd.is_none() && has_descriptor_ident(&self.monitored, name) {
return;
}
for stale in idents_for(&self.monitored, name, None) {
if stale != ident {
self.unregister_ident(name, stale);
}
}
self.monitored.insert(
ident,
MonitoredJail {
name: name.to_string(),
jid,
},
);
if let Some(ref kq) = self.kqueue {
let registered = match ident {
JailIdent::Descriptor(fd) => kq.register_jaildesc(fd),
JailIdent::Jid(jid) => kq.register_jail(jid),
};
match registered {
Ok(()) => println!("Warden: Monitoring jail '{name}' ({ident}) via kqueue"),
Err(e) => {
eprintln!("Warden: Failed to register kqueue for jail '{name}' ({ident}): {e}");
}
}
}
}
fn unregister_jail_jid(&mut self, name: &str, jid: i32) {
self.unregister_idents(name, Some(jid));
}
fn unregister_idents(&mut self, name: &str, only_jid: Option<i32>) {
for ident in idents_for(&self.monitored, name, only_jid) {
self.unregister_ident(name, ident);
}
}
fn unregister_ident(&mut self, name: &str, ident: JailIdent) {
self.monitored.remove(&ident);
if let Some(ref kq) = self.kqueue
&& let Err(e) = kq.unregister(ident)
{
eprintln!("Warden: Failed to unregister kqueue for jail '{name}' ({ident}): {e}");
}
}
fn process_kqueue_events(&self) -> Vec<(MonitoredJail, JailEvent)> {
let Some(kq) = &self.kqueue else {
return Vec::new();
};
let events = match kq.poll(Some(Duration::ZERO)) {
Ok(events) => events,
Err(e) => {
eprintln!("Warden: kqueue poll error: {e}");
return Vec::new();
}
};
let mut named_events = Vec::new();
for event in events {
if let Some(jail) = self.monitored.get(&event.ident()) {
named_events.push((jail.clone(), event));
}
}
named_events
}
pub async fn run(&mut self) {
println!("Warden: Starting jail supervisor");
let mut ticker = tokio::time::interval(KQUEUE_DRAIN_INTERVAL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
event = self.rx.recv() => {
match event {
Some(WardenEvent::Shutdown) | None => {
println!("Warden: Shutting down");
break;
}
Some(event) => self.handle_event(event).await,
}
}
_ = ticker.tick() => self.drain_kqueue(),
}
self.pump().await;
}
println!("Warden: Supervisor stopped");
}
fn drain_kqueue(&mut self) {
for (jail, event) in self.process_kqueue_events() {
let MonitoredJail { name, jid } = jail;
match event {
JailEvent::Remove { ident } => {
self.monitored.remove(&ident);
if self.intentional_stops.contains(jid) {
println!(
"Warden: Kernel reports jail '{name}' ({ident}) removed after requested stop"
);
} else {
println!("Warden: Kernel reports jail '{name}' ({ident}) removed");
self.handle_failure(&name);
}
}
JailEvent::Child { ident, coalesced } => {
let detail = if coalesced {
"multiple child jails"
} else {
"a child jail"
};
println!("Warden: Jail '{name}' ({ident}) created {detail}");
}
JailEvent::Set { .. } => {}
JailEvent::Attach { ident, coalesced } => {
if coalesced {
println!("Warden: Jail '{name}' ({ident}) had multiple attaches");
}
}
}
}
}
async fn pump(&mut self) {
for (name, supervision) in &mut self.supervisors {
if supervision.runner.tick(&self.clock).is_err() {
eprintln!("Warden: Monotonic clock went backwards supervising jail '{name}'");
}
if let Err(e) = supervision.runner.drain(SUPERVISION_STEPS).await {
eprintln!("Warden: Supervision of jail '{name}' failed: {e:?}");
}
supervision.settle();
supervision.arm_backoff(name, &self.clock);
}
}
async fn handle_event(&mut self, event: WardenEvent) {
match event {
WardenEvent::JailFailed { name } => {
println!("Warden: Jail '{name}' failed, initiating restart");
self.handle_failure(&name);
}
WardenEvent::JailHealthFailed { name } => {
println!("Warden: Jail '{name}' health check failed, initiating restart");
self.handle_failure(&name);
}
WardenEvent::JailHealthy { name } => {
self.shift_to_steady(&name, "healthy").await;
}
WardenEvent::QosTimeout { name } => {
self.shift_to_steady(&name, "timer").await;
}
WardenEvent::JailStarted { name } => {
println!("Warden: Jail '{name}' started successfully");
if let Some(supervision) = self.supervisors.get_mut(&name)
&& supervision.state() == SupervisorState::Backoff
{
supervision.enqueue(&name, SupervisorEvent::Recovered);
}
}
WardenEvent::JailStopped { name, jid } => {
println!("Warden: Jail '{name}' stopped intentionally");
self.supervisors.remove(&name);
self.unregister_jail_jid(&name, jid);
self.intentional_stops.clear(jid);
}
WardenEvent::RegisterJail {
name,
jid,
descriptor_fd,
} => {
self.register_jail_monitoring(&name, jid, descriptor_fd);
}
WardenEvent::Shutdown => {
}
}
}
#[allow(clippy::needless_pass_by_ref_mut)] async fn shift_to_steady(&mut self, name: &str, trigger: &str) {
let result = {
let br = self.bridge.lock().await;
br.shift_qos(name, crate::scope::QosPhase::Steady)
};
match result {
Ok(crate::bridge::ShiftOutcome::Shifted) => {
println!("Warden: Jail '{name}' shifted to steady QoS profile ({trigger})");
}
Ok(_) => {}
Err(e) => eprintln!("Warden: Failed to shift jail '{name}' to steady QoS profile: {e}"),
}
}
fn handle_failure(&mut self, name: &str) {
let bridge = Arc::clone(&self.bridge);
let tx = self.tx.clone();
let clock = &self.clock;
let supervision = self
.supervisors
.entry(name.to_string())
.or_insert_with(|| Supervision::new(name, bridge, tx, clock));
match supervision.state() {
SupervisorState::Healthy => supervision.enqueue(name, SupervisorEvent::Failed),
SupervisorState::Dead => {
eprintln!("Warden: Not restarting jail '{name}' (supervision exhausted)");
}
state => println!(
"Warden: Jail '{}' is already recovering ({})",
name,
state.name()
),
}
}
pub async fn request_shutdown(sender: &mpsc::Sender<WardenEvent>) {
let _ = sender.send(WardenEvent::Shutdown).await;
}
}
#[derive(Clone)]
pub struct WardenHandle {
sender: mpsc::Sender<WardenEvent>,
intentional_stops: IntentionalStops,
}
impl WardenHandle {
pub fn new(warden: &Warden) -> Self {
Self {
sender: warden.sender(),
intentional_stops: warden.intentional_stops.clone(),
}
}
pub fn mark_stop_requested(&self, jid: i32) {
self.intentional_stops.mark(jid);
}
pub fn clear_stop_requested(&self, jid: i32) {
self.intentional_stops.clear(jid);
}
fn channel_closed_err() -> crate::error::Error {
crate::error::Error::Io(std::io::Error::other("Warden channel closed"))
}
fn post(&self, event: WardenEvent) -> Result<()> {
self.sender.try_send(event).map_err(|e| match e {
mpsc::error::TrySendError::Closed(_) => Self::channel_closed_err(),
mpsc::error::TrySendError::Full(_) => {
crate::error::Error::Io(std::io::Error::other("Warden event queue full"))
}
})
}
pub fn notify_failure(&self, name: &str) -> Result<()> {
self.post(WardenEvent::JailFailed {
name: name.to_string(),
})
}
pub fn notify_health_failure(&self, name: &str) -> Result<()> {
self.post(WardenEvent::JailHealthFailed {
name: name.to_string(),
})
}
pub fn notify_healthy(&self, name: &str) -> Result<()> {
self.post(WardenEvent::JailHealthy {
name: name.to_string(),
})
}
pub async fn notify_qos_timeout(&self, name: &str) -> Result<()> {
self.sender
.send(WardenEvent::QosTimeout {
name: name.to_string(),
})
.await
.map_err(|_| Self::channel_closed_err())
}
pub fn notify_started(&self, name: &str) -> Result<()> {
self.post(WardenEvent::JailStarted {
name: name.to_string(),
})
}
pub fn notify_stopped(&self, name: &str, jid: i32) -> Result<()> {
self.post(WardenEvent::JailStopped {
name: name.to_string(),
jid,
})
}
pub fn register_jail(&self, name: &str, jid: i32, descriptor_fd: Option<i32>) -> Result<()> {
self.post(WardenEvent::RegisterJail {
name: name.to_string(),
jid,
descriptor_fd,
})
}
}
#[cfg(test)]
mod tests;