use std::collections::HashSet;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use throttle_machines::TokenBucketPolicy;
use tokio::sync::Mutex;
use crate::error::Result;
use crate::{bridge, jail, manifest, sickbay, warden};
type SharedPacer = Arc<std::sync::Mutex<sickbay::pacer::Pacer>>;
pub fn handle(
config_path: &(impl manifest::ConfigSource + ?Sized),
verbose: bool,
autostart_only: bool,
) -> Result<()> {
let config = manifest::load(config_path)?;
let project_prefix = format!("{}-", config.config.project_name());
let configured_jails = config.jails.clone();
let health_policy = config.config.rate_limit.health_policy()?;
let monitor_ping_url = config.config.monitor_ping_url.clone();
let pacer: Option<SharedPacer> = match &config.config.rate_limit.health_concurrency {
Some(cfg) => Some(Arc::new(std::sync::Mutex::new(cfg.build()?))),
None => None,
};
let usage_sample_secs = config.config.usage_sample_secs.unwrap_or(30);
let health_state_store = sickbay::HealthStateStore::from_config(Some(&config));
let bridge = bridge::Bridge::open(config, verbose)?;
let jails_for_health = if autostart_only {
let autostart = bridge
.autostart_order()?
.into_iter()
.collect::<HashSet<_>>();
configured_jails
.into_iter()
.filter(|jail| autostart.contains(&jail.name))
.collect::<Vec<_>>()
} else {
configured_jails
};
let bridge = Arc::new(Mutex::new(bridge));
let rt = tokio::runtime::Runtime::new()?;
rt.block_on(async {
let mut warden = warden::Warden::new(Arc::clone(&bridge));
let sender = warden.sender();
let warden_handle_for_orch = warden::WardenHandle::new(&warden);
let warden_handle_for_health = warden::WardenHandle::new(&warden);
{
let mut br = bridge.lock().await;
br.set_warden_handle(warden_handle_for_orch);
}
{
let mut br = bridge.lock().await;
let startup = if autostart_only {
br.reconcile_autostart(false)
} else {
br.up(None)
};
if let Err(e) = startup {
eprintln!("Warning: Failed to start some jails: {e}");
}
}
let full_jail_name = |name: &str| full_jail_name(&project_prefix, name);
for jail_def in &jails_for_health {
let full_name = full_jail_name(&jail_def.name);
if let Ok(jid) = jail::jail_getid(&full_name) {
let descriptor_fd = bridge.lock().await.jail_descriptor_fd(&full_name);
warden.register_jail_direct(&full_name, jid, descriptor_fd);
}
}
let mut warden = warden;
let warden_task = tokio::spawn(async move {
warden.run().await;
});
spawn_qos_timers(
&jails_for_health,
&project_prefix,
&warden_handle_for_health,
);
let health_stop_signals = spawn_health_monitors(
&bridge,
&jails_for_health,
&project_prefix,
health_policy,
&warden_handle_for_health,
pacer.as_ref(),
&health_state_store,
);
if usage_sample_secs > 0 {
let sampled: Vec<String> = jails_for_health
.iter()
.map(|jail_def| full_jail_name(&jail_def.name))
.collect();
spawn_usage_sampler(Arc::clone(&bridge), sampled, usage_sample_secs);
}
if let Some(url) = monitor_ping_url {
spawn_monitor_ping(url);
}
println!("Warden supervisor started. Press Ctrl+C to stop.");
tokio::signal::ctrl_c()
.await
.expect("Failed to listen for Ctrl+C");
println!("\nShutting down...");
for stop_signal in &health_stop_signals {
stop_signal.store(true, std::sync::atomic::Ordering::SeqCst);
}
if !health_stop_signals.is_empty() {
println!("Stopped {} health monitor(s)", health_stop_signals.len());
}
warden::Warden::request_shutdown(&sender).await;
if let Err(e) = warden_task.await {
eprintln!("Warning: Warden task failed: {e}");
}
});
Ok(())
}
fn full_jail_name(project_prefix: &str, name: &str) -> String {
if name.starts_with(project_prefix) {
name.to_string()
} else {
format!("{project_prefix}{name}")
}
}
fn spawn_qos_timers(
jails: &[manifest::JailDef],
project_prefix: &str,
warden_handle: &warden::WardenHandle,
) {
for jail_def in jails {
let Some(deadline) = jail_def
.qos
.as_ref()
.and_then(bridge::startup_deadline_secs)
else {
continue;
};
let full_name = full_jail_name(project_prefix, &jail_def.name);
let handle = warden_handle.clone();
println!("QoS startup budget for jail '{full_name}': {deadline}s");
tokio::spawn(async move {
tokio::time::sleep(tokio::time::Duration::from_secs(deadline)).await;
if let Err(e) = handle.notify_qos_timeout(&full_name).await {
eprintln!("Warning: Failed to signal QoS startup timeout for '{full_name}': {e}");
}
});
}
}
enum Cycle {
Ran(Result<sickbay::HealthStatus>),
Shed,
}
async fn run_cycle(
mut checker: sickbay::HealthChecker,
pacer: Option<&SharedPacer>,
) -> Option<(sickbay::HealthChecker, Cycle)> {
let report = |checker: &sickbay::HealthChecker, result: &Result<sickbay::HealthStatus>| {
if let Err(e) = result {
eprintln!("Health check error for {}: {}", checker.jail_name(), e);
}
};
let Some(pacer) = pacer else {
let result = checker.run_checks();
report(&checker, &result);
return Some((checker, Cycle::Ran(result)));
};
let admission = lock(pacer).enter();
let Some(admission) = admission else {
println!(
"Health pacer: shed a check cycle for '{}' (limit {})",
checker.jail_name(),
lock(pacer).limit()
);
return Some((checker, Cycle::Shed));
};
let timeout = checker.timeout();
let started = std::time::Instant::now();
let (checker, result) = tokio::task::spawn_blocking(move || {
let result = checker.run_checks();
(checker, result)
})
.await
.ok()?;
report(&checker, &result);
let moved = lock(pacer).leave(admission, started.elapsed(), timeout, result.is_err());
if let Some((before, after)) = moved {
println!(
"Health pacer: limit {before} -> {after} ({} in flight)",
lock(pacer).in_flight()
);
}
Some((checker, Cycle::Ran(result)))
}
fn save_snapshot(
store: &sickbay::HealthStateStore,
checker: &sickbay::HealthChecker,
status: sickbay::HealthStatus,
cycle: &Cycle,
) {
if !matches!(cycle, Cycle::Ran(Ok(_))) {
return;
}
let record = sickbay::HealthStateRecord::from_checker(checker, status);
if let Err(e) = store.save(&record) {
eprintln!(
"Warning: Failed to save health state for '{}': {e}",
checker.jail_name()
);
}
}
fn lock(pacer: &SharedPacer) -> std::sync::MutexGuard<'_, sickbay::pacer::Pacer> {
pacer
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn spawn_health_monitors(
bridge: &Arc<Mutex<bridge::Bridge>>,
jails: &[manifest::JailDef],
project_prefix: &str,
health_policy: TokenBucketPolicy,
warden_handle: &warden::WardenHandle,
pacer: Option<&SharedPacer>,
state_store: &sickbay::HealthStateStore,
) -> Vec<Arc<AtomicBool>> {
let mut health_stop_signals = Vec::new();
for jail_def in jails {
if !jail_def.healthcheck.enabled || jail_def.healthcheck.checks.is_empty() {
continue;
}
let full_name = full_jail_name(project_prefix, &jail_def.name);
let mut checker = sickbay::HealthChecker::with_rate_limit(
&full_name,
jail_def.healthcheck.clone(),
health_policy,
)
.with_warden_handle(warden_handle.clone());
if let Ok(jid) = jail::jail_getid(&full_name) {
checker = checker.with_jid(jid);
}
health_stop_signals.push(checker.stop_signal());
let state_store = state_store.clone();
let bridge = Arc::clone(bridge);
let pacer = pacer.cloned();
let name = full_name.clone();
tokio::spawn(async move {
let mut current = Some(checker);
while let Some(checker) = current.take().filter(|c| !c.is_stopped()) {
let Some((checker, cycle)) = run_cycle(checker, pacer.as_ref()).await else {
eprintln!("Health monitor for '{name}' lost its checker");
return;
};
let status = checker.status();
bridge
.lock()
.await
.record_health(checker.jail_name(), status);
save_snapshot(&state_store, &checker, status, &cycle);
if status == sickbay::HealthStatus::Failing {
eprintln!("Health check failing for jail '{}'", checker.jail_name());
}
tokio::time::sleep(tokio::time::Duration::from_secs(
checker.interval().as_secs(),
))
.await;
current = Some(checker);
}
println!("Health monitor stopped for jail '{name}'");
});
println!("Spawned health monitor for jail '{full_name}'");
}
health_stop_signals
}
fn spawn_usage_sampler(bridge: Arc<Mutex<bridge::Bridge>>, sampled: Vec<String>, every_secs: u64) {
tokio::spawn(async move {
loop {
tokio::time::sleep(tokio::time::Duration::from_secs(every_secs)).await;
let br = bridge.lock().await;
for name in &sampled {
br.sample_usage(name);
}
}
});
println!("Racct usage sampling every {every_secs}s.");
}
fn spawn_monitor_ping(url: String) {
std::thread::spawn(move || {
loop {
if let Err(e) = ureq::get(&url).call() {
eprintln!("Warning: monitor ping to {url} failed: {e}");
}
std::thread::sleep(std::time::Duration::from_secs(300));
}
});
println!("Dead-man ping enabled (every 300s).");
}
#[cfg(test)]
mod tests;