use crate::error::{Error, Result};
use crate::sickbay::recovery::{RecoveryAction, RecoveryConfig};
use crate::jail::ffi::{jail_getid, jail_remove};
use crate::warden::WardenHandle;
use breaker_machines::{CircuitBreaker, CircuitBuilder};
use serde::Deserialize;
use std::collections::HashMap;
use std::process::Command;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use throttle_machines::token_bucket;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealthStatus {
Starting,
Healthy,
Unhealthy,
Failing,
Suspended,
Unknown,
}
impl std::fmt::Display for HealthStatus {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HealthStatus::Starting => write!(f, "starting"),
HealthStatus::Healthy => write!(f, "healthy"),
HealthStatus::Unhealthy => write!(f, "unhealthy"),
HealthStatus::Failing => write!(f, "failing"),
HealthStatus::Suspended => write!(f, "suspended"),
HealthStatus::Unknown => write!(f, "unknown"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum CheckTarget {
Host,
#[default]
Jail,
}
#[derive(Debug, Clone, Deserialize)]
pub struct HealthCheck {
pub name: String,
pub command: String,
#[serde(default)]
pub target: CheckTarget,
#[serde(default = "default_interval")]
pub interval: u64,
#[serde(default = "default_timeout")]
pub timeout: u64,
#[serde(default = "default_start_period")]
pub start_period: u64,
#[serde(default = "default_retries")]
pub retries: u32,
pub recovery: Option<RecoveryConfig>,
}
fn default_interval() -> u64 {
30
}
fn default_timeout() -> u64 {
10
}
fn default_start_period() -> u64 {
60
}
fn default_retries() -> u32 {
3
}
#[cfg(test)]
impl HealthCheck {
pub fn new(name: &str, command: &str) -> Self {
Self {
name: name.to_string(),
command: command.to_string(),
target: CheckTarget::Jail,
interval: default_interval(),
timeout: default_timeout(),
start_period: default_start_period(),
retries: default_retries(),
recovery: None,
}
}
pub fn with_target(mut self, target: CheckTarget) -> Self {
self.target = target;
self
}
pub fn with_interval(mut self, interval: u64) -> Self {
self.interval = interval;
self
}
pub fn with_timeout(mut self, timeout: u64) -> Self {
self.timeout = timeout;
self
}
pub fn with_retries(mut self, retries: u32) -> Self {
self.retries = retries;
self
}
}
#[derive(Debug)]
pub struct CheckResult {
pub name: String,
pub passed: bool,
pub duration: Duration,
pub output: String,
pub timestamp: Instant,
}
impl CheckResult {
pub fn summary(&self) -> String {
let status = if self.passed { "ok" } else { "fail" };
let duration_ms = self.duration.as_millis();
if self.output.is_empty() {
format!("{}:{} ({}ms)", self.name, status, duration_ms)
} else {
format!(
"{}:{} ({}ms) - {}",
self.name,
status,
duration_ms,
self.output.lines().next().unwrap_or("")
)
}
}
pub fn age(&self) -> Duration {
self.timestamp.elapsed()
}
}
#[derive(Debug, Clone, Deserialize, Default)]
pub struct HealthCheckConfig {
#[serde(default)]
pub enabled: bool,
#[serde(default)]
pub checks: Vec<HealthCheck>,
}
#[cfg(test)]
impl HealthCheckConfig {
pub fn enabled() -> Self {
Self {
enabled: true,
checks: Vec::new(),
}
}
pub fn with_check(mut self, check: HealthCheck) -> Self {
self.checks.push(check);
self
}
}
#[derive(Debug)]
struct CheckState {
failures: u32,
last_result: Option<CheckResult>,
recovery_attempts: u32,
rate_limit_tokens: f64,
rate_limit_last_refill: f64,
}
impl CheckState {
fn new(capacity: f64) -> Self {
let now_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64();
Self {
failures: 0,
last_result: None,
recovery_attempts: 0,
rate_limit_tokens: capacity,
rate_limit_last_refill: now_secs,
}
}
}
impl Default for CheckState {
fn default() -> Self {
Self::new(5.0) }
}
pub struct HealthChecker {
jail_name: String,
jid: Option<i32>,
config: HealthCheckConfig,
status: HealthStatus,
started_at: Instant,
check_states: Vec<CheckState>,
stop_signal: Arc<AtomicBool>,
check_interval: Duration,
circuit_breakers: HashMap<String, CircuitBreaker>,
rate_limit_capacity: f64,
rate_limit_refill_rate: f64,
warden_handle: Option<WardenHandle>,
}
impl HealthChecker {
pub fn with_rate_limit(
jail_name: &str,
config: HealthCheckConfig,
rate_limit_capacity: f64,
rate_limit_refill_rate: f64,
) -> Self {
let check_count = config.checks.len();
let interval = config
.checks
.first()
.map(|c| c.interval)
.unwrap_or(30);
let circuit_breakers = config
.checks
.iter()
.map(|check| {
let breaker = CircuitBuilder::new(format!("health_{}_{}", jail_name, check.name))
.failure_threshold(check.retries as usize)
.success_threshold(2)
.half_open_timeout_secs(60.0)
.build();
(check.name.clone(), breaker)
})
.collect();
Self {
jail_name: jail_name.to_string(),
jid: None,
config,
status: HealthStatus::Unknown,
started_at: Instant::now(),
check_states: (0..check_count)
.map(|_| CheckState::new(rate_limit_capacity))
.collect(),
stop_signal: Arc::new(AtomicBool::new(false)),
check_interval: Duration::from_secs(interval),
circuit_breakers,
rate_limit_capacity,
rate_limit_refill_rate,
warden_handle: None,
}
}
pub fn with_warden_handle(mut self, handle: WardenHandle) -> Self {
self.warden_handle = Some(handle);
self
}
pub fn with_jid(mut self, jid: i32) -> Self {
self.jid = Some(jid);
self
}
pub fn status(&self) -> HealthStatus {
self.status
}
pub fn jail_name(&self) -> &str {
&self.jail_name
}
pub fn stop_signal(&self) -> Arc<AtomicBool> {
Arc::clone(&self.stop_signal)
}
pub fn is_stopped(&self) -> bool {
self.stop_signal.load(Ordering::SeqCst)
}
pub fn interval(&self) -> Duration {
self.check_interval
}
pub fn is_enabled(&self) -> bool {
self.config.enabled && !self.config.checks.is_empty()
}
pub fn run_checks(&mut self) -> Result<HealthStatus> {
if !self.is_enabled() {
return Ok(HealthStatus::Unknown);
}
let elapsed = self.started_at.elapsed().as_secs();
let in_start_period = self.config.checks.iter().any(|c| elapsed < c.start_period);
if in_start_period {
self.status = HealthStatus::Starting;
return Ok(self.status);
}
let mut any_failing = false;
let mut all_healthy = true;
let mut any_suspended = false;
let check_count = self.config.checks.len();
let mut results: Vec<Option<CheckResult>> = Vec::with_capacity(check_count);
let mut recovery_needed: Vec<(usize, RecoveryConfig)> = Vec::new();
let mut breaker_updates: Vec<(String, bool, f64)> = Vec::new();
let now_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64();
for (idx, check) in self.config.checks.iter().enumerate() {
let breaker_closed = self
.circuit_breakers
.get(&check.name)
.map(|b| b.is_closed())
.unwrap_or(true);
if !breaker_closed {
any_suspended = true;
all_healthy = false;
results.push(Some(CheckResult {
name: check.name.clone(),
passed: false,
duration: Duration::ZERO,
output: "Circuit breaker open - check suspended".to_string(),
timestamp: Instant::now(),
}));
continue;
}
let state = &self.check_states[idx];
let rate_result = token_bucket::check(
state.rate_limit_tokens,
state.rate_limit_last_refill,
now_secs,
self.rate_limit_capacity,
self.rate_limit_refill_rate,
);
if !rate_result.allowed {
self.check_states[idx].rate_limit_tokens = rate_result.new_tokens;
self.check_states[idx].rate_limit_last_refill = now_secs;
results.push(None);
continue;
}
self.check_states[idx].rate_limit_tokens = rate_result.new_tokens;
self.check_states[idx].rate_limit_last_refill = now_secs;
let result = self.execute_check(check)?;
let duration_secs = result.duration.as_secs_f64();
if result.passed {
self.check_states[idx].failures = 0;
self.check_states[idx].recovery_attempts = 0;
breaker_updates.push((check.name.clone(), true, duration_secs));
} else {
self.check_states[idx].failures += 1;
all_healthy = false;
breaker_updates.push((check.name.clone(), false, duration_secs));
if self.check_states[idx].failures >= check.retries {
any_failing = true;
if let Some(recovery) = &check.recovery {
recovery_needed.push((idx, recovery.clone()));
}
}
}
results.push(Some(result));
}
for (name, success, duration) in breaker_updates {
if let Some(breaker) = self.circuit_breakers.get_mut(&name) {
if success {
breaker.record_success(duration);
} else {
breaker.record_failure(duration);
}
}
}
for (idx, result) in results.into_iter().enumerate() {
if result.is_some() {
self.check_states[idx].last_result = result;
}
}
for (idx, recovery) in recovery_needed {
self.trigger_recovery(idx, &recovery)?;
}
self.status = if any_suspended && !any_failing {
HealthStatus::Suspended
} else if any_failing {
HealthStatus::Failing
} else if all_healthy {
HealthStatus::Healthy
} else {
HealthStatus::Unhealthy
};
Ok(self.status)
}
fn execute_check(&self, check: &HealthCheck) -> Result<CheckResult> {
let start = Instant::now();
let (passed, output) = match check.target {
CheckTarget::Host => self.execute_on_host(&check.command, check.timeout)?,
CheckTarget::Jail => {
if let Some(jid) = self.jid {
self.execute_in_jail(jid, &check.command, check.timeout)?
} else {
(false, "No jail ID available".to_string())
}
}
};
Ok(CheckResult {
name: check.name.clone(),
passed,
duration: start.elapsed(),
output,
timestamp: Instant::now(),
})
}
fn execute_on_host(&self, command: &str, timeout: u64) -> Result<(bool, String)> {
let mut child = Command::new("sh")
.args(["-c", command])
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.map_err(|e| Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "host".to_string(),
message: e.to_string(),
})?;
let timeout_duration = Duration::from_secs(timeout);
let start = Instant::now();
loop {
match child.try_wait() {
Ok(Some(status)) => {
let stdout = child
.stdout
.take()
.map(|mut s| {
let mut buf = String::new();
std::io::Read::read_to_string(&mut s, &mut buf).ok();
buf
})
.unwrap_or_default();
let stderr = child
.stderr
.take()
.map(|mut s| {
let mut buf = String::new();
std::io::Read::read_to_string(&mut s, &mut buf).ok();
buf
})
.unwrap_or_default();
let combined = format!("{}{}", stdout, stderr);
return Ok((status.success(), combined));
}
Ok(None) => {
if start.elapsed() > timeout_duration {
let _ = child.kill();
let _ = child.wait(); return Ok((
false,
format!("Health check timed out after {} seconds", timeout),
));
}
std::thread::sleep(Duration::from_millis(100));
}
Err(e) => {
return Err(Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "host".to_string(),
message: format!("Failed to wait for process: {}", e),
});
}
}
}
}
fn execute_in_jail(&self, jid: i32, command: &str, timeout: u64) -> Result<(bool, String)> {
let mut child = Command::new("/usr/sbin/jexec")
.arg(jid.to_string())
.args(["sh", "-c", command])
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.map_err(|e| Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "jail".to_string(),
message: format!("Failed to execute jexec: {}", e),
})?;
let timeout_duration = Duration::from_secs(timeout);
let start = Instant::now();
loop {
match child.try_wait() {
Ok(Some(status)) => {
let stdout = child
.stdout
.take()
.map(|mut s| {
let mut buf = String::new();
std::io::Read::read_to_string(&mut s, &mut buf).ok();
buf
})
.unwrap_or_default();
let stderr = child
.stderr
.take()
.map(|mut s| {
let mut buf = String::new();
std::io::Read::read_to_string(&mut s, &mut buf).ok();
buf
})
.unwrap_or_default();
let combined = format!("{}{}", stdout, stderr);
return Ok((status.success(), combined));
}
Ok(None) => {
if start.elapsed() > timeout_duration {
let _ = child.kill();
let _ = child.wait(); return Ok((
false,
format!("Health check timed out after {} seconds", timeout),
));
}
std::thread::sleep(Duration::from_millis(100));
}
Err(e) => {
return Err(Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "jail".to_string(),
message: format!("Failed to wait for jexec process: {}", e),
});
}
}
}
}
fn trigger_recovery(&mut self, check_idx: usize, config: &RecoveryConfig) -> Result<()> {
let state = &mut self.check_states[check_idx];
if state.recovery_attempts >= config.max_attempts {
eprintln!(
"Health check '{}' for jail '{}' exceeded max recovery attempts ({})",
self.config.checks[check_idx].name, self.jail_name, config.max_attempts
);
return Ok(());
}
state.recovery_attempts += 1;
println!(
"Triggering recovery action '{:?}' for jail '{}' (attempt {}/{})",
config.action, self.jail_name, state.recovery_attempts, config.max_attempts
);
if let Some(handle) = &self.warden_handle {
if let Err(e) = handle.notify_health_failure_blocking(&self.jail_name) {
eprintln!("Warning: Failed to notify Warden of health failure: {}", e);
}
}
match &config.action {
RecoveryAction::Restart => {
match jail_getid(&self.jail_name) {
Ok(jid) => {
println!("Recovery: Stopping jail '{}' (JID {})...", self.jail_name, jid);
if let Err(e) = jail_remove(jid) {
eprintln!(
"Recovery: Failed to stop jail '{}': {}",
self.jail_name, e
);
return Err(Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "recovery".to_string(),
message: format!("Failed to stop jail for restart: {}", e),
});
}
self.jid = None;
println!(
"Recovery: Jail '{}' stopped. Manual restart required via 'blackship up {}'",
self.jail_name, self.jail_name
);
println!(
"Recovery: Note: For automatic restart, use the 'supervise' command with Warden"
);
}
Err(e) => {
eprintln!(
"Recovery: Jail '{}' not found or already stopped: {}",
self.jail_name, e
);
}
}
}
RecoveryAction::Stop => {
match jail_getid(&self.jail_name) {
Ok(jid) => {
println!("Recovery: Stopping jail '{}' (JID {})...", self.jail_name, jid);
if let Err(e) = jail_remove(jid) {
eprintln!(
"Recovery: Failed to stop jail '{}': {}",
self.jail_name, e
);
return Err(Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "recovery".to_string(),
message: format!("Failed to stop jail: {}", e),
});
}
self.jid = None;
println!("Recovery: Jail '{}' stopped successfully", self.jail_name);
}
Err(e) => {
eprintln!(
"Recovery: Jail '{}' not found or already stopped: {}",
self.jail_name, e
);
}
}
}
RecoveryAction::Command(cmd) => {
println!("Recovery: Executing command for jail '{}'...", self.jail_name);
let output = Command::new("sh").args(["-c", cmd]).output().map_err(|e| {
Error::HealthCheckFailed {
jail: self.jail_name.clone(),
check: "recovery".to_string(),
message: e.to_string(),
}
})?;
if !output.status.success() {
eprintln!(
"Recovery command failed: {}",
String::from_utf8_lossy(&output.stderr)
);
} else {
println!("Recovery: Command executed successfully for jail '{}'", self.jail_name);
}
}
RecoveryAction::None => {}
}
Ok(())
}
pub fn get_check_results(&self) -> Vec<(&HealthCheck, Option<&CheckResult>, u32)> {
self.config
.checks
.iter()
.enumerate()
.map(|(idx, check)| {
(
check,
self.check_states[idx].last_result.as_ref(),
self.check_states[idx].failures,
)
})
.collect()
}
}
#[cfg(test)]
impl HealthChecker {
pub fn new(jail_name: &str, config: HealthCheckConfig) -> Self {
Self::with_rate_limit(jail_name, config, 5.0, 0.5)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_health_status_display() {
assert_eq!(HealthStatus::Healthy.to_string(), "healthy");
assert_eq!(HealthStatus::Failing.to_string(), "failing");
assert_eq!(HealthStatus::Starting.to_string(), "starting");
assert_eq!(HealthStatus::Suspended.to_string(), "suspended");
}
#[test]
fn test_health_check_builder() {
let check = HealthCheck::new("http", "curl -sf http://localhost:8080/health")
.with_target(CheckTarget::Jail)
.with_interval(15)
.with_timeout(5)
.with_retries(5);
assert_eq!(check.name, "http");
assert_eq!(check.interval, 15);
assert_eq!(check.timeout, 5);
assert_eq!(check.retries, 5);
}
#[test]
fn test_health_config_builder() {
let config = HealthCheckConfig::enabled()
.with_check(HealthCheck::new("test", "true"))
.with_check(HealthCheck::new("test2", "true"));
assert!(config.enabled);
assert_eq!(config.checks.len(), 2);
}
#[test]
fn test_health_checker_creation() {
let config = HealthCheckConfig::enabled().with_check(HealthCheck::new("test", "true"));
let checker = HealthChecker::new("testjail", config);
assert_eq!(checker.jail_name(), "testjail");
assert!(checker.is_enabled());
}
#[test]
fn test_health_check_deserialize() {
let toml = r#"
name = "http"
command = "curl -sf http://localhost/health"
target = "jail"
interval = 30
timeout = 10
start_period = 60
retries = 3
"#;
let check: HealthCheck = toml::from_str(toml).unwrap();
assert_eq!(check.name, "http");
assert_eq!(check.target, CheckTarget::Jail);
assert_eq!(check.interval, 30);
}
}