use std::collections::HashMap;
use std::process::Command;
use anyhow::{Context, Result, bail};
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct Daemon {
pub unit: String,
pub description: String,
pub load: String,
pub active: String,
pub sub: String,
pub main_pid: Option<u32>,
pub restarts: Option<u32>,
pub exec_status: Option<i32>,
pub memory_bytes: Option<u64>,
pub cpu_nsec: Option<u64>,
pub result: String,
pub unit_file_state: String,
pub fragment_path: String,
pub active_enter: String,
pub anomalies: Vec<Anomaly>,
}
impl Daemon {
pub fn severity(&self) -> u8 {
self.anomalies
.iter()
.map(|anomaly| anomaly.severity)
.max()
.unwrap_or(0)
}
pub fn anomaly_summary(&self) -> String {
if self.anomalies.is_empty() {
return "ok".to_string();
}
self.anomalies
.iter()
.take(3)
.map(|anomaly| anomaly.label.as_str())
.collect::<Vec<_>>()
.join(", ")
}
pub fn state_label(&self) -> String {
format!("{}/{}", self.active, self.sub)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Anomaly {
pub label: String,
pub detail: String,
pub severity: u8,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum DaemonAction {
Start,
Stop,
Restart,
Reload,
Enable,
Disable,
KillTerm,
KillKill,
}
impl DaemonAction {
pub fn label(self) -> &'static str {
match self {
Self::Start => "start",
Self::Stop => "stop",
Self::Restart => "restart",
Self::Reload => "reload",
Self::Enable => "enable",
Self::Disable => "disable",
Self::KillTerm => "term",
Self::KillKill => "kill",
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommandReport {
pub ok: bool,
pub command: String,
pub output: String,
}
pub fn load_daemons() -> Result<Vec<Daemon>> {
let output = command_output(
"systemctl",
&[
"list-units",
"--type=service",
"--all",
"--no-legend",
"--no-pager",
"--plain",
],
)
.context("failed to list systemd services")?;
let mut daemons = parse_list_units(&output);
enrich_daemons(&mut daemons)?;
for daemon in &mut daemons {
daemon.anomalies = detect_anomalies(daemon);
}
daemons.sort_by(|a, b| {
b.severity()
.cmp(&a.severity())
.then_with(|| a.unit.cmp(&b.unit))
});
Ok(daemons)
}
pub fn load_status(unit: &str) -> Result<Vec<String>> {
let output = command_output(
"systemctl",
&["status", unit, "--no-pager", "--lines=40", "--full"],
)
.with_context(|| format!("failed to read status for {unit}"))?;
Ok(output.lines().map(clean_line).collect())
}
pub fn load_journal(unit: &str, lookback_hours: u64, lines: usize) -> Result<Vec<String>> {
let since = if lookback_hours == 1 {
"1 hour ago".to_string()
} else {
format!("{lookback_hours} hours ago")
};
let line_count = lines.clamp(20, 2_000).to_string();
let output = command_output(
"journalctl",
&[
"-u",
unit,
"--since",
&since,
"-n",
&line_count,
"--no-pager",
"--output=short-iso",
],
)
.with_context(|| format!("failed to read journal for {unit}"))?;
let lines = output.lines().map(clean_line).collect::<Vec<_>>();
if lines.is_empty() {
Ok(vec![format!("no journal entries for {unit} since {since}")])
} else {
Ok(lines)
}
}
pub fn apply_action(unit: &str, action: DaemonAction) -> Result<CommandReport> {
let args = match action {
DaemonAction::Start => vec!["start", unit],
DaemonAction::Stop => vec!["stop", unit],
DaemonAction::Restart => vec!["restart", unit],
DaemonAction::Reload => vec!["reload", unit],
DaemonAction::Enable => vec!["enable", unit],
DaemonAction::Disable => vec!["disable", unit],
DaemonAction::KillTerm => vec!["kill", "--signal=TERM", unit],
DaemonAction::KillKill => vec!["kill", "--signal=KILL", unit],
};
let output = Command::new("systemctl")
.args(&args)
.output()
.with_context(|| format!("failed to run systemctl {} {unit}", action.label()))?;
let mut text = String::new();
text.push_str(&String::from_utf8_lossy(&output.stdout));
text.push_str(&String::from_utf8_lossy(&output.stderr));
let text = text.trim().to_string();
Ok(CommandReport {
ok: output.status.success(),
command: format!("systemctl {}", args.join(" ")),
output: if text.is_empty() {
"no output".to_string()
} else {
text
},
})
}
pub fn format_bytes(bytes: Option<u64>) -> String {
let Some(bytes) = bytes else {
return "-".to_string();
};
const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
let mut value = bytes as f64;
let mut unit = 0usize;
while value >= 1024.0 && unit + 1 < UNITS.len() {
value /= 1024.0;
unit += 1;
}
if unit == 0 {
format!("{} {}", bytes, UNITS[unit])
} else {
format!("{value:.1} {}", UNITS[unit])
}
}
pub fn format_cpu(cpu_nsec: Option<u64>) -> String {
let Some(nsec) = cpu_nsec else {
return "-".to_string();
};
let seconds = nsec / 1_000_000_000;
if seconds < 60 {
format!("{seconds}s")
} else if seconds < 3_600 {
format!("{}m", seconds / 60)
} else {
format!("{}h", seconds / 3_600)
}
}
fn enrich_daemons(daemons: &mut [Daemon]) -> Result<()> {
if daemons.is_empty() {
return Ok(());
}
let mut args = vec![
"show".to_string(),
"--property=Id,LoadState,ActiveState,SubState,Description,MainPID,NRestarts,ExecMainStatus,MemoryCurrent,CPUUsageNSec,Result,UnitFileState,FragmentPath,ActiveEnterTimestamp".to_string(),
"--no-pager".to_string(),
];
args.extend(daemons.iter().map(|daemon| daemon.unit.clone()));
let arg_refs = args.iter().map(String::as_str).collect::<Vec<_>>();
let output = command_output("systemctl", &arg_refs).context("failed to inspect services")?;
let blocks = parse_show_blocks(&output);
let index = daemons
.iter()
.enumerate()
.map(|(idx, daemon)| (daemon.unit.clone(), idx))
.collect::<HashMap<_, _>>();
for block in blocks {
let Some(unit) = block.get("Id") else {
continue;
};
let Some(idx) = index.get(unit).copied() else {
continue;
};
let daemon = &mut daemons[idx];
daemon.description = block
.get("Description")
.cloned()
.unwrap_or_else(|| daemon.description.clone());
daemon.load = block
.get("LoadState")
.cloned()
.unwrap_or_else(|| daemon.load.clone());
daemon.active = block
.get("ActiveState")
.cloned()
.unwrap_or_else(|| daemon.active.clone());
daemon.sub = block
.get("SubState")
.cloned()
.unwrap_or_else(|| daemon.sub.clone());
daemon.main_pid = parse_nonzero_u32(block.get("MainPID"));
daemon.restarts = parse_u32(block.get("NRestarts"));
daemon.exec_status = parse_i32(block.get("ExecMainStatus"));
daemon.memory_bytes = parse_nonzero_u64(block.get("MemoryCurrent"));
daemon.cpu_nsec = parse_nonzero_u64(block.get("CPUUsageNSec"));
daemon.result = block.get("Result").cloned().unwrap_or_default();
daemon.unit_file_state = block.get("UnitFileState").cloned().unwrap_or_default();
daemon.fragment_path = block.get("FragmentPath").cloned().unwrap_or_default();
daemon.active_enter = block
.get("ActiveEnterTimestamp")
.cloned()
.unwrap_or_default();
}
Ok(())
}
fn parse_list_units(output: &str) -> Vec<Daemon> {
output
.lines()
.filter_map(|line| {
let parts = line.split_whitespace().collect::<Vec<_>>();
if parts.len() < 4 || !parts[0].ends_with(".service") {
return None;
}
let description = if parts.len() > 4 {
parts[4..].join(" ")
} else {
String::new()
};
Some(Daemon {
unit: parts[0].to_string(),
load: parts[1].to_string(),
active: parts[2].to_string(),
sub: parts[3].to_string(),
description,
..Daemon::default()
})
})
.collect()
}
fn parse_show_blocks(output: &str) -> Vec<HashMap<String, String>> {
let mut blocks = Vec::new();
let mut block = HashMap::new();
for line in output.lines() {
if line.trim().is_empty() {
if !block.is_empty() {
blocks.push(block);
block = HashMap::new();
}
continue;
}
if let Some((key, value)) = line.split_once('=') {
block.insert(key.to_string(), value.to_string());
}
}
if !block.is_empty() {
blocks.push(block);
}
blocks
}
fn detect_anomalies(daemon: &Daemon) -> Vec<Anomaly> {
let mut anomalies = Vec::new();
if daemon.load == "not-found" || daemon.load == "error" {
anomalies.push(Anomaly {
label: daemon.load.clone(),
detail: "unit is referenced but not loadable".to_string(),
severity: 95,
});
}
if daemon.active == "failed" || daemon.result != "success" && !daemon.result.is_empty() {
anomalies.push(Anomaly {
label: "failed".to_string(),
detail: format!("result={}", blank_dash(&daemon.result)),
severity: 100,
});
}
if daemon.active == "activating" || daemon.active == "deactivating" {
anomalies.push(Anomaly {
label: daemon.active.clone(),
detail: "transitional service state".to_string(),
severity: 70,
});
}
if daemon.sub == "auto-restart" || daemon.sub == "start-pre" || daemon.sub == "stop-sigterm" {
anomalies.push(Anomaly {
label: daemon.sub.clone(),
detail: "systemd reports a noisy sub-state".to_string(),
severity: 80,
});
}
if daemon.active == "active" && daemon.sub == "running" && daemon.main_pid.is_none() {
anomalies.push(Anomaly {
label: "pid-missing".to_string(),
detail: "running service has no MainPID".to_string(),
severity: 60,
});
}
if let Some(restarts) = daemon.restarts {
if restarts >= 10 {
anomalies.push(Anomaly {
label: "restart-storm".to_string(),
detail: format!("{restarts} restarts recorded"),
severity: 90,
});
} else if restarts >= 3 {
anomalies.push(Anomaly {
label: "restarts".to_string(),
detail: format!("{restarts} restarts recorded"),
severity: 65,
});
}
}
if let Some(status) = daemon.exec_status {
if status != 0 && daemon.active != "active" {
anomalies.push(Anomaly {
label: "exit-code".to_string(),
detail: format!("ExecMainStatus={status}"),
severity: 75,
});
}
}
if let Some(bytes) = daemon.memory_bytes {
if bytes >= 2 * 1024 * 1024 * 1024 {
anomalies.push(Anomaly {
label: "high-mem".to_string(),
detail: format!("memory={}", format_bytes(Some(bytes))),
severity: 72,
});
}
}
anomalies
}
fn command_output(program: &str, args: &[&str]) -> Result<String> {
let output = Command::new(program)
.args(args)
.output()
.with_context(|| format!("failed to execute {program}"))?;
let mut text = String::new();
text.push_str(&String::from_utf8_lossy(&output.stdout));
text.push_str(&String::from_utf8_lossy(&output.stderr));
if !output.status.success() {
bail!("{program} {} failed: {}", args.join(" "), text.trim());
}
Ok(text)
}
fn parse_u32(value: Option<&String>) -> Option<u32> {
value.and_then(|value| value.parse::<u32>().ok())
}
fn parse_nonzero_u32(value: Option<&String>) -> Option<u32> {
parse_u32(value).filter(|value| *value != 0)
}
fn parse_i32(value: Option<&String>) -> Option<i32> {
value.and_then(|value| value.parse::<i32>().ok())
}
fn parse_nonzero_u64(value: Option<&String>) -> Option<u64> {
value
.and_then(|value| value.parse::<u64>().ok())
.filter(|value| *value != 0)
}
fn clean_line(line: &str) -> String {
line.chars()
.map(|ch| if ch.is_control() { ' ' } else { ch })
.collect::<String>()
}
fn blank_dash(value: &str) -> &str {
if value.is_empty() { "-" } else { value }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_systemctl_list_units() {
let input = "sshd.service loaded active running OpenSSH Daemon\nmissing.service not-found inactive dead missing.service";
let daemons = parse_list_units(input);
assert_eq!(daemons.len(), 2);
assert_eq!(daemons[0].unit, "sshd.service");
assert_eq!(daemons[0].description, "OpenSSH Daemon");
assert_eq!(daemons[1].load, "not-found");
}
#[test]
fn parses_systemctl_show_blocks() {
let input = "Id=a.service\nMainPID=42\n\nId=b.service\nMainPID=0\n";
let blocks = parse_show_blocks(input);
assert_eq!(blocks.len(), 2);
assert_eq!(blocks[0]["Id"], "a.service");
assert_eq!(blocks[1]["MainPID"], "0");
}
#[test]
fn failed_daemon_is_severe() {
let daemon = Daemon {
unit: "bad.service".to_string(),
active: "failed".to_string(),
result: "exit-code".to_string(),
..Daemon::default()
};
let anomalies = detect_anomalies(&daemon);
assert!(anomalies.iter().any(|anomaly| anomaly.label == "failed"));
}
}