use std::collections::{HashMap, VecDeque};
use crate::snapshot::{ProcessInfo, Snapshot};
use crate::source::ProcessSignal;
pub const HISTORY_LEN: usize = 1024;
const VIRTUAL_IFACE_PREFIXES: &[&str] = &["lo", "docker", "veth", "br-", "virbr"];
#[derive(Debug, Default, Clone)]
pub struct History {
samples: VecDeque<f64>,
}
impl History {
pub fn push(&mut self, value: f64) {
if self.samples.len() == HISTORY_LEN {
self.samples.pop_front();
}
self.samples.push_back(value);
}
pub fn last(&self, n: usize) -> Vec<f64> {
let skip = self.samples.len().saturating_sub(n);
self.samples.iter().skip(skip).copied().collect()
}
pub fn max_of_last(&self, n: usize) -> f64 {
self.last(n).into_iter().fold(0.0, f64::max)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SortKey {
Pid,
Name,
Threads,
User,
Memory,
Cpu,
Gpu,
Vram,
}
impl SortKey {
pub const ALL: [SortKey; 8] = [
SortKey::Pid,
SortKey::Name,
SortKey::Threads,
SortKey::User,
SortKey::Memory,
SortKey::Cpu,
SortKey::Gpu,
SortKey::Vram,
];
pub fn descending_by_default(self) -> bool {
!matches!(self, SortKey::Pid | SortKey::Name | SortKey::User)
}
}
#[derive(Debug, Clone)]
pub struct ProcessRow<'a> {
pub info: &'a ProcessInfo,
pub gpu_percent: Option<f32>,
pub vram_bytes: Option<u64>,
}
#[derive(Debug, Default, Clone)]
pub struct MetricHistory {
pub cpu: History,
pub memory: History,
pub gpus: Vec<History>,
pub net_rx: History,
pub net_tx: History,
pub net_interface: Option<String>,
}
impl MetricHistory {
pub fn record(&mut self, snapshot: &Snapshot) {
self.cpu.push(snapshot.cpu.total_percent as f64);
let memory = &snapshot.memory;
if memory.total_bytes > 0 {
self.memory
.push(memory.used_bytes as f64 / memory.total_bytes as f64 * 100.0);
}
self.gpus.resize_with(snapshot.gpus.len(), History::default);
for (history, gpu) in self.gpus.iter_mut().zip(&snapshot.gpus) {
history.push(gpu.utilization_percent.unwrap_or(0.0) as f64);
}
if self.net_interface.is_none() {
self.net_interface = default_interface(snapshot);
}
let (rx, tx) = self
.net_interface
.as_deref()
.and_then(|name| snapshot.networks.iter().find(|n| n.name == name))
.map(|n| (n.rx_bytes_per_sec, n.tx_bytes_per_sec))
.unwrap_or((0.0, 0.0));
self.net_rx.push(rx);
self.net_tx.push(tx);
}
pub fn select_interface(&mut self, name: &str) {
if self.net_interface.as_deref() != Some(name) {
self.net_interface = Some(name.to_owned());
self.net_rx = History::default();
self.net_tx = History::default();
}
}
pub fn next_interface(&mut self, snapshot: &Snapshot) -> Option<String> {
let names: Vec<&str> = snapshot.networks.iter().map(|n| n.name.as_str()).collect();
if names.is_empty() {
return None;
}
let current = self
.net_interface
.as_deref()
.and_then(|name| names.iter().position(|n| *n == name));
let next = names[current.map_or(0, |i| (i + 1) % names.len())].to_owned();
self.select_interface(&next);
Some(next)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Severity {
Warning,
Critical,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Alert {
pub severity: Severity,
pub text: String,
}
const FS_WARN_PERCENT: f64 = 90.0;
const FS_CRITICAL_PERCENT: f64 = 97.0;
const GPU_HOT_CELSIUS: f32 = 85.0;
pub fn alerts(snapshot: &Snapshot) -> Vec<Alert> {
let mut alerts = Vec::new();
let mut add = |severity, text: String| alerts.push(Alert { severity, text });
for fs in &snapshot.raid.btrfs {
let name = fs
.mount_points
.first()
.map_or(fs.fsid.as_str(), String::as_str);
let missing = fs.device_stats.iter().filter(|d| d.missing).count();
if missing > 0 {
add(
Severity::Critical,
format!("btrfs {name}: {missing} device(s) missing"),
);
}
let errors: u64 = fs.device_stats.iter().map(|d| d.total_errors()).sum();
if errors > 0 {
let devices: Vec<String> = fs
.device_stats
.iter()
.filter(|d| d.total_errors() > 0)
.map(|d| d.label())
.collect();
add(
Severity::Critical,
format!("btrfs {name}: {errors} I/O errors ({})", devices.join(", ")),
);
}
if let Some(scrub) = &fs.scrub {
if scrub.uncorrectable > 0 {
add(
Severity::Critical,
format!(
"btrfs {name}: scrub found {} uncorrectable errors",
scrub.uncorrectable
),
);
} else if scrub.errors > 0 {
add(
Severity::Warning,
format!("btrfs {name}: scrub corrected {} errors", scrub.corrected),
);
}
}
}
for md in &snapshot.raid.md {
if md.active && md.is_degraded() {
add(
Severity::Critical,
format!("{} ({}) degraded", md.name, md.level),
);
} else if !md.active {
add(Severity::Warning, format!("{} inactive", md.name));
}
if let Some(sync) = &md.sync {
add(
Severity::Warning,
format!("{} {} {:.0}%", md.name, sync.action, sync.percent),
);
}
}
for smart in &snapshot.smart {
if let Some(error) = &smart.error {
add(
Severity::Warning,
format!("{}: SMART unreadable ({error})", smart.device),
);
} else if smart.passed == Some(false) {
add(
Severity::Critical,
format!("{}: SMART failed", smart.device),
);
} else if smart.needs_attention() {
add(
Severity::Critical,
format!("{}: SMART warning", smart.device),
);
}
}
for fs in &snapshot.filesystems {
let used = fs.total_bytes.saturating_sub(fs.available_bytes);
let percent = used as f64 / fs.total_bytes.max(1) as f64 * 100.0;
let name = fs
.mount_points
.first()
.map_or(fs.device.as_str(), String::as_str);
if percent >= FS_CRITICAL_PERCENT {
add(Severity::Critical, format!("{name} {percent:.0}% full"));
} else if percent >= FS_WARN_PERCENT {
add(Severity::Warning, format!("{name} {percent:.0}% full"));
}
}
for gpu in &snapshot.gpus {
if let Some(t) = gpu.temperature_c.filter(|t| *t >= GPU_HOT_CELSIUS) {
add(Severity::Warning, format!("GPU {} at {t:.0}°C", gpu.index));
}
}
alerts.sort_by_key(|a| std::cmp::Reverse(a.severity));
alerts
}
#[derive(Debug, Clone, Copy)]
pub struct ProcessQuery<'q> {
pub sort_key: SortKey,
pub descending: bool,
pub filter: &'q str,
pub show_kernel_threads: bool,
}
pub fn process_rows<'a>(snapshot: &'a Snapshot, query: &ProcessQuery<'_>) -> Vec<ProcessRow<'a>> {
let mut gpu_usage: HashMap<u32, (Option<f32>, Option<u64>)> = HashMap::new();
for gpu in &snapshot.gpus {
for p in &gpu.processes {
let entry = gpu_usage.entry(p.pid).or_default();
if let Some(sm) = p.sm_percent {
entry.0 = Some(entry.0.map_or(sm, |v| v.max(sm)));
}
if let Some(mem) = p.memory_bytes {
entry.1 = Some(entry.1.unwrap_or(0) + mem);
}
}
}
let filter = query.filter.to_lowercase();
let mut rows: Vec<ProcessRow<'a>> = snapshot
.processes
.iter()
.filter(|p| query.show_kernel_threads || !p.kernel_thread)
.filter(|p| {
filter.is_empty()
|| p.name.to_lowercase().contains(&filter)
|| p.command.to_lowercase().contains(&filter)
|| p.user.to_lowercase().contains(&filter)
|| p.pid.to_string() == filter
})
.map(|info| {
let (gpu_percent, vram_bytes) = gpu_usage.get(&info.pid).copied().unwrap_or_default();
ProcessRow {
info,
gpu_percent,
vram_bytes,
}
})
.collect();
let key = query.sort_key;
rows.sort_by(|a, b| {
let ordering = match key {
SortKey::Pid => a.info.pid.cmp(&b.info.pid),
SortKey::Name => a.info.name.to_lowercase().cmp(&b.info.name.to_lowercase()),
SortKey::Threads => a.info.threads.cmp(&b.info.threads),
SortKey::User => a.info.user.cmp(&b.info.user),
SortKey::Memory => a.info.memory_bytes.cmp(&b.info.memory_bytes),
SortKey::Cpu => a.info.cpu_percent.total_cmp(&b.info.cpu_percent),
SortKey::Gpu => a
.gpu_percent
.unwrap_or(-1.0)
.total_cmp(&b.gpu_percent.unwrap_or(-1.0)),
SortKey::Vram => a.vram_bytes.cmp(&b.vram_bytes),
}
.then(a.info.pid.cmp(&b.info.pid));
if query.descending {
ordering.reverse()
} else {
ordering
}
});
rows
}
pub const LHM_HINT: &str = "temperatures need LibreHardwareMonitor";
pub fn lhm_help(error: &str) -> String {
format!(
"Windows needs LibreHardwareMonitor for temperatures, fans and CPU power.\n\
Start it and turn on Options → Remote Web Server → Run.\n\
([lhm] url = \"\" in the configuration hides this note.)\n\n\
{error}"
)
}
pub fn signal_name(signal: ProcessSignal) -> &'static str {
match signal {
ProcessSignal::Terminate => "SIGTERM",
ProcessSignal::Kill => "SIGKILL",
}
}
pub fn is_virtual_interface(name: &str) -> bool {
VIRTUAL_IFACE_PREFIXES.iter().any(|p| name.starts_with(p))
}
pub fn default_interface(snapshot: &Snapshot) -> Option<String> {
snapshot
.networks
.iter()
.filter(|n| !is_virtual_interface(&n.name))
.max_by_key(|n| n.rx_total_bytes + n.tx_total_bytes)
.or_else(|| snapshot.networks.first())
.map(|n| n.name.clone())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::snapshot::{GpuProcess, GpuStats, NetworkStats};
fn process(pid: u32, name: &str, cpu: f32, kernel: bool) -> ProcessInfo {
ProcessInfo {
pid,
name: name.to_owned(),
cpu_percent: cpu,
kernel_thread: kernel,
..Default::default()
}
}
fn query(sort_key: SortKey) -> ProcessQuery<'static> {
ProcessQuery {
sort_key,
descending: sort_key.descending_by_default(),
filter: "",
show_kernel_threads: false,
}
}
#[test]
fn history_keeps_newest_samples() {
let mut history = History::default();
for i in 0..(HISTORY_LEN + 5) {
history.push(i as f64);
}
assert_eq!(
history.last(2),
vec![(HISTORY_LEN + 3) as f64, (HISTORY_LEN + 4) as f64]
);
assert_eq!(history.last(HISTORY_LEN * 2).len(), HISTORY_LEN);
assert_eq!(history.max_of_last(3), (HISTORY_LEN + 4) as f64);
}
#[test]
fn rows_filter_sort_and_join_gpu_usage() {
let snapshot = Snapshot {
processes: vec![
process(10, "bash", 5.0, false),
process(2, "kworker/0:1", 50.0, true),
process(30, "python", 1.0, false),
],
gpus: vec![GpuStats {
processes: vec![GpuProcess {
pid: 30,
memory_bytes: Some(1 << 30),
sm_percent: Some(70.0),
}],
..Default::default()
}],
..Default::default()
};
let pids = |q: &ProcessQuery| {
process_rows(&snapshot, q)
.iter()
.map(|r| r.info.pid)
.collect::<Vec<_>>()
};
assert_eq!(pids(&query(SortKey::Cpu)), vec![10, 30]);
assert_eq!(pids(&query(SortKey::Vram)), vec![30, 10]);
assert_eq!(
pids(&ProcessQuery {
show_kernel_threads: true,
..query(SortKey::Cpu)
}),
vec![2, 10, 30]
);
assert_eq!(
pids(&ProcessQuery {
filter: "PYT",
..query(SortKey::Pid)
}),
vec![30]
);
let rows = process_rows(&snapshot, &query(SortKey::Pid));
assert_eq!(rows[1].vram_bytes, Some(1 << 30));
assert_eq!(rows[1].gpu_percent, Some(70.0));
}
#[test]
fn metric_history_records_and_switches_interfaces() {
let snapshot = Snapshot {
networks: vec![
NetworkStats {
name: "enp9s0".to_owned(),
rx_total_bytes: 10,
rx_bytes_per_sec: 100.0,
..Default::default()
},
NetworkStats {
name: "wlp7s0".to_owned(),
tx_bytes_per_sec: 5.0,
..Default::default()
},
],
gpus: vec![GpuStats {
utilization_percent: Some(40.0),
..Default::default()
}],
..Default::default()
};
let mut history = MetricHistory::default();
history.record(&snapshot);
assert_eq!(history.net_interface.as_deref(), Some("enp9s0"));
assert_eq!(history.net_rx.last(1), vec![100.0]);
assert_eq!(history.gpus[0].last(1), vec![40.0]);
assert_eq!(history.next_interface(&snapshot).as_deref(), Some("wlp7s0"));
assert!(history.net_rx.last(10).is_empty());
history.record(&snapshot);
assert_eq!(history.net_tx.last(1), vec![5.0]);
}
#[test]
fn alerts_flag_storage_problems_first() {
use crate::snapshot::{
BtrfsDeviceStats, BtrfsFilesystem, FilesystemUsage, MdArray, RaidStats, SmartDevice,
};
let snapshot = Snapshot {
filesystems: vec![FilesystemUsage {
mount_points: vec!["/data".to_owned()],
total_bytes: 100,
available_bytes: 8,
..Default::default()
}],
raid: RaidStats {
btrfs: vec![BtrfsFilesystem {
mount_points: vec!["/".to_owned()],
device_stats: vec![BtrfsDeviceStats {
devid: 1,
corruption_errs: 2,
..Default::default()
}],
..Default::default()
}],
md: vec![MdArray {
name: "md0".to_owned(),
active: true,
level: "raid1".to_owned(),
expected_devices: Some(2),
working_devices: Some(1),
..Default::default()
}],
},
smart: vec![SmartDevice {
device: "sda".to_owned(),
passed: Some(true),
..Default::default()
}],
..Default::default()
};
let alerts = alerts(&snapshot);
let texts: Vec<&str> = alerts.iter().map(|a| a.text.as_str()).collect();
assert_eq!(
texts,
vec![
"btrfs /: 2 I/O errors (devid 1)",
"md0 (raid1) degraded",
"/data 92% full"
]
);
assert_eq!(alerts[0].severity, Severity::Critical);
assert_eq!(alerts[2].severity, Severity::Warning);
assert!(super::alerts(&Snapshot::default()).is_empty());
}
#[test]
fn interface_helpers() {
assert!(is_virtual_interface("veth5a74356"));
assert!(!is_virtual_interface("enp9s0"));
let snapshot = Snapshot {
networks: vec![
NetworkStats {
name: "docker0".to_owned(),
rx_total_bytes: 1 << 40,
..Default::default()
},
NetworkStats {
name: "enp9s0".to_owned(),
rx_total_bytes: 1 << 30,
..Default::default()
},
],
..Default::default()
};
assert_eq!(default_interface(&snapshot).as_deref(), Some("enp9s0"));
assert_eq!(signal_name(ProcessSignal::Kill), "SIGKILL");
}
}