rigtop_core/

view.rs

//! Front-end helpers shared by the TUI and the GUI: graph history, process
//! table sorting/filtering joined with GPU usage, and small naming helpers.
//!
//! Nothing here depends on a UI toolkit.

use std::collections::{HashMap, HashSet, VecDeque};
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::PathBuf;
use std::time::{Duration, SystemTime};

use chrono::{DateTime, Local};
use serde::{Deserialize, Serialize};

use crate::config::AlertsConfig;
use crate::format;
use crate::snapshot::{GpuStats, OomKill, PcieLink, Pressure, ProcessInfo, Snapshot};
use crate::source::ProcessSignal;

/// Number of samples kept for graphs: over an hour at the default interval
/// of 1 s, and wider than any realistic terminal in braille dots (2 samples
/// per cell).
pub const HISTORY_LEN: usize = 4096;

/// Interface name prefixes skipped when choosing the default network
/// interface (container bridges and virtual links).
const VIRTUAL_IFACE_PREFIXES: &[&str] = &["lo", "docker", "veth", "br-", "virbr"];

/// Samples averaged into one point of the coarse history (a minute at the
/// default interval of 1 s).
pub const COARSE_FACTOR: usize = 60;
/// Points kept in the coarse history (a day at the default interval).
pub const COARSE_LEN: usize = 1440;

/// Fixed-size history of samples, newest last, plus a coarse history of
/// averages for long time spans.
#[derive(Debug, Default, Clone)]
pub struct History {
    samples: VecDeque<f64>,
    coarse: VecDeque<f64>,
    /// Sum and count of the samples not yet averaged into `coarse`.
    pending: (f64, usize),
}

impl History {
    /// Appends a sample, dropping the oldest when full.
    pub fn push(&mut self, value: f64) {
        if self.samples.len() == HISTORY_LEN {
            self.samples.pop_front();
        }
        self.samples.push_back(value);
        self.pending.0 += value;
        self.pending.1 += 1;
        if self.pending.1 == COARSE_FACTOR {
            if self.coarse.len() == COARSE_LEN {
                self.coarse.pop_front();
            }
            self.coarse.push_back(self.pending.0 / COARSE_FACTOR as f64);
            self.pending = (0.0, 0);
        }
    }

    /// The newest `n` samples (or fewer), oldest first.
    pub fn last(&self, n: usize) -> Vec<f64> {
        let skip = self.samples.len().saturating_sub(n);
        self.samples.iter().skip(skip).copied().collect()
    }

    /// Largest of the newest `n` samples.
    pub fn max_of_last(&self, n: usize) -> f64 {
        self.last(n).into_iter().fold(0.0, f64::max)
    }

    /// Values for a graph `columns` wide, oldest first. [`GraphSpan::Fit`]
    /// gives the newest sample per column; a time span gives the samples of
    /// that span (one every `interval`) spread over the columns, averaged
    /// where several share a column and interpolated where there are fewer
    /// samples than columns. While the history is shorter than the span,
    /// fewer values come back, so the graph fills up from the right.
    pub fn graph(&self, columns: usize, span: GraphSpan, interval: Duration) -> Vec<f64> {
        let GraphSpan::Secs(secs) = span else {
            return self.last(columns);
        };
        let wanted = ((secs as f64 / interval.as_secs_f64().max(0.001)).round() as usize).max(1);
        if wanted <= HISTORY_LEN {
            spread(&self.last(wanted), wanted, columns)
        } else {
            // Longer than the samples kept: from the averages.
            let wanted = wanted.div_ceil(COARSE_FACTOR);
            let skip = self.coarse.len().saturating_sub(wanted);
            let points: Vec<f64> = self.coarse.iter().skip(skip).copied().collect();
            spread(&points, wanted, columns)
        }
    }
}

/// `samples` (the newest of `wanted`) spread over `columns`: averaged where
/// several share a column, interpolated where there are fewer samples than
/// columns, and over a proportional share of the columns at the right while
/// fewer than `wanted` exist.
fn spread(samples: &[f64], wanted: usize, columns: usize) -> Vec<f64> {
    if columns == 0 || samples.is_empty() {
        return Vec::new();
    }
    // The share of the columns that the available samples cover.
    let shown = (columns as f64 * samples.len() as f64 / wanted.max(1) as f64).round() as usize;
    let shown = shown.clamp(1, columns);
    let per_column = samples.len() as f64 / shown as f64;
    if per_column < 1.0 {
        // Stretched: a straight line between neighbouring samples.
        let last = samples.len() - 1;
        return (0..shown)
            .map(|i| {
                let at = ((i as f64 + 0.5) * per_column - 0.5).clamp(0.0, last as f64);
                let (below, frac) = (at.floor() as usize, at.fract());
                let above = (below + 1).min(last);
                samples[below] * (1.0 - frac) + samples[above] * frac
            })
            .collect();
    }
    (0..shown)
        .map(|i| {
            let end = (((i + 1) as f64 * per_column) as usize).clamp(1, samples.len());
            let start = ((i as f64 * per_column) as usize).min(end - 1);
            let part = &samples[start..end];
            part.iter().sum::<f64>() / part.len() as f64
        })
        .collect()
}

/// How much time the graphs cover.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum GraphSpan {
    /// One sample per graph column, so the time covered grows with the
    /// width and the interval.
    #[default]
    Fit,
    /// The last this many seconds, spread over the width.
    Secs(u64),
}

impl GraphSpan {
    /// The choices [`GraphSpan::next`] steps through.
    pub const CHOICES: [GraphSpan; 7] = [
        GraphSpan::Fit,
        GraphSpan::Secs(60),
        GraphSpan::Secs(5 * 60),
        GraphSpan::Secs(15 * 60),
        GraphSpan::Secs(60 * 60),
        GraphSpan::Secs(6 * 60 * 60),
        GraphSpan::Secs(24 * 60 * 60),
    ];

    /// The choice after this one, back to the first after the last.
    pub fn next(self) -> Self {
        let index = Self::CHOICES
            .iter()
            .position(|c| *c == self)
            .map_or(0, |i| i + 1);
        Self::CHOICES[index % Self::CHOICES.len()]
    }

    /// Short name: "fit", "1m", "1h".
    pub fn label(self) -> String {
        match self {
            GraphSpan::Fit => "fit".to_owned(),
            GraphSpan::Secs(secs) if secs > 0 && secs % 3600 == 0 => format!("{}h", secs / 3600),
            GraphSpan::Secs(secs) if secs > 0 && secs % 60 == 0 => format!("{}m", secs / 60),
            GraphSpan::Secs(secs) => format!("{secs}s"),
        }
    }

    /// Seconds covered, 0 for [`GraphSpan::Fit`] (for storing the choice).
    pub fn as_secs(self) -> u64 {
        match self {
            GraphSpan::Fit => 0,
            GraphSpan::Secs(secs) => secs,
        }
    }

    /// The inverse of [`GraphSpan::as_secs`].
    pub fn from_secs(secs: u64) -> Self {
        if secs == 0 {
            GraphSpan::Fit
        } else {
            GraphSpan::Secs(secs)
        }
    }
}

/// Columns the process table can be sorted by.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SortKey {
    Pid,
    Name,
    Threads,
    User,
    /// Container name; processes outside containers last.
    Container,
    Memory,
    Cpu,
    /// Disk reads plus writes.
    Disk,
    Gpu,
    Vram,
}

impl SortKey {
    /// Sort keys in column order, used to step left/right.
    pub const ALL: [SortKey; 10] = [
        SortKey::Pid,
        SortKey::Name,
        SortKey::Threads,
        SortKey::User,
        SortKey::Container,
        SortKey::Memory,
        SortKey::Cpu,
        SortKey::Disk,
        SortKey::Gpu,
        SortKey::Vram,
    ];

    /// Whether "descending" is the natural order (largest consumers first).
    pub fn descending_by_default(self) -> bool {
        !matches!(
            self,
            SortKey::Pid | SortKey::Name | SortKey::User | SortKey::Container
        )
    }
}

/// One row of the process table: the process plus its GPU usage.
#[derive(Debug, Clone)]
pub struct ProcessRow<'a> {
    pub info: &'a ProcessInfo,
    /// Highest per-GPU SM utilization of the process.
    pub gpu_percent: Option<f32>,
    /// VRAM used by the process across all GPUs.
    pub vram_bytes: Option<u64>,
    /// In the tree view, the lines leading to the process ("│ ├ "), to be
    /// put before its name; empty otherwise.
    pub tree_prefix: String,
}

/// Graph histories of one host, updated from each snapshot.
#[derive(Debug, Default, Clone)]
pub struct MetricHistory {
    /// Total CPU usage (%).
    pub cpu: History,
    /// Used memory (% of total).
    pub memory: History,
    /// Utilization (%) per GPU index.
    pub gpus: Vec<History>,
    /// Receive rate of `net_interface` (bytes/s).
    pub net_rx: History,
    /// Transmit rate of `net_interface` (bytes/s).
    pub net_tx: History,
    /// Interface whose traffic is recorded (`None`: chosen on the next snapshot).
    pub net_interface: Option<String>,
}

impl MetricHistory {
    /// Appends the values of a new snapshot.
    pub fn record(&mut self, snapshot: &Snapshot) {
        self.cpu.push(snapshot.cpu.total_percent as f64);
        let memory = &snapshot.memory;
        if memory.total_bytes > 0 {
            self.memory
                .push(memory.used_bytes as f64 / memory.total_bytes as f64 * 100.0);
        }
        self.gpus.resize_with(snapshot.gpus.len(), History::default);
        for (history, gpu) in self.gpus.iter_mut().zip(&snapshot.gpus) {
            history.push(gpu.utilization_percent.unwrap_or(0.0) as f64);
        }
        if self.net_interface.is_none() {
            self.net_interface = default_interface(snapshot);
        }
        let (rx, tx) = self
            .net_interface
            .as_deref()
            .and_then(|name| snapshot.networks.iter().find(|n| n.name == name))
            .map(|n| (n.rx_bytes_per_sec, n.tx_bytes_per_sec))
            .unwrap_or((0.0, 0.0));
        self.net_rx.push(rx);
        self.net_tx.push(tx);
    }

    /// Switches the recorded network interface, clearing its history.
    pub fn select_interface(&mut self, name: &str) {
        if self.net_interface.as_deref() != Some(name) {
            self.net_interface = Some(name.to_owned());
            // The old samples belong to another interface.
            self.net_rx = History::default();
            self.net_tx = History::default();
        }
    }

    /// Switches to the interface after the current one in `snapshot` and
    /// returns its name.
    pub fn next_interface(&mut self, snapshot: &Snapshot) -> Option<String> {
        let names: Vec<&str> = snapshot.networks.iter().map(|n| n.name.as_str()).collect();
        if names.is_empty() {
            return None;
        }
        let current = self
            .net_interface
            .as_deref()
            .and_then(|name| names.iter().position(|n| *n == name));
        let next = names[current.map_or(0, |i| (i + 1) % names.len())].to_owned();
        self.select_interface(&next);
        Some(next)
    }
}

/// How urgent an [`Alert`] is.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Severity {
    /// Worth a look (e.g. a filesystem filling up, a RAID resync).
    Warning,
    /// Data at risk (errors, a missing device, a failing drive).
    Critical,
}

/// A condition on a host that needs attention, for overviews.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Alert {
    pub severity: Severity,
    pub text: String,
}

/// Identifies an alert across samples: its severity and text with numbers
/// masked, so "3 I/O errors" becoming "4 I/O errors" or a temperature
/// changing does not count as a new alert.
pub fn alert_key(alert: &Alert) -> String {
    let mut key = format!("{:?}:", alert.severity);
    let mut in_number = false;
    for c in alert.text.chars() {
        if c.is_ascii_digit() || (in_number && c == '.') {
            if !in_number {
                key.push('#');
            }
            in_number = true;
        } else {
            key.push(c);
            in_number = false;
        }
    }
    key
}

/// Whether an [`AlertEvent`] is an alert appearing or going away, or
/// something that happened once.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum AlertChange {
    Raised,
    Cleared,
    /// A one-off event, not a lasting condition (a watched process exited):
    /// never cleared. Older rigtops skip these lines of a saved log.
    Notice,
}

/// One entry of the [`AlertLog`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AlertEvent {
    /// When the change was seen, in milliseconds since the Unix epoch.
    pub at_ms: u64,
    pub host: String,
    pub change: AlertChange,
    /// The alert as raised, or as last seen before it cleared.
    pub alert: Alert,
}

/// Events kept by an [`AlertLog`].
const ALERT_LOG_LEN: usize = 500;

/// The alerts of every host as they appear and clear, so that what
/// happened while nobody watched can be looked up later.
#[derive(Debug, Default)]
pub struct AlertLog {
    /// Newest last.
    events: VecDeque<AlertEvent>,
    /// Each host's alerts at its previous update.
    active: HashMap<String, Vec<Alert>>,
    /// Alerts raised since [`AlertLog::mark_read`], and the worst of them.
    unread: (usize, Option<Severity>),
    /// Where events are kept between runs (JSON lines), if anywhere.
    file: Option<PathBuf>,
}

/// Where a front-end ("tui", "gui") keeps its alert history: the state
/// directory (`~/.local/state/rigtop/` on Linux), else the local data
/// directory (macOS, Windows).
pub fn alert_log_path(front_end: &str) -> Option<PathBuf> {
    dirs::state_dir()
        .or_else(dirs::data_local_dir)
        .map(|dir| dir.join("rigtop").join(format!("alerts-{front_end}.jsonl")))
}

impl AlertLog {
    /// A log kept in `path`: the events of earlier runs are loaded, alerts
    /// they left raised count as still present (so a restart does not log
    /// them again), and new events are appended. A file that cannot be
    /// read or written leaves the log working in memory.
    pub fn persistent(path: PathBuf) -> Self {
        let mut log = Self::default();
        let loaded: Vec<AlertEvent> = fs::read_to_string(&path)
            .map(|text| {
                text.lines()
                    .filter_map(|line| serde_json::from_str(line).ok())
                    .collect()
            })
            .unwrap_or_default();
        let keep = loaded.len().saturating_sub(ALERT_LOG_LEN);
        for event in loaded.iter().filter(|e| e.change != AlertChange::Notice) {
            let active = log.active.entry(event.host.clone()).or_default();
            let key = alert_key(&event.alert);
            active.retain(|a| alert_key(a) != key);
            if event.change == AlertChange::Raised {
                active.push(event.alert.clone());
            }
        }
        log.events = loaded.into_iter().skip(keep).collect();
        // Rewrite a file grown well past what is shown.
        if keep > ALERT_LOG_LEN {
            let text: String = log
                .events
                .iter()
                .filter_map(|e| serde_json::to_string(e).ok())
                .map(|line| line + "\n")
                .collect();
            let tmp = path.with_extension("jsonl.tmp");
            let _ = fs::write(&tmp, text).and_then(|()| fs::rename(&tmp, &path));
        }
        log.file = Some(path);
        log
    }

    /// Appends `event` to the file, if the log has one.
    fn save(&self, event: &AlertEvent) {
        let Some(path) = &self.file else {
            return;
        };
        let Ok(line) = serde_json::to_string(event) else {
            return;
        };
        if let Some(dir) = path.parent() {
            let _ = fs::create_dir_all(dir);
        }
        // One write per line, so that lines from concurrent runs never mix.
        let _ = OpenOptions::new()
            .create(true)
            .append(true)
            .open(path)
            .and_then(|mut file| file.write_all(format!("{line}\n").as_bytes()));
    }

    /// Records how the alerts of `host` changed since its previous update,
    /// and returns those changes.
    pub fn update(&mut self, host: &str, alerts: &[Alert], at_ms: u64) -> Vec<AlertEvent> {
        let previous = self.active.remove(host).unwrap_or_default();
        let keys = |list: &[Alert]| -> HashSet<String> { list.iter().map(alert_key).collect() };
        let (before, now) = (keys(&previous), keys(alerts));
        let raised = alerts.iter().filter(|a| !before.contains(&alert_key(a)));
        let cleared = previous.iter().filter(|a| !now.contains(&alert_key(a)));
        let mut changes: Vec<(AlertChange, &Alert)> = raised
            .map(|a| (AlertChange::Raised, a))
            .chain(cleared.map(|a| (AlertChange::Cleared, a)))
            .collect();
        // Cleared first, so that a changed alert reads "went away, came back".
        changes.sort_by_key(|(change, _)| *change == AlertChange::Raised);
        let mut new_events = Vec::with_capacity(changes.len());
        for (change, alert) in changes {
            new_events.push(self.push(AlertEvent {
                at_ms,
                host: host.to_owned(),
                change,
                alert: alert.clone(),
            }));
        }
        self.active.insert(host.to_owned(), alerts.to_vec());
        new_events
    }

    /// [`AlertLog::update`] for `alerts` that leave out `snoozed` ones (see
    /// [`crate::snooze`]): an alert snoozed while it was raised leaves
    /// quietly instead of logging "cleared", and is raised again if it is
    /// still there when the snooze ends.
    pub fn update_snoozed(
        &mut self,
        host: &str,
        alerts: &[Alert],
        snoozed: &[Alert],
        at_ms: u64,
    ) -> Vec<AlertEvent> {
        if let Some(previous) = self.active.get_mut(host) {
            let quiet: HashSet<String> = snoozed.iter().map(alert_key).collect();
            previous.retain(|a| !quiet.contains(&alert_key(a)));
        }
        self.update(host, alerts, at_ms)
    }

    /// Records a one-off event of `host` ([`AlertChange::Notice`]), such as
    /// a watched process exiting, and returns it.
    pub fn record(&mut self, host: &str, alert: Alert, at_ms: u64) -> AlertEvent {
        self.push(AlertEvent {
            at_ms,
            host: host.to_owned(),
            change: AlertChange::Notice,
            alert,
        })
    }

    /// Adds `event` to the log (and its file), counting it as unread unless
    /// it is a clearing.
    fn push(&mut self, event: AlertEvent) -> AlertEvent {
        if event.change != AlertChange::Cleared {
            self.unread.0 += 1;
            self.unread.1 = self.unread.1.max(Some(event.alert.severity));
        }
        if self.events.len() == ALERT_LOG_LEN {
            self.events.pop_front();
        }
        self.save(&event);
        self.events.push_back(event.clone());
        event
    }

    /// All events, oldest first.
    pub fn events(&self) -> impl DoubleEndedIterator<Item = &AlertEvent> {
        self.events.iter()
    }

    /// Alerts raised since the log was last read, and the most severe one.
    pub fn unread(&self) -> (usize, Option<Severity>) {
        self.unread
    }

    /// Marks every event as read.
    pub fn mark_read(&mut self) {
        self.unread = (0, None);
    }
}

/// When an [`AlertEvent`] happened, as short as is clear: "14:03:12"
/// today, "Tue 14:03" this week, "Oct 21 14:03" before.
pub fn event_time(at_ms: u64, now: SystemTime) -> String {
    let at: DateTime<Local> = (SystemTime::UNIX_EPOCH + Duration::from_millis(at_ms)).into();
    let today: DateTime<Local> = now.into();
    let days = (today.date_naive() - at.date_naive()).num_days();
    if days == 0 {
        at.format("%H:%M:%S").to_string()
    } else if (1..6).contains(&days) {
        at.format("%a %H:%M").to_string()
    } else {
        at.format("%b %d %H:%M").to_string()
    }
}

/// Whether `process` is one `pattern` names: its name, or the file name
/// of a word of its command line (`train.py` for `python3 jobs/train.py`).
pub fn process_matches(process: &ProcessInfo, pattern: &str) -> bool {
    let pattern = pattern.trim();
    !pattern.is_empty()
        && (process.name == pattern
            || process
                .command
                .split_whitespace()
                .any(|word| word == pattern || word.rsplit(['/', '\\']).next() == Some(pattern)))
}

/// A process to announce when it exits.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WatchedProcess {
    pub pid: u32,
    /// Its name when the watch began; another name under the same pid
    /// means it exited and the pid was reused.
    pub name: String,
    /// What the exit names it by, when not `name` (e.g. the script).
    pub label: Option<String>,
    /// When it started (milliseconds since the Unix epoch, by the host's
    /// run time).
    pub started_ms: u64,
}

/// What is known about a [`WatchedProcess`] from a snapshot.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WatchState {
    Running,
    Exited,
    /// The source does not report it (yet): an agent before 0.19, or the
    /// request has not reached the source.
    Unknown,
}

/// The processes of one host to announce when they exit. The front-end
/// passes [`ExitWatch::pids`] to the source
/// ([`crate::DataSource::watch_processes`]), which then keeps them in
/// snapshots whatever the process limit.
#[derive(Debug, Clone, Default)]
pub struct ExitWatch {
    processes: Vec<WatchedProcess>,
}

impl ExitWatch {
    /// Whether `pid` is watched.
    pub fn contains(&self, pid: u32) -> bool {
        self.processes.iter().any(|w| w.pid == pid)
    }

    /// Starts or stops watching `process`; true when it is now watched.
    pub fn toggle(&mut self, process: &ProcessInfo, now_ms: u64) -> bool {
        if self.contains(process.pid) {
            self.processes.retain(|w| w.pid != process.pid);
            return false;
        }
        self.watch_as(process, None, now_ms);
        true
    }

    /// Starts watching `process` unless it is watched already, naming it
    /// `label` when it exits. True when it was not watched before.
    pub fn watch_as(&mut self, process: &ProcessInfo, label: Option<&str>, now_ms: u64) -> bool {
        if self.contains(process.pid) {
            return false;
        }
        self.processes.push(WatchedProcess {
            pid: process.pid,
            name: process.name.clone(),
            label: label.map(str::to_owned),
            started_ms: now_ms.saturating_sub(process.run_time_secs * 1000),
        });
        true
    }

    /// The watched pids, for the source.
    pub fn pids(&self) -> Vec<u32> {
        self.processes.iter().map(|w| w.pid).collect()
    }

    /// What `snapshot` says about the watched process `pid`.
    pub fn state(&self, snapshot: &Snapshot, pid: u32) -> WatchState {
        match self.processes.iter().find(|w| w.pid == pid) {
            Some(watched) => watch_state(snapshot, watched),
            None => WatchState::Unknown,
        }
    }

    /// Stops watching the processes that exited by `snapshot` and returns
    /// an alert for each: "train.py (pid 1234) exited after 3h12m".
    pub fn check(&mut self, snapshot: &Snapshot, now_ms: u64) -> Vec<Alert> {
        let mut exited = Vec::new();
        self.processes.retain(|watched| {
            if watch_state(snapshot, watched) != WatchState::Exited {
                return true;
            }
            let ran = now_ms.saturating_sub(watched.started_ms) / 1000;
            exited.push(Alert {
                severity: Severity::Warning,
                text: format!(
                    "{} (pid {}) exited after {}",
                    watched.label.as_deref().unwrap_or(&watched.name),
                    watched.pid,
                    format::duration(ran)
                ),
            });
            false
        });
        exited
    }
}

/// Whether `watched` still runs according to `snapshot`. Running only
/// when the source reports it, since only then will its exit be seen.
fn watch_state(snapshot: &Snapshot, watched: &WatchedProcess) -> WatchState {
    let reported = snapshot.watched_pids.contains(&watched.pid);
    match snapshot.processes.iter().find(|p| p.pid == watched.pid) {
        // The pid now belongs to another program.
        Some(process) if process.name != watched.name => WatchState::Exited,
        Some(_) if reported => WatchState::Running,
        None if reported => WatchState::Exited,
        _ => WatchState::Unknown,
    }
}

/// What a process detail view shows, as label and value rows: who started
/// it, who runs it, since when, what it uses, and its whole command line.
/// `None` once the process is gone.
pub fn process_details(
    snapshot: &Snapshot,
    pid: u32,
    now: SystemTime,
) -> Option<Vec<(&'static str, String)>> {
    let by_pid: HashMap<u32, &ProcessInfo> =
        snapshot.processes.iter().map(|p| (p.pid, p)).collect();
    let process = by_pid.get(&pid)?;
    let mut rows = vec![("Program", format!("{} (pid {})", process.name, process.pid))];

    // Parents up to the first process, oldest first; bounded in case of a
    // loop in inconsistent data.
    let mut chain = Vec::new();
    let mut parent = process.parent_pid;
    while let Some(ppid) = parent.filter(|_| chain.len() < 32) {
        match by_pid.get(&ppid) {
            Some(p) => {
                chain.push(format!("{} ({})", p.name, p.pid));
                parent = p.parent_pid.filter(|&next| next != p.pid);
            }
            None => {
                chain.push(ppid.to_string());
                break;
            }
        }
    }
    if !chain.is_empty() {
        chain.reverse();
        rows.push(("Parents", chain.join(" › ")));
    }
    let children = snapshot
        .processes
        .iter()
        .filter(|p| p.parent_pid == Some(pid))
        .count();
    if children > 0 {
        rows.push(("Children", children.to_string()));
    }
    rows.push(("User", process.user.clone()));
    if let Some(container) = &process.container {
        rows.push(("Container", container.clone()));
    }
    if !process.state.is_empty() {
        rows.push(("State", process.state.clone()));
    }
    if let Some(nice) = process.nice {
        rows.push(("Nice", nice.to_string()));
    }
    let started = now - Duration::from_secs(process.run_time_secs);
    rows.push(("Running", since_note(started, now)));
    rows.push(("CPU", format!("{:.1}%", process.cpu_percent)));
    rows.push(("Memory", format::bytes(process.memory_bytes)));
    if let Some(threads) = process.threads {
        rows.push(("Threads", threads.to_string()));
    }
    rows.push((
        "Disk",
        format!(
            "read {}, write {}",
            format::rate(process.read_bytes_per_sec),
            format::rate(process.write_bytes_per_sec)
        ),
    ));
    for gpu in &snapshot.gpus {
        if let Some(usage) = gpu.processes.iter().find(|p| p.pid == pid) {
            let mut text = format!("GPU {}", gpu.index);
            if let Some(sm) = usage.sm_percent {
                text.push_str(&format!(" {sm:.0}%"));
            }
            if let Some(memory) = usage.memory_bytes {
                text.push_str(&format!(", {} VRAM", format::bytes(memory)));
            }
            rows.push(("GPU", text));
        }
    }
    let command = if process.command.is_empty() {
        format!("[{}]", process.name)
    } else {
        process.command.clone()
    };
    rows.push(("Command", command));
    Some(rows)
}

/// Where a host goes in an overview sorted by attention: 0 for one that is
/// unreachable or has a critical alert, 1 for one with stale data or a
/// warning, 2 for the rest.
pub fn attention_rank(alerts: &[Alert], unreachable: bool, stale: bool) -> u8 {
    let worst = alerts.iter().map(|a| a.severity).max();
    if unreachable || worst == Some(Severity::Critical) {
        0
    } else if stale || worst == Some(Severity::Warning) {
        1
    } else {
        2
    }
}

/// The GPUs worth a line on an overview card: those with a utilization or
/// memory reading (an Intel iGPU, whose utilization needs perf, would only
/// say "--%"). The host view lists them all.
pub fn card_gpus(snapshot: &Snapshot) -> impl Iterator<Item = &GpuStats> {
    snapshot
        .gpus
        .iter()
        .filter(|g| g.utilization_percent.is_some() || g.memory_used_bytes.is_some())
}

/// A GPU's name with its power draw, for a card's detail line:
/// "NVIDIA GeForce RTX 4090 · 52/450W".
pub fn gpu_detail(gpu: &GpuStats) -> String {
    let mut text = gpu.name.clone();
    match (gpu.power_w, gpu.power_limit_w) {
        (Some(w), Some(limit)) => text.push_str(&format!(" · {w:.0}/{limit:.0}W")),
        (Some(w), None) => text.push_str(&format!(" · {w:.0}W")),
        _ => {}
    }
    text
}

/// Short notes for a GPU's title line, with how much attention each needs
/// (`None`: just information): running at its power limit, slowed down by
/// the hardware, ECC memory errors.
pub fn gpu_notes(gpu: &GpuStats) -> Vec<(String, Option<Severity>)> {
    let mut notes = Vec::new();
    let slowdown: Vec<&str> = gpu.slowdown_reasons().collect();
    if !slowdown.is_empty() {
        notes.push((
            format!("slowed: {}", slowdown.join(", ")),
            Some(Severity::Warning),
        ));
    } else if !gpu.throttle_reasons.is_empty() {
        notes.push(("power cap".to_owned(), None));
    }
    if let Some(ecc) = gpu.ecc {
        if ecc.uncorrected > 0 {
            notes.push((
                format!("ECC {} uncorrected", ecc.uncorrected),
                Some(Severity::Critical),
            ));
        } else if ecc.corrected > 0 {
            notes.push((format!("ECC {} corrected", ecc.corrected), None));
        }
    }
    notes
}

/// A GPU's PCIe link for a tooltip: "PCIe gen 4 x8 (up to gen 4 x16)".
pub fn pcie_text(link: &PcieLink) -> String {
    let now = format!("PCIe gen {} x{}", link.generation, link.width);
    if (link.generation, link.width) == (link.max_generation, link.max_width) {
        now
    } else {
        format!(
            "{now} (up to gen {} x{}; the link slows down when idle)",
            link.max_generation, link.max_width
        )
    }
}

/// How long it has been since `since` (e.g. a host's last data) and when
/// that was, in local time: "5m (since 14:03)", "2d03h (since Tue 14:03)".
pub fn since_note(since: SystemTime, now: SystemTime) -> String {
    let secs = now.duration_since(since).map_or(0, |d| d.as_secs());
    let span = match secs {
        0..60 => format!("{secs}s"),
        60..3_600 => format!("{}m", secs / 60),
        3_600..86_400 => format!("{}h{:02}m", secs / 3_600, secs % 3_600 / 60),
        _ => format!("{}d{:02}h", secs / 86_400, secs % 86_400 / 3_600),
    };
    let at: DateTime<Local> = since.into();
    let today: DateTime<Local> = now.into();
    let clock = if at.date_naive() == today.date_naive() {
        at.format("%H:%M")
    } else if secs < 6 * 86_400 {
        at.format("%a %H:%M")
    } else {
        at.format("%b %d %H:%M")
    };
    format!("{span} (since {clock})")
}

/// "agent 0.8.2" when `snapshot` comes from a rigtop of another version
/// than this one (an agent not updated yet), else `None`.
pub fn version_note(snapshot: &Snapshot) -> Option<String> {
    let version = &snapshot.host.rigtop_version;
    (!version.is_empty() && version != env!("CARGO_PKG_VERSION"))
        .then(|| format!("agent {version}"))
}

/// Hover text explaining a [`version_note`].
pub fn version_hint(snapshot: &Snapshot) -> String {
    format!(
        "this host runs rigtop {}, this machine {}: install the same version on both \
         for every reading",
        snapshot.host.rigtop_version,
        env!("CARGO_PKG_VERSION")
    )
}

/// Shown next to the uptime of a host that needs a reboot to finish an
/// update ([`crate::snapshot::HostInfo::reboot_required`]).
pub const REBOOT_NOTE: &str = "reboot needed";

/// Hover text for [`REBOOT_NOTE`], with the reason when known.
pub fn reboot_hint(reason: &str) -> String {
    let mut hint = "a reboot is needed to finish an update".to_owned();
    if !reason.is_empty() {
        hint.push_str(&format!(" ({reason})"));
    }
    hint
}

/// How a disk's request time looks: yellow from 100 ms, red from 500 ms
/// (an SSD answers in well under a millisecond, a busy hard disk in tens).
pub fn latency_severity(ms: f32) -> Option<Severity> {
    if ms >= 500.0 {
        Some(Severity::Critical)
    } else if ms >= 100.0 {
        Some(Severity::Warning)
    } else {
        None
    }
}

/// Round trips at least this long (milliseconds) are shown as slow.
pub const SLOW_LINK_MS: f32 = 500.0;

/// The round trip of a remote host's link, for its title ("12ms"), and
/// whether it is slow.
pub fn link_rtt_text(snapshot: &Snapshot) -> Option<(String, bool)> {
    let ms = snapshot.link_rtt_ms?;
    Some((format::latency(ms), ms >= SLOW_LINK_MS))
}

/// Hover text for [`link_rtt_text`].
pub const LINK_RTT_HINT: &str = "round trip to the agent over SSH (every 5 s)";

/// Label before the pressure readings ("Stall cpu 3% mem 0% io 6%").
pub const PRESSURE_LABEL: &str = "Stall";

/// One pressure reading for display: the share of the last 10 seconds in
/// which some task waited for the resource.
#[derive(Debug, Clone, PartialEq)]
pub struct PressureItem {
    /// "cpu", "mem" or "io".
    pub name: &'static str,
    /// e.g. "6%".
    pub value: String,
    /// Its alert threshold is reached.
    pub warn: bool,
}

/// The readings shown next to the load average: those of at least 1%, or
/// raising an alert under `limits` (marked). Empty while nothing waits.
pub fn pressure_items(pressure: &Pressure, limits: &AlertsConfig) -> Vec<PressureItem> {
    let over = |value: f32, limit: f64| limit > 0.0 && value as f64 >= limit;
    let item = |name, value: f32, warn| PressureItem {
        name,
        value: format!("{value:.0}%"),
        warn,
    };
    let items = [
        item("cpu", pressure.cpu.some_avg10, false),
        item(
            "mem",
            pressure.memory.some_avg10,
            over(
                pressure.memory.some_avg60,
                limits.memory_pressure_warning_percent,
            ),
        ),
        item(
            "io",
            pressure.io.some_avg10,
            over(pressure.io.full_avg60, limits.io_pressure_warning_percent),
        ),
    ];
    items
        .into_iter()
        .filter(|item| item.warn || item.value != "0%")
        .collect()
}

/// Hover text for the pressure readings: what they mean, with the "full"
/// figures and the minute averages.
pub fn pressure_details(pressure: &Pressure) -> String {
    let line = |name: &str, levels: &crate::snapshot::PressureLevels| {
        format!(
            "{name}: some {:.0}% / {:.0}%, full {:.0}% / {:.0}%",
            levels.some_avg10, levels.some_avg60, levels.full_avg10, levels.full_avg60
        )
    };
    [
        "Share of time tasks waited (10 s / 60 s averages); some: at least one \
         task, full: all of them"
            .to_owned(),
        line("cpu", &pressure.cpu),
        line("memory", &pressure.memory),
        line("io", &pressure.io),
    ]
    .join("\n")
}

/// [`alerts_with`] the default thresholds.
pub fn alerts(snapshot: &Snapshot) -> Vec<Alert> {
    alerts_with(snapshot, &AlertsConfig::default())
}

/// Conditions in `snapshot` that need attention, most severe first:
/// storage errors and failing drives, degraded RAID, full filesystems, OOM
/// kills, full memory, hot CPUs, GPUs and drives, judged by `limits` (0
/// turns a threshold off).
pub fn alerts_with(snapshot: &Snapshot, limits: &AlertsConfig) -> Vec<Alert> {
    // A threshold of 0 (or less) is turned off.
    let reached = |value: f64, limit: f64| limit > 0.0 && value >= limit;
    let mut alerts = Vec::new();
    let mut add = |severity, text: String| alerts.push(Alert { severity, text });
    for fs in &snapshot.raid.btrfs {
        let name = fs
            .mount_points
            .first()
            .map_or(fs.fsid.as_str(), String::as_str);
        let missing = fs.device_stats.iter().filter(|d| d.missing).count();
        if missing > 0 {
            add(
                Severity::Critical,
                format!("btrfs {name}: {missing} device(s) missing"),
            );
        }
        let errors: u64 = fs.device_stats.iter().map(|d| d.total_errors()).sum();
        if errors > 0 {
            // Which drives: the one to replace.
            let devices: Vec<String> = fs
                .device_stats
                .iter()
                .filter(|d| d.total_errors() > 0)
                .map(|d| d.label())
                .collect();
            add(
                Severity::Critical,
                format!("btrfs {name}: {errors} I/O errors ({})", devices.join(", ")),
            );
        }
        if let Some(scrub) = &fs.scrub {
            if scrub.uncorrectable > 0 {
                add(
                    Severity::Critical,
                    format!(
                        "btrfs {name}: scrub found {} uncorrectable errors",
                        scrub.uncorrectable
                    ),
                );
            } else if scrub.errors > 0 {
                add(
                    Severity::Warning,
                    format!("btrfs {name}: scrub corrected {} errors", scrub.corrected),
                );
            }
        }
    }
    for md in &snapshot.raid.md {
        if md.active && md.is_degraded() {
            add(
                Severity::Critical,
                format!("{} ({}) degraded", md.name, md.level),
            );
        } else if !md.active {
            add(Severity::Warning, format!("{} inactive", md.name));
        }
        if let Some(sync) = &md.sync {
            add(
                Severity::Warning,
                format!("{} {} {:.0}%", md.name, sync.action, sync.percent),
            );
        }
    }
    for pool in &snapshot.raid.zfs {
        if pool.is_failing() {
            add(
                Severity::Critical,
                format!("zfs {} {}", pool.name, pool.state),
            );
        } else if !pool.is_online() {
            add(
                Severity::Warning,
                format!("zfs {} {}", pool.name, pool.state),
            );
        }
    }
    for unit in &snapshot.failed_units {
        if !limits.ignored_units.contains(unit) {
            add(Severity::Warning, format!("systemd: {unit} failed"));
        }
    }
    for smart in &snapshot.smart {
        if let Some(error) = &smart.error {
            add(
                Severity::Warning,
                format!("{}: SMART unreadable ({error})", smart.device),
            );
        } else if smart.passed == Some(false) {
            add(
                Severity::Critical,
                format!("{}: SMART failed", smart.device),
            );
        } else if smart.needs_attention() {
            add(
                Severity::Critical,
                format!("{}: SMART warning", smart.device),
            );
        }
    }
    for fs in &snapshot.filesystems {
        let used = fs.total_bytes.saturating_sub(fs.available_bytes);
        let percent = used as f64 / fs.total_bytes.max(1) as f64 * 100.0;
        let name = fs
            .mount_points
            .first()
            .map_or(fs.device.as_str(), String::as_str);
        if reached(percent, limits.filesystem_critical_percent) {
            add(Severity::Critical, format!("{name} {percent:.0}% full"));
        } else if reached(percent, limits.filesystem_warning_percent) {
            add(Severity::Warning, format!("{name} {percent:.0}% full"));
        }
        let soon = limits.filesystem_full_warning_hours * 3600.0;
        if let Some(secs) = fs.full_in_secs
            && soon > 0.0
            && secs as f64 <= soon
        {
            add(
                Severity::Warning,
                format!("{name} full in ~{}", format::duration(secs)),
            );
        }
    }
    if let Some(text) = oom_alert(&snapshot.memory.oom_kills) {
        add(Severity::Critical, text);
    }
    let memory = &snapshot.memory;
    let percent = memory.used_bytes as f64 / memory.total_bytes.max(1) as f64 * 100.0;
    if memory.total_bytes > 0 && reached(percent, limits.memory_warning_percent) {
        add(Severity::Warning, format!("memory {percent:.0}% used"));
    }
    if let Some(pressure) = &snapshot.pressure {
        let memory = pressure.memory.some_avg60 as f64;
        if reached(memory, limits.memory_pressure_warning_percent) {
            add(Severity::Warning, format!("memory pressure {memory:.0}%"));
        }
        let io = pressure.io.full_avg60 as f64;
        if reached(io, limits.io_pressure_warning_percent) {
            add(Severity::Warning, format!("I/O pressure {io:.0}%"));
        }
    }
    let hottest_cpu = snapshot
        .cpu
        .temperatures
        .iter()
        .map(|t| t.celsius)
        .chain(snapshot.cpu.per_core_celsius.iter().flatten().copied())
        .fold(f32::NAN, f32::max);
    if reached(hottest_cpu as f64, limits.cpu_warning_celsius as f64) {
        add(Severity::Warning, format!("CPU at {hottest_cpu:.0}°C"));
    }
    for gpu in &snapshot.gpus {
        if let Some(t) = gpu
            .temperature_c
            .filter(|t| reached(*t as f64, limits.gpu_warning_celsius as f64))
        {
            add(Severity::Warning, format!("GPU {} at {t:.0}°C", gpu.index));
        }
        if let (Some(used), Some(total)) = (gpu.memory_used_bytes, gpu.memory_total_bytes)
            && total > 0
        {
            let percent = used as f64 / total as f64 * 100.0;
            if reached(percent, limits.gpu_memory_warning_percent) {
                add(
                    Severity::Warning,
                    format!("GPU {} memory {percent:.0}% used", gpu.index),
                );
            }
        }
        let slowdown: Vec<&str> = gpu.slowdown_reasons().collect();
        if !slowdown.is_empty() {
            add(
                Severity::Warning,
                format!("GPU {} slowed down ({})", gpu.index, slowdown.join(", ")),
            );
        }
        if let Some(ecc) = gpu.ecc.filter(|e| e.uncorrected > 0) {
            add(
                Severity::Critical,
                format!(
                    "GPU {}: {} uncorrected ECC errors",
                    gpu.index, ecc.uncorrected
                ),
            );
        }
    }
    for (device, celsius) in drive_temperatures(snapshot) {
        if reached(celsius as f64, limits.drive_warning_celsius as f64) {
            add(Severity::Warning, format!("{device} at {celsius:.0}°C"));
        }
    }
    // Stable sort keeps the order above within each severity.
    alerts.sort_by_key(|a| std::cmp::Reverse(a.severity));
    alerts
}

/// The alert for processes the OOM killer stopped: the likely ones by
/// name ("OOM killer stopped python3 (likely)"), so that another victim is
/// a new alert.
fn oom_alert(kills: &[OomKill]) -> Option<String> {
    let mut names: Vec<&str> = Vec::new();
    for name in kills
        .iter()
        .rev()
        .filter_map(|k| k.likely_process.as_deref())
    {
        if !names.contains(&name) {
            names.push(name);
        }
    }
    let likely = if names.is_empty() {
        String::new()
    } else {
        format!(" (likely {})", names.join(", "))
    };
    match kills.len() {
        0 => None,
        1 => Some(match names.first() {
            Some(name) => format!("OOM killer stopped {name} (likely)"),
            None => "OOM killer stopped a process".to_owned(),
        }),
        n => Some(format!("OOM killer stopped {n} processes{likely}")),
    }
}

/// (device, °C) of each drive: SMART results, then NVMe drives without
/// them from their hwmon sensor ("Composite", the drive's own summary).
fn drive_temperatures(snapshot: &Snapshot) -> Vec<(String, f32)> {
    let mut drives: Vec<(String, f32)> = snapshot
        .smart
        .iter()
        .filter_map(|s| Some((s.device.clone(), s.temperature_c?)))
        .collect();
    for chip in snapshot.sensors.iter().filter(|c| c.name == "nvme") {
        // The chip key is "nvme:<device>".
        let device = chip.key.split_once(':').map_or(chip.key.as_str(), |k| k.1);
        if drives.iter().any(|(d, _)| d == device) {
            continue;
        }
        let composite = chip
            .temperatures
            .iter()
            .find(|t| t.label == "Composite")
            .or(chip.temperatures.first());
        if let Some(t) = composite {
            drives.push((device.to_owned(), t.celsius));
        }
    }
    drives
}

/// pid -> (highest SM %, total VRAM) across the GPUs.
fn gpu_usage_by_pid(snapshot: &Snapshot) -> HashMap<u32, (Option<f32>, Option<u64>)> {
    let mut gpu_usage: HashMap<u32, (Option<f32>, Option<u64>)> = HashMap::new();
    for gpu in &snapshot.gpus {
        for p in &gpu.processes {
            let entry = gpu_usage.entry(p.pid).or_default();
            if let Some(sm) = p.sm_percent {
                entry.0 = Some(entry.0.map_or(sm, |v| v.max(sm)));
            }
            if let Some(mem) = p.memory_bytes {
                entry.1 = Some(entry.1.unwrap_or(0) + mem);
            }
        }
    }
    gpu_usage
}

/// What the processes of one container use together.
#[derive(Debug, Clone, PartialEq)]
pub struct ContainerUsage {
    pub name: String,
    pub processes: usize,
    pub cpu_percent: f32,
    pub memory_bytes: u64,
    /// Summed over its processes, where any uses a GPU.
    pub gpu_percent: Option<f32>,
    pub vram_bytes: Option<u64>,
}

/// The containers of `snapshot`'s processes with what they use, the
/// busiest (CPU, then memory) first. Empty when no process runs in one.
pub fn container_usage(snapshot: &Snapshot) -> Vec<ContainerUsage> {
    let gpu_usage = gpu_usage_by_pid(snapshot);
    let mut by_name: HashMap<&str, ContainerUsage> = HashMap::new();
    for process in &snapshot.processes {
        let Some(name) = process.container.as_deref() else {
            continue;
        };
        let usage = by_name.entry(name).or_insert_with(|| ContainerUsage {
            name: name.to_owned(),
            processes: 0,
            cpu_percent: 0.0,
            memory_bytes: 0,
            gpu_percent: None,
            vram_bytes: None,
        });
        usage.processes += 1;
        usage.cpu_percent += process.cpu_percent;
        usage.memory_bytes += process.memory_bytes;
        if let Some((gpu, vram)) = gpu_usage.get(&process.pid) {
            if let Some(gpu) = gpu {
                usage.gpu_percent = Some(usage.gpu_percent.unwrap_or(0.0) + gpu);
            }
            if let Some(vram) = vram {
                usage.vram_bytes = Some(usage.vram_bytes.unwrap_or(0) + vram);
            }
        }
    }
    let mut list: Vec<ContainerUsage> = by_name.into_values().collect();
    list.sort_by(|a, b| {
        b.cpu_percent
            .total_cmp(&a.cpu_percent)
            .then(b.memory_bytes.cmp(&a.memory_bytes))
            .then(a.name.cmp(&b.name))
    });
    list
}

/// How the process table is filtered and sorted.
#[derive(Debug, Clone, Copy)]
pub struct ProcessQuery<'q> {
    /// Column to sort by.
    pub sort_key: SortKey,
    /// Largest first when true.
    pub descending: bool,
    /// Case-insensitive substring of name, command or user, or an exact pid.
    pub filter: &'q str,
    /// Include kernel threads (`kworker` and friends).
    pub show_kernel_threads: bool,
    /// Children under their parents, siblings sorted by `sort_key`. A
    /// filter keeps the ancestors of what it matches.
    pub tree: bool,
}

/// Processes of `snapshot` after filtering and sorting, joined with GPU usage.
pub fn process_rows<'a>(snapshot: &'a Snapshot, query: &ProcessQuery<'_>) -> Vec<ProcessRow<'a>> {
    let gpu_usage = gpu_usage_by_pid(snapshot);

    let filter = query.filter.to_lowercase();
    let matches = |p: &ProcessInfo| {
        filter.is_empty()
            || p.name.to_lowercase().contains(&filter)
            || p.command.to_lowercase().contains(&filter)
            || p.user.to_lowercase().contains(&filter)
            || p.container
                .as_ref()
                .is_some_and(|c| c.to_lowercase().contains(&filter))
            || p.pid.to_string() == filter
    };
    let candidates = snapshot
        .processes
        .iter()
        .filter(|p| query.show_kernel_threads || !p.kernel_thread);
    let shown: Vec<&ProcessInfo> = if query.tree && !filter.is_empty() {
        with_ancestors(candidates.collect(), matches)
    } else {
        candidates.filter(|p| matches(p)).collect()
    };
    let mut rows: Vec<ProcessRow<'a>> = shown
        .into_iter()
        .map(|info| {
            let (gpu_percent, vram_bytes) = gpu_usage.get(&info.pid).copied().unwrap_or_default();
            ProcessRow {
                info,
                gpu_percent,
                vram_bytes,
                tree_prefix: String::new(),
            }
        })
        .collect();

    let key = query.sort_key;
    rows.sort_by(|a, b| {
        let ordering = match key {
            SortKey::Pid => a.info.pid.cmp(&b.info.pid),
            SortKey::Name => a.info.name.to_lowercase().cmp(&b.info.name.to_lowercase()),
            SortKey::Threads => a.info.threads.cmp(&b.info.threads),
            SortKey::User => a.info.user.cmp(&b.info.user),
            SortKey::Container => match (&a.info.container, &b.info.container) {
                (Some(a), Some(b)) => a.cmp(b),
                (Some(_), None) => std::cmp::Ordering::Less,
                (None, Some(_)) => std::cmp::Ordering::Greater,
                (None, None) => std::cmp::Ordering::Equal,
            },
            SortKey::Memory => a.info.memory_bytes.cmp(&b.info.memory_bytes),
            SortKey::Cpu => a.info.cpu_percent.total_cmp(&b.info.cpu_percent),
            SortKey::Disk => disk_rate(a.info).total_cmp(&disk_rate(b.info)),
            SortKey::Gpu => a
                .gpu_percent
                .unwrap_or(-1.0)
                .total_cmp(&b.gpu_percent.unwrap_or(-1.0)),
            SortKey::Vram => a.vram_bytes.cmp(&b.vram_bytes),
        }
        // Stable tie-breaker so rows do not jump between frames.
        .then(a.info.pid.cmp(&b.info.pid));
        if query.descending {
            ordering.reverse()
        } else {
            ordering
        }
    });
    if query.tree { into_tree(rows) } else { rows }
}

/// The processes `matches` accepts, plus their ancestors among `processes`
/// (so a filtered tree keeps its shape), in the order of `processes`.
fn with_ancestors(
    processes: Vec<&ProcessInfo>,
    matches: impl Fn(&ProcessInfo) -> bool,
) -> Vec<&ProcessInfo> {
    let by_pid: HashMap<u32, &ProcessInfo> = processes.iter().map(|p| (p.pid, *p)).collect();
    let mut keep: HashSet<u32> = HashSet::new();
    for process in processes.iter().filter(|p| matches(p)) {
        let mut current = Some(*process);
        // Stops at a process already kept, which also ends parent loops.
        while let Some(p) = current
            && keep.insert(p.pid)
        {
            current = p.parent_pid.and_then(|pid| by_pid.get(&pid).copied());
        }
    }
    processes
        .into_iter()
        .filter(|p| keep.contains(&p.pid))
        .collect()
}

/// Reorders sorted `rows` depth-first under their parents, keeping the
/// sorted order among siblings, and fills in [`ProcessRow::tree_prefix`].
/// Processes whose parent is not among the rows are roots.
fn into_tree(rows: Vec<ProcessRow<'_>>) -> Vec<ProcessRow<'_>> {
    let index: HashMap<u32, usize> = rows
        .iter()
        .enumerate()
        .map(|(i, row)| (row.info.pid, i))
        .collect();
    let mut children: Vec<Vec<usize>> = vec![Vec::new(); rows.len()];
    let mut roots = Vec::new();
    for (i, row) in rows.iter().enumerate() {
        match row.info.parent_pid.and_then(|pid| index.get(&pid)) {
            Some(&parent) if parent != i => children[parent].push(i),
            _ => roots.push(i),
        }
    }
    let mut order: Vec<(usize, String)> = Vec::with_capacity(rows.len());
    let mut visited = vec![false; rows.len()];
    // Processes in a parent loop have no root; they start trees of their own.
    let starts: Vec<usize> = roots.into_iter().chain(0..rows.len()).collect();
    for start in starts {
        // (row, lines of the ancestors, whether it is its parent's last child)
        let mut stack: Vec<(usize, String, Option<bool>)> = vec![(start, String::new(), None)];
        while let Some((i, lead, last)) = stack.pop() {
            if std::mem::replace(&mut visited[i], true) {
                continue;
            }
            let (prefix, child_lead) = match last {
                None => (String::new(), String::new()),
                Some(true) => (format!("{lead}└ "), format!("{lead}  ")),
                Some(false) => (format!("{lead}├ "), format!("{lead}│ ")),
            };
            order.push((i, prefix));
            let kids = &children[i];
            for (k, &child) in kids.iter().enumerate().rev() {
                stack.push((child, child_lead.clone(), Some(k + 1 == kids.len())));
            }
        }
    }
    let mut rows: Vec<Option<ProcessRow<'_>>> = rows.into_iter().map(Some).collect();
    order
        .into_iter()
        .filter_map(|(i, prefix)| {
            let mut row = rows[i].take()?;
            row.tree_prefix = prefix;
            Some(row)
        })
        .collect()
}

/// Disk reads plus writes of a process (bytes/s), the Disk column.
pub fn disk_rate(process: &ProcessInfo) -> f64 {
    process.read_bytes_per_sec + process.write_bytes_per_sec
}

/// The Disk column's text: the total rate, blank when idle (also where the
/// counters cannot be read, e.g. other users' processes on Linux).
pub fn disk_rate_text(process: &ProcessInfo) -> String {
    let rate = disk_rate(process);
    if rate > 0.0 {
        format::rate(rate)
    } else {
        String::new()
    }
}

/// Shown where the CPU temperatures go when LibreHardwareMonitor cannot be
/// read (Windows; see [`Snapshot::lhm_error`]).
pub const LHM_HINT: &str = "temperatures need LibreHardwareMonitor";

/// Explanation behind [`LHM_HINT`], given the reason it could not be read.
pub fn lhm_help(error: &str) -> String {
    format!(
        "Windows needs LibreHardwareMonitor for temperatures, fans and CPU power.\n\
         Start it and turn on Options → Remote Web Server → Run.\n\
         ([lhm] url = \"\" in the configuration hides this note.)\n\n\
         {error}"
    )
}

/// Name of a signal as shown to the user ("renice" for a priority; see
/// [`action_text`] for the value).
pub fn signal_name(signal: ProcessSignal) -> &'static str {
    match signal {
        ProcessSignal::Terminate => "SIGTERM",
        ProcessSignal::Kill => "SIGKILL",
        ProcessSignal::Stop => "SIGSTOP",
        ProcessSignal::Continue => "SIGCONT",
        ProcessSignal::Nice(_) => "renice",
    }
}

/// What was done, for status lines: "SIGSTOP", "nice 10".
pub fn action_text(signal: ProcessSignal) -> String {
    match signal {
        ProcessSignal::Nice(nice) => format!("nice {nice}"),
        other => signal_name(other).to_owned(),
    }
}

/// Steps of the priority keys and menu items.
pub const NICE_STEP: i32 = 5;

/// The nice value one step lower (`later`) or higher in priority than
/// `process`'s (0 when unknown), within -20..=19; `None` at the end.
pub fn nice_step(process: &ProcessInfo, later: bool) -> Option<i32> {
    let current = process.nice.unwrap_or(0);
    let next = if later {
        (current + NICE_STEP).min(19)
    } else {
        (current - NICE_STEP).max(-20)
    };
    (next != current).then_some(next)
}

/// Whether `process` is paused (SIGSTOP, or stopped by a debugger).
pub fn is_stopped(process: &ProcessInfo) -> bool {
    process.state == "Stopped" || process.state == "Tracing"
}

/// Returns true for container bridges, veth pairs and loopback, which are
/// skipped when choosing and listing interfaces.
pub fn is_virtual_interface(name: &str) -> bool {
    VIRTUAL_IFACE_PREFIXES.iter().any(|p| name.starts_with(p))
}

/// Picks the physical interface with the most traffic so far.
pub fn default_interface(snapshot: &Snapshot) -> Option<String> {
    snapshot
        .networks
        .iter()
        .filter(|n| !is_virtual_interface(&n.name))
        .max_by_key(|n| n.rx_total_bytes + n.tx_total_bytes)
        .or_else(|| snapshot.networks.first())
        .map(|n| n.name.clone())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::snapshot::{EccErrors, GpuProcess, NetworkStats};

    #[test]
    fn process_details_rows() {
        let mut snapshot = Snapshot::default();
        let mut shell = process(20, "zsh", 0.0, false);
        shell.parent_pid = Some(1);
        let mut python = process(30, "python", 99.5, false);
        python.parent_pid = Some(20);
        python.command = "python train.py --epochs 10".to_owned();
        python.container = Some("trainer".to_owned());
        python.run_time_secs = 3_660;
        snapshot.processes = vec![process(1, "systemd", 0.0, false), shell, python];
        snapshot.gpus = vec![GpuStats {
            index: 0,
            processes: vec![GpuProcess {
                pid: 30,
                memory_bytes: Some(2 << 30),
                sm_percent: Some(97.0),
            }],
            ..Default::default()
        }];
        let now = SystemTime::now();
        let rows = process_details(&snapshot, 30, now).unwrap();
        let get = |label: &str| {
            rows.iter()
                .find(|(l, _)| *l == label)
                .map(|(_, v)| v.as_str())
        };
        assert_eq!(get("Program"), Some("python (pid 30)"));
        assert_eq!(get("Parents"), Some("systemd (1) › zsh (20)"));
        assert_eq!(get("Children"), None);
        assert_eq!(get("Container"), Some("trainer"));
        assert!(get("Running").unwrap().starts_with("1h01m (since "));
        assert_eq!(get("GPU"), Some("GPU 0 97%, 2.0G VRAM"));
        assert_eq!(get("Command"), Some("python train.py --epochs 10"));
        let shell = process_details(&snapshot, 20, now).unwrap();
        assert!(shell.contains(&("Children", "1".to_owned())));
        assert!(process_details(&snapshot, 99, now).is_none());
    }

    #[test]
    fn attention_ranks() {
        let alert = |severity| Alert {
            severity,
            text: String::new(),
        };
        assert_eq!(attention_rank(&[], false, false), 2);
        assert_eq!(attention_rank(&[alert(Severity::Warning)], false, false), 1);
        assert_eq!(attention_rank(&[], false, true), 1);
        assert_eq!(
            attention_rank(
                &[alert(Severity::Warning), alert(Severity::Critical)],
                false,
                false
            ),
            0
        );
        assert_eq!(attention_rank(&[], true, false), 0);
    }

    #[test]
    fn gpu_notes_and_pcie() {
        let mut gpu = GpuStats {
            throttle_reasons: vec!["power cap".to_owned()],
            ..Default::default()
        };
        assert_eq!(gpu_notes(&gpu), [("power cap".to_owned(), None)]);
        gpu.throttle_reasons.push("thermal".to_owned());
        gpu.ecc = Some(EccErrors {
            corrected: 3,
            uncorrected: 0,
        });
        assert_eq!(
            gpu_notes(&gpu),
            [
                ("slowed: thermal".to_owned(), Some(Severity::Warning)),
                ("ECC 3 corrected".to_owned(), None)
            ]
        );
        let full = PcieLink {
            generation: 4,
            width: 16,
            max_generation: 4,
            max_width: 16,
        };
        assert_eq!(pcie_text(&full), "PCIe gen 4 x16");
        let idle = PcieLink {
            generation: 1,
            width: 8,
            ..full
        };
        assert!(pcie_text(&idle).starts_with("PCIe gen 1 x8 (up to gen 4 x16;"));
    }

    fn process(pid: u32, name: &str, cpu: f32, kernel: bool) -> ProcessInfo {
        ProcessInfo {
            pid,
            name: name.to_owned(),
            cpu_percent: cpu,
            kernel_thread: kernel,
            ..Default::default()
        }
    }

    fn query(sort_key: SortKey) -> ProcessQuery<'static> {
        ProcessQuery {
            sort_key,
            descending: sort_key.descending_by_default(),
            filter: "",
            show_kernel_threads: false,
            tree: false,
        }
    }

    #[test]
    fn history_keeps_newest_samples() {
        let mut history = History::default();
        for i in 0..(HISTORY_LEN + 5) {
            history.push(i as f64);
        }
        assert_eq!(
            history.last(2),
            vec![(HISTORY_LEN + 3) as f64, (HISTORY_LEN + 4) as f64]
        );
        assert_eq!(history.last(HISTORY_LEN * 2).len(), HISTORY_LEN);
        assert_eq!(history.max_of_last(3), (HISTORY_LEN + 4) as f64);
    }

    #[test]
    fn graph_spans_spread_samples_over_the_width() {
        let second = Duration::from_secs(1);
        let mut history = History::default();
        for i in 0..120 {
            history.push(i as f64);
        }
        // Fit: one sample per column, the newest ones.
        assert_eq!(
            history.graph(3, GraphSpan::Fit, second),
            vec![117.0, 118.0, 119.0]
        );
        // A minute on 30 columns: two samples averaged per column.
        let minute = history.graph(30, GraphSpan::Secs(60), second);
        assert_eq!(minute.len(), 30);
        assert_eq!(minute[0], 60.5);
        assert_eq!(minute[29], 118.5);
        // Fewer samples than columns: interpolated between them.
        let stretched = history.graph(120, GraphSpan::Secs(60), Duration::from_secs(2));
        assert_eq!(stretched.len(), 120);
        assert_eq!(&stretched[..2], &[90.0, 90.0]);
        assert_eq!(&stretched[2..4], &[90.125, 90.375]);
        assert_eq!(stretched[119], 119.0);
        assert!(stretched.windows(2).all(|w| w[0] <= w[1]));
        // An hour with two minutes recorded fills the right 1/30.
        let hour = history.graph(300, GraphSpan::Secs(3600), second);
        assert_eq!(hour.len(), 10);
        assert_eq!(hour[0], 5.5);
        assert!(
            History::default()
                .graph(10, GraphSpan::Secs(60), second)
                .is_empty()
        );
    }

    #[test]
    fn long_spans_draw_from_the_averages() {
        let second = Duration::from_secs(1);
        let mut history = History::default();
        // Two hours: the first at 0, the second at 100.
        for i in 0..7200 {
            history.push(if i < 3600 { 0.0 } else { 100.0 });
        }
        // 6 hours with 2 recorded: the right third, half low and half high.
        let six = history.graph(90, GraphSpan::Secs(6 * 3600), second);
        assert_eq!(six.len(), 30);
        assert_eq!((six[0], six[29]), (0.0, 100.0));
        // An hour still comes from the samples themselves.
        let hour = history.graph(60, GraphSpan::Secs(3600), second);
        assert!(hour.iter().all(|v| *v == 100.0));
        // The averages stay bounded.
        for i in 0..(COARSE_LEN * COARSE_FACTOR + 500) {
            history.push(i as f64);
        }
        assert_eq!(history.coarse.len(), COARSE_LEN);
    }

    #[test]
    fn graph_span_choices_cycle() {
        let labels: Vec<String> = std::iter::successors(Some(GraphSpan::Fit), |s| Some(s.next()))
            .take(6)
            .map(GraphSpan::label)
            .collect();
        assert_eq!(labels, ["fit", "1m", "5m", "15m", "1h", "6h"]);
        assert_eq!(GraphSpan::Secs(86_400).label(), "24h");
        assert_eq!(GraphSpan::Secs(86_400).next(), GraphSpan::Fit);
        assert_eq!(
            GraphSpan::from_secs(GraphSpan::Secs(300).as_secs()),
            GraphSpan::Secs(300)
        );
        assert_eq!(GraphSpan::from_secs(0), GraphSpan::Fit);
        // A stored span that is not a choice starts over.
        assert_eq!(GraphSpan::Secs(7).next(), GraphSpan::Fit);
    }

    #[test]
    fn rows_filter_sort_and_join_gpu_usage() {
        let snapshot = Snapshot {
            processes: vec![
                process(10, "bash", 5.0, false),
                process(2, "kworker/0:1", 50.0, true),
                process(30, "python", 1.0, false),
            ],
            gpus: vec![GpuStats {
                processes: vec![GpuProcess {
                    pid: 30,
                    memory_bytes: Some(1 << 30),
                    sm_percent: Some(70.0),
                }],
                ..Default::default()
            }],
            ..Default::default()
        };
        let pids = |q: &ProcessQuery| {
            process_rows(&snapshot, q)
                .iter()
                .map(|r| r.info.pid)
                .collect::<Vec<_>>()
        };
        assert_eq!(pids(&query(SortKey::Cpu)), vec![10, 30]);
        assert_eq!(pids(&query(SortKey::Vram)), vec![30, 10]);
        assert_eq!(
            pids(&ProcessQuery {
                show_kernel_threads: true,
                ..query(SortKey::Cpu)
            }),
            vec![2, 10, 30]
        );
        assert_eq!(
            pids(&ProcessQuery {
                filter: "PYT",
                ..query(SortKey::Pid)
            }),
            vec![30]
        );
        let rows = process_rows(&snapshot, &query(SortKey::Pid));
        assert_eq!(rows[1].vram_bytes, Some(1 << 30));
        assert_eq!(rows[1].gpu_percent, Some(70.0));
    }

    #[test]
    fn since_notes_say_how_long_and_since_when() {
        let now = SystemTime::now();
        let ago = |secs: u64| now - Duration::from_secs(secs);
        assert!(since_note(ago(42), now).starts_with("42s (since "));
        assert!(since_note(ago(5 * 60), now).starts_with("5m (since "));
        assert!(since_note(ago(3 * 3600 + 7 * 60), now).starts_with("3h07m (since "));
        assert!(since_note(ago(2 * 86_400 + 3 * 3600), now).starts_with("2d03h (since "));
        // A clock that moved backwards does not underflow.
        assert!(since_note(now + Duration::from_secs(5), now).starts_with("0s"));
    }

    #[test]
    fn tree_puts_children_under_parents() {
        let child = |pid: u32, parent: u32, name: &str, cpu: f32| ProcessInfo {
            parent_pid: Some(parent),
            ..process(pid, name, cpu, false)
        };
        let snapshot = Snapshot {
            processes: vec![
                process(1, "init", 0.0, false),
                child(10, 1, "sshd", 1.0),
                child(11, 10, "bash", 2.0),
                child(12, 11, "python", 50.0),
                child(20, 1, "cron", 3.0),
                // Parent not listed (e.g. not readable): a root of its own.
                child(30, 999, "orphan", 4.0),
                // A parent loop must not hang or drop the processes.
                child(40, 41, "loop-a", 0.0),
                child(41, 40, "loop-b", 0.0),
            ],
            ..Default::default()
        };
        let tree = |filter| {
            process_rows(
                &snapshot,
                &ProcessQuery {
                    filter,
                    tree: true,
                    ..query(SortKey::Cpu)
                },
            )
            .iter()
            .map(|r| format!("{}{}", r.tree_prefix, r.info.name))
            .collect::<Vec<_>>()
        };
        // Roots and siblings by CPU, largest first (ties: higher pid first).
        assert_eq!(
            tree(""),
            vec![
                "orphan",
                "init",
                "├ cron",
                "└ sshd",
                "  └ bash",
                "    └ python",
                "loop-b",
                "└ loop-a",
            ]
        );
        // A filter keeps the ancestors of what it matches.
        assert_eq!(
            tree("pyth"),
            vec!["init", "└ sshd", "  └ bash", "    └ python"]
        );
    }

    #[test]
    fn disk_column_sums_reads_and_writes() {
        let reader = ProcessInfo {
            read_bytes_per_sec: 3.0 * 1024.0 * 1024.0,
            ..process(10, "reader", 0.0, false)
        };
        let writer = ProcessInfo {
            read_bytes_per_sec: 1024.0 * 1024.0,
            write_bytes_per_sec: 4.0 * 1024.0 * 1024.0,
            ..process(20, "writer", 0.0, false)
        };
        let idle = process(30, "idle", 0.0, false);
        assert_eq!(disk_rate_text(&writer), "5.0M/s");
        assert_eq!(disk_rate_text(&idle), "");
        let snapshot = Snapshot {
            processes: vec![idle, reader, writer],
            ..Default::default()
        };
        let pids: Vec<u32> = process_rows(&snapshot, &query(SortKey::Disk))
            .iter()
            .map(|r| r.info.pid)
            .collect();
        assert_eq!(pids, vec![20, 10, 30]);
    }

    #[test]
    fn metric_history_records_and_switches_interfaces() {
        let snapshot = Snapshot {
            networks: vec![
                NetworkStats {
                    name: "enp9s0".to_owned(),
                    rx_total_bytes: 10,
                    rx_bytes_per_sec: 100.0,
                    ..Default::default()
                },
                NetworkStats {
                    name: "wlp7s0".to_owned(),
                    tx_bytes_per_sec: 5.0,
                    ..Default::default()
                },
            ],
            gpus: vec![GpuStats {
                utilization_percent: Some(40.0),
                ..Default::default()
            }],
            ..Default::default()
        };
        let mut history = MetricHistory::default();
        history.record(&snapshot);
        assert_eq!(history.net_interface.as_deref(), Some("enp9s0"));
        assert_eq!(history.net_rx.last(1), vec![100.0]);
        assert_eq!(history.gpus[0].last(1), vec![40.0]);
        assert_eq!(history.next_interface(&snapshot).as_deref(), Some("wlp7s0"));
        // Switching clears the samples of the previous interface.
        assert!(history.net_rx.last(10).is_empty());
        history.record(&snapshot);
        assert_eq!(history.net_tx.last(1), vec![5.0]);
    }

    #[test]
    fn alerts_flag_storage_problems_first() {
        use crate::snapshot::{
            BtrfsDeviceStats, BtrfsFilesystem, FilesystemUsage, MdArray, RaidStats, SmartDevice,
            ZfsPool,
        };
        let snapshot = Snapshot {
            filesystems: vec![FilesystemUsage {
                mount_points: vec!["/data".to_owned()],
                total_bytes: 100,
                available_bytes: 8,
                ..Default::default()
            }],
            raid: RaidStats {
                btrfs: vec![BtrfsFilesystem {
                    mount_points: vec!["/".to_owned()],
                    device_stats: vec![BtrfsDeviceStats {
                        devid: 1,
                        corruption_errs: 2,
                        ..Default::default()
                    }],
                    ..Default::default()
                }],
                md: vec![MdArray {
                    name: "md0".to_owned(),
                    active: true,
                    level: "raid1".to_owned(),
                    expected_devices: Some(2),
                    working_devices: Some(1),
                    ..Default::default()
                }],
                zfs: vec![
                    ZfsPool {
                        name: "tank".to_owned(),
                        state: "DEGRADED".to_owned(),
                    },
                    ZfsPool {
                        name: "cold".to_owned(),
                        state: "OFFLINE".to_owned(),
                    },
                    ZfsPool {
                        name: "fast".to_owned(),
                        state: "ONLINE".to_owned(),
                    },
                ],
            },
            smart: vec![SmartDevice {
                device: "sda".to_owned(),
                passed: Some(true),
                ..Default::default()
            }],
            ..Default::default()
        };
        let alerts = alerts(&snapshot);
        let texts: Vec<&str> = alerts.iter().map(|a| a.text.as_str()).collect();
        assert_eq!(
            texts,
            vec![
                "btrfs /: 2 I/O errors (devid 1)",
                "md0 (raid1) degraded",
                "zfs tank DEGRADED",
                "zfs cold OFFLINE",
                "/data 92% full"
            ]
        );
        assert_eq!(alerts[0].severity, Severity::Critical);
        assert_eq!(alerts[2].severity, Severity::Critical);
        assert_eq!(alerts[3].severity, Severity::Warning);
        assert!(super::alerts(&Snapshot::default()).is_empty());
    }

    #[test]
    fn event_times_are_as_short_as_is_clear() {
        let now = SystemTime::now();
        let ms = |t: SystemTime| {
            t.duration_since(SystemTime::UNIX_EPOCH)
                .unwrap()
                .as_millis() as u64
        };
        // "14:03:12" today; a weekday this week; a date before.
        assert_eq!(event_time(ms(now), now).len(), 8);
        let week_ago = now - Duration::from_secs(10 * 86_400);
        assert!(event_time(ms(week_ago), now).contains(' '));
        assert_ne!(event_time(ms(week_ago), now).len(), 8);
    }

    #[test]
    fn snoozing_a_raised_alert_logs_no_clearing() {
        let disk = Alert {
            severity: Severity::Warning,
            text: "sdc: SMART warning".to_owned(),
        };
        let mut log = AlertLog::default();
        assert_eq!(log.update("nas", std::slice::from_ref(&disk), 1).len(), 1);
        // Snoozed: it leaves quietly.
        assert!(
            log.update_snoozed("nas", &[], std::slice::from_ref(&disk), 2)
                .is_empty()
        );
        assert!(
            log.update_snoozed("nas", &[], std::slice::from_ref(&disk), 3)
                .is_empty()
        );
        // The snooze ended with the alert still there: raised again.
        let events = log.update_snoozed("nas", std::slice::from_ref(&disk), &[], 4);
        assert_eq!(events.len(), 1);
        assert_eq!(events[0].change, AlertChange::Raised);
        // A real clearing is still logged.
        assert_eq!(
            log.update_snoozed("nas", &[], &[], 5)[0].change,
            AlertChange::Cleared
        );
    }

    #[test]
    fn alert_history_survives_a_restart() {
        let tmp = tempfile::tempdir().unwrap();
        let path = tmp.path().join("state/alerts-tui.jsonl");
        let degraded = Alert {
            severity: Severity::Critical,
            text: "md0 (raid1) degraded".to_owned(),
        };
        let full = Alert {
            severity: Severity::Warning,
            text: "/srv 92% full".to_owned(),
        };
        let mut log = AlertLog::persistent(path.clone());
        log.update("nas", &[degraded.clone(), full.clone()], 1_000);
        log.update("nas", std::slice::from_ref(&degraded), 2_000);
        assert_eq!(log.events().count(), 3);

        // The next run sees the same events, and the alert still present
        // is not logged again.
        let mut log = AlertLog::persistent(path.clone());
        assert_eq!(log.events().count(), 3);
        assert_eq!(log.unread(), (0, None));
        assert!(
            log.update("nas", std::slice::from_ref(&degraded), 3_000)
                .is_empty()
        );
        assert_eq!(log.events().count(), 3);
        let changes = log.update("nas", &[], 4_000);
        assert_eq!(changes.len(), 1);
        assert_eq!(changes[0].alert, degraded);
        let last = log.events().last().unwrap();
        assert_eq!((last.change, last.at_ms), (AlertChange::Cleared, 4_000));
        assert_eq!(AlertLog::persistent(path.clone()).events().count(), 4);

        // A one-off notice is unread, kept, and not taken for a lasting
        // alert by the next run (which would log it cleared).
        let exited = Alert {
            severity: Severity::Warning,
            text: "train.py (pid 7) exited after 3h".to_owned(),
        };
        let mut log = AlertLog::persistent(path.clone());
        log.record("nas", exited.clone(), 5_000);
        assert_eq!(log.unread(), (1, Some(Severity::Warning)));
        let mut log = AlertLog::persistent(path);
        assert!(log.update("nas", &[], 6_000).is_empty());
        let last = log.events().last().unwrap();
        assert_eq!((last.change, &last.alert), (AlertChange::Notice, &exited));
    }

    #[test]
    fn cards_skip_gpus_without_readings() {
        let snapshot = Snapshot {
            gpus: vec![
                GpuStats {
                    index: 0,
                    name: "Intel UHD Graphics".to_owned(),
                    temperature_c: Some(50.0),
                    ..Default::default()
                },
                GpuStats {
                    index: 1,
                    utilization_percent: Some(3.0),
                    ..Default::default()
                },
            ],
            ..Default::default()
        };
        let shown: Vec<u32> = card_gpus(&snapshot).map(|g| g.index).collect();
        assert_eq!(shown, [1]);
    }

    #[test]
    fn containers_add_up_their_processes() {
        use crate::snapshot::GpuProcess;
        let process = |pid, container: Option<&str>, cpu, mem| ProcessInfo {
            pid,
            container: container.map(str::to_owned),
            cpu_percent: cpu,
            memory_bytes: mem,
            ..Default::default()
        };
        let snapshot = Snapshot {
            processes: vec![
                process(1, None, 50.0, 100),
                process(2, Some("db"), 3.0, 500),
                process(3, Some("web"), 10.0, 200),
                process(4, Some("web"), 5.0, 300),
            ],
            gpus: vec![GpuStats {
                processes: vec![GpuProcess {
                    pid: 3,
                    sm_percent: Some(40.0),
                    memory_bytes: Some(1 << 30),
                }],
                ..Default::default()
            }],
            ..Default::default()
        };
        let usage = container_usage(&snapshot);
        assert_eq!(usage.len(), 2, "processes outside containers are left out");
        assert_eq!(usage[0].name, "web");
        assert_eq!(usage[0].processes, 2);
        assert_eq!(usage[0].cpu_percent, 15.0);
        assert_eq!(usage[0].memory_bytes, 500);
        assert_eq!(usage[0].gpu_percent, Some(40.0));
        assert_eq!(usage[0].vram_bytes, Some(1 << 30));
        assert_eq!(usage[1].gpu_percent, None);
        assert!(container_usage(&Snapshot::default()).is_empty());
    }

    #[test]
    fn processes_match_by_name_or_file() {
        let process = ProcessInfo {
            name: "python3".to_owned(),
            command: "/usr/bin/python3 /home/me/jobs/train.py --epochs 3".to_owned(),
            ..Default::default()
        };
        assert!(process_matches(&process, "python3"));
        assert!(process_matches(&process, "train.py"));
        assert!(process_matches(&process, " train.py "));
        assert!(process_matches(&process, "/home/me/jobs/train.py"));
        assert!(!process_matches(&process, "train"));
        assert!(!process_matches(&process, "epochs"));
        assert!(!process_matches(&process, ""));
    }

    #[test]
    fn exit_watch_reports_exits_it_can_tell() {
        let process = |pid: u32, name: &str| ProcessInfo {
            pid,
            name: name.to_owned(),
            run_time_secs: 3_600,
            ..Default::default()
        };
        let mut watch = ExitWatch::default();
        assert!(watch.toggle(&process(7, "train.py"), 10_000_000));
        assert!(watch.toggle(&process(8, "make"), 10_000_000));
        assert!(!watch.toggle(&process(8, "make"), 10_000_000));
        assert_eq!(watch.pids(), [7]);

        // An agent that does not watch (before 0.19): gone from a limited
        // list says nothing.
        let mut snapshot = Snapshot::default();
        assert_eq!(watch.state(&snapshot, 7), WatchState::Unknown);
        assert!(watch.check(&snapshot, 10_000_000).is_empty());
        // Listed but not reported: its exit would go unnoticed.
        snapshot.processes = vec![process(7, "train.py")];
        assert_eq!(watch.state(&snapshot, 7), WatchState::Unknown);

        snapshot.watched_pids = vec![7];
        snapshot.processes = vec![process(7, "train.py")];
        assert_eq!(watch.state(&snapshot, 7), WatchState::Running);
        assert!(watch.check(&snapshot, 10_000_000).is_empty());

        // Gone while watched: exited, after its run time plus the wait.
        snapshot.processes.clear();
        assert_eq!(watch.state(&snapshot, 7), WatchState::Exited);
        let alerts = watch.check(&snapshot, 10_000_000 + 1_800_000);
        assert_eq!(alerts.len(), 1);
        assert_eq!(alerts[0].text, "train.py (pid 7) exited after 1h30m");
        assert_eq!(watch.pids(), Vec::<u32>::new(), "reported once");

        // The pid taken by another program also means it exited.
        watch.toggle(&process(9, "rsync"), 0);
        snapshot.processes = vec![process(9, "bash")];
        assert_eq!(watch.check(&snapshot, 0).len(), 1);

        // Watched under another name (the script, not the interpreter).
        let python = process(12, "python3");
        assert!(watch.watch_as(&python, Some("train.py"), 0));
        assert!(!watch.watch_as(&python, Some("train.py"), 0), "once");
        snapshot.processes.clear();
        snapshot.watched_pids = vec![12];
        assert_eq!(
            watch.check(&snapshot, 60_000)[0].text,
            "train.py (pid 12) exited after 01:00"
        );
    }

    #[test]
    fn failed_units_warn_unless_ignored() {
        let snapshot = Snapshot {
            failed_units: vec!["nginx.service".to_owned(), "wait-online.service".to_owned()],
            ..Default::default()
        };
        let limits = AlertsConfig {
            ignored_units: vec!["wait-online.service".to_owned()],
            ..Default::default()
        };
        let alerts = alerts_with(&snapshot, &limits);
        assert_eq!(alerts.len(), 1);
        assert_eq!(alerts[0].text, "systemd: nginx.service failed");
        assert_eq!(alerts[0].severity, Severity::Warning);
    }

    #[test]
    fn version_notes_only_for_other_versions() {
        let mut snapshot = Snapshot::default();
        assert_eq!(version_note(&snapshot), None);
        snapshot.host.rigtop_version = env!("CARGO_PKG_VERSION").to_owned();
        assert_eq!(version_note(&snapshot), None);
        snapshot.host.rigtop_version = "0.1.0".to_owned();
        assert_eq!(version_note(&snapshot).as_deref(), Some("agent 0.1.0"));
        assert!(version_hint(&snapshot).contains("rigtop 0.1.0"));
    }

    #[test]
    fn alert_log_records_alerts_appearing_and_clearing() {
        let alert = |severity, text: &str| Alert {
            severity,
            text: text.to_owned(),
        };
        let full = alert(Severity::Warning, "/data 91% full");
        let errors = alert(Severity::Critical, "btrfs /: 2 I/O errors (nvme1n1)");
        let mut log = AlertLog::default();
        log.update("nas", std::slice::from_ref(&full), 1_000);
        // Numbers changing is the same alert.
        log.update(
            "nas",
            &[alert(Severity::Warning, "/data 92% full"), errors.clone()],
            2_000,
        );
        log.update("zefat", &[], 2_000);
        assert_eq!(log.unread(), (2, Some(Severity::Critical)));
        log.mark_read();
        log.update("nas", std::slice::from_ref(&errors), 3_000);
        assert_eq!(log.unread(), (0, None));
        let summary: Vec<(u64, AlertChange, &str)> = log
            .events()
            .map(|e| (e.at_ms, e.change, e.alert.text.as_str()))
            .collect();
        assert_eq!(
            summary,
            [
                (1_000, AlertChange::Raised, "/data 91% full"),
                (
                    2_000,
                    AlertChange::Raised,
                    "btrfs /: 2 I/O errors (nvme1n1)"
                ),
                (3_000, AlertChange::Cleared, "/data 92% full"),
            ]
        );
        for i in 0..(ALERT_LOG_LEN as u64 + 10) {
            log.update(
                "flappy",
                &[alert(Severity::Warning, if i % 2 == 0 { "a" } else { "b" })],
                i,
            );
        }
        assert_eq!(log.events().count(), ALERT_LOG_LEN);
    }

    #[test]
    fn filesystems_filling_up_warn() {
        use crate::snapshot::FilesystemUsage;
        let filling = |full_in_secs| Snapshot {
            filesystems: vec![FilesystemUsage {
                mount_points: vec!["/srv".to_owned()],
                total_bytes: 100,
                available_bytes: 50,
                full_in_secs,
                ..Default::default()
            }],
            ..Default::default()
        };
        let texts = |snapshot: &Snapshot, limits: &AlertsConfig| -> Vec<String> {
            alerts_with(snapshot, limits)
                .into_iter()
                .map(|a| a.text)
                .collect()
        };
        let defaults = AlertsConfig::default();
        assert_eq!(
            texts(&filling(Some(5 * 3600)), &defaults),
            ["/srv full in ~5h00m"]
        );
        assert!(texts(&filling(Some(30 * 3600)), &defaults).is_empty());
        assert!(texts(&filling(None), &defaults).is_empty());
        let off = AlertsConfig {
            filesystem_full_warning_hours: 0.0,
            ..Default::default()
        };
        assert!(texts(&filling(Some(60)), &off).is_empty());
    }

    #[test]
    fn alert_thresholds_come_from_the_configuration() {
        use crate::snapshot::{FilesystemUsage, GpuStats, SensorChip, SmartDevice, Temperature};
        let temp = |label: &str, celsius| Temperature {
            label: label.to_owned(),
            celsius,
            ..Default::default()
        };
        let mut snapshot = Snapshot {
            filesystems: vec![FilesystemUsage {
                mount_points: vec!["/data".to_owned()],
                total_bytes: 100,
                available_bytes: 8,
                ..Default::default()
            }],
            gpus: vec![GpuStats {
                index: 0,
                temperature_c: Some(80.0),
                ..Default::default()
            }],
            smart: vec![SmartDevice {
                device: "nvme0".to_owned(),
                temperature_c: Some(72.0),
                ..Default::default()
            }],
            // nvme0 also has SMART (reported once); nvme1 only its sensor.
            sensors: vec![
                SensorChip {
                    key: "nvme:nvme0".to_owned(),
                    name: "nvme".to_owned(),
                    temperatures: vec![temp("Composite", 71.0)],
                    ..Default::default()
                },
                SensorChip {
                    key: "nvme:nvme1".to_owned(),
                    name: "nvme".to_owned(),
                    temperatures: vec![temp("Sensor 1", 90.0), temp("Composite", 74.0)],
                    ..Default::default()
                },
            ],
            ..Default::default()
        };
        snapshot.cpu.temperatures = vec![temp("Tctl", 97.0), temp("Tccd1", 90.0)];
        let texts_of = |snapshot: &Snapshot| -> Vec<String> {
            alerts_with(snapshot, &AlertsConfig::default())
                .into_iter()
                .map(|a| a.text)
                .collect()
        };
        let texts = |limits: &AlertsConfig| -> Vec<String> {
            alerts_with(&snapshot, limits)
                .into_iter()
                .map(|a| a.text)
                .collect()
        };
        assert_eq!(
            texts(&AlertsConfig::default()),
            [
                "/data 92% full",
                "CPU at 97°C",
                "nvme0 at 72°C",
                "nvme1 at 74°C"
            ]
        );
        let custom = AlertsConfig {
            filesystem_warning_percent: 0.0,
            cpu_warning_celsius: 0.0,
            gpu_warning_celsius: 75.0,
            drive_warning_celsius: 73.0,
            ..Default::default()
        };
        assert_eq!(texts(&custom), ["GPU 0 at 80°C", "nvme1 at 74°C"]);

        // Hardware slowdowns warn; running at the power limit does not.
        let mut gpu = snapshot.gpus[0].clone();
        gpu.throttle_reasons = vec!["power cap".to_owned()];
        let mut capped = snapshot.clone();
        capped.gpus = vec![gpu.clone()];
        assert!(!texts_of(&capped).iter().any(|t| t.contains("slowed")));
        gpu.throttle_reasons.push("hw thermal".to_owned());
        gpu.ecc = Some(EccErrors {
            corrected: 5,
            uncorrected: 2,
        });
        capped.gpus = vec![gpu];
        let alerts = alerts_with(&capped, &AlertsConfig::default());
        assert_eq!(alerts[0].text, "GPU 0: 2 uncorrected ECC errors");
        assert_eq!(alerts[0].severity, Severity::Critical);
        assert!(
            alerts
                .iter()
                .any(|a| a.text == "GPU 0 slowed down (hw thermal)")
        );
        let critical = AlertsConfig {
            filesystem_critical_percent: 90.0,
            ..custom
        };
        assert_eq!(
            alerts_with(&snapshot, &critical)[0],
            Alert {
                severity: Severity::Critical,
                text: "/data 92% full".to_owned()
            }
        );
    }

    #[test]
    fn pressure_alerts_and_readings() {
        use crate::snapshot::PressureLevels;
        let levels = |some: f32, full: f32| PressureLevels {
            some_avg10: some,
            some_avg60: some,
            full_avg10: full,
            full_avg60: full,
        };
        let mut snapshot = Snapshot {
            pressure: Some(Pressure {
                cpu: levels(40.0, 0.0),
                memory: levels(0.2, 0.1),
                io: levels(80.0, 12.0),
            }),
            ..Default::default()
        };
        let limits = AlertsConfig::default();
        // A busy CPU and a long copy (some I/O, little full) are fine.
        assert!(alerts_with(&snapshot, &limits).is_empty());
        let names = |snapshot: &Snapshot| -> Vec<(&'static str, String, bool)> {
            pressure_items(snapshot.pressure.as_ref().unwrap(), &limits)
                .into_iter()
                .map(|i| (i.name, i.value, i.warn))
                .collect()
        };
        // Memory under 1% is left out.
        assert_eq!(
            names(&snapshot),
            [
                ("cpu", "40%".to_owned(), false),
                ("io", "80%".to_owned(), false)
            ]
        );

        snapshot.pressure = Some(Pressure {
            cpu: levels(0.0, 0.0),
            memory: levels(25.0, 4.0),
            io: levels(90.0, 45.0),
        });
        let texts: Vec<String> = alerts_with(&snapshot, &limits)
            .into_iter()
            .map(|a| a.text)
            .collect();
        assert_eq!(texts, ["memory pressure 25%", "I/O pressure 45%"]);
        assert_eq!(
            names(&snapshot),
            [
                ("mem", "25%".to_owned(), true),
                ("io", "90%".to_owned(), true)
            ]
        );
        assert!(
            pressure_details(snapshot.pressure.as_ref().unwrap())
                .contains("io: some 90% / 90%, full 45% / 45%")
        );
    }

    #[test]
    fn memory_alerts() {
        use crate::snapshot::{MemoryStats, OomKill};
        let kill = |name: Option<&str>| OomKill {
            at_ms: 1,
            likely_process: name.map(str::to_owned),
        };
        let mut snapshot = Snapshot {
            memory: MemoryStats {
                total_bytes: 100,
                used_bytes: 93,
                oom_kills: vec![kill(Some("python3"))],
                ..Default::default()
            },
            gpus: vec![GpuStats {
                index: 1,
                memory_used_bytes: Some(99),
                memory_total_bytes: Some(100),
                ..Default::default()
            }],
            ..Default::default()
        };
        let texts = |snapshot: &Snapshot, limits: &AlertsConfig| -> Vec<(Severity, String)> {
            alerts_with(snapshot, limits)
                .into_iter()
                .map(|a| (a.severity, a.text))
                .collect()
        };
        // VRAM is off by default (frameworks reserve it all).
        assert_eq!(
            texts(&snapshot, &AlertsConfig::default()),
            [
                (
                    Severity::Critical,
                    "OOM killer stopped python3 (likely)".to_owned()
                ),
                (Severity::Warning, "memory 93% used".to_owned()),
            ]
        );
        let vram = AlertsConfig {
            memory_warning_percent: 0.0,
            gpu_memory_warning_percent: 98.0,
            ..Default::default()
        };
        snapshot.memory.oom_kills = vec![kill(Some("bash")), kill(None), kill(Some("python3"))];
        assert_eq!(
            texts(&snapshot, &vram),
            [
                (
                    Severity::Critical,
                    "OOM killer stopped 3 processes (likely python3, bash)".to_owned()
                ),
                (Severity::Warning, "GPU 1 memory 99% used".to_owned()),
            ]
        );
        snapshot.memory.oom_kills = vec![kill(None)];
        assert_eq!(texts(&snapshot, &vram)[0].1, "OOM killer stopped a process");
    }

    #[test]
    fn interface_helpers() {
        assert!(is_virtual_interface("veth5a74356"));
        assert!(!is_virtual_interface("enp9s0"));
        let snapshot = Snapshot {
            networks: vec![
                NetworkStats {
                    name: "docker0".to_owned(),
                    rx_total_bytes: 1 << 40,
                    ..Default::default()
                },
                NetworkStats {
                    name: "enp9s0".to_owned(),
                    rx_total_bytes: 1 << 30,
                    ..Default::default()
                },
            ],
            ..Default::default()
        };
        assert_eq!(default_interface(&snapshot).as_deref(), Some("enp9s0"));
        assert_eq!(signal_name(ProcessSignal::Kill), "SIGKILL");
    }
}