Skip to main content

rigtop_core/
view.rs

1//! Front-end helpers shared by the TUI and the GUI: graph history, process
2//! table sorting/filtering joined with GPU usage, and small naming helpers.
3//!
4//! Nothing here depends on a UI toolkit.
5
6use std::collections::{HashMap, HashSet, VecDeque};
7use std::time::Duration;
8
9use crate::config::AlertsConfig;
10use crate::snapshot::{ProcessInfo, Snapshot};
11use crate::source::ProcessSignal;
12
13/// Number of samples kept for graphs: over an hour at the default interval
14/// of 1 s, and wider than any realistic terminal in braille dots (2 samples
15/// per cell).
16pub const HISTORY_LEN: usize = 4096;
17
18/// Interface name prefixes skipped when choosing the default network
19/// interface (container bridges and virtual links).
20const VIRTUAL_IFACE_PREFIXES: &[&str] = &["lo", "docker", "veth", "br-", "virbr"];
21
22/// Samples averaged into one point of the coarse history (a minute at the
23/// default interval of 1 s).
24pub const COARSE_FACTOR: usize = 60;
25/// Points kept in the coarse history (a day at the default interval).
26pub const COARSE_LEN: usize = 1440;
27
28/// Fixed-size history of samples, newest last, plus a coarse history of
29/// averages for long time spans.
30#[derive(Debug, Default, Clone)]
31pub struct History {
32    samples: VecDeque<f64>,
33    coarse: VecDeque<f64>,
34    /// Sum and count of the samples not yet averaged into `coarse`.
35    pending: (f64, usize),
36}
37
38impl History {
39    /// Appends a sample, dropping the oldest when full.
40    pub fn push(&mut self, value: f64) {
41        if self.samples.len() == HISTORY_LEN {
42            self.samples.pop_front();
43        }
44        self.samples.push_back(value);
45        self.pending.0 += value;
46        self.pending.1 += 1;
47        if self.pending.1 == COARSE_FACTOR {
48            if self.coarse.len() == COARSE_LEN {
49                self.coarse.pop_front();
50            }
51            self.coarse.push_back(self.pending.0 / COARSE_FACTOR as f64);
52            self.pending = (0.0, 0);
53        }
54    }
55
56    /// The newest `n` samples (or fewer), oldest first.
57    pub fn last(&self, n: usize) -> Vec<f64> {
58        let skip = self.samples.len().saturating_sub(n);
59        self.samples.iter().skip(skip).copied().collect()
60    }
61
62    /// Largest of the newest `n` samples.
63    pub fn max_of_last(&self, n: usize) -> f64 {
64        self.last(n).into_iter().fold(0.0, f64::max)
65    }
66
67    /// Values for a graph `columns` wide, oldest first. [`GraphSpan::Fit`]
68    /// gives the newest sample per column; a time span gives the samples of
69    /// that span (one every `interval`) spread over the columns, averaged
70    /// where several share a column and interpolated where there are fewer
71    /// samples than columns. While the history is shorter than the span,
72    /// fewer values come back, so the graph fills up from the right.
73    pub fn graph(&self, columns: usize, span: GraphSpan, interval: Duration) -> Vec<f64> {
74        let GraphSpan::Secs(secs) = span else {
75            return self.last(columns);
76        };
77        let wanted = ((secs as f64 / interval.as_secs_f64().max(0.001)).round() as usize).max(1);
78        if wanted <= HISTORY_LEN {
79            spread(&self.last(wanted), wanted, columns)
80        } else {
81            // Longer than the samples kept: from the averages.
82            let wanted = wanted.div_ceil(COARSE_FACTOR);
83            let skip = self.coarse.len().saturating_sub(wanted);
84            let points: Vec<f64> = self.coarse.iter().skip(skip).copied().collect();
85            spread(&points, wanted, columns)
86        }
87    }
88}
89
90/// `samples` (the newest of `wanted`) spread over `columns`: averaged where
91/// several share a column, interpolated where there are fewer samples than
92/// columns, and over a proportional share of the columns at the right while
93/// fewer than `wanted` exist.
94fn spread(samples: &[f64], wanted: usize, columns: usize) -> Vec<f64> {
95    if columns == 0 || samples.is_empty() {
96        return Vec::new();
97    }
98    // The share of the columns that the available samples cover.
99    let shown = (columns as f64 * samples.len() as f64 / wanted.max(1) as f64).round() as usize;
100    let shown = shown.clamp(1, columns);
101    let per_column = samples.len() as f64 / shown as f64;
102    if per_column < 1.0 {
103        // Stretched: a straight line between neighbouring samples.
104        let last = samples.len() - 1;
105        return (0..shown)
106            .map(|i| {
107                let at = ((i as f64 + 0.5) * per_column - 0.5).clamp(0.0, last as f64);
108                let (below, frac) = (at.floor() as usize, at.fract());
109                let above = (below + 1).min(last);
110                samples[below] * (1.0 - frac) + samples[above] * frac
111            })
112            .collect();
113    }
114    (0..shown)
115        .map(|i| {
116            let end = (((i + 1) as f64 * per_column) as usize).clamp(1, samples.len());
117            let start = ((i as f64 * per_column) as usize).min(end - 1);
118            let part = &samples[start..end];
119            part.iter().sum::<f64>() / part.len() as f64
120        })
121        .collect()
122}
123
124/// How much time the graphs cover.
125#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
126pub enum GraphSpan {
127    /// One sample per graph column, so the time covered grows with the
128    /// width and the interval.
129    #[default]
130    Fit,
131    /// The last this many seconds, spread over the width.
132    Secs(u64),
133}
134
135impl GraphSpan {
136    /// The choices [`GraphSpan::next`] steps through.
137    pub const CHOICES: [GraphSpan; 7] = [
138        GraphSpan::Fit,
139        GraphSpan::Secs(60),
140        GraphSpan::Secs(5 * 60),
141        GraphSpan::Secs(15 * 60),
142        GraphSpan::Secs(60 * 60),
143        GraphSpan::Secs(6 * 60 * 60),
144        GraphSpan::Secs(24 * 60 * 60),
145    ];
146
147    /// The choice after this one, back to the first after the last.
148    pub fn next(self) -> Self {
149        let index = Self::CHOICES
150            .iter()
151            .position(|c| *c == self)
152            .map_or(0, |i| i + 1);
153        Self::CHOICES[index % Self::CHOICES.len()]
154    }
155
156    /// Short name: "fit", "1m", "1h".
157    pub fn label(self) -> String {
158        match self {
159            GraphSpan::Fit => "fit".to_owned(),
160            GraphSpan::Secs(secs) if secs > 0 && secs % 3600 == 0 => format!("{}h", secs / 3600),
161            GraphSpan::Secs(secs) if secs > 0 && secs % 60 == 0 => format!("{}m", secs / 60),
162            GraphSpan::Secs(secs) => format!("{secs}s"),
163        }
164    }
165
166    /// Seconds covered, 0 for [`GraphSpan::Fit`] (for storing the choice).
167    pub fn as_secs(self) -> u64 {
168        match self {
169            GraphSpan::Fit => 0,
170            GraphSpan::Secs(secs) => secs,
171        }
172    }
173
174    /// The inverse of [`GraphSpan::as_secs`].
175    pub fn from_secs(secs: u64) -> Self {
176        if secs == 0 {
177            GraphSpan::Fit
178        } else {
179            GraphSpan::Secs(secs)
180        }
181    }
182}
183
184/// Columns the process table can be sorted by.
185#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186pub enum SortKey {
187    Pid,
188    Name,
189    Threads,
190    User,
191    Memory,
192    Cpu,
193    Gpu,
194    Vram,
195}
196
197impl SortKey {
198    /// Sort keys in column order, used to step left/right.
199    pub const ALL: [SortKey; 8] = [
200        SortKey::Pid,
201        SortKey::Name,
202        SortKey::Threads,
203        SortKey::User,
204        SortKey::Memory,
205        SortKey::Cpu,
206        SortKey::Gpu,
207        SortKey::Vram,
208    ];
209
210    /// Whether "descending" is the natural order (largest consumers first).
211    pub fn descending_by_default(self) -> bool {
212        !matches!(self, SortKey::Pid | SortKey::Name | SortKey::User)
213    }
214}
215
216/// One row of the process table: the process plus its GPU usage.
217#[derive(Debug, Clone)]
218pub struct ProcessRow<'a> {
219    pub info: &'a ProcessInfo,
220    /// Highest per-GPU SM utilization of the process.
221    pub gpu_percent: Option<f32>,
222    /// VRAM used by the process across all GPUs.
223    pub vram_bytes: Option<u64>,
224}
225
226/// Graph histories of one host, updated from each snapshot.
227#[derive(Debug, Default, Clone)]
228pub struct MetricHistory {
229    /// Total CPU usage (%).
230    pub cpu: History,
231    /// Used memory (% of total).
232    pub memory: History,
233    /// Utilization (%) per GPU index.
234    pub gpus: Vec<History>,
235    /// Receive rate of `net_interface` (bytes/s).
236    pub net_rx: History,
237    /// Transmit rate of `net_interface` (bytes/s).
238    pub net_tx: History,
239    /// Interface whose traffic is recorded (`None`: chosen on the next snapshot).
240    pub net_interface: Option<String>,
241}
242
243impl MetricHistory {
244    /// Appends the values of a new snapshot.
245    pub fn record(&mut self, snapshot: &Snapshot) {
246        self.cpu.push(snapshot.cpu.total_percent as f64);
247        let memory = &snapshot.memory;
248        if memory.total_bytes > 0 {
249            self.memory
250                .push(memory.used_bytes as f64 / memory.total_bytes as f64 * 100.0);
251        }
252        self.gpus.resize_with(snapshot.gpus.len(), History::default);
253        for (history, gpu) in self.gpus.iter_mut().zip(&snapshot.gpus) {
254            history.push(gpu.utilization_percent.unwrap_or(0.0) as f64);
255        }
256        if self.net_interface.is_none() {
257            self.net_interface = default_interface(snapshot);
258        }
259        let (rx, tx) = self
260            .net_interface
261            .as_deref()
262            .and_then(|name| snapshot.networks.iter().find(|n| n.name == name))
263            .map(|n| (n.rx_bytes_per_sec, n.tx_bytes_per_sec))
264            .unwrap_or((0.0, 0.0));
265        self.net_rx.push(rx);
266        self.net_tx.push(tx);
267    }
268
269    /// Switches the recorded network interface, clearing its history.
270    pub fn select_interface(&mut self, name: &str) {
271        if self.net_interface.as_deref() != Some(name) {
272            self.net_interface = Some(name.to_owned());
273            // The old samples belong to another interface.
274            self.net_rx = History::default();
275            self.net_tx = History::default();
276        }
277    }
278
279    /// Switches to the interface after the current one in `snapshot` and
280    /// returns its name.
281    pub fn next_interface(&mut self, snapshot: &Snapshot) -> Option<String> {
282        let names: Vec<&str> = snapshot.networks.iter().map(|n| n.name.as_str()).collect();
283        if names.is_empty() {
284            return None;
285        }
286        let current = self
287            .net_interface
288            .as_deref()
289            .and_then(|name| names.iter().position(|n| *n == name));
290        let next = names[current.map_or(0, |i| (i + 1) % names.len())].to_owned();
291        self.select_interface(&next);
292        Some(next)
293    }
294}
295
296/// How urgent an [`Alert`] is.
297#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
298pub enum Severity {
299    /// Worth a look (e.g. a filesystem filling up, a RAID resync).
300    Warning,
301    /// Data at risk (errors, a missing device, a failing drive).
302    Critical,
303}
304
305/// A condition on a host that needs attention, for overviews.
306#[derive(Debug, Clone, PartialEq, Eq)]
307pub struct Alert {
308    pub severity: Severity,
309    pub text: String,
310}
311
312/// Identifies an alert across samples: its severity and text with numbers
313/// masked, so "3 I/O errors" becoming "4 I/O errors" or a temperature
314/// changing does not count as a new alert.
315pub fn alert_key(alert: &Alert) -> String {
316    let mut key = format!("{:?}:", alert.severity);
317    let mut in_number = false;
318    for c in alert.text.chars() {
319        if c.is_ascii_digit() || (in_number && c == '.') {
320            if !in_number {
321                key.push('#');
322            }
323            in_number = true;
324        } else {
325            key.push(c);
326            in_number = false;
327        }
328    }
329    key
330}
331
332/// Whether an [`AlertEvent`] is an alert appearing or going away.
333#[derive(Debug, Clone, Copy, PartialEq, Eq)]
334pub enum AlertChange {
335    Raised,
336    Cleared,
337}
338
339/// One entry of the [`AlertLog`].
340#[derive(Debug, Clone, PartialEq, Eq)]
341pub struct AlertEvent {
342    /// When the change was seen, in milliseconds since the Unix epoch.
343    pub at_ms: u64,
344    pub host: String,
345    pub change: AlertChange,
346    /// The alert as raised, or as last seen before it cleared.
347    pub alert: Alert,
348}
349
350/// Events kept by an [`AlertLog`].
351const ALERT_LOG_LEN: usize = 500;
352
353/// The alerts of every host as they appear and clear, so that what
354/// happened while nobody watched can be looked up later.
355#[derive(Debug, Default)]
356pub struct AlertLog {
357    /// Newest last.
358    events: VecDeque<AlertEvent>,
359    /// Each host's alerts at its previous update.
360    active: HashMap<String, Vec<Alert>>,
361    /// Alerts raised since [`AlertLog::mark_read`], and the worst of them.
362    unread: (usize, Option<Severity>),
363}
364
365impl AlertLog {
366    /// Records how the alerts of `host` changed since its previous update.
367    pub fn update(&mut self, host: &str, alerts: &[Alert], at_ms: u64) {
368        let previous = self.active.remove(host).unwrap_or_default();
369        let keys = |list: &[Alert]| -> HashSet<String> { list.iter().map(alert_key).collect() };
370        let (before, now) = (keys(&previous), keys(alerts));
371        let raised = alerts.iter().filter(|a| !before.contains(&alert_key(a)));
372        let cleared = previous.iter().filter(|a| !now.contains(&alert_key(a)));
373        let mut changes: Vec<(AlertChange, &Alert)> = raised
374            .map(|a| (AlertChange::Raised, a))
375            .chain(cleared.map(|a| (AlertChange::Cleared, a)))
376            .collect();
377        // Cleared first, so that a changed alert reads "went away, came back".
378        changes.sort_by_key(|(change, _)| *change == AlertChange::Raised);
379        for (change, alert) in changes {
380            if change == AlertChange::Raised {
381                self.unread.0 += 1;
382                self.unread.1 = self.unread.1.max(Some(alert.severity));
383            }
384            if self.events.len() == ALERT_LOG_LEN {
385                self.events.pop_front();
386            }
387            self.events.push_back(AlertEvent {
388                at_ms,
389                host: host.to_owned(),
390                change,
391                alert: alert.clone(),
392            });
393        }
394        self.active.insert(host.to_owned(), alerts.to_vec());
395    }
396
397    /// All events, oldest first.
398    pub fn events(&self) -> impl DoubleEndedIterator<Item = &AlertEvent> {
399        self.events.iter()
400    }
401
402    /// Alerts raised since the log was last read, and the most severe one.
403    pub fn unread(&self) -> (usize, Option<Severity>) {
404        self.unread
405    }
406
407    /// Marks every event as read.
408    pub fn mark_read(&mut self) {
409        self.unread = (0, None);
410    }
411}
412
413/// "agent 0.8.2" when `snapshot` comes from a rigtop of another version
414/// than this one (an agent not updated yet), else `None`.
415pub fn version_note(snapshot: &Snapshot) -> Option<String> {
416    let version = &snapshot.host.rigtop_version;
417    (!version.is_empty() && version != env!("CARGO_PKG_VERSION"))
418        .then(|| format!("agent {version}"))
419}
420
421/// Hover text explaining a [`version_note`].
422pub fn version_hint(snapshot: &Snapshot) -> String {
423    format!(
424        "this host runs rigtop {}, this machine {}: install the same version on both \
425         for every reading",
426        snapshot.host.rigtop_version,
427        env!("CARGO_PKG_VERSION")
428    )
429}
430
431/// [`alerts_with`] the default thresholds.
432pub fn alerts(snapshot: &Snapshot) -> Vec<Alert> {
433    alerts_with(snapshot, &AlertsConfig::default())
434}
435
436/// Conditions in `snapshot` that need attention, most severe first:
437/// storage errors and failing drives, degraded RAID, full filesystems, hot
438/// CPUs, GPUs and drives, judged by `limits` (0 turns a threshold off).
439pub fn alerts_with(snapshot: &Snapshot, limits: &AlertsConfig) -> Vec<Alert> {
440    // A threshold of 0 (or less) is turned off.
441    let reached = |value: f64, limit: f64| limit > 0.0 && value >= limit;
442    let mut alerts = Vec::new();
443    let mut add = |severity, text: String| alerts.push(Alert { severity, text });
444    for fs in &snapshot.raid.btrfs {
445        let name = fs
446            .mount_points
447            .first()
448            .map_or(fs.fsid.as_str(), String::as_str);
449        let missing = fs.device_stats.iter().filter(|d| d.missing).count();
450        if missing > 0 {
451            add(
452                Severity::Critical,
453                format!("btrfs {name}: {missing} device(s) missing"),
454            );
455        }
456        let errors: u64 = fs.device_stats.iter().map(|d| d.total_errors()).sum();
457        if errors > 0 {
458            // Which drives: the one to replace.
459            let devices: Vec<String> = fs
460                .device_stats
461                .iter()
462                .filter(|d| d.total_errors() > 0)
463                .map(|d| d.label())
464                .collect();
465            add(
466                Severity::Critical,
467                format!("btrfs {name}: {errors} I/O errors ({})", devices.join(", ")),
468            );
469        }
470        if let Some(scrub) = &fs.scrub {
471            if scrub.uncorrectable > 0 {
472                add(
473                    Severity::Critical,
474                    format!(
475                        "btrfs {name}: scrub found {} uncorrectable errors",
476                        scrub.uncorrectable
477                    ),
478                );
479            } else if scrub.errors > 0 {
480                add(
481                    Severity::Warning,
482                    format!("btrfs {name}: scrub corrected {} errors", scrub.corrected),
483                );
484            }
485        }
486    }
487    for md in &snapshot.raid.md {
488        if md.active && md.is_degraded() {
489            add(
490                Severity::Critical,
491                format!("{} ({}) degraded", md.name, md.level),
492            );
493        } else if !md.active {
494            add(Severity::Warning, format!("{} inactive", md.name));
495        }
496        if let Some(sync) = &md.sync {
497            add(
498                Severity::Warning,
499                format!("{} {} {:.0}%", md.name, sync.action, sync.percent),
500            );
501        }
502    }
503    for smart in &snapshot.smart {
504        if let Some(error) = &smart.error {
505            add(
506                Severity::Warning,
507                format!("{}: SMART unreadable ({error})", smart.device),
508            );
509        } else if smart.passed == Some(false) {
510            add(
511                Severity::Critical,
512                format!("{}: SMART failed", smart.device),
513            );
514        } else if smart.needs_attention() {
515            add(
516                Severity::Critical,
517                format!("{}: SMART warning", smart.device),
518            );
519        }
520    }
521    for fs in &snapshot.filesystems {
522        let used = fs.total_bytes.saturating_sub(fs.available_bytes);
523        let percent = used as f64 / fs.total_bytes.max(1) as f64 * 100.0;
524        let name = fs
525            .mount_points
526            .first()
527            .map_or(fs.device.as_str(), String::as_str);
528        if reached(percent, limits.filesystem_critical_percent) {
529            add(Severity::Critical, format!("{name} {percent:.0}% full"));
530        } else if reached(percent, limits.filesystem_warning_percent) {
531            add(Severity::Warning, format!("{name} {percent:.0}% full"));
532        }
533    }
534    let hottest_cpu = snapshot
535        .cpu
536        .temperatures
537        .iter()
538        .map(|t| t.celsius)
539        .fold(f32::NAN, f32::max);
540    if reached(hottest_cpu as f64, limits.cpu_warning_celsius as f64) {
541        add(Severity::Warning, format!("CPU at {hottest_cpu:.0}°C"));
542    }
543    for gpu in &snapshot.gpus {
544        if let Some(t) = gpu
545            .temperature_c
546            .filter(|t| reached(*t as f64, limits.gpu_warning_celsius as f64))
547        {
548            add(Severity::Warning, format!("GPU {} at {t:.0}°C", gpu.index));
549        }
550    }
551    for (device, celsius) in drive_temperatures(snapshot) {
552        if reached(celsius as f64, limits.drive_warning_celsius as f64) {
553            add(Severity::Warning, format!("{device} at {celsius:.0}°C"));
554        }
555    }
556    // Stable sort keeps the order above within each severity.
557    alerts.sort_by_key(|a| std::cmp::Reverse(a.severity));
558    alerts
559}
560
561/// (device, °C) of each drive: SMART results, then NVMe drives without
562/// them from their hwmon sensor ("Composite", the drive's own summary).
563fn drive_temperatures(snapshot: &Snapshot) -> Vec<(String, f32)> {
564    let mut drives: Vec<(String, f32)> = snapshot
565        .smart
566        .iter()
567        .filter_map(|s| Some((s.device.clone(), s.temperature_c?)))
568        .collect();
569    for chip in snapshot.sensors.iter().filter(|c| c.name == "nvme") {
570        // The chip key is "nvme:<device>".
571        let device = chip.key.split_once(':').map_or(chip.key.as_str(), |k| k.1);
572        if drives.iter().any(|(d, _)| d == device) {
573            continue;
574        }
575        let composite = chip
576            .temperatures
577            .iter()
578            .find(|t| t.label == "Composite")
579            .or(chip.temperatures.first());
580        if let Some(t) = composite {
581            drives.push((device.to_owned(), t.celsius));
582        }
583    }
584    drives
585}
586
587/// How the process table is filtered and sorted.
588#[derive(Debug, Clone, Copy)]
589pub struct ProcessQuery<'q> {
590    /// Column to sort by.
591    pub sort_key: SortKey,
592    /// Largest first when true.
593    pub descending: bool,
594    /// Case-insensitive substring of name, command or user, or an exact pid.
595    pub filter: &'q str,
596    /// Include kernel threads (`kworker` and friends).
597    pub show_kernel_threads: bool,
598}
599
600/// Processes of `snapshot` after filtering and sorting, joined with GPU usage.
601pub fn process_rows<'a>(snapshot: &'a Snapshot, query: &ProcessQuery<'_>) -> Vec<ProcessRow<'a>> {
602    // pid -> (max SM %, total VRAM) across GPUs.
603    let mut gpu_usage: HashMap<u32, (Option<f32>, Option<u64>)> = HashMap::new();
604    for gpu in &snapshot.gpus {
605        for p in &gpu.processes {
606            let entry = gpu_usage.entry(p.pid).or_default();
607            if let Some(sm) = p.sm_percent {
608                entry.0 = Some(entry.0.map_or(sm, |v| v.max(sm)));
609            }
610            if let Some(mem) = p.memory_bytes {
611                entry.1 = Some(entry.1.unwrap_or(0) + mem);
612            }
613        }
614    }
615
616    let filter = query.filter.to_lowercase();
617    let mut rows: Vec<ProcessRow<'a>> = snapshot
618        .processes
619        .iter()
620        .filter(|p| query.show_kernel_threads || !p.kernel_thread)
621        .filter(|p| {
622            filter.is_empty()
623                || p.name.to_lowercase().contains(&filter)
624                || p.command.to_lowercase().contains(&filter)
625                || p.user.to_lowercase().contains(&filter)
626                || p.pid.to_string() == filter
627        })
628        .map(|info| {
629            let (gpu_percent, vram_bytes) = gpu_usage.get(&info.pid).copied().unwrap_or_default();
630            ProcessRow {
631                info,
632                gpu_percent,
633                vram_bytes,
634            }
635        })
636        .collect();
637
638    let key = query.sort_key;
639    rows.sort_by(|a, b| {
640        let ordering = match key {
641            SortKey::Pid => a.info.pid.cmp(&b.info.pid),
642            SortKey::Name => a.info.name.to_lowercase().cmp(&b.info.name.to_lowercase()),
643            SortKey::Threads => a.info.threads.cmp(&b.info.threads),
644            SortKey::User => a.info.user.cmp(&b.info.user),
645            SortKey::Memory => a.info.memory_bytes.cmp(&b.info.memory_bytes),
646            SortKey::Cpu => a.info.cpu_percent.total_cmp(&b.info.cpu_percent),
647            SortKey::Gpu => a
648                .gpu_percent
649                .unwrap_or(-1.0)
650                .total_cmp(&b.gpu_percent.unwrap_or(-1.0)),
651            SortKey::Vram => a.vram_bytes.cmp(&b.vram_bytes),
652        }
653        // Stable tie-breaker so rows do not jump between frames.
654        .then(a.info.pid.cmp(&b.info.pid));
655        if query.descending {
656            ordering.reverse()
657        } else {
658            ordering
659        }
660    });
661    rows
662}
663
664/// Shown where the CPU temperatures go when LibreHardwareMonitor cannot be
665/// read (Windows; see [`Snapshot::lhm_error`]).
666pub const LHM_HINT: &str = "temperatures need LibreHardwareMonitor";
667
668/// Explanation behind [`LHM_HINT`], given the reason it could not be read.
669pub fn lhm_help(error: &str) -> String {
670    format!(
671        "Windows needs LibreHardwareMonitor for temperatures, fans and CPU power.\n\
672         Start it and turn on Options → Remote Web Server → Run.\n\
673         ([lhm] url = \"\" in the configuration hides this note.)\n\n\
674         {error}"
675    )
676}
677
678/// Name of a signal as shown to the user.
679pub fn signal_name(signal: ProcessSignal) -> &'static str {
680    match signal {
681        ProcessSignal::Terminate => "SIGTERM",
682        ProcessSignal::Kill => "SIGKILL",
683    }
684}
685
686/// Returns true for container bridges, veth pairs and loopback, which are
687/// skipped when choosing and listing interfaces.
688pub fn is_virtual_interface(name: &str) -> bool {
689    VIRTUAL_IFACE_PREFIXES.iter().any(|p| name.starts_with(p))
690}
691
692/// Picks the physical interface with the most traffic so far.
693pub fn default_interface(snapshot: &Snapshot) -> Option<String> {
694    snapshot
695        .networks
696        .iter()
697        .filter(|n| !is_virtual_interface(&n.name))
698        .max_by_key(|n| n.rx_total_bytes + n.tx_total_bytes)
699        .or_else(|| snapshot.networks.first())
700        .map(|n| n.name.clone())
701}
702
703#[cfg(test)]
704mod tests {
705    use super::*;
706    use crate::snapshot::{GpuProcess, GpuStats, NetworkStats};
707
708    fn process(pid: u32, name: &str, cpu: f32, kernel: bool) -> ProcessInfo {
709        ProcessInfo {
710            pid,
711            name: name.to_owned(),
712            cpu_percent: cpu,
713            kernel_thread: kernel,
714            ..Default::default()
715        }
716    }
717
718    fn query(sort_key: SortKey) -> ProcessQuery<'static> {
719        ProcessQuery {
720            sort_key,
721            descending: sort_key.descending_by_default(),
722            filter: "",
723            show_kernel_threads: false,
724        }
725    }
726
727    #[test]
728    fn history_keeps_newest_samples() {
729        let mut history = History::default();
730        for i in 0..(HISTORY_LEN + 5) {
731            history.push(i as f64);
732        }
733        assert_eq!(
734            history.last(2),
735            vec![(HISTORY_LEN + 3) as f64, (HISTORY_LEN + 4) as f64]
736        );
737        assert_eq!(history.last(HISTORY_LEN * 2).len(), HISTORY_LEN);
738        assert_eq!(history.max_of_last(3), (HISTORY_LEN + 4) as f64);
739    }
740
741    #[test]
742    fn graph_spans_spread_samples_over_the_width() {
743        let second = Duration::from_secs(1);
744        let mut history = History::default();
745        for i in 0..120 {
746            history.push(i as f64);
747        }
748        // Fit: one sample per column, the newest ones.
749        assert_eq!(
750            history.graph(3, GraphSpan::Fit, second),
751            vec![117.0, 118.0, 119.0]
752        );
753        // A minute on 30 columns: two samples averaged per column.
754        let minute = history.graph(30, GraphSpan::Secs(60), second);
755        assert_eq!(minute.len(), 30);
756        assert_eq!(minute[0], 60.5);
757        assert_eq!(minute[29], 118.5);
758        // Fewer samples than columns: interpolated between them.
759        let stretched = history.graph(120, GraphSpan::Secs(60), Duration::from_secs(2));
760        assert_eq!(stretched.len(), 120);
761        assert_eq!(&stretched[..2], &[90.0, 90.0]);
762        assert_eq!(&stretched[2..4], &[90.125, 90.375]);
763        assert_eq!(stretched[119], 119.0);
764        assert!(stretched.windows(2).all(|w| w[0] <= w[1]));
765        // An hour with two minutes recorded fills the right 1/30.
766        let hour = history.graph(300, GraphSpan::Secs(3600), second);
767        assert_eq!(hour.len(), 10);
768        assert_eq!(hour[0], 5.5);
769        assert!(
770            History::default()
771                .graph(10, GraphSpan::Secs(60), second)
772                .is_empty()
773        );
774    }
775
776    #[test]
777    fn long_spans_draw_from_the_averages() {
778        let second = Duration::from_secs(1);
779        let mut history = History::default();
780        // Two hours: the first at 0, the second at 100.
781        for i in 0..7200 {
782            history.push(if i < 3600 { 0.0 } else { 100.0 });
783        }
784        // 6 hours with 2 recorded: the right third, half low and half high.
785        let six = history.graph(90, GraphSpan::Secs(6 * 3600), second);
786        assert_eq!(six.len(), 30);
787        assert_eq!((six[0], six[29]), (0.0, 100.0));
788        // An hour still comes from the samples themselves.
789        let hour = history.graph(60, GraphSpan::Secs(3600), second);
790        assert!(hour.iter().all(|v| *v == 100.0));
791        // The averages stay bounded.
792        for i in 0..(COARSE_LEN * COARSE_FACTOR + 500) {
793            history.push(i as f64);
794        }
795        assert_eq!(history.coarse.len(), COARSE_LEN);
796    }
797
798    #[test]
799    fn graph_span_choices_cycle() {
800        let labels: Vec<String> = std::iter::successors(Some(GraphSpan::Fit), |s| Some(s.next()))
801            .take(6)
802            .map(GraphSpan::label)
803            .collect();
804        assert_eq!(labels, ["fit", "1m", "5m", "15m", "1h", "6h"]);
805        assert_eq!(GraphSpan::Secs(86_400).label(), "24h");
806        assert_eq!(GraphSpan::Secs(86_400).next(), GraphSpan::Fit);
807        assert_eq!(
808            GraphSpan::from_secs(GraphSpan::Secs(300).as_secs()),
809            GraphSpan::Secs(300)
810        );
811        assert_eq!(GraphSpan::from_secs(0), GraphSpan::Fit);
812        // A stored span that is not a choice starts over.
813        assert_eq!(GraphSpan::Secs(7).next(), GraphSpan::Fit);
814    }
815
816    #[test]
817    fn rows_filter_sort_and_join_gpu_usage() {
818        let snapshot = Snapshot {
819            processes: vec![
820                process(10, "bash", 5.0, false),
821                process(2, "kworker/0:1", 50.0, true),
822                process(30, "python", 1.0, false),
823            ],
824            gpus: vec![GpuStats {
825                processes: vec![GpuProcess {
826                    pid: 30,
827                    memory_bytes: Some(1 << 30),
828                    sm_percent: Some(70.0),
829                }],
830                ..Default::default()
831            }],
832            ..Default::default()
833        };
834        let pids = |q: &ProcessQuery| {
835            process_rows(&snapshot, q)
836                .iter()
837                .map(|r| r.info.pid)
838                .collect::<Vec<_>>()
839        };
840        assert_eq!(pids(&query(SortKey::Cpu)), vec![10, 30]);
841        assert_eq!(pids(&query(SortKey::Vram)), vec![30, 10]);
842        assert_eq!(
843            pids(&ProcessQuery {
844                show_kernel_threads: true,
845                ..query(SortKey::Cpu)
846            }),
847            vec![2, 10, 30]
848        );
849        assert_eq!(
850            pids(&ProcessQuery {
851                filter: "PYT",
852                ..query(SortKey::Pid)
853            }),
854            vec![30]
855        );
856        let rows = process_rows(&snapshot, &query(SortKey::Pid));
857        assert_eq!(rows[1].vram_bytes, Some(1 << 30));
858        assert_eq!(rows[1].gpu_percent, Some(70.0));
859    }
860
861    #[test]
862    fn metric_history_records_and_switches_interfaces() {
863        let snapshot = Snapshot {
864            networks: vec![
865                NetworkStats {
866                    name: "enp9s0".to_owned(),
867                    rx_total_bytes: 10,
868                    rx_bytes_per_sec: 100.0,
869                    ..Default::default()
870                },
871                NetworkStats {
872                    name: "wlp7s0".to_owned(),
873                    tx_bytes_per_sec: 5.0,
874                    ..Default::default()
875                },
876            ],
877            gpus: vec![GpuStats {
878                utilization_percent: Some(40.0),
879                ..Default::default()
880            }],
881            ..Default::default()
882        };
883        let mut history = MetricHistory::default();
884        history.record(&snapshot);
885        assert_eq!(history.net_interface.as_deref(), Some("enp9s0"));
886        assert_eq!(history.net_rx.last(1), vec![100.0]);
887        assert_eq!(history.gpus[0].last(1), vec![40.0]);
888        assert_eq!(history.next_interface(&snapshot).as_deref(), Some("wlp7s0"));
889        // Switching clears the samples of the previous interface.
890        assert!(history.net_rx.last(10).is_empty());
891        history.record(&snapshot);
892        assert_eq!(history.net_tx.last(1), vec![5.0]);
893    }
894
895    #[test]
896    fn alerts_flag_storage_problems_first() {
897        use crate::snapshot::{
898            BtrfsDeviceStats, BtrfsFilesystem, FilesystemUsage, MdArray, RaidStats, SmartDevice,
899        };
900        let snapshot = Snapshot {
901            filesystems: vec![FilesystemUsage {
902                mount_points: vec!["/data".to_owned()],
903                total_bytes: 100,
904                available_bytes: 8,
905                ..Default::default()
906            }],
907            raid: RaidStats {
908                btrfs: vec![BtrfsFilesystem {
909                    mount_points: vec!["/".to_owned()],
910                    device_stats: vec![BtrfsDeviceStats {
911                        devid: 1,
912                        corruption_errs: 2,
913                        ..Default::default()
914                    }],
915                    ..Default::default()
916                }],
917                md: vec![MdArray {
918                    name: "md0".to_owned(),
919                    active: true,
920                    level: "raid1".to_owned(),
921                    expected_devices: Some(2),
922                    working_devices: Some(1),
923                    ..Default::default()
924                }],
925            },
926            smart: vec![SmartDevice {
927                device: "sda".to_owned(),
928                passed: Some(true),
929                ..Default::default()
930            }],
931            ..Default::default()
932        };
933        let alerts = alerts(&snapshot);
934        let texts: Vec<&str> = alerts.iter().map(|a| a.text.as_str()).collect();
935        assert_eq!(
936            texts,
937            vec![
938                "btrfs /: 2 I/O errors (devid 1)",
939                "md0 (raid1) degraded",
940                "/data 92% full"
941            ]
942        );
943        assert_eq!(alerts[0].severity, Severity::Critical);
944        assert_eq!(alerts[2].severity, Severity::Warning);
945        assert!(super::alerts(&Snapshot::default()).is_empty());
946    }
947
948    #[test]
949    fn version_notes_only_for_other_versions() {
950        let mut snapshot = Snapshot::default();
951        assert_eq!(version_note(&snapshot), None);
952        snapshot.host.rigtop_version = env!("CARGO_PKG_VERSION").to_owned();
953        assert_eq!(version_note(&snapshot), None);
954        snapshot.host.rigtop_version = "0.1.0".to_owned();
955        assert_eq!(version_note(&snapshot).as_deref(), Some("agent 0.1.0"));
956        assert!(version_hint(&snapshot).contains("rigtop 0.1.0"));
957    }
958
959    #[test]
960    fn alert_log_records_alerts_appearing_and_clearing() {
961        let alert = |severity, text: &str| Alert {
962            severity,
963            text: text.to_owned(),
964        };
965        let full = alert(Severity::Warning, "/data 91% full");
966        let errors = alert(Severity::Critical, "btrfs /: 2 I/O errors (nvme1n1)");
967        let mut log = AlertLog::default();
968        log.update("nas", std::slice::from_ref(&full), 1_000);
969        // Numbers changing is the same alert.
970        log.update(
971            "nas",
972            &[alert(Severity::Warning, "/data 92% full"), errors.clone()],
973            2_000,
974        );
975        log.update("zefat", &[], 2_000);
976        assert_eq!(log.unread(), (2, Some(Severity::Critical)));
977        log.mark_read();
978        log.update("nas", std::slice::from_ref(&errors), 3_000);
979        assert_eq!(log.unread(), (0, None));
980        let summary: Vec<(u64, AlertChange, &str)> = log
981            .events()
982            .map(|e| (e.at_ms, e.change, e.alert.text.as_str()))
983            .collect();
984        assert_eq!(
985            summary,
986            [
987                (1_000, AlertChange::Raised, "/data 91% full"),
988                (
989                    2_000,
990                    AlertChange::Raised,
991                    "btrfs /: 2 I/O errors (nvme1n1)"
992                ),
993                (3_000, AlertChange::Cleared, "/data 92% full"),
994            ]
995        );
996        for i in 0..(ALERT_LOG_LEN as u64 + 10) {
997            log.update(
998                "flappy",
999                &[alert(Severity::Warning, if i % 2 == 0 { "a" } else { "b" })],
1000                i,
1001            );
1002        }
1003        assert_eq!(log.events().count(), ALERT_LOG_LEN);
1004    }
1005
1006    #[test]
1007    fn alert_thresholds_come_from_the_configuration() {
1008        use crate::snapshot::{FilesystemUsage, GpuStats, SensorChip, SmartDevice, Temperature};
1009        let temp = |label: &str, celsius| Temperature {
1010            label: label.to_owned(),
1011            celsius,
1012            ..Default::default()
1013        };
1014        let mut snapshot = Snapshot {
1015            filesystems: vec![FilesystemUsage {
1016                mount_points: vec!["/data".to_owned()],
1017                total_bytes: 100,
1018                available_bytes: 8,
1019                ..Default::default()
1020            }],
1021            gpus: vec![GpuStats {
1022                index: 0,
1023                temperature_c: Some(80.0),
1024                ..Default::default()
1025            }],
1026            smart: vec![SmartDevice {
1027                device: "nvme0".to_owned(),
1028                temperature_c: Some(72.0),
1029                ..Default::default()
1030            }],
1031            // nvme0 also has SMART (reported once); nvme1 only its sensor.
1032            sensors: vec![
1033                SensorChip {
1034                    key: "nvme:nvme0".to_owned(),
1035                    name: "nvme".to_owned(),
1036                    temperatures: vec![temp("Composite", 71.0)],
1037                    ..Default::default()
1038                },
1039                SensorChip {
1040                    key: "nvme:nvme1".to_owned(),
1041                    name: "nvme".to_owned(),
1042                    temperatures: vec![temp("Sensor 1", 90.0), temp("Composite", 74.0)],
1043                    ..Default::default()
1044                },
1045            ],
1046            ..Default::default()
1047        };
1048        snapshot.cpu.temperatures = vec![temp("Tctl", 97.0), temp("Tccd1", 90.0)];
1049        let texts = |limits: &AlertsConfig| -> Vec<String> {
1050            alerts_with(&snapshot, limits)
1051                .into_iter()
1052                .map(|a| a.text)
1053                .collect()
1054        };
1055        assert_eq!(
1056            texts(&AlertsConfig::default()),
1057            [
1058                "/data 92% full",
1059                "CPU at 97°C",
1060                "nvme0 at 72°C",
1061                "nvme1 at 74°C"
1062            ]
1063        );
1064        let custom = AlertsConfig {
1065            filesystem_warning_percent: 0.0,
1066            cpu_warning_celsius: 0.0,
1067            gpu_warning_celsius: 75.0,
1068            drive_warning_celsius: 73.0,
1069            ..Default::default()
1070        };
1071        assert_eq!(texts(&custom), ["GPU 0 at 80°C", "nvme1 at 74°C"]);
1072        let critical = AlertsConfig {
1073            filesystem_critical_percent: 90.0,
1074            ..custom
1075        };
1076        assert_eq!(
1077            alerts_with(&snapshot, &critical)[0],
1078            Alert {
1079                severity: Severity::Critical,
1080                text: "/data 92% full".to_owned()
1081            }
1082        );
1083    }
1084
1085    #[test]
1086    fn interface_helpers() {
1087        assert!(is_virtual_interface("veth5a74356"));
1088        assert!(!is_virtual_interface("enp9s0"));
1089        let snapshot = Snapshot {
1090            networks: vec![
1091                NetworkStats {
1092                    name: "docker0".to_owned(),
1093                    rx_total_bytes: 1 << 40,
1094                    ..Default::default()
1095                },
1096                NetworkStats {
1097                    name: "enp9s0".to_owned(),
1098                    rx_total_bytes: 1 << 30,
1099                    ..Default::default()
1100                },
1101            ],
1102            ..Default::default()
1103        };
1104        assert_eq!(default_interface(&snapshot).as_deref(), Some("enp9s0"));
1105        assert_eq!(signal_name(ProcessSignal::Kill), "SIGKILL");
1106    }
1107}