1use std::collections::{HashMap, HashSet, VecDeque};
7use std::time::Duration;
8
9use crate::config::AlertsConfig;
10use crate::snapshot::{ProcessInfo, Snapshot};
11use crate::source::ProcessSignal;
12
13pub const HISTORY_LEN: usize = 4096;
17
18const VIRTUAL_IFACE_PREFIXES: &[&str] = &["lo", "docker", "veth", "br-", "virbr"];
21
22pub const COARSE_FACTOR: usize = 60;
25pub const COARSE_LEN: usize = 1440;
27
28#[derive(Debug, Default, Clone)]
31pub struct History {
32 samples: VecDeque<f64>,
33 coarse: VecDeque<f64>,
34 pending: (f64, usize),
36}
37
38impl History {
39 pub fn push(&mut self, value: f64) {
41 if self.samples.len() == HISTORY_LEN {
42 self.samples.pop_front();
43 }
44 self.samples.push_back(value);
45 self.pending.0 += value;
46 self.pending.1 += 1;
47 if self.pending.1 == COARSE_FACTOR {
48 if self.coarse.len() == COARSE_LEN {
49 self.coarse.pop_front();
50 }
51 self.coarse.push_back(self.pending.0 / COARSE_FACTOR as f64);
52 self.pending = (0.0, 0);
53 }
54 }
55
56 pub fn last(&self, n: usize) -> Vec<f64> {
58 let skip = self.samples.len().saturating_sub(n);
59 self.samples.iter().skip(skip).copied().collect()
60 }
61
62 pub fn max_of_last(&self, n: usize) -> f64 {
64 self.last(n).into_iter().fold(0.0, f64::max)
65 }
66
67 pub fn graph(&self, columns: usize, span: GraphSpan, interval: Duration) -> Vec<f64> {
74 let GraphSpan::Secs(secs) = span else {
75 return self.last(columns);
76 };
77 let wanted = ((secs as f64 / interval.as_secs_f64().max(0.001)).round() as usize).max(1);
78 if wanted <= HISTORY_LEN {
79 spread(&self.last(wanted), wanted, columns)
80 } else {
81 let wanted = wanted.div_ceil(COARSE_FACTOR);
83 let skip = self.coarse.len().saturating_sub(wanted);
84 let points: Vec<f64> = self.coarse.iter().skip(skip).copied().collect();
85 spread(&points, wanted, columns)
86 }
87 }
88}
89
90fn spread(samples: &[f64], wanted: usize, columns: usize) -> Vec<f64> {
95 if columns == 0 || samples.is_empty() {
96 return Vec::new();
97 }
98 let shown = (columns as f64 * samples.len() as f64 / wanted.max(1) as f64).round() as usize;
100 let shown = shown.clamp(1, columns);
101 let per_column = samples.len() as f64 / shown as f64;
102 if per_column < 1.0 {
103 let last = samples.len() - 1;
105 return (0..shown)
106 .map(|i| {
107 let at = ((i as f64 + 0.5) * per_column - 0.5).clamp(0.0, last as f64);
108 let (below, frac) = (at.floor() as usize, at.fract());
109 let above = (below + 1).min(last);
110 samples[below] * (1.0 - frac) + samples[above] * frac
111 })
112 .collect();
113 }
114 (0..shown)
115 .map(|i| {
116 let end = (((i + 1) as f64 * per_column) as usize).clamp(1, samples.len());
117 let start = ((i as f64 * per_column) as usize).min(end - 1);
118 let part = &samples[start..end];
119 part.iter().sum::<f64>() / part.len() as f64
120 })
121 .collect()
122}
123
124#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
126pub enum GraphSpan {
127 #[default]
130 Fit,
131 Secs(u64),
133}
134
135impl GraphSpan {
136 pub const CHOICES: [GraphSpan; 7] = [
138 GraphSpan::Fit,
139 GraphSpan::Secs(60),
140 GraphSpan::Secs(5 * 60),
141 GraphSpan::Secs(15 * 60),
142 GraphSpan::Secs(60 * 60),
143 GraphSpan::Secs(6 * 60 * 60),
144 GraphSpan::Secs(24 * 60 * 60),
145 ];
146
147 pub fn next(self) -> Self {
149 let index = Self::CHOICES
150 .iter()
151 .position(|c| *c == self)
152 .map_or(0, |i| i + 1);
153 Self::CHOICES[index % Self::CHOICES.len()]
154 }
155
156 pub fn label(self) -> String {
158 match self {
159 GraphSpan::Fit => "fit".to_owned(),
160 GraphSpan::Secs(secs) if secs > 0 && secs % 3600 == 0 => format!("{}h", secs / 3600),
161 GraphSpan::Secs(secs) if secs > 0 && secs % 60 == 0 => format!("{}m", secs / 60),
162 GraphSpan::Secs(secs) => format!("{secs}s"),
163 }
164 }
165
166 pub fn as_secs(self) -> u64 {
168 match self {
169 GraphSpan::Fit => 0,
170 GraphSpan::Secs(secs) => secs,
171 }
172 }
173
174 pub fn from_secs(secs: u64) -> Self {
176 if secs == 0 {
177 GraphSpan::Fit
178 } else {
179 GraphSpan::Secs(secs)
180 }
181 }
182}
183
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186pub enum SortKey {
187 Pid,
188 Name,
189 Threads,
190 User,
191 Memory,
192 Cpu,
193 Gpu,
194 Vram,
195}
196
197impl SortKey {
198 pub const ALL: [SortKey; 8] = [
200 SortKey::Pid,
201 SortKey::Name,
202 SortKey::Threads,
203 SortKey::User,
204 SortKey::Memory,
205 SortKey::Cpu,
206 SortKey::Gpu,
207 SortKey::Vram,
208 ];
209
210 pub fn descending_by_default(self) -> bool {
212 !matches!(self, SortKey::Pid | SortKey::Name | SortKey::User)
213 }
214}
215
216#[derive(Debug, Clone)]
218pub struct ProcessRow<'a> {
219 pub info: &'a ProcessInfo,
220 pub gpu_percent: Option<f32>,
222 pub vram_bytes: Option<u64>,
224}
225
226#[derive(Debug, Default, Clone)]
228pub struct MetricHistory {
229 pub cpu: History,
231 pub memory: History,
233 pub gpus: Vec<History>,
235 pub net_rx: History,
237 pub net_tx: History,
239 pub net_interface: Option<String>,
241}
242
243impl MetricHistory {
244 pub fn record(&mut self, snapshot: &Snapshot) {
246 self.cpu.push(snapshot.cpu.total_percent as f64);
247 let memory = &snapshot.memory;
248 if memory.total_bytes > 0 {
249 self.memory
250 .push(memory.used_bytes as f64 / memory.total_bytes as f64 * 100.0);
251 }
252 self.gpus.resize_with(snapshot.gpus.len(), History::default);
253 for (history, gpu) in self.gpus.iter_mut().zip(&snapshot.gpus) {
254 history.push(gpu.utilization_percent.unwrap_or(0.0) as f64);
255 }
256 if self.net_interface.is_none() {
257 self.net_interface = default_interface(snapshot);
258 }
259 let (rx, tx) = self
260 .net_interface
261 .as_deref()
262 .and_then(|name| snapshot.networks.iter().find(|n| n.name == name))
263 .map(|n| (n.rx_bytes_per_sec, n.tx_bytes_per_sec))
264 .unwrap_or((0.0, 0.0));
265 self.net_rx.push(rx);
266 self.net_tx.push(tx);
267 }
268
269 pub fn select_interface(&mut self, name: &str) {
271 if self.net_interface.as_deref() != Some(name) {
272 self.net_interface = Some(name.to_owned());
273 self.net_rx = History::default();
275 self.net_tx = History::default();
276 }
277 }
278
279 pub fn next_interface(&mut self, snapshot: &Snapshot) -> Option<String> {
282 let names: Vec<&str> = snapshot.networks.iter().map(|n| n.name.as_str()).collect();
283 if names.is_empty() {
284 return None;
285 }
286 let current = self
287 .net_interface
288 .as_deref()
289 .and_then(|name| names.iter().position(|n| *n == name));
290 let next = names[current.map_or(0, |i| (i + 1) % names.len())].to_owned();
291 self.select_interface(&next);
292 Some(next)
293 }
294}
295
296#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
298pub enum Severity {
299 Warning,
301 Critical,
303}
304
305#[derive(Debug, Clone, PartialEq, Eq)]
307pub struct Alert {
308 pub severity: Severity,
309 pub text: String,
310}
311
312pub fn alert_key(alert: &Alert) -> String {
316 let mut key = format!("{:?}:", alert.severity);
317 let mut in_number = false;
318 for c in alert.text.chars() {
319 if c.is_ascii_digit() || (in_number && c == '.') {
320 if !in_number {
321 key.push('#');
322 }
323 in_number = true;
324 } else {
325 key.push(c);
326 in_number = false;
327 }
328 }
329 key
330}
331
332#[derive(Debug, Clone, Copy, PartialEq, Eq)]
334pub enum AlertChange {
335 Raised,
336 Cleared,
337}
338
339#[derive(Debug, Clone, PartialEq, Eq)]
341pub struct AlertEvent {
342 pub at_ms: u64,
344 pub host: String,
345 pub change: AlertChange,
346 pub alert: Alert,
348}
349
350const ALERT_LOG_LEN: usize = 500;
352
353#[derive(Debug, Default)]
356pub struct AlertLog {
357 events: VecDeque<AlertEvent>,
359 active: HashMap<String, Vec<Alert>>,
361 unread: (usize, Option<Severity>),
363}
364
365impl AlertLog {
366 pub fn update(&mut self, host: &str, alerts: &[Alert], at_ms: u64) {
368 let previous = self.active.remove(host).unwrap_or_default();
369 let keys = |list: &[Alert]| -> HashSet<String> { list.iter().map(alert_key).collect() };
370 let (before, now) = (keys(&previous), keys(alerts));
371 let raised = alerts.iter().filter(|a| !before.contains(&alert_key(a)));
372 let cleared = previous.iter().filter(|a| !now.contains(&alert_key(a)));
373 let mut changes: Vec<(AlertChange, &Alert)> = raised
374 .map(|a| (AlertChange::Raised, a))
375 .chain(cleared.map(|a| (AlertChange::Cleared, a)))
376 .collect();
377 changes.sort_by_key(|(change, _)| *change == AlertChange::Raised);
379 for (change, alert) in changes {
380 if change == AlertChange::Raised {
381 self.unread.0 += 1;
382 self.unread.1 = self.unread.1.max(Some(alert.severity));
383 }
384 if self.events.len() == ALERT_LOG_LEN {
385 self.events.pop_front();
386 }
387 self.events.push_back(AlertEvent {
388 at_ms,
389 host: host.to_owned(),
390 change,
391 alert: alert.clone(),
392 });
393 }
394 self.active.insert(host.to_owned(), alerts.to_vec());
395 }
396
397 pub fn events(&self) -> impl DoubleEndedIterator<Item = &AlertEvent> {
399 self.events.iter()
400 }
401
402 pub fn unread(&self) -> (usize, Option<Severity>) {
404 self.unread
405 }
406
407 pub fn mark_read(&mut self) {
409 self.unread = (0, None);
410 }
411}
412
413pub fn version_note(snapshot: &Snapshot) -> Option<String> {
416 let version = &snapshot.host.rigtop_version;
417 (!version.is_empty() && version != env!("CARGO_PKG_VERSION"))
418 .then(|| format!("agent {version}"))
419}
420
421pub fn version_hint(snapshot: &Snapshot) -> String {
423 format!(
424 "this host runs rigtop {}, this machine {}: install the same version on both \
425 for every reading",
426 snapshot.host.rigtop_version,
427 env!("CARGO_PKG_VERSION")
428 )
429}
430
431pub fn alerts(snapshot: &Snapshot) -> Vec<Alert> {
433 alerts_with(snapshot, &AlertsConfig::default())
434}
435
436pub fn alerts_with(snapshot: &Snapshot, limits: &AlertsConfig) -> Vec<Alert> {
440 let reached = |value: f64, limit: f64| limit > 0.0 && value >= limit;
442 let mut alerts = Vec::new();
443 let mut add = |severity, text: String| alerts.push(Alert { severity, text });
444 for fs in &snapshot.raid.btrfs {
445 let name = fs
446 .mount_points
447 .first()
448 .map_or(fs.fsid.as_str(), String::as_str);
449 let missing = fs.device_stats.iter().filter(|d| d.missing).count();
450 if missing > 0 {
451 add(
452 Severity::Critical,
453 format!("btrfs {name}: {missing} device(s) missing"),
454 );
455 }
456 let errors: u64 = fs.device_stats.iter().map(|d| d.total_errors()).sum();
457 if errors > 0 {
458 let devices: Vec<String> = fs
460 .device_stats
461 .iter()
462 .filter(|d| d.total_errors() > 0)
463 .map(|d| d.label())
464 .collect();
465 add(
466 Severity::Critical,
467 format!("btrfs {name}: {errors} I/O errors ({})", devices.join(", ")),
468 );
469 }
470 if let Some(scrub) = &fs.scrub {
471 if scrub.uncorrectable > 0 {
472 add(
473 Severity::Critical,
474 format!(
475 "btrfs {name}: scrub found {} uncorrectable errors",
476 scrub.uncorrectable
477 ),
478 );
479 } else if scrub.errors > 0 {
480 add(
481 Severity::Warning,
482 format!("btrfs {name}: scrub corrected {} errors", scrub.corrected),
483 );
484 }
485 }
486 }
487 for md in &snapshot.raid.md {
488 if md.active && md.is_degraded() {
489 add(
490 Severity::Critical,
491 format!("{} ({}) degraded", md.name, md.level),
492 );
493 } else if !md.active {
494 add(Severity::Warning, format!("{} inactive", md.name));
495 }
496 if let Some(sync) = &md.sync {
497 add(
498 Severity::Warning,
499 format!("{} {} {:.0}%", md.name, sync.action, sync.percent),
500 );
501 }
502 }
503 for smart in &snapshot.smart {
504 if let Some(error) = &smart.error {
505 add(
506 Severity::Warning,
507 format!("{}: SMART unreadable ({error})", smart.device),
508 );
509 } else if smart.passed == Some(false) {
510 add(
511 Severity::Critical,
512 format!("{}: SMART failed", smart.device),
513 );
514 } else if smart.needs_attention() {
515 add(
516 Severity::Critical,
517 format!("{}: SMART warning", smart.device),
518 );
519 }
520 }
521 for fs in &snapshot.filesystems {
522 let used = fs.total_bytes.saturating_sub(fs.available_bytes);
523 let percent = used as f64 / fs.total_bytes.max(1) as f64 * 100.0;
524 let name = fs
525 .mount_points
526 .first()
527 .map_or(fs.device.as_str(), String::as_str);
528 if reached(percent, limits.filesystem_critical_percent) {
529 add(Severity::Critical, format!("{name} {percent:.0}% full"));
530 } else if reached(percent, limits.filesystem_warning_percent) {
531 add(Severity::Warning, format!("{name} {percent:.0}% full"));
532 }
533 }
534 let hottest_cpu = snapshot
535 .cpu
536 .temperatures
537 .iter()
538 .map(|t| t.celsius)
539 .fold(f32::NAN, f32::max);
540 if reached(hottest_cpu as f64, limits.cpu_warning_celsius as f64) {
541 add(Severity::Warning, format!("CPU at {hottest_cpu:.0}°C"));
542 }
543 for gpu in &snapshot.gpus {
544 if let Some(t) = gpu
545 .temperature_c
546 .filter(|t| reached(*t as f64, limits.gpu_warning_celsius as f64))
547 {
548 add(Severity::Warning, format!("GPU {} at {t:.0}°C", gpu.index));
549 }
550 }
551 for (device, celsius) in drive_temperatures(snapshot) {
552 if reached(celsius as f64, limits.drive_warning_celsius as f64) {
553 add(Severity::Warning, format!("{device} at {celsius:.0}°C"));
554 }
555 }
556 alerts.sort_by_key(|a| std::cmp::Reverse(a.severity));
558 alerts
559}
560
561fn drive_temperatures(snapshot: &Snapshot) -> Vec<(String, f32)> {
564 let mut drives: Vec<(String, f32)> = snapshot
565 .smart
566 .iter()
567 .filter_map(|s| Some((s.device.clone(), s.temperature_c?)))
568 .collect();
569 for chip in snapshot.sensors.iter().filter(|c| c.name == "nvme") {
570 let device = chip.key.split_once(':').map_or(chip.key.as_str(), |k| k.1);
572 if drives.iter().any(|(d, _)| d == device) {
573 continue;
574 }
575 let composite = chip
576 .temperatures
577 .iter()
578 .find(|t| t.label == "Composite")
579 .or(chip.temperatures.first());
580 if let Some(t) = composite {
581 drives.push((device.to_owned(), t.celsius));
582 }
583 }
584 drives
585}
586
587#[derive(Debug, Clone, Copy)]
589pub struct ProcessQuery<'q> {
590 pub sort_key: SortKey,
592 pub descending: bool,
594 pub filter: &'q str,
596 pub show_kernel_threads: bool,
598}
599
600pub fn process_rows<'a>(snapshot: &'a Snapshot, query: &ProcessQuery<'_>) -> Vec<ProcessRow<'a>> {
602 let mut gpu_usage: HashMap<u32, (Option<f32>, Option<u64>)> = HashMap::new();
604 for gpu in &snapshot.gpus {
605 for p in &gpu.processes {
606 let entry = gpu_usage.entry(p.pid).or_default();
607 if let Some(sm) = p.sm_percent {
608 entry.0 = Some(entry.0.map_or(sm, |v| v.max(sm)));
609 }
610 if let Some(mem) = p.memory_bytes {
611 entry.1 = Some(entry.1.unwrap_or(0) + mem);
612 }
613 }
614 }
615
616 let filter = query.filter.to_lowercase();
617 let mut rows: Vec<ProcessRow<'a>> = snapshot
618 .processes
619 .iter()
620 .filter(|p| query.show_kernel_threads || !p.kernel_thread)
621 .filter(|p| {
622 filter.is_empty()
623 || p.name.to_lowercase().contains(&filter)
624 || p.command.to_lowercase().contains(&filter)
625 || p.user.to_lowercase().contains(&filter)
626 || p.pid.to_string() == filter
627 })
628 .map(|info| {
629 let (gpu_percent, vram_bytes) = gpu_usage.get(&info.pid).copied().unwrap_or_default();
630 ProcessRow {
631 info,
632 gpu_percent,
633 vram_bytes,
634 }
635 })
636 .collect();
637
638 let key = query.sort_key;
639 rows.sort_by(|a, b| {
640 let ordering = match key {
641 SortKey::Pid => a.info.pid.cmp(&b.info.pid),
642 SortKey::Name => a.info.name.to_lowercase().cmp(&b.info.name.to_lowercase()),
643 SortKey::Threads => a.info.threads.cmp(&b.info.threads),
644 SortKey::User => a.info.user.cmp(&b.info.user),
645 SortKey::Memory => a.info.memory_bytes.cmp(&b.info.memory_bytes),
646 SortKey::Cpu => a.info.cpu_percent.total_cmp(&b.info.cpu_percent),
647 SortKey::Gpu => a
648 .gpu_percent
649 .unwrap_or(-1.0)
650 .total_cmp(&b.gpu_percent.unwrap_or(-1.0)),
651 SortKey::Vram => a.vram_bytes.cmp(&b.vram_bytes),
652 }
653 .then(a.info.pid.cmp(&b.info.pid));
655 if query.descending {
656 ordering.reverse()
657 } else {
658 ordering
659 }
660 });
661 rows
662}
663
664pub const LHM_HINT: &str = "temperatures need LibreHardwareMonitor";
667
668pub fn lhm_help(error: &str) -> String {
670 format!(
671 "Windows needs LibreHardwareMonitor for temperatures, fans and CPU power.\n\
672 Start it and turn on Options → Remote Web Server → Run.\n\
673 ([lhm] url = \"\" in the configuration hides this note.)\n\n\
674 {error}"
675 )
676}
677
678pub fn signal_name(signal: ProcessSignal) -> &'static str {
680 match signal {
681 ProcessSignal::Terminate => "SIGTERM",
682 ProcessSignal::Kill => "SIGKILL",
683 }
684}
685
686pub fn is_virtual_interface(name: &str) -> bool {
689 VIRTUAL_IFACE_PREFIXES.iter().any(|p| name.starts_with(p))
690}
691
692pub fn default_interface(snapshot: &Snapshot) -> Option<String> {
694 snapshot
695 .networks
696 .iter()
697 .filter(|n| !is_virtual_interface(&n.name))
698 .max_by_key(|n| n.rx_total_bytes + n.tx_total_bytes)
699 .or_else(|| snapshot.networks.first())
700 .map(|n| n.name.clone())
701}
702
703#[cfg(test)]
704mod tests {
705 use super::*;
706 use crate::snapshot::{GpuProcess, GpuStats, NetworkStats};
707
708 fn process(pid: u32, name: &str, cpu: f32, kernel: bool) -> ProcessInfo {
709 ProcessInfo {
710 pid,
711 name: name.to_owned(),
712 cpu_percent: cpu,
713 kernel_thread: kernel,
714 ..Default::default()
715 }
716 }
717
718 fn query(sort_key: SortKey) -> ProcessQuery<'static> {
719 ProcessQuery {
720 sort_key,
721 descending: sort_key.descending_by_default(),
722 filter: "",
723 show_kernel_threads: false,
724 }
725 }
726
727 #[test]
728 fn history_keeps_newest_samples() {
729 let mut history = History::default();
730 for i in 0..(HISTORY_LEN + 5) {
731 history.push(i as f64);
732 }
733 assert_eq!(
734 history.last(2),
735 vec![(HISTORY_LEN + 3) as f64, (HISTORY_LEN + 4) as f64]
736 );
737 assert_eq!(history.last(HISTORY_LEN * 2).len(), HISTORY_LEN);
738 assert_eq!(history.max_of_last(3), (HISTORY_LEN + 4) as f64);
739 }
740
741 #[test]
742 fn graph_spans_spread_samples_over_the_width() {
743 let second = Duration::from_secs(1);
744 let mut history = History::default();
745 for i in 0..120 {
746 history.push(i as f64);
747 }
748 assert_eq!(
750 history.graph(3, GraphSpan::Fit, second),
751 vec![117.0, 118.0, 119.0]
752 );
753 let minute = history.graph(30, GraphSpan::Secs(60), second);
755 assert_eq!(minute.len(), 30);
756 assert_eq!(minute[0], 60.5);
757 assert_eq!(minute[29], 118.5);
758 let stretched = history.graph(120, GraphSpan::Secs(60), Duration::from_secs(2));
760 assert_eq!(stretched.len(), 120);
761 assert_eq!(&stretched[..2], &[90.0, 90.0]);
762 assert_eq!(&stretched[2..4], &[90.125, 90.375]);
763 assert_eq!(stretched[119], 119.0);
764 assert!(stretched.windows(2).all(|w| w[0] <= w[1]));
765 let hour = history.graph(300, GraphSpan::Secs(3600), second);
767 assert_eq!(hour.len(), 10);
768 assert_eq!(hour[0], 5.5);
769 assert!(
770 History::default()
771 .graph(10, GraphSpan::Secs(60), second)
772 .is_empty()
773 );
774 }
775
776 #[test]
777 fn long_spans_draw_from_the_averages() {
778 let second = Duration::from_secs(1);
779 let mut history = History::default();
780 for i in 0..7200 {
782 history.push(if i < 3600 { 0.0 } else { 100.0 });
783 }
784 let six = history.graph(90, GraphSpan::Secs(6 * 3600), second);
786 assert_eq!(six.len(), 30);
787 assert_eq!((six[0], six[29]), (0.0, 100.0));
788 let hour = history.graph(60, GraphSpan::Secs(3600), second);
790 assert!(hour.iter().all(|v| *v == 100.0));
791 for i in 0..(COARSE_LEN * COARSE_FACTOR + 500) {
793 history.push(i as f64);
794 }
795 assert_eq!(history.coarse.len(), COARSE_LEN);
796 }
797
798 #[test]
799 fn graph_span_choices_cycle() {
800 let labels: Vec<String> = std::iter::successors(Some(GraphSpan::Fit), |s| Some(s.next()))
801 .take(6)
802 .map(GraphSpan::label)
803 .collect();
804 assert_eq!(labels, ["fit", "1m", "5m", "15m", "1h", "6h"]);
805 assert_eq!(GraphSpan::Secs(86_400).label(), "24h");
806 assert_eq!(GraphSpan::Secs(86_400).next(), GraphSpan::Fit);
807 assert_eq!(
808 GraphSpan::from_secs(GraphSpan::Secs(300).as_secs()),
809 GraphSpan::Secs(300)
810 );
811 assert_eq!(GraphSpan::from_secs(0), GraphSpan::Fit);
812 assert_eq!(GraphSpan::Secs(7).next(), GraphSpan::Fit);
814 }
815
816 #[test]
817 fn rows_filter_sort_and_join_gpu_usage() {
818 let snapshot = Snapshot {
819 processes: vec![
820 process(10, "bash", 5.0, false),
821 process(2, "kworker/0:1", 50.0, true),
822 process(30, "python", 1.0, false),
823 ],
824 gpus: vec![GpuStats {
825 processes: vec![GpuProcess {
826 pid: 30,
827 memory_bytes: Some(1 << 30),
828 sm_percent: Some(70.0),
829 }],
830 ..Default::default()
831 }],
832 ..Default::default()
833 };
834 let pids = |q: &ProcessQuery| {
835 process_rows(&snapshot, q)
836 .iter()
837 .map(|r| r.info.pid)
838 .collect::<Vec<_>>()
839 };
840 assert_eq!(pids(&query(SortKey::Cpu)), vec![10, 30]);
841 assert_eq!(pids(&query(SortKey::Vram)), vec![30, 10]);
842 assert_eq!(
843 pids(&ProcessQuery {
844 show_kernel_threads: true,
845 ..query(SortKey::Cpu)
846 }),
847 vec![2, 10, 30]
848 );
849 assert_eq!(
850 pids(&ProcessQuery {
851 filter: "PYT",
852 ..query(SortKey::Pid)
853 }),
854 vec![30]
855 );
856 let rows = process_rows(&snapshot, &query(SortKey::Pid));
857 assert_eq!(rows[1].vram_bytes, Some(1 << 30));
858 assert_eq!(rows[1].gpu_percent, Some(70.0));
859 }
860
861 #[test]
862 fn metric_history_records_and_switches_interfaces() {
863 let snapshot = Snapshot {
864 networks: vec![
865 NetworkStats {
866 name: "enp9s0".to_owned(),
867 rx_total_bytes: 10,
868 rx_bytes_per_sec: 100.0,
869 ..Default::default()
870 },
871 NetworkStats {
872 name: "wlp7s0".to_owned(),
873 tx_bytes_per_sec: 5.0,
874 ..Default::default()
875 },
876 ],
877 gpus: vec![GpuStats {
878 utilization_percent: Some(40.0),
879 ..Default::default()
880 }],
881 ..Default::default()
882 };
883 let mut history = MetricHistory::default();
884 history.record(&snapshot);
885 assert_eq!(history.net_interface.as_deref(), Some("enp9s0"));
886 assert_eq!(history.net_rx.last(1), vec![100.0]);
887 assert_eq!(history.gpus[0].last(1), vec![40.0]);
888 assert_eq!(history.next_interface(&snapshot).as_deref(), Some("wlp7s0"));
889 assert!(history.net_rx.last(10).is_empty());
891 history.record(&snapshot);
892 assert_eq!(history.net_tx.last(1), vec![5.0]);
893 }
894
895 #[test]
896 fn alerts_flag_storage_problems_first() {
897 use crate::snapshot::{
898 BtrfsDeviceStats, BtrfsFilesystem, FilesystemUsage, MdArray, RaidStats, SmartDevice,
899 };
900 let snapshot = Snapshot {
901 filesystems: vec![FilesystemUsage {
902 mount_points: vec!["/data".to_owned()],
903 total_bytes: 100,
904 available_bytes: 8,
905 ..Default::default()
906 }],
907 raid: RaidStats {
908 btrfs: vec![BtrfsFilesystem {
909 mount_points: vec!["/".to_owned()],
910 device_stats: vec![BtrfsDeviceStats {
911 devid: 1,
912 corruption_errs: 2,
913 ..Default::default()
914 }],
915 ..Default::default()
916 }],
917 md: vec![MdArray {
918 name: "md0".to_owned(),
919 active: true,
920 level: "raid1".to_owned(),
921 expected_devices: Some(2),
922 working_devices: Some(1),
923 ..Default::default()
924 }],
925 },
926 smart: vec![SmartDevice {
927 device: "sda".to_owned(),
928 passed: Some(true),
929 ..Default::default()
930 }],
931 ..Default::default()
932 };
933 let alerts = alerts(&snapshot);
934 let texts: Vec<&str> = alerts.iter().map(|a| a.text.as_str()).collect();
935 assert_eq!(
936 texts,
937 vec![
938 "btrfs /: 2 I/O errors (devid 1)",
939 "md0 (raid1) degraded",
940 "/data 92% full"
941 ]
942 );
943 assert_eq!(alerts[0].severity, Severity::Critical);
944 assert_eq!(alerts[2].severity, Severity::Warning);
945 assert!(super::alerts(&Snapshot::default()).is_empty());
946 }
947
948 #[test]
949 fn version_notes_only_for_other_versions() {
950 let mut snapshot = Snapshot::default();
951 assert_eq!(version_note(&snapshot), None);
952 snapshot.host.rigtop_version = env!("CARGO_PKG_VERSION").to_owned();
953 assert_eq!(version_note(&snapshot), None);
954 snapshot.host.rigtop_version = "0.1.0".to_owned();
955 assert_eq!(version_note(&snapshot).as_deref(), Some("agent 0.1.0"));
956 assert!(version_hint(&snapshot).contains("rigtop 0.1.0"));
957 }
958
959 #[test]
960 fn alert_log_records_alerts_appearing_and_clearing() {
961 let alert = |severity, text: &str| Alert {
962 severity,
963 text: text.to_owned(),
964 };
965 let full = alert(Severity::Warning, "/data 91% full");
966 let errors = alert(Severity::Critical, "btrfs /: 2 I/O errors (nvme1n1)");
967 let mut log = AlertLog::default();
968 log.update("nas", std::slice::from_ref(&full), 1_000);
969 log.update(
971 "nas",
972 &[alert(Severity::Warning, "/data 92% full"), errors.clone()],
973 2_000,
974 );
975 log.update("zefat", &[], 2_000);
976 assert_eq!(log.unread(), (2, Some(Severity::Critical)));
977 log.mark_read();
978 log.update("nas", std::slice::from_ref(&errors), 3_000);
979 assert_eq!(log.unread(), (0, None));
980 let summary: Vec<(u64, AlertChange, &str)> = log
981 .events()
982 .map(|e| (e.at_ms, e.change, e.alert.text.as_str()))
983 .collect();
984 assert_eq!(
985 summary,
986 [
987 (1_000, AlertChange::Raised, "/data 91% full"),
988 (
989 2_000,
990 AlertChange::Raised,
991 "btrfs /: 2 I/O errors (nvme1n1)"
992 ),
993 (3_000, AlertChange::Cleared, "/data 92% full"),
994 ]
995 );
996 for i in 0..(ALERT_LOG_LEN as u64 + 10) {
997 log.update(
998 "flappy",
999 &[alert(Severity::Warning, if i % 2 == 0 { "a" } else { "b" })],
1000 i,
1001 );
1002 }
1003 assert_eq!(log.events().count(), ALERT_LOG_LEN);
1004 }
1005
1006 #[test]
1007 fn alert_thresholds_come_from_the_configuration() {
1008 use crate::snapshot::{FilesystemUsage, GpuStats, SensorChip, SmartDevice, Temperature};
1009 let temp = |label: &str, celsius| Temperature {
1010 label: label.to_owned(),
1011 celsius,
1012 ..Default::default()
1013 };
1014 let mut snapshot = Snapshot {
1015 filesystems: vec![FilesystemUsage {
1016 mount_points: vec!["/data".to_owned()],
1017 total_bytes: 100,
1018 available_bytes: 8,
1019 ..Default::default()
1020 }],
1021 gpus: vec![GpuStats {
1022 index: 0,
1023 temperature_c: Some(80.0),
1024 ..Default::default()
1025 }],
1026 smart: vec![SmartDevice {
1027 device: "nvme0".to_owned(),
1028 temperature_c: Some(72.0),
1029 ..Default::default()
1030 }],
1031 sensors: vec![
1033 SensorChip {
1034 key: "nvme:nvme0".to_owned(),
1035 name: "nvme".to_owned(),
1036 temperatures: vec![temp("Composite", 71.0)],
1037 ..Default::default()
1038 },
1039 SensorChip {
1040 key: "nvme:nvme1".to_owned(),
1041 name: "nvme".to_owned(),
1042 temperatures: vec![temp("Sensor 1", 90.0), temp("Composite", 74.0)],
1043 ..Default::default()
1044 },
1045 ],
1046 ..Default::default()
1047 };
1048 snapshot.cpu.temperatures = vec![temp("Tctl", 97.0), temp("Tccd1", 90.0)];
1049 let texts = |limits: &AlertsConfig| -> Vec<String> {
1050 alerts_with(&snapshot, limits)
1051 .into_iter()
1052 .map(|a| a.text)
1053 .collect()
1054 };
1055 assert_eq!(
1056 texts(&AlertsConfig::default()),
1057 [
1058 "/data 92% full",
1059 "CPU at 97°C",
1060 "nvme0 at 72°C",
1061 "nvme1 at 74°C"
1062 ]
1063 );
1064 let custom = AlertsConfig {
1065 filesystem_warning_percent: 0.0,
1066 cpu_warning_celsius: 0.0,
1067 gpu_warning_celsius: 75.0,
1068 drive_warning_celsius: 73.0,
1069 ..Default::default()
1070 };
1071 assert_eq!(texts(&custom), ["GPU 0 at 80°C", "nvme1 at 74°C"]);
1072 let critical = AlertsConfig {
1073 filesystem_critical_percent: 90.0,
1074 ..custom
1075 };
1076 assert_eq!(
1077 alerts_with(&snapshot, &critical)[0],
1078 Alert {
1079 severity: Severity::Critical,
1080 text: "/data 92% full".to_owned()
1081 }
1082 );
1083 }
1084
1085 #[test]
1086 fn interface_helpers() {
1087 assert!(is_virtual_interface("veth5a74356"));
1088 assert!(!is_virtual_interface("enp9s0"));
1089 let snapshot = Snapshot {
1090 networks: vec![
1091 NetworkStats {
1092 name: "docker0".to_owned(),
1093 rx_total_bytes: 1 << 40,
1094 ..Default::default()
1095 },
1096 NetworkStats {
1097 name: "enp9s0".to_owned(),
1098 rx_total_bytes: 1 << 30,
1099 ..Default::default()
1100 },
1101 ],
1102 ..Default::default()
1103 };
1104 assert_eq!(default_interface(&snapshot).as_deref(), Some("enp9s0"));
1105 assert_eq!(signal_name(ProcessSignal::Kill), "SIGKILL");
1106 }
1107}