Skip to main content

rigtop_core/
snapshot.rs

1//! Serializable snapshot of everything rigtop displays.
2//!
3//! A [`Snapshot`] is the single unit of data exchanged between a data source
4//! (local collectors, or a remote `rigtop --agent` over SSH) and a front-end
5//! (TUI or GUI). Every type here derives `Serialize`/`Deserialize` so that the
6//! same structure can be streamed over the wire unchanged.
7//!
8//! Conventions:
9//! - Sizes are in bytes, rates in bytes per second.
10//! - Temperatures are in degrees Celsius, power in watts, fan speed in RPM.
11//! - Percentages are in the range `0.0..=100.0` (per-core values are not summed).
12//! - Values that a platform or device cannot provide are `None` (or an empty
13//!   list) rather than zero, so that front-ends can hide them.
14//! - Every struct is `#[serde(default)]`: fields missing from the input take
15//!   their default, so a client can read snapshots from an older or newer
16//!   `rigtop --agent` as long as fields are only added.
17
18use serde::{Deserialize, Serialize};
19
20/// One full measurement of a host.
21#[derive(Debug, Clone, Default, Serialize, Deserialize)]
22#[serde(default)]
23pub struct Snapshot {
24    /// Wall-clock time of the measurement, in milliseconds since the Unix epoch.
25    pub timestamp_ms: u64,
26    /// Static-ish information about the host.
27    pub host: HostInfo,
28    /// CPU load, frequency and temperature.
29    pub cpu: CpuStats,
30    /// Physical memory and swap.
31    pub memory: MemoryStats,
32    /// Discrete and integrated GPUs that could be queried.
33    pub gpus: Vec<GpuStats>,
34    /// Raw hardware sensor chips (temperatures and fans), grouped by chip.
35    pub sensors: Vec<SensorChip>,
36    /// Power draw summary.
37    pub power: PowerStats,
38    /// I/O activity per physical block device.
39    pub disks: Vec<DiskIo>,
40    /// Capacity of mounted filesystems.
41    pub filesystems: Vec<FilesystemUsage>,
42    /// Software RAID state (btrfs and mdadm).
43    pub raid: RaidStats,
44    /// SMART health per drive, from files written by a privileged timer.
45    pub smart: Vec<SmartDevice>,
46    /// Network interface traffic.
47    pub networks: Vec<NetworkStats>,
48    /// Process list. May be truncated by the source (see [`Snapshot::truncate_processes`]).
49    pub processes: Vec<ProcessInfo>,
50    /// Total number of processes on the host, even if `processes` was truncated.
51    pub process_count: usize,
52    /// Processes the client asked to hear about when they exit (see
53    /// [`crate::DataSource::watch_processes`]): listed in `processes`
54    /// whatever the limit while they run, so one of them missing has
55    /// exited. Empty from agents before 0.19.
56    pub watched_pids: Vec<u32>,
57    /// Linux: systemd units in the failed state (e.g. `"nginx.service"`).
58    pub failed_units: Vec<String>,
59    /// Remote hosts: the round trip of the link to the agent (milliseconds),
60    /// measured by the client; agents leave it out.
61    pub link_rtt_ms: Option<f32>,
62    /// The host's public IPv4 address as an outside service sees it, when
63    /// the client asked for it (see [`crate::DataSource::set_public_ip`]).
64    pub public_ip: Option<String>,
65    /// Linux: how much of the time tasks waited for CPU, memory or I/O
66    /// (pressure stall information). `None` without it (other platforms,
67    /// kernels built or booted without PSI, agents before 0.21).
68    pub pressure: Option<Pressure>,
69    /// Windows: why LibreHardwareMonitor, the source of temperatures, fans
70    /// and CPU power, could not be read. `None` when it was read, when it is
71    /// disabled in the configuration, and on other platforms.
72    pub lhm_error: Option<String>,
73    /// UPSes known to Network UPS Tools' upsd at the configured address
74    /// (`[ups] address`). Empty without NUT, and from agents before 0.30.
75    pub ups: Vec<UpsStats>,
76}
77
78impl Snapshot {
79    /// Keeps only the `limit` processes with the highest CPU usage.
80    ///
81    /// Used to reduce the payload when a host is shown in an overview rather
82    /// than in detail. Kernel threads are dropped first: the overview does not
83    /// show them, and busy `kworker`s would otherwise take places in the top
84    /// `limit` and leave the list short. `process_count` keeps the original
85    /// total. Processes in `watched_pids` stay, after the top `limit`.
86    pub fn truncate_processes(&mut self, limit: usize) {
87        self.processes.retain(|p| !p.kernel_thread);
88        if self.processes.len() <= limit {
89            return;
90        }
91        let (watched, mut others): (Vec<ProcessInfo>, Vec<ProcessInfo>) =
92            std::mem::take(&mut self.processes)
93                .into_iter()
94                .partition(|p| self.watched_pids.contains(&p.pid));
95        if others.len() > limit {
96            // Partial sort is enough: only the top `limit` entries matter.
97            others.select_nth_unstable_by(limit, |a, b| b.cpu_percent.total_cmp(&a.cpu_percent));
98            others.truncate(limit);
99        }
100        others.sort_unstable_by(|a, b| b.cpu_percent.total_cmp(&a.cpu_percent));
101        others.extend(watched);
102        self.processes = others;
103    }
104}
105
106/// Host identification and uptime.
107#[derive(Debug, Clone, Default, Serialize, Deserialize)]
108#[serde(default)]
109pub struct HostInfo {
110    /// Host name as reported by the OS.
111    pub hostname: String,
112    /// Human readable OS name and version (e.g. "Linux 26.04 Ubuntu").
113    pub os: String,
114    /// Kernel version string.
115    pub kernel: String,
116    /// Seconds since boot.
117    pub uptime_secs: u64,
118    /// Version of the rigtop that took the snapshot (the agent's, for a
119    /// remote host). Agents before 0.9.1 leave it out; the remote source
120    /// fills it in from the agent's greeting.
121    pub rigtop_version: String,
122    /// A reboot is needed to finish an update (Linux), with the reason
123    /// (possibly empty), e.g. "updated: linux-image-7.0.0-39-generic".
124    pub reboot_required: Option<String>,
125}
126
127/// CPU-wide and per-core statistics.
128#[derive(Debug, Clone, Default, Serialize, Deserialize)]
129#[serde(default)]
130pub struct CpuStats {
131    /// Brand string of the first CPU (e.g. "AMD Ryzen 9 9950X3D 16-Core Processor").
132    pub model: String,
133    /// Usage across all logical cores.
134    pub total_percent: f32,
135    /// Usage of each logical core, in OS order.
136    pub per_core_percent: Vec<f32>,
137    /// Current frequency of each logical core in MHz.
138    pub per_core_mhz: Vec<u64>,
139    /// 1, 5 and 15 minute load averages; `None` on platforms without them
140    /// (Windows).
141    pub load_average: Option<[f64; 3]>,
142    /// CPU temperature readings, e.g. `Tctl`, `Tccd1`, `Tccd2` on AMD or
143    /// `Package id 0` on Intel. Per-core readings that could be matched to
144    /// logical cores are in [`CpuStats::per_core_celsius`] instead.
145    pub temperatures: Vec<Temperature>,
146    /// Temperature of the physical core of each logical core, in OS order
147    /// (Intel's coretemp). Empty when the CPU has no per-core sensors.
148    pub per_core_celsius: Vec<Option<f32>>,
149}
150
151/// Physical memory and swap usage.
152#[derive(Debug, Clone, Default, Serialize, Deserialize)]
153#[serde(default)]
154pub struct MemoryStats {
155    /// Installed (visible) memory.
156    pub total_bytes: u64,
157    /// Memory in use by processes (total minus available).
158    pub used_bytes: u64,
159    /// Memory that can be allocated without swapping.
160    pub available_bytes: u64,
161    /// Completely unused memory.
162    pub free_bytes: u64,
163    /// Page cache and reclaimable slab, where the platform reports it.
164    pub cached_bytes: Option<u64>,
165    /// Total swap space.
166    pub swap_total_bytes: u64,
167    /// Swap space in use.
168    pub swap_used_bytes: u64,
169    /// Processes the kernel's OOM killer stopped in the last hour, seen
170    /// since rigtop started (Linux), oldest first.
171    pub oom_kills: Vec<OomKill>,
172}
173
174/// Pressure stall information: the share of time tasks were stalled
175/// waiting for a resource, from `/proc/pressure/`.
176#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
177#[serde(default)]
178pub struct Pressure {
179    pub cpu: PressureLevels,
180    pub memory: PressureLevels,
181    pub io: PressureLevels,
182}
183
184/// Stall percentages of one resource. `some`: at least one task waited
185/// (others may have run); `full`: every task that wanted to run waited, so
186/// the time was lost. The kernel's 10- and 60-second averages.
187#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
188#[serde(default)]
189pub struct PressureLevels {
190    pub some_avg10: f32,
191    pub some_avg60: f32,
192    pub full_avg10: f32,
193    pub full_avg60: f32,
194}
195
196/// One process stopped by the kernel for lack of memory.
197#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
198#[serde(default)]
199pub struct OomKill {
200    /// When it was noticed (milliseconds since the Unix epoch).
201    pub at_ms: u64,
202    /// Name of the process that most likely was stopped: the largest one
203    /// that went away at the same time (the kernel log saying which needs
204    /// root).
205    pub likely_process: Option<String>,
206}
207
208/// A single GPU.
209#[derive(Debug, Clone, Default, Serialize, Deserialize)]
210#[serde(default)]
211pub struct GpuStats {
212    /// Position in the GPU list: NVIDIA GPUs first (their NVML index),
213    /// then AMD and Intel ones.
214    pub index: u32,
215    /// Vendor name ("NVIDIA", "AMD", ...).
216    pub vendor: String,
217    /// Marketing name of the device.
218    pub name: String,
219    /// Integrated graphics (an APU or Intel iGPU), whose power is part of
220    /// the CPU package and not reported here.
221    pub integrated: bool,
222    /// Core utilization (unknown for Intel without privileges).
223    pub utilization_percent: Option<f32>,
224    /// Memory controller utilization.
225    pub memory_utilization_percent: Option<f32>,
226    /// VRAM in use.
227    pub memory_used_bytes: Option<u64>,
228    /// Total VRAM.
229    pub memory_total_bytes: Option<u64>,
230    /// Core temperature.
231    pub temperature_c: Option<f32>,
232    /// Current board power draw.
233    pub power_w: Option<f32>,
234    /// Enforced power limit.
235    pub power_limit_w: Option<f32>,
236    /// Fan speed as a percentage of the maximum (first fan).
237    pub fan_percent: Option<f32>,
238    /// Graphics clock in MHz.
239    pub clock_mhz: Option<u32>,
240    /// Why the clocks are held down right now (NVIDIA): "power cap",
241    /// "thermal", "hw thermal", "power brake" or "hw slowdown". Empty when
242    /// the GPU runs freely or idles. Text, so that newer agents can add
243    /// reasons older clients do not know.
244    pub throttle_reasons: Vec<String>,
245    /// ECC memory errors since the driver loaded (NVIDIA, with ECC on).
246    pub ecc: Option<EccErrors>,
247    /// The PCIe link (NVIDIA).
248    pub pcie: Option<PcieLink>,
249    /// Processes currently using this GPU.
250    pub processes: Vec<GpuProcess>,
251}
252
253impl GpuStats {
254    /// Throttle reasons that mean the hardware is slowing the GPU down, as
255    /// opposed to running at its power limit as designed.
256    pub fn slowdown_reasons(&self) -> impl Iterator<Item = &str> {
257        self.throttle_reasons
258            .iter()
259            .map(String::as_str)
260            .filter(|r| *r != "power cap")
261    }
262}
263
264/// ECC memory error counts of a GPU.
265#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
266#[serde(default)]
267pub struct EccErrors {
268    /// Errors the ECC corrected.
269    pub corrected: u64,
270    /// Errors it could not correct (data was corrupted).
271    pub uncorrected: u64,
272}
273
274/// A GPU's PCIe link: what it runs at now (lower when idle, to save power)
275/// and the most the GPU and its slot allow.
276#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
277#[serde(default)]
278pub struct PcieLink {
279    pub generation: u32,
280    pub width: u32,
281    pub max_generation: u32,
282    pub max_width: u32,
283}
284
285/// GPU usage attributed to one process.
286#[derive(Debug, Clone, Default, Serialize, Deserialize)]
287#[serde(default)]
288pub struct GpuProcess {
289    /// OS process id.
290    pub pid: u32,
291    /// VRAM used by the process, if the driver reports it.
292    pub memory_bytes: Option<u64>,
293    /// Streaming multiprocessor (core) utilization by the process.
294    pub sm_percent: Option<f32>,
295}
296
297/// A hardware monitoring chip (a Linux hwmon device) and its readings.
298#[derive(Debug, Clone, Default, Serialize, Deserialize)]
299#[serde(default)]
300pub struct SensorChip {
301    /// Stable identifier, `"<driver name>:<device name>"`
302    /// (e.g. `"nct6686:nct6683.2592"`, `"nvme:nvme0"`).
303    ///
304    /// Unlike `hwmonN`, this survives reboots, so it is used as the key for
305    /// user-defined labels.
306    pub key: String,
307    /// Driver-provided chip name (e.g. `"k10temp"`, `"nvme"`, `"spd5118"`).
308    pub name: String,
309    /// Temperature channels.
310    pub temperatures: Vec<Temperature>,
311    /// Fan channels.
312    pub fans: Vec<Fan>,
313}
314
315/// One temperature channel.
316#[derive(Debug, Clone, Default, Serialize, Deserialize)]
317#[serde(default)]
318pub struct Temperature {
319    /// Stable key `"<chip key>/temp<N>"` used to look up user-defined names.
320    /// Empty for readings that do not come from a hwmon chip.
321    pub key: String,
322    /// Driver label (e.g. `"Tccd1"`), or a generic `"temp<N>"` when none exists.
323    pub label: String,
324    /// Current reading.
325    pub celsius: f32,
326    /// High threshold, if exposed.
327    pub max_celsius: Option<f32>,
328    /// Critical threshold, if exposed.
329    pub critical_celsius: Option<f32>,
330}
331
332/// One fan channel.
333#[derive(Debug, Clone, Default, Serialize, Deserialize)]
334#[serde(default)]
335pub struct Fan {
336    /// Stable key `"<chip key>/fan<N>"` used to look up user-defined names.
337    pub key: String,
338    /// Driver label, or a generic `"fan<N>"` when none exists.
339    pub label: String,
340    /// Current speed (0 when stopped or not connected).
341    pub rpm: u32,
342}
343
344/// Power draw summary.
345///
346/// There is no portable way to measure whole-system power without external
347/// hardware, so the total is the sum of the CPU package and discrete GPUs,
348/// which dominate consumption on typical workstations.
349#[derive(Debug, Clone, Default, Serialize, Deserialize)]
350#[serde(default)]
351pub struct PowerStats {
352    /// CPU package power (RAPL on Linux). `None` before the second sample or
353    /// when RAPL is unavailable.
354    pub cpu_package_w: Option<f32>,
355    /// Sum of power reported by discrete GPUs.
356    pub gpus_w: Option<f32>,
357    /// Power an AMD APU's graphics driver reports (amdgpu `PPT`). Per the
358    /// kernel documentation it covers the whole SoC, CPU cores included, so
359    /// it overlaps the CPU package and only stands in for it when RAPL cannot
360    /// be read (see [`PowerStats::cpu_or_soc_w`]).
361    pub igpu_w: Option<f32>,
362    /// `cpu_package_w` (or `igpu_w` without it) `+ gpus_w`, counting only the
363    /// parts that are available.
364    pub total_w: Option<f32>,
365    /// Laptop battery, if there is one.
366    pub battery: Option<BatteryStats>,
367}
368
369/// Below this, an APU reading is not whole-SoC power: desktop Ryzen CPUs
370/// with a small iGPU report only a few milliwatts there, while a laptop
371/// APU's SoC never idles under a watt.
372const MIN_SOC_W: f32 = 1.0;
373
374impl PowerStats {
375    /// The CPU figure to show: `("CPU", RAPL)`, else `("SoC", APU PPT)`.
376    pub fn cpu_or_soc_w(&self) -> (&'static str, Option<f32>) {
377        match (self.cpu_package_w, self.igpu_w) {
378            (Some(w), _) => ("CPU", Some(w)),
379            (None, Some(w)) if w >= MIN_SOC_W => ("SoC", Some(w)),
380            _ => ("CPU", None),
381        }
382    }
383}
384
385/// Charge and power of the system's batteries (summed when there are
386/// several), from `/sys/class/power_supply` on Linux.
387#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
388#[serde(default)]
389pub struct BatteryStats {
390    /// Charge in percent.
391    pub percent: Option<f32>,
392    /// Kernel status: `Charging`, `Discharging`, `Full`, `Not charging`...
393    pub status: String,
394    /// Charge or discharge power (not signed; see `status`).
395    pub power_w: Option<f32>,
396    /// Until empty while discharging, until full while charging, at the
397    /// recent (smoothed) rate.
398    pub time_left_secs: Option<u64>,
399    /// Full charge as a share of the design capacity (wear), in percent.
400    pub health_percent: Option<f32>,
401}
402
403/// A UPS, as Network UPS Tools' `upsd` reports it.
404#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
405#[serde(default)]
406pub struct UpsStats {
407    /// upsd's name for it (`ups` in `upsc ups@localhost`).
408    pub name: String,
409    /// Maker and model (`APC Back-UPS ES 700`), else its description in
410    /// NUT's `ups.conf`.
411    pub model: Option<String>,
412    /// The flags of `ups.status`: `OL` on line power, `OB` on battery, `LB`
413    /// low battery, `CHRG` charging, `RB` replace the battery, `OVER`
414    /// overloaded, `BYPASS`, `OFF`, `FSD` forced shutdown...
415    pub status: Vec<String>,
416    /// Battery charge in percent.
417    pub charge_percent: Option<f32>,
418    /// How long the battery would last at the present load.
419    pub runtime_secs: Option<u64>,
420    /// Load in percent of the UPS's capacity.
421    pub load_percent: Option<f32>,
422    /// Power drawn by what is plugged in: `ups.realpower`, else the load
423    /// times `ups.realpower.nominal`.
424    pub power_w: Option<f32>,
425    /// Mains voltage.
426    pub input_voltage: Option<f32>,
427    /// Why there are no readings (`data stale`, `driver not connected`,
428    /// `upsd not reachable (...)`).
429    pub error: Option<String>,
430}
431
432impl UpsStats {
433    /// Whether `ups.status` has `flag` (`OB`).
434    pub fn has(&self, flag: &str) -> bool {
435        self.status.iter().any(|s| s == flag)
436    }
437
438    /// "UPS" for the usual one named `ups`, else "UPS <name>".
439    pub fn label(&self) -> String {
440        if self.name.eq_ignore_ascii_case("ups") || self.name.is_empty() {
441            "UPS".to_owned()
442        } else {
443            format!("UPS {}", self.name)
444        }
445    }
446
447    /// Short form for status lines: `100% 73W` on line power, `on battery
448    /// 72% 25m` during a power failure, `no data` without readings.
449    pub fn summary(&self) -> String {
450        if self.error.is_some() {
451            return "no data".to_owned();
452        }
453        let mut parts = Vec::new();
454        if self.has("OB") {
455            parts.push("on battery".to_owned());
456        }
457        if let Some(charge) = self.charge_percent {
458            parts.push(format!("{charge:.0}%"));
459        }
460        if self.has("OB") {
461            parts.extend(self.runtime_secs.map(crate::format::hours_minutes));
462        } else if let Some(power) = self.power_w {
463            parts.push(format!("{power:.0}W"));
464        } else if let Some(load) = self.load_percent {
465            parts.push(format!("load {load:.0}%"));
466        }
467        if parts.is_empty() {
468            parts.push(self.status.join(" "));
469        }
470        parts.join(" ")
471    }
472
473    /// Hover text: model, state, battery, load and mains voltage.
474    pub fn details(&self) -> String {
475        let mut lines = vec![match &self.model {
476            Some(model) => format!("{} ({model})", self.label()),
477            None => self.label(),
478        }];
479        if let Some(error) = &self.error {
480            lines.push(format!("no data: {error}"));
481            return lines.join("\n");
482        }
483        let state: Vec<&str> = self
484            .status
485            .iter()
486            .map(|flag| match flag.as_str() {
487                "OL" => "on line power",
488                "OB" => "on battery",
489                "LB" => "battery low",
490                "HB" => "battery high",
491                "RB" => "replace the battery",
492                "CHRG" => "charging",
493                "DISCHRG" => "discharging",
494                "BYPASS" => "on bypass",
495                "CAL" => "calibrating",
496                "OFF" => "output off",
497                "OVER" => "overloaded",
498                "TRIM" => "trimming high mains voltage",
499                "BOOST" => "boosting low mains voltage",
500                "FSD" => "forced shutdown",
501                other => other,
502            })
503            .collect();
504        if !state.is_empty() {
505            lines.push(state.join(", "));
506        }
507        let mut battery = Vec::new();
508        if let Some(charge) = self.charge_percent {
509            battery.push(format!("{charge:.0}%"));
510        }
511        if let Some(secs) = self.runtime_secs {
512            battery.push(format!(
513                "{} at this load",
514                crate::format::hours_minutes(secs)
515            ));
516        }
517        if !battery.is_empty() {
518            lines.push(format!("battery {}", battery.join(", ")));
519        }
520        let mut load = Vec::new();
521        if let Some(percent) = self.load_percent {
522            load.push(format!("{percent:.0}%"));
523        }
524        if let Some(power) = self.power_w {
525            load.push(format!("{power:.0} W"));
526        }
527        if !load.is_empty() {
528            lines.push(format!("load {}", load.join(", ")));
529        }
530        if let Some(volts) = self.input_voltage {
531            lines.push(format!("mains {volts:.0} V"));
532        }
533        lines.join("\n")
534    }
535}
536
537impl BatteryStats {
538    /// Short form for status lines: `72% -12W 3h10m` while discharging
539    /// (with the time left when known), `+20W` while charging.
540    pub fn summary(&self) -> String {
541        let percent = self
542            .percent
543            .map_or("--%".to_owned(), |p| format!("{p:.0}%"));
544        let sign = match self.status.as_str() {
545            "Discharging" => "-",
546            "Charging" => "+",
547            _ => return percent,
548        };
549        let mut text = match self.power_w {
550            Some(w) if w >= 0.5 => format!("{percent} {sign}{w:.0}W"),
551            _ => percent,
552        };
553        if let Some(secs) = self.time_left_secs {
554            text.push(' ');
555            text.push_str(&crate::format::hours_minutes(secs));
556        }
557        text
558    }
559
560    /// The health worth showing next to the charge: below 80% of the design
561    /// capacity, where batteries are usually considered worn.
562    pub fn health_note(&self) -> Option<String> {
563        self.health_percent
564            .filter(|h| *h < 80.0)
565            .map(|h| format!("health {h:.0}%"))
566    }
567
568    /// Hover text: what the time means, and the health.
569    pub fn details(&self) -> String {
570        let mut lines = Vec::new();
571        if let Some(secs) = self.time_left_secs {
572            let until = if self.status == "Charging" {
573                "full"
574            } else {
575                "empty"
576            };
577            lines.push(format!(
578                "{} until {until} at the recent rate",
579                crate::format::hours_minutes(secs)
580            ));
581        }
582        if let Some(health) = self.health_percent {
583            lines.push(format!("full charge {health:.0}% of the design capacity"));
584        }
585        if lines.is_empty() {
586            lines.push(self.status.clone());
587        }
588        lines.join("\n")
589    }
590}
591
592/// I/O activity of a block device.
593#[derive(Debug, Clone, Default, Serialize, Deserialize)]
594#[serde(default)]
595pub struct DiskIo {
596    /// Kernel device name (e.g. `"nvme0n1"`, `"sda"`).
597    pub name: String,
598    /// Read throughput.
599    pub read_bytes_per_sec: f64,
600    /// Write throughput.
601    pub write_bytes_per_sec: f64,
602    /// Fraction of time the device had I/O in flight, where the platform
603    /// reports it (Linux).
604    pub busy_percent: Option<f32>,
605    /// Average time one request took (queue and service), over the requests
606    /// completed since the last sample (Linux). `None` without any.
607    pub latency_ms: Option<f32>,
608    /// Drive temperature (NVMe composite sensor), if available.
609    pub temperature_c: Option<f32>,
610}
611
612/// Capacity of one filesystem.
613///
614/// Filesystems mounted at several places (e.g. btrfs subvolumes) appear once,
615/// with all mount points listed.
616#[derive(Debug, Clone, Default, Serialize, Deserialize)]
617#[serde(default)]
618pub struct FilesystemUsage {
619    /// Source device (e.g. `"/dev/nvme1n1p1"`).
620    pub device: String,
621    /// Filesystem type (e.g. `"btrfs"`, `"ext4"`).
622    pub fs_type: String,
623    /// All mount points of this filesystem, shortest first.
624    pub mount_points: Vec<String>,
625    /// Total size.
626    pub total_bytes: u64,
627    /// Space available to unprivileged users.
628    pub available_bytes: u64,
629    /// Seconds until it is full at the growth of the last half hour; `None`
630    /// when usage does not grow, without enough history yet, and from
631    /// agents before 0.19 (see [`crate::fill`]).
632    pub full_in_secs: Option<u64>,
633}
634
635/// Returns true if `device` (a disk such as `"sda"`, or an NVMe controller
636/// such as `"nvme0"`) is the drive behind the block device `disk`
637/// (`"sda"`, `"nvme0n1"`).
638///
639/// `"nvme1"` does not match `"nvme10n1"`: after the controller name only the
640/// namespace suffix (`n<N>`) may follow.
641pub fn device_matches_disk(device: &str, disk: &str) -> bool {
642    match disk.strip_prefix(device) {
643        Some("") => true,
644        Some(rest) => device.starts_with("nvme") && rest.starts_with('n'),
645        None => false,
646    }
647}
648
649/// SMART health of one drive, parsed from `smartctl -j -a` output.
650///
651/// Fields a drive or protocol does not report are `None` (NVMe drives have
652/// no sector reallocation counters, ATA drives no "percentage used").
653#[derive(Debug, Clone, Default, Serialize, Deserialize)]
654#[serde(default)]
655pub struct SmartDevice {
656    /// Kernel device name (`"nvme0"`, `"sda"`). See [`device_matches_disk`].
657    pub device: String,
658    /// Protocol reported by smartctl (`"NVMe"`, `"ATA"`, `"SCSI"`).
659    pub protocol: String,
660    /// Drive model.
661    pub model: String,
662    /// Overall health self-assessment; `None` if smartctl could not tell.
663    pub passed: Option<bool>,
664    /// Current drive temperature.
665    pub temperature_c: Option<f32>,
666    /// Accumulated power-on time.
667    pub power_on_hours: Option<u64>,
668    /// NVMe wear estimate (may exceed 100).
669    pub percentage_used: Option<u32>,
670    /// NVMe spare capacity left.
671    pub available_spare_percent: Option<u32>,
672    /// NVMe spare threshold below which the drive reports a warning.
673    pub available_spare_threshold_percent: Option<u32>,
674    /// NVMe critical warning bit field (0 when healthy).
675    pub critical_warning: Option<u32>,
676    /// NVMe unrecovered data integrity errors.
677    pub media_errors: Option<u64>,
678    /// ATA attribute 5 (Reallocated_Sector_Ct), raw value.
679    pub reallocated_sectors: Option<u64>,
680    /// ATA attribute 197 (Current_Pending_Sector), raw value.
681    pub pending_sectors: Option<u64>,
682    /// ATA attribute 198 (Offline_Uncorrectable), raw value.
683    pub offline_uncorrectable: Option<u64>,
684    /// Total host writes, where the drive reports them.
685    pub data_written_bytes: Option<u64>,
686    /// When smartctl ran, in milliseconds since the Unix epoch. Lets front-ends
687    /// flag stale results (e.g. the timer stopped running).
688    pub collected_at_ms: Option<u64>,
689    /// Error reported by smartctl (e.g. the device could not be opened).
690    pub error: Option<String>,
691}
692
693impl SmartDevice {
694    /// Returns true if anything in the SMART data calls for attention.
695    pub fn needs_attention(&self) -> bool {
696        let positive = |v: Option<u64>| v.is_some_and(|v| v > 0);
697        let spare_low = match (
698            self.available_spare_percent,
699            self.available_spare_threshold_percent,
700        ) {
701            (Some(spare), Some(threshold)) => spare < threshold,
702            _ => false,
703        };
704        self.passed == Some(false)
705            || self.error.is_some()
706            || self.critical_warning.is_some_and(|w| w != 0)
707            || positive(self.media_errors)
708            || positive(self.reallocated_sectors)
709            || positive(self.pending_sectors)
710            || positive(self.offline_uncorrectable)
711            || self.percentage_used.is_some_and(|p| p >= 100)
712            || spare_low
713    }
714}
715
716/// Software RAID state.
717#[derive(Debug, Clone, Default, Serialize, Deserialize)]
718#[serde(default)]
719pub struct RaidStats {
720    /// Mounted btrfs filesystems.
721    pub btrfs: Vec<BtrfsFilesystem>,
722    /// Linux md (mdadm) arrays.
723    pub md: Vec<MdArray>,
724    /// Imported ZFS pools.
725    pub zfs: Vec<ZfsPool>,
726}
727
728/// A ZFS pool and its health.
729#[derive(Debug, Clone, Default, Serialize, Deserialize)]
730#[serde(default)]
731pub struct ZfsPool {
732    /// Pool name (e.g. `"tank"`).
733    pub name: String,
734    /// Health as `zpool status` shows it: `ONLINE`, `DEGRADED`, `FAULTED`,
735    /// `OFFLINE`, `UNAVAIL`, `REMOVED` or `SUSPENDED`.
736    pub state: String,
737    /// Its vdevs and devices as `zpool status` lists them, top to bottom
738    /// (empty where `zpool` could not be run, and from agents before 0.26).
739    pub devices: Vec<ZfsDevice>,
740    /// The last or running scrub or resilver ("scrub repaired 0B in
741    /// 00:00:01 with 0 errors on ...", "resilver in progress ... 42.1%
742    /// done"), from the `scan:` line.
743    pub scan: Option<String>,
744    /// Permanent data errors (`errors: N data errors`); `None` unknown.
745    pub data_errors: Option<u64>,
746    /// Size and space in use (`zpool list`).
747    pub size_bytes: Option<u64>,
748    pub allocated_bytes: Option<u64>,
749}
750
751/// One row of a pool's configuration in `zpool status`: a vdev (raidz1-0,
752/// mirror-0) or a device under one.
753#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
754#[serde(default)]
755pub struct ZfsDevice {
756    /// As listed: "raidz1-0", "mirror-1", "ata-WDC_WD80...", "/var/tmp/log1".
757    pub name: String,
758    /// Nesting under the pool: 0 for top-level vdevs, 1 for their devices.
759    pub depth: u8,
760    /// The section it is in: "" for data, else "logs", "cache", "spares",
761    /// "special" or "dedup".
762    pub section: String,
763    /// ONLINE, DEGRADED, OFFLINE, FAULTED, UNAVAIL, REMOVED; AVAIL or
764    /// INUSE for spares.
765    pub state: String,
766    /// Read, write and checksum errors since the last `zpool clear`.
767    pub read_errors: u64,
768    pub write_errors: u64,
769    pub checksum_errors: u64,
770}
771
772impl ZfsDevice {
773    /// All its errors.
774    pub fn errors(&self) -> u64 {
775        self.read_errors + self.write_errors + self.checksum_errors
776    }
777
778    /// Whether its state needs attention (spares waiting are fine).
779    pub fn is_unhealthy(&self) -> bool {
780        !matches!(self.state.as_str(), "ONLINE" | "AVAIL" | "INUSE" | "")
781    }
782
783    /// Whether it is a vdev grouping devices (raidz, mirror, ...) rather
784    /// than a device.
785    pub fn is_group(&self) -> bool {
786        ["raidz", "mirror", "draid", "replacing", "spare-"]
787            .iter()
788            .any(|kind| self.name.starts_with(kind))
789    }
790}
791
792impl ZfsPool {
793    /// The layout of its data vdevs: "raidz1", "mirror", "2 × mirror",
794    /// "stripe" (devices directly under the pool), or `None` when unknown.
795    pub fn layout(&self) -> Option<String> {
796        let top: Vec<&ZfsDevice> = self
797            .devices
798            .iter()
799            .filter(|d| d.depth == 0 && d.section.is_empty())
800            .collect();
801        let first = top.first()?;
802        if !first.is_group() {
803            return Some("stripe".to_owned());
804        }
805        let kind = first
806            .name
807            .rsplit_once('-')
808            .map_or(first.name.as_str(), |(k, _)| k);
809        let same = top.iter().filter(|d| d.name.starts_with(kind)).count();
810        Some(if top.len() > 1 && same == top.len() {
811            format!("{} × {kind}", top.len())
812        } else {
813            kind.to_owned()
814        })
815    }
816
817    /// Whether `zpool status` reported data errors or device errors, which
818    /// a healthy-looking ONLINE state does not show.
819    pub fn has_errors(&self) -> bool {
820        self.data_errors.unwrap_or(0) > 0 || self.devices.iter().any(|d| d.errors() > 0)
821    }
822
823    /// Its devices (not vdevs) with errors or in a bad state.
824    pub fn troubled_devices(&self) -> impl Iterator<Item = &ZfsDevice> {
825        self.devices
826            .iter()
827            .filter(|d| !d.is_group() && (d.errors() > 0 || d.is_unhealthy()))
828    }
829
830    /// Whether the pool is healthy.
831    pub fn is_online(&self) -> bool {
832        self.state == "ONLINE"
833    }
834
835    /// Whether the pool lost redundancy or data is at risk (worse than an
836    /// administratively offline pool).
837    pub fn is_failing(&self) -> bool {
838        matches!(
839            self.state.as_str(),
840            "DEGRADED" | "FAULTED" | "UNAVAIL" | "REMOVED" | "SUSPENDED"
841        )
842    }
843}
844
845/// A btrfs filesystem as seen in `/sys/fs/btrfs/<fsid>`.
846#[derive(Debug, Clone, Default, Serialize, Deserialize)]
847#[serde(default)]
848pub struct BtrfsFilesystem {
849    /// Filesystem UUID.
850    pub fsid: String,
851    /// Filesystem label (may be empty).
852    pub label: String,
853    /// Mount points found in the mount table for this filesystem.
854    pub mount_points: Vec<String>,
855    /// Member block devices (e.g. `"nvme0n1"`, `"nvme1n1p1"`).
856    pub devices: Vec<String>,
857    /// Per-device error counters, keyed by btrfs device id. Each carries its
858    /// device name when it could be looked up (see [`BtrfsDeviceStats::name`]);
859    /// the list is not aligned with `devices`.
860    pub device_stats: Vec<BtrfsDeviceStats>,
861    /// Allocation per block group type (data, metadata, system).
862    pub allocations: Vec<BtrfsAllocation>,
863    /// Result of the most recent scrub, if it is known. Error counters in
864    /// `device_stats` only grow when data is read, so the scrub time tells
865    /// when all data was last verified.
866    pub scrub: Option<BtrfsScrub>,
867}
868
869/// Outcome of the last btrfs scrub (summed over devices).
870#[derive(Debug, Clone, Default, Serialize, Deserialize)]
871#[serde(default)]
872pub struct BtrfsScrub {
873    /// Start time, in milliseconds since the Unix epoch.
874    pub started_at_ms: u64,
875    /// Run time in seconds.
876    pub duration_secs: u64,
877    /// `"finished"`, `"canceled"` or `"unfinished"` (running or interrupted).
878    pub state: String,
879    /// Bytes verified.
880    pub bytes_scrubbed: u64,
881    /// Read, checksum, verify and superblock errors found.
882    pub errors: u64,
883    /// Errors that were repaired from a good copy.
884    pub corrected: u64,
885    /// Errors that could not be repaired.
886    pub uncorrectable: u64,
887}
888
889impl BtrfsFilesystem {
890    /// Returns true if any member is missing or has a non-zero error counter.
891    pub fn has_problems(&self) -> bool {
892        self.device_stats
893            .iter()
894            .any(|d| d.missing || d.total_errors() > 0)
895    }
896}
897
898/// Error counters of one btrfs member device.
899#[derive(Debug, Clone, Default, Serialize, Deserialize)]
900#[serde(default)]
901pub struct BtrfsDeviceStats {
902    /// btrfs device id (1-based).
903    pub devid: u64,
904    /// Device name (e.g. `"nvme0n1"`), from the `BTRFS_IOC_DEV_INFO` ioctl
905    /// on a mount point; empty when unknown (not mounted, device missing).
906    pub name: String,
907    /// Size of the device as btrfs uses it.
908    pub size_bytes: Option<u64>,
909    /// Space allocated on the device.
910    pub used_bytes: Option<u64>,
911    /// Whether the device is missing from the filesystem.
912    pub missing: bool,
913    /// Write I/O errors.
914    pub write_errs: u64,
915    /// Read I/O errors.
916    pub read_errs: u64,
917    /// Flush errors.
918    pub flush_errs: u64,
919    /// Checksum mismatches.
920    pub corruption_errs: u64,
921    /// Generation mismatches (stale data).
922    pub generation_errs: u64,
923}
924
925impl BtrfsDeviceStats {
926    /// Name for display: the device name, else `devid N`.
927    pub fn label(&self) -> String {
928        if self.name.is_empty() {
929            format!("devid {}", self.devid)
930        } else {
931            self.name.clone()
932        }
933    }
934
935    /// Sum of all error counters.
936    pub fn total_errors(&self) -> u64 {
937        self.write_errs
938            + self.read_errs
939            + self.flush_errs
940            + self.corruption_errs
941            + self.generation_errs
942    }
943}
944
945/// Space allocation of one btrfs block group type and RAID profile.
946#[derive(Debug, Clone, Default, Serialize, Deserialize)]
947#[serde(default)]
948pub struct BtrfsAllocation {
949    /// Block group type: `"data"`, `"metadata"` or `"system"`.
950    pub kind: String,
951    /// RAID profile (e.g. `"single"`, `"raid0"`, `"raid1c3"`).
952    pub profile: String,
953    /// Space allocated to this profile (logical size).
954    pub total_bytes: u64,
955    /// Space used within the allocation (logical size).
956    pub used_bytes: u64,
957}
958
959/// One Linux md array parsed from `/proc/mdstat`.
960#[derive(Debug, Clone, Default, Serialize, Deserialize)]
961#[serde(default)]
962pub struct MdArray {
963    /// Array name (e.g. `"md0"`).
964    pub name: String,
965    /// Whether the array is active.
966    pub active: bool,
967    /// Whether the array is read-only (`active (read-only)` / `(auto-read-only)`).
968    pub read_only: bool,
969    /// RAID level (e.g. `"raid1"`); empty for inactive arrays.
970    pub level: String,
971    /// Member devices.
972    pub members: Vec<MdMember>,
973    /// Number of devices the array should have (`[n/m]` -> n).
974    pub expected_devices: Option<u32>,
975    /// Number of devices currently in sync (`[n/m]` -> m).
976    pub working_devices: Option<u32>,
977    /// Ongoing resync/recovery/reshape/check, if any.
978    pub sync: Option<MdSync>,
979}
980
981impl MdArray {
982    /// Returns true if the array runs with fewer devices than expected or has a failed member.
983    pub fn is_degraded(&self) -> bool {
984        let short = match (self.expected_devices, self.working_devices) {
985            (Some(expected), Some(working)) => working < expected,
986            _ => false,
987        };
988        short || self.members.iter().any(|m| m.faulty)
989    }
990}
991
992/// A member device of an md array.
993#[derive(Debug, Clone, Default, Serialize, Deserialize)]
994#[serde(default)]
995pub struct MdMember {
996    /// Device name (e.g. `"sda1"`).
997    pub name: String,
998    /// Role number within the array.
999    pub role: u32,
1000    /// Marked faulty `(F)`.
1001    pub faulty: bool,
1002    /// Spare device `(S)`.
1003    pub spare: bool,
1004}
1005
1006/// Progress of an md background operation.
1007#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1008#[serde(default)]
1009pub struct MdSync {
1010    /// Operation name: `"resync"`, `"recovery"`, `"reshape"` or `"check"`.
1011    pub action: String,
1012    /// Completion percentage.
1013    pub percent: f32,
1014    /// Estimated minutes until completion.
1015    pub finish_minutes: Option<f32>,
1016    /// Current speed in KiB/s.
1017    pub speed_kib_per_sec: Option<u64>,
1018}
1019
1020/// Traffic of one network interface.
1021#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1022#[serde(default)]
1023pub struct NetworkStats {
1024    /// Interface name (e.g. `"enp9s0"`).
1025    pub name: String,
1026    /// Receive throughput.
1027    pub rx_bytes_per_sec: f64,
1028    /// Transmit throughput.
1029    pub tx_bytes_per_sec: f64,
1030    /// Bytes received since the interface came up.
1031    pub rx_total_bytes: u64,
1032    /// Bytes transmitted since the interface came up.
1033    pub tx_total_bytes: u64,
1034    /// Negotiated link speed in Mbit/s (wired links), if known.
1035    pub link_speed_mbps: Option<u32>,
1036    /// Whether the link is up, if the OS reports it.
1037    pub link_up: Option<bool>,
1038    /// Wi-Fi link details, for wireless interfaces.
1039    pub wireless: Option<WirelessLink>,
1040    /// The interface's addresses with their prefix length
1041    /// (`"192.168.1.10/24"`), IPv4 first; IPv6 link-local ones are left out.
1042    pub addresses: Vec<String>,
1043}
1044
1045/// State of a Wi-Fi link.
1046#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1047#[serde(default)]
1048pub struct WirelessLink {
1049    /// Current receive bit rate in Mbit/s.
1050    pub rx_bitrate_mbps: Option<f32>,
1051    /// Current transmit bit rate in Mbit/s.
1052    pub tx_bitrate_mbps: Option<f32>,
1053    /// Signal strength in dBm.
1054    pub signal_dbm: Option<i32>,
1055}
1056
1057/// One process.
1058#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1059#[serde(default)]
1060pub struct ProcessInfo {
1061    /// Process id.
1062    pub pid: u32,
1063    /// Parent process id.
1064    pub parent_pid: Option<u32>,
1065    /// Short name (executable name).
1066    pub name: String,
1067    /// Full command line joined by spaces.
1068    pub command: String,
1069    /// Owner user name (or numeric uid when it cannot be resolved).
1070    pub user: String,
1071    /// Scheduling state (e.g. "Run", "Sleep", "Zombie").
1072    pub state: String,
1073    /// CPU usage; 100 means one full core, so multi-threaded processes may exceed 100.
1074    pub cpu_percent: f32,
1075    /// Resident memory.
1076    pub memory_bytes: u64,
1077    /// Number of threads, where known.
1078    pub threads: Option<u32>,
1079    /// Nice value, -20 (first) to 19 (last), where known (Linux).
1080    pub nice: Option<i32>,
1081    /// Disk read throughput.
1082    pub read_bytes_per_sec: f64,
1083    /// Disk write throughput.
1084    pub write_bytes_per_sec: f64,
1085    /// Seconds since the process started.
1086    pub run_time_secs: u64,
1087    /// Kernel thread (e.g. `kworker`), which front-ends may hide.
1088    pub kernel_thread: bool,
1089    /// The container it runs in (Docker, Podman, ... name, or short id),
1090    /// if any.
1091    pub container: Option<String>,
1092}
1093
1094#[cfg(test)]
1095mod tests {
1096    use super::*;
1097
1098    fn process(pid: u32, cpu: f32) -> ProcessInfo {
1099        ProcessInfo {
1100            pid,
1101            cpu_percent: cpu,
1102            ..Default::default()
1103        }
1104    }
1105
1106    #[test]
1107    fn cpu_power_falls_back_to_a_laptop_apu_only() {
1108        let power = |cpu: Option<f32>, igpu: Option<f32>| PowerStats {
1109            cpu_package_w: cpu,
1110            igpu_w: igpu,
1111            ..Default::default()
1112        };
1113        assert_eq!(
1114            power(Some(30.0), Some(6.0)).cpu_or_soc_w(),
1115            ("CPU", Some(30.0))
1116        );
1117        assert_eq!(power(None, Some(6.0)).cpu_or_soc_w(), ("SoC", Some(6.0)));
1118        // A desktop Ryzen iGPU's few milliwatts are not the SoC.
1119        assert_eq!(power(None, Some(0.007)).cpu_or_soc_w(), ("CPU", None));
1120        assert_eq!(power(None, None).cpu_or_soc_w(), ("CPU", None));
1121    }
1122
1123    #[test]
1124    fn truncate_keeps_top_cpu_processes_in_order() {
1125        let mut snapshot = Snapshot {
1126            processes: vec![
1127                process(1, 5.0),
1128                process(2, 50.0),
1129                process(3, 0.0),
1130                process(4, 20.0),
1131            ],
1132            process_count: 4,
1133            ..Default::default()
1134        };
1135        snapshot.truncate_processes(2);
1136        let pids: Vec<u32> = snapshot.processes.iter().map(|p| p.pid).collect();
1137        assert_eq!(pids, vec![2, 4]);
1138        // The total is preserved for display ("2 of 4 processes").
1139        assert_eq!(snapshot.process_count, 4);
1140    }
1141
1142    #[test]
1143    fn truncate_leaves_out_kernel_threads() {
1144        let kworker = |pid: u32, cpu: f32| ProcessInfo {
1145            kernel_thread: true,
1146            ..process(pid, cpu)
1147        };
1148        let mut snapshot = Snapshot {
1149            processes: vec![
1150                kworker(1, 9.0),
1151                process(2, 5.0),
1152                kworker(3, 8.0),
1153                process(4, 1.0),
1154                process(5, 0.0),
1155            ],
1156            process_count: 5,
1157            ..Default::default()
1158        };
1159        snapshot.truncate_processes(3);
1160        let pids: Vec<u32> = snapshot.processes.iter().map(|p| p.pid).collect();
1161        assert_eq!(pids, vec![2, 4, 5], "busy kworkers take no places");
1162        assert_eq!(snapshot.process_count, 5);
1163    }
1164
1165    #[test]
1166    fn truncate_keeps_watched_processes() {
1167        let mut snapshot = Snapshot {
1168            processes: (1..=6).map(|pid| process(pid, pid as f32)).collect(),
1169            watched_pids: vec![1, 99],
1170            ..Default::default()
1171        };
1172        snapshot.truncate_processes(2);
1173        let pids: Vec<u32> = snapshot.processes.iter().map(|p| p.pid).collect();
1174        assert_eq!(pids, vec![6, 5, 1]);
1175    }
1176
1177    #[test]
1178    fn snapshot_round_trips_through_json() {
1179        let snapshot = Snapshot {
1180            timestamp_ms: 42,
1181            processes: vec![process(7, 1.5)],
1182            ..Default::default()
1183        };
1184        let json = serde_json::to_string(&snapshot).unwrap();
1185        let back: Snapshot = serde_json::from_str(&json).unwrap();
1186        assert_eq!(back.timestamp_ms, 42);
1187        assert_eq!(back.processes[0].pid, 7);
1188    }
1189
1190    #[test]
1191    fn device_matching_handles_nvme_namespaces() {
1192        assert!(device_matches_disk("sda", "sda"));
1193        assert!(device_matches_disk("nvme0", "nvme0n1"));
1194        assert!(device_matches_disk("nvme1", "nvme1n1"));
1195        assert!(!device_matches_disk("nvme1", "nvme10n1"));
1196        assert!(!device_matches_disk("sd", "sda"));
1197        assert!(!device_matches_disk("sda", "sdb"));
1198    }
1199
1200    #[test]
1201    fn smart_attention_flags() {
1202        let healthy = SmartDevice {
1203            passed: Some(true),
1204            media_errors: Some(0),
1205            available_spare_percent: Some(100),
1206            available_spare_threshold_percent: Some(10),
1207            ..Default::default()
1208        };
1209        assert!(!healthy.needs_attention());
1210        let worn = SmartDevice {
1211            available_spare_percent: Some(5),
1212            ..healthy.clone()
1213        };
1214        assert!(worn.needs_attention());
1215        let reallocated = SmartDevice {
1216            reallocated_sectors: Some(8),
1217            ..healthy
1218        };
1219        assert!(reallocated.needs_attention());
1220    }
1221
1222    #[test]
1223    fn md_array_degraded_detection() {
1224        let mut array = MdArray {
1225            expected_devices: Some(2),
1226            working_devices: Some(2),
1227            ..Default::default()
1228        };
1229        assert!(!array.is_degraded());
1230        array.working_devices = Some(1);
1231        assert!(array.is_degraded());
1232    }
1233}