rigtop_core/snapshot.rs
1//! Serializable snapshot of everything rigtop displays.
2//!
3//! A [`Snapshot`] is the single unit of data exchanged between a data source
4//! (local collectors, or a remote `rigtop --agent` over SSH) and a front-end
5//! (TUI or GUI). Every type here derives `Serialize`/`Deserialize` so that the
6//! same structure can be streamed over the wire unchanged.
7//!
8//! Conventions:
9//! - Sizes are in bytes, rates in bytes per second.
10//! - Temperatures are in degrees Celsius, power in watts, fan speed in RPM.
11//! - Percentages are in the range `0.0..=100.0` (per-core values are not summed).
12//! - Values that a platform or device cannot provide are `None` (or an empty
13//! list) rather than zero, so that front-ends can hide them.
14//! - Every struct is `#[serde(default)]`: fields missing from the input take
15//! their default, so a client can read snapshots from an older or newer
16//! `rigtop --agent` as long as fields are only added.
17
18use serde::{Deserialize, Serialize};
19
20/// One full measurement of a host.
21#[derive(Debug, Clone, Default, Serialize, Deserialize)]
22#[serde(default)]
23pub struct Snapshot {
24 /// Wall-clock time of the measurement, in milliseconds since the Unix epoch.
25 pub timestamp_ms: u64,
26 /// Static-ish information about the host.
27 pub host: HostInfo,
28 /// CPU load, frequency and temperature.
29 pub cpu: CpuStats,
30 /// Physical memory and swap.
31 pub memory: MemoryStats,
32 /// Discrete and integrated GPUs that could be queried.
33 pub gpus: Vec<GpuStats>,
34 /// Raw hardware sensor chips (temperatures and fans), grouped by chip.
35 pub sensors: Vec<SensorChip>,
36 /// Power draw summary.
37 pub power: PowerStats,
38 /// I/O activity per physical block device.
39 pub disks: Vec<DiskIo>,
40 /// Capacity of mounted filesystems.
41 pub filesystems: Vec<FilesystemUsage>,
42 /// Software RAID state (btrfs and mdadm).
43 pub raid: RaidStats,
44 /// SMART health per drive, from files written by a privileged timer.
45 pub smart: Vec<SmartDevice>,
46 /// Network interface traffic.
47 pub networks: Vec<NetworkStats>,
48 /// Process list. May be truncated by the source (see [`Snapshot::truncate_processes`]).
49 pub processes: Vec<ProcessInfo>,
50 /// Total number of processes on the host, even if `processes` was truncated.
51 pub process_count: usize,
52 /// Processes the client asked to hear about when they exit (see
53 /// [`crate::DataSource::watch_processes`]): listed in `processes`
54 /// whatever the limit while they run, so one of them missing has
55 /// exited. Empty from agents before 0.19.
56 pub watched_pids: Vec<u32>,
57 /// Linux: systemd units in the failed state (e.g. `"nginx.service"`).
58 pub failed_units: Vec<String>,
59 /// Remote hosts: the round trip of the link to the agent (milliseconds),
60 /// measured by the client; agents leave it out.
61 pub link_rtt_ms: Option<f32>,
62 /// The host's public IPv4 address as an outside service sees it, when
63 /// the client asked for it (see [`crate::DataSource::set_public_ip`]).
64 pub public_ip: Option<String>,
65 /// Linux: how much of the time tasks waited for CPU, memory or I/O
66 /// (pressure stall information). `None` without it (other platforms,
67 /// kernels built or booted without PSI, agents before 0.21).
68 pub pressure: Option<Pressure>,
69 /// Windows: why LibreHardwareMonitor, the source of temperatures, fans
70 /// and CPU power, could not be read. `None` when it was read, when it is
71 /// disabled in the configuration, and on other platforms.
72 pub lhm_error: Option<String>,
73 /// UPSes known to Network UPS Tools' upsd at the configured address
74 /// (`[ups] address`). Empty without NUT, and from agents before 0.30.
75 pub ups: Vec<UpsStats>,
76}
77
78impl Snapshot {
79 /// Keeps only the `limit` processes with the highest CPU usage.
80 ///
81 /// Used to reduce the payload when a host is shown in an overview rather
82 /// than in detail. Kernel threads are dropped first: the overview does not
83 /// show them, and busy `kworker`s would otherwise take places in the top
84 /// `limit` and leave the list short. `process_count` keeps the original
85 /// total. Processes in `watched_pids` stay, after the top `limit`.
86 pub fn truncate_processes(&mut self, limit: usize) {
87 self.processes.retain(|p| !p.kernel_thread);
88 if self.processes.len() <= limit {
89 return;
90 }
91 let (watched, mut others): (Vec<ProcessInfo>, Vec<ProcessInfo>) =
92 std::mem::take(&mut self.processes)
93 .into_iter()
94 .partition(|p| self.watched_pids.contains(&p.pid));
95 if others.len() > limit {
96 // Partial sort is enough: only the top `limit` entries matter.
97 others.select_nth_unstable_by(limit, |a, b| b.cpu_percent.total_cmp(&a.cpu_percent));
98 others.truncate(limit);
99 }
100 others.sort_unstable_by(|a, b| b.cpu_percent.total_cmp(&a.cpu_percent));
101 others.extend(watched);
102 self.processes = others;
103 }
104}
105
106/// Host identification and uptime.
107#[derive(Debug, Clone, Default, Serialize, Deserialize)]
108#[serde(default)]
109pub struct HostInfo {
110 /// Host name as reported by the OS.
111 pub hostname: String,
112 /// Human readable OS name and version (e.g. "Linux 26.04 Ubuntu").
113 pub os: String,
114 /// Kernel version string.
115 pub kernel: String,
116 /// Seconds since boot.
117 pub uptime_secs: u64,
118 /// Version of the rigtop that took the snapshot (the agent's, for a
119 /// remote host). Agents before 0.9.1 leave it out; the remote source
120 /// fills it in from the agent's greeting.
121 pub rigtop_version: String,
122 /// A reboot is needed to finish an update (Linux), with the reason
123 /// (possibly empty), e.g. "updated: linux-image-7.0.0-39-generic".
124 pub reboot_required: Option<String>,
125}
126
127/// CPU-wide and per-core statistics.
128#[derive(Debug, Clone, Default, Serialize, Deserialize)]
129#[serde(default)]
130pub struct CpuStats {
131 /// Brand string of the first CPU (e.g. "AMD Ryzen 9 9950X3D 16-Core Processor").
132 pub model: String,
133 /// Usage across all logical cores.
134 pub total_percent: f32,
135 /// Usage of each logical core, in OS order.
136 pub per_core_percent: Vec<f32>,
137 /// Current frequency of each logical core in MHz.
138 pub per_core_mhz: Vec<u64>,
139 /// 1, 5 and 15 minute load averages; `None` on platforms without them
140 /// (Windows).
141 pub load_average: Option<[f64; 3]>,
142 /// CPU temperature readings, e.g. `Tctl`, `Tccd1`, `Tccd2` on AMD or
143 /// `Package id 0` on Intel. Per-core readings that could be matched to
144 /// logical cores are in [`CpuStats::per_core_celsius`] instead.
145 pub temperatures: Vec<Temperature>,
146 /// Temperature of the physical core of each logical core, in OS order
147 /// (Intel's coretemp). Empty when the CPU has no per-core sensors.
148 pub per_core_celsius: Vec<Option<f32>>,
149}
150
151/// Physical memory and swap usage.
152#[derive(Debug, Clone, Default, Serialize, Deserialize)]
153#[serde(default)]
154pub struct MemoryStats {
155 /// Installed (visible) memory.
156 pub total_bytes: u64,
157 /// Memory in use by processes (total minus available).
158 pub used_bytes: u64,
159 /// Memory that can be allocated without swapping.
160 pub available_bytes: u64,
161 /// Completely unused memory.
162 pub free_bytes: u64,
163 /// Page cache and reclaimable slab, where the platform reports it.
164 pub cached_bytes: Option<u64>,
165 /// Total swap space.
166 pub swap_total_bytes: u64,
167 /// Swap space in use.
168 pub swap_used_bytes: u64,
169 /// Processes the kernel's OOM killer stopped in the last hour, seen
170 /// since rigtop started (Linux), oldest first.
171 pub oom_kills: Vec<OomKill>,
172}
173
174/// Pressure stall information: the share of time tasks were stalled
175/// waiting for a resource, from `/proc/pressure/`.
176#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
177#[serde(default)]
178pub struct Pressure {
179 pub cpu: PressureLevels,
180 pub memory: PressureLevels,
181 pub io: PressureLevels,
182}
183
184/// Stall percentages of one resource. `some`: at least one task waited
185/// (others may have run); `full`: every task that wanted to run waited, so
186/// the time was lost. The kernel's 10- and 60-second averages.
187#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
188#[serde(default)]
189pub struct PressureLevels {
190 pub some_avg10: f32,
191 pub some_avg60: f32,
192 pub full_avg10: f32,
193 pub full_avg60: f32,
194}
195
196/// One process stopped by the kernel for lack of memory.
197#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
198#[serde(default)]
199pub struct OomKill {
200 /// When it was noticed (milliseconds since the Unix epoch).
201 pub at_ms: u64,
202 /// Name of the process that most likely was stopped: the largest one
203 /// that went away at the same time (the kernel log saying which needs
204 /// root).
205 pub likely_process: Option<String>,
206}
207
208/// A single GPU.
209#[derive(Debug, Clone, Default, Serialize, Deserialize)]
210#[serde(default)]
211pub struct GpuStats {
212 /// Position in the GPU list: NVIDIA GPUs first (their NVML index),
213 /// then AMD and Intel ones.
214 pub index: u32,
215 /// Vendor name ("NVIDIA", "AMD", ...).
216 pub vendor: String,
217 /// Marketing name of the device.
218 pub name: String,
219 /// Integrated graphics (an APU or Intel iGPU), whose power is part of
220 /// the CPU package and not reported here.
221 pub integrated: bool,
222 /// Core utilization (unknown for Intel without privileges).
223 pub utilization_percent: Option<f32>,
224 /// Memory controller utilization.
225 pub memory_utilization_percent: Option<f32>,
226 /// VRAM in use.
227 pub memory_used_bytes: Option<u64>,
228 /// Total VRAM.
229 pub memory_total_bytes: Option<u64>,
230 /// Core temperature.
231 pub temperature_c: Option<f32>,
232 /// Current board power draw.
233 pub power_w: Option<f32>,
234 /// Enforced power limit.
235 pub power_limit_w: Option<f32>,
236 /// Fan speed as a percentage of the maximum (first fan).
237 pub fan_percent: Option<f32>,
238 /// Graphics clock in MHz.
239 pub clock_mhz: Option<u32>,
240 /// Why the clocks are held down right now (NVIDIA): "power cap",
241 /// "thermal", "hw thermal", "power brake" or "hw slowdown". Empty when
242 /// the GPU runs freely or idles. Text, so that newer agents can add
243 /// reasons older clients do not know.
244 pub throttle_reasons: Vec<String>,
245 /// ECC memory errors since the driver loaded (NVIDIA, with ECC on).
246 pub ecc: Option<EccErrors>,
247 /// The PCIe link (NVIDIA).
248 pub pcie: Option<PcieLink>,
249 /// Processes currently using this GPU.
250 pub processes: Vec<GpuProcess>,
251}
252
253impl GpuStats {
254 /// Throttle reasons that mean the hardware is slowing the GPU down, as
255 /// opposed to running at its power limit as designed.
256 pub fn slowdown_reasons(&self) -> impl Iterator<Item = &str> {
257 self.throttle_reasons
258 .iter()
259 .map(String::as_str)
260 .filter(|r| *r != "power cap")
261 }
262}
263
264/// ECC memory error counts of a GPU.
265#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
266#[serde(default)]
267pub struct EccErrors {
268 /// Errors the ECC corrected.
269 pub corrected: u64,
270 /// Errors it could not correct (data was corrupted).
271 pub uncorrected: u64,
272}
273
274/// A GPU's PCIe link: what it runs at now (lower when idle, to save power)
275/// and the most the GPU and its slot allow.
276#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
277#[serde(default)]
278pub struct PcieLink {
279 pub generation: u32,
280 pub width: u32,
281 pub max_generation: u32,
282 pub max_width: u32,
283}
284
285/// GPU usage attributed to one process.
286#[derive(Debug, Clone, Default, Serialize, Deserialize)]
287#[serde(default)]
288pub struct GpuProcess {
289 /// OS process id.
290 pub pid: u32,
291 /// VRAM used by the process, if the driver reports it.
292 pub memory_bytes: Option<u64>,
293 /// Streaming multiprocessor (core) utilization by the process.
294 pub sm_percent: Option<f32>,
295}
296
297/// A hardware monitoring chip (a Linux hwmon device) and its readings.
298#[derive(Debug, Clone, Default, Serialize, Deserialize)]
299#[serde(default)]
300pub struct SensorChip {
301 /// Stable identifier, `"<driver name>:<device name>"`
302 /// (e.g. `"nct6686:nct6683.2592"`, `"nvme:nvme0"`).
303 ///
304 /// Unlike `hwmonN`, this survives reboots, so it is used as the key for
305 /// user-defined labels.
306 pub key: String,
307 /// Driver-provided chip name (e.g. `"k10temp"`, `"nvme"`, `"spd5118"`).
308 pub name: String,
309 /// Temperature channels.
310 pub temperatures: Vec<Temperature>,
311 /// Fan channels.
312 pub fans: Vec<Fan>,
313}
314
315/// One temperature channel.
316#[derive(Debug, Clone, Default, Serialize, Deserialize)]
317#[serde(default)]
318pub struct Temperature {
319 /// Stable key `"<chip key>/temp<N>"` used to look up user-defined names.
320 /// Empty for readings that do not come from a hwmon chip.
321 pub key: String,
322 /// Driver label (e.g. `"Tccd1"`), or a generic `"temp<N>"` when none exists.
323 pub label: String,
324 /// Current reading.
325 pub celsius: f32,
326 /// High threshold, if exposed.
327 pub max_celsius: Option<f32>,
328 /// Critical threshold, if exposed.
329 pub critical_celsius: Option<f32>,
330}
331
332/// One fan channel.
333#[derive(Debug, Clone, Default, Serialize, Deserialize)]
334#[serde(default)]
335pub struct Fan {
336 /// Stable key `"<chip key>/fan<N>"` used to look up user-defined names.
337 pub key: String,
338 /// Driver label, or a generic `"fan<N>"` when none exists.
339 pub label: String,
340 /// Current speed (0 when stopped or not connected).
341 pub rpm: u32,
342}
343
344/// Power draw summary.
345///
346/// There is no portable way to measure whole-system power without external
347/// hardware, so the total is the sum of the CPU package and discrete GPUs,
348/// which dominate consumption on typical workstations.
349#[derive(Debug, Clone, Default, Serialize, Deserialize)]
350#[serde(default)]
351pub struct PowerStats {
352 /// CPU package power (RAPL on Linux). `None` before the second sample or
353 /// when RAPL is unavailable.
354 pub cpu_package_w: Option<f32>,
355 /// Sum of power reported by discrete GPUs.
356 pub gpus_w: Option<f32>,
357 /// Power an AMD APU's graphics driver reports (amdgpu `PPT`). Per the
358 /// kernel documentation it covers the whole SoC, CPU cores included, so
359 /// it overlaps the CPU package and only stands in for it when RAPL cannot
360 /// be read (see [`PowerStats::cpu_or_soc_w`]).
361 pub igpu_w: Option<f32>,
362 /// `cpu_package_w` (or `igpu_w` without it) `+ gpus_w`, counting only the
363 /// parts that are available.
364 pub total_w: Option<f32>,
365 /// Laptop battery, if there is one.
366 pub battery: Option<BatteryStats>,
367}
368
369/// Below this, an APU reading is not whole-SoC power: desktop Ryzen CPUs
370/// with a small iGPU report only a few milliwatts there, while a laptop
371/// APU's SoC never idles under a watt.
372const MIN_SOC_W: f32 = 1.0;
373
374impl PowerStats {
375 /// The CPU figure to show: `("CPU", RAPL)`, else `("SoC", APU PPT)`.
376 pub fn cpu_or_soc_w(&self) -> (&'static str, Option<f32>) {
377 match (self.cpu_package_w, self.igpu_w) {
378 (Some(w), _) => ("CPU", Some(w)),
379 (None, Some(w)) if w >= MIN_SOC_W => ("SoC", Some(w)),
380 _ => ("CPU", None),
381 }
382 }
383}
384
385/// Charge and power of the system's batteries (summed when there are
386/// several), from `/sys/class/power_supply` on Linux.
387#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
388#[serde(default)]
389pub struct BatteryStats {
390 /// Charge in percent.
391 pub percent: Option<f32>,
392 /// Kernel status: `Charging`, `Discharging`, `Full`, `Not charging`...
393 pub status: String,
394 /// Charge or discharge power (not signed; see `status`).
395 pub power_w: Option<f32>,
396 /// Until empty while discharging, until full while charging, at the
397 /// recent (smoothed) rate.
398 pub time_left_secs: Option<u64>,
399 /// Full charge as a share of the design capacity (wear), in percent.
400 pub health_percent: Option<f32>,
401}
402
403/// A UPS, as Network UPS Tools' `upsd` reports it.
404#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
405#[serde(default)]
406pub struct UpsStats {
407 /// upsd's name for it (`ups` in `upsc ups@localhost`).
408 pub name: String,
409 /// Maker and model (`APC Back-UPS ES 700`), else its description in
410 /// NUT's `ups.conf`.
411 pub model: Option<String>,
412 /// The flags of `ups.status`: `OL` on line power, `OB` on battery, `LB`
413 /// low battery, `CHRG` charging, `RB` replace the battery, `OVER`
414 /// overloaded, `BYPASS`, `OFF`, `FSD` forced shutdown...
415 pub status: Vec<String>,
416 /// Battery charge in percent.
417 pub charge_percent: Option<f32>,
418 /// How long the battery would last at the present load.
419 pub runtime_secs: Option<u64>,
420 /// Load in percent of the UPS's capacity.
421 pub load_percent: Option<f32>,
422 /// Power drawn by what is plugged in: `ups.realpower`, else the load
423 /// times `ups.realpower.nominal`.
424 pub power_w: Option<f32>,
425 /// Mains voltage.
426 pub input_voltage: Option<f32>,
427 /// Why there are no readings (`data stale`, `driver not connected`,
428 /// `upsd not reachable (...)`).
429 pub error: Option<String>,
430}
431
432impl UpsStats {
433 /// Whether `ups.status` has `flag` (`OB`).
434 pub fn has(&self, flag: &str) -> bool {
435 self.status.iter().any(|s| s == flag)
436 }
437
438 /// "UPS" for the usual one named `ups`, else "UPS <name>".
439 pub fn label(&self) -> String {
440 if self.name.eq_ignore_ascii_case("ups") || self.name.is_empty() {
441 "UPS".to_owned()
442 } else {
443 format!("UPS {}", self.name)
444 }
445 }
446
447 /// Short form for status lines: `100% 73W` on line power, `on battery
448 /// 72% 25m` during a power failure, `no data` without readings.
449 pub fn summary(&self) -> String {
450 if self.error.is_some() {
451 return "no data".to_owned();
452 }
453 let mut parts = Vec::new();
454 if self.has("OB") {
455 parts.push("on battery".to_owned());
456 }
457 if let Some(charge) = self.charge_percent {
458 parts.push(format!("{charge:.0}%"));
459 }
460 if self.has("OB") {
461 parts.extend(self.runtime_secs.map(crate::format::hours_minutes));
462 } else if let Some(power) = self.power_w {
463 parts.push(format!("{power:.0}W"));
464 } else if let Some(load) = self.load_percent {
465 parts.push(format!("load {load:.0}%"));
466 }
467 if parts.is_empty() {
468 parts.push(self.status.join(" "));
469 }
470 parts.join(" ")
471 }
472
473 /// Hover text: model, state, battery, load and mains voltage.
474 pub fn details(&self) -> String {
475 let mut lines = vec![match &self.model {
476 Some(model) => format!("{} ({model})", self.label()),
477 None => self.label(),
478 }];
479 if let Some(error) = &self.error {
480 lines.push(format!("no data: {error}"));
481 return lines.join("\n");
482 }
483 let state: Vec<&str> = self
484 .status
485 .iter()
486 .map(|flag| match flag.as_str() {
487 "OL" => "on line power",
488 "OB" => "on battery",
489 "LB" => "battery low",
490 "HB" => "battery high",
491 "RB" => "replace the battery",
492 "CHRG" => "charging",
493 "DISCHRG" => "discharging",
494 "BYPASS" => "on bypass",
495 "CAL" => "calibrating",
496 "OFF" => "output off",
497 "OVER" => "overloaded",
498 "TRIM" => "trimming high mains voltage",
499 "BOOST" => "boosting low mains voltage",
500 "FSD" => "forced shutdown",
501 other => other,
502 })
503 .collect();
504 if !state.is_empty() {
505 lines.push(state.join(", "));
506 }
507 let mut battery = Vec::new();
508 if let Some(charge) = self.charge_percent {
509 battery.push(format!("{charge:.0}%"));
510 }
511 if let Some(secs) = self.runtime_secs {
512 battery.push(format!(
513 "{} at this load",
514 crate::format::hours_minutes(secs)
515 ));
516 }
517 if !battery.is_empty() {
518 lines.push(format!("battery {}", battery.join(", ")));
519 }
520 let mut load = Vec::new();
521 if let Some(percent) = self.load_percent {
522 load.push(format!("{percent:.0}%"));
523 }
524 if let Some(power) = self.power_w {
525 load.push(format!("{power:.0} W"));
526 }
527 if !load.is_empty() {
528 lines.push(format!("load {}", load.join(", ")));
529 }
530 if let Some(volts) = self.input_voltage {
531 lines.push(format!("mains {volts:.0} V"));
532 }
533 lines.join("\n")
534 }
535}
536
537impl BatteryStats {
538 /// Short form for status lines: `72% -12W 3h10m` while discharging
539 /// (with the time left when known), `+20W` while charging.
540 pub fn summary(&self) -> String {
541 let percent = self
542 .percent
543 .map_or("--%".to_owned(), |p| format!("{p:.0}%"));
544 let sign = match self.status.as_str() {
545 "Discharging" => "-",
546 "Charging" => "+",
547 _ => return percent,
548 };
549 let mut text = match self.power_w {
550 Some(w) if w >= 0.5 => format!("{percent} {sign}{w:.0}W"),
551 _ => percent,
552 };
553 if let Some(secs) = self.time_left_secs {
554 text.push(' ');
555 text.push_str(&crate::format::hours_minutes(secs));
556 }
557 text
558 }
559
560 /// The health worth showing next to the charge: below 80% of the design
561 /// capacity, where batteries are usually considered worn.
562 pub fn health_note(&self) -> Option<String> {
563 self.health_percent
564 .filter(|h| *h < 80.0)
565 .map(|h| format!("health {h:.0}%"))
566 }
567
568 /// Hover text: what the time means, and the health.
569 pub fn details(&self) -> String {
570 let mut lines = Vec::new();
571 if let Some(secs) = self.time_left_secs {
572 let until = if self.status == "Charging" {
573 "full"
574 } else {
575 "empty"
576 };
577 lines.push(format!(
578 "{} until {until} at the recent rate",
579 crate::format::hours_minutes(secs)
580 ));
581 }
582 if let Some(health) = self.health_percent {
583 lines.push(format!("full charge {health:.0}% of the design capacity"));
584 }
585 if lines.is_empty() {
586 lines.push(self.status.clone());
587 }
588 lines.join("\n")
589 }
590}
591
592/// I/O activity of a block device.
593#[derive(Debug, Clone, Default, Serialize, Deserialize)]
594#[serde(default)]
595pub struct DiskIo {
596 /// Kernel device name (e.g. `"nvme0n1"`, `"sda"`).
597 pub name: String,
598 /// Read throughput.
599 pub read_bytes_per_sec: f64,
600 /// Write throughput.
601 pub write_bytes_per_sec: f64,
602 /// Fraction of time the device had I/O in flight, where the platform
603 /// reports it (Linux).
604 pub busy_percent: Option<f32>,
605 /// Average time one request took (queue and service), over the requests
606 /// completed since the last sample (Linux). `None` without any.
607 pub latency_ms: Option<f32>,
608 /// Drive temperature (NVMe composite sensor), if available.
609 pub temperature_c: Option<f32>,
610}
611
612/// Capacity of one filesystem.
613///
614/// Filesystems mounted at several places (e.g. btrfs subvolumes) appear once,
615/// with all mount points listed.
616#[derive(Debug, Clone, Default, Serialize, Deserialize)]
617#[serde(default)]
618pub struct FilesystemUsage {
619 /// Source device (e.g. `"/dev/nvme1n1p1"`).
620 pub device: String,
621 /// Filesystem type (e.g. `"btrfs"`, `"ext4"`).
622 pub fs_type: String,
623 /// All mount points of this filesystem, shortest first.
624 pub mount_points: Vec<String>,
625 /// Total size.
626 pub total_bytes: u64,
627 /// Space available to unprivileged users.
628 pub available_bytes: u64,
629 /// Seconds until it is full at the growth of the last half hour; `None`
630 /// when usage does not grow, without enough history yet, and from
631 /// agents before 0.19 (see [`crate::fill`]).
632 pub full_in_secs: Option<u64>,
633}
634
635/// Returns true if `device` (a disk such as `"sda"`, or an NVMe controller
636/// such as `"nvme0"`) is the drive behind the block device `disk`
637/// (`"sda"`, `"nvme0n1"`).
638///
639/// `"nvme1"` does not match `"nvme10n1"`: after the controller name only the
640/// namespace suffix (`n<N>`) may follow.
641pub fn device_matches_disk(device: &str, disk: &str) -> bool {
642 match disk.strip_prefix(device) {
643 Some("") => true,
644 Some(rest) => device.starts_with("nvme") && rest.starts_with('n'),
645 None => false,
646 }
647}
648
649/// SMART health of one drive, parsed from `smartctl -j -a` output.
650///
651/// Fields a drive or protocol does not report are `None` (NVMe drives have
652/// no sector reallocation counters, ATA drives no "percentage used").
653#[derive(Debug, Clone, Default, Serialize, Deserialize)]
654#[serde(default)]
655pub struct SmartDevice {
656 /// Kernel device name (`"nvme0"`, `"sda"`). See [`device_matches_disk`].
657 pub device: String,
658 /// Protocol reported by smartctl (`"NVMe"`, `"ATA"`, `"SCSI"`).
659 pub protocol: String,
660 /// Drive model.
661 pub model: String,
662 /// Overall health self-assessment; `None` if smartctl could not tell.
663 pub passed: Option<bool>,
664 /// Current drive temperature.
665 pub temperature_c: Option<f32>,
666 /// Accumulated power-on time.
667 pub power_on_hours: Option<u64>,
668 /// NVMe wear estimate (may exceed 100).
669 pub percentage_used: Option<u32>,
670 /// NVMe spare capacity left.
671 pub available_spare_percent: Option<u32>,
672 /// NVMe spare threshold below which the drive reports a warning.
673 pub available_spare_threshold_percent: Option<u32>,
674 /// NVMe critical warning bit field (0 when healthy).
675 pub critical_warning: Option<u32>,
676 /// NVMe unrecovered data integrity errors.
677 pub media_errors: Option<u64>,
678 /// ATA attribute 5 (Reallocated_Sector_Ct), raw value.
679 pub reallocated_sectors: Option<u64>,
680 /// ATA attribute 197 (Current_Pending_Sector), raw value.
681 pub pending_sectors: Option<u64>,
682 /// ATA attribute 198 (Offline_Uncorrectable), raw value.
683 pub offline_uncorrectable: Option<u64>,
684 /// Total host writes, where the drive reports them.
685 pub data_written_bytes: Option<u64>,
686 /// When smartctl ran, in milliseconds since the Unix epoch. Lets front-ends
687 /// flag stale results (e.g. the timer stopped running).
688 pub collected_at_ms: Option<u64>,
689 /// Error reported by smartctl (e.g. the device could not be opened).
690 pub error: Option<String>,
691}
692
693impl SmartDevice {
694 /// Returns true if anything in the SMART data calls for attention.
695 pub fn needs_attention(&self) -> bool {
696 let positive = |v: Option<u64>| v.is_some_and(|v| v > 0);
697 let spare_low = match (
698 self.available_spare_percent,
699 self.available_spare_threshold_percent,
700 ) {
701 (Some(spare), Some(threshold)) => spare < threshold,
702 _ => false,
703 };
704 self.passed == Some(false)
705 || self.error.is_some()
706 || self.critical_warning.is_some_and(|w| w != 0)
707 || positive(self.media_errors)
708 || positive(self.reallocated_sectors)
709 || positive(self.pending_sectors)
710 || positive(self.offline_uncorrectable)
711 || self.percentage_used.is_some_and(|p| p >= 100)
712 || spare_low
713 }
714}
715
716/// Software RAID state.
717#[derive(Debug, Clone, Default, Serialize, Deserialize)]
718#[serde(default)]
719pub struct RaidStats {
720 /// Mounted btrfs filesystems.
721 pub btrfs: Vec<BtrfsFilesystem>,
722 /// Linux md (mdadm) arrays.
723 pub md: Vec<MdArray>,
724 /// Imported ZFS pools.
725 pub zfs: Vec<ZfsPool>,
726}
727
728/// A ZFS pool and its health.
729#[derive(Debug, Clone, Default, Serialize, Deserialize)]
730#[serde(default)]
731pub struct ZfsPool {
732 /// Pool name (e.g. `"tank"`).
733 pub name: String,
734 /// Health as `zpool status` shows it: `ONLINE`, `DEGRADED`, `FAULTED`,
735 /// `OFFLINE`, `UNAVAIL`, `REMOVED` or `SUSPENDED`.
736 pub state: String,
737 /// Its vdevs and devices as `zpool status` lists them, top to bottom
738 /// (empty where `zpool` could not be run, and from agents before 0.26).
739 pub devices: Vec<ZfsDevice>,
740 /// The last or running scrub or resilver ("scrub repaired 0B in
741 /// 00:00:01 with 0 errors on ...", "resilver in progress ... 42.1%
742 /// done"), from the `scan:` line.
743 pub scan: Option<String>,
744 /// Permanent data errors (`errors: N data errors`); `None` unknown.
745 pub data_errors: Option<u64>,
746 /// Size and space in use (`zpool list`).
747 pub size_bytes: Option<u64>,
748 pub allocated_bytes: Option<u64>,
749}
750
751/// One row of a pool's configuration in `zpool status`: a vdev (raidz1-0,
752/// mirror-0) or a device under one.
753#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
754#[serde(default)]
755pub struct ZfsDevice {
756 /// As listed: "raidz1-0", "mirror-1", "ata-WDC_WD80...", "/var/tmp/log1".
757 pub name: String,
758 /// Nesting under the pool: 0 for top-level vdevs, 1 for their devices.
759 pub depth: u8,
760 /// The section it is in: "" for data, else "logs", "cache", "spares",
761 /// "special" or "dedup".
762 pub section: String,
763 /// ONLINE, DEGRADED, OFFLINE, FAULTED, UNAVAIL, REMOVED; AVAIL or
764 /// INUSE for spares.
765 pub state: String,
766 /// Read, write and checksum errors since the last `zpool clear`.
767 pub read_errors: u64,
768 pub write_errors: u64,
769 pub checksum_errors: u64,
770}
771
772impl ZfsDevice {
773 /// All its errors.
774 pub fn errors(&self) -> u64 {
775 self.read_errors + self.write_errors + self.checksum_errors
776 }
777
778 /// Whether its state needs attention (spares waiting are fine).
779 pub fn is_unhealthy(&self) -> bool {
780 !matches!(self.state.as_str(), "ONLINE" | "AVAIL" | "INUSE" | "")
781 }
782
783 /// Whether it is a vdev grouping devices (raidz, mirror, ...) rather
784 /// than a device.
785 pub fn is_group(&self) -> bool {
786 ["raidz", "mirror", "draid", "replacing", "spare-"]
787 .iter()
788 .any(|kind| self.name.starts_with(kind))
789 }
790}
791
792impl ZfsPool {
793 /// The layout of its data vdevs: "raidz1", "mirror", "2 × mirror",
794 /// "stripe" (devices directly under the pool), or `None` when unknown.
795 pub fn layout(&self) -> Option<String> {
796 let top: Vec<&ZfsDevice> = self
797 .devices
798 .iter()
799 .filter(|d| d.depth == 0 && d.section.is_empty())
800 .collect();
801 let first = top.first()?;
802 if !first.is_group() {
803 return Some("stripe".to_owned());
804 }
805 let kind = first
806 .name
807 .rsplit_once('-')
808 .map_or(first.name.as_str(), |(k, _)| k);
809 let same = top.iter().filter(|d| d.name.starts_with(kind)).count();
810 Some(if top.len() > 1 && same == top.len() {
811 format!("{} × {kind}", top.len())
812 } else {
813 kind.to_owned()
814 })
815 }
816
817 /// Whether `zpool status` reported data errors or device errors, which
818 /// a healthy-looking ONLINE state does not show.
819 pub fn has_errors(&self) -> bool {
820 self.data_errors.unwrap_or(0) > 0 || self.devices.iter().any(|d| d.errors() > 0)
821 }
822
823 /// Its devices (not vdevs) with errors or in a bad state.
824 pub fn troubled_devices(&self) -> impl Iterator<Item = &ZfsDevice> {
825 self.devices
826 .iter()
827 .filter(|d| !d.is_group() && (d.errors() > 0 || d.is_unhealthy()))
828 }
829
830 /// Whether the pool is healthy.
831 pub fn is_online(&self) -> bool {
832 self.state == "ONLINE"
833 }
834
835 /// Whether the pool lost redundancy or data is at risk (worse than an
836 /// administratively offline pool).
837 pub fn is_failing(&self) -> bool {
838 matches!(
839 self.state.as_str(),
840 "DEGRADED" | "FAULTED" | "UNAVAIL" | "REMOVED" | "SUSPENDED"
841 )
842 }
843}
844
845/// A btrfs filesystem as seen in `/sys/fs/btrfs/<fsid>`.
846#[derive(Debug, Clone, Default, Serialize, Deserialize)]
847#[serde(default)]
848pub struct BtrfsFilesystem {
849 /// Filesystem UUID.
850 pub fsid: String,
851 /// Filesystem label (may be empty).
852 pub label: String,
853 /// Mount points found in the mount table for this filesystem.
854 pub mount_points: Vec<String>,
855 /// Member block devices (e.g. `"nvme0n1"`, `"nvme1n1p1"`).
856 pub devices: Vec<String>,
857 /// Per-device error counters, keyed by btrfs device id. Each carries its
858 /// device name when it could be looked up (see [`BtrfsDeviceStats::name`]);
859 /// the list is not aligned with `devices`.
860 pub device_stats: Vec<BtrfsDeviceStats>,
861 /// Allocation per block group type (data, metadata, system).
862 pub allocations: Vec<BtrfsAllocation>,
863 /// Result of the most recent scrub, if it is known. Error counters in
864 /// `device_stats` only grow when data is read, so the scrub time tells
865 /// when all data was last verified.
866 pub scrub: Option<BtrfsScrub>,
867}
868
869/// Outcome of the last btrfs scrub (summed over devices).
870#[derive(Debug, Clone, Default, Serialize, Deserialize)]
871#[serde(default)]
872pub struct BtrfsScrub {
873 /// Start time, in milliseconds since the Unix epoch.
874 pub started_at_ms: u64,
875 /// Run time in seconds.
876 pub duration_secs: u64,
877 /// `"finished"`, `"canceled"` or `"unfinished"` (running or interrupted).
878 pub state: String,
879 /// Bytes verified.
880 pub bytes_scrubbed: u64,
881 /// Read, checksum, verify and superblock errors found.
882 pub errors: u64,
883 /// Errors that were repaired from a good copy.
884 pub corrected: u64,
885 /// Errors that could not be repaired.
886 pub uncorrectable: u64,
887}
888
889impl BtrfsFilesystem {
890 /// Returns true if any member is missing or has a non-zero error counter.
891 pub fn has_problems(&self) -> bool {
892 self.device_stats
893 .iter()
894 .any(|d| d.missing || d.total_errors() > 0)
895 }
896}
897
898/// Error counters of one btrfs member device.
899#[derive(Debug, Clone, Default, Serialize, Deserialize)]
900#[serde(default)]
901pub struct BtrfsDeviceStats {
902 /// btrfs device id (1-based).
903 pub devid: u64,
904 /// Device name (e.g. `"nvme0n1"`), from the `BTRFS_IOC_DEV_INFO` ioctl
905 /// on a mount point; empty when unknown (not mounted, device missing).
906 pub name: String,
907 /// Size of the device as btrfs uses it.
908 pub size_bytes: Option<u64>,
909 /// Space allocated on the device.
910 pub used_bytes: Option<u64>,
911 /// Whether the device is missing from the filesystem.
912 pub missing: bool,
913 /// Write I/O errors.
914 pub write_errs: u64,
915 /// Read I/O errors.
916 pub read_errs: u64,
917 /// Flush errors.
918 pub flush_errs: u64,
919 /// Checksum mismatches.
920 pub corruption_errs: u64,
921 /// Generation mismatches (stale data).
922 pub generation_errs: u64,
923}
924
925impl BtrfsDeviceStats {
926 /// Name for display: the device name, else `devid N`.
927 pub fn label(&self) -> String {
928 if self.name.is_empty() {
929 format!("devid {}", self.devid)
930 } else {
931 self.name.clone()
932 }
933 }
934
935 /// Sum of all error counters.
936 pub fn total_errors(&self) -> u64 {
937 self.write_errs
938 + self.read_errs
939 + self.flush_errs
940 + self.corruption_errs
941 + self.generation_errs
942 }
943}
944
945/// Space allocation of one btrfs block group type and RAID profile.
946#[derive(Debug, Clone, Default, Serialize, Deserialize)]
947#[serde(default)]
948pub struct BtrfsAllocation {
949 /// Block group type: `"data"`, `"metadata"` or `"system"`.
950 pub kind: String,
951 /// RAID profile (e.g. `"single"`, `"raid0"`, `"raid1c3"`).
952 pub profile: String,
953 /// Space allocated to this profile (logical size).
954 pub total_bytes: u64,
955 /// Space used within the allocation (logical size).
956 pub used_bytes: u64,
957}
958
959/// One Linux md array parsed from `/proc/mdstat`.
960#[derive(Debug, Clone, Default, Serialize, Deserialize)]
961#[serde(default)]
962pub struct MdArray {
963 /// Array name (e.g. `"md0"`).
964 pub name: String,
965 /// Whether the array is active.
966 pub active: bool,
967 /// Whether the array is read-only (`active (read-only)` / `(auto-read-only)`).
968 pub read_only: bool,
969 /// RAID level (e.g. `"raid1"`); empty for inactive arrays.
970 pub level: String,
971 /// Member devices.
972 pub members: Vec<MdMember>,
973 /// Number of devices the array should have (`[n/m]` -> n).
974 pub expected_devices: Option<u32>,
975 /// Number of devices currently in sync (`[n/m]` -> m).
976 pub working_devices: Option<u32>,
977 /// Ongoing resync/recovery/reshape/check, if any.
978 pub sync: Option<MdSync>,
979}
980
981impl MdArray {
982 /// Returns true if the array runs with fewer devices than expected or has a failed member.
983 pub fn is_degraded(&self) -> bool {
984 let short = match (self.expected_devices, self.working_devices) {
985 (Some(expected), Some(working)) => working < expected,
986 _ => false,
987 };
988 short || self.members.iter().any(|m| m.faulty)
989 }
990}
991
992/// A member device of an md array.
993#[derive(Debug, Clone, Default, Serialize, Deserialize)]
994#[serde(default)]
995pub struct MdMember {
996 /// Device name (e.g. `"sda1"`).
997 pub name: String,
998 /// Role number within the array.
999 pub role: u32,
1000 /// Marked faulty `(F)`.
1001 pub faulty: bool,
1002 /// Spare device `(S)`.
1003 pub spare: bool,
1004}
1005
1006/// Progress of an md background operation.
1007#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1008#[serde(default)]
1009pub struct MdSync {
1010 /// Operation name: `"resync"`, `"recovery"`, `"reshape"` or `"check"`.
1011 pub action: String,
1012 /// Completion percentage.
1013 pub percent: f32,
1014 /// Estimated minutes until completion.
1015 pub finish_minutes: Option<f32>,
1016 /// Current speed in KiB/s.
1017 pub speed_kib_per_sec: Option<u64>,
1018}
1019
1020/// Traffic of one network interface.
1021#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1022#[serde(default)]
1023pub struct NetworkStats {
1024 /// Interface name (e.g. `"enp9s0"`).
1025 pub name: String,
1026 /// Receive throughput.
1027 pub rx_bytes_per_sec: f64,
1028 /// Transmit throughput.
1029 pub tx_bytes_per_sec: f64,
1030 /// Bytes received since the interface came up.
1031 pub rx_total_bytes: u64,
1032 /// Bytes transmitted since the interface came up.
1033 pub tx_total_bytes: u64,
1034 /// Negotiated link speed in Mbit/s (wired links), if known.
1035 pub link_speed_mbps: Option<u32>,
1036 /// Whether the link is up, if the OS reports it.
1037 pub link_up: Option<bool>,
1038 /// Wi-Fi link details, for wireless interfaces.
1039 pub wireless: Option<WirelessLink>,
1040 /// The interface's addresses with their prefix length
1041 /// (`"192.168.1.10/24"`), IPv4 first; IPv6 link-local ones are left out.
1042 pub addresses: Vec<String>,
1043}
1044
1045/// State of a Wi-Fi link.
1046#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1047#[serde(default)]
1048pub struct WirelessLink {
1049 /// Current receive bit rate in Mbit/s.
1050 pub rx_bitrate_mbps: Option<f32>,
1051 /// Current transmit bit rate in Mbit/s.
1052 pub tx_bitrate_mbps: Option<f32>,
1053 /// Signal strength in dBm.
1054 pub signal_dbm: Option<i32>,
1055}
1056
1057/// One process.
1058#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1059#[serde(default)]
1060pub struct ProcessInfo {
1061 /// Process id.
1062 pub pid: u32,
1063 /// Parent process id.
1064 pub parent_pid: Option<u32>,
1065 /// Short name (executable name).
1066 pub name: String,
1067 /// Full command line joined by spaces.
1068 pub command: String,
1069 /// Owner user name (or numeric uid when it cannot be resolved).
1070 pub user: String,
1071 /// Scheduling state (e.g. "Run", "Sleep", "Zombie").
1072 pub state: String,
1073 /// CPU usage; 100 means one full core, so multi-threaded processes may exceed 100.
1074 pub cpu_percent: f32,
1075 /// Resident memory.
1076 pub memory_bytes: u64,
1077 /// Number of threads, where known.
1078 pub threads: Option<u32>,
1079 /// Nice value, -20 (first) to 19 (last), where known (Linux).
1080 pub nice: Option<i32>,
1081 /// Disk read throughput.
1082 pub read_bytes_per_sec: f64,
1083 /// Disk write throughput.
1084 pub write_bytes_per_sec: f64,
1085 /// Seconds since the process started.
1086 pub run_time_secs: u64,
1087 /// Kernel thread (e.g. `kworker`), which front-ends may hide.
1088 pub kernel_thread: bool,
1089 /// The container it runs in (Docker, Podman, ... name, or short id),
1090 /// if any.
1091 pub container: Option<String>,
1092}
1093
1094#[cfg(test)]
1095mod tests {
1096 use super::*;
1097
1098 fn process(pid: u32, cpu: f32) -> ProcessInfo {
1099 ProcessInfo {
1100 pid,
1101 cpu_percent: cpu,
1102 ..Default::default()
1103 }
1104 }
1105
1106 #[test]
1107 fn cpu_power_falls_back_to_a_laptop_apu_only() {
1108 let power = |cpu: Option<f32>, igpu: Option<f32>| PowerStats {
1109 cpu_package_w: cpu,
1110 igpu_w: igpu,
1111 ..Default::default()
1112 };
1113 assert_eq!(
1114 power(Some(30.0), Some(6.0)).cpu_or_soc_w(),
1115 ("CPU", Some(30.0))
1116 );
1117 assert_eq!(power(None, Some(6.0)).cpu_or_soc_w(), ("SoC", Some(6.0)));
1118 // A desktop Ryzen iGPU's few milliwatts are not the SoC.
1119 assert_eq!(power(None, Some(0.007)).cpu_or_soc_w(), ("CPU", None));
1120 assert_eq!(power(None, None).cpu_or_soc_w(), ("CPU", None));
1121 }
1122
1123 #[test]
1124 fn truncate_keeps_top_cpu_processes_in_order() {
1125 let mut snapshot = Snapshot {
1126 processes: vec![
1127 process(1, 5.0),
1128 process(2, 50.0),
1129 process(3, 0.0),
1130 process(4, 20.0),
1131 ],
1132 process_count: 4,
1133 ..Default::default()
1134 };
1135 snapshot.truncate_processes(2);
1136 let pids: Vec<u32> = snapshot.processes.iter().map(|p| p.pid).collect();
1137 assert_eq!(pids, vec![2, 4]);
1138 // The total is preserved for display ("2 of 4 processes").
1139 assert_eq!(snapshot.process_count, 4);
1140 }
1141
1142 #[test]
1143 fn truncate_leaves_out_kernel_threads() {
1144 let kworker = |pid: u32, cpu: f32| ProcessInfo {
1145 kernel_thread: true,
1146 ..process(pid, cpu)
1147 };
1148 let mut snapshot = Snapshot {
1149 processes: vec![
1150 kworker(1, 9.0),
1151 process(2, 5.0),
1152 kworker(3, 8.0),
1153 process(4, 1.0),
1154 process(5, 0.0),
1155 ],
1156 process_count: 5,
1157 ..Default::default()
1158 };
1159 snapshot.truncate_processes(3);
1160 let pids: Vec<u32> = snapshot.processes.iter().map(|p| p.pid).collect();
1161 assert_eq!(pids, vec![2, 4, 5], "busy kworkers take no places");
1162 assert_eq!(snapshot.process_count, 5);
1163 }
1164
1165 #[test]
1166 fn truncate_keeps_watched_processes() {
1167 let mut snapshot = Snapshot {
1168 processes: (1..=6).map(|pid| process(pid, pid as f32)).collect(),
1169 watched_pids: vec![1, 99],
1170 ..Default::default()
1171 };
1172 snapshot.truncate_processes(2);
1173 let pids: Vec<u32> = snapshot.processes.iter().map(|p| p.pid).collect();
1174 assert_eq!(pids, vec![6, 5, 1]);
1175 }
1176
1177 #[test]
1178 fn snapshot_round_trips_through_json() {
1179 let snapshot = Snapshot {
1180 timestamp_ms: 42,
1181 processes: vec![process(7, 1.5)],
1182 ..Default::default()
1183 };
1184 let json = serde_json::to_string(&snapshot).unwrap();
1185 let back: Snapshot = serde_json::from_str(&json).unwrap();
1186 assert_eq!(back.timestamp_ms, 42);
1187 assert_eq!(back.processes[0].pid, 7);
1188 }
1189
1190 #[test]
1191 fn device_matching_handles_nvme_namespaces() {
1192 assert!(device_matches_disk("sda", "sda"));
1193 assert!(device_matches_disk("nvme0", "nvme0n1"));
1194 assert!(device_matches_disk("nvme1", "nvme1n1"));
1195 assert!(!device_matches_disk("nvme1", "nvme10n1"));
1196 assert!(!device_matches_disk("sd", "sda"));
1197 assert!(!device_matches_disk("sda", "sdb"));
1198 }
1199
1200 #[test]
1201 fn smart_attention_flags() {
1202 let healthy = SmartDevice {
1203 passed: Some(true),
1204 media_errors: Some(0),
1205 available_spare_percent: Some(100),
1206 available_spare_threshold_percent: Some(10),
1207 ..Default::default()
1208 };
1209 assert!(!healthy.needs_attention());
1210 let worn = SmartDevice {
1211 available_spare_percent: Some(5),
1212 ..healthy.clone()
1213 };
1214 assert!(worn.needs_attention());
1215 let reallocated = SmartDevice {
1216 reallocated_sectors: Some(8),
1217 ..healthy
1218 };
1219 assert!(reallocated.needs_attention());
1220 }
1221
1222 #[test]
1223 fn md_array_degraded_detection() {
1224 let mut array = MdArray {
1225 expected_devices: Some(2),
1226 working_devices: Some(2),
1227 ..Default::default()
1228 };
1229 assert!(!array.is_degraded());
1230 array.working_devices = Some(1);
1231 assert!(array.is_degraded());
1232 }
1233}