monitrs_core/diagnostics/rules/
storage.rs

1//! Sustained block-device busy state, where the platform supports it (§11.2, §7.3).
2
3use crate::model::{
4    MeasuredValue, Measurement, MetricState, PressureId, PressureState, Severity, SystemSnapshot,
5};
6use crate::units::{Ellipsis, format_age, truncate_tail};
7
8use super::super::{DiagnosticRule, Evidence, Finding, HistoryWindow, Thresholds};
9use super::{SUSTAINED_CONFIDENCE, as_percent};
10
11/// Rule id for a device that has been busy for a sustained period.
12pub const DISK_SUSTAINED_BUSY: &str = "disk.sustained_busy";
13
14/// Display width a device name is truncated to in the summary.
15const DEVICE_NAME_WIDTH: usize = 24;
16
17/// How many devices the summary names.
18const SUMMARY_DEVICES: usize = 3;
19
20/// A block device busy for a sustained period (§11.2).
21///
22/// # Why this rule reads the radar instead of counting history
23///
24/// History retains device *throughput*, not the busy share (§8.5) — a busy
25/// percentage per device per sample would grow every retained sample with the
26/// machine's device count. The sustained-ness of device busy is therefore
27/// established by the hysteresis the [`PressureEngine`](super::super::PressureEngine)
28/// already applies to the disk signal, and this rule reports it with the evidence
29/// the engine kept. The consequence is an ordering requirement, documented on the
30/// engine: pressure must be derived before the rules run. Until it is, the signal
31/// is warming up and this rule stays silent — which is the correct answer, not a
32/// missed detection.
33///
34/// The finding says the device is busy. §11.3 forbids concluding anything about the
35/// health of the hardware from it: a device at 100% busy is usually a device doing
36/// its job.
37#[derive(Clone, Copy, Debug)]
38pub struct DiskBusyRule {
39    thresholds: Thresholds,
40}
41
42impl DiskBusyRule {
43    /// Builds the rule from sanitized thresholds.
44    #[must_use]
45    pub const fn new(thresholds: Thresholds) -> Self {
46        Self { thresholds }
47    }
48}
49
50impl DiagnosticRule for DiskBusyRule {
51    fn id(&self) -> &'static str {
52        DISK_SUSTAINED_BUSY
53    }
54
55    fn evaluate(&self, current: &SystemSnapshot, _history: &HistoryWindow<'_>) -> Option<Finding> {
56        let signal = current.pressure.signal(PressureId::Disk)?;
57        let state = *signal.state.fresh()?;
58        let severity = match state {
59            PressureState::Normal => return None,
60            PressureState::Watch => Severity::Watch,
61            PressureState::Critical => Severity::Critical,
62        };
63        let threshold = if severity == Severity::Critical {
64            self.thresholds.disk_busy_critical_percent
65        } else {
66            self.thresholds.disk_busy_watch_percent
67        };
68
69        let mut evidence = vec![Evidence::current(Measurement::new(
70            "threshold",
71            MeasuredValue::Percent(as_percent(threshold)),
72        ))];
73        if let Some(raw) = signal.raw {
74            evidence.push(Evidence::current(raw));
75        }
76        if let Some(held) = signal.held_for {
77            evidence.push(Evidence::current(Measurement::new(
78                "held for",
79                MeasuredValue::Duration(held),
80            )));
81        }
82
83        // Name the devices that actually reported a busy share, so the finding
84        // points at a device rather than at "the disk".
85        let mut devices: Vec<(&str, crate::units::Percent)> = current
86            .disks
87            .iter()
88            .filter_map(|disk| disk.busy.fresh().map(|busy| (&*disk.device, *busy)))
89            .collect();
90        devices.sort_by(|left, right| {
91            right
92                .1
93                .value()
94                .total_cmp(&left.1.value())
95                .then_with(|| left.0.cmp(right.0))
96        });
97        let named: Vec<String> = devices
98            .iter()
99            .take(SUMMARY_DEVICES)
100            .map(|(device, busy)| {
101                format!(
102                    "{} {busy}",
103                    truncate_tail(device, DEVICE_NAME_WIDTH, Ellipsis::Ascii)
104                )
105            })
106            .collect();
107        let held = signal
108            .held_for
109            .map_or_else(|| "the sustained window".to_owned(), format_age);
110        let observed = if named.is_empty() {
111            String::new()
112        } else {
113            format!(" Busiest observed devices: {}.", named.join(", "))
114        };
115        let summary = format!(
116            "A block device has been at or above {} busy for {held}. Device busy is the share of \
117             wall time with at least one request in flight; it is not filesystem capacity, and a \
118             device working hard is not a device in trouble.{observed}",
119            as_percent(threshold),
120        );
121
122        Some(
123            Finding::new(
124                DISK_SUSTAINED_BUSY,
125                severity,
126                "Disk device sustained busy",
127                summary,
128                SUSTAINED_CONFIDENCE,
129            )
130            .with_evidence(evidence),
131        )
132    }
133}
134
135/// Whether a snapshot's disk signal has been derived yet.
136///
137/// Exposed for the runtime's benefit: a caller that wants disk findings must run
138/// the pressure engine first, and this is how it can assert that it did.
139#[must_use]
140pub fn disk_signal_ready(current: &SystemSnapshot) -> bool {
141    current
142        .pressure
143        .signal(PressureId::Disk)
144        .is_some_and(|signal| matches!(signal.state, MetricState::Available(_)))
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150    use crate::diagnostics::PressureEngine;
151    use crate::diagnostics::fixtures::{Timeline, set_cpu, set_disk_busy, snapshot};
152    use core::time::Duration;
153
154    fn rule() -> DiskBusyRule {
155        DiskBusyRule::new(Thresholds::default().sanitized())
156    }
157
158    /// Runs the engine over `count` snapshots whose busiest device reports `busy`,
159    /// returning the last snapshot with its pressure filled in — the order the
160    /// runtime uses.
161    fn derived(busy: f32, count: usize) -> (Timeline, SystemSnapshot) {
162        let mut engine = PressureEngine::default();
163        let mut timeline = Timeline::new(Duration::from_secs(1));
164        let mut current = timeline.push(|snapshot| {
165            set_cpu(snapshot, 5.0);
166            set_disk_busy(snapshot, "nvme0n1", busy);
167        });
168        current.pressure = engine.observe(&current);
169        for _ in 1..count {
170            let mut next = timeline.push(|snapshot| {
171                set_cpu(snapshot, 5.0);
172                set_disk_busy(snapshot, "nvme0n1", busy);
173            });
174            next.pressure = engine.observe(&next);
175            current = next;
176        }
177        (timeline, current)
178    }
179
180    #[test]
181    fn a_quiet_device_produces_no_finding() {
182        let (timeline, current) = derived(4.0, 20);
183        assert!(rule().evaluate(&current, &timeline.window()).is_none());
184    }
185
186    #[test]
187    fn a_device_busy_for_the_sustained_window_is_a_finding() {
188        let (timeline, current) = derived(85.0, 20);
189        let finding = rule()
190            .evaluate(&current, &timeline.window())
191            .expect("the disk signal has escalated");
192
193        assert_eq!(finding.severity, Severity::Watch);
194        assert_eq!(finding.rule_id, DISK_SUSTAINED_BUSY);
195        assert!(
196            finding.summary.contains("nvme0n1 85%"),
197            "the finding must name the device: {}",
198            finding.summary
199        );
200        assert!(
201            finding.summary.contains("not filesystem capacity"),
202            "§7.3 keeps the two metrics apart: {}",
203            finding.summary
204        );
205    }
206
207    #[test]
208    fn a_saturated_device_escalates_to_critical() {
209        let (timeline, current) = derived(99.0, 20);
210        let finding = rule()
211            .evaluate(&current, &timeline.window())
212            .expect("the disk signal has escalated");
213        assert_eq!(finding.severity, Severity::Critical);
214        let labels: Vec<&str> = finding
215            .evidence
216            .iter()
217            .map(|item| item.measurement.label)
218            .collect();
219        assert!(labels.contains(&"device busy"), "{labels:?}");
220        assert!(labels.contains(&"held for"), "{labels:?}");
221    }
222
223    #[test]
224    fn a_brief_burst_does_not_fire_because_the_signal_has_not_escalated() {
225        let (timeline, current) = derived(99.0, 5);
226        assert!(
227            rule().evaluate(&current, &timeline.window()).is_none(),
228            "five samples cannot sustain a ten-sample condition"
229        );
230    }
231
232    #[test]
233    fn nothing_fires_before_pressure_has_been_derived() {
234        let mut timeline = Timeline::new(Duration::from_secs(1));
235        let current = timeline.push_many(20, |snapshot| set_disk_busy(snapshot, "nvme0n1", 99.0));
236        assert!(!disk_signal_ready(&current));
237        assert!(
238            rule().evaluate(&current, &timeline.window()).is_none(),
239            "the collector's warming-up radar carries no state to report"
240        );
241    }
242
243    #[test]
244    fn a_platform_without_a_busy_figure_never_fires() {
245        let mut engine = PressureEngine::default();
246        let mut timeline = Timeline::new(Duration::from_secs(1));
247        let mut current = snapshot();
248        for _ in 0..20 {
249            let mut next = timeline.push(|snapshot| {
250                set_disk_busy(snapshot, "disk0", 99.0);
251                if let Some(device) = snapshot.disks.first_mut() {
252                    device.busy = MetricState::Unsupported;
253                }
254            });
255            next.pressure = engine.observe(&next);
256            current = next;
257        }
258        assert!(!disk_signal_ready(&current));
259        assert!(rule().evaluate(&current, &timeline.window()).is_none());
260    }
261
262    #[test]
263    fn the_busiest_device_is_named_first() {
264        let mut engine = PressureEngine::default();
265        let mut timeline = Timeline::new(Duration::from_secs(1));
266        let mut current = snapshot();
267        for _ in 0..20 {
268            let mut next = timeline.push(|snapshot| {
269                set_disk_busy(snapshot, "nvme0n1", 30.0);
270                set_disk_busy(snapshot, "nvme1n1", 96.0);
271            });
272            next.pressure = engine.observe(&next);
273            current = next;
274        }
275        let finding = rule()
276            .evaluate(&current, &timeline.window())
277            .expect("the busiest device escalated the signal");
278        assert!(
279            finding
280                .summary
281                .contains("Busiest observed devices: nvme1n1 96%, nvme0n1 30%"),
282            "{}",
283            finding.summary
284        );
285    }
286}
monitrs_core/diagnostics/rules/storage.rs

monitrs_core/diagnostics/rules/
storage.rs
