monitrs_core/diagnostics/rules/
psi.rs

1//! Linux pressure-stall information rules (§11.2, §9.2).
2//!
3//! PSI is the one place where the kernel does the hard part for us: it reports how
4//! much time tasks spent *stalled* waiting for a resource, which is a far better
5//! signal than utilization. Two properties follow, and both shape these rules:
6//!
7//! * A single read of `some avg10` already summarizes ten seconds, so the evidence
8//!   window is those ten seconds rather than one sample — no extra counting needed.
9//! * PSI exists only on Linux, and only on kernels built with it. Everywhere else
10//!   these rules produce nothing at all, which is not the same as reporting that
11//!   there is no pressure (§4).
12
13use core::time::Duration;
14
15use crate::model::{MeasuredValue, Measurement, PressureId, PsiResource, Severity, SystemSnapshot};
16
17use super::super::{
18    DiagnosticRule, Evidence, Finding, HistoryWindow, Thresholds, TimeWindow, signals,
19};
20use super::{SUSTAINED_CONFIDENCE, as_percent, escalate, ratio};
21
22/// Rule id for elevated Linux memory PSI.
23pub const PSI_MEMORY_ELEVATED: &str = "psi.memory_elevated";
24/// Rule id for elevated Linux I/O PSI.
25pub const PSI_IO_ELEVATED: &str = "psi.io_elevated";
26
27/// The spans the three `some` averages cover.
28const AVG10: Duration = Duration::from_secs(10);
29const AVG60: Duration = Duration::from_secs(60);
30const AVG300: Duration = Duration::from_secs(300);
31
32/// Builds the finding shared by both PSI rules.
33///
34/// `noun` is the resource as it appears in prose, `waiting_for` completes the
35/// sentence describing what a stall means for that resource. Neither string draws a
36/// conclusion beyond what PSI measures (§11.3).
37fn evaluate_psi(
38    rule_id: &'static str,
39    id: PressureId,
40    title: &'static str,
41    waiting_for: &'static str,
42    thresholds: &Thresholds,
43    current: &SystemSnapshot,
44) -> Option<Finding> {
45    let psi = current.pressure.psi.fresh()?;
46    let resource: &PsiResource = signals::psi_resource(psi, id);
47    let some10 = f64::from(resource.some_avg10.value());
48
49    let severity = escalate(
50        some10 >= f64::from(thresholds.psi_watch_percent),
51        some10 >= f64::from(thresholds.psi_critical_percent),
52    )?;
53    let threshold = if severity == Severity::Critical {
54        thresholds.psi_critical_percent
55    } else {
56        thresholds.psi_watch_percent
57    };
58
59    let mut evidence = vec![
60        Evidence::new(
61            Measurement::new("some avg10", MeasuredValue::Percent(resource.some_avg10)),
62            TimeWindow::moving_average(AVG10),
63        ),
64        Evidence::new(
65            Measurement::new("some avg60", MeasuredValue::Percent(resource.some_avg60)),
66            TimeWindow::moving_average(AVG60),
67        ),
68        Evidence::new(
69            Measurement::new("some avg300", MeasuredValue::Percent(resource.some_avg300)),
70            TimeWindow::moving_average(AVG300),
71        ),
72        Evidence::current(Measurement::new(
73            "threshold",
74            MeasuredValue::Percent(as_percent(threshold)),
75        )),
76        Evidence::current(Measurement::new(
77            "total stalled",
78            MeasuredValue::Duration(resource.total_stalled),
79        )),
80    ];
81    // `full` is absent for some resources on some kernels, so it is extra evidence
82    // rather than part of the condition (§4).
83    if let Some(full) = resource.full_avg10.fresh() {
84        evidence.push(Evidence::new(
85            Measurement::new("full avg10", MeasuredValue::Percent(*full)),
86            TimeWindow::moving_average(AVG10),
87        ));
88    }
89
90    let multiple = ratio(some10, f64::from(threshold))
91        .map_or_else(String::new, |value| format!(" ({value:.1}x the threshold)"));
92    let summary = format!(
93        "The kernel reports at least one task stalled {waiting_for} for {} of the last 10 seconds{multiple}, \
94         and {} of the last 60. A pressure share is stalled time, not utilization.",
95        resource.some_avg10, resource.some_avg60,
96    );
97
98    Some(
99        Finding::new(rule_id, severity, title, summary, SUSTAINED_CONFIDENCE)
100            .with_evidence(evidence),
101    )
102}
103
104/// Linux memory PSI elevated (§11.2).
105///
106/// Reports stalled time waiting on memory reclaim. It deliberately stops there:
107/// §11.3 forbids concluding anything about an impending kill or an allocation
108/// pattern from a stall share.
109#[derive(Clone, Copy, Debug)]
110pub struct MemoryPsiElevatedRule {
111    thresholds: Thresholds,
112}
113
114impl MemoryPsiElevatedRule {
115    /// Builds the rule from sanitized thresholds.
116    #[must_use]
117    pub const fn new(thresholds: Thresholds) -> Self {
118        Self { thresholds }
119    }
120}
121
122impl DiagnosticRule for MemoryPsiElevatedRule {
123    fn id(&self) -> &'static str {
124        PSI_MEMORY_ELEVATED
125    }
126
127    fn evaluate(&self, current: &SystemSnapshot, _history: &HistoryWindow<'_>) -> Option<Finding> {
128        evaluate_psi(
129            PSI_MEMORY_ELEVATED,
130            PressureId::PsiMemory,
131            "Linux memory pressure stalls elevated",
132            "on memory reclaim",
133            &self.thresholds,
134            current,
135        )
136    }
137}
138
139/// Linux I/O PSI elevated (§11.2).
140#[derive(Clone, Copy, Debug)]
141pub struct IoPsiElevatedRule {
142    thresholds: Thresholds,
143}
144
145impl IoPsiElevatedRule {
146    /// Builds the rule from sanitized thresholds.
147    #[must_use]
148    pub const fn new(thresholds: Thresholds) -> Self {
149        Self { thresholds }
150    }
151}
152
153impl DiagnosticRule for IoPsiElevatedRule {
154    fn id(&self) -> &'static str {
155        PSI_IO_ELEVATED
156    }
157
158    fn evaluate(&self, current: &SystemSnapshot, _history: &HistoryWindow<'_>) -> Option<Finding> {
159        evaluate_psi(
160            PSI_IO_ELEVATED,
161            PressureId::PsiIo,
162            "Linux I/O pressure stalls elevated",
163            "on block i/o",
164            &self.thresholds,
165            current,
166        )
167    }
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173    use crate::diagnostics::fixtures::{Timeline, set_psi, snapshot};
174    use crate::model::{Confidence, MetricState};
175
176    fn memory_rule() -> MemoryPsiElevatedRule {
177        MemoryPsiElevatedRule::new(Thresholds::default().sanitized())
178    }
179
180    fn io_rule() -> IoPsiElevatedRule {
181        IoPsiElevatedRule::new(Thresholds::default().sanitized())
182    }
183
184    #[test]
185    fn no_psi_data_produces_no_finding_on_either_rule() {
186        let timeline = Timeline::new(Duration::from_secs(1));
187        let window = timeline.window();
188        let snapshot = snapshot();
189        assert!(snapshot.pressure.psi.fresh().is_none());
190        assert!(memory_rule().evaluate(&snapshot, &window).is_none());
191        assert!(io_rule().evaluate(&snapshot, &window).is_none());
192    }
193
194    #[test]
195    fn quiet_psi_produces_no_finding() {
196        let timeline = Timeline::new(Duration::from_secs(1));
197        let mut snapshot = snapshot();
198        set_psi(&mut snapshot, 30.0, 0.4, 0.9);
199        assert!(
200            memory_rule()
201                .evaluate(&snapshot, &timeline.window())
202                .is_none()
203        );
204        assert!(io_rule().evaluate(&snapshot, &timeline.window()).is_none());
205    }
206
207    #[test]
208    fn each_rule_reads_only_its_own_resource() {
209        let timeline = Timeline::new(Duration::from_secs(1));
210        let mut snapshot = snapshot();
211        set_psi(&mut snapshot, 90.0, 12.0, 0.0);
212
213        let memory = memory_rule()
214            .evaluate(&snapshot, &timeline.window())
215            .expect("memory psi is elevated");
216        assert_eq!(memory.rule_id, PSI_MEMORY_ELEVATED);
217        assert_eq!(memory.severity, Severity::Watch);
218        assert!(
219            io_rule().evaluate(&snapshot, &timeline.window()).is_none(),
220            "an idle i/o resource must not inherit the memory reading"
221        );
222    }
223
224    #[test]
225    fn elevated_io_psi_escalates_to_critical() {
226        let timeline = Timeline::new(Duration::from_secs(1));
227        let mut snapshot = snapshot();
228        set_psi(&mut snapshot, 0.0, 0.0, 62.0);
229        let finding = io_rule()
230            .evaluate(&snapshot, &timeline.window())
231            .expect("62% stalled is critical");
232        assert_eq!(finding.severity, Severity::Critical);
233        assert_eq!(finding.title, "Linux I/O pressure stalls elevated");
234    }
235
236    #[test]
237    fn the_evidence_window_is_the_span_the_average_covers() {
238        let timeline = Timeline::new(Duration::from_secs(1));
239        let mut snapshot = snapshot();
240        set_psi(&mut snapshot, 0.0, 45.0, 0.0);
241        let finding = memory_rule()
242            .evaluate(&snapshot, &timeline.window())
243            .expect("elevated memory psi");
244
245        let avg10 = finding
246            .evidence
247            .iter()
248            .find(|item| item.measurement.label == "some avg10")
249            .expect("avg10 is evidence");
250        assert_eq!(avg10.window.span, AVG10);
251        assert!(
252            !avg10.window.is_current_sample(),
253            "one read of avg10 still covers ten seconds"
254        );
255
256        let labels: Vec<&str> = finding
257            .evidence
258            .iter()
259            .map(|item| item.measurement.label)
260            .collect();
261        assert!(labels.contains(&"some avg60"), "{labels:?}");
262        assert!(labels.contains(&"full avg10"), "{labels:?}");
263        assert!(labels.contains(&"total stalled"), "{labels:?}");
264    }
265
266    #[test]
267    fn a_kernel_without_the_full_figure_still_produces_a_finding() {
268        let timeline = Timeline::new(Duration::from_secs(1));
269        let mut snapshot = snapshot();
270        set_psi(&mut snapshot, 0.0, 45.0, 0.0);
271        if let MetricState::Available(psi) = &mut snapshot.pressure.psi {
272            psi.memory.full_avg10 = MetricState::Unsupported;
273        }
274        let finding = memory_rule()
275            .evaluate(&snapshot, &timeline.window())
276            .expect("some avg10 is enough to fire");
277        assert!(
278            !finding
279                .evidence
280                .iter()
281                .any(|item| item.measurement.label == "full avg10")
282        );
283    }
284
285    #[test]
286    fn the_summary_says_a_pressure_share_is_not_a_utilization() {
287        let timeline = Timeline::new(Duration::from_secs(1));
288        let mut snapshot = snapshot();
289        set_psi(&mut snapshot, 0.0, 45.0, 0.0);
290        let finding = memory_rule()
291            .evaluate(&snapshot, &timeline.window())
292            .expect("elevated memory psi");
293        assert!(
294            finding.summary.contains("not utilization"),
295            "{}",
296            finding.summary
297        );
298        assert_eq!(finding.confidence, Confidence::Medium);
299    }
300
301    #[test]
302    fn stale_psi_does_not_produce_a_finding() {
303        let timeline = Timeline::new(Duration::from_secs(1));
304        let mut snapshot = snapshot();
305        set_psi(&mut snapshot, 0.0, 99.0, 0.0);
306        snapshot.pressure.psi = snapshot.pressure.psi.into_stale(Duration::from_secs(5));
307        assert!(
308            memory_rule()
309                .evaluate(&snapshot, &timeline.window())
310                .is_none(),
311            "a retained value is not a current measurement (§4)"
312        );
313    }
314}
monitrs_core/diagnostics/rules/psi.rs

monitrs_core/diagnostics/rules/
psi.rs
