monitrs_core/model/
health.rs

1//! Collector health and self-overhead.
2//!
3//! §26: *a system monitor must measure and expose its own overhead.* §16.1 sets
4//! budgets for it, §11.2 has diagnostic rules that fire when they are exceeded,
5//! and §7.5 renders the result. All three read this type.
6
7use core::time::Duration;
8
9use crate::model::MetricState;
10use crate::units::Percent;
11
12/// Which sampling tier a measurement belongs to (§8.6).
13#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
14#[cfg_attr(feature = "serde", derive(serde::Serialize))]
15#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
16pub enum Tier {
17    /// CPU, memory, processes, network and disk counters. Default 1 s.
18    Fast,
19    /// Filesystem capacity, static device state, sensors. Default 5 s.
20    Medium,
21    /// Users, device lists, cgroup metadata. Default 30 s.
22    Slow,
23    /// Selected-process details, ancestry, open files.
24    OnDemand,
25}
26
27impl Tier {
28    /// Lower-case label.
29    #[must_use]
30    pub const fn label(self) -> &'static str {
31        match self {
32            Self::Fast => "fast",
33            Self::Medium => "medium",
34            Self::Slow => "slow",
35            Self::OnDemand => "on demand",
36        }
37    }
38
39    /// All tiers, in the order the Inspect screen lists them.
40    pub const ALL: [Self; 4] = [Self::Fast, Self::Medium, Self::Slow, Self::OnDemand];
41}
42
43/// Timing and failure counts for one tier.
44#[derive(Clone, Copy, Debug, Default, PartialEq)]
45#[cfg_attr(feature = "serde", derive(serde::Serialize))]
46pub struct TierHealth {
47    /// How long the most recent collection took.
48    pub last_duration: Duration,
49    /// The slowest collection observed in this run.
50    pub max_duration: Duration,
51    /// A running estimate of the 95th percentile, against the §16.1 budget.
52    pub p95_duration: Duration,
53    /// Completed collections.
54    pub completed: u64,
55    /// Collections that returned an error.
56    pub failed: u64,
57    /// How long ago this tier last completed, for staleness display.
58    pub since_last: Option<Duration>,
59}
60
61impl TierHealth {
62    /// Whether this tier has produced at least one successful collection.
63    #[must_use]
64    pub const fn has_sampled(&self) -> bool {
65        self.completed > 0
66    }
67}
68
69/// A recurring collector problem, aggregated rather than logged per occurrence.
70///
71/// §9.2 forbids logging one error per vanished process, and the same reasoning
72/// applies on screen: a repeated failure is one row with a count, not a flood.
73#[derive(Clone, Debug, PartialEq)]
74#[cfg_attr(feature = "serde", derive(serde::Serialize))]
75pub struct CollectorIssue {
76    /// Where it came from, e.g. `"/proc/diskstats"`.
77    pub source: Box<str>,
78    /// What went wrong.
79    pub message: Box<str>,
80    /// How many times it has happened.
81    pub occurrences: u32,
82    /// How long ago it last happened.
83    pub last_seen: Option<Duration>,
84}
85
86/// monitrs's own resource use.
87#[derive(Clone, Copy, Debug, PartialEq)]
88#[cfg_attr(feature = "serde", derive(serde::Serialize))]
89pub struct SelfOverhead {
90    /// Our own CPU usage, core-normalized. Budget: median < 1% (§16.1).
91    pub cpu: Percent,
92    /// Our own resident memory. Budget: < 50 MiB by default (§16.1).
93    pub rss_bytes: u64,
94    /// Bytes the history ring currently occupies, against the configured budget.
95    pub history_bytes: u64,
96    /// Open file descriptors, watched for the unbounded-growth check (§16.1).
97    pub open_files: MetricState<u32>,
98}
99
100/// The maximum number of distinct issues retained.
101///
102/// Bounded because §10.3 forbids unbounded accumulation anywhere in the
103/// pipeline, and this list is written from the sampler thread.
104pub const MAX_RETAINED_ISSUES: usize = 16;
105
106/// Overall collector health.
107#[derive(Clone, Debug, Default, PartialEq)]
108#[cfg_attr(feature = "serde", derive(serde::Serialize))]
109pub struct CollectorHealth {
110    /// Fast tier timing.
111    pub fast: TierHealth,
112    /// Medium tier timing.
113    pub medium: TierHealth,
114    /// Slow tier timing.
115    pub slow: TierHealth,
116    /// On-demand detail worker timing.
117    pub on_demand: TierHealth,
118    /// Snapshots dropped because the channel was full.
119    pub dropped_samples: u64,
120    /// Snapshots superseded before the UI rendered them (§10.3).
121    pub coalesced_samples: u64,
122    /// How far behind live the most recent snapshot is.
123    ///
124    /// Rendered in the header when it exceeds the sample interval, which is what
125    /// §16.2 means by "display collector lag".
126    pub lag: Duration,
127    /// Distinct problems, at most [`MAX_RETAINED_ISSUES`].
128    pub issues: Vec<CollectorIssue>,
129    /// Our own overhead.
130    pub self_overhead: Option<SelfOverhead>,
131}
132
133impl CollectorHealth {
134    /// Timing for one tier.
135    #[must_use]
136    pub const fn tier(&self, tier: Tier) -> &TierHealth {
137        match tier {
138            Tier::Fast => &self.fast,
139            Tier::Medium => &self.medium,
140            Tier::Slow => &self.slow,
141            Tier::OnDemand => &self.on_demand,
142        }
143    }
144
145    /// Records an issue, merging it into an existing entry when the source and
146    /// message match, and dropping it once the list is full.
147    ///
148    /// Dropping rather than evicting keeps the *first* distinct failures, which
149    /// are usually the root cause; a later flood cannot push them out.
150    pub fn record_issue(&mut self, source: &str, message: &str, since_start: Duration) {
151        if let Some(existing) = self
152            .issues
153            .iter_mut()
154            .find(|issue| &*issue.source == source && &*issue.message == message)
155        {
156            existing.occurrences = existing.occurrences.saturating_add(1);
157            existing.last_seen = Some(since_start);
158            return;
159        }
160        if self.issues.len() >= MAX_RETAINED_ISSUES {
161            return;
162        }
163        self.issues.push(CollectorIssue {
164            source: source.into(),
165            message: message.into(),
166            occurrences: 1,
167            last_seen: Some(since_start),
168        });
169    }
170
171    /// Whether the collector is behind by more than one sample interval.
172    #[must_use]
173    pub fn is_behind(&self, interval: Duration) -> bool {
174        self.lag > interval
175    }
176}
177
178#[cfg(test)]
179mod tests {
180    use super::*;
181
182    #[test]
183    fn repeated_issues_are_aggregated_rather_than_duplicated() {
184        let mut health = CollectorHealth::default();
185        for _ in 0..1_000 {
186            health.record_issue("/proc/diskstats", "read failed", Duration::from_secs(1));
187        }
188        assert_eq!(health.issues.len(), 1);
189        assert_eq!(health.issues.first().map(|i| i.occurrences), Some(1_000));
190    }
191
192    #[test]
193    fn distinct_issues_are_kept_separate() {
194        let mut health = CollectorHealth::default();
195        health.record_issue("/proc/diskstats", "read failed", Duration::ZERO);
196        health.record_issue("/proc/net/dev", "read failed", Duration::ZERO);
197        health.record_issue("/proc/diskstats", "parse failed", Duration::ZERO);
198        assert_eq!(health.issues.len(), 3);
199    }
200
201    #[test]
202    fn the_issue_list_is_bounded_and_keeps_the_earliest_distinct_failures() {
203        let mut health = CollectorHealth::default();
204        for index in 0..(MAX_RETAINED_ISSUES * 4) {
205            health.record_issue("source", &format!("failure {index}"), Duration::ZERO);
206        }
207        assert_eq!(health.issues.len(), MAX_RETAINED_ISSUES);
208        assert_eq!(
209            health.issues.first().map(|i| &*i.message),
210            Some("failure 0"),
211            "a later flood must not evict the root cause"
212        );
213    }
214
215    #[test]
216    fn lag_is_reported_only_beyond_one_interval() {
217        let mut health = CollectorHealth {
218            lag: Duration::from_millis(900),
219            ..CollectorHealth::default()
220        };
221        assert!(!health.is_behind(Duration::from_secs(1)));
222        health.lag = Duration::from_millis(1_100);
223        assert!(health.is_behind(Duration::from_secs(1)));
224    }
225
226    #[test]
227    fn a_fresh_health_record_has_not_sampled_any_tier() {
228        let health = CollectorHealth::default();
229        for tier in Tier::ALL {
230            assert!(!health.tier(tier).has_sampled(), "{tier:?}");
231        }
232        assert_eq!(health.dropped_samples, 0);
233        assert_eq!(health.coalesced_samples, 0);
234    }
235}
monitrs_core/model/health.rs

monitrs_core/model/
health.rs
