rill_ml/diagnostics/
training_summary.rs

1//! Training summary statistics.
2//!
3//! Maintains bounded-memory summary statistics about the training process
4//! without storing raw samples. Useful for diagnostics and monitoring.
5//!
6//! Space complexity: `O(1)`.
7
8use crate::error::{RillError, ensure_finite};
9use crate::stats::ExponentiallyWeightedMean;
10use crate::traits::OnlineStatistic;
11
12/// Configuration for [`TrainingSummary`].
13#[derive(Debug, Clone)]
14#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
15pub struct TrainingSummaryConfig {
16    /// Alpha for the exponentially weighted recent error.
17    ///
18    /// Must be in `(0, 1]`. Smaller values give a longer memory.
19    pub error_alpha: f64,
20}
21
22impl Default for TrainingSummaryConfig {
23    fn default() -> Self {
24        Self { error_alpha: 0.1 }
25    }
26}
27
28/// Bounded-memory summary of a training process.
29///
30/// Tracks counts, recent/best errors, model switches, resets, and load
31/// failures. Does not store raw samples.
32///
33/// # Examples
34///
35/// ```
36/// use rill_ml::diagnostics::TrainingSummary;
37///
38/// let mut summary = TrainingSummary::default();
39/// summary.record_sample();
40/// summary.record_error(0.5).unwrap();
41/// summary.set_baseline_error(0.8).unwrap();
42///
43/// assert_eq!(summary.total_samples(), 1);
44/// assert!(summary.beats_baseline().unwrap());
45/// ```
46#[derive(Debug, Clone)]
47#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
48pub struct TrainingSummary {
49    total_samples: u64,
50    rejected_samples: u64,
51    error_ew: ExponentiallyWeightedMean,
52    best_error: Option<f64>,
53    baseline_error: Option<f64>,
54    model_switches: u64,
55    reset_count: u64,
56    load_failures: u64,
57}
58
59impl TrainingSummary {
60    /// Create a new training summary with the given configuration.
61    pub fn new(config: TrainingSummaryConfig) -> Result<Self, RillError> {
62        Ok(Self {
63            total_samples: 0,
64            rejected_samples: 0,
65            error_ew: ExponentiallyWeightedMean::new(config.error_alpha)?,
66            best_error: None,
67            baseline_error: None,
68            model_switches: 0,
69            reset_count: 0,
70            load_failures: 0,
71        })
72    }
73
74    /// Record that a sample was processed.
75    pub fn record_sample(&mut self) {
76        self.total_samples += 1;
77    }
78
79    /// Record that an input was rejected (invalid, non-finite, etc.).
80    pub fn record_rejection(&mut self) {
81        self.rejected_samples += 1;
82    }
83
84    /// Record an error from a prediction.
85    ///
86    /// The absolute value is taken, so signed errors are accepted.
87    /// Updates the recent error (EW mean) and the best (minimum) error.
88    pub fn record_error(&mut self, error: f64) -> Result<(), RillError> {
89        ensure_finite("error", error)?;
90        let abs_error = error.abs();
91        self.error_ew.update(abs_error)?;
92        match self.best_error {
93            None => self.best_error = Some(abs_error),
94            Some(b) if abs_error < b => self.best_error = Some(abs_error),
95            _ => {}
96        }
97        Ok(())
98    }
99
100    /// Set the baseline error for comparison.
101    pub fn set_baseline_error(&mut self, error: f64) -> Result<(), RillError> {
102        ensure_finite("baseline_error", error)?;
103        self.baseline_error = Some(error.abs());
104        Ok(())
105    }
106
107    /// Record that the active model was switched.
108    pub fn record_switch(&mut self) {
109        self.model_switches += 1;
110    }
111
112    /// Record that the model was reset.
113    pub fn record_reset(&mut self) {
114        self.reset_count += 1;
115    }
116
117    /// Record that a state load failed.
118    pub fn record_load_failure(&mut self) {
119        self.load_failures += 1;
120    }
121
122    /// Total samples processed.
123    pub const fn total_samples(&self) -> u64 {
124        self.total_samples
125    }
126
127    /// Samples rejected due to invalid input.
128    pub const fn rejected_samples(&self) -> u64 {
129        self.rejected_samples
130    }
131
132    /// Recent error (EW mean of absolute errors), or `None` if no errors recorded.
133    pub fn recent_error(&self) -> Option<f64> {
134        if self.error_ew.count() == 0 {
135            None
136        } else {
137            Some(self.error_ew.value())
138        }
139    }
140
141    /// Best (minimum) error observed, or `None` if no errors recorded.
142    pub const fn best_error(&self) -> Option<f64> {
143        self.best_error
144    }
145
146    /// Baseline error for comparison, or `None` if not set.
147    pub const fn baseline_error(&self) -> Option<f64> {
148        self.baseline_error
149    }
150
151    /// Number of times the active model was switched.
152    pub const fn model_switches(&self) -> u64 {
153        self.model_switches
154    }
155
156    /// Number of times the model was reset.
157    pub const fn reset_count(&self) -> u64 {
158        self.reset_count
159    }
160
161    /// Number of state load failures.
162    pub const fn load_failures(&self) -> u64 {
163        self.load_failures
164    }
165
166    /// Whether the model is currently beating the baseline.
167    ///
168    /// Returns `None` if either recent error or baseline error is unavailable.
169    pub fn beats_baseline(&self) -> Option<bool> {
170        match (self.recent_error(), self.baseline_error) {
171            (Some(recent), Some(baseline)) => Some(recent < baseline),
172            _ => None,
173        }
174    }
175
176    /// Reset all summary statistics.
177    pub fn reset(&mut self) {
178        self.total_samples = 0;
179        self.rejected_samples = 0;
180        self.error_ew.reset();
181        self.best_error = None;
182        self.baseline_error = None;
183        self.model_switches = 0;
184        self.reset_count = 0;
185        self.load_failures = 0;
186    }
187}
188
189impl Default for TrainingSummary {
190    fn default() -> Self {
191        Self::new(TrainingSummaryConfig::default()).expect("default config is valid")
192    }
193}
194
195#[cfg(test)]
196mod tests {
197    use super::*;
198
199    #[test]
200    fn default_summary_has_no_data() {
201        let s = TrainingSummary::default();
202        assert_eq!(s.total_samples(), 0);
203        assert_eq!(s.rejected_samples(), 0);
204        assert_eq!(s.recent_error(), None);
205        assert_eq!(s.best_error(), None);
206        assert_eq!(s.baseline_error(), None);
207        assert_eq!(s.beats_baseline(), None);
208        assert_eq!(s.model_switches(), 0);
209        assert_eq!(s.reset_count(), 0);
210        assert_eq!(s.load_failures(), 0);
211    }
212
213    #[test]
214    fn record_error_updates_recent_and_best() {
215        let mut s = TrainingSummary::default();
216        s.record_error(10.0).unwrap();
217        s.record_error(5.0).unwrap();
218        s.record_error(8.0).unwrap();
219        assert_eq!(s.best_error(), Some(5.0));
220        // EW mean with alpha=0.1: seed=10, then 0.1*5+0.9*10=9.5, then 0.1*8+0.9*9.5=9.35
221        assert!((s.recent_error().unwrap() - 9.35).abs() < 1e-9);
222    }
223
224    #[test]
225    fn beats_baseline_comparison() {
226        let mut s = TrainingSummary::default();
227        s.record_error(5.0).unwrap();
228        s.set_baseline_error(10.0).unwrap();
229        assert_eq!(s.beats_baseline(), Some(true));
230
231        s.set_baseline_error(3.0).unwrap();
232        assert_eq!(s.beats_baseline(), Some(false));
233    }
234
235    #[test]
236    fn beats_baseline_none_without_errors() {
237        let mut s = TrainingSummary::default();
238        s.set_baseline_error(10.0).unwrap();
239        assert_eq!(s.beats_baseline(), None);
240    }
241
242    #[test]
243    fn counts_tracked_correctly() {
244        let mut s = TrainingSummary::default();
245        s.record_sample();
246        s.record_sample();
247        s.record_rejection();
248        s.record_switch();
249        s.record_switch();
250        s.record_reset();
251        s.record_load_failure();
252        s.record_load_failure();
253        s.record_load_failure();
254        assert_eq!(s.total_samples(), 2);
255        assert_eq!(s.rejected_samples(), 1);
256        assert_eq!(s.model_switches(), 2);
257        assert_eq!(s.reset_count(), 1);
258        assert_eq!(s.load_failures(), 3);
259    }
260
261    #[test]
262    fn reset_clears_all() {
263        let mut s = TrainingSummary::default();
264        s.record_sample();
265        s.record_error(1.0).unwrap();
266        s.set_baseline_error(2.0).unwrap();
267        s.record_switch();
268        s.record_reset();
269        s.record_load_failure();
270        s.record_rejection();
271        s.reset();
272        assert_eq!(s.total_samples(), 0);
273        assert_eq!(s.rejected_samples(), 0);
274        assert_eq!(s.recent_error(), None);
275        assert_eq!(s.best_error(), None);
276        assert_eq!(s.baseline_error(), None);
277        assert_eq!(s.model_switches(), 0);
278        assert_eq!(s.reset_count(), 0);
279        assert_eq!(s.load_failures(), 0);
280    }
281
282    #[test]
283    fn non_finite_error_rejected() {
284        let mut s = TrainingSummary::default();
285        assert!(s.record_error(f64::NAN).is_err());
286        assert!(s.record_error(f64::INFINITY).is_err());
287        assert!(s.record_error(f64::NEG_INFINITY).is_err());
288    }
289
290    #[test]
291    fn non_finite_baseline_rejected() {
292        let mut s = TrainingSummary::default();
293        assert!(s.set_baseline_error(f64::NAN).is_err());
294        assert!(s.set_baseline_error(f64::INFINITY).is_err());
295    }
296
297    #[test]
298    fn invalid_alpha_rejected() {
299        let config = TrainingSummaryConfig { error_alpha: 0.0 };
300        assert!(TrainingSummary::new(config).is_err());
301    }
302
303    #[test]
304    fn negative_error_uses_absolute_value() {
305        let mut s = TrainingSummary::default();
306        s.record_error(-5.0).unwrap();
307        assert_eq!(s.best_error(), Some(5.0));
308        assert!((s.recent_error().unwrap() - 5.0).abs() < 1e-12);
309    }
310
311    #[test]
312    fn custom_alpha_changes_memory() {
313        let config = TrainingSummaryConfig { error_alpha: 1.0 };
314        let mut s = TrainingSummary::new(config).unwrap();
315        s.record_error(10.0).unwrap();
316        s.record_error(5.0).unwrap();
317        s.record_error(8.0).unwrap();
318        // alpha=1.0 tracks last value
319        assert!((s.recent_error().unwrap() - 8.0).abs() < 1e-12);
320    }
321
322    #[cfg(feature = "serde")]
323    #[test]
324    fn serde_roundtrip() {
325        let mut s = TrainingSummary::default();
326        s.record_sample();
327        s.record_sample();
328        s.record_error(2.0).unwrap();
329        s.record_error(1.5).unwrap();
330        s.set_baseline_error(3.0).unwrap();
331        s.record_switch();
332        let json = serde_json::to_string(&s).unwrap();
333        let restored: TrainingSummary = serde_json::from_str(&json).unwrap();
334        assert_eq!(restored.total_samples(), 2);
335        assert_eq!(restored.best_error(), Some(1.5));
336        assert_eq!(restored.baseline_error(), Some(3.0));
337        assert_eq!(restored.model_switches(), 1);
338        assert!(restored.beats_baseline().unwrap());
339    }
340}
rill_ml/diagnostics/training_summary.rs

rill_ml/diagnostics/
training_summary.rs
