rill_ml/diagnostics/
training_summary.rs

1//! Training summary statistics.
2//!
3//! Maintains bounded-memory summary statistics about the training process
4//! without storing raw samples. Useful for diagnostics and monitoring.
5//!
6//! Space complexity: `O(1)`.
7
8use crate::error::{RillError, checked_increment, ensure_finite};
9use crate::stats::ExponentiallyWeightedMean;
10use crate::traits::OnlineStatistic;
11
12/// Configuration for [`TrainingSummary`].
13#[derive(Debug, Clone)]
14#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
15#[non_exhaustive]
16pub struct TrainingSummaryConfig {
17    /// Alpha for the exponentially weighted recent error.
18    ///
19    /// Must be in `(0, 1]`. Smaller values give a longer memory.
20    pub error_alpha: f64,
21}
22
23impl Default for TrainingSummaryConfig {
24    fn default() -> Self {
25        Self { error_alpha: 0.1 }
26    }
27}
28
29/// Bounded-memory summary of a training process.
30///
31/// Tracks counts, recent/best errors, model switches, resets, and load
32/// failures. Does not store raw samples.
33///
34/// # Examples
35///
36/// ```
37/// use rill_ml::diagnostics::TrainingSummary;
38///
39/// let mut summary = TrainingSummary::default();
40/// summary.record_sample().unwrap();
41/// summary.record_error(0.5).unwrap();
42/// summary.set_baseline_error(0.8).unwrap();
43///
44/// assert_eq!(summary.total_samples(), 1);
45/// assert!(summary.beats_baseline().unwrap());
46/// ```
47#[derive(Debug, Clone)]
48#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
49pub struct TrainingSummary {
50    total_samples: u64,
51    rejected_samples: u64,
52    error_ew: ExponentiallyWeightedMean,
53    best_error: Option<f64>,
54    baseline_error: Option<f64>,
55    model_switches: u64,
56    reset_count: u64,
57    load_failures: u64,
58}
59
60impl TrainingSummary {
61    /// Create a new training summary with the given configuration.
62    pub fn new(config: TrainingSummaryConfig) -> Result<Self, RillError> {
63        Ok(Self {
64            total_samples: 0,
65            rejected_samples: 0,
66            error_ew: ExponentiallyWeightedMean::new(config.error_alpha)?,
67            best_error: None,
68            baseline_error: None,
69            model_switches: 0,
70            reset_count: 0,
71            load_failures: 0,
72        })
73    }
74
75    /// Record that a sample was processed.
76    pub fn record_sample(&mut self) -> Result<(), RillError> {
77        self.total_samples = checked_increment(self.total_samples, "total_samples")?;
78        Ok(())
79    }
80
81    /// Record that an input was rejected (invalid, non-finite, etc.).
82    pub fn record_rejection(&mut self) -> Result<(), RillError> {
83        self.rejected_samples = checked_increment(self.rejected_samples, "rejected_samples")?;
84        Ok(())
85    }
86
87    /// Record an error from a prediction.
88    ///
89    /// The absolute value is taken, so signed errors are accepted.
90    /// Updates the recent error (EW mean) and the best (minimum) error.
91    pub fn record_error(&mut self, error: f64) -> Result<(), RillError> {
92        ensure_finite("error", error)?;
93        let abs_error = error.abs();
94        self.error_ew.update(abs_error)?;
95        match self.best_error {
96            None => self.best_error = Some(abs_error),
97            Some(b) if abs_error < b => self.best_error = Some(abs_error),
98            _ => {}
99        }
100        Ok(())
101    }
102
103    /// Set the baseline error for comparison.
104    pub fn set_baseline_error(&mut self, error: f64) -> Result<(), RillError> {
105        ensure_finite("baseline_error", error)?;
106        self.baseline_error = Some(error.abs());
107        Ok(())
108    }
109
110    /// Record that the active model was switched.
111    pub fn record_switch(&mut self) -> Result<(), RillError> {
112        self.model_switches = checked_increment(self.model_switches, "model_switches")?;
113        Ok(())
114    }
115
116    /// Record that the model was reset.
117    pub fn record_reset(&mut self) -> Result<(), RillError> {
118        self.reset_count = checked_increment(self.reset_count, "reset_count")?;
119        Ok(())
120    }
121
122    /// Record that a state load failed.
123    pub fn record_load_failure(&mut self) -> Result<(), RillError> {
124        self.load_failures = checked_increment(self.load_failures, "load_failures")?;
125        Ok(())
126    }
127
128    /// Total samples processed.
129    pub const fn total_samples(&self) -> u64 {
130        self.total_samples
131    }
132
133    /// Samples rejected due to invalid input.
134    pub const fn rejected_samples(&self) -> u64 {
135        self.rejected_samples
136    }
137
138    /// Recent error (EW mean of absolute errors), or `None` if no errors recorded.
139    pub fn recent_error(&self) -> Option<f64> {
140        if self.error_ew.count() == 0 {
141            None
142        } else {
143            Some(self.error_ew.value())
144        }
145    }
146
147    /// Best (minimum) error observed, or `None` if no errors recorded.
148    pub const fn best_error(&self) -> Option<f64> {
149        self.best_error
150    }
151
152    /// Baseline error for comparison, or `None` if not set.
153    pub const fn baseline_error(&self) -> Option<f64> {
154        self.baseline_error
155    }
156
157    /// Number of times the active model was switched.
158    pub const fn model_switches(&self) -> u64 {
159        self.model_switches
160    }
161
162    /// Number of times the model was reset.
163    pub const fn reset_count(&self) -> u64 {
164        self.reset_count
165    }
166
167    /// Number of state load failures.
168    pub const fn load_failures(&self) -> u64 {
169        self.load_failures
170    }
171
172    /// Whether the model is currently beating the baseline.
173    ///
174    /// Returns `None` if either recent error or baseline error is unavailable.
175    pub fn beats_baseline(&self) -> Option<bool> {
176        match (self.recent_error(), self.baseline_error) {
177            (Some(recent), Some(baseline)) => Some(recent < baseline),
178            _ => None,
179        }
180    }
181
182    /// Reset all summary statistics.
183    pub fn reset(&mut self) {
184        self.total_samples = 0;
185        self.rejected_samples = 0;
186        self.error_ew.reset();
187        self.best_error = None;
188        self.baseline_error = None;
189        self.model_switches = 0;
190        self.reset_count = 0;
191        self.load_failures = 0;
192    }
193}
194
195impl Default for TrainingSummary {
196    fn default() -> Self {
197        Self::new(TrainingSummaryConfig::default()).expect("default config is valid")
198    }
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204
205    #[test]
206    fn default_summary_has_no_data() {
207        let s = TrainingSummary::default();
208        assert_eq!(s.total_samples(), 0);
209        assert_eq!(s.rejected_samples(), 0);
210        assert_eq!(s.recent_error(), None);
211        assert_eq!(s.best_error(), None);
212        assert_eq!(s.baseline_error(), None);
213        assert_eq!(s.beats_baseline(), None);
214        assert_eq!(s.model_switches(), 0);
215        assert_eq!(s.reset_count(), 0);
216        assert_eq!(s.load_failures(), 0);
217    }
218
219    #[test]
220    fn record_error_updates_recent_and_best() {
221        let mut s = TrainingSummary::default();
222        s.record_error(10.0).unwrap();
223        s.record_error(5.0).unwrap();
224        s.record_error(8.0).unwrap();
225        assert_eq!(s.best_error(), Some(5.0));
226        // EW mean with alpha=0.1: seed=10, then 0.1*5+0.9*10=9.5, then 0.1*8+0.9*9.5=9.35
227        assert!((s.recent_error().unwrap() - 9.35).abs() < 1e-9);
228    }
229
230    #[test]
231    fn beats_baseline_comparison() {
232        let mut s = TrainingSummary::default();
233        s.record_error(5.0).unwrap();
234        s.set_baseline_error(10.0).unwrap();
235        assert_eq!(s.beats_baseline(), Some(true));
236
237        s.set_baseline_error(3.0).unwrap();
238        assert_eq!(s.beats_baseline(), Some(false));
239    }
240
241    #[test]
242    fn beats_baseline_none_without_errors() {
243        let mut s = TrainingSummary::default();
244        s.set_baseline_error(10.0).unwrap();
245        assert_eq!(s.beats_baseline(), None);
246    }
247
248    #[test]
249    fn counts_tracked_correctly() {
250        let mut s = TrainingSummary::default();
251        s.record_sample().unwrap();
252        s.record_sample().unwrap();
253        s.record_rejection().unwrap();
254        s.record_switch().unwrap();
255        s.record_switch().unwrap();
256        s.record_reset().unwrap();
257        s.record_load_failure().unwrap();
258        s.record_load_failure().unwrap();
259        s.record_load_failure().unwrap();
260        assert_eq!(s.total_samples(), 2);
261        assert_eq!(s.rejected_samples(), 1);
262        assert_eq!(s.model_switches(), 2);
263        assert_eq!(s.reset_count(), 1);
264        assert_eq!(s.load_failures(), 3);
265    }
266
267    #[test]
268    fn reset_clears_all() {
269        let mut s = TrainingSummary::default();
270        s.record_sample().unwrap();
271        s.record_error(1.0).unwrap();
272        s.set_baseline_error(2.0).unwrap();
273        s.record_switch().unwrap();
274        s.record_reset().unwrap();
275        s.record_load_failure().unwrap();
276        s.record_rejection().unwrap();
277        s.reset();
278        assert_eq!(s.total_samples(), 0);
279        assert_eq!(s.rejected_samples(), 0);
280        assert_eq!(s.recent_error(), None);
281        assert_eq!(s.best_error(), None);
282        assert_eq!(s.baseline_error(), None);
283        assert_eq!(s.model_switches(), 0);
284        assert_eq!(s.reset_count(), 0);
285        assert_eq!(s.load_failures(), 0);
286    }
287
288    #[test]
289    fn non_finite_error_rejected() {
290        let mut s = TrainingSummary::default();
291        assert!(s.record_error(f64::NAN).is_err());
292        assert!(s.record_error(f64::INFINITY).is_err());
293        assert!(s.record_error(f64::NEG_INFINITY).is_err());
294    }
295
296    #[test]
297    fn non_finite_baseline_rejected() {
298        let mut s = TrainingSummary::default();
299        assert!(s.set_baseline_error(f64::NAN).is_err());
300        assert!(s.set_baseline_error(f64::INFINITY).is_err());
301    }
302
303    #[test]
304    fn invalid_alpha_rejected() {
305        let config = TrainingSummaryConfig { error_alpha: 0.0 };
306        assert!(TrainingSummary::new(config).is_err());
307    }
308
309    #[test]
310    fn negative_error_uses_absolute_value() {
311        let mut s = TrainingSummary::default();
312        s.record_error(-5.0).unwrap();
313        assert_eq!(s.best_error(), Some(5.0));
314        assert!((s.recent_error().unwrap() - 5.0).abs() < 1e-12);
315    }
316
317    #[test]
318    fn custom_alpha_changes_memory() {
319        let config = TrainingSummaryConfig { error_alpha: 1.0 };
320        let mut s = TrainingSummary::new(config).unwrap();
321        s.record_error(10.0).unwrap();
322        s.record_error(5.0).unwrap();
323        s.record_error(8.0).unwrap();
324        // alpha=1.0 tracks last value
325        assert!((s.recent_error().unwrap() - 8.0).abs() < 1e-12);
326    }
327
328    #[cfg(feature = "serde")]
329    #[test]
330    fn serde_roundtrip() {
331        let mut s = TrainingSummary::default();
332        s.record_sample().unwrap();
333        s.record_sample().unwrap();
334        s.record_error(2.0).unwrap();
335        s.record_error(1.5).unwrap();
336        s.set_baseline_error(3.0).unwrap();
337        s.record_switch().unwrap();
338        let json = serde_json::to_string(&s).unwrap();
339        let restored: TrainingSummary = serde_json::from_str(&json).unwrap();
340        assert_eq!(restored.total_samples(), 2);
341        assert_eq!(restored.best_error(), Some(1.5));
342        assert_eq!(restored.baseline_error(), Some(3.0));
343        assert_eq!(restored.model_switches(), 1);
344        assert!(restored.beats_baseline().unwrap());
345    }
346}
rill_ml/diagnostics/training_summary.rs

rill_ml/diagnostics/
training_summary.rs
