rill_ml/diagnostics/
training_summary.rs

1//! Training summary statistics.
2//!
3//! Maintains bounded-memory summary statistics about the training process
4//! without storing raw samples. Useful for diagnostics and monitoring.
5//!
6//! Space complexity: `O(1)`.
7
8use crate::error::{RillError, checked_increment, ensure_finite};
9use crate::stats::ExponentiallyWeightedMean;
10use crate::traits::OnlineStatistic;
11
12/// Configuration for [`TrainingSummary`].
13#[derive(Debug, Clone)]
14#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
15pub struct TrainingSummaryConfig {
16    /// Alpha for the exponentially weighted recent error.
17    ///
18    /// Must be in `(0, 1]`. Smaller values give a longer memory.
19    pub error_alpha: f64,
20}
21
22impl Default for TrainingSummaryConfig {
23    fn default() -> Self {
24        Self { error_alpha: 0.1 }
25    }
26}
27
28/// Bounded-memory summary of a training process.
29///
30/// Tracks counts, recent/best errors, model switches, resets, and load
31/// failures. Does not store raw samples.
32///
33/// # Examples
34///
35/// ```
36/// use rill_ml::diagnostics::TrainingSummary;
37///
38/// let mut summary = TrainingSummary::default();
39/// summary.record_sample().unwrap();
40/// summary.record_error(0.5).unwrap();
41/// summary.set_baseline_error(0.8).unwrap();
42///
43/// assert_eq!(summary.total_samples(), 1);
44/// assert!(summary.beats_baseline().unwrap());
45/// ```
46#[derive(Debug, Clone)]
47#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
48pub struct TrainingSummary {
49    total_samples: u64,
50    rejected_samples: u64,
51    error_ew: ExponentiallyWeightedMean,
52    best_error: Option<f64>,
53    baseline_error: Option<f64>,
54    model_switches: u64,
55    reset_count: u64,
56    load_failures: u64,
57}
58
59impl TrainingSummary {
60    /// Create a new training summary with the given configuration.
61    pub fn new(config: TrainingSummaryConfig) -> Result<Self, RillError> {
62        Ok(Self {
63            total_samples: 0,
64            rejected_samples: 0,
65            error_ew: ExponentiallyWeightedMean::new(config.error_alpha)?,
66            best_error: None,
67            baseline_error: None,
68            model_switches: 0,
69            reset_count: 0,
70            load_failures: 0,
71        })
72    }
73
74    /// Record that a sample was processed.
75    pub fn record_sample(&mut self) -> Result<(), RillError> {
76        self.total_samples = checked_increment(self.total_samples, "total_samples")?;
77        Ok(())
78    }
79
80    /// Record that an input was rejected (invalid, non-finite, etc.).
81    pub fn record_rejection(&mut self) -> Result<(), RillError> {
82        self.rejected_samples = checked_increment(self.rejected_samples, "rejected_samples")?;
83        Ok(())
84    }
85
86    /// Record an error from a prediction.
87    ///
88    /// The absolute value is taken, so signed errors are accepted.
89    /// Updates the recent error (EW mean) and the best (minimum) error.
90    pub fn record_error(&mut self, error: f64) -> Result<(), RillError> {
91        ensure_finite("error", error)?;
92        let abs_error = error.abs();
93        self.error_ew.update(abs_error)?;
94        match self.best_error {
95            None => self.best_error = Some(abs_error),
96            Some(b) if abs_error < b => self.best_error = Some(abs_error),
97            _ => {}
98        }
99        Ok(())
100    }
101
102    /// Set the baseline error for comparison.
103    pub fn set_baseline_error(&mut self, error: f64) -> Result<(), RillError> {
104        ensure_finite("baseline_error", error)?;
105        self.baseline_error = Some(error.abs());
106        Ok(())
107    }
108
109    /// Record that the active model was switched.
110    pub fn record_switch(&mut self) -> Result<(), RillError> {
111        self.model_switches = checked_increment(self.model_switches, "model_switches")?;
112        Ok(())
113    }
114
115    /// Record that the model was reset.
116    pub fn record_reset(&mut self) -> Result<(), RillError> {
117        self.reset_count = checked_increment(self.reset_count, "reset_count")?;
118        Ok(())
119    }
120
121    /// Record that a state load failed.
122    pub fn record_load_failure(&mut self) -> Result<(), RillError> {
123        self.load_failures = checked_increment(self.load_failures, "load_failures")?;
124        Ok(())
125    }
126
127    /// Total samples processed.
128    pub const fn total_samples(&self) -> u64 {
129        self.total_samples
130    }
131
132    /// Samples rejected due to invalid input.
133    pub const fn rejected_samples(&self) -> u64 {
134        self.rejected_samples
135    }
136
137    /// Recent error (EW mean of absolute errors), or `None` if no errors recorded.
138    pub fn recent_error(&self) -> Option<f64> {
139        if self.error_ew.count() == 0 {
140            None
141        } else {
142            Some(self.error_ew.value())
143        }
144    }
145
146    /// Best (minimum) error observed, or `None` if no errors recorded.
147    pub const fn best_error(&self) -> Option<f64> {
148        self.best_error
149    }
150
151    /// Baseline error for comparison, or `None` if not set.
152    pub const fn baseline_error(&self) -> Option<f64> {
153        self.baseline_error
154    }
155
156    /// Number of times the active model was switched.
157    pub const fn model_switches(&self) -> u64 {
158        self.model_switches
159    }
160
161    /// Number of times the model was reset.
162    pub const fn reset_count(&self) -> u64 {
163        self.reset_count
164    }
165
166    /// Number of state load failures.
167    pub const fn load_failures(&self) -> u64 {
168        self.load_failures
169    }
170
171    /// Whether the model is currently beating the baseline.
172    ///
173    /// Returns `None` if either recent error or baseline error is unavailable.
174    pub fn beats_baseline(&self) -> Option<bool> {
175        match (self.recent_error(), self.baseline_error) {
176            (Some(recent), Some(baseline)) => Some(recent < baseline),
177            _ => None,
178        }
179    }
180
181    /// Reset all summary statistics.
182    pub fn reset(&mut self) {
183        self.total_samples = 0;
184        self.rejected_samples = 0;
185        self.error_ew.reset();
186        self.best_error = None;
187        self.baseline_error = None;
188        self.model_switches = 0;
189        self.reset_count = 0;
190        self.load_failures = 0;
191    }
192}
193
194impl Default for TrainingSummary {
195    fn default() -> Self {
196        Self::new(TrainingSummaryConfig::default()).expect("default config is valid")
197    }
198}
199
200#[cfg(test)]
201mod tests {
202    use super::*;
203
204    #[test]
205    fn default_summary_has_no_data() {
206        let s = TrainingSummary::default();
207        assert_eq!(s.total_samples(), 0);
208        assert_eq!(s.rejected_samples(), 0);
209        assert_eq!(s.recent_error(), None);
210        assert_eq!(s.best_error(), None);
211        assert_eq!(s.baseline_error(), None);
212        assert_eq!(s.beats_baseline(), None);
213        assert_eq!(s.model_switches(), 0);
214        assert_eq!(s.reset_count(), 0);
215        assert_eq!(s.load_failures(), 0);
216    }
217
218    #[test]
219    fn record_error_updates_recent_and_best() {
220        let mut s = TrainingSummary::default();
221        s.record_error(10.0).unwrap();
222        s.record_error(5.0).unwrap();
223        s.record_error(8.0).unwrap();
224        assert_eq!(s.best_error(), Some(5.0));
225        // EW mean with alpha=0.1: seed=10, then 0.1*5+0.9*10=9.5, then 0.1*8+0.9*9.5=9.35
226        assert!((s.recent_error().unwrap() - 9.35).abs() < 1e-9);
227    }
228
229    #[test]
230    fn beats_baseline_comparison() {
231        let mut s = TrainingSummary::default();
232        s.record_error(5.0).unwrap();
233        s.set_baseline_error(10.0).unwrap();
234        assert_eq!(s.beats_baseline(), Some(true));
235
236        s.set_baseline_error(3.0).unwrap();
237        assert_eq!(s.beats_baseline(), Some(false));
238    }
239
240    #[test]
241    fn beats_baseline_none_without_errors() {
242        let mut s = TrainingSummary::default();
243        s.set_baseline_error(10.0).unwrap();
244        assert_eq!(s.beats_baseline(), None);
245    }
246
247    #[test]
248    fn counts_tracked_correctly() {
249        let mut s = TrainingSummary::default();
250        s.record_sample().unwrap();
251        s.record_sample().unwrap();
252        s.record_rejection().unwrap();
253        s.record_switch().unwrap();
254        s.record_switch().unwrap();
255        s.record_reset().unwrap();
256        s.record_load_failure().unwrap();
257        s.record_load_failure().unwrap();
258        s.record_load_failure().unwrap();
259        assert_eq!(s.total_samples(), 2);
260        assert_eq!(s.rejected_samples(), 1);
261        assert_eq!(s.model_switches(), 2);
262        assert_eq!(s.reset_count(), 1);
263        assert_eq!(s.load_failures(), 3);
264    }
265
266    #[test]
267    fn reset_clears_all() {
268        let mut s = TrainingSummary::default();
269        s.record_sample().unwrap();
270        s.record_error(1.0).unwrap();
271        s.set_baseline_error(2.0).unwrap();
272        s.record_switch().unwrap();
273        s.record_reset().unwrap();
274        s.record_load_failure().unwrap();
275        s.record_rejection().unwrap();
276        s.reset();
277        assert_eq!(s.total_samples(), 0);
278        assert_eq!(s.rejected_samples(), 0);
279        assert_eq!(s.recent_error(), None);
280        assert_eq!(s.best_error(), None);
281        assert_eq!(s.baseline_error(), None);
282        assert_eq!(s.model_switches(), 0);
283        assert_eq!(s.reset_count(), 0);
284        assert_eq!(s.load_failures(), 0);
285    }
286
287    #[test]
288    fn non_finite_error_rejected() {
289        let mut s = TrainingSummary::default();
290        assert!(s.record_error(f64::NAN).is_err());
291        assert!(s.record_error(f64::INFINITY).is_err());
292        assert!(s.record_error(f64::NEG_INFINITY).is_err());
293    }
294
295    #[test]
296    fn non_finite_baseline_rejected() {
297        let mut s = TrainingSummary::default();
298        assert!(s.set_baseline_error(f64::NAN).is_err());
299        assert!(s.set_baseline_error(f64::INFINITY).is_err());
300    }
301
302    #[test]
303    fn invalid_alpha_rejected() {
304        let config = TrainingSummaryConfig { error_alpha: 0.0 };
305        assert!(TrainingSummary::new(config).is_err());
306    }
307
308    #[test]
309    fn negative_error_uses_absolute_value() {
310        let mut s = TrainingSummary::default();
311        s.record_error(-5.0).unwrap();
312        assert_eq!(s.best_error(), Some(5.0));
313        assert!((s.recent_error().unwrap() - 5.0).abs() < 1e-12);
314    }
315
316    #[test]
317    fn custom_alpha_changes_memory() {
318        let config = TrainingSummaryConfig { error_alpha: 1.0 };
319        let mut s = TrainingSummary::new(config).unwrap();
320        s.record_error(10.0).unwrap();
321        s.record_error(5.0).unwrap();
322        s.record_error(8.0).unwrap();
323        // alpha=1.0 tracks last value
324        assert!((s.recent_error().unwrap() - 8.0).abs() < 1e-12);
325    }
326
327    #[cfg(feature = "serde")]
328    #[test]
329    fn serde_roundtrip() {
330        let mut s = TrainingSummary::default();
331        s.record_sample().unwrap();
332        s.record_sample().unwrap();
333        s.record_error(2.0).unwrap();
334        s.record_error(1.5).unwrap();
335        s.set_baseline_error(3.0).unwrap();
336        s.record_switch().unwrap();
337        let json = serde_json::to_string(&s).unwrap();
338        let restored: TrainingSummary = serde_json::from_str(&json).unwrap();
339        assert_eq!(restored.total_samples(), 2);
340        assert_eq!(restored.best_error(), Some(1.5));
341        assert_eq!(restored.baseline_error(), Some(3.0));
342        assert_eq!(restored.model_switches(), 1);
343        assert!(restored.beats_baseline().unwrap());
344    }
345}
rill_ml/diagnostics/training_summary.rs

rill_ml/diagnostics/
training_summary.rs
