rill_ml/diagnostics/
prediction_report.rs

1//! Unified prediction report.
2//!
3//! Combines [`ResidualInterval`], [`WarmupTracker`], and [`TrainingSummary`]
4//! into a single diagnostic wrapper that produces an immutable
5//! [`PredictionReport`] for each prediction. This keeps the base model API
6//! clean: a model returns a plain prediction, and the caller can wrap it with
7//! [`PredictionReporter`] to obtain intervals, confidence levels, and
8//! warmup/baseline comparisons.
9//!
10//! Space complexity: `O(1)`.
11
12use crate::diagnostics::prediction_interval::{ResidualInterval, ResidualIntervalConfig};
13use crate::diagnostics::training_summary::{TrainingSummary, TrainingSummaryConfig};
14use crate::diagnostics::warmup::{WarmupConfig, WarmupState, WarmupTracker};
15use crate::error::{RillError, ensure_finite};
16
17/// Coarse confidence level derived from the warmup state and baseline
18/// comparison.
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
21pub enum Confidence {
22    /// No data has been observed yet.
23    None,
24    /// The model is still warming up or has degraded.
25    Low,
26    /// The model is usable but not yet stable.
27    Medium,
28    /// The model is stable and beating the baseline.
29    High,
30}
31
32impl Confidence {
33    /// Returns a short, stable string identifier.
34    ///
35    /// Possible return values: `"none"`, `"low"`, `"medium"`, `"high"`.
36    pub const fn as_str(&self) -> &'static str {
37        match self {
38            Confidence::None => "none",
39            Confidence::Low => "low",
40            Confidence::Medium => "medium",
41            Confidence::High => "high",
42        }
43    }
44}
45
46/// An immutable snapshot of diagnostics for a single prediction.
47#[derive(Debug, Clone)]
48#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
49pub struct PredictionReport {
50    prediction: f64,
51    lower_bound: Option<f64>,
52    upper_bound: Option<f64>,
53    confidence: Confidence,
54    samples_seen: u64,
55    recent_error: Option<f64>,
56    baseline_error: Option<f64>,
57    warmup_state: WarmupState,
58    beats_baseline: Option<bool>,
59}
60
61impl PredictionReport {
62    /// The prediction that this report was generated for.
63    pub const fn prediction(&self) -> f64 {
64        self.prediction
65    }
66
67    /// Lower bound of the prediction interval, or `None` if insufficient data.
68    pub const fn lower_bound(&self) -> Option<f64> {
69        self.lower_bound
70    }
71
72    /// Upper bound of the prediction interval, or `None` if insufficient data.
73    pub const fn upper_bound(&self) -> Option<f64> {
74        self.upper_bound
75    }
76
77    /// Coarse confidence level for this prediction.
78    pub const fn confidence(&self) -> Confidence {
79        self.confidence
80    }
81
82    /// Total number of samples observed so far.
83    pub const fn samples_seen(&self) -> u64 {
84        self.samples_seen
85    }
86
87    /// Recent (EW mean) absolute error, or `None` if no errors recorded.
88    pub const fn recent_error(&self) -> Option<f64> {
89        self.recent_error
90    }
91
92    /// Baseline error for comparison, or `None` if not set.
93    pub const fn baseline_error(&self) -> Option<f64> {
94        self.baseline_error
95    }
96
97    /// Current warmup state of the model.
98    pub const fn warmup_state(&self) -> WarmupState {
99        self.warmup_state
100    }
101
102    /// Whether the model is currently beating the baseline.
103    ///
104    /// Returns `None` if either recent error or baseline error is unavailable.
105    pub const fn beats_baseline(&self) -> Option<bool> {
106        self.beats_baseline
107    }
108}
109
110/// Diagnostic wrapper that integrates interval estimation, warmup tracking,
111/// and training summary statistics.
112///
113/// Produces a [`PredictionReport`] for each prediction without storing raw
114/// samples. The underlying model API is not affected: callers feed
115/// `(prediction, truth)` pairs via [`observe`](Self::observe) and request a
116/// report via [`report`](Self::report) when needed.
117///
118/// # Examples
119///
120/// ```
121/// use rill_ml::diagnostics::PredictionReporter;
122///
123/// let mut reporter = PredictionReporter::default();
124/// reporter.observe(10.0, 11.0).unwrap();
125/// reporter.observe(10.0, 9.0).unwrap();
126///
127/// let report = reporter.report(10.0).unwrap();
128/// assert_eq!(report.prediction(), 10.0);
129/// assert!(report.lower_bound().is_some());
130/// assert_eq!(report.samples_seen(), 2);
131/// ```
132#[derive(Debug, Clone)]
133#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
134pub struct PredictionReporter {
135    interval: ResidualInterval,
136    warmup: WarmupTracker,
137    summary: TrainingSummary,
138}
139
140impl PredictionReporter {
141    /// Create a new reporter with the given configurations.
142    ///
143    /// Each sub-component is constructed independently; configuration errors
144    /// are propagated as [`RillError`].
145    pub fn new(
146        interval_config: ResidualIntervalConfig,
147        warmup_config: WarmupConfig,
148        summary_config: TrainingSummaryConfig,
149    ) -> Result<Self, RillError> {
150        Ok(Self {
151            interval: ResidualInterval::new(interval_config)?,
152            warmup: WarmupTracker::new(warmup_config)?,
153            summary: TrainingSummary::new(summary_config)?,
154        })
155    }
156
157    /// Observe a prediction and its ground truth.
158    ///
159    /// Updates the interval estimator, warmup tracker, and training summary.
160    /// Non-finite inputs are rejected before any state is mutated.
161    pub fn observe(&mut self, prediction: f64, truth: f64) -> Result<(), RillError> {
162        self.interval.observe(prediction, truth)?;
163        let error = (truth - prediction).abs();
164        self.warmup.observe_sample(Some(error))?;
165        self.summary.record_error(error)?;
166        self.summary.record_sample();
167        Ok(())
168    }
169
170    /// Set the baseline error for comparison.
171    ///
172    /// Propagates to both the warmup tracker and the training summary.
173    pub fn set_baseline(&mut self, baseline: f64) -> Result<(), RillError> {
174        self.warmup.set_baseline(baseline)?;
175        self.summary.set_baseline_error(baseline)?;
176        Ok(())
177    }
178
179    /// Build an immutable report for the given prediction.
180    ///
181    /// If the interval estimator has insufficient data, the bounds are set to
182    /// `None` and no error is returned. Non-finite `prediction` values are
183    /// rejected.
184    pub fn report(&self, prediction: f64) -> Result<PredictionReport, RillError> {
185        ensure_finite("prediction", prediction)?;
186
187        let (lower_bound, upper_bound) = match self.interval.interval(prediction) {
188            Ok(iv) => (Some(iv.lower()), Some(iv.upper())),
189            Err(RillError::InsufficientData) => (None, None),
190            Err(e) => return Err(e),
191        };
192
193        let warmup_state = self.warmup.state();
194        let beats_baseline = self.summary.beats_baseline();
195        let samples_seen = self.summary.total_samples();
196        let recent_error = self.summary.recent_error();
197        let baseline_error = self.summary.baseline_error();
198
199        let confidence = match warmup_state {
200            WarmupState::NoData => Confidence::None,
201            WarmupState::WarmingUp | WarmupState::Degraded => Confidence::Low,
202            WarmupState::Usable => Confidence::Medium,
203            WarmupState::Stable => {
204                if matches!(beats_baseline, Some(true)) {
205                    Confidence::High
206                } else {
207                    Confidence::Medium
208                }
209            }
210        };
211
212        Ok(PredictionReport {
213            prediction,
214            lower_bound,
215            upper_bound,
216            confidence,
217            samples_seen,
218            recent_error,
219            baseline_error,
220            warmup_state,
221            beats_baseline,
222        })
223    }
224
225    /// Borrow the underlying training summary.
226    pub fn summary(&self) -> &TrainingSummary {
227        &self.summary
228    }
229
230    /// Current warmup state.
231    pub fn warmup_state(&self) -> WarmupState {
232        self.warmup.state()
233    }
234
235    /// Recent (EW mean) absolute error, or `None` if no errors recorded.
236    pub fn recent_error(&self) -> Option<f64> {
237        self.summary.recent_error()
238    }
239
240    /// Reset all three sub-components to their initial state.
241    pub fn reset(&mut self) {
242        self.interval.reset();
243        self.warmup.reset();
244        self.summary.reset();
245    }
246}
247
248impl Default for PredictionReporter {
249    fn default() -> Self {
250        Self::new(
251            ResidualIntervalConfig::default(),
252            WarmupConfig::default(),
253            TrainingSummaryConfig::default(),
254        )
255        .expect("default configs are valid")
256    }
257}
258
259#[cfg(test)]
260mod tests {
261    use super::*;
262
263    #[test]
264    fn confidence_as_str() {
265        assert_eq!(Confidence::None.as_str(), "none");
266        assert_eq!(Confidence::Low.as_str(), "low");
267        assert_eq!(Confidence::Medium.as_str(), "medium");
268        assert_eq!(Confidence::High.as_str(), "high");
269    }
270
271    #[test]
272    fn default_reporter_no_data() {
273        let reporter = PredictionReporter::default();
274        let r = reporter.report(0.0).unwrap();
275        assert_eq!(r.prediction(), 0.0);
276        assert_eq!(r.lower_bound(), None);
277        assert_eq!(r.upper_bound(), None);
278        assert_eq!(r.confidence(), Confidence::None);
279        assert_eq!(r.warmup_state(), WarmupState::NoData);
280        assert_eq!(r.samples_seen(), 0);
281        assert_eq!(r.recent_error(), None);
282        assert_eq!(r.baseline_error(), None);
283        assert_eq!(r.beats_baseline(), None);
284    }
285
286    #[test]
287    fn observe_then_report() {
288        let mut reporter = PredictionReporter::default();
289        reporter.observe(10.0, 11.0).unwrap(); // |error| = 1.0
290        reporter.observe(10.0, 9.0).unwrap(); // |error| = 1.0
291        let r = reporter.report(10.0).unwrap();
292        assert_eq!(r.prediction(), 10.0);
293        assert!(r.lower_bound().is_some());
294        assert!(r.upper_bound().is_some());
295        assert!(r.lower_bound().unwrap() < 10.0);
296        assert!(r.upper_bound().unwrap() > 10.0);
297        assert_eq!(r.samples_seen(), 2);
298        assert!(r.recent_error().is_some());
299    }
300
301    #[test]
302    fn set_baseline_enables_comparison() {
303        let mut reporter = PredictionReporter::default();
304        reporter.observe(0.0, 1.0).unwrap();
305        let r = reporter.report(0.0).unwrap();
306        assert_eq!(r.beats_baseline(), None);
307        assert_eq!(r.baseline_error(), None);
308        reporter.set_baseline(2.0).unwrap();
309        let r = reporter.report(0.0).unwrap();
310        assert_eq!(r.baseline_error(), Some(2.0));
311        assert_eq!(r.beats_baseline(), Some(true)); // recent_error=1.0 < 2.0
312    }
313
314    #[test]
315    fn confidence_progression() {
316        let warmup_config = WarmupConfig {
317            warming_up_threshold: 2,
318            usable_threshold: 5,
319            stable_threshold: 10,
320            degraded_error_ratio: 2.0,
321        };
322        let summary_config = TrainingSummaryConfig { error_alpha: 1.0 };
323        let mut reporter = PredictionReporter::new(
324            ResidualIntervalConfig::default(),
325            warmup_config,
326            summary_config,
327        )
328        .unwrap();
329
330        // NoData: no observations yet.
331        let r = reporter.report(0.0).unwrap();
332        assert_eq!(r.warmup_state(), WarmupState::NoData);
333        assert_eq!(r.confidence(), Confidence::None);
334
335        // WarmingUp: 1 sample (< warming_up_threshold=2).
336        reporter.observe(0.0, 0.5).unwrap();
337        let r = reporter.report(0.0).unwrap();
338        assert_eq!(r.warmup_state(), WarmupState::WarmingUp);
339        assert_eq!(r.confidence(), Confidence::Low);
340
341        // Usable: 5 samples, no baseline.
342        for _ in 0..4 {
343            reporter.observe(0.0, 0.5).unwrap();
344        }
345        let r = reporter.report(0.0).unwrap();
346        assert_eq!(r.warmup_state(), WarmupState::Usable);
347        assert_eq!(r.confidence(), Confidence::Medium);
348
349        // Set baseline; still Usable because samples < stable_threshold.
350        reporter.set_baseline(1.0).unwrap();
351        let r = reporter.report(0.0).unwrap();
352        assert_eq!(r.warmup_state(), WarmupState::Usable);
353        assert_eq!(r.confidence(), Confidence::Medium);
354
355        // Stable: 10 samples and recent_error (0.5) <= baseline (1.0).
356        for _ in 0..5 {
357            reporter.observe(0.0, 0.5).unwrap();
358        }
359        let r = reporter.report(0.0).unwrap();
360        assert_eq!(r.warmup_state(), WarmupState::Stable);
361        assert_eq!(r.confidence(), Confidence::High);
362    }
363
364    #[test]
365    fn degraded_state() {
366        let mut reporter = PredictionReporter::default();
367        reporter.set_baseline(0.4).unwrap();
368        // error=1.0 > baseline(0.4) * ratio(2.0) = 0.8
369        for _ in 0..5 {
370            reporter.observe(0.0, 1.0).unwrap();
371        }
372        let r = reporter.report(0.0).unwrap();
373        assert_eq!(r.warmup_state(), WarmupState::Degraded);
374        assert_eq!(r.confidence(), Confidence::Low);
375    }
376
377    #[test]
378    fn reset_clears_all() {
379        let mut reporter = PredictionReporter::default();
380        reporter.observe(10.0, 12.0).unwrap();
381        reporter.set_baseline(2.0).unwrap();
382        reporter.reset();
383        let r = reporter.report(0.0).unwrap();
384        assert_eq!(r.lower_bound(), None);
385        assert_eq!(r.upper_bound(), None);
386        assert_eq!(r.confidence(), Confidence::None);
387        assert_eq!(r.warmup_state(), WarmupState::NoData);
388        assert_eq!(r.samples_seen(), 0);
389        assert_eq!(r.recent_error(), None);
390        assert_eq!(r.baseline_error(), None);
391        assert_eq!(r.beats_baseline(), None);
392    }
393
394    #[test]
395    fn report_with_non_finite_prediction_errors() {
396        let mut reporter = PredictionReporter::default();
397        reporter.observe(0.0, 1.0).unwrap();
398        assert!(reporter.report(f64::NAN).is_err());
399        assert!(reporter.report(f64::INFINITY).is_err());
400        assert!(reporter.report(f64::NEG_INFINITY).is_err());
401    }
402
403    #[test]
404    fn observe_with_non_finite_rejected() {
405        let mut reporter = PredictionReporter::default();
406        assert!(reporter.observe(0.0, f64::NAN).is_err());
407        assert!(reporter.observe(0.0, f64::INFINITY).is_err());
408        assert!(reporter.observe(f64::NAN, 0.0).is_err());
409        // No state should have been recorded.
410        let r = reporter.report(0.0).unwrap();
411        assert_eq!(r.samples_seen(), 0);
412        assert_eq!(r.warmup_state(), WarmupState::NoData);
413    }
414
415    #[test]
416    fn samples_seen_tracked() {
417        let mut reporter = PredictionReporter::default();
418        for i in 0..10 {
419            reporter.observe(0.0, i as f64).unwrap();
420        }
421        let r = reporter.report(0.0).unwrap();
422        assert_eq!(r.samples_seen(), 10);
423    }
424
425    #[cfg(feature = "serde")]
426    #[test]
427    fn serde_roundtrip() {
428        let mut reporter = PredictionReporter::default();
429        reporter.observe(10.0, 12.0).unwrap();
430        reporter.observe(10.0, 9.0).unwrap();
431        reporter.set_baseline(3.0).unwrap();
432
433        let json = serde_json::to_string(&reporter).unwrap();
434        let restored: PredictionReporter = serde_json::from_str(&json).unwrap();
435
436        let r = restored.report(10.0).unwrap();
437        assert_eq!(r.samples_seen(), 2);
438        assert_eq!(r.baseline_error(), Some(3.0));
439        assert!(r.beats_baseline().is_some());
440    }
441}
rill_ml/diagnostics/prediction_report.rs

rill_ml/diagnostics/
prediction_report.rs
