u_analytics/
accuracy.rs

1//! Accuracy of a model's predictions against observations.
2//!
3//! The errors are `eᵢ = observedᵢ − predictedᵢ` (positive when the model
4//! predicts too little), from a regression's fitted values, a forecast, or any
5//! other model.
6//!
7//! # Examples
8//!
9//! ```
10//! use u_analytics::accuracy::prediction_errors;
11//!
12//! let observed = [10.0, 12.0, 11.0, 13.0];
13//! let predicted = [10.5, 11.5, 11.0, 12.0];
14//! let e = prediction_errors(&observed, &predicted, 0).unwrap();
15//! assert!((e.mean_error - 0.25).abs() < 1e-12);
16//! assert!((e.mean_absolute_error - 0.5).abs() < 1e-12);
17//! ```
18
19/// Error measures of predictions against observations.
20#[derive(Debug, Clone, Copy, PartialEq)]
21pub struct PredictionErrors {
22    /// Number of observations n.
23    pub n: usize,
24    /// Mean error (bias), Σeᵢ / n.
25    pub mean_error: f64,
26    /// Mean absolute error, Σ|eᵢ| / n.
27    pub mean_absolute_error: f64,
28    /// Root mean squared error, √(Σeᵢ² / (n − p)).
29    pub rmse: f64,
30    /// Coefficient of variation of the RMSE, `rmse / ȳ` with ȳ the mean of
31    /// the observations; `None` when ȳ is 0. A fraction (× 100 for percent).
32    pub cv_rmse: Option<f64>,
33    /// Normalized mean bias error, Σeᵢ / ((n − p) · ȳ); `None` when ȳ is 0.
34    /// A fraction (× 100 for percent); positive when the predictions are low.
35    pub nmbe: Option<f64>,
36    /// The p the RMSE and NMBE denominators subtract.
37    pub parameters: usize,
38}
39
40/// Error measures of `predicted` against `observed`.
41///
42/// `parameters` is the number of parameters p the model estimated from these
43/// observations; the RMSE and the NMBE divide by n − p, the residual degrees
44/// of freedom (0 gives the plain means over n). With p the regression's
45/// coefficient count these are the CV(RMSE) and NMBE of ASHRAE Guideline 14.
46///
47/// # Returns
48///
49/// `None` when the slices differ in length, are empty, hold a non-finite
50/// value, or `parameters` is not below n.
51///
52/// # References
53///
54/// ASHRAE Guideline 14-2014, "Measurement of Energy, Demand, and Water
55/// Savings", Annex B (CV(RMSE), NMBE).
56/// Hyndman & Koehler (2006). "Another look at measures of forecast
57/// accuracy", *International Journal of Forecasting* 22(4), 679-688.
58pub fn prediction_errors(
59    observed: &[f64],
60    predicted: &[f64],
61    parameters: usize,
62) -> Option<PredictionErrors> {
63    let n = observed.len();
64    if n == 0 || n != predicted.len() || parameters >= n {
65        return None;
66    }
67    if observed.iter().chain(predicted).any(|v| !v.is_finite()) {
68        return None;
69    }
70    let nf = n as f64;
71    let df = (n - parameters) as f64;
72    let errors = observed.iter().zip(predicted).map(|(o, p)| o - p);
73    let sum: f64 = errors.clone().sum();
74    let sum_abs: f64 = errors.clone().map(f64::abs).sum();
75    let sum_sq: f64 = errors.map(|e| e * e).sum();
76    let mean_observed = observed.iter().sum::<f64>() / nf;
77    let rmse = (sum_sq / df).sqrt();
78    let relative = |v: f64| (mean_observed != 0.0).then(|| v / mean_observed);
79    Some(PredictionErrors {
80        n,
81        mean_error: sum / nf,
82        mean_absolute_error: sum_abs / nf,
83        rmse,
84        cv_rmse: relative(rmse),
85        nmbe: relative(sum / df),
86        parameters,
87    })
88}
89
90#[cfg(test)]
91mod tests {
92    use super::*;
93
94    #[test]
95    fn the_definitions_by_hand() {
96        // e = (1, -1, 2, 0): Σe = 2, Σ|e| = 4, Σe² = 6, ȳ = 10.
97        let observed = [11.0, 9.0, 12.0, 8.0];
98        let predicted = [10.0, 10.0, 10.0, 8.0];
99        let e = prediction_errors(&observed, &predicted, 0).unwrap();
100        assert_eq!(e.n, 4);
101        assert!((e.mean_error - 0.5).abs() < 1e-15);
102        assert!((e.mean_absolute_error - 1.0).abs() < 1e-15);
103        assert!((e.rmse - (6.0f64 / 4.0).sqrt()).abs() < 1e-15);
104        assert!((e.cv_rmse.unwrap() - (1.5f64).sqrt() / 10.0).abs() < 1e-15);
105        assert!((e.nmbe.unwrap() - 2.0 / 40.0).abs() < 1e-15);
106        // Two estimated parameters: the denominators become n − p = 2.
107        let e = prediction_errors(&observed, &predicted, 2).unwrap();
108        assert!((e.rmse - 3.0f64.sqrt()).abs() < 1e-15);
109        assert!((e.nmbe.unwrap() - 2.0 / 20.0).abs() < 1e-15);
110        assert!((e.mean_error - 0.5).abs() < 1e-15, "the means stay over n");
111    }
112
113    #[test]
114    fn ratios_need_a_nonzero_mean() {
115        let e = prediction_errors(&[1.0, -1.0], &[0.0, 0.0], 0).unwrap();
116        assert_eq!((e.cv_rmse, e.nmbe), (None, None));
117        assert!((e.rmse - 1.0).abs() < 1e-15);
118    }
119
120    #[test]
121    fn refusals() {
122        assert!(prediction_errors(&[], &[], 0).is_none());
123        assert!(prediction_errors(&[1.0], &[1.0, 2.0], 0).is_none());
124        assert!(prediction_errors(&[1.0, 2.0], &[1.0, 2.0], 2).is_none());
125        assert!(prediction_errors(&[1.0, f64::NAN], &[1.0, 2.0], 0).is_none());
126    }
127
128    #[test]
129    fn a_change_of_unit_scales_the_absolute_measures_only() {
130        let observed = [101.0, 98.5, 103.2, 99.9, 100.4];
131        let predicted = [100.0, 99.0, 102.0, 100.5, 100.0];
132        let a = prediction_errors(&observed, &predicted, 1).unwrap();
133        for k in [1e-9, 1e9] {
134            let o: Vec<f64> = observed.iter().map(|v| v * k).collect();
135            let p: Vec<f64> = predicted.iter().map(|v| v * k).collect();
136            let b = prediction_errors(&o, &p, 1).unwrap();
137            assert!((b.rmse / k - a.rmse).abs() < 1e-9 * a.rmse);
138            assert!((b.cv_rmse.unwrap() - a.cv_rmse.unwrap()).abs() < 1e-12);
139            assert!((b.nmbe.unwrap() - a.nmbe.unwrap()).abs() < 1e-12);
140        }
141    }
142}
u_analytics/accuracy.rs

u_analytics/
accuracy.rs
