Skip to main content

solow_regression/
penalized_cv.rs

1//! Cross-validated penalised-regression variants:
2//! [`RidgeCV`], [`LassoCV`], [`ElasticNetCV`].
3//!
4//! Each searches over a user-supplied `alpha` grid (and, for
5//! `ElasticNetCV`, an `l1_ratio` grid) using solow-cv-style K-fold
6//! cross-validation with the mean-squared-error criterion, refits the
7//! full estimator at the best-CV `alpha`, and reports both the
8//! selected hyperparameter and the per-alpha CV score.
9//!
10//! # References
11//!
12//! * Hoerl-Kennard (1970), Tibshirani (1996), Zou-Hastie (2005) — see
13//!   [`crate::penalized`] for the underlying estimators.
14
15use ndarray::{Array1, Array2, ArrayView1, ArrayView2};
16use solow_core::{Error, Result};
17
18use crate::penalized::{ElasticNet, Lasso, Ridge};
19
20/// Cross-validated Ridge regression.
21#[derive(Clone, Debug)]
22pub struct RidgeCV {
23    /// The fitted [`Ridge`] at the CV-optimal α.
24    pub fit: Ridge,
25    /// α values searched, in the order supplied.
26    pub alphas: Vec<f64>,
27    /// Mean CV MSE per α (lower is better).
28    pub mean_cv_mse: Vec<f64>,
29    /// Best α (minimises mean CV MSE).
30    pub best_alpha: f64,
31}
32
33impl RidgeCV {
34    /// Fit RidgeCV over the given `alphas` with `k`-fold cross-validation.
35    pub fn fit(
36        y: ArrayView1<'_, f64>,
37        x: ArrayView2<'_, f64>,
38        alphas: &[f64],
39        k: usize,
40        fit_intercept: bool,
41    ) -> Result<Self> {
42        if alphas.is_empty() {
43            return Err(Error::Value(
44                "RidgeCV::fit: alphas must be non-empty".into(),
45            ));
46        }
47        if k < 2 || k > y.len() {
48            return Err(Error::Value(format!(
49                "RidgeCV::fit: k must be in [2, n] (got {k})"
50            )));
51        }
52        let folds = kfold_indices(y.len(), k);
53        let mut mean_cv_mse = Vec::with_capacity(alphas.len());
54        for &alpha in alphas {
55            let mut sum_mse = 0.0_f64;
56            for (train, test) in &folds {
57                let (y_tr, x_tr) = subset(y, x, train);
58                let (y_te, x_te) = subset(y, x, test);
59                let fit = Ridge::fit(y_tr.view(), x_tr.view(), alpha, fit_intercept)?;
60                let pred = fit.predict(x_te.view())?;
61                let mse: f64 = pred
62                    .iter()
63                    .zip(y_te.iter())
64                    .map(|(a, b)| (a - b).powi(2))
65                    .sum::<f64>()
66                    / y_te.len() as f64;
67                sum_mse += mse;
68            }
69            mean_cv_mse.push(sum_mse / folds.len() as f64);
70        }
71        let (best_i, _) = argmin(&mean_cv_mse);
72        let best_alpha = alphas[best_i];
73        let fit = Ridge::fit(y, x, best_alpha, fit_intercept)?;
74        Ok(Self {
75            fit,
76            alphas: alphas.to_vec(),
77            mean_cv_mse,
78            best_alpha,
79        })
80    }
81}
82
83/// Cross-validated Lasso.
84#[derive(Clone, Debug)]
85pub struct LassoCV {
86    /// Fitted Lasso at the best α.
87    pub fit: Lasso,
88    /// α values searched.
89    pub alphas: Vec<f64>,
90    /// Mean CV MSE per α.
91    pub mean_cv_mse: Vec<f64>,
92    /// Best α.
93    pub best_alpha: f64,
94}
95
96impl LassoCV {
97    /// Fit LassoCV.
98    pub fn fit(
99        y: ArrayView1<'_, f64>,
100        x: ArrayView2<'_, f64>,
101        alphas: &[f64],
102        k: usize,
103        fit_intercept: bool,
104    ) -> Result<Self> {
105        if alphas.is_empty() {
106            return Err(Error::Value(
107                "LassoCV::fit: alphas must be non-empty".into(),
108            ));
109        }
110        if k < 2 || k > y.len() {
111            return Err(Error::Value(format!(
112                "LassoCV::fit: k must be in [2, n] (got {k})"
113            )));
114        }
115        let folds = kfold_indices(y.len(), k);
116        let mut mean_cv_mse = Vec::with_capacity(alphas.len());
117        for &alpha in alphas {
118            let mut sum_mse = 0.0_f64;
119            for (train, test) in &folds {
120                let (y_tr, x_tr) = subset(y, x, train);
121                let (y_te, x_te) = subset(y, x, test);
122                let fit = Lasso::fit(y_tr.view(), x_tr.view(), alpha, fit_intercept)?;
123                let pred = fit.predict(x_te.view())?;
124                let mse: f64 = pred
125                    .iter()
126                    .zip(y_te.iter())
127                    .map(|(a, b)| (a - b).powi(2))
128                    .sum::<f64>()
129                    / y_te.len() as f64;
130                sum_mse += mse;
131            }
132            mean_cv_mse.push(sum_mse / folds.len() as f64);
133        }
134        let (best_i, _) = argmin(&mean_cv_mse);
135        let best_alpha = alphas[best_i];
136        let fit = Lasso::fit(y, x, best_alpha, fit_intercept)?;
137        Ok(Self {
138            fit,
139            alphas: alphas.to_vec(),
140            mean_cv_mse,
141            best_alpha,
142        })
143    }
144}
145
146/// Cross-validated ElasticNet with a 2-D `(alpha, l1_ratio)` grid.
147#[derive(Clone, Debug)]
148pub struct ElasticNetCV {
149    /// Fitted ElasticNet at the best `(α, ρ)`.
150    pub fit: ElasticNet,
151    /// α grid searched.
152    pub alphas: Vec<f64>,
153    /// `l1_ratio` grid searched.
154    pub l1_ratios: Vec<f64>,
155    /// Mean CV MSE per `(alpha_idx, l1_ratio_idx)` — row-major.
156    pub mean_cv_mse: Vec<Vec<f64>>,
157    /// Best `(alpha, l1_ratio)` pair.
158    pub best_alpha: f64,
159    /// See [`Self::best_alpha`].
160    pub best_l1_ratio: f64,
161}
162
163impl ElasticNetCV {
164    /// Fit ElasticNetCV over the product grid.
165    pub fn fit(
166        y: ArrayView1<'_, f64>,
167        x: ArrayView2<'_, f64>,
168        alphas: &[f64],
169        l1_ratios: &[f64],
170        k: usize,
171        fit_intercept: bool,
172    ) -> Result<Self> {
173        if alphas.is_empty() || l1_ratios.is_empty() {
174            return Err(Error::Value(
175                "ElasticNetCV::fit: alphas and l1_ratios must be non-empty".into(),
176            ));
177        }
178        if k < 2 || k > y.len() {
179            return Err(Error::Value(format!(
180                "ElasticNetCV::fit: k must be in [2, n] (got {k})"
181            )));
182        }
183        let folds = kfold_indices(y.len(), k);
184        let mut grid: Vec<Vec<f64>> = Vec::with_capacity(alphas.len());
185        let (mut best_i, mut best_j) = (0usize, 0usize);
186        let mut best_score = f64::INFINITY;
187        for (i, &alpha) in alphas.iter().enumerate() {
188            let mut row = Vec::with_capacity(l1_ratios.len());
189            for (j, &rho) in l1_ratios.iter().enumerate() {
190                let mut sum_mse = 0.0_f64;
191                for (train, test) in &folds {
192                    let (y_tr, x_tr) = subset(y, x, train);
193                    let (y_te, x_te) = subset(y, x, test);
194                    let fit = ElasticNet::fit(y_tr.view(), x_tr.view(), alpha, rho, fit_intercept)?;
195                    let pred = fit.predict(x_te.view())?;
196                    let mse: f64 = pred
197                        .iter()
198                        .zip(y_te.iter())
199                        .map(|(a, b)| (a - b).powi(2))
200                        .sum::<f64>()
201                        / y_te.len() as f64;
202                    sum_mse += mse;
203                }
204                let mean = sum_mse / folds.len() as f64;
205                row.push(mean);
206                if mean < best_score {
207                    best_score = mean;
208                    best_i = i;
209                    best_j = j;
210                }
211            }
212            grid.push(row);
213        }
214        let best_alpha = alphas[best_i];
215        let best_l1_ratio = l1_ratios[best_j];
216        let fit = ElasticNet::fit(y, x, best_alpha, best_l1_ratio, fit_intercept)?;
217        Ok(Self {
218            fit,
219            alphas: alphas.to_vec(),
220            l1_ratios: l1_ratios.to_vec(),
221            mean_cv_mse: grid,
222            best_alpha,
223            best_l1_ratio,
224        })
225    }
226}
227
228// ---------------------------------------------------------------------------
229// Helpers — a self-contained K-fold splitter so this module doesn't need a
230// dependency on solow-cv (which would be a cycle: solow-cv would then depend
231// on solow-regression via consumers).
232// ---------------------------------------------------------------------------
233
234fn kfold_indices(n: usize, k: usize) -> Vec<(Vec<usize>, Vec<usize>)> {
235    let mut out = Vec::with_capacity(k);
236    for fold in 0..k {
237        let start = fold * n / k;
238        let end = (fold + 1) * n / k;
239        let test: Vec<usize> = (start..end).collect();
240        let train: Vec<usize> = (0..n).filter(|i| !(start..end).contains(i)).collect();
241        out.push((train, test));
242    }
243    out
244}
245
246fn subset(
247    y: ArrayView1<'_, f64>,
248    x: ArrayView2<'_, f64>,
249    idx: &[usize],
250) -> (Array1<f64>, Array2<f64>) {
251    let d = x.ncols();
252    let mut sub_y = Array1::<f64>::zeros(idx.len());
253    let mut sub_x = Array2::<f64>::zeros((idx.len(), d));
254    for (r, &i) in idx.iter().enumerate() {
255        sub_y[r] = y[i];
256        for j in 0..d {
257            sub_x[[r, j]] = x[[i, j]];
258        }
259    }
260    (sub_y, sub_x)
261}
262
263fn argmin(v: &[f64]) -> (usize, f64) {
264    let mut best_i = 0usize;
265    let mut best_v = f64::INFINITY;
266    for (i, &val) in v.iter().enumerate() {
267        if val < best_v {
268            best_v = val;
269            best_i = i;
270        }
271    }
272    (best_i, best_v)
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278    use ndarray::array;
279
280    #[test]
281    fn ridgecv_picks_a_reasonable_alpha() {
282        // Small linear signal — RidgeCV over a wide α grid should land
283        // near α = 0 (small regularisation).
284        let x = array![
285            [1.0],
286            [2.0],
287            [3.0],
288            [4.0],
289            [5.0],
290            [6.0],
291            [7.0],
292            [8.0],
293            [9.0]
294        ];
295        let y = x.column(0).mapv(|v| 2.0 * v + 1.0);
296        let alphas = vec![0.001, 0.01, 0.1, 1.0, 10.0, 100.0];
297        let cv = RidgeCV::fit(y.view(), x.view(), &alphas, 3, true).unwrap();
298        assert!(alphas.contains(&cv.best_alpha));
299        // Fit predictions are close to y.
300        let pred = cv.fit.predict(x.view()).unwrap();
301        let mse: f64 = pred
302            .iter()
303            .zip(y.iter())
304            .map(|(a, b)| (a - b).powi(2))
305            .sum::<f64>()
306            / y.len() as f64;
307        let mean_y: f64 = y.iter().sum::<f64>() / y.len() as f64;
308        let var_y: f64 = y.iter().map(|v| (v - mean_y).powi(2)).sum::<f64>() / y.len() as f64;
309        assert!(mse < 0.1 * var_y, "MSE = {mse}");
310    }
311
312    #[test]
313    fn lassocv_returns_a_valid_alpha_from_the_grid() {
314        let x = array![
315            [1.0, 5.0],
316            [2.0, -3.0],
317            [3.0, 4.0],
318            [4.0, 0.0],
319            [5.0, 2.0],
320            [6.0, -1.0],
321            [7.0, 3.0],
322            [8.0, 0.5],
323            [9.0, -2.0]
324        ];
325        let y = x.column(0).mapv(|v| 3.0 * v + 1.0);
326        let alphas = vec![0.01, 0.1, 1.0];
327        let cv = LassoCV::fit(y.view(), x.view(), &alphas, 3, true).unwrap();
328        assert!(alphas.contains(&cv.best_alpha));
329        // We have `alphas.len()` mean-MSE entries.
330        assert_eq!(cv.mean_cv_mse.len(), alphas.len());
331    }
332
333    #[test]
334    fn elasticnetcv_searches_the_full_grid() {
335        let x = array![
336            [1.0, 5.0],
337            [2.0, -3.0],
338            [3.0, 4.0],
339            [4.0, 0.0],
340            [5.0, 2.0],
341            [6.0, -1.0],
342            [7.0, 3.0],
343            [8.0, 0.5],
344            [9.0, -2.0]
345        ];
346        let y = x.column(0).mapv(|v| 3.0 * v + 1.0);
347        let alphas = vec![0.01, 0.1];
348        let l1s = vec![0.0, 0.5, 1.0];
349        let cv = ElasticNetCV::fit(y.view(), x.view(), &alphas, &l1s, 3, true).unwrap();
350        assert!(alphas.contains(&cv.best_alpha));
351        assert!(l1s.contains(&cv.best_l1_ratio));
352        assert_eq!(cv.mean_cv_mse.len(), alphas.len());
353        assert_eq!(cv.mean_cv_mse[0].len(), l1s.len());
354    }
355}