rusty_machine/learning/toolkit/
regularization.rs

1//! Regularization Module
2//!
3//! This module contains some base utility methods for regularization
4//! within machine learning algorithms.
5//!
6//! The module contains a `Regularization` enum which provides access to
7//! `L1`, `L2` and `ElasticNet` regularization.
8//!
9//! # Examples
10//!
11//! ```
12//! use rusty_machine::learning::toolkit::regularization::Regularization;
13//!
14//! let reg = Regularization::L1(0.5);
15//! ```
16
17use linalg::Metric;
18use linalg::{Matrix, MatrixSlice, BaseMatrix};
19use libnum::{FromPrimitive, Float};
20
21/// Model Regularization
22#[derive(Debug, Clone, Copy)]
23pub enum Regularization<T: Float> {
24    /// L1 Regularization
25    L1(T),
26    /// L2 Regularization
27    L2(T),
28    /// Elastic Net Regularization (L1 and L2)
29    ElasticNet(T, T),
30    /// No Regularization
31    None,
32}
33
34impl<T: Float + FromPrimitive> Regularization<T> {
35    /// Compute the regularization addition to the cost.
36    pub fn reg_cost(&self, mat: MatrixSlice<T>) -> T {
37        match *self {
38            Regularization::L1(x) => Self::l1_reg_cost(&mat, x),
39            Regularization::L2(x) => Self::l2_reg_cost(&mat, x),
40            Regularization::ElasticNet(x, y) => {
41                Self::l1_reg_cost(&mat, x) + Self::l2_reg_cost(&mat, y)
42            }
43            Regularization::None => T::zero(),
44        }
45    }
46
47    /// Compute the regularization addition to the gradient.
48    pub fn reg_grad(&self, mat: MatrixSlice<T>) -> Matrix<T> {
49        match *self {
50            Regularization::L1(x) => Self::l1_reg_grad(&mat, x),
51            Regularization::L2(x) => Self::l2_reg_grad(&mat, x),
52            Regularization::ElasticNet(x, y) => {
53                Self::l1_reg_grad(&mat, x) + Self::l2_reg_grad(&mat, y)
54            }
55            Regularization::None => Matrix::zeros(mat.rows(), mat.cols()),
56        }
57    }
58
59    fn l1_reg_cost(mat: &MatrixSlice<T>, x: T) -> T {
60        // TODO: This won't be regularized. Need to unroll...
61        let l1_norm = mat.iter()
62            .fold(T::zero(), |acc, y| acc + y.abs());
63        l1_norm * x / ((T::one() + T::one()) * FromPrimitive::from_usize(mat.rows()).unwrap())
64    }
65
66    fn l1_reg_grad(mat: &MatrixSlice<T>, x: T) -> Matrix<T> {
67        let m_2 = (T::one() + T::one()) * FromPrimitive::from_usize(mat.rows()).unwrap();
68        let out_mat_data = mat.iter()
69            .map(|y| {
70                if y.is_sign_negative() {
71                    -x / m_2
72                } else {
73                    x / m_2
74                }
75            })
76            .collect::<Vec<_>>();
77        Matrix::new(mat.rows(), mat.cols(), out_mat_data)
78    }
79
80    fn l2_reg_cost(mat: &MatrixSlice<T>, x: T) -> T {
81        mat.norm() * x / ((T::one() + T::one()) * FromPrimitive::from_usize(mat.rows()).unwrap())
82    }
83
84    fn l2_reg_grad(mat: &MatrixSlice<T>, x: T) -> Matrix<T> {
85        mat * (x / FromPrimitive::from_usize(mat.rows()).unwrap())
86    }
87}
88
89#[cfg(test)]
90mod tests {
91    use super::Regularization;
92    use linalg::{Matrix, BaseMatrix};
93    use linalg::Metric;
94
95    #[test]
96    fn test_no_reg() {
97        let input_mat = Matrix::new(3, 4, (0..12).map(|x| x as f64).collect::<Vec<_>>());
98        let mat_slice = input_mat.as_slice();
99
100        let no_reg: Regularization<f64> = Regularization::None;
101
102        let a = no_reg.reg_cost(mat_slice);
103        let b = no_reg.reg_grad(mat_slice);
104
105        assert_eq!(a, 0f64);
106        assert_eq!(b, Matrix::zeros(3, 4));
107    }
108
109    #[test]
110    fn test_l1_reg() {
111        let input_mat = Matrix::new(3, 4, (0..12).map(|x| x as f64 - 3f64).collect::<Vec<_>>());
112        let mat_slice = input_mat.as_slice();
113
114        let no_reg: Regularization<f64> = Regularization::L1(0.5);
115
116        let a = no_reg.reg_cost(mat_slice);
117        let b = no_reg.reg_grad(mat_slice);
118
119        assert!((a - (42f64 / 12f64)) < 1e-18);
120
121        let true_grad = vec![-1., -1., -1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]
122            .into_iter()
123            .map(|x| x / 12f64)
124            .collect::<Vec<_>>();
125
126        for eps in (b - Matrix::new(3, 4, true_grad)).into_vec() {
127            assert!(eps < 1e-18);
128        }
129    }
130
131    #[test]
132    fn test_l2_reg() {
133        let input_mat = Matrix::new(3, 4, (0..12).map(|x| x as f64 - 3f64).collect::<Vec<_>>());
134        let mat_slice = input_mat.as_slice();
135
136        let no_reg: Regularization<f64> = Regularization::L2(0.5);
137
138        let a = no_reg.reg_cost(mat_slice);
139        let b = no_reg.reg_grad(mat_slice);
140
141        assert!((a - (input_mat.norm() / 12f64)) < 1e-18);
142
143        let true_grad = &input_mat / 6f64;
144        for eps in (b - true_grad).into_vec() {
145            assert!(eps < 1e-18);
146        }
147    }
148
149    #[test]
150    fn test_elastic_net_reg() {
151        let input_mat = Matrix::new(3, 4, (0..12).map(|x| x as f64 - 3f64).collect::<Vec<_>>());
152        let mat_slice = input_mat.as_slice();
153
154        let no_reg: Regularization<f64> = Regularization::ElasticNet(0.5, 0.25);
155
156        let a = no_reg.reg_cost(mat_slice);
157        let b = no_reg.reg_grad(mat_slice);
158
159        assert!(a - ((input_mat.norm() / 24f64) + (42f64 / 12f64)) < 1e-18);
160
161        let l1_true_grad = Matrix::new(3, 4,
162            vec![-1., -1., -1., 1., 1., 1., 1., 1., 1., 1., 1., 1.]
163            .into_iter()
164            .map(|x| x / 12f64)
165            .collect::<Vec<_>>());
166        let l2_true_grad = &input_mat / 12f64;
167
168        for eps in (b - l1_true_grad - l2_true_grad)
169            .into_vec() {
170            // Slightly lower boundary than others - more numerical error as more ops.
171            assert!(eps < 1e-12);
172        }
173    }
174}
rusty_machine/learning/toolkit/regularization.rs

rusty_machine/learning/toolkit/
regularization.rs
