ipfrs_tensorlogic/
regularizer.rs

1//! L1/L2/ElasticNet regularization for tensor parameters.
2//!
3//! Provides regularization penalty and gradient computation to prevent
4//! overfitting during tensor-based optimization and training loops.
5
6/// Type of regularization to apply.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum RegularizerType {
9    /// L1 (Lasso): penalty = lambda * sum(|w|)
10    L1,
11    /// L2 (Ridge): penalty = lambda * sum(w^2)
12    L2,
13    /// ElasticNet: alpha * L1 + (1 - alpha) * L2
14    ElasticNet,
15}
16
17/// Configuration for a `TensorRegularizer`.
18#[derive(Debug, Clone)]
19pub struct RegularizerConfig {
20    /// Which regularization strategy to use.
21    pub reg_type: RegularizerType,
22    /// Regularization strength (default 0.01).
23    pub lambda: f64,
24    /// L1 ratio for ElasticNet (default 0.5).
25    /// 0.0 = pure L2, 1.0 = pure L1.
26    pub elastic_alpha: f64,
27}
28
29impl Default for RegularizerConfig {
30    fn default() -> Self {
31        Self {
32            reg_type: RegularizerType::L2,
33            lambda: 0.01,
34            elastic_alpha: 0.5,
35        }
36    }
37}
38
39/// Runtime statistics for a `TensorRegularizer`.
40#[derive(Debug, Clone)]
41pub struct RegularizerStats {
42    /// The regularizer type in use.
43    pub reg_type: RegularizerType,
44    /// The lambda (regularization strength).
45    pub lambda: f64,
46    /// Total number of penalty/gradient computations performed.
47    pub computations: u64,
48}
49
50/// L1/L2/ElasticNet regularization for tensor weight vectors.
51///
52/// # Examples
53///
54/// ```
55/// use ipfrs_tensorlogic::regularizer::{TensorRegularizer, RegularizerConfig, RegularizerType};
56///
57/// let config = RegularizerConfig {
58///     reg_type: RegularizerType::L1,
59///     lambda: 0.1,
60///     elastic_alpha: 0.5,
61/// };
62/// let mut reg = TensorRegularizer::new(config);
63///
64/// let weights = vec![1.0, -2.0, 3.0];
65/// let penalty = reg.penalty(&weights);
66/// assert!((penalty - 0.6).abs() < 1e-12); // 0.1 * (1 + 2 + 3)
67/// ```
68pub struct TensorRegularizer {
69    config: RegularizerConfig,
70    computations: u64,
71}
72
73impl TensorRegularizer {
74    /// Create a new regularizer from the given configuration.
75    pub fn new(config: RegularizerConfig) -> Self {
76        Self {
77            config,
78            computations: 0,
79        }
80    }
81
82    // ------------------------------------------------------------------
83    // Dispatched methods (delegate to the configured type)
84    // ------------------------------------------------------------------
85
86    /// Compute the regularization penalty for the given weight vector.
87    pub fn penalty(&mut self, weights: &[f64]) -> f64 {
88        self.computations += 1;
89        match self.config.reg_type {
90            RegularizerType::L1 => self.l1_penalty(weights),
91            RegularizerType::L2 => self.l2_penalty(weights),
92            RegularizerType::ElasticNet => self.elastic_penalty(weights),
93        }
94    }
95
96    /// Compute the regularization gradient for the given weight vector.
97    pub fn gradient(&mut self, weights: &[f64]) -> Vec<f64> {
98        self.computations += 1;
99        match self.config.reg_type {
100            RegularizerType::L1 => self.l1_gradient(weights),
101            RegularizerType::L2 => self.l2_gradient(weights),
102            RegularizerType::ElasticNet => self.elastic_gradient(weights),
103        }
104    }
105
106    // ------------------------------------------------------------------
107    // L1
108    // ------------------------------------------------------------------
109
110    /// L1 penalty: `lambda * sum(|w_i|)`.
111    pub fn l1_penalty(&self, weights: &[f64]) -> f64 {
112        let sum_abs: f64 = weights.iter().map(|w| w.abs()).sum();
113        self.config.lambda * sum_abs
114    }
115
116    /// L1 gradient: `lambda * sign(w_i)`.
117    ///
118    /// The sub-gradient at zero is defined as 0.0.
119    pub fn l1_gradient(&self, weights: &[f64]) -> Vec<f64> {
120        weights
121            .iter()
122            .map(|&w| {
123                if w > 0.0 {
124                    self.config.lambda
125                } else if w < 0.0 {
126                    -self.config.lambda
127                } else {
128                    0.0
129                }
130            })
131            .collect()
132    }
133
134    // ------------------------------------------------------------------
135    // L2
136    // ------------------------------------------------------------------
137
138    /// L2 penalty: `lambda * sum(w_i^2)`.
139    pub fn l2_penalty(&self, weights: &[f64]) -> f64 {
140        let sum_sq: f64 = weights.iter().map(|w| w * w).sum();
141        self.config.lambda * sum_sq
142    }
143
144    /// L2 gradient: `2 * lambda * w_i`.
145    pub fn l2_gradient(&self, weights: &[f64]) -> Vec<f64> {
146        weights
147            .iter()
148            .map(|&w| 2.0 * self.config.lambda * w)
149            .collect()
150    }
151
152    // ------------------------------------------------------------------
153    // ElasticNet
154    // ------------------------------------------------------------------
155
156    /// ElasticNet penalty: `alpha * L1(w) + (1 - alpha) * L2(w)`.
157    pub fn elastic_penalty(&self, weights: &[f64]) -> f64 {
158        let alpha = self.config.elastic_alpha;
159        alpha * self.l1_penalty(weights) + (1.0 - alpha) * self.l2_penalty(weights)
160    }
161
162    /// ElasticNet gradient: `alpha * L1_grad(w) + (1 - alpha) * L2_grad(w)`.
163    pub fn elastic_gradient(&self, weights: &[f64]) -> Vec<f64> {
164        let alpha = self.config.elastic_alpha;
165        let l1 = self.l1_gradient(weights);
166        let l2 = self.l2_gradient(weights);
167        l1.iter()
168            .zip(l2.iter())
169            .map(|(&a, &b)| alpha * a + (1.0 - alpha) * b)
170            .collect()
171    }
172
173    // ------------------------------------------------------------------
174    // Stats
175    // ------------------------------------------------------------------
176
177    /// Return runtime statistics.
178    pub fn stats(&self) -> RegularizerStats {
179        RegularizerStats {
180            reg_type: self.config.reg_type,
181            lambda: self.config.lambda,
182            computations: self.computations,
183        }
184    }
185}
186
187// ======================================================================
188// Tests
189// ======================================================================
190
191#[cfg(test)]
192mod tests {
193    use super::*;
194
195    fn make(reg_type: RegularizerType, lambda: f64, elastic_alpha: f64) -> TensorRegularizer {
196        TensorRegularizer::new(RegularizerConfig {
197            reg_type,
198            lambda,
199            elastic_alpha,
200        })
201    }
202
203    // ------------------------------------------------------------------
204    // L1 penalty
205    // ------------------------------------------------------------------
206
207    #[test]
208    fn l1_penalty_positive_weights() {
209        let mut r = make(RegularizerType::L1, 0.1, 0.5);
210        let p = r.penalty(&[1.0, 2.0, 3.0]);
211        assert!((p - 0.6).abs() < 1e-12);
212    }
213
214    #[test]
215    fn l1_penalty_negative_weights() {
216        let mut r = make(RegularizerType::L1, 0.1, 0.5);
217        let p = r.penalty(&[-1.0, -2.0, -3.0]);
218        assert!((p - 0.6).abs() < 1e-12);
219    }
220
221    #[test]
222    fn l1_penalty_mixed_weights() {
223        let mut r = make(RegularizerType::L1, 0.5, 0.5);
224        let p = r.penalty(&[1.0, -2.0, 0.0]);
225        assert!((p - 1.5).abs() < 1e-12); // 0.5 * (1+2+0)
226    }
227
228    #[test]
229    fn l1_penalty_zero_weights() {
230        let mut r = make(RegularizerType::L1, 0.1, 0.5);
231        let p = r.penalty(&[0.0, 0.0, 0.0]);
232        assert!((p - 0.0).abs() < 1e-12);
233    }
234
235    #[test]
236    fn l1_penalty_single_weight() {
237        let mut r = make(RegularizerType::L1, 0.2, 0.5);
238        let p = r.penalty(&[5.0]);
239        assert!((p - 1.0).abs() < 1e-12); // 0.2 * 5
240    }
241
242    #[test]
243    fn l1_penalty_empty_weights() {
244        let mut r = make(RegularizerType::L1, 1.0, 0.5);
245        let p = r.penalty(&[]);
246        assert!((p - 0.0).abs() < 1e-12);
247    }
248
249    // ------------------------------------------------------------------
250    // L1 gradient
251    // ------------------------------------------------------------------
252
253    #[test]
254    fn l1_gradient_correctness() {
255        let mut r = make(RegularizerType::L1, 0.1, 0.5);
256        let g = r.gradient(&[3.0, -2.0, 0.0]);
257        assert!((g[0] - 0.1).abs() < 1e-12);
258        assert!((g[1] - (-0.1)).abs() < 1e-12);
259        assert!((g[2] - 0.0).abs() < 1e-12);
260    }
261
262    #[test]
263    fn l1_gradient_all_positive() {
264        let r = make(RegularizerType::L1, 0.5, 0.5);
265        let g = r.l1_gradient(&[1.0, 2.0, 3.0]);
266        assert!(g.iter().all(|&v| (v - 0.5).abs() < 1e-12));
267    }
268
269    #[test]
270    fn l1_gradient_all_negative() {
271        let r = make(RegularizerType::L1, 0.5, 0.5);
272        let g = r.l1_gradient(&[-1.0, -2.0, -3.0]);
273        assert!(g.iter().all(|&v| (v - (-0.5)).abs() < 1e-12));
274    }
275
276    // ------------------------------------------------------------------
277    // L2 penalty
278    // ------------------------------------------------------------------
279
280    #[test]
281    fn l2_penalty_positive_weights() {
282        let mut r = make(RegularizerType::L2, 0.1, 0.5);
283        let p = r.penalty(&[1.0, 2.0, 3.0]);
284        // 0.1 * (1 + 4 + 9) = 1.4
285        assert!((p - 1.4).abs() < 1e-12);
286    }
287
288    #[test]
289    fn l2_penalty_negative_weights() {
290        let mut r = make(RegularizerType::L2, 0.1, 0.5);
291        let p = r.penalty(&[-1.0, -2.0, -3.0]);
292        assert!((p - 1.4).abs() < 1e-12);
293    }
294
295    #[test]
296    fn l2_penalty_zero_weights() {
297        let mut r = make(RegularizerType::L2, 0.1, 0.5);
298        let p = r.penalty(&[0.0, 0.0]);
299        assert!((p - 0.0).abs() < 1e-12);
300    }
301
302    #[test]
303    fn l2_penalty_single_weight() {
304        let mut r = make(RegularizerType::L2, 0.5, 0.5);
305        let p = r.penalty(&[4.0]);
306        assert!((p - 8.0).abs() < 1e-12); // 0.5 * 16
307    }
308
309    // ------------------------------------------------------------------
310    // L2 gradient
311    // ------------------------------------------------------------------
312
313    #[test]
314    fn l2_gradient_correctness() {
315        let mut r = make(RegularizerType::L2, 0.1, 0.5);
316        let g = r.gradient(&[1.0, -2.0, 3.0]);
317        // 2 * 0.1 * w
318        assert!((g[0] - 0.2).abs() < 1e-12);
319        assert!((g[1] - (-0.4)).abs() < 1e-12);
320        assert!((g[2] - 0.6).abs() < 1e-12);
321    }
322
323    #[test]
324    fn l2_gradient_zero() {
325        let r = make(RegularizerType::L2, 0.5, 0.5);
326        let g = r.l2_gradient(&[0.0, 0.0]);
327        assert!(g.iter().all(|&v| v.abs() < 1e-12));
328    }
329
330    // ------------------------------------------------------------------
331    // ElasticNet penalty
332    // ------------------------------------------------------------------
333
334    #[test]
335    fn elastic_penalty_balanced() {
336        let mut r = make(RegularizerType::ElasticNet, 0.1, 0.5);
337        let w = [1.0, -2.0, 3.0];
338        let expected = 0.5 * r.l1_penalty(&w) + 0.5 * r.l2_penalty(&w);
339        let p = r.penalty(&w);
340        assert!((p - expected).abs() < 1e-12);
341    }
342
343    #[test]
344    fn elastic_penalty_pure_l1() {
345        // alpha = 1.0 => pure L1
346        let mut elastic = make(RegularizerType::ElasticNet, 0.1, 1.0);
347        let mut l1 = make(RegularizerType::L1, 0.1, 0.5);
348        let w = [2.0, -3.0, 0.5];
349        let pe = elastic.penalty(&w);
350        let pl = l1.penalty(&w);
351        assert!((pe - pl).abs() < 1e-12);
352    }
353
354    #[test]
355    fn elastic_penalty_pure_l2() {
356        // alpha = 0.0 => pure L2
357        let mut elastic = make(RegularizerType::ElasticNet, 0.1, 0.0);
358        let mut l2 = make(RegularizerType::L2, 0.1, 0.5);
359        let w = [2.0, -3.0, 0.5];
360        let pe = elastic.penalty(&w);
361        let pl = l2.penalty(&w);
362        assert!((pe - pl).abs() < 1e-12);
363    }
364
365    #[test]
366    fn elastic_penalty_zero_weights() {
367        let mut r = make(RegularizerType::ElasticNet, 0.5, 0.3);
368        let p = r.penalty(&[0.0, 0.0]);
369        assert!((p - 0.0).abs() < 1e-12);
370    }
371
372    // ------------------------------------------------------------------
373    // ElasticNet gradient
374    // ------------------------------------------------------------------
375
376    #[test]
377    fn elastic_gradient_balanced() {
378        let mut r = make(RegularizerType::ElasticNet, 0.1, 0.5);
379        let w = [1.0, -2.0, 3.0];
380        let l1g = r.l1_gradient(&w);
381        let l2g = r.l2_gradient(&w);
382        let expected: Vec<f64> = l1g
383            .iter()
384            .zip(l2g.iter())
385            .map(|(&a, &b)| 0.5 * a + 0.5 * b)
386            .collect();
387        let g = r.gradient(&w);
388        for (i, (&got, &exp)) in g.iter().zip(expected.iter()).enumerate() {
389            assert!(
390                (got - exp).abs() < 1e-12,
391                "mismatch at index {i}: got {got}, expected {exp}"
392            );
393        }
394    }
395
396    #[test]
397    fn elastic_gradient_pure_l1() {
398        let mut elastic = make(RegularizerType::ElasticNet, 0.2, 1.0);
399        let mut l1 = make(RegularizerType::L1, 0.2, 0.5);
400        let w = [1.0, -1.0, 0.0];
401        let ge = elastic.gradient(&w);
402        let gl = l1.gradient(&w);
403        for (a, b) in ge.iter().zip(gl.iter()) {
404            assert!((a - b).abs() < 1e-12);
405        }
406    }
407
408    #[test]
409    fn elastic_gradient_pure_l2() {
410        let mut elastic = make(RegularizerType::ElasticNet, 0.2, 0.0);
411        let mut l2 = make(RegularizerType::L2, 0.2, 0.5);
412        let w = [1.0, -1.0, 0.5];
413        let ge = elastic.gradient(&w);
414        let gl = l2.gradient(&w);
415        for (a, b) in ge.iter().zip(gl.iter()) {
416            assert!((a - b).abs() < 1e-12);
417        }
418    }
419
420    // ------------------------------------------------------------------
421    // Lambda scaling
422    // ------------------------------------------------------------------
423
424    #[test]
425    fn lambda_scaling_l1() {
426        let r1 = make(RegularizerType::L1, 0.1, 0.5);
427        let r2 = make(RegularizerType::L1, 0.2, 0.5);
428        let w = [1.0, 2.0, 3.0];
429        let p1 = r1.l1_penalty(&w);
430        let p2 = r2.l1_penalty(&w);
431        assert!((p2 / p1 - 2.0).abs() < 1e-12);
432    }
433
434    #[test]
435    fn lambda_scaling_l2() {
436        let r1 = make(RegularizerType::L2, 0.1, 0.5);
437        let r2 = make(RegularizerType::L2, 0.3, 0.5);
438        let w = [1.0, 2.0];
439        let p1 = r1.l2_penalty(&w);
440        let p2 = r2.l2_penalty(&w);
441        assert!((p2 / p1 - 3.0).abs() < 1e-12);
442    }
443
444    // ------------------------------------------------------------------
445    // Stats tracking
446    // ------------------------------------------------------------------
447
448    #[test]
449    fn stats_initial() {
450        let r = make(RegularizerType::L2, 0.01, 0.5);
451        let s = r.stats();
452        assert_eq!(s.reg_type, RegularizerType::L2);
453        assert!((s.lambda - 0.01).abs() < 1e-12);
454        assert_eq!(s.computations, 0);
455    }
456
457    #[test]
458    fn stats_after_operations() {
459        let mut r = make(RegularizerType::L1, 0.1, 0.5);
460        r.penalty(&[1.0]);
461        r.penalty(&[2.0]);
462        r.gradient(&[3.0]);
463        let s = r.stats();
464        assert_eq!(s.computations, 3);
465    }
466
467    // ------------------------------------------------------------------
468    // Default config
469    // ------------------------------------------------------------------
470
471    #[test]
472    fn default_config() {
473        let cfg = RegularizerConfig::default();
474        assert_eq!(cfg.reg_type, RegularizerType::L2);
475        assert!((cfg.lambda - 0.01).abs() < 1e-12);
476        assert!((cfg.elastic_alpha - 0.5).abs() < 1e-12);
477    }
478
479    // ------------------------------------------------------------------
480    // Edge cases
481    // ------------------------------------------------------------------
482
483    #[test]
484    fn large_weight_l2() {
485        let mut r = make(RegularizerType::L2, 1.0, 0.5);
486        let p = r.penalty(&[1000.0]);
487        assert!((p - 1_000_000.0).abs() < 1e-6);
488    }
489
490    #[test]
491    fn very_small_lambda() {
492        let mut r = make(RegularizerType::L1, 1e-10, 0.5);
493        let p = r.penalty(&[1.0, 2.0, 3.0]);
494        assert!((p - 6e-10).abs() < 1e-20);
495    }
496
497    #[test]
498    fn elastic_alpha_0_25() {
499        let mut r = make(RegularizerType::ElasticNet, 1.0, 0.25);
500        let w = [2.0];
501        // 0.25 * 1.0 * 2.0 + 0.75 * 1.0 * 4.0 = 0.5 + 3.0 = 3.5
502        let p = r.penalty(&w);
503        assert!((p - 3.5).abs() < 1e-12);
504    }
505
506    #[test]
507    fn gradient_empty_weights() {
508        let mut r = make(RegularizerType::L2, 0.1, 0.5);
509        let g = r.gradient(&[]);
510        assert!(g.is_empty());
511    }
512}
ipfrs_tensorlogic/regularizer.rs

ipfrs_tensorlogic/
regularizer.rs
