Skip to main content

coeus_nn/
regularization.rs

1// ── Regularization modules (G-041) ──
2//
3// AlphaDropout, FeatureAlphaDropout, GaussianNoise, LocalResponseNorm.
4// All are parameter-free regularization layers matching PyTorch's API surface.
5
6use crate::module::{Module, ModuleError};
7use coeus_autograd::Var;
8use coeus_core::Float;
9use coeus_tensor::Tensor;
10
11// ── AlphaDropout ──────────────────────────────────────────────────────────────
12
13/// Alpha-dropout: a dropout variant designed for SELU activation networks.
14///
15/// During training, replaces dropped elements with the negative saturation value
16/// `-alpha * lambda` and rescales the remaining elements so that the expected
17/// value is preserved. `alpha` = 1.6732632423543772026 (SELU ELU parameter)
18/// and `lambda` = 1.0507009873554804934 (SELU scale parameter).
19///
20/// In eval mode (or when `p == 0`) the layer is an identity.
21#[derive(Clone, Debug)]
22pub struct AlphaDropout {
23    /// Dropout probability (elements zeroed per forward call).
24    pub p: f64,
25    /// Whether the layer is in training mode.
26    pub is_training: bool,
27    /// RNG seed for Bernoulli sampling.
28    pub seed: u64,
29}
30
31impl AlphaDropout {
32    /// SELU ELU-parameter (α).
33    const SELU_ALPHA: f64 = 1.673_263_242_354_377_2;
34    /// SELU scale-parameter (λ).
35    const SELU_LAMBDA: f64 = 1.050_700_987_355_480_5;
36
37    /// Create a new AlphaDropout layer.
38    pub fn new(p: f64) -> Self {
39        assert!((0.0..1.0).contains(&p), "p must be in [0, 1)");
40        Self {
41            p,
42            is_training: true,
43            seed: 42,
44        }
45    }
46
47    /// Set training/eval mode.
48    pub fn set_training(&mut self, mode: bool) {
49        self.is_training = mode;
50    }
51}
52
53fn alpha_dropout_with_mask<
54    T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
55    B: coeus_ops::BackendOps<T> + Default,
56>(
57    input: &Var<T, B>,
58    p: f64,
59    seed: u64,
60    feature_wise: bool,
61) -> Result<Var<T, B>, ModuleError<B::Error>> {
62    let shape = input.tensor.shape_cloned();
63    if feature_wise && shape.len() < 2 {
64        return Err(ModuleError::InvalidRank {
65            module: "FeatureAlphaDropout",
66            expected: "at least 2",
67            actual: shape.len(),
68        });
69    }
70
71    let numel = shape.iter().product::<usize>();
72    let spatial = shape.get(2..).map_or(1, |dims| dims.iter().product());
73    let channels = shape.get(1).copied().unwrap_or(1);
74    let mut rng = coeus_autograd::ops::nn::dropout::Xorshift64::new(seed);
75    let mut keep = Vec::with_capacity(numel);
76
77    if feature_wise {
78        for _batch in 0..shape[0] {
79            for _channel in 0..channels {
80                let kept = rng.next_unit() >= p;
81                keep.extend(std::iter::repeat_n(
82                    if kept { T::ONE } else { T::ZERO },
83                    spatial,
84                ));
85            }
86        }
87    } else {
88        keep.extend((0..numel).map(|_| {
89            if rng.next_unit() >= p {
90                T::ONE
91            } else {
92                T::ZERO
93            }
94        }));
95    }
96
97    let alpha_prime = -(AlphaDropout::SELU_ALPHA * AlphaDropout::SELU_LAMBDA);
98    let keep_probability = 1.0 - p;
99    let scale = 1.0 / (keep_probability * (1.0 + p * alpha_prime * alpha_prime)).sqrt();
100    let shift = -scale * p * alpha_prime;
101    let saturation = <T as coeus_core::FloatElement>::from_f64(alpha_prime);
102    let backend = B::default();
103    let keep_tensor = Tensor::from_slice_on(shape.clone(), &keep, &backend)?;
104    let saturation_data = keep
105        .iter()
106        .map(|&value| {
107            if value == T::ZERO {
108                saturation
109            } else {
110                T::ZERO
111            }
112        })
113        .collect::<Vec<_>>();
114    let keep_var = Var::new(keep_tensor, false)?;
115    let saturation_var = Var::new(
116        Tensor::from_slice_on(shape, &saturation_data, &backend)?,
117        false,
118    )?;
119    let selected = coeus_autograd::add(&coeus_autograd::mul(input, &keep_var)?, &saturation_var)?;
120    Ok(coeus_autograd::scalar_add(
121        &coeus_autograd::scalar_mul(&selected, <T as coeus_core::FloatElement>::from_f64(scale))?,
122        <T as coeus_core::FloatElement>::from_f64(shift),
123    )?)
124}
125
126impl<
127        T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
128        B: coeus_ops::BackendOps<T> + Default,
129    > Module<T, B> for AlphaDropout
130{
131    fn parameters(&self) -> Vec<Var<T, B>> {
132        vec![]
133    }
134
135    fn train(&mut self, mode: bool) {
136        self.set_training(mode);
137    }
138
139    /// Alpha-dropout forward pass.
140    ///
141    /// In training mode each element is independently dropped with probability `p`.
142    /// Dropped elements are replaced with the SELU saturation value
143    /// `α' = -λ·α ≈ -1.7581`.  A single affine shift `(a, b)` is then applied
144    /// element-wise so that the output has the same mean and variance as the
145    /// input: `a = 1/sqrt(1 - p*(1-α'^2*(1-p^2)))`, `b = -a*(p*α' + 0)`.
146    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
147        if !self.is_training || self.p == 0.0 {
148            return Ok(input.clone());
149        }
150        alpha_dropout_with_mask(input, self.p, self.seed, false)
151    }
152}
153
154// ── FeatureAlphaDropout ───────────────────────────────────────────────────────
155
156/// Feature alpha-dropout: channel-wise alpha-dropout for 2D+ tensors.
157///
158/// Same as `AlphaDropout` but drops entire feature maps (channels) rather
159/// than individual elements, matching `torch.nn.FeatureAlphaDropout`.
160#[derive(Clone, Debug)]
161pub struct FeatureAlphaDropout {
162    /// Dropout probability per feature map.
163    pub p: f64,
164    /// Training mode flag.
165    pub is_training: bool,
166    /// RNG seed.
167    pub seed: u64,
168}
169
170impl FeatureAlphaDropout {
171    /// Create a new FeatureAlphaDropout layer.
172    pub fn new(p: f64) -> Self {
173        assert!((0.0..1.0).contains(&p), "p must be in [0, 1)");
174        Self {
175            p,
176            is_training: true,
177            seed: 42,
178        }
179    }
180
181    /// Set training/eval mode.
182    pub fn set_training(&mut self, mode: bool) {
183        self.is_training = mode;
184    }
185}
186
187impl<
188        T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
189        B: coeus_ops::BackendOps<T> + Default,
190    > Module<T, B> for FeatureAlphaDropout
191{
192    fn parameters(&self) -> Vec<Var<T, B>> {
193        vec![]
194    }
195
196    fn train(&mut self, mode: bool) {
197        self.set_training(mode);
198    }
199
200    /// Feature alpha-dropout forward: same as AlphaDropout but applied channel-wise.
201    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
202        if !self.is_training || self.p == 0.0 {
203            return Ok(input.clone());
204        }
205        alpha_dropout_with_mask(input, self.p, self.seed, true)
206    }
207}
208
209// ── GaussianNoise ─────────────────────────────────────────────────────────────
210
211/// Gaussian noise regularisation layer.
212///
213/// During training, adds i.i.d. Gaussian noise `N(0, std^2)` element-wise.
214/// In eval mode the layer is an identity (no noise added).
215#[derive(Clone, Debug)]
216pub struct GaussianNoise {
217    /// Standard deviation of the injected Gaussian noise.
218    pub std: f64,
219    /// Training mode flag.
220    pub is_training: bool,
221}
222
223impl GaussianNoise {
224    /// Create a GaussianNoise layer with given standard deviation.
225    pub fn new(std: f64) -> Self {
226        assert!(std >= 0.0, "std must be non-negative");
227        Self {
228            std,
229            is_training: true,
230        }
231    }
232
233    /// Set training/eval mode.
234    pub fn set_training(&mut self, mode: bool) {
235        self.is_training = mode;
236    }
237}
238
239impl<
240        T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
241        B: coeus_ops::BackendOps<T> + Default,
242    > Module<T, B> for GaussianNoise
243{
244    fn parameters(&self) -> Vec<Var<T, B>> {
245        vec![]
246    }
247
248    fn train(&mut self, mode: bool) {
249        self.set_training(mode);
250    }
251
252    /// Add Gaussian noise during training; identity during evaluation.
253    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
254        if !self.is_training || self.std == 0.0 {
255            return Ok(input.clone());
256        }
257        let backend = B::default();
258        let shape = input.tensor.shape_cloned();
259        let numel = shape.iter().product::<usize>();
260
261        // Box-Muller transform: two uniform samples → one normal sample.
262        let mut rng = coeus_autograd::ops::nn::dropout::Xorshift64::new(42);
263        let mut noise = vec![T::ZERO; numel];
264        let std = self.std;
265        let mut i = 0;
266        while i < numel {
267            let u1 = rng.next_unit().max(1e-10);
268            let u2 = rng.next_unit();
269            let r = (-2.0 * u1.ln()).sqrt() * std;
270            noise[i] = <T as coeus_core::FloatElement>::from_f64(
271                r * (2.0 * std::f64::consts::PI * u2).cos(),
272            );
273            if i + 1 < numel {
274                noise[i + 1] = <T as coeus_core::FloatElement>::from_f64(
275                    r * (2.0 * std::f64::consts::PI * u2).sin(),
276                );
277            }
278            i += 2;
279        }
280
281        let noise_tensor = Tensor::from_slice_on(shape, &noise, &backend)?;
282        let noise_var = Var::new(noise_tensor, false)?;
283        Ok(coeus_autograd::add(input, &noise_var)?)
284    }
285}
286
287// ── LocalResponseNorm ─────────────────────────────────────────────────────────
288
289/// Local Response Normalization (LRN).
290///
291/// Normalises within a local neighbourhood across channels:
292/// `b[n,c,h,w] = a[n,c,h,w] / (k + alpha/size * sum(a[n,j,h,w]^2))^beta`
293/// where `j` runs over `[max(0,c-size/2), min(C,c+size/2+1))`.
294///
295/// Matches `torch.nn.LocalResponseNorm(size, alpha, beta, k)`.
296#[derive(Clone, Debug)]
297pub struct LocalResponseNorm {
298    /// Number of adjacent channels to normalise across.
299    pub size: usize,
300    /// Scaling factor in the denominator.
301    pub alpha: f64,
302    /// Exponent.
303    pub beta: f64,
304    /// Additive constant for numerical stability.
305    pub k: f64,
306}
307
308impl LocalResponseNorm {
309    /// Create a LocalResponseNorm layer with PyTorch defaults.
310    ///
311    /// `torch.nn.LocalResponseNorm(size)` uses `alpha=0.0001`, `beta=0.75`, `k=1.0`.
312    pub fn new(size: usize) -> Self {
313        Self {
314            size,
315            alpha: 0.0001,
316            beta: 0.75,
317            k: 1.0,
318        }
319    }
320
321    /// Create with full hyperparameter control.
322    pub fn with_params(size: usize, alpha: f64, beta: f64, k: f64) -> Self {
323        assert!(size >= 1, "size must be >= 1");
324        Self {
325            size,
326            alpha,
327            beta,
328            k,
329        }
330    }
331}
332
333impl<
334        T: Float + coeus_core::FloatElement + std::ops::Neg<Output = T> + coeus_cpu::RealScalar,
335        B: coeus_ops::BackendOps<T> + Default,
336    > Module<T, B> for LocalResponseNorm
337where
338    B::DeviceBuffer<T>:
339        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
340{
341    fn parameters(&self) -> Vec<Var<T, B>> {
342        vec![]
343    }
344
345    /// LRN forward pass — differentiable.
346    ///
347    /// Supports 2D (`[N, C]`), 3D (`[N, C, L]`), and 4D (`[N, C, H, W]`) inputs.
348    /// Built entirely from autograd ops so gradients flow to the input: the
349    /// cross-channel windowed sum-of-squares is a constant band-matrix product
350    /// (differentiable through the squared activations), and the `^beta`
351    /// denominator uses the differentiable `pow` (the base `k + .. >= k > 0`,
352    /// so the `exp(beta*ln(.))` it computes is well defined).
353    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
354        let shape = input.tensor.shape_cloned();
355        if !(2..=4).contains(&shape.len()) {
356            return Err(ModuleError::InvalidRank {
357                module: "LocalResponseNorm",
358                expected: "2 to 4",
359                actual: shape.len(),
360            });
361        }
362        let n = shape[0];
363        let c = shape[1];
364        let spatial: usize = shape[2..].iter().product::<usize>().max(1);
365        let half = self.size / 2;
366        let backend = B::default();
367
368        // View as [N, C, spatial] (channel axis = dim 1); square it.
369        let x3 = coeus_autograd::reshape(input, [n, c, spatial])?;
370        let sq = coeus_autograd::mul(&x3, &x3)?;
371
372        // Constant band matrix M [C, C], M[i, j] = 1 iff |i - j| <= half. Then
373        // `M @ sq` over the channel axis is each channel's squared response
374        // summed across its size-neighbourhood (with boundary clamping implicit
375        // in the band), differentiable through `sq`.
376        let mut m_data = vec![T::ZERO; c * c];
377        for i in 0..c {
378            let lo = i.saturating_sub(half);
379            let hi = (i + half + 1).min(c);
380            for cell in m_data[i * c + lo..i * c + hi].iter_mut() {
381                *cell = T::ONE;
382            }
383        }
384        let m = Var::new(Tensor::from_slice_on([c, c], &m_data, &backend)?, false)?;
385
386        // windowed = M @ sq:  [N,C,S] -> [C,N*S] -> M@ -> [C,N*S] -> [N,C,S].
387        let sq_cns = coeus_autograd::permute(&sq, &[1, 0, 2])?;
388        let sq_2d = coeus_autograd::reshape(&sq_cns, [c, n * spatial])?;
389        let win_2d = coeus_autograd::matmul(&m, &sq_2d)?;
390        let win_cns = coeus_autograd::reshape(&win_2d, [c, n, spatial])?;
391        let windowed = coeus_autograd::permute(&win_cns, &[1, 0, 2])?;
392
393        // denom = (k + (alpha / size) * windowed)^beta;  y = x / denom.
394        // The count is exact at f64 precision and the quotient is rounded once
395        // into the element type.
396        let scaled = coeus_autograd::scalar_mul(
397            &windowed,
398            <T as coeus_core::FloatElement>::from_f64(
399                self.alpha / <f64 as coeus_core::FloatElement>::from_count(self.size),
400            ),
401        )?;
402        let denom = coeus_autograd::pow(
403            &coeus_autograd::scalar_add(
404                &scaled,
405                <T as coeus_core::FloatElement>::from_f64(self.k),
406            )?,
407            self.beta,
408        )?;
409        let y3 = coeus_autograd::div(&x3, &denom)?;
410        Ok(coeus_autograd::reshape(&y3, shape)?)
411    }
412}