Skip to main content

coeus_nn/normalization/
batchnorm.rs

1//! Generic N-dimensional batch normalization.
2//!
3//! [`BatchNorm`](crate::normalization::batchnorm::BatchNorm) is the one implementation behind the
4//! [`BatchNorm1d`](crate::normalization::BatchNorm1d),
5//! [`BatchNorm2d`](crate::normalization::BatchNorm2d) and
6//! [`BatchNorm3d`](crate::normalization::BatchNorm3d) aliases: the forward
7//! pass is const-generic over the spatial rank `DIM`.
8//!
9//! The permutation/reshape tables below deliberately mirror the `match DIM`
10//! helpers in `coeus-autograd`'s `ops/nn/normalization/batchnorm/bn_nd.rs`.
11//! Those helpers are `pub(crate)` to `coeus-autograd` and so cannot be named
12//! across the crate boundary, so the *table* (not the orchestration) is
13//! duplicated here on purpose.
14
15use super::validation;
16use crate::module::{Module, ModuleError, ModuleResult};
17use coeus_autograd::Var;
18use coeus_core::{Float, MoiraiBackend, Scalar};
19use coeus_tensor::Tensor;
20use std::cell::RefCell;
21
22/// A pair of tensors on the same backend.
23type TensorPair<T, B> = (Tensor<T, B>, Tensor<T, B>);
24
25/// Cached per-batch-size constants entry: `(m, m_const, corr_t)`.
26type MCacheEntry<T, B> = (usize, Tensor<T, B>, Tensor<T, B>);
27
28/// Cached per-batch-size constants, keyed by the batch size they were built for.
29type MCache<T, B> = RefCell<Option<MCacheEntry<T, B>>>;
30
31/// Diagnostic module name for a given spatial rank.
32const fn module_name<const DIM: usize>() -> &'static str {
33    match DIM {
34        1 => "BatchNorm1d",
35        2 => "BatchNorm2d",
36        3 => "BatchNorm3d",
37        _ => "BatchNormNd",
38    }
39}
40
41/// Expected-rank description for the rank check (`1-D` also accepts `[N, C]`).
42const fn rank_str<const DIM: usize>() -> &'static str {
43    match DIM {
44        1 => "2 or 3",
45        2 => "4",
46        3 => "5",
47        _ => "configured batch-normalization rank",
48    }
49}
50
51// ── Permute/reshape dispatch (mirrors `coeus-autograd`'s `bn_nd.rs`) ──
52
53/// `[N, C, spatial...] -> [N, spatial..., C]`.
54fn permute_to_nhwc<T: Scalar, B: coeus_ops::BackendOps<T> + Default, const DIM: usize>(
55    tensor: &Tensor<T, B>,
56    backend: &B,
57) -> Result<Tensor<T, B>, B::Error> {
58    match DIM {
59        1 => tensor.permute(&[0, 2, 1]).to_contiguous_on(backend),
60        2 => tensor.permute(&[0, 2, 3, 1]).to_contiguous_on(backend),
61        3 => tensor.permute(&[0, 2, 3, 4, 1]).to_contiguous_on(backend),
62        _ => panic!("BatchNorm permute_to_nhwc: unsupported DIM {DIM}"),
63    }
64}
65
66/// `[N, spatial..., C] -> [N, C, spatial...]`.
67fn permute_from_nhwc<T: Scalar, B: coeus_ops::BackendOps<T> + Default, const DIM: usize>(
68    tensor: &Tensor<T, B>,
69    backend: &B,
70) -> Result<Tensor<T, B>, B::Error> {
71    match DIM {
72        1 => tensor.permute(&[0, 2, 1]).to_contiguous_on(backend),
73        2 => tensor.permute(&[0, 3, 1, 2]).to_contiguous_on(backend),
74        3 => tensor.permute(&[0, 4, 1, 2, 3]).to_contiguous_on(backend),
75        _ => panic!("BatchNorm permute_from_nhwc: unsupported DIM {DIM}"),
76    }
77}
78
79/// `[M, C] -> [N, spatial..., C]`.
80fn reshape_from_flat<T: Scalar, B: coeus_ops::BackendOps<T>, const DIM: usize>(
81    tensor: Tensor<T, B>,
82    n: usize,
83    spatial: &[usize],
84    c: usize,
85) -> Result<Tensor<T, B>, B::Error> {
86    match DIM {
87        1 => Ok(tensor.reshape([n, spatial[0], c])),
88        2 => Ok(tensor.reshape([n, spatial[0], spatial[1], c])),
89        3 => Ok(tensor.reshape([n, spatial[0], spatial[1], spatial[2], c])),
90        _ => panic!("BatchNorm reshape_from_flat: unsupported DIM {DIM}"),
91    }
92}
93
94/// N-dimensional batch normalization for `DIM`-spatial-rank inputs.
95///
96/// `BatchNorm<T, B, 1>` normalizes `[N, C]` or `[N, C, L]` inputs (the
97/// `[N, C]` form is the degenerate `L = 1` case, handled by the squeeze
98/// adapter in [`Module::forward`]); `BatchNorm<T, B, 2>` handles
99/// `[N, C, H, W]` and `BatchNorm<T, B, 3>` handles `[N, C, D, H, W]`.
100/// Prefer the [`BatchNorm1d`](crate::normalization::BatchNorm1d),
101/// [`BatchNorm2d`](crate::normalization::BatchNorm2d) and
102/// [`BatchNorm3d`](crate::normalization::BatchNorm3d) aliases.
103///
104/// Running stats are updated during each forward call in training mode; in
105/// eval mode (`is_training = false`) the frozen `running_mean`/`running_var`
106/// are used instead.
107#[derive(Clone)]
108pub struct BatchNorm<
109    T: Float,
110    B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
111    const DIM: usize = 1,
112> {
113    /// Number of channels (C dimension).
114    pub num_features: usize,
115    /// Learnable scale (gamma): `[C]`.
116    pub weight: Var<T, B>,
117    /// Learnable shift (beta): `[C]`.
118    pub bias: Var<T, B>,
119    /// Numerical stability constant added to variance.
120    pub eps: f64,
121    /// Exponential moving average factor for running stats.
122    pub momentum: f64,
123    /// Whether the layer is in training mode (updates running stats when true).
124    pub is_training: bool,
125    /// Running mean `[C]`.
126    pub running_mean: RefCell<Tensor<T, B>>,
127    /// Running variance `[C]`.
128    pub running_var: RefCell<Tensor<T, B>>,
129    /// Cached epsilon tensor: `[1]`.
130    eps_t: Tensor<T, B>,
131    /// Cached momentum tensor: `[1]`.
132    mom_t: Tensor<T, B>,
133    /// Cached 1 - momentum tensor: `[1]`.
134    one_minus_mom_t: Tensor<T, B>,
135    /// Cached -0.5 constant tensor: `[1]`.
136    minus_half: Tensor<T, B>,
137    /// Cached 2.0 constant tensor: `[1]`.
138    two_const: Tensor<T, B>,
139    /// Cached ones tensor of shape `[1, C]`.
140    ones_c: Tensor<T, B>,
141    /// Cached spatial batch size m constants: `(m, m_const, corr_t)`.
142    m_cache: MCache<T, B>,
143}
144
145impl<
146        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
147        B: coeus_ops::BackendOps<T> + Default,
148        const DIM: usize,
149    > BatchNorm<T, B, DIM>
150{
151    /// Create with ones weight, zeros bias, and initialized running stats.
152    pub fn new(
153        num_features: usize,
154        eps: f64,
155        momentum: f64,
156    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
157        let backend = B::default();
158        Self::from_parts(
159            num_features,
160            Var::new(Tensor::ones_on([num_features], &backend)?, true)?,
161            Var::new(Tensor::zeros_on([num_features], &backend)?, true)?,
162            eps,
163            momentum,
164            Tensor::zeros_on([num_features], &backend)?,
165            Tensor::ones_on([num_features], &backend)?,
166        )
167    }
168
169    /// Construct from pre-existing weight, bias, and running-stat tensors (e.g. after checkpoint load).
170    pub fn from_parts(
171        num_features: usize,
172        weight: Var<T, B>,
173        bias: Var<T, B>,
174        eps: f64,
175        momentum: f64,
176        running_mean: Tensor<T, B>,
177        running_var: Tensor<T, B>,
178    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
179        let backend = B::default();
180        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
181        let mom_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(momentum), &backend)?;
182        let one_minus_mom_t = Tensor::full_on(
183            [1],
184            coeus_core::FloatElement::from_f64(1.0 - momentum),
185            &backend,
186        )?;
187        let minus_half = Tensor::full_on([1], coeus_core::FloatElement::from_f64(-0.5), &backend)?;
188        let two_const = Tensor::full_on([1], coeus_core::FloatElement::from_f64(2.0), &backend)?;
189        let ones_c = Tensor::ones_on([1, num_features], &backend)?;
190        Ok(Self {
191            num_features,
192            weight,
193            bias,
194            eps,
195            momentum,
196            is_training: true,
197            running_mean: RefCell::new(running_mean),
198            running_var: RefCell::new(running_var),
199            eps_t,
200            mom_t,
201            one_minus_mom_t,
202            minus_half,
203            two_const,
204            ones_c,
205            m_cache: RefCell::new(None),
206        })
207    }
208
209    /// Set training/eval mode.
210    pub fn set_training(&mut self, mode: bool) {
211        self.is_training = mode;
212    }
213
214    /// Retrieve (or build and cache) the per-batch `m` constants `(m_const, corr_t)`.
215    fn m_constants(&self, m: usize, backend: &B) -> ModuleResult<TensorPair<T, B>, B> {
216        let module: &str = module_name::<DIM>();
217        let mut cache = self
218            .m_cache
219            .try_borrow_mut()
220            .map_err(|_| validation::state_borrow(module, "m_cache"))?;
221        if let Some((cached_m, ref cached_m_const, ref cached_corr_t)) = *cache {
222            if cached_m == m {
223                return Ok((cached_m_const.clone(), cached_corr_t.clone()));
224            }
225        }
226        let m_const = Tensor::full_on([1], coeus_core::FloatElement::from_count(m), backend)?;
227        // The unbiased correction `m/(m-1)`: both counts are exact at f64
228        // precision and the quotient is rounded once into the element type.
229        let correction = if m > 1 {
230            <f64 as coeus_core::FloatElement>::from_count(m)
231                / <f64 as coeus_core::FloatElement>::from_count(m - 1)
232        } else {
233            1.0
234        };
235        let corr_t = Tensor::full_on(
236            [1],
237            <T as coeus_core::FloatElement>::from_f64(correction),
238            backend,
239        )?;
240        *cache = Some((m, m_const.clone(), corr_t.clone()));
241        Ok((m_const, corr_t))
242    }
243}
244
245impl<
246        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
247        B: coeus_ops::BackendOps<T> + Default,
248        const DIM: usize,
249    > Module<T, B> for BatchNorm<T, B, DIM>
250{
251    fn parameters(&self) -> Vec<Var<T, B>> {
252        vec![self.weight.clone(), self.bias.clone()]
253    }
254
255    fn train(&mut self, mode: bool) {
256        self.is_training = mode;
257    }
258
259    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
260        let module: &str = module_name::<DIM>();
261        let shape = input.tensor.shape();
262        // 1-D additionally accepts the rank-2 `[N, C]` form (the `L = 1` case).
263        let rank_ok = shape.len() == DIM + 2 || (DIM == 1 && shape.len() == 2);
264        if !rank_ok {
265            return Err(validation::invalid_rank(
266                module,
267                rank_str::<DIM>(),
268                shape.len(),
269            ));
270        }
271        let c = shape[1];
272        if c != self.num_features {
273            return Err(validation::channel_mismatch(module, self.num_features, c));
274        }
275        if !self.eps.is_finite() || self.eps < 0.0 {
276            return Err(ModuleError::InvalidEpsilon { module });
277        }
278        for (parameter, actual) in [
279            ("weight", self.weight.tensor.shape()),
280            ("bias", self.bias.tensor.shape()),
281        ] {
282            if actual != [self.num_features] {
283                return Err(validation::shape_mismatch(
284                    module,
285                    parameter,
286                    &[self.num_features],
287                    actual,
288                ));
289            }
290        }
291        // PyTorch `nn.BatchNorm1d` accepts both `[N, C]` and `[N, C, L]` inputs;
292        // the `[N, C]` form is the degenerate case `L = 1`.  Squeeze-unsqueeze via
293        // autograd-tracked `reshape` so the 2D path stays differentiable and the
294        // existing 3D kernel runs unchanged on the reshaped tensor.
295        let input_is_2d = DIM == 1 && shape.len() == 2;
296        let upstream: Var<T, B> = if input_is_2d {
297            let n = shape[0];
298            let c = shape[1];
299            // [N, C] -> [N, C, 1]; preserve grad-creator by going through `reshape`.
300            coeus_autograd::reshape(input, vec![n, c, 1])?
301        } else {
302            input.clone()
303        };
304        let out = self.forward_nd(&upstream)?;
305        if input_is_2d {
306            let n = shape[0];
307            let c = shape[1];
308            // [N, C, 1] -> [N, C]; preserve grad-creator.
309            Ok(coeus_autograd::reshape(&out, vec![n, c])?)
310        } else {
311            Ok(out)
312        }
313    }
314}
315
316impl<
317        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
318        B: coeus_ops::BackendOps<T> + Default,
319        const DIM: usize,
320    > BatchNorm<T, B, DIM>
321{
322    /// Rank-`DIM` forward path: `[N, C, spatial...] -> [N, C, spatial...]`.
323    /// Separated from the `Module` trait surface so the 2D-input adapter above
324    /// can call it without going through the trait vtable.
325    fn forward_nd(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
326        let module: &str = module_name::<DIM>();
327        let shape = input.tensor.shape();
328        let n = shape[0];
329        let c = shape[1];
330        let mut spatial = [1usize; 3];
331        for (slot, &extent) in spatial.iter_mut().zip(shape[2..].iter()) {
332            *slot = extent;
333        }
334        let backend = B::default();
335        let m_spatial: usize = spatial.iter().product();
336        let m = n * m_spatial; // spatial batch size
337
338        // ── Eval mode: use running stats without updating them ──
339        if !self.is_training {
340            let rm = self
341                .running_mean
342                .try_borrow()
343                .map_err(|_| validation::state_borrow(module, "running_mean"))?;
344            let rv = self
345                .running_var
346                .try_borrow()
347                .map_err(|_| validation::state_borrow(module, "running_var"))?;
348            for (parameter, actual) in [("running_mean", rm.shape()), ("running_var", rv.shape())] {
349                if actual != [self.num_features] {
350                    return Err(validation::shape_mismatch(
351                        module,
352                        parameter,
353                        &[self.num_features],
354                        actual,
355                    ));
356                }
357            }
358            // Normalize using running stats: (x - running_mean) / coeus_core::FloatElement::sqrt(running_var + eps)
359            let nhwc = permute_to_nhwc::<T, B, DIM>(&input.tensor, &backend);
360            let flat = nhwc?.reshape([m, c]);
361            let rm_row = rm.reshape([1, c]);
362            let rv_row = rv.reshape([1, c]);
363            let mut istdev = rv_row.clone();
364            coeus_ops::add_assign(&mut istdev, &self.eps_t, &backend)
365                .map_err(|source| validation::backend(module, source))?;
366            coeus_ops::sqrt_assign(&mut istdev, &backend)
367                .map_err(|source| validation::backend(module, source))?;
368            let ones = Tensor::ones_on([1, c], &backend)?;
369            let mut istdev_inv = ones;
370            coeus_ops::div_assign(&mut istdev_inv, &istdev, &backend)
371                .map_err(|source| validation::backend(module, source))?;
372            let xmu = coeus_ops::sub(&flat, &rm_row, &backend)?;
373            let x_hat = coeus_ops::mul(&xmu, &istdev_inv, &backend)?;
374            let w_r = self.weight.tensor.reshape([1, c]);
375            let b_r = self.bias.tensor.reshape([1, c]);
376            let mut y = coeus_ops::mul(&x_hat, &w_r, &backend)?;
377            coeus_ops::add_assign(&mut y, &b_r, &backend)
378                .map_err(|source| validation::backend(module, source))?;
379            let y_nhwc = reshape_from_flat::<T, B, DIM>(y, n, &spatial, c)?;
380            let out_tensor = permute_from_nhwc::<T, B, DIM>(&y_nhwc, &backend)?;
381            return Ok(Var::new(out_tensor, false)?);
382        }
383
384        if m < 2 {
385            return Err(validation::insufficient_elements(module, 2, m));
386        }
387
388        let (m_const, corr_t) = self.m_constants(m, &backend)?;
389
390        // ── View as [M, C] via NCHW... → NHWC... → [M, C] ──
391        let nhwc = permute_to_nhwc::<T, B, DIM>(&input.tensor, &backend);
392        let flat = nhwc?.reshape([m, c]); // [M, C]
393
394        // ── Per-channel mean [1, C] ──
395        let mean_t = coeus_ops::mean_axis(&flat, 0, &backend)
396            .map_err(|source| validation::backend(module, source))?; // [1, C]
397
398        // ── Centered: x - mu [M, C] ──
399        let xmu = coeus_ops::sub(&flat, &mean_t, &backend)?;
400
401        // ── Per-channel variance [1, C] ──
402        let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
403        let var_t = coeus_ops::mean_axis(&xmu_sq, 0, &backend)
404            .map_err(|source| validation::backend(module, source))?; // [1, C]
405
406        // ── 1/coeus_core::FloatElement::sqrt(var + eps) [1, C] ──
407        let mut stdev = var_t.clone();
408        coeus_ops::add_assign(&mut stdev, &self.eps_t, &backend)
409            .map_err(|source| validation::backend(module, source))?;
410        coeus_ops::sqrt_assign(&mut stdev, &backend)
411            .map_err(|source| validation::backend(module, source))?;
412
413        let mut istdev = self.ones_c.clone();
414        coeus_ops::div_assign(&mut istdev, &stdev, &backend)
415            .map_err(|source| validation::backend(module, source))?; // [1, C]
416
417        // ── x_hat = xmu * istdev [M, C] ──
418        let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
419
420        // ── y = gamma * x_hat + beta ──
421        let w_reshaped = self.weight.tensor.reshape([1, c]);
422        let b_reshaped = self.bias.tensor.reshape([1, c]);
423        let mut y_flat = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
424        coeus_ops::add_assign(&mut y_flat, &b_reshaped, &backend)
425            .map_err(|source| validation::backend(module, source))?;
426
427        // ── Output: [M, C] → [N, ..., C] → permute → [N, C, ...] ──
428        let y_nhwc = reshape_from_flat::<T, B, DIM>(y_flat, n, &spatial, c)?;
429        let out_tensor = permute_from_nhwc::<T, B, DIM>(&y_nhwc, &backend)?;
430
431        // ── Update running stats (exponential moving average) ──
432        let mut next_mean = self
433            .running_mean
434            .try_borrow()
435            .map_err(|_| validation::state_borrow(module, "running_mean"))?
436            .clone();
437        let mut next_var = self
438            .running_var
439            .try_borrow()
440            .map_err(|_| validation::state_borrow(module, "running_var"))?
441            .clone();
442        for (parameter, actual) in [
443            ("running_mean", next_mean.shape()),
444            ("running_var", next_var.shape()),
445        ] {
446            if actual != [self.num_features] {
447                return Err(validation::shape_mismatch(
448                    module,
449                    parameter,
450                    &[self.num_features],
451                    actual,
452                ));
453            }
454        }
455        let mean_c = mean_t.reshape([c]);
456        let var_c = var_t.reshape([c]);
457        coeus_ops::mul_assign(&mut next_mean, &self.one_minus_mom_t, &backend)
458            .map_err(|source| validation::backend(module, source))?;
459        let term_mean = coeus_ops::mul(&mean_c, &self.mom_t, &backend)?;
460        coeus_ops::add_assign(&mut next_mean, &term_mean, &backend)
461            .map_err(|source| validation::backend(module, source))?;
462        coeus_ops::mul_assign(&mut next_var, &self.one_minus_mom_t, &backend)
463            .map_err(|source| validation::backend(module, source))?;
464        let var_corrected = coeus_ops::mul(&var_c, &corr_t, &backend)?;
465        let term_var = coeus_ops::mul(&var_corrected, &self.mom_t, &backend)?;
466        coeus_ops::add_assign(&mut next_var, &term_var, &backend)
467            .map_err(|source| validation::backend(module, source))?;
468        let mut running_mean = self
469            .running_mean
470            .try_borrow_mut()
471            .map_err(|_| validation::state_borrow(module, "running_mean"))?;
472        let mut running_var = self
473            .running_var
474            .try_borrow_mut()
475            .map_err(|_| validation::state_borrow(module, "running_var"))?;
476        *running_mean = next_mean;
477        *running_var = next_var;
478
479        Ok(coeus_autograd::batchnorm_nd::<T, B, DIM>(
480            input,
481            &self.weight,
482            &self.bias,
483            coeus_autograd::BatchNormArgs {
484                out_tensor,
485                x_hat,
486                xmu,
487                istdev,
488                m_const,
489                minus_half: self.minus_half.clone(),
490                two_const: self.two_const.clone(),
491                n,
492                c,
493                spatial,
494                m,
495            },
496        )?)
497    }
498}