Skip to main content

coeus_nn/normalization/
instancenorm.rs

1//! Instance normalization layers.
2//!
3//! [`InstanceNorm`](crate::normalization::instancenorm::InstanceNorm) is the single const-generic implementation behind the
4//! [`InstanceNorm1d`], [`InstanceNorm2d`] and [`InstanceNorm3d`] aliases: each
5//! sample/channel slice is normalized across its spatial dimensions
6//! independently. This is equivalent to group normalization with one channel
7//! per group.
8
9use super::validation;
10use crate::module::{Module, ModuleError};
11use coeus_autograd::Var;
12use coeus_core::{Float, MoiraiBackend};
13use coeus_tensor::Tensor;
14use std::cell::RefCell;
15
16/// Diagnostic module name for a given spatial rank.
17const fn module_name<const DIM: usize>() -> &'static str {
18    match DIM {
19        1 => "InstanceNorm1d",
20        2 => "InstanceNorm2d",
21        3 => "InstanceNorm3d",
22        _ => "InstanceNormNd",
23    }
24}
25
26/// Expected-rank description for the rank check (`1-D` also accepts `[N, C]`).
27const fn rank_str<const DIM: usize>() -> &'static str {
28    match DIM {
29        1 => "2 or 3",
30        2 => "4",
31        3 => "5",
32        _ => "configured instance-normalization rank",
33    }
34}
35
36// ── Shared cache ──────────────────────────────────────────────────────────────
37
38#[derive(Clone)]
39struct InstanceNormCache<T: Float, B: coeus_ops::BackendOps<T> + Default> {
40    spatial: usize,
41    ln_weight: Var<T, B>,
42    ln_bias: Var<T, B>,
43    eps_t: Tensor<T, B>,
44    d_const: Tensor<T, B>,
45    ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
46}
47
48/// Rebuild `cache` if the spatial size changed.
49fn ensure_cache<T: Float + coeus_core::FloatElement, B: coeus_ops::BackendOps<T> + Default>(
50    cache: &mut Option<InstanceNormCache<T, B>>,
51    spatial: usize,
52    eps: f64,
53) -> Result<(), B::Error> {
54    let needs_rebuild = cache.as_ref().is_none_or(|c| c.spatial != spatial);
55    if needs_rebuild {
56        let backend = B::default();
57        let ln_weight = Var::new(Tensor::ones_on([spatial], &backend)?, false)?;
58        let ln_bias = Var::new(Tensor::zeros_on([spatial], &backend)?, false)?;
59        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
60        let d_const =
61            Tensor::full_on([1], coeus_core::FloatElement::from_count(spatial), &backend)?;
62        *cache = Some(InstanceNormCache {
63            spatial,
64            ln_weight,
65            ln_bias,
66            eps_t,
67            d_const,
68            ones_cache: RefCell::new(None),
69        });
70    }
71    Ok(())
72}
73
74// ── Shared normalization body ─────────────────────────────────────────────────
75//
76// Input is already reshaped to `[N*C, spatial]` by the caller.
77// `weight`/`bias` are the per-channel affine parameters ([C]).
78
79fn instance_norm_forward<
80    T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
81    B: coeus_ops::BackendOps<T> + Default,
82>(
83    flat: &Var<T, B>,
84    weight: &Var<T, B>,
85    bias: &Var<T, B>,
86    cache: &InstanceNormCache<T, B>,
87    n: usize,
88    c: usize,
89    orig_shape: coeus_core::Shape,
90) -> Result<Var<T, B>, ModuleError<B::Error>> {
91    const MODULE: &str = "InstanceNorm";
92    let spatial = cache.spatial;
93    let backend = B::default();
94    let n_channels = n * c;
95
96    let mean_t = coeus_ops::mean_axis(&flat.tensor, 1, &backend)
97        .map_err(|source| validation::backend(MODULE, source))?; // [N*C, 1]
98    let xmu = coeus_ops::sub(&flat.tensor, &mean_t, &backend)?;
99    let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
100    let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
101        .map_err(|source| validation::backend(MODULE, source))?; // population var
102    coeus_ops::add_assign(&mut stdev, &cache.eps_t, &backend)
103        .map_err(|source| validation::backend(MODULE, source))?;
104    coeus_ops::sqrt_assign(&mut stdev, &backend)
105        .map_err(|source| validation::backend(MODULE, source))?;
106
107    let ones = {
108        let mut o_cache = cache
109            .ones_cache
110            .try_borrow_mut()
111            .map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
112        match &*o_cache {
113            Some((cached_n, ref cached_ones)) if *cached_n == n_channels => cached_ones.clone(),
114            _ => {
115                let ones = Tensor::ones_on([n_channels, 1], &backend)?;
116                *o_cache = Some((n_channels, ones.clone()));
117                ones
118            }
119        }
120    };
121    let mut istdev = ones;
122    coeus_ops::div_assign(&mut istdev, &stdev, &backend)
123        .map_err(|source| validation::backend(MODULE, source))?;
124
125    let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
126
127    let w_reshaped = cache.ln_weight.tensor.reshape([1, spatial]);
128    let b_reshaped = cache.ln_bias.tensor.reshape([1, spatial]);
129    let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
130    coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
131        .map_err(|source| validation::backend(MODULE, source))?;
132
133    let normed_flat = coeus_autograd::layernorm(
134        flat,
135        &cache.ln_weight,
136        &cache.ln_bias,
137        out_tensor,
138        x_hat,
139        istdev,
140        cache.d_const.clone(),
141    )?;
142
143    let mut bshape = vec![1usize; orig_shape.len()];
144    bshape[1] = c;
145    let normed = coeus_autograd::reshape(&normed_flat, orig_shape)?;
146    let wv = coeus_autograd::reshape(weight, bshape.as_slice())?;
147    let bv = coeus_autograd::reshape(bias, bshape.as_slice())?;
148    let scaled = coeus_autograd::mul(&normed, &wv)?;
149    Ok(coeus_autograd::add(&scaled, &bv)?)
150}
151
152// ── Generic layer ─────────────────────────────────────────────────────────────
153
154/// Instance Normalization for `DIM`-spatial-rank inputs.
155///
156/// `InstanceNorm<T, B, DIM>` normalizes each sample/channel slice over its
157/// spatial dimensions independently. `DIM = 1` accepts `[N, C]` or `[N, C, L]`,
158/// `DIM = 2` accepts `[N, C, H, W]` and `DIM = 3` accepts `[N, C, D, H, W]`.
159/// Prefer the [`InstanceNorm1d`], [`InstanceNorm2d`] and [`InstanceNorm3d`]
160/// aliases.
161#[derive(Clone)]
162pub struct InstanceNorm<
163    T: Float,
164    B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
165    const DIM: usize = 1,
166> {
167    /// Trainable scale (gamma): shape `[num_features]`.
168    pub weight: Var<T, B>,
169    /// Trainable shift (beta): shape `[num_features]`.
170    pub bias: Var<T, B>,
171    /// Number of channels.
172    pub num_features: usize,
173    /// Numerical stability constant added to variance.
174    pub eps: f64,
175    cache: RefCell<Option<InstanceNormCache<T, B>>>,
176}
177
178impl<
179        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
180        B: coeus_ops::BackendOps<T> + Default,
181        const DIM: usize,
182    > InstanceNorm<T, B, DIM>
183{
184    /// Create an `InstanceNorm` layer for `DIM` spatial dimensions.
185    pub fn new(
186        num_features: usize,
187        eps: f64,
188    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
189        let backend = B::default();
190        Ok(Self {
191            weight: Var::new(Tensor::ones_on([num_features], &backend)?, true)?,
192            bias: Var::new(Tensor::zeros_on([num_features], &backend)?, true)?,
193            num_features,
194            eps,
195            cache: RefCell::new(None),
196        })
197    }
198}
199
200/// Implements the [`Module`] interface for every [`InstanceNorm`](crate::normalization::instancenorm::InstanceNorm) rank.
201impl<
202        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
203        B: coeus_ops::BackendOps<T> + Default,
204        const DIM: usize,
205    > Module<T, B> for InstanceNorm<T, B, DIM>
206{
207    fn parameters(&self) -> Vec<Var<T, B>> {
208        vec![self.weight.clone(), self.bias.clone()]
209    }
210
211    /// Forward: input `[N, C, ...spatial]` (and `[N, C]` for `DIM = 1`).
212    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
213        let module: &str = module_name::<DIM>();
214        let shape = input.tensor.shape_cloned();
215        let rank_ok = shape.len() == DIM + 2 || (DIM == 1 && shape.len() == 2);
216        if !rank_ok {
217            return Err(validation::invalid_rank(
218                module,
219                rank_str::<DIM>(),
220                shape.len(),
221            ));
222        }
223        let n = shape[0];
224        let c = shape[1];
225        if c != self.num_features {
226            return Err(validation::channel_mismatch(module, self.num_features, c));
227        }
228        for (parameter, actual) in [
229            ("weight", self.weight.tensor.shape()),
230            ("bias", self.bias.tensor.shape()),
231        ] {
232            if actual != [c] {
233                return Err(validation::shape_mismatch(module, parameter, &[c], actual));
234            }
235        }
236        if !self.eps.is_finite() || self.eps < 0.0 {
237            return Err(ModuleError::InvalidEpsilon { module });
238        }
239        // Spatial extent is the product of the trailing axes; the empty product
240        // for the `[N, C]` form is the degenerate spatial size 1.
241        let spatial: usize = shape[2..].iter().product();
242        let flat = coeus_autograd::reshape(input, [n * c, spatial])?;
243
244        let mut cache = self
245            .cache
246            .try_borrow_mut()
247            .map_err(|_| validation::state_borrow(module, "cache"))?;
248        ensure_cache::<T, B>(&mut *cache, spatial, self.eps)?;
249        let Some(cache) = cache.as_ref() else {
250            unreachable!("invariant: ensure_cache initializes the InstanceNorm cache")
251        };
252
253        instance_norm_forward(&flat, &self.weight, &self.bias, cache, n, c, shape)
254    }
255}
256
257// ── Fixed-rank aliases ────────────────────────────────────────────────────────
258
259/// Instance Normalization for 1D inputs `[N, C, L]` or `[N, C]`.
260///
261/// Normalizes over the `L` (spatial) dimension independently per sample per channel.
262///
263/// # Examples
264///
265/// ```
266/// use coeus_nn::{InstanceNorm1d, Module};
267/// use coeus_autograd::Var;
268/// use coeus_tensor::Tensor;
269/// use coeus_core::SequentialBackend;
270///
271/// let in1 = InstanceNorm1d::<f32, SequentialBackend>::new(4, 1e-5)
272///     .expect("invariant: example channel count and epsilon are valid");
273/// let x = Var::new(
274///     Tensor::ones_on([2, 4, 8], &SequentialBackend::new())
275///         .expect("invariant: example shape allocation succeeds"),
276///     false,
277/// ).expect("invariant: constant example variable needs no gradient buffer");
278/// let y = in1.forward(&x).expect("valid InstanceNorm1d input");
279/// assert_eq!(y.tensor.shape(), &[2, 4, 8]);
280/// ```
281pub type InstanceNorm1d<T, B = MoiraiBackend> = InstanceNorm<T, B, 1>;
282
283/// Instance Normalization for 2D inputs `[N, C, H, W]`.
284///
285/// Normalizes over the `H × W` spatial dimensions independently per sample per channel.
286///
287/// # Examples
288///
289/// ```
290/// use coeus_nn::{InstanceNorm2d, Module};
291/// use coeus_autograd::Var;
292/// use coeus_tensor::Tensor;
293/// use coeus_core::SequentialBackend;
294///
295/// let in2 = InstanceNorm2d::<f32, SequentialBackend>::new(4, 1e-5)
296///     .expect("invariant: example channel count and epsilon are valid");
297/// let x = Var::new(
298///     Tensor::ones_on([2, 4, 8, 8], &SequentialBackend::new())
299///         .expect("invariant: example shape allocation succeeds"),
300///     false,
301/// ).expect("invariant: constant example variable needs no gradient buffer");
302/// let y = in2.forward(&x).expect("valid InstanceNorm2d input");
303/// assert_eq!(y.tensor.shape(), &[2, 4, 8, 8]);
304/// ```
305pub type InstanceNorm2d<T, B = MoiraiBackend> = InstanceNorm<T, B, 2>;
306
307/// Instance Normalization for 3D inputs `[N, C, D, H, W]`.
308///
309/// Normalizes over the `D × H × W` spatial volume independently per sample per channel.
310///
311/// # Examples
312///
313/// ```
314/// use coeus_nn::{InstanceNorm3d, Module};
315/// use coeus_autograd::Var;
316/// use coeus_tensor::Tensor;
317/// use coeus_core::SequentialBackend;
318///
319/// let in3 = InstanceNorm3d::<f32, SequentialBackend>::new(4, 1e-5)
320///     .expect("invariant: example channel count and epsilon are valid");
321/// let x = Var::new(
322///     Tensor::ones_on([1, 4, 4, 4, 4], &SequentialBackend::new())
323///         .expect("invariant: example shape allocation succeeds"),
324///     false,
325/// ).expect("invariant: constant example variable needs no gradient buffer");
326/// let y = in3.forward(&x).expect("valid InstanceNorm3d input");
327/// assert_eq!(y.tensor.shape(), &[1, 4, 4, 4, 4]);
328/// ```
329pub type InstanceNorm3d<T, B = MoiraiBackend> = InstanceNorm<T, B, 3>;