coeus-nn 0.14.0

Neural-network layers and operators for Coeus.
Documentation
//! Instance normalization layers.
//!
//! [`InstanceNorm`](crate::normalization::instancenorm::InstanceNorm) is the single const-generic implementation behind the
//! [`InstanceNorm1d`], [`InstanceNorm2d`] and [`InstanceNorm3d`] aliases: each
//! sample/channel slice is normalized across its spatial dimensions
//! independently. This is equivalent to group normalization with one channel
//! per group.

use super::validation;
use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::{Float, MoiraiBackend};
use coeus_tensor::Tensor;
use std::cell::RefCell;

/// Diagnostic module name for a given spatial rank.
const fn module_name<const DIM: usize>() -> &'static str {
    match DIM {
        1 => "InstanceNorm1d",
        2 => "InstanceNorm2d",
        3 => "InstanceNorm3d",
        _ => "InstanceNormNd",
    }
}

/// Expected-rank description for the rank check (`1-D` also accepts `[N, C]`).
const fn rank_str<const DIM: usize>() -> &'static str {
    match DIM {
        1 => "2 or 3",
        2 => "4",
        3 => "5",
        _ => "configured instance-normalization rank",
    }
}

// ── Shared cache ──────────────────────────────────────────────────────────────

#[derive(Clone)]
struct InstanceNormCache<T: Float, B: coeus_ops::BackendOps<T> + Default> {
    spatial: usize,
    ln_weight: Var<T, B>,
    ln_bias: Var<T, B>,
    eps_t: Tensor<T, B>,
    d_const: Tensor<T, B>,
    ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
}

/// Rebuild `cache` if the spatial size changed.
fn ensure_cache<T: Float + coeus_core::FloatElement, B: coeus_ops::BackendOps<T> + Default>(
    cache: &mut Option<InstanceNormCache<T, B>>,
    spatial: usize,
    eps: f64,
) -> Result<(), B::Error> {
    let needs_rebuild = cache.as_ref().is_none_or(|c| c.spatial != spatial);
    if needs_rebuild {
        let backend = B::default();
        let ln_weight = Var::new(Tensor::ones_on([spatial], &backend)?, false)?;
        let ln_bias = Var::new(Tensor::zeros_on([spatial], &backend)?, false)?;
        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
        let d_const =
            Tensor::full_on([1], coeus_core::FloatElement::from_count(spatial), &backend)?;
        *cache = Some(InstanceNormCache {
            spatial,
            ln_weight,
            ln_bias,
            eps_t,
            d_const,
            ones_cache: RefCell::new(None),
        });
    }
    Ok(())
}

// ── Shared normalization body ─────────────────────────────────────────────────
//
// Input is already reshaped to `[N*C, spatial]` by the caller.
// `weight`/`bias` are the per-channel affine parameters ([C]).

fn instance_norm_forward<
    T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
    B: coeus_ops::BackendOps<T> + Default,
>(
    flat: &Var<T, B>,
    weight: &Var<T, B>,
    bias: &Var<T, B>,
    cache: &InstanceNormCache<T, B>,
    n: usize,
    c: usize,
    orig_shape: coeus_core::Shape,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
    const MODULE: &str = "InstanceNorm";
    let spatial = cache.spatial;
    let backend = B::default();
    let n_channels = n * c;

    let mean_t = coeus_ops::mean_axis(&flat.tensor, 1, &backend)
        .map_err(|source| validation::backend(MODULE, source))?; // [N*C, 1]
    let xmu = coeus_ops::sub(&flat.tensor, &mean_t, &backend)?;
    let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
    let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
        .map_err(|source| validation::backend(MODULE, source))?; // population var
    coeus_ops::add_assign(&mut stdev, &cache.eps_t, &backend)
        .map_err(|source| validation::backend(MODULE, source))?;
    coeus_ops::sqrt_assign(&mut stdev, &backend)
        .map_err(|source| validation::backend(MODULE, source))?;

    let ones = {
        let mut o_cache = cache
            .ones_cache
            .try_borrow_mut()
            .map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
        match &*o_cache {
            Some((cached_n, ref cached_ones)) if *cached_n == n_channels => cached_ones.clone(),
            _ => {
                let ones = Tensor::ones_on([n_channels, 1], &backend)?;
                *o_cache = Some((n_channels, ones.clone()));
                ones
            }
        }
    };
    let mut istdev = ones;
    coeus_ops::div_assign(&mut istdev, &stdev, &backend)
        .map_err(|source| validation::backend(MODULE, source))?;

    let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;

    let w_reshaped = cache.ln_weight.tensor.reshape([1, spatial]);
    let b_reshaped = cache.ln_bias.tensor.reshape([1, spatial]);
    let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
    coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
        .map_err(|source| validation::backend(MODULE, source))?;

    let normed_flat = coeus_autograd::layernorm(
        flat,
        &cache.ln_weight,
        &cache.ln_bias,
        out_tensor,
        x_hat,
        istdev,
        cache.d_const.clone(),
    )?;

    let mut bshape = vec![1usize; orig_shape.len()];
    bshape[1] = c;
    let normed = coeus_autograd::reshape(&normed_flat, orig_shape)?;
    let wv = coeus_autograd::reshape(weight, bshape.as_slice())?;
    let bv = coeus_autograd::reshape(bias, bshape.as_slice())?;
    let scaled = coeus_autograd::mul(&normed, &wv)?;
    Ok(coeus_autograd::add(&scaled, &bv)?)
}

// ── Generic layer ─────────────────────────────────────────────────────────────

/// Instance Normalization for `DIM`-spatial-rank inputs.
///
/// `InstanceNorm<T, B, DIM>` normalizes each sample/channel slice over its
/// spatial dimensions independently. `DIM = 1` accepts `[N, C]` or `[N, C, L]`,
/// `DIM = 2` accepts `[N, C, H, W]` and `DIM = 3` accepts `[N, C, D, H, W]`.
/// Prefer the [`InstanceNorm1d`], [`InstanceNorm2d`] and [`InstanceNorm3d`]
/// aliases.
#[derive(Clone)]
pub struct InstanceNorm<
    T: Float,
    B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
    const DIM: usize = 1,
> {
    /// Trainable scale (gamma): shape `[num_features]`.
    pub weight: Var<T, B>,
    /// Trainable shift (beta): shape `[num_features]`.
    pub bias: Var<T, B>,
    /// Number of channels.
    pub num_features: usize,
    /// Numerical stability constant added to variance.
    pub eps: f64,
    cache: RefCell<Option<InstanceNormCache<T, B>>>,
}

impl<
        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
        B: coeus_ops::BackendOps<T> + Default,
        const DIM: usize,
    > InstanceNorm<T, B, DIM>
{
    /// Create an `InstanceNorm` layer for `DIM` spatial dimensions.
    pub fn new(
        num_features: usize,
        eps: f64,
    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
        let backend = B::default();
        Ok(Self {
            weight: Var::new(Tensor::ones_on([num_features], &backend)?, true)?,
            bias: Var::new(Tensor::zeros_on([num_features], &backend)?, true)?,
            num_features,
            eps,
            cache: RefCell::new(None),
        })
    }
}

/// Implements the [`Module`] interface for every [`InstanceNorm`](crate::normalization::instancenorm::InstanceNorm) rank.
impl<
        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
        B: coeus_ops::BackendOps<T> + Default,
        const DIM: usize,
    > Module<T, B> for InstanceNorm<T, B, DIM>
{
    fn parameters(&self) -> Vec<Var<T, B>> {
        vec![self.weight.clone(), self.bias.clone()]
    }

    /// Forward: input `[N, C, ...spatial]` (and `[N, C]` for `DIM = 1`).
    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
        let module: &str = module_name::<DIM>();
        let shape = input.tensor.shape_cloned();
        let rank_ok = shape.len() == DIM + 2 || (DIM == 1 && shape.len() == 2);
        if !rank_ok {
            return Err(validation::invalid_rank(
                module,
                rank_str::<DIM>(),
                shape.len(),
            ));
        }
        let n = shape[0];
        let c = shape[1];
        if c != self.num_features {
            return Err(validation::channel_mismatch(module, self.num_features, c));
        }
        for (parameter, actual) in [
            ("weight", self.weight.tensor.shape()),
            ("bias", self.bias.tensor.shape()),
        ] {
            if actual != [c] {
                return Err(validation::shape_mismatch(module, parameter, &[c], actual));
            }
        }
        if !self.eps.is_finite() || self.eps < 0.0 {
            return Err(ModuleError::InvalidEpsilon { module });
        }
        // Spatial extent is the product of the trailing axes; the empty product
        // for the `[N, C]` form is the degenerate spatial size 1.
        let spatial: usize = shape[2..].iter().product();
        let flat = coeus_autograd::reshape(input, [n * c, spatial])?;

        let mut cache = self
            .cache
            .try_borrow_mut()
            .map_err(|_| validation::state_borrow(module, "cache"))?;
        ensure_cache::<T, B>(&mut *cache, spatial, self.eps)?;
        let Some(cache) = cache.as_ref() else {
            unreachable!("invariant: ensure_cache initializes the InstanceNorm cache")
        };

        instance_norm_forward(&flat, &self.weight, &self.bias, cache, n, c, shape)
    }
}

// ── Fixed-rank aliases ────────────────────────────────────────────────────────

/// Instance Normalization for 1D inputs `[N, C, L]` or `[N, C]`.
///
/// Normalizes over the `L` (spatial) dimension independently per sample per channel.
///
/// # Examples
///
/// ```
/// use coeus_nn::{InstanceNorm1d, Module};
/// use coeus_autograd::Var;
/// use coeus_tensor::Tensor;
/// use coeus_core::SequentialBackend;
///
/// let in1 = InstanceNorm1d::<f32, SequentialBackend>::new(4, 1e-5)
///     .expect("invariant: example channel count and epsilon are valid");
/// let x = Var::new(
///     Tensor::ones_on([2, 4, 8], &SequentialBackend::new())
///         .expect("invariant: example shape allocation succeeds"),
///     false,
/// ).expect("invariant: constant example variable needs no gradient buffer");
/// let y = in1.forward(&x).expect("valid InstanceNorm1d input");
/// assert_eq!(y.tensor.shape(), &[2, 4, 8]);
/// ```
pub type InstanceNorm1d<T, B = MoiraiBackend> = InstanceNorm<T, B, 1>;

/// Instance Normalization for 2D inputs `[N, C, H, W]`.
///
/// Normalizes over the `H × W` spatial dimensions independently per sample per channel.
///
/// # Examples
///
/// ```
/// use coeus_nn::{InstanceNorm2d, Module};
/// use coeus_autograd::Var;
/// use coeus_tensor::Tensor;
/// use coeus_core::SequentialBackend;
///
/// let in2 = InstanceNorm2d::<f32, SequentialBackend>::new(4, 1e-5)
///     .expect("invariant: example channel count and epsilon are valid");
/// let x = Var::new(
///     Tensor::ones_on([2, 4, 8, 8], &SequentialBackend::new())
///         .expect("invariant: example shape allocation succeeds"),
///     false,
/// ).expect("invariant: constant example variable needs no gradient buffer");
/// let y = in2.forward(&x).expect("valid InstanceNorm2d input");
/// assert_eq!(y.tensor.shape(), &[2, 4, 8, 8]);
/// ```
pub type InstanceNorm2d<T, B = MoiraiBackend> = InstanceNorm<T, B, 2>;

/// Instance Normalization for 3D inputs `[N, C, D, H, W]`.
///
/// Normalizes over the `D × H × W` spatial volume independently per sample per channel.
///
/// # Examples
///
/// ```
/// use coeus_nn::{InstanceNorm3d, Module};
/// use coeus_autograd::Var;
/// use coeus_tensor::Tensor;
/// use coeus_core::SequentialBackend;
///
/// let in3 = InstanceNorm3d::<f32, SequentialBackend>::new(4, 1e-5)
///     .expect("invariant: example channel count and epsilon are valid");
/// let x = Var::new(
///     Tensor::ones_on([1, 4, 4, 4, 4], &SequentialBackend::new())
///         .expect("invariant: example shape allocation succeeds"),
///     false,
/// ).expect("invariant: constant example variable needs no gradient buffer");
/// let y = in3.forward(&x).expect("valid InstanceNorm3d input");
/// assert_eq!(y.tensor.shape(), &[1, 4, 4, 4, 4]);
/// ```
pub type InstanceNorm3d<T, B = MoiraiBackend> = InstanceNorm<T, B, 3>;