Skip to main content

coeus_nn/normalization/
rmsnorm.rs

1//! RMS normalization layers and helpers.
2//!
3//! [`RMSNorm`] rescales activations by their root-mean-square value over the
4//! last dimension without subtracting the mean.
5
6use super::validation;
7use crate::module::{Module, ModuleError};
8use coeus_autograd::Var;
9use coeus_core::{Float, MoiraiBackend};
10use coeus_tensor::Tensor;
11
12/// Functional (stateless) RMS normalization.
13///
14/// Applies RMSNorm over the last dimension of a rank-2 input `[N, D]`.
15/// `weight` defaults to ones of shape `[D]`.
16///
17/// # Errors
18///
19/// Returns a typed module or backend failure when the input is not rank two,
20/// the weight shape differs from the trailing dimension, epsilon is invalid,
21/// or a backend operation fails.
22pub fn rms_norm<
23    T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
24    B: coeus_ops::BackendOps<T> + Default,
25>(
26    input: &Var<T, B>,
27    weight: Option<&Var<T, B>>,
28    eps: f64,
29) -> Result<Var<T, B>, ModuleError<B::Error>> {
30    let d = input.tensor.shape().last().copied().unwrap_or(1);
31    let backend = B::default();
32    let w = match weight {
33        Some(weight) => weight.clone(),
34        None => Var::new(Tensor::ones_on([d], &backend)?, false)?,
35    };
36    RMSNorm::from_parts(w, eps)?.forward(input)
37}
38
39/// Root Mean Square Normalization (RMSNorm) module.
40///
41/// Applies RMSNorm over the last dimension of a 2D tensor `[N, D]`.
42#[derive(Clone)]
43pub struct RMSNorm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
44    /// Trainable scale parameter gamma: `[D]`.
45    pub weight: Var<T, B>,
46    /// Small value for numerical stability.
47    pub eps: f64,
48    /// Cached epsilon tensor: `[1]`.
49    eps_t: Tensor<T, B>,
50}
51
52impl<T: coeus_core::FloatElement + Float, B: coeus_ops::BackendOps<T> + Default> RMSNorm<T, B> {
53    /// Create a new RMSNorm layer for a given feature dimension.
54    pub fn new(normalized_shape: usize, eps: f64) -> Result<Self, B::Error> {
55        let backend = B::default();
56        let weight = Var::new(Tensor::ones_on([normalized_shape], &backend)?, true)?;
57        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
58        Ok(Self { weight, eps, eps_t })
59    }
60
61    /// Create an RMSNorm layer from existing parameters.
62    pub fn from_parts(weight: Var<T, B>, eps: f64) -> Result<Self, B::Error> {
63        let backend = B::default();
64        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
65        Ok(Self { weight, eps, eps_t })
66    }
67}
68
69/// Implements the [`crate::module::Module`] interface for [`RMSNorm`].
70impl<
71        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
72        B: coeus_ops::BackendOps<T> + Default,
73    > Module<T, B> for RMSNorm<T, B>
74{
75    fn parameters(&self) -> Vec<Var<T, B>> {
76        vec![self.weight.clone()]
77    }
78
79    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
80        const MODULE: &str = "RMSNorm";
81        let shape = input.tensor.shape_cloned();
82        if shape.len() != 2 {
83            return Err(validation::invalid_rank(MODULE, "2", shape.len()));
84        }
85        let _n = shape[0];
86        let d = shape[1];
87        if self.weight.tensor.shape() != [d] {
88            return Err(validation::shape_mismatch(
89                MODULE,
90                "weight",
91                &[d],
92                self.weight.tensor.shape(),
93            ));
94        }
95        if !self.eps.is_finite() || self.eps < 0.0 {
96            return Err(ModuleError::InvalidEpsilon { module: MODULE });
97        }
98        let backend = B::default();
99
100        // ── Mean square ──
101        let x_sq = coeus_ops::mul(&input.tensor, &input.tensor, &backend)?;
102        let mut rms = coeus_ops::mean_axis(&x_sq, 1, &backend)
103            .map_err(|source| validation::backend(MODULE, source))?; // [N, 1]
104
105        // ── RMS ──
106        coeus_ops::add_assign(&mut rms, &self.eps_t, &backend)
107            .map_err(|source| validation::backend(MODULE, source))?;
108        coeus_ops::sqrt_assign(&mut rms, &backend)
109            .map_err(|source| validation::backend(MODULE, source))?;
110
111        // ── Normalize ──
112        let x_hat = coeus_ops::div(&input.tensor, &rms, &backend)?; // [N, D]
113
114        // ── Scale ──
115        let w_reshaped = self.weight.tensor.reshape([1, d]);
116        let out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
117
118        Ok(coeus_autograd::rmsnorm(
119            input,
120            &self.weight,
121            out_tensor,
122            x_hat,
123            rms,
124        )?)
125    }
126}