coeus_nn/normalization/
rmsnorm.rs1use super::validation;
7use crate::module::{Module, ModuleError};
8use coeus_autograd::Var;
9use coeus_core::{Float, MoiraiBackend};
10use coeus_tensor::Tensor;
11
12pub fn rms_norm<
23 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
24 B: coeus_ops::BackendOps<T> + Default,
25>(
26 input: &Var<T, B>,
27 weight: Option<&Var<T, B>>,
28 eps: f64,
29) -> Result<Var<T, B>, ModuleError<B::Error>> {
30 let d = input.tensor.shape().last().copied().unwrap_or(1);
31 let backend = B::default();
32 let w = match weight {
33 Some(weight) => weight.clone(),
34 None => Var::new(Tensor::ones_on([d], &backend)?, false)?,
35 };
36 RMSNorm::from_parts(w, eps)?.forward(input)
37}
38
39#[derive(Clone)]
43pub struct RMSNorm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
44 pub weight: Var<T, B>,
46 pub eps: f64,
48 eps_t: Tensor<T, B>,
50}
51
52impl<T: coeus_core::FloatElement + Float, B: coeus_ops::BackendOps<T> + Default> RMSNorm<T, B> {
53 pub fn new(normalized_shape: usize, eps: f64) -> Result<Self, B::Error> {
55 let backend = B::default();
56 let weight = Var::new(Tensor::ones_on([normalized_shape], &backend)?, true)?;
57 let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
58 Ok(Self { weight, eps, eps_t })
59 }
60
61 pub fn from_parts(weight: Var<T, B>, eps: f64) -> Result<Self, B::Error> {
63 let backend = B::default();
64 let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
65 Ok(Self { weight, eps, eps_t })
66 }
67}
68
69impl<
71 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
72 B: coeus_ops::BackendOps<T> + Default,
73 > Module<T, B> for RMSNorm<T, B>
74{
75 fn parameters(&self) -> Vec<Var<T, B>> {
76 vec![self.weight.clone()]
77 }
78
79 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
80 const MODULE: &str = "RMSNorm";
81 let shape = input.tensor.shape_cloned();
82 if shape.len() != 2 {
83 return Err(validation::invalid_rank(MODULE, "2", shape.len()));
84 }
85 let _n = shape[0];
86 let d = shape[1];
87 if self.weight.tensor.shape() != [d] {
88 return Err(validation::shape_mismatch(
89 MODULE,
90 "weight",
91 &[d],
92 self.weight.tensor.shape(),
93 ));
94 }
95 if !self.eps.is_finite() || self.eps < 0.0 {
96 return Err(ModuleError::InvalidEpsilon { module: MODULE });
97 }
98 let backend = B::default();
99
100 let x_sq = coeus_ops::mul(&input.tensor, &input.tensor, &backend)?;
102 let mut rms = coeus_ops::mean_axis(&x_sq, 1, &backend)
103 .map_err(|source| validation::backend(MODULE, source))?; coeus_ops::add_assign(&mut rms, &self.eps_t, &backend)
107 .map_err(|source| validation::backend(MODULE, source))?;
108 coeus_ops::sqrt_assign(&mut rms, &backend)
109 .map_err(|source| validation::backend(MODULE, source))?;
110
111 let x_hat = coeus_ops::div(&input.tensor, &rms, &backend)?; let w_reshaped = self.weight.tensor.reshape([1, d]);
116 let out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
117
118 Ok(coeus_autograd::rmsnorm(
119 input,
120 &self.weight,
121 out_tensor,
122 x_hat,
123 rms,
124 )?)
125 }
126}