Skip to main content

coeus_nn/normalization/
layernorm.rs

1//! Layer normalization over a configured trailing shape.
2
3use super::validation;
4use crate::module::{Module, ModuleError};
5use coeus_autograd::Var;
6use coeus_core::{Float, MoiraiBackend};
7use coeus_tensor::Tensor;
8use std::cell::RefCell;
9
10/// Shape of the trailing dimensions normalized by [`LayerNorm`].
11#[derive(Clone, Debug, PartialEq, Eq)]
12pub struct NormalizedShape(Vec<usize>);
13
14impl NormalizedShape {
15    /// Borrow the configured trailing dimensions.
16    #[must_use]
17    pub fn as_slice(&self) -> &[usize] {
18        &self.0
19    }
20
21    fn is_valid(&self) -> bool {
22        !self.0.is_empty() && self.0.iter().all(|&dimension| dimension > 0)
23    }
24}
25
26impl From<usize> for NormalizedShape {
27    fn from(dimension: usize) -> Self {
28        Self(vec![dimension])
29    }
30}
31
32impl From<Vec<usize>> for NormalizedShape {
33    fn from(dimensions: Vec<usize>) -> Self {
34        Self(dimensions)
35    }
36}
37
38impl From<&[usize]> for NormalizedShape {
39    fn from(dimensions: &[usize]) -> Self {
40        Self(dimensions.to_vec())
41    }
42}
43
44impl<const N: usize> From<[usize; N]> for NormalizedShape {
45    fn from(dimensions: [usize; N]) -> Self {
46        Self(dimensions.into())
47    }
48}
49
50/// Functional (stateless) layer normalization.
51///
52/// Normalizes the configured trailing dimensions of every input with rank at
53/// least two. `weight` and `bias` default to ones and zeros with the requested
54/// `normalized_shape`.
55///
56/// # Errors
57///
58/// Returns a typed module or backend failure when the input rank, trailing
59/// dimensions, affine parameter shapes, or epsilon violate the LayerNorm
60/// contract, or when a backend operation fails.
61pub fn layer_norm<
62    T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
63    B: coeus_ops::BackendOps<T> + Default,
64>(
65    input: &Var<T, B>,
66    normalized_shape: impl Into<NormalizedShape>,
67    weight: Option<&Var<T, B>>,
68    bias: Option<&Var<T, B>>,
69    eps: f64,
70) -> Result<Var<T, B>, ModuleError<B::Error>>
71where
72    B::DeviceBuffer<T>:
73        coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
74{
75    let normalized_shape = normalized_shape.into();
76    if !normalized_shape.is_valid() {
77        return Err(validation::shape_mismatch(
78            "LayerNorm",
79            "normalized_shape",
80            &[1],
81            normalized_shape.as_slice(),
82        ));
83    }
84    let backend = B::default();
85    let w = match weight {
86        Some(weight) => weight.clone(),
87        None => Var::new(
88            Tensor::ones_on(normalized_shape.as_slice(), &backend)?,
89            false,
90        )?,
91    };
92    let b = match bias {
93        Some(bias) => bias.clone(),
94        None => Var::new(
95            Tensor::zeros_on(normalized_shape.as_slice(), &backend)?,
96            false,
97        )?,
98    };
99    for (parameter, actual) in [("weight", w.tensor.shape()), ("bias", b.tensor.shape())] {
100        if actual != normalized_shape.as_slice() {
101            return Err(validation::shape_mismatch(
102                "LayerNorm",
103                parameter,
104                normalized_shape.as_slice(),
105                actual,
106            ));
107        }
108    }
109    LayerNorm::from_parts(w, b, eps)?.forward(input)
110}
111
112/// Layer Normalization module.
113///
114/// Applies Layer Normalization over a configured trailing shape of tensors
115/// with rank two or greater.
116#[derive(Clone)]
117pub struct LayerNorm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
118    /// Trainable scale parameter gamma: `normalized_shape`.
119    pub weight: Var<T, B>,
120    /// Trainable shift parameter beta: `normalized_shape`.
121    pub bias: Var<T, B>,
122    /// Small value for numerical stability.
123    pub eps: f64,
124    /// Cached epsilon tensor: `[1]`.
125    eps_t: Tensor<T, B>,
126    /// Cached normalized-element count: `[1]`.
127    d_const: Tensor<T, B>,
128    /// Cached ones tensor of shape `[N, 1]`: (N, ones_tensor).
129    ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
130}
131
132impl<T: coeus_core::FloatElement + Float, B: coeus_ops::BackendOps<T> + Default> LayerNorm<T, B> {
133    /// Create a new LayerNorm layer for a single feature dimension.
134    pub fn new(
135        normalized_shape: usize,
136        eps: f64,
137    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
138        Ok(Self::from_shape(normalized_shape, eps)?)
139    }
140
141    /// Create a LayerNorm layer for one or more trailing dimensions.
142    pub fn from_shape(
143        normalized_shape: impl Into<NormalizedShape>,
144        eps: f64,
145    ) -> Result<Self, B::Error> {
146        let normalized_shape = normalized_shape.into();
147        let backend = B::default();
148        let weight = Var::new(
149            Tensor::ones_on(normalized_shape.as_slice(), &backend)?,
150            true,
151        )?;
152        let bias = Var::new(
153            Tensor::zeros_on(normalized_shape.as_slice(), &backend)?,
154            true,
155        )?;
156        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
157        let d_const = Tensor::full_on(
158            [1],
159            coeus_core::FloatElement::from_count(weight.tensor.numel()),
160            &backend,
161        )?;
162        Ok(Self {
163            weight,
164            bias,
165            eps,
166            eps_t,
167            d_const,
168            ones_cache: RefCell::new(None),
169        })
170    }
171
172    /// Create a LayerNorm layer from existing affine parameters.
173    pub fn from_parts(weight: Var<T, B>, bias: Var<T, B>, eps: f64) -> Result<Self, B::Error> {
174        let backend = B::default();
175        let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
176        let d_const = Tensor::full_on(
177            [1],
178            coeus_core::FloatElement::from_count(weight.tensor.numel()),
179            &backend,
180        )?;
181        Ok(Self {
182            weight,
183            bias,
184            eps,
185            eps_t,
186            d_const,
187            ones_cache: RefCell::new(None),
188        })
189    }
190
191    fn normalize_flat(
192        &self,
193        input: &Var<T, B>,
194        normalized_size: usize,
195    ) -> Result<Var<T, B>, ModuleError<B::Error>>
196    where
197        T: coeus_cpu::RealScalar,
198    {
199        const MODULE: &str = "LayerNorm";
200        let batch = input.tensor.shape()[0];
201        let backend = B::default();
202
203        let mean_t = coeus_ops::mean_axis(&input.tensor, 1, &backend)
204            .map_err(|source| validation::backend(MODULE, source))?;
205        let xmu = coeus_ops::sub(&input.tensor, &mean_t, &backend)?;
206        let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
207        let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
208            .map_err(|source| validation::backend(MODULE, source))?;
209
210        coeus_ops::add_assign(&mut stdev, &self.eps_t, &backend)
211            .map_err(|source| validation::backend(MODULE, source))?;
212        coeus_ops::sqrt_assign(&mut stdev, &backend)
213            .map_err(|source| validation::backend(MODULE, source))?;
214
215        let ones = {
216            let mut cache = self
217                .ones_cache
218                .try_borrow_mut()
219                .map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
220            if let Some((cached_batch, ref cached_ones)) = *cache {
221                if cached_batch == batch {
222                    cached_ones.clone()
223                } else {
224                    let ones = Tensor::ones_on([batch, 1], &backend)?;
225                    *cache = Some((batch, ones.clone()));
226                    ones
227                }
228            } else {
229                let ones = Tensor::ones_on([batch, 1], &backend)?;
230                *cache = Some((batch, ones.clone()));
231                ones
232            }
233        };
234        let mut istdev = ones;
235        coeus_ops::div_assign(&mut istdev, &stdev, &backend)
236            .map_err(|source| validation::backend(MODULE, source))?;
237
238        let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
239        let w_reshaped = self.weight.tensor.reshape([1, normalized_size]);
240        let b_reshaped = self.bias.tensor.reshape([1, normalized_size]);
241        let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
242        coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
243            .map_err(|source| validation::backend(MODULE, source))?;
244
245        Ok(coeus_autograd::layernorm(
246            input,
247            &self.weight,
248            &self.bias,
249            out_tensor,
250            x_hat,
251            istdev,
252            self.d_const.clone(),
253        )?)
254    }
255}
256
257/// Implements the [`crate::module::Module`] interface for [`LayerNorm`].
258impl<
259        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
260        B: coeus_ops::BackendOps<T> + Default,
261    > Module<T, B> for LayerNorm<T, B>
262{
263    fn parameters(&self) -> Vec<Var<T, B>> {
264        vec![self.weight.clone(), self.bias.clone()]
265    }
266
267    /// Write updated `(weight, bias)` values back to this module.
268    fn load_parameters(&mut self, params: &[Var<T, B>]) {
269        self.weight = params[0].clone();
270        self.bias = params[1].clone();
271    }
272
273    /// Forward pass over the configured trailing dimensions.
274    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
275        self.forward_nd(input)
276    }
277}
278
279impl<
280        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
281        B: coeus_ops::BackendOps<T> + Default,
282    > LayerNorm<T, B>
283{
284    /// Forward pass for any rank ≥ 2 input.
285    ///
286    /// The configured suffix is flattened into one normalized feature axis for
287    /// the provider kernel, then the original input shape is restored through
288    /// tracked reshape operations.
289    ///
290    /// # Errors
291    ///
292    /// Returns a typed module or backend failure when the input rank, trailing
293    /// dimensions, affine parameter shapes, or epsilon violate the contract,
294    /// or when normalization execution fails.
295    pub fn forward_nd(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
296        const MODULE: &str = "LayerNorm";
297        let input_shape = input.tensor.shape_cloned();
298        if input_shape.len() < 2 {
299            return Err(validation::invalid_rank(
300                MODULE,
301                "at least 2",
302                input_shape.len(),
303            ));
304        }
305
306        let normalized_shape = self.weight.tensor.shape_cloned();
307        if normalized_shape.is_empty() || normalized_shape.contains(&0) {
308            return Err(validation::shape_mismatch(
309                MODULE,
310                "normalized_shape",
311                &[1],
312                &normalized_shape,
313            ));
314        }
315        for (parameter, actual) in [
316            ("weight", self.weight.tensor.shape()),
317            ("bias", self.bias.tensor.shape()),
318        ] {
319            if actual != &*normalized_shape {
320                return Err(validation::shape_mismatch(
321                    MODULE,
322                    parameter,
323                    &normalized_shape,
324                    actual,
325                ));
326            }
327        }
328        if !self.eps.is_finite() || self.eps < 0.0 {
329            return Err(ModuleError::InvalidEpsilon { module: MODULE });
330        }
331
332        let Some(suffix_start) = input_shape.len().checked_sub(normalized_shape.len()) else {
333            return Err(validation::shape_mismatch(
334                MODULE,
335                "input trailing dimensions",
336                &normalized_shape,
337                &input_shape,
338            ));
339        };
340        let input_suffix = &input_shape[suffix_start..];
341        if input_suffix != &*normalized_shape {
342            return Err(validation::shape_mismatch(
343                MODULE,
344                "input trailing dimensions",
345                &normalized_shape,
346                input_suffix,
347            ));
348        }
349
350        let normalized_size = self.weight.tensor.numel();
351        let Some(batch) = input.tensor.numel().checked_div(normalized_size) else {
352            return Err(validation::shape_mismatch(
353                MODULE,
354                "input",
355                &normalized_shape,
356                &input_shape,
357            ));
358        };
359        if input_shape.len() == 2 && normalized_shape.len() == 1 {
360            return self.normalize_flat(input, normalized_size);
361        }
362
363        let flat = coeus_autograd::reshape(input, [batch, normalized_size])?;
364        let normalized = self.normalize_flat(&flat, normalized_size)?;
365        Ok(coeus_autograd::reshape(&normalized, input_shape)?)
366    }
367}