1use super::validation;
4use crate::module::{Module, ModuleError};
5use coeus_autograd::Var;
6use coeus_core::{Float, MoiraiBackend};
7use coeus_tensor::Tensor;
8use std::cell::RefCell;
9
10#[derive(Clone, Debug, PartialEq, Eq)]
12pub struct NormalizedShape(Vec<usize>);
13
14impl NormalizedShape {
15 #[must_use]
17 pub fn as_slice(&self) -> &[usize] {
18 &self.0
19 }
20
21 fn is_valid(&self) -> bool {
22 !self.0.is_empty() && self.0.iter().all(|&dimension| dimension > 0)
23 }
24}
25
26impl From<usize> for NormalizedShape {
27 fn from(dimension: usize) -> Self {
28 Self(vec![dimension])
29 }
30}
31
32impl From<Vec<usize>> for NormalizedShape {
33 fn from(dimensions: Vec<usize>) -> Self {
34 Self(dimensions)
35 }
36}
37
38impl From<&[usize]> for NormalizedShape {
39 fn from(dimensions: &[usize]) -> Self {
40 Self(dimensions.to_vec())
41 }
42}
43
44impl<const N: usize> From<[usize; N]> for NormalizedShape {
45 fn from(dimensions: [usize; N]) -> Self {
46 Self(dimensions.into())
47 }
48}
49
50pub fn layer_norm<
62 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
63 B: coeus_ops::BackendOps<T> + Default,
64>(
65 input: &Var<T, B>,
66 normalized_shape: impl Into<NormalizedShape>,
67 weight: Option<&Var<T, B>>,
68 bias: Option<&Var<T, B>>,
69 eps: f64,
70) -> Result<Var<T, B>, ModuleError<B::Error>>
71where
72 B::DeviceBuffer<T>:
73 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
74{
75 let normalized_shape = normalized_shape.into();
76 if !normalized_shape.is_valid() {
77 return Err(validation::shape_mismatch(
78 "LayerNorm",
79 "normalized_shape",
80 &[1],
81 normalized_shape.as_slice(),
82 ));
83 }
84 let backend = B::default();
85 let w = match weight {
86 Some(weight) => weight.clone(),
87 None => Var::new(
88 Tensor::ones_on(normalized_shape.as_slice(), &backend)?,
89 false,
90 )?,
91 };
92 let b = match bias {
93 Some(bias) => bias.clone(),
94 None => Var::new(
95 Tensor::zeros_on(normalized_shape.as_slice(), &backend)?,
96 false,
97 )?,
98 };
99 for (parameter, actual) in [("weight", w.tensor.shape()), ("bias", b.tensor.shape())] {
100 if actual != normalized_shape.as_slice() {
101 return Err(validation::shape_mismatch(
102 "LayerNorm",
103 parameter,
104 normalized_shape.as_slice(),
105 actual,
106 ));
107 }
108 }
109 LayerNorm::from_parts(w, b, eps)?.forward(input)
110}
111
112#[derive(Clone)]
117pub struct LayerNorm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
118 pub weight: Var<T, B>,
120 pub bias: Var<T, B>,
122 pub eps: f64,
124 eps_t: Tensor<T, B>,
126 d_const: Tensor<T, B>,
128 ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
130}
131
132impl<T: coeus_core::FloatElement + Float, B: coeus_ops::BackendOps<T> + Default> LayerNorm<T, B> {
133 pub fn new(
135 normalized_shape: usize,
136 eps: f64,
137 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
138 Ok(Self::from_shape(normalized_shape, eps)?)
139 }
140
141 pub fn from_shape(
143 normalized_shape: impl Into<NormalizedShape>,
144 eps: f64,
145 ) -> Result<Self, B::Error> {
146 let normalized_shape = normalized_shape.into();
147 let backend = B::default();
148 let weight = Var::new(
149 Tensor::ones_on(normalized_shape.as_slice(), &backend)?,
150 true,
151 )?;
152 let bias = Var::new(
153 Tensor::zeros_on(normalized_shape.as_slice(), &backend)?,
154 true,
155 )?;
156 let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
157 let d_const = Tensor::full_on(
158 [1],
159 coeus_core::FloatElement::from_count(weight.tensor.numel()),
160 &backend,
161 )?;
162 Ok(Self {
163 weight,
164 bias,
165 eps,
166 eps_t,
167 d_const,
168 ones_cache: RefCell::new(None),
169 })
170 }
171
172 pub fn from_parts(weight: Var<T, B>, bias: Var<T, B>, eps: f64) -> Result<Self, B::Error> {
174 let backend = B::default();
175 let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
176 let d_const = Tensor::full_on(
177 [1],
178 coeus_core::FloatElement::from_count(weight.tensor.numel()),
179 &backend,
180 )?;
181 Ok(Self {
182 weight,
183 bias,
184 eps,
185 eps_t,
186 d_const,
187 ones_cache: RefCell::new(None),
188 })
189 }
190
191 fn normalize_flat(
192 &self,
193 input: &Var<T, B>,
194 normalized_size: usize,
195 ) -> Result<Var<T, B>, ModuleError<B::Error>>
196 where
197 T: coeus_cpu::RealScalar,
198 {
199 const MODULE: &str = "LayerNorm";
200 let batch = input.tensor.shape()[0];
201 let backend = B::default();
202
203 let mean_t = coeus_ops::mean_axis(&input.tensor, 1, &backend)
204 .map_err(|source| validation::backend(MODULE, source))?;
205 let xmu = coeus_ops::sub(&input.tensor, &mean_t, &backend)?;
206 let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
207 let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
208 .map_err(|source| validation::backend(MODULE, source))?;
209
210 coeus_ops::add_assign(&mut stdev, &self.eps_t, &backend)
211 .map_err(|source| validation::backend(MODULE, source))?;
212 coeus_ops::sqrt_assign(&mut stdev, &backend)
213 .map_err(|source| validation::backend(MODULE, source))?;
214
215 let ones = {
216 let mut cache = self
217 .ones_cache
218 .try_borrow_mut()
219 .map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
220 if let Some((cached_batch, ref cached_ones)) = *cache {
221 if cached_batch == batch {
222 cached_ones.clone()
223 } else {
224 let ones = Tensor::ones_on([batch, 1], &backend)?;
225 *cache = Some((batch, ones.clone()));
226 ones
227 }
228 } else {
229 let ones = Tensor::ones_on([batch, 1], &backend)?;
230 *cache = Some((batch, ones.clone()));
231 ones
232 }
233 };
234 let mut istdev = ones;
235 coeus_ops::div_assign(&mut istdev, &stdev, &backend)
236 .map_err(|source| validation::backend(MODULE, source))?;
237
238 let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
239 let w_reshaped = self.weight.tensor.reshape([1, normalized_size]);
240 let b_reshaped = self.bias.tensor.reshape([1, normalized_size]);
241 let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
242 coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
243 .map_err(|source| validation::backend(MODULE, source))?;
244
245 Ok(coeus_autograd::layernorm(
246 input,
247 &self.weight,
248 &self.bias,
249 out_tensor,
250 x_hat,
251 istdev,
252 self.d_const.clone(),
253 )?)
254 }
255}
256
257impl<
259 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
260 B: coeus_ops::BackendOps<T> + Default,
261 > Module<T, B> for LayerNorm<T, B>
262{
263 fn parameters(&self) -> Vec<Var<T, B>> {
264 vec![self.weight.clone(), self.bias.clone()]
265 }
266
267 fn load_parameters(&mut self, params: &[Var<T, B>]) {
269 self.weight = params[0].clone();
270 self.bias = params[1].clone();
271 }
272
273 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
275 self.forward_nd(input)
276 }
277}
278
279impl<
280 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
281 B: coeus_ops::BackendOps<T> + Default,
282 > LayerNorm<T, B>
283{
284 pub fn forward_nd(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
296 const MODULE: &str = "LayerNorm";
297 let input_shape = input.tensor.shape_cloned();
298 if input_shape.len() < 2 {
299 return Err(validation::invalid_rank(
300 MODULE,
301 "at least 2",
302 input_shape.len(),
303 ));
304 }
305
306 let normalized_shape = self.weight.tensor.shape_cloned();
307 if normalized_shape.is_empty() || normalized_shape.contains(&0) {
308 return Err(validation::shape_mismatch(
309 MODULE,
310 "normalized_shape",
311 &[1],
312 &normalized_shape,
313 ));
314 }
315 for (parameter, actual) in [
316 ("weight", self.weight.tensor.shape()),
317 ("bias", self.bias.tensor.shape()),
318 ] {
319 if actual != &*normalized_shape {
320 return Err(validation::shape_mismatch(
321 MODULE,
322 parameter,
323 &normalized_shape,
324 actual,
325 ));
326 }
327 }
328 if !self.eps.is_finite() || self.eps < 0.0 {
329 return Err(ModuleError::InvalidEpsilon { module: MODULE });
330 }
331
332 let Some(suffix_start) = input_shape.len().checked_sub(normalized_shape.len()) else {
333 return Err(validation::shape_mismatch(
334 MODULE,
335 "input trailing dimensions",
336 &normalized_shape,
337 &input_shape,
338 ));
339 };
340 let input_suffix = &input_shape[suffix_start..];
341 if input_suffix != &*normalized_shape {
342 return Err(validation::shape_mismatch(
343 MODULE,
344 "input trailing dimensions",
345 &normalized_shape,
346 input_suffix,
347 ));
348 }
349
350 let normalized_size = self.weight.tensor.numel();
351 let Some(batch) = input.tensor.numel().checked_div(normalized_size) else {
352 return Err(validation::shape_mismatch(
353 MODULE,
354 "input",
355 &normalized_shape,
356 &input_shape,
357 ));
358 };
359 if input_shape.len() == 2 && normalized_shape.len() == 1 {
360 return self.normalize_flat(input, normalized_size);
361 }
362
363 let flat = coeus_autograd::reshape(input, [batch, normalized_size])?;
364 let normalized = self.normalize_flat(&flat, normalized_size)?;
365 Ok(coeus_autograd::reshape(&normalized, input_shape)?)
366 }
367}