use super::validation;
use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::{Float, MoiraiBackend};
use coeus_tensor::Tensor;
use std::cell::RefCell;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct NormalizedShape(Vec<usize>);
impl NormalizedShape {
#[must_use]
pub fn as_slice(&self) -> &[usize] {
&self.0
}
fn is_valid(&self) -> bool {
!self.0.is_empty() && self.0.iter().all(|&dimension| dimension > 0)
}
}
impl From<usize> for NormalizedShape {
fn from(dimension: usize) -> Self {
Self(vec![dimension])
}
}
impl From<Vec<usize>> for NormalizedShape {
fn from(dimensions: Vec<usize>) -> Self {
Self(dimensions)
}
}
impl From<&[usize]> for NormalizedShape {
fn from(dimensions: &[usize]) -> Self {
Self(dimensions.to_vec())
}
}
impl<const N: usize> From<[usize; N]> for NormalizedShape {
fn from(dimensions: [usize; N]) -> Self {
Self(dimensions.into())
}
}
pub fn layer_norm<
T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
>(
input: &Var<T, B>,
normalized_shape: impl Into<NormalizedShape>,
weight: Option<&Var<T, B>>,
bias: Option<&Var<T, B>>,
eps: f64,
) -> Result<Var<T, B>, ModuleError<B::Error>>
where
B::DeviceBuffer<T>:
coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
{
let normalized_shape = normalized_shape.into();
if !normalized_shape.is_valid() {
return Err(validation::shape_mismatch(
"LayerNorm",
"normalized_shape",
&[1],
normalized_shape.as_slice(),
));
}
let backend = B::default();
let w = match weight {
Some(weight) => weight.clone(),
None => Var::new(
Tensor::ones_on(normalized_shape.as_slice(), &backend)?,
false,
)?,
};
let b = match bias {
Some(bias) => bias.clone(),
None => Var::new(
Tensor::zeros_on(normalized_shape.as_slice(), &backend)?,
false,
)?,
};
for (parameter, actual) in [("weight", w.tensor.shape()), ("bias", b.tensor.shape())] {
if actual != normalized_shape.as_slice() {
return Err(validation::shape_mismatch(
"LayerNorm",
parameter,
normalized_shape.as_slice(),
actual,
));
}
}
LayerNorm::from_parts(w, b, eps)?.forward(input)
}
#[derive(Clone)]
pub struct LayerNorm<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
pub weight: Var<T, B>,
pub bias: Var<T, B>,
pub eps: f64,
eps_t: Tensor<T, B>,
d_const: Tensor<T, B>,
ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
}
impl<T: coeus_core::FloatElement + Float, B: coeus_ops::BackendOps<T> + Default> LayerNorm<T, B> {
pub fn new(
normalized_shape: usize,
eps: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>> {
Ok(Self::from_shape(normalized_shape, eps)?)
}
pub fn from_shape(
normalized_shape: impl Into<NormalizedShape>,
eps: f64,
) -> Result<Self, B::Error> {
let normalized_shape = normalized_shape.into();
let backend = B::default();
let weight = Var::new(
Tensor::ones_on(normalized_shape.as_slice(), &backend)?,
true,
)?;
let bias = Var::new(
Tensor::zeros_on(normalized_shape.as_slice(), &backend)?,
true,
)?;
let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
let d_const = Tensor::full_on(
[1],
coeus_core::FloatElement::from_count(weight.tensor.numel()),
&backend,
)?;
Ok(Self {
weight,
bias,
eps,
eps_t,
d_const,
ones_cache: RefCell::new(None),
})
}
pub fn from_parts(weight: Var<T, B>, bias: Var<T, B>, eps: f64) -> Result<Self, B::Error> {
let backend = B::default();
let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
let d_const = Tensor::full_on(
[1],
coeus_core::FloatElement::from_count(weight.tensor.numel()),
&backend,
)?;
Ok(Self {
weight,
bias,
eps,
eps_t,
d_const,
ones_cache: RefCell::new(None),
})
}
fn normalize_flat(
&self,
input: &Var<T, B>,
normalized_size: usize,
) -> Result<Var<T, B>, ModuleError<B::Error>>
where
T: coeus_cpu::RealScalar,
{
const MODULE: &str = "LayerNorm";
let batch = input.tensor.shape()[0];
let backend = B::default();
let mean_t = coeus_ops::mean_axis(&input.tensor, 1, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let xmu = coeus_ops::sub(&input.tensor, &mean_t, &backend)?;
let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
coeus_ops::add_assign(&mut stdev, &self.eps_t, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
coeus_ops::sqrt_assign(&mut stdev, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let ones = {
let mut cache = self
.ones_cache
.try_borrow_mut()
.map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
if let Some((cached_batch, ref cached_ones)) = *cache {
if cached_batch == batch {
cached_ones.clone()
} else {
let ones = Tensor::ones_on([batch, 1], &backend)?;
*cache = Some((batch, ones.clone()));
ones
}
} else {
let ones = Tensor::ones_on([batch, 1], &backend)?;
*cache = Some((batch, ones.clone()));
ones
}
};
let mut istdev = ones;
coeus_ops::div_assign(&mut istdev, &stdev, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
let w_reshaped = self.weight.tensor.reshape([1, normalized_size]);
let b_reshaped = self.bias.tensor.reshape([1, normalized_size]);
let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
Ok(coeus_autograd::layernorm(
input,
&self.weight,
&self.bias,
out_tensor,
x_hat,
istdev,
self.d_const.clone(),
)?)
}
}
impl<
T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
> Module<T, B> for LayerNorm<T, B>
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![self.weight.clone(), self.bias.clone()]
}
fn load_parameters(&mut self, params: &[Var<T, B>]) {
self.weight = params[0].clone();
self.bias = params[1].clone();
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.forward_nd(input)
}
}
impl<
T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
> LayerNorm<T, B>
{
pub fn forward_nd(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
const MODULE: &str = "LayerNorm";
let input_shape = input.tensor.shape_cloned();
if input_shape.len() < 2 {
return Err(validation::invalid_rank(
MODULE,
"at least 2",
input_shape.len(),
));
}
let normalized_shape = self.weight.tensor.shape_cloned();
if normalized_shape.is_empty() || normalized_shape.contains(&0) {
return Err(validation::shape_mismatch(
MODULE,
"normalized_shape",
&[1],
&normalized_shape,
));
}
for (parameter, actual) in [
("weight", self.weight.tensor.shape()),
("bias", self.bias.tensor.shape()),
] {
if actual != &*normalized_shape {
return Err(validation::shape_mismatch(
MODULE,
parameter,
&normalized_shape,
actual,
));
}
}
if !self.eps.is_finite() || self.eps < 0.0 {
return Err(ModuleError::InvalidEpsilon { module: MODULE });
}
let Some(suffix_start) = input_shape.len().checked_sub(normalized_shape.len()) else {
return Err(validation::shape_mismatch(
MODULE,
"input trailing dimensions",
&normalized_shape,
&input_shape,
));
};
let input_suffix = &input_shape[suffix_start..];
if input_suffix != &*normalized_shape {
return Err(validation::shape_mismatch(
MODULE,
"input trailing dimensions",
&normalized_shape,
input_suffix,
));
}
let normalized_size = self.weight.tensor.numel();
let Some(batch) = input.tensor.numel().checked_div(normalized_size) else {
return Err(validation::shape_mismatch(
MODULE,
"input",
&normalized_shape,
&input_shape,
));
};
if input_shape.len() == 2 && normalized_shape.len() == 1 {
return self.normalize_flat(input, normalized_size);
}
let flat = coeus_autograd::reshape(input, [batch, normalized_size])?;
let normalized = self.normalize_flat(&flat, normalized_size)?;
Ok(coeus_autograd::reshape(&normalized, input_shape)?)
}
}