use crate::backend_authoring::SupportsDType;
use crate::err::{Error, Result};
use crate::exec::catalog::op;
use crate::nn::{Module, Param};
use crate::shapes::idx::{FromEnd, Here};
use crate::shapes::shape_ops::ReduceKeepAt;
use crate::shapes::{Dim, Dyn, DynShape, Shape, ShapeValue};
use crate::tensor::backend::Execute;
use crate::tensor::base::Tensor;
use crate::tensor::device::Device;
use crate::tensor::dtype::DType;
use core::marker::PhantomData;
pub trait RMSNormShape: Shape + DynShape {
type Channels: Dim;
type BuildArg: crate::tensor::arg_into::NotUnit + Clone;
type ParamShape: Shape<Arg = Self::BuildArg> + DynShape;
fn build_args(target: <Self::Channels as Dim>::Arg) -> Self::BuildArg;
}
impl<C: Dim> RMSNormShape for crate::shapes::shape::DimCons<C, crate::shapes::shape::Nil> {
type Channels = C;
type BuildArg = (<C as Dim>::Arg, ());
type ParamShape = crate::shapes::shape::DimCons<C, crate::shapes::shape::Nil>;
fn build_args(target: <Self::Channels as Dim>::Arg) -> Self::BuildArg {
(target, ())
}
}
impl RMSNormShape for Dyn {
type Channels = usize;
type BuildArg = alloc::vec::Vec<usize>;
type ParamShape = Dyn;
fn build_args(target: usize) -> Self::BuildArg {
alloc::vec![target]
}
}
use crate::nn::param::{Frozen, TrainState, Trainable};
#[derive(Debug, Clone)]
#[incin_macros::module(internal)]
pub struct RMSNorm<
S: RMSNormShape,
B: crate::tensor::backend::VariableBackend,
K: DType = f32,
Train: TrainState = Trainable,
> {
pub weight: Param<S::ParamShape, B, K, Train>,
#[module(ignore)]
pub eps: f32,
#[module(ignore)]
_phantom: PhantomData<(S, B, K, Train)>,
}
impl<S: RMSNormShape, B: crate::tensor::backend::VariableBackend, K: DType, Train: TrainState>
RMSNorm<S, B, K, Train>
{
pub fn from_raw_parts(weight: Param<S::ParamShape, B, K, Train>, eps: f32) -> Self {
Self {
weight,
eps,
_phantom: PhantomData,
}
}
pub fn freeze(self) -> RMSNorm<S, B, K, Frozen> {
RMSNorm {
weight: self.weight.freeze(),
eps: self.eps,
_phantom: PhantomData,
}
}
pub fn unfreeze(self) -> RMSNorm<S, B, K, Trainable> {
RMSNorm {
weight: self.weight.unfreeze(),
eps: self.eps,
_phantom: PhantomData,
}
}
}
#[derive(Debug, Clone)]
pub struct RMSNormBuilder<S: RMSNormShape, Train: TrainState = Trainable> {
pub shape: ShapeValue<S>,
pub eps: f32,
pub weight_init: crate::nn::init::Init,
pub _train: PhantomData<Train>,
}
pub fn rms_norm<S: RMSNormShape>(shape: ShapeValue<S>, eps: f32) -> RMSNormBuilder<S> {
RMSNormBuilder {
shape,
eps,
weight_init: crate::nn::init::ones(),
_train: PhantomData,
}
}
impl<S: RMSNormShape, Train: TrainState> RMSNormBuilder<S, Train> {
pub fn weight_init(mut self, init: crate::nn::init::Init) -> Self {
self.weight_init = init;
self
}
pub fn frozen(self) -> RMSNormBuilder<S, Frozen> {
RMSNormBuilder {
shape: self.shape,
eps: self.eps,
weight_init: self.weight_init,
_train: PhantomData,
}
}
}
impl<
S: RMSNormShape,
B: crate::tensor::backend::VariableBackend
+ crate::tensor::backend::SupportsDType<K>
+ crate::nn::param::ParameterInit<K>,
K: DType,
> RMSNorm<S, B, K, Trainable>
{
pub fn build<A>(args: A) -> Result<Self>
where
B: SupportsDType<K>,
A: crate::tensor::arg_into::LayerArgInto<(
<S::Channels as Dim>::Arg,
<K as DType>::Arg,
<B::Device as Device>::Arg,
f32,
)>,
{
let (channels, dtype, device, eps) = args.into_layer_arg();
let shape = S::build_args(channels);
let weight = Param::<S::ParamShape, B, K, Trainable>::ones_raw(
crate::tensor::arg_into::TensorArgsData {
shape,
dtype,
device,
grad: (),
},
)?;
Ok(Self {
weight,
eps,
_phantom: PhantomData,
})
}
}
impl<
S: RMSNormShape,
InS: Shape + DynShape + crate::shapes::EndsWith<S::Channels> + ReduceKeepAt<FromEnd<Here>>,
B: crate::tensor::backend::VariableBackend
+ crate::exec::Capabilities
+ Execute<op::Mul>
+ Execute<op::Div>
+ Execute<op::DivScalar>
+ Execute<op::Sqrt>
+ Execute<op::SumKeepDim>
+ Execute<op::AddScalar>,
K: DType,
Train: TrainState,
> Module<Tensor<InS, B, K>> for RMSNorm<S, B, K, Train>
where
<InS as ReduceKeepAt<FromEnd<Here>>>::Output: DynShape,
<B as Execute<op::SumKeepDim>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Mul>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Div>>::Output: Into<B::Storage<K>>,
<B as Execute<op::DivScalar>>::Output: Into<B::Storage<K>>,
<B as Execute<op::Sqrt>>::Output: Into<B::Storage<K>>,
<B as Execute<op::AddScalar>>::Output: Into<B::Storage<K>>,
{
type Output = Tensor<InS, B, K, Train::TensorGrad>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<InS, B, K>) -> core::result::Result<Self::Output, Error> {
let weight = self.weight.as_tensor()?.into_dyn();
let channels = x.shape_buf().as_ref().last().copied().ok_or_else(|| {
Error::Shape(crate::shapes::error::ShapeError::RankMismatch {
operation: crate::shapes::error::OperationKind::MeanDim,
expected: crate::shapes::error::RankExpectation::AtLeast(1),
actual: 0,
})
})?;
if channels == 0 {
return Err(Error::Shape(
crate::shapes::error::ShapeError::InvalidParameter {
operation: crate::shapes::error::OperationKind::MeanDim,
parameter: "channels",
value: channels,
},
));
}
let x_sq = x.mul_exact(&x)?;
let mean_sq = x_sq
.sum_keepdim_at::<FromEnd<Here>>()?
.div_scalar(channels as f64)?
.into_dyn();
let var = mean_sq.add_scalar(self.eps)?;
let std = var.sqrt()?;
let dyn_x = x.into_dyn();
let normed = dyn_x.broadcast_div(&std)?;
let out = normed.broadcast_mul(&weight)?;
out.into_shape::<InS>()
}
}