use crate::backend_authoring::SupportsDType;
use crate::dist::placement::Local;
use crate::err::{Error, Result};
use crate::exec::catalog::{LayerNormAttributes, op};
use crate::exec::context::ExecutionContext;
use crate::exec::dispatch;
use crate::exec::request::TensorHandle;
use crate::nn::module::ShapeInfo;
use crate::nn::{Module, Param};
use crate::shapes::{Dim, Dyn, DynShape, Shape, ShapeValue};
use crate::tensor::backend::Execute;
use crate::tensor::base::Tensor;
use crate::tensor::device::Device;
use crate::tensor::dtype::DType;
use alloc::string::String;
use core::marker::PhantomData;
pub trait LayerNormShape: Shape + DynShape {
type Channels: Dim;
type BuildArg: crate::tensor::arg_into::NotUnit + Clone;
type ParamShape: Shape<Arg = Self::BuildArg> + DynShape;
fn build_args(target: <Self::Channels as Dim>::Arg) -> Self::BuildArg;
}
impl<C: Dim> LayerNormShape for crate::shapes::shape::DimCons<C, crate::shapes::shape::Nil> {
type Channels = C;
type BuildArg = (<C as Dim>::Arg, ());
type ParamShape = crate::shapes::shape::DimCons<C, crate::shapes::shape::Nil>;
fn build_args(target: <Self::Channels as Dim>::Arg) -> Self::BuildArg {
(target, ())
}
}
impl LayerNormShape for Dyn {
type Channels = usize;
type BuildArg = alloc::vec::Vec<usize>;
type ParamShape = Dyn;
fn build_args(target: usize) -> Self::BuildArg {
alloc::vec![target]
}
}
use crate::nn::param::{Frozen, TrainState, Trainable};
#[derive(Debug)]
#[incin_macros::module(internal)]
pub struct LayerNorm<
S: LayerNormShape,
B: crate::tensor::backend::VariableBackend,
K: DType = f32,
Train: TrainState = Trainable,
> {
pub weight: Param<S::ParamShape, B, K, Train>,
pub bias: Param<S::ParamShape, B, K, Train>,
#[module(ignore)]
pub eps: f32,
#[module(ignore)]
_phantom: PhantomData<(S, B, K, Train)>,
}
impl<S, B, K, Train> ShapeInfo for LayerNorm<S, B, K, Train>
where
S: LayerNormShape,
B: crate::tensor::backend::VariableBackend,
K: DType,
Train: TrainState,
{
fn shape_info(&self) -> Option<String> {
None
}
}
impl<S: LayerNormShape, B: crate::tensor::backend::VariableBackend, K: DType, Train: TrainState>
LayerNorm<S, B, K, Train>
{
pub fn from_raw_parts(
weight: Param<S::ParamShape, B, K, Train>,
bias: Param<S::ParamShape, B, K, Train>,
eps: f32,
) -> Self {
Self {
weight,
bias,
eps,
_phantom: PhantomData,
}
}
pub fn freeze(self) -> LayerNorm<S, B, K, Frozen> {
LayerNorm {
weight: self.weight.freeze(),
bias: self.bias.freeze(),
eps: self.eps,
_phantom: PhantomData,
}
}
pub fn unfreeze(self) -> LayerNorm<S, B, K, Trainable> {
LayerNorm {
weight: self.weight.unfreeze(),
bias: self.bias.unfreeze(),
eps: self.eps,
_phantom: PhantomData,
}
}
}
#[derive(Debug, Clone)]
pub struct LayerNormBuilder<S: LayerNormShape, Train: TrainState = Trainable> {
pub shape: ShapeValue<S>,
pub eps: f32,
pub weight_init: crate::nn::init::Init,
pub bias_init: crate::nn::init::Init,
pub _train: PhantomData<Train>,
}
pub fn layer_norm<S: LayerNormShape>(shape: ShapeValue<S>, eps: f32) -> LayerNormBuilder<S> {
LayerNormBuilder {
shape,
eps,
weight_init: crate::nn::init::ones(),
bias_init: crate::nn::init::zeros(),
_train: PhantomData,
}
}
impl<S: LayerNormShape, Train: TrainState> LayerNormBuilder<S, Train> {
pub fn weight_init(mut self, init: crate::nn::init::Init) -> Self {
self.weight_init = init;
self
}
pub fn bias_init(mut self, init: crate::nn::init::Init) -> Self {
self.bias_init = init;
self
}
pub fn frozen(self) -> LayerNormBuilder<S, Frozen> {
LayerNormBuilder {
shape: self.shape,
eps: self.eps,
weight_init: self.weight_init,
bias_init: self.bias_init,
_train: PhantomData,
}
}
}
impl<
S: LayerNormShape,
B: crate::tensor::backend::VariableBackend
+ crate::tensor::backend::SupportsDType<K>
+ crate::nn::param::ParameterInit<K>,
K: DType,
> LayerNorm<S, B, K, Trainable>
where
B: SupportsDType<K>,
<K as DType>::Arg: Clone,
<B::Device as Device>::Arg: Clone,
{
pub fn build<A>(args: A) -> Result<Self>
where
A: crate::tensor::arg_into::LayerArgInto<(
<S::Channels as Dim>::Arg,
<K as DType>::Arg,
<B::Device as Device>::Arg,
f32,
)>,
{
let (channels, dtype, device, eps) = args.into_layer_arg();
let shape = S::build_args(channels);
let weight = Param::<S::ParamShape, B, K, Trainable>::ones_raw(
crate::tensor::arg_into::TensorArgsData {
shape: shape.clone(),
dtype: dtype.clone(),
device: device.clone(),
grad: (),
},
)?;
let bias = Param::<S::ParamShape, B, K, Trainable>::zeros_raw(
crate::tensor::arg_into::TensorArgsData {
shape,
dtype,
device,
grad: (),
},
)?;
Ok(Self {
weight,
bias,
eps,
_phantom: PhantomData,
})
}
}
impl<
S: LayerNormShape,
InS: Shape + DynShape + crate::shapes::EndsWith<S::Channels>,
B: crate::tensor::backend::VariableBackend + crate::exec::Capabilities + Execute<op::LayerNorm>,
K: DType,
Train: TrainState,
> Module<Tensor<InS, B, K>> for LayerNorm<S, B, K, Train>
where
<B as Execute<op::LayerNorm>>::Output: Into<B::Storage<K>>,
{
type Output = Tensor<InS, B, K>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<InS, B, K>) -> core::result::Result<Self::Output, Error> {
let weight = self.weight.as_tensor()?;
let bias = self.bias.as_tensor()?;
let inputs = [
TensorHandle::from_storage::<B, K, Local>(x.inner()),
TensorHandle::from_storage::<B, K, Local>(weight.inner()),
TensorHandle::from_storage::<B, K, Local>(bias.inner()),
];
let context = ExecutionContext::from_scope(B::default());
let out_inner = dispatch::execute_shaped::<op::LayerNorm, B, InS>(
&context,
LayerNormAttributes {
normalized_shape: weight.shape_buf().as_ref().to_vec(),
epsilon: self.eps as f64,
has_bias: true,
},
&inputs,
&x._shape,
)
.map_err(crate::err::Error::from)?;
Tensor::from_shape_value(
out_inner.into(),
x._shape.clone(),
x._dtype.clone(),
weight._device,
*x.grad_field(),
)
}
}