Struct dfdx::nn::LayerNorm1D

pub struct LayerNorm1D<const M: usize> {
    pub gamma: Tensor1D<M>,
    pub beta: Tensor1D<M>,
    pub epsilon: f32,
}

Implements layer normalization as described in Layer Normalization.

This calls normalize() on the last axis of the input to normalize to 0 mean and unit std dev, and then does an element-wise affine transform using learnable parameters Self::gamma and Self::beta.

Self::epsilon is passed to normalize() and added to the variance to ensure big enough numbers. It defaults to 1e-5.

Generics

• M The size of the affine transform tensors.

Examples

let model: LayerNorm1D<5> = Default::default();
let x: Tensor1D<5> = Default::default();
let _: Tensor1D<5> = model.forward(x);

Fields

gamma: Tensor1D<M>

beta: Tensor1D<M>

epsilon: f32

Trait Implementations

impl<const M: usize> CanUpdateWithGradients for LayerNorm1D<M>

fn update<G: GradientProvider>(
    &mut self,
    grads: &mut G,
    unused: &mut UnusedTensors
)

Updates Self::gamma and Self::beta.

impl<const M: usize> Clone for LayerNorm1D<M>

fn clone(&self) -> LayerNorm1D<M>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<const M: usize> Debug for LayerNorm1D<M>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<const M: usize> Default for LayerNorm1D<M>

fn default() -> Self

Fills Self::gamma with 1s and Self::beta with 0s and sets Self::epsilon to 1e-5.

impl<const M: usize> LoadFromNpz for LayerNorm1D<M>

fn read<R: Read + Seek>(
    &mut self,
    p: &str,
    r: &mut ZipArchive<R>
) -> Result<(), NpzError>

Reads this object from a ZipArchive. r with a base filename of filename_prefix.

fn load<P: AsRef<Path>>(&mut self, path: P) -> Result<(), NpzError>

Loads data from a .npz zip archive at the specified path.

impl<H: Tape, const M: usize> Module<Tensor1D<M, H>> for LayerNorm1D<M>

fn forward(&self, x: Tensor1D<M, H>) -> Self::Output

Calls:

1. normalize() with Self::epsilon
2. mul() with Self::gamma
3. add() with Self::beta

type Output = Tensor1D<M, H>

The type that this unit produces given Input.

impl<H: Tape, const B: usize, const M: usize> Module<Tensor2D<B, M, H>> for LayerNorm1D<M>

fn forward(&self, x: Tensor2D<B, M, H>) -> Self::Output

Calls:

1. normalize() with Self::epsilon.
2. mul() with Self::gamma
3. add() with Self::beta

type Output = Tensor2D<B, M, H>

The type that this unit produces given Input.

impl<H: Tape, const B: usize, const S: usize, const M: usize> Module<Tensor3D<B, S, M, H>> for LayerNorm1D<M>

fn forward(&self, x: Tensor3D<B, S, M, H>) -> Self::Output

Calls:

1. normalize() with Self::epsilon.
2. add() with Self::gamma
3. add() with Self::beta

type Output = Tensor3D<B, S, M, H>

The type that this unit produces given Input.

impl<T, const M: usize> ModuleMut<T> for LayerNorm1D<M>where
    Self: Module<T>,

type Output = <LayerNorm1D<M> as Module<T>>::Output

The type that this unit produces given Input.

fn forward_mut(&mut self, input: T) -> Self::Output

Forward Input through the module and produce ModuleMut::Output.

impl<const M: usize> ResetParams for LayerNorm1D<M>

fn reset_params<R: Rng>(&mut self, _: &mut R)

Fills Self::gamma with 1s and Self::beta with 0s.

impl<const M: usize> SaveToNpz for LayerNorm1D<M>

fn write<W: Write + Seek>(&self, p: &str, w: &mut ZipWriter<W>) -> ZipResult<()>

Write this object into ZipWriter w with a base filename of filename_prefix.

fn save<P: AsRef<Path>>(&self, path: P) -> ZipResult<()>

Save this object into the .npz file determined located at path.