use burn::{
module::Param,
nn::{
Initializer,
LayerNormConfig,
},
prelude::*,
};
use crate::burner::tensor::Widening;
#[derive(Module, Debug)]
pub struct LayerNorm {
pub gamma: Param<Tensor<1>>,
pub beta: Option<Param<Tensor<1>>>,
pub epsilon: f64,
}
impl LayerNorm {
pub fn init(
config: &LayerNormConfig,
device: &Device,
) -> Self {
Self {
gamma: Initializer::Ones.init([config.d_model], device),
beta: config
.bias
.then(|| Initializer::Zeros.init([config.d_model], device)),
epsilon: config.epsilon,
}
}
pub fn forward<const D: usize>(
&self,
input: Tensor<D>,
) -> Tensor<D> {
Widening::F32.around(input.dtype(), |w| {
burn::tensor::module::layer_norm(
w.widen(input),
w.widen(self.gamma.val()),
self.beta.as_ref().map(|b| w.widen(b.val())),
self.epsilon,
)
})
}
}