use burn::{
config::Config,
module::Module,
nn::{
LayerNorm,
Linear,
LinearConfig,
activation::{
Activation,
ActivationConfig,
},
},
prelude::Tensor,
tensor::Device,
};
use crate::{
burner::module::ModuleInit,
errors::BunsenResult,
};
pub fn layer_norm_mlp(
layer_norm: &LayerNorm,
mlp: &Mlp,
x: Tensor<3>,
) -> Tensor<3> {
mlp.forward(layer_norm.forward(x))
}
pub trait MlpMeta {
fn n_embed(&self) -> usize;
fn act_exponent(&self) -> Option<f64>;
}
#[derive(Config, Debug)]
pub struct MlpConfig {
pub n_embed: usize,
#[config(default = "4")]
pub expansion_factor: usize,
#[config(default = "ActivationConfig::Relu")]
pub activation: ActivationConfig,
#[config(default = "None")]
pub act_exponent: Option<f64>,
#[config(default = "false")]
pub bias: bool,
}
impl MlpMeta for MlpConfig {
fn n_embed(&self) -> usize {
self.n_embed
}
fn act_exponent(&self) -> Option<f64> {
self.act_exponent
}
}
impl MlpConfig {
pub fn hidden_size(&self) -> usize {
self.n_embed * self.expansion_factor
}
}
impl ModuleInit<Mlp> for MlpConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<Mlp> {
let linear1 = LinearConfig::new(self.n_embed(), self.hidden_size())
.with_bias(self.bias)
.init(device);
let linear2 = LinearConfig::new(self.hidden_size(), self.n_embed())
.with_bias(self.bias)
.init(device);
Ok(Mlp {
linear1,
act: self.activation.init(device),
act_exponent: self.act_exponent,
linear2,
})
}
}
#[derive(Module, Debug)]
pub struct Mlp {
pub linear1: Linear,
pub act: Activation,
pub act_exponent: Option<f64>,
pub linear2: Linear,
}
impl MlpMeta for Mlp {
fn n_embed(&self) -> usize {
self.linear1.weight.dims()[0]
}
fn act_exponent(&self) -> Option<f64> {
self.act_exponent
}
}
impl Mlp {
pub fn forward(
&self,
x: Tensor<3>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, time] = crate::contracts::unpack_shape_contract!(
["batch", "time", "embed"],
&x,
&["batch", "time"],
&[("embed", self.n_embed())]
);
let x = self.linear1.forward(x);
let x = self.act.forward(x);
let x = match self.act_exponent {
Some(exp) => x.powf_scalar(exp),
None => x,
};
let x = self.linear2.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "embed"],
&x,
&[("batch", batch), ("time", time), ("embed", self.n_embed())]
);
x
}
}
#[cfg(test)]
mod tests {
use burn::tensor::Distribution;
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
performance_device,
},
};
#[test]
fn test_mlp_config() {
let cfg = MlpConfig::new(3);
assert_eq!(cfg.n_embed, 3);
assert_eq!(cfg.expansion_factor, 4);
assert_eq!(cfg.n_embed(), 3);
assert_eq!(cfg.hidden_size(), 3 * 4);
}
#[test]
#[serial]
fn test_mlp() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
for activation in [ActivationConfig::Relu, ActivationConfig::Gelu] {
for ef in [4, 3] {
let b = 2;
let t = 3;
let n_embed = 10;
let cfg = MlpConfig::new(n_embed)
.with_expansion_factor(ef)
.with_activation(activation.clone())
.with_act_exponent(Some(2.0));
let mlp: Mlp = cfg.init(&device);
assert_eq!(mlp.n_embed(), n_embed);
let input = Tensor::random([b, t, n_embed], Distribution::Default, &device);
let output = mlp.forward(input.clone());
let x = input;
assert_shape_contract!(
["batch", "time", "embed"],
&x.dims(),
&[("batch", b), ("time", t), ("embed", n_embed)]
);
let x = mlp.linear1.forward(x);
assert_shape_contract!(
["batch", "time", "hidden"],
&x.dims(),
&[("batch", b), ("time", t), ("hidden", ef * n_embed)]
);
let x = mlp.act.forward(x);
assert_shape_contract!(
["batch", "time", "hidden"],
&x.dims(),
&[("batch", b), ("time", t), ("hidden", ef * n_embed)]
);
let x = x.clone() * x;
let x = mlp.linear2.forward(x);
assert_shape_contract!(
["batch", "time", "embed"],
&x.dims(),
&[("batch", b), ("time", t), ("embed", n_embed)]
);
output.to_data().assert_eq(&x.to_data(), true);
}
}
}
}