use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNorm,
LayerNormConfig,
activation::ActivationConfig,
attention::{
MultiHeadAttention,
MultiHeadAttentionConfig,
},
},
tensor::Device,
};
use crate::{
blocks::transformers::mlp::{
Mlp,
MlpConfig,
layer_norm_mlp,
},
burner::module::ModuleInit,
ops::transformers::attention::layer_norm_self_attn,
};
pub trait ResidualEncoderAttentionBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn dropout(&self) -> f64;
}
#[derive(Config, Debug)]
pub struct ResidualEncoderAttentionBlockConfig {
pub d_model: usize,
#[config(default = "64")]
pub d_head: usize,
#[config(default = "0.0")]
pub dropout: f64,
#[config(default = "4")]
pub expansion_factor: usize,
#[config(default = "false")]
pub key_bias: bool,
}
impl ResidualEncoderAttentionBlockMeta for ResidualEncoderAttentionBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn dropout(&self) -> f64 {
self.dropout
}
}
impl ModuleInit<ResidualEncoderAttentionBlock> for ResidualEncoderAttentionBlockConfig {
fn try_init(
&self,
device: &Device,
) -> crate::errors::BunsenResult<ResidualEncoderAttentionBlock> {
let mha_cfg =
MultiHeadAttentionConfig::new(self.d_model, self.n_heads()).with_dropout(self.dropout);
let ln_cfg = LayerNormConfig::new(self.d_model);
let mut attn = mha_cfg.init(device);
if !self.key_bias {
attn.key.bias = None;
}
let mlp: Mlp = MlpConfig::new(self.d_model)
.with_expansion_factor(self.expansion_factor)
.with_activation(ActivationConfig::Gelu)
.with_bias(true)
.try_init(device)?;
Ok(ResidualEncoderAttentionBlock {
attn_ln: ln_cfg.init(device),
attn,
mlp_ln: ln_cfg.init(device),
mlp,
})
}
}
#[derive(Module, Debug)]
pub struct ResidualEncoderAttentionBlock {
pub attn_ln: LayerNorm,
pub attn: MultiHeadAttention,
pub mlp_ln: LayerNorm,
pub mlp: Mlp,
}
impl ResidualEncoderAttentionBlockMeta for ResidualEncoderAttentionBlock {
fn d_model(&self) -> usize {
self.attn.d_model
}
fn n_heads(&self) -> usize {
self.attn.n_heads
}
fn dropout(&self) -> f64 {
self.attn.dropout.prob
}
}
impl ResidualEncoderAttentionBlock {
pub fn forward(
&self,
x: Tensor<3>,
) -> Tensor<3> {
self.forward_masked(x, None)
}
pub fn forward_masked(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, burn::prelude::Bool>>,
) -> Tensor<3> {
let self_attn = layer_norm_self_attn(&self.attn_ln, &self.attn, x.clone(), mask);
let x = x + self_attn.context;
let mlp = layer_norm_mlp(&self.mlp_ln, &self.mlp, x.clone());
x + mlp
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::Shape,
tensor::Distribution,
};
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::DeviceMemoryGuard,
};
#[test]
#[serial_test::serial]
fn test_residual_decoder_forward() {
use crate::support::testing::performance_device;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let cfg = ResidualEncoderAttentionBlockConfig::new(d_model);
let n_heads = cfg.n_heads();
assert_eq!(cfg.d_model(), d_model);
assert_eq!(cfg.n_heads(), n_heads);
let block: ResidualEncoderAttentionBlock = cfg.init(&device);
assert_eq!(block.d_model(), d_model);
assert_eq!(block.n_heads(), n_heads);
let batch = 2;
let seq_len = 10;
let shape: Shape = [batch, seq_len, d_model].into();
let x: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let output = block.forward(x.clone());
let expected = {
let self_attn = layer_norm_self_attn(&block.attn_ln, &block.attn, x.clone(), None);
let x = x + self_attn.context;
let mlp = layer_norm_mlp(&block.mlp_ln, &block.mlp, x.clone());
x + mlp
};
output
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.into_data(), Default::default());
assert_shape_contract!(
["batch", "seq_len", "d_model"],
&output,
&[("batch", batch), ("seq_len", seq_len), ("d_model", d_model),],
);
}
}