use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNormConfig,
activation::ActivationConfig,
attention::{
MhaInput,
MultiHeadAttention,
MultiHeadAttentionConfig,
},
norm::{
Normalization,
NormalizationConfig,
},
},
tensor::Device,
};
use crate::{
blocks::transformers::mlp::{
Mlp,
MlpConfig,
},
burner::module::{
ModuleInit,
Rekey,
},
contracts::unpack_shape_contract,
errors::BunsenResult,
ops::transformers::attention::length_mask,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum NormPlacement {
Pre,
Post,
}
pub trait ResidualEncoderAttentionBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn dropout(&self) -> f64;
fn norm_placement(&self) -> NormPlacement;
}
#[derive(Config, Debug)]
pub struct ResidualEncoderAttentionBlockConfig {
pub d_model: usize,
#[config(default = "64")]
pub d_head: usize,
#[config(default = "0.0")]
pub dropout: f64,
#[config(default = "4")]
pub expansion_factor: usize,
#[config(default = "false")]
pub key_bias: bool,
#[config(default = "Some(NormalizationConfig::Layer(LayerNormConfig::new(0)))")]
pub norm: Option<NormalizationConfig>,
#[config(default = "NormPlacement::Pre")]
pub norm_placement: NormPlacement,
#[config(default = "ActivationConfig::Gelu")]
pub activation: ActivationConfig,
}
impl ResidualEncoderAttentionBlockMeta for ResidualEncoderAttentionBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn dropout(&self) -> f64 {
self.dropout
}
fn norm_placement(&self) -> NormPlacement {
self.norm_placement
}
}
impl ModuleInit<ResidualEncoderAttentionBlock> for ResidualEncoderAttentionBlockConfig {
fn try_init(
&self,
device: &Device,
) -> crate::errors::BunsenResult<ResidualEncoderAttentionBlock> {
let mha_cfg = MultiHeadAttentionConfig::new(self.d_model, self.n_heads())
.with_dropout(self.dropout)
.with_key_bias(self.key_bias);
let norm = || {
self.norm
.as_ref()
.map(|n| n.clone().with_num_features(self.d_model).init(device))
};
let attn = mha_cfg.init(device);
let mlp: Mlp = MlpConfig::new(self.d_model)
.with_expansion_factor(self.expansion_factor)
.with_activation(self.activation.clone())
.with_bias(true)
.try_init(device)?;
Ok(ResidualEncoderAttentionBlock {
attn_norm: norm(),
attn,
mlp_norm: norm(),
mlp,
norm_placement: self.norm_placement,
})
}
}
#[derive(Module, Debug)]
pub struct ResidualEncoderAttentionBlock {
pub attn_norm: Option<Normalization>,
pub attn: MultiHeadAttention,
pub mlp_norm: Option<Normalization>,
pub mlp: Mlp,
#[module(skip)]
pub norm_placement: NormPlacement,
}
fn normalize(
norm: &Option<Normalization>,
x: Tensor<3>,
) -> Tensor<3> {
match norm {
Some(norm) => norm.forward(x),
None => x,
}
}
impl ResidualEncoderAttentionBlockMeta for ResidualEncoderAttentionBlock {
fn d_model(&self) -> usize {
self.attn.d_model
}
fn n_heads(&self) -> usize {
self.attn.n_heads
}
fn dropout(&self) -> f64 {
self.attn.dropout.prob
}
fn norm_placement(&self) -> NormPlacement {
self.norm_placement
}
}
impl ResidualEncoderAttentionBlock {
pub fn forward(
&self,
x: Tensor<3>,
) -> Tensor<3> {
self.forward_masked(x, None)
}
pub fn forward_masked(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, burn::prelude::Bool>>,
) -> Tensor<3> {
let attend = |x: Tensor<3>| {
let input = MhaInput::self_attn(x);
let input = match mask {
Some(mask) => input.mask_attn(mask),
None => input,
};
self.attn.forward(input).context
};
match self.norm_placement {
NormPlacement::Pre => {
let x = x.clone() + attend(normalize(&self.attn_norm, x));
x.clone() + self.mlp.forward(normalize(&self.mlp_norm, x))
}
NormPlacement::Post => {
let x = normalize(&self.attn_norm, x.clone() + attend(x));
normalize(&self.mlp_norm, x.clone() + self.mlp.forward(x))
}
}
}
}
#[derive(Config, Debug)]
pub struct ResidualEncoderTransformerConfig {
pub block: ResidualEncoderAttentionBlockConfig,
pub n_layers: usize,
}
pub trait ResidualEncoderTransformerMeta {
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
}
impl ResidualEncoderTransformerMeta for ResidualEncoderTransformerConfig {
fn d_model(&self) -> usize {
self.block.d_model()
}
fn n_layers(&self) -> usize {
self.n_layers
}
}
impl ModuleInit<ResidualEncoderTransformer> for ResidualEncoderTransformerConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ResidualEncoderTransformer> {
Ok(ResidualEncoderTransformer {
layers: (0..self.n_layers)
.map(|_| self.block.try_init(device))
.collect::<BunsenResult<_>>()?,
})
}
}
#[derive(Module, Debug)]
pub struct ResidualEncoderTransformer {
pub layers: Vec<ResidualEncoderAttentionBlock>,
}
impl ResidualEncoderTransformerMeta for ResidualEncoderTransformer {
fn d_model(&self) -> usize {
self.layers.first().map_or(0, |l| l.d_model())
}
fn n_layers(&self) -> usize {
self.layers.len()
}
}
impl ResidualEncoderTransformer {
pub fn forward(
&self,
x: Tensor<3>,
lengths: Option<&[usize]>,
) -> Tensor<3> {
let [batch, frames] = unpack_shape_contract!(
["batch", "frames", "d_model"],
&x.dims(),
&["batch", "frames"],
&[("d_model", self.d_model())],
);
let mask = lengths.map(|lengths| {
length_mask(lengths, frames, &x.device())
.bool_not()
.unsqueeze_dim::<3>(1)
.expand([batch, frames, frames])
});
self.layers
.iter()
.fold(x, |x, layer| layer.forward_masked(x, mask.clone()))
}
}
impl ResidualEncoderAttentionBlock {
pub fn whisper_hf_rekey() -> Rekey {
Rekey::keep_rest()
.rename("self_attn.q_proj.", "attn.query.")
.rename("self_attn.k_proj.", "attn.key.")
.rename("self_attn.v_proj.", "attn.value.")
.rename("self_attn.out_proj.", "attn.output.")
.rename("self_attn_layer_norm.", "attn_norm.")
.rename("final_layer_norm.", "mlp_norm.")
.rename("fc1.", "mlp.linear1.")
.rename("fc2.", "mlp.linear2.")
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::Shape,
tensor::{
Distribution,
Tolerance,
},
};
use super::*;
use crate::{
blocks::transformers::mlp::norm_mlp,
contracts::assert_shape_contract,
ops::transformers::attention::norm_self_attn,
support::testing::{
DeviceMemoryGuard,
cpu_device,
},
};
fn layer_norm(norm: &Option<Normalization>) -> &burn::nn::LayerNorm {
match norm {
Some(Normalization::Layer(norm)) => norm,
other => panic!("not a LayerNorm: {other:?}"),
}
}
#[test]
fn test_defaults_are_whispers() {
let cfg = ResidualEncoderAttentionBlockConfig::new(64);
assert!(matches!(cfg.norm, Some(NormalizationConfig::Layer(_))));
assert_eq!(cfg.norm_placement, NormPlacement::Pre);
assert!(matches!(cfg.activation, ActivationConfig::Gelu));
assert!(!cfg.key_bias);
let block: ResidualEncoderAttentionBlock = cfg.init(&crate::support::testing::cpu_device());
assert_eq!(block.norm_placement(), NormPlacement::Pre);
assert_eq!(layer_norm(&block.attn_norm).gamma.dims(), [64]);
assert_eq!(layer_norm(&block.mlp_norm).gamma.dims(), [64]);
assert!(block.attn.key.bias.is_none());
}
#[test]
fn test_norm_is_configurable() {
use burn::nn::RmsNormConfig;
let device = crate::support::testing::cpu_device();
let rms: ResidualEncoderAttentionBlock = ResidualEncoderAttentionBlockConfig::new(32)
.with_d_head(8)
.with_norm(Some(RmsNormConfig::new(0).into()))
.init(&device);
match &rms.attn_norm {
Some(Normalization::Rms(norm)) => assert_eq!(norm.gamma.dims(), [32]),
other => panic!("not an RmsNorm: {other:?}"),
}
let x: Tensor<3> = Tensor::random([2, 5, 32], Distribution::Default, &device);
for placement in [NormPlacement::Pre, NormPlacement::Post] {
let block: ResidualEncoderAttentionBlock = ResidualEncoderAttentionBlockConfig::new(32)
.with_d_head(8)
.with_norm(None)
.with_norm_placement(placement)
.init(&device);
assert!(block.attn_norm.is_none() && block.mlp_norm.is_none());
assert!(block.num_params() < rms.num_params());
let expected = {
let x = x.clone() + block.attn.forward(MhaInput::self_attn(x.clone())).context;
x.clone() + block.mlp.forward(x)
};
block
.forward(x.clone())
.into_data()
.assert_approx_eq::<f64>(&expected.into_data(), Default::default());
}
}
#[test]
fn test_post_norm_forward() {
let device = crate::support::testing::cpu_device();
let cfg = ResidualEncoderAttentionBlockConfig::new(32)
.with_d_head(8)
.with_key_bias(true)
.with_norm_placement(NormPlacement::Post)
.with_activation(ActivationConfig::Relu);
let block: ResidualEncoderAttentionBlock = cfg.init(&device);
assert!(block.attn.key.bias.is_some());
assert_eq!(block.norm_placement(), NormPlacement::Post);
let x: Tensor<3> = Tensor::random([2, 7, 32], Distribution::Default, &device);
let mask: Tensor<3> = Tensor::random([1, 7, 7], Distribution::Bernoulli(0.3), &device);
let mask = mask.bool();
let output = block.forward_masked(x.clone(), Some(mask.clone()));
let expected = {
let context = block
.attn
.forward(MhaInput::self_attn(x.clone()).mask_attn(mask))
.context;
let x = layer_norm(&block.attn_norm).forward(x + context);
layer_norm(&block.mlp_norm).forward(x.clone() + block.mlp.forward(x))
};
output
.into_data()
.assert_approx_eq::<f64>(&expected.into_data(), Default::default());
}
#[test]
#[serial_test::serial]
fn test_residual_encoder_forward() {
use crate::support::testing::performance_device;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let cfg = ResidualEncoderAttentionBlockConfig::new(d_model);
let n_heads = cfg.n_heads();
assert_eq!(cfg.d_model(), d_model);
assert_eq!(cfg.n_heads(), n_heads);
let block: ResidualEncoderAttentionBlock = cfg.init(&device);
assert_eq!(block.d_model(), d_model);
assert_eq!(block.n_heads(), n_heads);
let batch = 2;
let seq_len = 10;
let shape: Shape = [batch, seq_len, d_model].into();
let x: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let output = block.forward(x.clone());
let expected = {
let self_attn = norm_self_attn(
block.attn_norm.as_ref().unwrap(),
&block.attn,
x.clone(),
None,
);
let x = x + self_attn.context;
let mlp = norm_mlp(block.mlp_norm.as_ref().unwrap(), &block.mlp, x.clone());
x + mlp
};
output
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.into_data(), Default::default());
assert_shape_contract!(
["batch", "seq_len", "d_model"],
&output,
&[("batch", batch), ("seq_len", seq_len), ("d_model", d_model),],
);
}
fn toy_transformer() -> ResidualEncoderTransformer {
ResidualEncoderTransformerConfig::new(
ResidualEncoderAttentionBlockConfig::new(8)
.with_d_head(4)
.with_key_bias(true)
.with_norm_placement(NormPlacement::Post),
2,
)
.init(&cpu_device())
}
#[test]
fn test_transformer_layers_and_padding() {
let device = cpu_device();
let model = toy_transformer();
assert_eq!(model.layers.len(), 2);
assert_eq!(model.d_model(), 8);
let x = Tensor::<3>::random([2, 6, 8], Distribution::Default, &device);
let whole = model.forward(x.clone(), None);
let by_hand = model.layers[1].forward(model.layers[0].forward(x.clone()));
whole
.to_data()
.assert_approx_eq::<f32>(&by_hand.to_data(), Tolerance::absolute(1e-6));
let batched = model.forward(x.clone(), Some(&[6, 4]));
let alone = model.forward(x.slice([1..2, 0..4]), None);
batched
.slice([1..2, 0..4])
.to_data()
.assert_approx_eq::<f32>(&alone.to_data(), Tolerance::absolute(1e-5));
}
}