use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNorm,
LayerNormConfig,
activation::ActivationConfig,
attention::{
MultiHeadAttention,
MultiHeadAttentionConfig,
},
},
prelude::Bool,
tensor::Device,
};
use super::WHISPER_DEFAULT_D_MODEL;
use crate::{
blocks::transformers::mlp::{
Mlp,
MlpConfig,
layer_norm_mlp,
},
burner::module::ModuleInit,
errors::BunsenResult,
ops::transformers::attention::{
AttnKvPair,
layer_norm_cross_attn,
layer_norm_cross_attn_w_kv_cache_grouped,
layer_norm_self_attn,
layer_norm_self_attn_w_kv_cache,
project_kv_pair,
},
};
pub trait ResidualDecoderAttentionBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn dropout(&self) -> f64;
}
#[derive(Config, Debug)]
pub struct ResidualDecoderAttentionBlockConfig {
pub d_model: usize,
#[config(defaul_value = "WHISPER_DEFAULT_D_MODEL")]
pub d_head: usize,
#[config(default = "0.0")]
pub dropout: f64,
}
impl ResidualDecoderAttentionBlockMeta for ResidualDecoderAttentionBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn dropout(&self) -> f64 {
self.dropout
}
}
impl ModuleInit<ResidualDecoderAttentionBlock> for ResidualDecoderAttentionBlockConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ResidualDecoderAttentionBlock> {
let mha_cfg =
MultiHeadAttentionConfig::new(self.d_model, self.n_heads()).with_dropout(self.dropout);
let ln_cfg = LayerNormConfig::new(self.d_model);
let mut attn = mha_cfg.init(device);
let mut cross_attn = mha_cfg.init(device);
attn.key.bias = None;
cross_attn.key.bias = None;
let mlp: Mlp = MlpConfig::new(self.d_model)
.with_activation(ActivationConfig::Gelu)
.with_bias(true)
.try_init(device)?;
Ok(ResidualDecoderAttentionBlock {
attn_ln: ln_cfg.init(device),
attn,
cross_attn_ln: ln_cfg.init(device),
cross_attn,
mlp_ln: ln_cfg.init(device),
mlp,
})
}
}
#[derive(Module, Debug)]
pub struct ResidualDecoderAttentionBlock {
pub attn_ln: LayerNorm,
pub attn: MultiHeadAttention,
pub cross_attn_ln: LayerNorm,
pub cross_attn: MultiHeadAttention,
pub mlp_ln: LayerNorm,
pub mlp: Mlp,
}
impl ResidualDecoderAttentionBlockMeta for ResidualDecoderAttentionBlock {
fn d_model(&self) -> usize {
self.attn.d_model
}
fn n_heads(&self) -> usize {
self.attn.n_heads
}
fn dropout(&self) -> f64 {
self.attn.dropout.prob
}
}
#[derive(Debug, Clone)]
pub struct DecodeRecord {
pub output: Tensor<3>,
pub ca_weights: Tensor<4>,
}
impl DecodeRecord {
pub fn batch_size(&self) -> usize {
self.output.shape()[0]
}
pub fn d_model(&self) -> usize {
self.output.shape()[2]
}
pub fn seq_len(&self) -> usize {
self.output.shape()[1]
}
}
impl ResidualDecoderAttentionBlock {
pub fn forward(
&self,
x: Tensor<3>,
xa: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
) -> DecodeRecord {
let self_attn = layer_norm_self_attn(&self.attn_ln, &self.attn, x.clone(), mask);
let x = x + self_attn.context;
let cross_attn =
layer_norm_cross_attn(&self.cross_attn_ln, &self.cross_attn, x.clone(), xa);
let x = x + cross_attn.context;
let mlp = layer_norm_mlp(&self.mlp_ln, &self.mlp, x.clone());
let x = x + mlp;
DecodeRecord {
output: x,
ca_weights: cross_attn.weights,
}
}
pub fn forward_w_kv_cache(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
self_kv: &mut Option<AttnKvPair>,
cross_kv: &AttnKvPair,
) -> Tensor<3> {
self.forward_w_kv_cache_grouped(x, mask, self_kv, cross_kv, 1)
}
pub fn forward_w_kv_cache_grouped(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
self_kv: &mut Option<AttnKvPair>,
cross_kv: &AttnKvPair,
group: usize,
) -> Tensor<3> {
let x = x.clone().add(layer_norm_self_attn_w_kv_cache(
&self.attn_ln,
&self.attn,
x,
mask,
self_kv,
));
let x = x.clone().add(layer_norm_cross_attn_w_kv_cache_grouped(
&self.cross_attn_ln,
&self.cross_attn,
x,
cross_kv,
group,
));
let mlp = layer_norm_mlp(&self.mlp_ln, &self.mlp, x.clone());
x + mlp
}
pub fn build_cross_kv(
&self,
xa: Tensor<3>,
) -> AttnKvPair {
project_kv_pair(&self.cross_attn, xa)
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::Shape,
tensor::Distribution,
};
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::DeviceMemoryGuard,
};
#[test]
#[serial_test::serial]
fn test_residual_decoder_forward() {
use crate::support::testing::performance_device;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let cfg = ResidualDecoderAttentionBlockConfig::new(d_model);
let n_heads = cfg.n_heads();
assert_eq!(cfg.d_model(), d_model);
assert_eq!(cfg.n_heads(), n_heads);
let block: ResidualDecoderAttentionBlock = cfg.init(&device);
assert_eq!(block.d_model(), d_model);
assert_eq!(block.n_heads(), n_heads);
let batch = 2;
let seq_len = 10;
let shape: Shape = [batch, seq_len, d_model].into();
let x: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let xa: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let mask: Tensor<3> =
Tensor::random([1, seq_len, seq_len], Distribution::Bernoulli(0.5), &device);
let mask = mask.bool();
let result = block.forward(x.clone(), xa.clone(), mask.clone().into());
assert_eq!(result.batch_size(), batch);
assert_eq!(result.d_model(), d_model);
assert_eq!(result.seq_len(), seq_len);
let expected = {
let self_attn =
layer_norm_self_attn(&block.attn_ln, &block.attn, x.clone(), mask.clone().into());
let x = x + self_attn.context;
let cross_attn = layer_norm_cross_attn(
&block.cross_attn_ln,
&block.cross_attn,
x.clone(),
xa.clone(),
);
let x = x + cross_attn.context;
let mlp = layer_norm_mlp(&block.mlp_ln, &block.mlp, x.clone());
let x = x + mlp;
DecodeRecord {
output: x,
ca_weights: cross_attn.weights,
}
};
result
.output
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.output.to_data(), Default::default());
result
.ca_weights
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.ca_weights.to_data(), Default::default());
assert_shape_contract!(
["batch", "seq_len", "d_model"],
&result.output,
&[("batch", batch), ("seq_len", seq_len), ("d_model", d_model),],
);
assert_shape_contract!(
["batch", "n_heads", "seq_len", "seq_len"],
&result.ca_weights,
&[("batch", batch), ("n_heads", n_heads), ("seq_len", seq_len)],
);
}
}