use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNormConfig,
activation::ActivationConfig,
attention::{
MultiHeadAttention,
MultiHeadAttentionConfig,
},
norm::{
Normalization,
NormalizationConfig,
},
},
prelude::Bool,
tensor::Device,
};
use crate::{
blocks::transformers::{
encoders::residual::ResidualEncoderAttentionBlock,
mlp::{
Mlp,
MlpConfig,
norm_mlp,
},
},
burner::module::{
ModuleInit,
Rekey,
},
errors::BunsenResult,
ops::transformers::attention::{
KVCache,
norm_cross_attn,
norm_cross_attn_w_kv_cache,
norm_self_attn,
norm_self_attn_w_kv_cache,
project_kv,
},
};
pub trait CrossDecoderBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn dropout(&self) -> f64;
}
#[derive(Config, Debug)]
pub struct CrossDecoderBlockConfig {
pub d_model: usize,
#[config(default = "64")]
pub d_head: usize,
#[config(default = "0.0")]
pub dropout: f64,
#[config(default = "NormalizationConfig::Layer(LayerNormConfig::new(0))")]
pub norm: NormalizationConfig,
}
impl CrossDecoderBlockMeta for CrossDecoderBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn dropout(&self) -> f64 {
self.dropout
}
}
impl ModuleInit<CrossDecoderBlock> for CrossDecoderBlockConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<CrossDecoderBlock> {
let mha_cfg = MultiHeadAttentionConfig::new(self.d_model, self.n_heads())
.with_dropout(self.dropout)
.with_key_bias(false);
let norm = || {
self.norm
.clone()
.with_num_features(self.d_model)
.init(device)
};
let attn = mha_cfg.init(device);
let cross_attn = mha_cfg.init(device);
let mlp: Mlp = MlpConfig::new(self.d_model)
.with_activation(ActivationConfig::Gelu)
.with_bias(true)
.try_init(device)?;
Ok(CrossDecoderBlock {
attn_norm: norm(),
attn,
cross_attn_norm: norm(),
cross_attn,
mlp_norm: norm(),
mlp,
})
}
}
#[derive(Module, Debug)]
pub struct CrossDecoderBlock {
pub attn_norm: Normalization,
pub attn: MultiHeadAttention,
pub cross_attn_norm: Normalization,
pub cross_attn: MultiHeadAttention,
pub mlp_norm: Normalization,
pub mlp: Mlp,
}
impl CrossDecoderBlockMeta for CrossDecoderBlock {
fn d_model(&self) -> usize {
self.attn.d_model
}
fn n_heads(&self) -> usize {
self.attn.n_heads
}
fn dropout(&self) -> f64 {
self.attn.dropout.prob
}
}
#[derive(Debug, Clone)]
pub struct CrossDecoderRecord {
pub output: Tensor<3>,
pub ca_weights: Tensor<4>,
}
impl CrossDecoderRecord {
pub fn batch_size(&self) -> usize {
self.output.shape()[0]
}
pub fn d_model(&self) -> usize {
self.output.shape()[2]
}
pub fn seq_len(&self) -> usize {
self.output.shape()[1]
}
}
impl CrossDecoderBlock {
pub fn forward(
&self,
x: Tensor<3>,
xa: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
) -> CrossDecoderRecord {
let self_attn = norm_self_attn(&self.attn_norm, &self.attn, x.clone(), mask);
let x = x + self_attn.context;
let cross_attn = norm_cross_attn(&self.cross_attn_norm, &self.cross_attn, x.clone(), xa);
let x = x + cross_attn.context;
let mlp = norm_mlp(&self.mlp_norm, &self.mlp, x.clone());
let x = x + mlp;
CrossDecoderRecord {
output: x,
ca_weights: cross_attn.weights,
}
}
pub fn forward_w_kv_cache(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
cache: &mut KVCache,
layer: usize,
) -> Tensor<3> {
let x = x.clone().add(norm_self_attn_w_kv_cache(
&self.attn_norm,
&self.attn,
x,
mask,
cache,
layer,
));
let x = x.clone().add(norm_cross_attn_w_kv_cache(
&self.cross_attn_norm,
&self.cross_attn,
x,
cache,
layer,
));
let mlp = norm_mlp(&self.mlp_norm, &self.mlp, x.clone());
x + mlp
}
pub fn cross_kv(
&self,
xa: Tensor<3>,
) -> (Tensor<4>, Tensor<4>) {
project_kv(&self.cross_attn, xa)
}
pub fn whisper_hf_rekey() -> Rekey {
Rekey::keep_rest()
.rename("encoder_attn.q_proj.", "cross_attn.query.")
.rename("encoder_attn.k_proj.", "cross_attn.key.")
.rename("encoder_attn.v_proj.", "cross_attn.value.")
.rename("encoder_attn.out_proj.", "cross_attn.output.")
.rename("encoder_attn_layer_norm.", "cross_attn_norm.")
.mount("", "", ResidualEncoderAttentionBlock::whisper_hf_rekey())
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::Shape,
tensor::Distribution,
};
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::DeviceMemoryGuard,
};
#[test]
#[serial_test::serial]
fn test_cross_decoder_forward() {
use crate::support::testing::performance_device;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let cfg = CrossDecoderBlockConfig::new(d_model);
let n_heads = cfg.n_heads();
assert_eq!(cfg.d_model(), d_model);
assert_eq!(cfg.n_heads(), n_heads);
let block: CrossDecoderBlock = cfg.init(&device);
assert_eq!(block.d_model(), d_model);
assert_eq!(block.n_heads(), n_heads);
for attn in [&block.attn, &block.cross_attn] {
assert!(attn.key.bias.is_none());
assert!(attn.query.bias.is_some());
assert!(attn.value.bias.is_some());
assert!(attn.output.bias.is_some());
}
let batch = 2;
let seq_len = 10;
let shape: Shape = [batch, seq_len, d_model].into();
let x: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let xa: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let mask: Tensor<3> =
Tensor::random([1, seq_len, seq_len], Distribution::Bernoulli(0.5), &device);
let mask = mask.bool();
let result = block.forward(x.clone(), xa.clone(), mask.clone().into());
assert_eq!(result.batch_size(), batch);
assert_eq!(result.d_model(), d_model);
assert_eq!(result.seq_len(), seq_len);
let expected = {
let self_attn = norm_self_attn(
&block.attn_norm,
&block.attn,
x.clone(),
mask.clone().into(),
);
let x = x + self_attn.context;
let cross_attn = norm_cross_attn(
&block.cross_attn_norm,
&block.cross_attn,
x.clone(),
xa.clone(),
);
let x = x + cross_attn.context;
let mlp = norm_mlp(&block.mlp_norm, &block.mlp, x.clone());
let x = x + mlp;
CrossDecoderRecord {
output: x,
ca_weights: cross_attn.weights,
}
};
result
.output
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.output.to_data(), Default::default());
result
.ca_weights
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.ca_weights.to_data(), Default::default());
assert_shape_contract!(
["batch", "seq_len", "d_model"],
&result.output,
&[("batch", batch), ("seq_len", seq_len), ("d_model", d_model),],
);
assert_shape_contract!(
["batch", "n_heads", "seq_len", "seq_len"],
&result.ca_weights,
&[("batch", batch), ("n_heads", n_heads), ("seq_len", seq_len)],
);
}
}