use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNormConfig,
attention::{
MultiHeadAttention,
MultiHeadAttentionConfig,
},
},
prelude::Bool,
tensor::Device,
};
use crate::{
blocks::{
activation::BunsenActivationConfig,
norm::BunsenNormalizationConfig,
transformers::{
encoders::residual::ResidualEncoderAttentionBlock,
mlp::{
TransformerMlp,
TransformerMlpConfig,
},
sublayer::{
CrossAttnKvSublayer,
CrossAttnSublayer,
PlainSublayer,
ResidualSublayer,
ResidualSublayerConfig,
SelfAttnKvSublayer,
SelfAttnSublayer,
},
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::BunsenResult,
ops::transformers::attention::{
KVCache,
project_kv,
},
};
pub trait CrossDecoderBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn dropout(&self) -> f64;
}
#[derive(Config, Debug)]
pub struct CrossDecoderBlockConfig {
pub d_model: usize,
#[config(default = "64")]
pub d_head: usize,
#[config(default = "0.0")]
pub dropout: f64,
#[config(default = "BunsenNormalizationConfig::Layer(LayerNormConfig::new(0))")]
pub norm: BunsenNormalizationConfig,
}
impl CrossDecoderBlockMeta for CrossDecoderBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn dropout(&self) -> f64 {
self.dropout
}
}
impl ModuleInit<CrossDecoderBlock> for CrossDecoderBlockConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<CrossDecoderBlock> {
let mha_cfg = MultiHeadAttentionConfig::new(self.d_model, self.n_heads())
.with_dropout(self.dropout)
.with_key_bias(false);
let sublayer = ResidualSublayerConfig::pre_norm(Some(self.norm.clone()));
let attn = mha_cfg.init(device);
let cross_attn = mha_cfg.init(device);
let mlp: TransformerMlp = TransformerMlpConfig::new(self.d_model)
.with_activation(BunsenActivationConfig::Gelu)
.with_bias(true)
.try_init(device)?;
Ok(CrossDecoderBlock {
attn: sublayer.init(attn, self.d_model, device),
cross_attn: sublayer.init(cross_attn, self.d_model, device),
mlp: sublayer.init(mlp, self.d_model, device),
})
}
}
#[derive(Module, Debug)]
pub struct CrossDecoderBlock {
pub attn: ResidualSublayer<MultiHeadAttention>,
pub cross_attn: ResidualSublayer<MultiHeadAttention>,
pub mlp: ResidualSublayer<TransformerMlp>,
}
impl CrossDecoderBlockMeta for CrossDecoderBlock {
fn d_model(&self) -> usize {
self.attn.branch.d_model
}
fn n_heads(&self) -> usize {
self.attn.branch.n_heads
}
fn dropout(&self) -> f64 {
self.attn.branch.dropout.prob
}
}
#[derive(Debug, Clone)]
pub struct CrossDecoderRecord {
pub output: Tensor<3>,
pub ca_weights: Tensor<4>,
}
impl CrossDecoderRecord {
pub fn batch_size(&self) -> usize {
self.output.shape()[0]
}
pub fn d_model(&self) -> usize {
self.output.shape()[2]
}
pub fn seq_len(&self) -> usize {
self.output.shape()[1]
}
}
impl CrossDecoderBlock {
pub fn forward(
&self,
x: Tensor<3>,
xa: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
) -> CrossDecoderRecord {
let x = self.attn.self_attn(x, mask);
let (x, ca_weights) = self.cross_attn.cross_attn(x, xa);
let x = self.mlp.forward(x);
CrossDecoderRecord {
output: x,
ca_weights,
}
}
pub fn forward_w_kv_cache(
&self,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
cache: &mut KVCache,
layer: usize,
) -> Tensor<3> {
let x = self.attn.self_attn_w_kv_cache(x, mask, cache, layer);
let x = self.cross_attn.cross_attn_w_kv_cache(x, cache, layer);
self.mlp.forward(x)
}
pub fn cross_kv(
&self,
xa: Tensor<3>,
) -> (Tensor<4>, Tensor<4>) {
project_kv(&self.cross_attn.branch, xa)
}
pub fn whisper_transformers_rekey() -> Rekey {
Rekey::keep_rest()
.rename("encoder_attn.q_proj.", "cross_attn.query.")
.rename("encoder_attn.k_proj.", "cross_attn.key.")
.rename("encoder_attn.v_proj.", "cross_attn.value.")
.rename("encoder_attn.out_proj.", "cross_attn.output.")
.rename("encoder_attn_layer_norm.", "cross_attn_norm.")
.mount(
"",
"",
ResidualEncoderAttentionBlock::whisper_transformers_rekey(),
)
}
}
impl CanonicalKeys for CrossDecoderBlock {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest()
.rename("attn_norm.", "attn.pre.")
.rename("attn.", "attn.branch.")
.rename("cross_attn_norm.", "cross_attn.pre.")
.rename("cross_attn.", "cross_attn.branch.")
.rename("mlp_norm.", "mlp.pre.")
.rename("mlp.", "mlp.branch.")
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::Shape,
tensor::Distribution,
};
use super::*;
use crate::{
contracts::assert_shape_contract,
ops::transformers::attention::{
cross_attn,
self_attn,
},
support::testing::DeviceMemoryGuard,
};
#[test]
#[serial_test::serial]
fn test_cross_decoder_forward() {
use crate::support::testing::performance_device;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let cfg = CrossDecoderBlockConfig::new(d_model);
let n_heads = cfg.n_heads();
assert_eq!(cfg.d_model(), d_model);
assert_eq!(cfg.n_heads(), n_heads);
let block: CrossDecoderBlock = cfg.init(&device);
assert_eq!(block.d_model(), d_model);
assert_eq!(block.n_heads(), n_heads);
for attn in [&block.attn.branch, &block.cross_attn.branch] {
assert!(attn.key.bias.is_none());
assert!(attn.query.bias.is_some());
assert!(attn.value.bias.is_some());
assert!(attn.output.bias.is_some());
}
let batch = 2;
let seq_len = 10;
let shape: Shape = [batch, seq_len, d_model].into();
let x: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let xa: Tensor<3> = Tensor::random(shape.clone(), Distribution::Default, &device);
let mask: Tensor<3> =
Tensor::random([1, seq_len, seq_len], Distribution::Bernoulli(0.5), &device);
let mask = mask.bool();
let result = block.forward(x.clone(), xa.clone(), mask.clone().into());
assert_eq!(result.batch_size(), batch);
assert_eq!(result.d_model(), d_model);
assert_eq!(result.seq_len(), seq_len);
let expected = {
let pre = |s: &ResidualSublayer<MultiHeadAttention>, x: Tensor<3>| {
s.pre.as_ref().unwrap().forward(x)
};
let self_out = self_attn(
&block.attn.branch,
pre(&block.attn, x.clone()),
mask.clone().into(),
);
let x = x + self_out.context;
let cross_out = cross_attn(
&block.cross_attn.branch,
pre(&block.cross_attn, x.clone()),
xa.clone(),
);
let x = x + cross_out.context;
let x = x.clone()
+ block
.mlp
.branch
.forward(block.mlp.pre.as_ref().unwrap().forward(x));
CrossDecoderRecord {
output: x,
ca_weights: cross_out.weights,
}
};
result
.output
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.output.to_data(), Default::default());
result
.ca_weights
.clone()
.into_data()
.assert_approx_eq::<f64>(&expected.ca_weights.to_data(), Default::default());
assert_shape_contract!(
["batch", "seq_len", "d_model"],
&result.output,
&[("batch", batch), ("seq_len", seq_len), ("d_model", d_model),],
);
assert_shape_contract!(
["batch", "n_heads", "seq_len", "seq_len"],
&result.ca_weights,
&[("batch", batch), ("n_heads", n_heads), ("seq_len", seq_len)],
);
}
}