use burn::{
Tensor,
config::Config,
module::Module,
nn::LayerNormConfig,
prelude::Bool,
tensor::Device,
};
use super::{
ConformerConvolution,
ConformerConvolutionConfig,
ConformerFeedForward,
ConformerFeedForwardConfig,
};
use crate::{
blocks::{
norm::{
BunsenNormalization,
BunsenNormalizationConfig,
},
transformers::{
attention::rel_pos::{
RelPosAttnSublayer,
RelPosSelfAttention,
RelPosSelfAttentionConfig,
RelPosSelfAttentionMeta,
},
encoders::conformer::ConformerConvSublayer,
sublayer::{
ResidualSublayer,
ResidualSublayerConfig,
},
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::BunsenResult,
};
pub trait ConformerBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct ConformerBlockConfig {
pub d_model: usize,
pub n_heads: usize,
pub d_ff: usize,
pub conv_kernel_size: usize,
#[config(default = "true")]
pub bias: bool,
#[config(default = "true")]
pub conv_bias: bool,
#[config(default = "BunsenNormalizationConfig::Layer(LayerNormConfig::new(0))")]
pub norm: BunsenNormalizationConfig,
}
impl ConformerBlockMeta for ConformerBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.n_heads
}
}
impl ModuleInit<ConformerBlock> for ConformerBlockConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ConformerBlock> {
let norm = || {
self.norm
.clone()
.with_num_features(self.d_model)
.init(device)
};
let ff = || {
ConformerFeedForwardConfig::new(self.d_model, self.d_ff)
.with_bias(self.bias)
.try_init(device)
};
let ff_sublayer = ResidualSublayerConfig::pre_norm(Some(self.norm.clone()));
Ok(ConformerBlock {
feed_forward1: ff_sublayer.init(ff()?, self.d_model, device),
attn: ff_sublayer.init(
RelPosSelfAttentionConfig::new(self.d_model, self.n_heads)
.with_bias(self.bias)
.try_init(device)?,
self.d_model,
device,
),
conv: ff_sublayer.init(
ConformerConvolutionConfig::new(self.d_model, self.conv_kernel_size)
.with_bias(self.conv_bias)
.try_init(device)?,
self.d_model,
device,
),
feed_forward2: ff_sublayer.init(ff()?, self.d_model, device),
out_norm: norm(),
})
}
}
#[derive(Module, Debug)]
pub struct ConformerBlock {
pub feed_forward1: ResidualSublayer<ConformerFeedForward>,
pub attn: ResidualSublayer<RelPosSelfAttention>,
pub conv: ResidualSublayer<ConformerConvolution>,
pub feed_forward2: ResidualSublayer<ConformerFeedForward>,
pub out_norm: BunsenNormalization,
}
impl ConformerBlockMeta for ConformerBlock {
fn d_model(&self) -> usize {
self.attn.branch.d_model()
}
fn n_heads(&self) -> usize {
self.attn.branch.n_heads()
}
}
impl ConformerBlock {
pub fn forward(
&self,
x: Tensor<3>,
positions: Tensor<3>,
valid: Option<Tensor<2, Bool>>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, time] = crate::contracts::unpack_shape_contract!(
["batch", "time", "d_model"],
&x.dims(),
&["batch", "time"],
&[("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
[1, 2 * "time" - 1, "d_model"],
&positions.dims(),
&[("time", time), ("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
if let Some(valid) = &valid {
crate::contracts::assert_shape_contract_periodically!(
["batch", "time"],
&valid.dims(),
&[("batch", batch), ("time", time)],
);
}
let x = self
.feed_forward1
.residual_update_with(x, |ff, z| ff.forward(z).mul_scalar(0.5));
let x = self.attn.rel_pos_attn(x, positions, valid.clone());
let x = self.conv.conformer_conv(x, valid);
let x = self
.feed_forward2
.residual_update_with(x, |ff, z| ff.forward(z).mul_scalar(0.5));
let y = self.out_norm.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[
("batch", batch),
("time", time),
("d_model", self.d_model())
],
);
y
}
}
impl CanonicalKeys for ConformerBlock {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest()
.rename("feed_forward1_norm.", "feed_forward1.pre.")
.rename("feed_forward1.", "feed_forward1.branch.")
.rename("attn_norm.", "attn.pre.")
.rename("attn.", "attn.branch.")
.rename("conv_norm.", "conv.pre.")
.rename("conv.", "conv.branch.")
.rename("feed_forward2_norm.", "feed_forward2.pre.")
.rename("feed_forward2.", "feed_forward2.branch.")
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = ConformerBlockConfig::new(8, 2, 16, 3)
.with_bias(false)
.with_conv_bias(false);
let module: ConformerBlock = config.init(&cpu_device());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_heads(), config.n_heads());
}
}