use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNorm,
LayerNormConfig,
},
prelude::Bool,
tensor::Device,
};
use super::{
ConformerConvolution,
ConformerConvolutionConfig,
ConformerFeedForward,
ConformerFeedForwardConfig,
};
use crate::{
blocks::transformers::attention::rel_pos::{
RelPosSelfAttention,
RelPosSelfAttentionConfig,
RelPosSelfAttentionMeta,
},
burner::module::ModuleInit,
errors::BunsenResult,
};
pub trait ConformerBlockMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct ConformerBlockConfig {
pub d_model: usize,
pub n_heads: usize,
pub d_ff: usize,
pub conv_kernel_size: usize,
#[config(default = "true")]
pub bias: bool,
#[config(default = "true")]
pub conv_bias: bool,
}
impl ConformerBlockMeta for ConformerBlockConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.n_heads
}
}
impl ModuleInit<ConformerBlock> for ConformerBlockConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ConformerBlock> {
let norm = || LayerNormConfig::new(self.d_model).init(device);
let ff = || {
ConformerFeedForwardConfig::new(self.d_model, self.d_ff)
.with_bias(self.bias)
.try_init(device)
};
Ok(ConformerBlock {
norm_feed_forward1: norm(),
feed_forward1: ff()?,
norm_self_att: norm(),
self_attn: RelPosSelfAttentionConfig::new(self.d_model, self.n_heads)
.with_bias(self.bias)
.try_init(device)?,
norm_conv: norm(),
conv: ConformerConvolutionConfig::new(self.d_model, self.conv_kernel_size)
.with_bias(self.conv_bias)
.try_init(device)?,
norm_feed_forward2: norm(),
feed_forward2: ff()?,
norm_out: norm(),
})
}
}
#[derive(Module, Debug)]
pub struct ConformerBlock {
pub norm_feed_forward1: LayerNorm,
pub feed_forward1: ConformerFeedForward,
pub norm_self_att: LayerNorm,
pub self_attn: RelPosSelfAttention,
pub norm_conv: LayerNorm,
pub conv: ConformerConvolution,
pub norm_feed_forward2: LayerNorm,
pub feed_forward2: ConformerFeedForward,
pub norm_out: LayerNorm,
}
impl ConformerBlockMeta for ConformerBlock {
fn d_model(&self) -> usize {
self.self_attn.d_model()
}
fn n_heads(&self) -> usize {
self.self_attn.n_heads()
}
}
impl ConformerBlock {
pub fn forward(
&self,
x: Tensor<3>,
positions: Tensor<3>,
valid: Option<Tensor<2, Bool>>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, time] = crate::contracts::unpack_shape_contract!(
["batch", "time", "d_model"],
&x.dims(),
&["batch", "time"],
&[("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
[1, 2 * "time" - 1, "d_model"],
&positions.dims(),
&[("time", time), ("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
if let Some(valid) = &valid {
crate::contracts::assert_shape_contract_periodically!(
["batch", "time"],
&valid.dims(),
&[("batch", batch), ("time", time)],
);
}
let x = x.clone()
+ self
.feed_forward1
.forward(self.norm_feed_forward1.forward(x))
.mul_scalar(0.5);
let x = x.clone()
+ self
.self_attn
.forward(self.norm_self_att.forward(x), positions, valid.clone());
let x = x.clone() + self.conv.forward(self.norm_conv.forward(x), valid);
let x = x.clone()
+ self
.feed_forward2
.forward(self.norm_feed_forward2.forward(x))
.mul_scalar(0.5);
let y = self.norm_out.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[
("batch", batch),
("time", time),
("d_model", self.d_model())
],
);
y
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = ConformerBlockConfig::new(8, 2, 16, 3)
.with_bias(false)
.with_conv_bias(false);
let module: ConformerBlock = config.init(&cpu_device());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_heads(), config.n_heads());
}
}