use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
},
tensor::{
Device,
activation::silu,
},
};
use crate::{
blocks::transformers::sublayer::SublayerBranch,
burner::module::ModuleInit,
errors::BunsenResult,
};
pub trait ConformerFeedForwardMeta {
fn d_model(&self) -> usize;
fn d_ff(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct ConformerFeedForwardConfig {
pub d_model: usize,
pub d_ff: usize,
#[config(default = "true")]
pub bias: bool,
}
impl ConformerFeedForwardMeta for ConformerFeedForwardConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn d_ff(&self) -> usize {
self.d_ff
}
}
impl ModuleInit<ConformerFeedForward> for ConformerFeedForwardConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ConformerFeedForward> {
Ok(ConformerFeedForward {
linear1: LinearConfig::new(self.d_model, self.d_ff)
.with_bias(self.bias)
.init(device),
linear2: LinearConfig::new(self.d_ff, self.d_model)
.with_bias(self.bias)
.init(device),
})
}
}
#[derive(Module, Debug)]
pub struct ConformerFeedForward {
pub linear1: Linear,
pub linear2: Linear,
}
impl ConformerFeedForwardMeta for ConformerFeedForward {
fn d_model(&self) -> usize {
self.linear1.weight.dims()[0]
}
fn d_ff(&self) -> usize {
self.linear1.weight.dims()[1]
}
}
impl ConformerFeedForward {
pub fn forward(
&self,
x: Tensor<3>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, time] = crate::contracts::unpack_shape_contract!(
["batch", "time", "d_model"],
&x.dims(),
&["batch", "time"],
&[("d_model", self.d_model())],
);
let y = self.linear2.forward(silu(self.linear1.forward(x)));
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[
("batch", batch),
("time", time),
("d_model", self.d_model())
],
);
y
}
}
impl SublayerBranch<(), 3> for ConformerFeedForward {
type Output = Tensor<3>;
fn call_branch(
&self,
z: Tensor<3>,
(): (),
) -> Tensor<3> {
self.forward(z)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = ConformerFeedForwardConfig::new(6, 10).with_bias(false);
let module: ConformerFeedForward = config.init(&cpu_device());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.d_ff(), config.d_ff());
assert!(module.linear1.bias.is_none());
let y = module.forward(Tensor::zeros([2, 3, 6], &cpu_device()));
assert_eq!(y.dims(), [2, 3, 6]);
}
}