use burn::{
Tensor,
config::Config,
module::Module,
tensor::Device,
};
use super::{
ConformerBlock,
ConformerBlockConfig,
DwStridingSubsampling,
DwStridingSubsamplingConfig,
DwStridingSubsamplingMeta,
};
use crate::{
burner::module::ModuleInit,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::transformers::attention::{
length_mask,
rel_position_table,
},
};
pub trait ConformerEncoderMeta {
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
fn subsampling_factor(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct ConformerEncoderConfig {
pub subsampling: DwStridingSubsamplingConfig,
pub block: ConformerBlockConfig,
pub n_layers: usize,
#[config(default = "false")]
pub scale_input: bool,
}
impl ConformerEncoderMeta for ConformerEncoderConfig {
fn d_model(&self) -> usize {
self.block.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn subsampling_factor(&self) -> usize {
self.subsampling.factor()
}
}
impl ModuleInit<ConformerEncoder> for ConformerEncoderConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ConformerEncoder> {
if self.subsampling.d_model != self.block.d_model {
return Err(ConstraintError::new(
"ConformerEncoderConfig",
"",
Rule::Relation {
lhs: (
"subsampling.d_model".into(),
self.subsampling.d_model.to_string(),
),
op: "==",
rhs: ("block.d_model".into(), self.block.d_model.to_string()),
},
)
.into());
}
Ok(ConformerEncoder {
subsampling: self.subsampling.try_init(device)?,
layers: (0..self.n_layers)
.map(|_| self.block.try_init(device))
.collect::<BunsenResult<_>>()?,
scale_input: self.scale_input,
})
}
}
#[derive(Module, Debug)]
pub struct ConformerEncoder {
pub subsampling: DwStridingSubsampling,
pub layers: Vec<ConformerBlock>,
pub scale_input: bool,
}
impl ConformerEncoderMeta for ConformerEncoder {
fn d_model(&self) -> usize {
self.subsampling.d_model()
}
fn n_layers(&self) -> usize {
self.layers.len()
}
fn subsampling_factor(&self) -> usize {
self.subsampling.factor()
}
}
impl ConformerEncoder {
pub fn output_len(
&self,
len: usize,
) -> usize {
self.subsampling.output_len(len)
}
pub fn forward(
&self,
features: Tensor<3>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
#[cfg(any(debug_assertions, test))]
let [batch, time_in] = crate::contracts::unpack_shape_contract!(
["batch", "n_mels", "time"],
&features.dims(),
&["batch", "time"],
);
let padded = lengths.is_some();
let (mut x, lengths) = self.subsampling.forward(features, lengths);
if self.scale_input {
x = x.mul_scalar((self.d_model() as f64).sqrt());
}
let [_, time, d_model] = x.dims();
let device = x.device();
let positions = rel_position_table(time, d_model, &device);
let valid = padded.then(|| length_mask(&lengths, time, &device));
for layer in &self.layers {
x = layer.forward(x, positions.clone(), valid.clone());
}
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&x.dims(),
&[
("batch", batch),
("time", self.output_len(time_in)),
("d_model", self.d_model()),
],
);
(x, lengths)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(24, 4, 16),
ConformerBlockConfig::new(16, 2, 32, 5),
2,
);
let module: ConformerEncoder = config.init(&cpu_device());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_layers(), config.n_layers());
assert_eq!(module.subsampling_factor(), config.subsampling_factor());
let (y, lengths) = module.forward(Tensor::zeros([1, 24, 20], &cpu_device()), None);
assert_eq!(y.dims(), [1, 3, 16]);
assert_eq!(lengths, [3]);
}
#[test]
fn test_widths_must_agree() {
let config = ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(24, 4, 12),
ConformerBlockConfig::new(16, 2, 32, 5),
2,
);
assert!(ModuleInit::<ConformerEncoder>::try_init(&config, &cpu_device()).is_err());
}
}