use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
PaddingConfig2d,
conv::Conv2dConfig,
},
tensor::Device,
};
use crate::{
blocks::conv::{
ConvBlock2d,
ConvBlock2dConfig,
},
burner::module::ModuleInit,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::transformers::attention::length_mask,
};
pub trait DwStridingSubsamplingMeta {
fn channels(&self) -> usize;
fn d_model(&self) -> usize;
fn n_stages(&self) -> usize;
fn factor(&self) -> usize {
1 << self.n_stages()
}
fn output_len(
&self,
len: usize,
) -> usize {
(0..self.n_stages()).fold(len, |len, _| if len == 0 { 0 } else { (len - 1) / 2 + 1 })
}
fn n_freq_out(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct DwStridingSubsamplingConfig {
pub n_mels: usize,
pub channels: usize,
pub d_model: usize,
#[config(default = "3")]
pub n_stages: usize,
}
impl DwStridingSubsamplingMeta for DwStridingSubsamplingConfig {
fn channels(&self) -> usize {
self.channels
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_stages(&self) -> usize {
self.n_stages
}
fn n_freq_out(&self) -> usize {
self.output_len(self.n_mels)
}
}
impl ModuleInit<DwStridingSubsampling> for DwStridingSubsamplingConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<DwStridingSubsampling> {
if self.n_stages == 0 {
return Err(ConstraintError::new(
"DwStridingSubsamplingConfig",
"n_stages",
Rule::ZeroOrEmpty,
)
.into());
}
let c = self.channels;
let strided = |c_in: usize, groups: usize| {
Conv2dConfig::new([c_in, c], [3, 3])
.with_stride([2, 2])
.with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1))
.with_groups(groups)
};
let stem = ConvBlock2dConfig::new(strided(1, 1)).try_init(device)?;
let mut stages = Vec::with_capacity(self.n_stages - 1);
for _ in 1..self.n_stages {
stages.push(DwStridingStage {
depthwise: ConvBlock2dConfig::new(strided(c, c))
.with_act(None)
.try_init(device)?,
pointwise: ConvBlock2dConfig::new(Conv2dConfig::new([c, c], [1, 1]))
.try_init(device)?,
});
}
Ok(DwStridingSubsampling {
stem,
stages,
linear: LinearConfig::new(c * self.n_freq_out(), self.d_model).init(device),
})
}
}
#[derive(Module, Debug)]
pub struct DwStridingStage {
pub depthwise: ConvBlock2d,
pub pointwise: ConvBlock2d,
}
#[derive(Module, Debug)]
pub struct DwStridingSubsampling {
pub stem: ConvBlock2d,
pub stages: Vec<DwStridingStage>,
pub linear: Linear,
}
impl DwStridingSubsamplingMeta for DwStridingSubsampling {
fn channels(&self) -> usize {
self.stem.conv.weight.dims()[0]
}
fn d_model(&self) -> usize {
self.linear.weight.dims()[1]
}
fn n_stages(&self) -> usize {
self.stages.len() + 1
}
fn n_freq_out(&self) -> usize {
self.linear.weight.dims()[0] / self.channels()
}
}
impl DwStridingSubsampling {
pub fn forward(
&self,
features: Tensor<3>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
let [batch, _, time] = features.dims();
let device = features.device();
let mut lengths: Vec<usize> = match lengths {
Some(lengths) => {
assert_eq!(lengths.len(), batch, "one length per batch item");
lengths.to_vec()
}
None => vec![time; batch],
};
let step = |lengths: &mut Vec<usize>| {
for len in lengths.iter_mut() {
*len = if *len == 0 { 0 } else { (*len - 1) / 2 + 1 };
}
};
let masked = |x: Tensor<4>, lengths: &[usize]| {
let t = x.dims()[2];
x * length_mask(lengths, t, &device)
.float()
.reshape([batch as i32, 1, t as i32, 1])
};
let mut x = features.swap_dims(1, 2).unsqueeze_dim::<4>(1);
x = self.stem.forward(x);
step(&mut lengths);
x = masked(x, &lengths);
for stage in &self.stages {
x = stage.depthwise.forward(x);
step(&mut lengths);
x = masked(x, &lengths);
x = masked(stage.pointwise.forward(x), &lengths);
}
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "channels", "time", "n_freq"],
&x.dims(),
&[
("batch", batch),
("channels", self.channels()),
("time", self.output_len(time)),
("n_freq", self.n_freq_out()),
],
);
let [_, c, t, f] = x.dims();
let x = x.swap_dims(1, 2).reshape([batch, t, c * f]);
let y = self.linear.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[
("batch", batch),
("time", self.output_len(time)),
("d_model", self.d_model()),
],
);
(y, lengths)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = DwStridingSubsamplingConfig::new(24, 4, 16);
assert_eq!(config.factor(), 8);
assert_eq!(config.n_freq_out(), 3);
assert_eq!(config.output_len(53), 7);
assert_eq!(config.output_len(37), 5);
assert_eq!(config.output_len(8), 1);
assert_eq!(config.output_len(0), 0);
let module: DwStridingSubsampling = config.init(&cpu_device());
assert_eq!(module.channels(), config.channels());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_stages(), config.n_stages());
assert_eq!(module.n_freq_out(), config.n_freq_out());
let (y, lengths) =
module.forward(Tensor::zeros([2, 24, 53], &cpu_device()), Some(&[53, 37]));
assert_eq!(y.dims(), [2, 7, 16]);
assert_eq!(lengths, [7, 5]);
}
}