use alloc::{
string::{
String,
ToString,
},
vec::Vec,
};
use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
PaddingConfig2d,
conv::Conv2dConfig,
},
tensor::Device,
};
use crate::{
blocks::conv::{
ConvBlock2d,
ConvBlock2dConfig,
},
burner::module::{
ModuleInit,
Rekey,
TensorLoadTransform,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::transformers::attention::length_mask,
};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum DwStridingFlattenOrder {
#[default]
ChannelsFreqs,
FreqsChannels,
}
pub trait DwStridingSubsamplingMeta {
fn n_mels(&self) -> usize;
fn channels(&self) -> usize;
fn d_model(&self) -> usize;
fn n_stages(&self) -> usize;
fn factor(&self) -> usize {
1 << self.n_stages()
}
fn output_len(
&self,
len: usize,
) -> usize {
(0..self.n_stages()).fold(len, |len, _| if len == 0 { 0 } else { (len - 1) / 2 + 1 })
}
fn n_freq_out(&self) -> usize;
fn flatten(&self) -> DwStridingFlattenOrder;
fn sequential_rekey(
&self,
prefix: &'static str,
) -> Rekey {
let n_stages = self.n_stages();
Rekey::keep_rest().custom(move |name| {
let (k, param) = name.strip_prefix(prefix)?.split_once('.')?;
let module = k
.parse()
.ok()
.and_then(|k| DwStridingSubsampling::module_path(k, n_stages));
Some(module.map(|module| format!("{module}.{param}")))
})
}
fn reflatten_from(
&self,
from: DwStridingFlattenOrder,
rekey: Rekey,
) -> Rekey {
if from == self.flatten() {
return rekey;
}
let (c, f) = (self.channels(), self.n_freq_out());
let (a, b) = match from {
DwStridingFlattenOrder::ChannelsFreqs => (c, f),
DwStridingFlattenOrder::FreqsChannels => (f, c),
};
rekey.transform("linear.weight", TensorLoadTransform::regroup_columns(a, b))
}
}
#[derive(Config, Debug)]
pub struct DwStridingSubsamplingConfig {
pub n_mels: usize,
pub channels: usize,
pub d_model: usize,
#[config(default = "3")]
pub n_stages: usize,
#[config(default = "DwStridingFlattenOrder::ChannelsFreqs")]
pub flatten: DwStridingFlattenOrder,
}
impl DwStridingSubsamplingMeta for DwStridingSubsamplingConfig {
fn n_mels(&self) -> usize {
self.n_mels
}
fn channels(&self) -> usize {
self.channels
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_stages(&self) -> usize {
self.n_stages
}
fn n_freq_out(&self) -> usize {
self.output_len(self.n_mels)
}
fn flatten(&self) -> DwStridingFlattenOrder {
self.flatten
}
}
impl ModuleInit<DwStridingSubsampling> for DwStridingSubsamplingConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<DwStridingSubsampling> {
if self.n_stages == 0 {
return Err(ConstraintError::new(
"DwStridingSubsamplingConfig",
"n_stages",
Rule::ZeroOrEmpty,
)
.into());
}
let c = self.channels;
let strided = |c_in: usize, groups: usize| {
Conv2dConfig::new([c_in, c], [3, 3])
.with_stride([2, 2])
.with_padding(PaddingConfig2d::Explicit(1, 1, 1, 1))
.with_groups(groups)
};
let stem = ConvBlock2dConfig::new(strided(1, 1)).try_init(device)?;
let mut stages = Vec::with_capacity(self.n_stages - 1);
for _ in 1..self.n_stages {
stages.push(DwStridingStage {
depthwise: ConvBlock2dConfig::new(strided(c, c))
.with_act(None)
.try_init(device)?,
pointwise: ConvBlock2dConfig::new(Conv2dConfig::new([c, c], [1, 1]))
.try_init(device)?,
});
}
Ok(DwStridingSubsampling {
stem,
stages,
linear: LinearConfig::new(c * self.n_freq_out(), self.d_model).init(device),
n_mels: self.n_mels,
flatten: self.flatten,
})
}
}
#[derive(Module, Debug)]
pub struct DwStridingStage {
pub depthwise: ConvBlock2d,
pub pointwise: ConvBlock2d,
}
#[derive(Module, Debug)]
pub struct DwStridingSubsampling {
pub stem: ConvBlock2d,
pub stages: Vec<DwStridingStage>,
pub linear: Linear,
#[module(skip)]
pub n_mels: usize,
#[module(skip)]
pub flatten: DwStridingFlattenOrder,
}
impl DwStridingSubsamplingMeta for DwStridingSubsampling {
fn n_mels(&self) -> usize {
self.n_mels
}
fn channels(&self) -> usize {
self.stem.conv.weight.dims()[0]
}
fn d_model(&self) -> usize {
self.linear.weight.dims()[1]
}
fn n_stages(&self) -> usize {
self.stages.len() + 1
}
fn n_freq_out(&self) -> usize {
self.linear.weight.dims()[0] / self.channels()
}
fn flatten(&self) -> DwStridingFlattenOrder {
self.flatten
}
}
impl DwStridingSubsampling {
fn module_path(
k: usize,
n_stages: usize,
) -> Option<String> {
match k {
0 => Some("stem.conv".to_string()),
k if k >= 2 && (k - 2) / 3 + 1 < n_stages => match (k - 2) % 3 {
0 => Some(format!("stages.{}.depthwise.conv", (k - 2) / 3)),
1 => Some(format!("stages.{}.pointwise.conv", (k - 2) / 3)),
_ => None,
},
_ => None,
}
}
pub fn forward(
&self,
features: Tensor<3>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
let [batch, _, time] = features.dims();
let device = features.device();
let mut lengths: Vec<usize> = match lengths {
Some(lengths) => {
assert_eq!(lengths.len(), batch, "one length per batch item");
lengths.to_vec()
}
None => vec![time; batch],
};
let step = |lengths: &mut Vec<usize>| {
for len in lengths.iter_mut() {
*len = if *len == 0 { 0 } else { (*len - 1) / 2 + 1 };
}
};
let masked = |x: Tensor<4>, lengths: &[usize]| {
let t = x.dims()[2];
x * length_mask(lengths, t, &device)
.float()
.reshape([batch as i32, 1, t as i32, 1])
};
let mut x = features.swap_dims(1, 2).unsqueeze_dim::<4>(1);
x = self.stem.forward(x);
step(&mut lengths);
x = masked(x, &lengths);
for stage in &self.stages {
x = stage.depthwise.forward(x);
step(&mut lengths);
x = masked(x, &lengths);
x = masked(stage.pointwise.forward(x), &lengths);
}
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "channels", "time", "n_freq"],
&x.dims(),
&[
("batch", batch),
("channels", self.channels()),
("time", self.output_len(time)),
("n_freq", self.n_freq_out()),
],
);
let [_, c, t, f] = x.dims();
let x = match self.flatten {
DwStridingFlattenOrder::ChannelsFreqs => x.swap_dims(1, 2).reshape([batch, t, c * f]),
DwStridingFlattenOrder::FreqsChannels => {
x.permute([0, 2, 3, 1]).reshape([batch, t, f * c])
}
};
let y = self.linear.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[
("batch", batch),
("time", self.output_len(time)),
("d_model", self.d_model()),
],
);
(y, lengths)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = DwStridingSubsamplingConfig::new(24, 4, 16);
assert_eq!(config.factor(), 8);
assert_eq!(config.n_freq_out(), 3);
assert_eq!(config.output_len(53), 7);
assert_eq!(config.output_len(37), 5);
assert_eq!(config.output_len(8), 1);
assert_eq!(config.output_len(0), 0);
let module: DwStridingSubsampling = config.init(&cpu_device());
assert_eq!(module.n_mels(), 24);
assert_eq!(module.n_mels(), config.n_mels());
assert_eq!(module.channels(), config.channels());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_stages(), config.n_stages());
assert_eq!(module.n_freq_out(), config.n_freq_out());
let (y, lengths) =
module.forward(Tensor::zeros([2, 24, 53], &cpu_device()), Some(&[53, 37]));
assert_eq!(y.dims(), [2, 7, 16]);
assert_eq!(lengths, [7, 5]);
}
}