use alloc::{
string::ToString,
vec::Vec,
};
use burn::{
Tensor,
config::Config,
module::Module,
tensor::Device,
};
use crate::{
blocks::{
layout::BoundaryPermutes,
signals::{
EncoderSubsampling,
EncoderSubsamplingConfig,
EncoderSubsamplingMeta,
},
transformers::encoders::{
EncoderTransformer,
EncoderTransformerConfig,
EncoderTransformerMeta,
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
},
};
pub trait SubsampledEncoderMeta {
fn n_mels(&self) -> usize;
fn d_model(&self) -> usize;
fn subsampling_factor(&self) -> usize;
fn n_layers(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct SubsampledEncoderConfig {
pub subsampling: EncoderSubsamplingConfig,
pub transformer: EncoderTransformerConfig,
#[config(default = "BoundaryPermutes::IDENTITY")]
pub layout: BoundaryPermutes,
}
impl SubsampledEncoderMeta for SubsampledEncoderConfig {
fn n_mels(&self) -> usize {
self.subsampling.n_input()
}
fn d_model(&self) -> usize {
self.transformer.d_model()
}
fn subsampling_factor(&self) -> usize {
self.subsampling.factor()
}
fn n_layers(&self) -> usize {
self.transformer.n_layers()
}
}
impl ModuleInit<SubsampledEncoder> for SubsampledEncoderConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<SubsampledEncoder> {
if self.subsampling.d_model() != self.transformer.d_model() {
return Err(ConstraintError::new(
"SubsampledEncoderConfig",
"",
Rule::Relation {
lhs: (
"subsampling.d_model()".into(),
self.subsampling.d_model().to_string(),
),
op: "==",
rhs: (
"transformer.d_model()".into(),
self.transformer.d_model().to_string(),
),
},
)
.into());
}
Ok(SubsampledEncoder {
subsampling: self.subsampling.try_init(device)?,
transformer: self.transformer.try_init(device)?,
layout: self.layout,
})
}
}
#[derive(Module, Debug)]
pub struct SubsampledEncoder {
pub subsampling: EncoderSubsampling,
pub transformer: EncoderTransformer,
#[module(skip)]
pub layout: BoundaryPermutes,
}
impl SubsampledEncoderMeta for SubsampledEncoder {
fn n_mels(&self) -> usize {
self.subsampling.n_input()
}
fn d_model(&self) -> usize {
self.transformer.d_model()
}
fn subsampling_factor(&self) -> usize {
self.subsampling.factor()
}
fn n_layers(&self) -> usize {
self.transformer.n_layers()
}
}
impl SubsampledEncoder {
pub fn output_len(
&self,
len: usize,
) -> usize {
self.subsampling.output_len(len)
}
pub fn forward(
&self,
features: Tensor<3>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
let padded = lengths.is_some();
let (x, lengths) = self.pre_encode(features, lengths);
let x = self.forward_pre_encoded(x, padded.then_some(lengths.as_slice()));
(x, lengths)
}
pub fn pre_encode(
&self,
features: Tensor<3>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
self.subsampling
.forward(self.layout.pre_apply(features), lengths)
}
pub fn forward_pre_encoded(
&self,
embs: Tensor<3>,
lengths: Option<&[usize]>,
) -> Tensor<3> {
self.layout
.post_apply(self.transformer.forward(embs, lengths))
}
}
impl CanonicalKeys for SubsampledEncoder {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("transformer.", &self.transformer)
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use super::*;
use crate::{
blocks::{
signals::{
DwStridingSubsamplingConfig,
FeatureStackingConfig,
},
transformers::encoders::{
conformer::{
ConformerBlockConfig,
ConformerTransformerConfig,
},
encoder_transformer::test_rope_stack,
},
},
support::testing::cpu_device,
};
fn conformer(layout: BoundaryPermutes) -> SubsampledEncoderConfig {
SubsampledEncoderConfig::new(
EncoderSubsamplingConfig::DwStriding(
DwStridingSubsamplingConfig::new(16, 4, 8).with_n_stages(2),
),
ConformerTransformerConfig::new(ConformerBlockConfig::new(8, 2, 16, 3), 2).into(),
)
.with_layout(layout)
}
#[test]
#[serial_test::serial]
fn test_layout_bridges_the_callers_order() {
let device = cpu_device();
device.seed(5);
let native: SubsampledEncoder = conformer(BoundaryPermutes::IDENTITY).init(&device);
assert_eq!(native.n_mels(), 16);
assert_eq!(native.subsampling_factor(), 4);
assert_eq!(native.n_layers(), 2);
let mels = Tensor::<3>::random([2, 16, 20], Distribution::Default, &device);
let (want, want_lengths) = native.forward(mels.clone(), Some(&[20, 13]));
assert_eq!(want.dims(), [2, native.output_len(20), 8]);
let mut swapped = native.clone();
swapped.layout = BoundaryPermutes::SWAP_IN;
let (got, got_lengths) = swapped.forward(mels.swap_dims(1, 2), Some(&[20, 13]));
assert_eq!(got_lengths, want_lengths);
got.to_data()
.assert_approx_eq::<f32>(&want.to_data(), Tolerance::absolute(1e-6));
}
#[test]
#[serial_test::serial]
#[cfg(feature = "store")]
fn test_paths() {
use burn::store::ModuleSnapshot;
let device = cpu_device();
let stacking = SubsampledEncoderConfig::new(
EncoderSubsamplingConfig::Stacking(FeatureStackingConfig::new(6, 8).with_factor(2)),
test_rope_stack(8, 2, 1).into(),
);
for config in [conformer(BoundaryPermutes::SWAP_IN), stacking] {
let encoder: SubsampledEncoder = config.init(&device);
let paths: Vec<String> = encoder
.collect(None, None, true)
.into_iter()
.map(|t| t.name)
.collect();
assert!(!paths.is_empty());
assert!(
paths
.iter()
.all(|p| p.starts_with("subsampling.") || p.starts_with("transformer.")),
"{paths:?}"
);
assert!(paths.iter().any(|p| p.starts_with("transformer.layers.0.")));
}
}
#[test]
#[serial_test::serial]
fn test_widths_must_agree() {
let config = SubsampledEncoderConfig::new(
EncoderSubsamplingConfig::Stacking(FeatureStackingConfig::new(6, 16).with_factor(2)),
test_rope_stack(8, 2, 1).into(),
);
assert!(config.try_init(&cpu_device()).is_err());
}
}