use burn::{
Tensor,
config::Config,
module::Module,
tensor::Device,
};
use crate::{
burner::module::ModuleInit,
errors::BunsenResult,
ops::{
signal::{
StftWindowConfig,
perceptive_audio::{
PaddingMode,
PerceptiveAudioConverter,
PerceptiveAudioConverterMeta,
PerceptiveAudioConverterOptions,
},
},
transformers::attention::length_mask,
},
};
pub trait ParakeetFrontEndMeta {
fn sample_rate(&self) -> usize;
fn hop(&self) -> usize;
fn n_mels(&self) -> usize;
fn frame_len(
&self,
samples: usize,
) -> usize {
samples / self.hop()
}
}
#[derive(Config, Debug)]
pub struct ParakeetFrontEndConfig {
#[config(default = "16000")]
pub sample_rate: usize,
#[config(default = "400")]
pub win_length: usize,
#[config(default = "160")]
pub hop: usize,
#[config(default = "128")]
pub n_mels: usize,
#[config(default = "0.97")]
pub preemphasis: f64,
#[config(default = "5.960464477539063e-8")]
pub log_guard: f64,
#[config(default = "1e-5")]
pub norm_epsilon: f64,
}
impl ParakeetFrontEndMeta for ParakeetFrontEndConfig {
fn sample_rate(&self) -> usize {
self.sample_rate
}
fn hop(&self) -> usize {
self.hop
}
fn n_mels(&self) -> usize {
self.n_mels
}
}
impl ParakeetFrontEndConfig {
pub fn converter_options(&self) -> PerceptiveAudioConverterOptions {
PerceptiveAudioConverterOptions::new()
.with_sample_rate(self.sample_rate)
.with_n_fft(self.win_length)
.with_window(StftWindowConfig::Hann { periodic: false })
.with_pad_to_pow2(true)
.with_n_mels(self.n_mels)
.with_hop(self.hop)
.with_start_padding(PaddingMode::Zero)
.with_end_padding(PaddingMode::Zero)
}
}
impl ModuleInit<ParakeetFrontEnd> for ParakeetFrontEndConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ParakeetFrontEnd> {
Ok(ParakeetFrontEnd {
config: self.clone(),
converter: self.converter_options().try_init(device)?,
})
}
}
#[derive(Module, Debug)]
pub struct ParakeetFrontEnd {
#[module(skip)]
pub config: ParakeetFrontEndConfig,
pub converter: PerceptiveAudioConverter,
}
impl ParakeetFrontEndMeta for ParakeetFrontEnd {
fn sample_rate(&self) -> usize {
self.converter.sample_rate()
}
fn hop(&self) -> usize {
self.converter.hop()
}
fn n_mels(&self) -> usize {
self.converter.n_mels()
}
}
impl ParakeetFrontEnd {
pub fn forward(
&self,
waveforms: Tensor<2>,
lengths: Option<&[usize]>,
) -> (Tensor<3>, Vec<usize>) {
let [batch, samples] = waveforms.dims();
let device = waveforms.device();
let lengths: Vec<usize> = match lengths {
Some(lengths) => {
assert_eq!(lengths.len(), batch, "one length per batch item");
lengths.to_vec()
}
None => vec![samples; batch],
};
let a = self.config.preemphasis;
let x = if a != 0.0 && samples > 1 {
let head = waveforms.clone().slice_dim(1, 0..1);
let rest = waveforms.clone().slice_dim(1, 1..)
- waveforms.slice_dim(1, 0..samples - 1).mul_scalar(a);
Tensor::cat(vec![head, rest], 1)
} else {
waveforms
};
let x = x * length_mask(&lengths, samples, &device).float();
let half = self.config.win_length / 2;
let x = Tensor::cat(
vec![
Tensor::zeros([batch, half], &device),
x,
Tensor::zeros([batch, half], &device),
],
1,
);
let mel = self
.converter
.mel(self.converter.spectrum(self.converter.frame(x)));
let logmel = mel.add_scalar(self.config.log_guard).log();
let [_, frames, _] = logmel.dims();
let frame_lengths: Vec<usize> = lengths.iter().map(|&n| self.frame_len(n)).collect();
let valid = length_mask(&frame_lengths, frames, &device)
.float()
.reshape([batch, frames, 1]);
let counts: Vec<f32> = frame_lengths.iter().map(|&n| n as f32).collect();
let counts = Tensor::<1>::from_floats(counts.as_slice(), &device).reshape([batch, 1, 1]);
let mean = (logmel.clone() * valid.clone()).sum_dim(1) / counts.clone();
let centered = (logmel - mean) * valid.clone();
let variance = centered.clone().powi_scalar(2).sum_dim(1) / (counts - 1.0).clamp_min(1.0);
let std = variance.sqrt().add_scalar(self.config.norm_epsilon);
let features = (centered / std) * valid;
let features = features.swap_dims(1, 2);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "n_mels", "frames"],
&features.dims(),
&[
("batch", batch),
("n_mels", self.n_mels()),
("frames", samples / self.hop() + 1),
],
);
(features, frame_lengths)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta() {
let config = ParakeetFrontEndConfig::new()
.with_hop(80)
.with_n_mels(40)
.with_sample_rate(8000);
let module: ParakeetFrontEnd = config.init(&cpu_device());
assert_eq!(module.sample_rate(), config.sample_rate());
assert_eq!(module.hop(), config.hop());
assert_eq!(module.n_mels(), config.n_mels());
assert_eq!(module.frame_len(1000), 12);
}
#[test]
fn test_shapes() {
let module: ParakeetFrontEnd = ParakeetFrontEndConfig::new().init(&cpu_device());
let (features, lengths) = module.forward(
Tensor::zeros([2, 16000], &cpu_device()),
Some(&[16000, 8000]),
);
assert_eq!(features.dims(), [2, 128, 101]);
assert_eq!(lengths, [100, 50]);
}
#[cfg(feature = "store_safetensors")]
#[test]
fn test_matches_transformers() {
use burn::{
store::{
ModuleStore,
SafetensorsStore,
bridge,
},
tensor::Tolerance,
};
use crate::support::testing::performance_device;
let path = concat!(
env!("CARGO_MANIFEST_DIR"),
"/testdata/parakeet/front_end.safetensors"
);
let mut store = SafetensorsStore::from_file(path);
let all = store.get_all_tensors().expect("read the fixture").clone();
let device = performance_device();
let get = |k: &str| bridge::to_data(&all[k]).expect("read a fixture tensor");
let waveforms = Tensor::<2>::from_data(get("waveforms"), &device);
let lengths: Vec<usize> = get("lengths")
.convert::<f32>()
.iter::<f32>()
.map(|n| n as usize)
.collect();
let expected = Tensor::<3>::from_data(get("features"), &device);
let mask = Tensor::<2>::from_data(get("mask"), &device);
let module: ParakeetFrontEnd = ParakeetFrontEndConfig::new().init(&device);
let (features, frame_lengths) = module.forward(waveforms, Some(&lengths));
let expected_lengths: Vec<usize> = mask
.sum_dim(1)
.into_data()
.iter::<f32>()
.map(|n| n as usize)
.collect();
assert_eq!(frame_lengths, expected_lengths);
features
.swap_dims(1, 2)
.to_data_as::<f32>()
.assert_approx_eq::<f32>(
&expected.to_data_as::<f32>(),
Tolerance::rel_abs(1e-3, 2e-3),
);
}
}