use burn::{
Tensor,
config::Config,
tensor::Device,
};
use crate::{
burner::module::ModuleInit,
errors::{
BunsenError,
BunsenResult,
ConstraintError,
},
kits::speech::{
asr_driver::{
LevelStatistic,
MaxLevel,
},
whisper::driver::support::drop_last_frame,
},
ops::signal::perceptive_audio::{
AffineCompress,
PerceptiveAudioConverter,
PerceptiveAudioConverterOptions,
},
};
#[derive(Config, Debug, PartialEq)]
pub struct WhisperFrontEndConfig {
#[config(default = "16_000")]
pub sample_rate: usize,
#[config(default = "10")]
pub hop_ms: usize,
#[config(default = "25")]
pub window_ms: usize,
#[config(default = "8.0")]
pub range_clamp_db: f64,
}
impl WhisperFrontEndConfig {
pub fn hop(&self) -> usize {
self.sample_rate * self.hop_ms / 1000
}
pub fn n_fft(&self) -> usize {
self.sample_rate * self.window_ms / 1000
}
pub fn validate(&self) -> BunsenResult<()> {
const OWNER: &str = "WhisperFrontEndConfig";
for (field, value) in [
("sample_rate", self.sample_rate),
("hop_ms", self.hop_ms),
("window_ms", self.window_ms),
] {
if value == 0 {
return Err(ConstraintError::zero_or_empty(OWNER, field).into());
}
}
for (field, what, ms) in [
("hop_ms", "hop", self.hop_ms),
("window_ms", "window", self.window_ms),
] {
if !(self.sample_rate * ms).is_multiple_of(1000) {
return Err(BunsenError::illegal(format!(
"{OWNER}.{field}: a {ms} ms {what} is not a whole number of samples at {} Hz",
self.sample_rate,
)));
}
}
Ok(())
}
pub fn try_init_audio_converter(
&self,
n_mels: usize,
device: &Device,
) -> BunsenResult<PerceptiveAudioConverter> {
self.mel_converter_options(n_mels)?.try_init(device)
}
pub fn mel_converter_options(
&self,
n_mels: usize,
) -> BunsenResult<PerceptiveAudioConverterOptions> {
self.validate()?;
Ok(PerceptiveAudioConverterOptions::default()
.with_sample_rate(self.sample_rate)
.with_n_fft(self.n_fft())
.with_hop(self.hop())
.with_n_mels(n_mels))
}
pub fn package_window(
&self,
window: Tensor<3>,
reference: Tensor<1>,
) -> Tensor<3> {
let [batch, _, _] = window.dims();
assert_eq!(reference.dims(), [batch], "one reference per batch row");
let floor = reference
.sub_scalar(self.range_clamp_db)
.reshape([batch, 1, 1])
.expand(window.dims());
let floored = window.max_pair(floor);
AffineCompress::default()
.apply(floored)
.swap_dims(1, 2)
}
pub fn package_mels(
&self,
joined: Tensor<3>,
) -> Tensor<3> {
let window = drop_last_frame(joined);
let reference = MaxLevel::of(window.clone()).max;
self.package_window(window, reference)
}
}
#[cfg(test)]
mod tests {
use burn::{
prelude::TensorData,
tensor::Tolerance,
};
use super::*;
use crate::{
kits::speech::whisper::blocks::{
AUDIO_ENCODER_STRIDE,
WhisperTokenLayoutConfig,
},
support::testing::{
assert_tensor_close_to_vec,
cpu_device,
},
};
#[test]
fn test_defaults_are_upstreams() {
let front_end = WhisperFrontEndConfig::new();
assert_eq!(front_end.sample_rate, 16_000);
assert_eq!((front_end.hop(), front_end.n_fft()), (160, 400));
assert_eq!(front_end.range_clamp_db, 8.0);
assert!(front_end.validate().is_ok());
}
#[test]
fn test_grid_scales_with_the_rate() {
let at = |rate: usize| WhisperFrontEndConfig::new().with_sample_rate(rate);
assert_eq!((at(8_000).hop(), at(8_000).n_fft()), (80, 200));
assert!(at(8_000).validate().is_ok());
assert_eq!((at(48_000).hop(), at(48_000).n_fft()), (480, 1_200));
assert!(at(44_100).validate().is_err(), "not a whole hop");
assert!(at(0).validate().is_err());
assert!(at(16_000).with_hop_ms(0).validate().is_err());
assert!(
at(16_000).with_window_ms(3).validate().is_ok(),
"3 ms is 48 whole samples"
);
assert!(
at(16_000)
.with_hop_ms(3)
.with_window_ms(7)
.validate()
.is_ok(),
"any whole-sample grid passes; the 10 ms / 25 ms pairing is convention, not checked"
);
assert!(
at(22_050).with_hop_ms(10).validate().is_err(),
"220.5 samples"
);
}
fn front_end() -> WhisperFrontEndConfig {
WhisperFrontEndConfig::new()
}
fn package_mels(joined: Tensor<3>) -> Tensor<3> {
front_end().package_mels(joined)
}
fn package_window(
window: Tensor<3>,
reference: Tensor<1>,
) -> Tensor<3> {
front_end().package_window(window, reference)
}
#[test]
fn test_mel_options_derives_the_grid_from_the_rate() {
let base = PerceptiveAudioConverterOptions::default();
let opts = front_end().mel_converter_options(80).unwrap();
assert_eq!(opts.sample_rate, base.sample_rate);
assert_eq!(opts.n_mels, base.n_mels);
assert_eq!(opts.n_fft, base.n_fft);
assert_eq!(opts.hop, base.hop);
let opts = front_end()
.with_sample_rate(8_000)
.mel_converter_options(128)
.unwrap();
assert_eq!(opts.sample_rate, 8_000);
assert_eq!(opts.n_mels, 128);
assert_eq!(opts.n_fft, 200);
assert_eq!(opts.hop, 80);
assert_eq!(opts.window, base.window);
assert_eq!(opts.mel_scale, base.mel_scale);
assert_eq!(opts.filter_norm, base.filter_norm);
assert_eq!(opts.log_base, base.log_base);
let opts = front_end()
.with_sample_rate(48_000)
.mel_converter_options(80)
.unwrap();
assert_eq!((opts.n_fft, opts.hop), (1_200, 480));
let opts = front_end()
.with_hop_ms(20)
.with_window_ms(50)
.mel_converter_options(80)
.unwrap();
assert_eq!((opts.n_fft, opts.hop), (800, 320));
assert!(
front_end()
.with_sample_rate(44_100)
.mel_converter_options(80)
.is_err(),
"not a whole hop"
);
assert!(
front_end()
.with_sample_rate(0)
.mel_converter_options(80)
.is_err()
);
}
#[test]
fn test_hop_is_one_timestamp_step_per_encoder_position() {
let hop_seconds = front_end().hop_ms as f64 / 1000.0;
let step_seconds =
WhisperTokenLayoutConfig::new().timestamp_step_seconds / AUDIO_ENCODER_STRIDE as f64;
assert!((hop_seconds - step_seconds).abs() < 1e-12);
}
#[test]
fn test_package_window_uses_the_configured_range() {
let device = cpu_device();
let window: Tensor<3> =
Tensor::from_data(TensorData::new(vec![0.0_f64, -20.0], [1, 2, 1]), &device);
let reference: Tensor<1> = Tensor::from_data(TensorData::new(vec![0.0_f64], [1]), &device);
assert_tensor_close_to_vec(
&front_end()
.with_range_clamp_db(4.0)
.package_window(window, reference),
&[1.0, 0.0],
Tolerance::absolute(1e-6),
);
}
#[test]
fn test_package_mels_shape_over_a_range() {
let device = cpu_device();
for batch in 1..4 {
for n_mels in 1..5 {
for frames in 2..12 {
let joined: Tensor<3> = Tensor::zeros([batch, frames, n_mels], &device);
assert_eq!(
package_mels(joined).dims(),
[batch, n_mels, frames - 1],
"batch {batch}, n_mels {n_mels}, frames {frames}",
);
}
}
}
}
#[test]
fn test_package_mels_ignores_the_dropped_frame_over_a_range() {
let device = cpu_device();
let n_mels = 2;
for frames in 2..10 {
let mut data = vec![0.0_f64; frames * n_mels];
for slot in data.iter_mut().skip((frames - 1) * n_mels) {
*slot = 99.0;
}
let joined: Tensor<3> =
Tensor::from_data(TensorData::new(data, [1, frames, n_mels]), &device);
assert_tensor_close_to_vec(
&package_mels(joined),
&vec![1.0; n_mels * (frames - 1)],
Tolerance::absolute(1e-6),
);
}
}
#[test]
#[should_panic(expected = "at least 2 frames")]
fn test_package_mels_rejects_a_single_frame() {
let device = cpu_device();
let joined: Tensor<3> = Tensor::zeros([1, 1, 4], &device);
let _ = package_mels(joined);
}
#[test]
#[should_panic(expected = "at least 2 frames")]
fn test_package_mels_rejects_zero_frames() {
let device = cpu_device();
let joined: Tensor<3> = Tensor::zeros([1, 0, 4], &device);
let _ = package_mels(joined);
}
#[test]
fn test_package_mels_clamps_after_dropping_the_frame() {
let device = cpu_device();
let joined: Tensor<3> = Tensor::from_data(
TensorData::new(vec![0.0_f64, -20.0, 99.0], [1, 3, 1]),
&device,
);
assert_tensor_close_to_vec(
&package_mels(joined),
&[1.0, -1.0],
Tolerance::absolute(1e-6),
);
}
#[test]
fn test_split_packaging_equals_package_mels() {
let device = cpu_device();
let (batch, frames, n_mels) = (3, 7, 4);
let data: Vec<f64> = (0..batch * frames * n_mels)
.map(|k| ((k * 37) % 23) as f64 - 15.0 + (k / (frames * n_mels)) as f64 * 3.0)
.collect();
let joined: Tensor<3> =
Tensor::from_data(TensorData::new(data, [batch, frames, n_mels]), &device);
let whole = package_mels(joined.clone());
let window = drop_last_frame(joined);
let split = package_window(window.clone(), MaxLevel::of(window).max);
assert_eq!(split.dims(), [batch, n_mels, frames - 1]);
split.to_data().assert_eq(&whole.to_data(), true);
}
#[test]
fn test_package_window_floors_each_row_against_its_reference() {
let device = cpu_device();
let window: Tensor<3> = Tensor::from_data(
TensorData::new(vec![0.0_f64, -20.0, 0.0, -20.0], [2, 2, 1]),
&device,
);
let reference: Tensor<1> =
Tensor::from_data(TensorData::new(vec![0.0_f64, 12.0], [2]), &device);
assert_tensor_close_to_vec(
&package_window(window, reference),
&[1.0, -1.0, 2.0, 2.0],
Tolerance::absolute(1e-6),
);
}
#[test]
fn test_package_window_never_clips_above_the_floor() {
let device = cpu_device();
let window: Tensor<3> =
Tensor::from_data(TensorData::new(vec![4.0_f64, 0.0], [1, 2, 1]), &device);
let reference: Tensor<1> = Tensor::from_data(TensorData::new(vec![0.0_f64], [1]), &device);
assert_tensor_close_to_vec(
&package_window(window, reference),
&[2.0, 1.0],
Tolerance::absolute(1e-6),
);
}
}