use burn::{
Tensor,
config::Config,
module::Module,
prelude::TensorData,
tensor::Device,
};
use crate::{
burner::module::ModuleInit,
errors::BunsenResult,
kits::speech::asr::whisper::{
blocks::WhisperFrontEndConfig,
driver::support::drop_last_frame,
},
ops::signal::perceptive_audio::{
PerceptiveAudioConverter,
PerceptiveAudioConverterMeta,
},
};
pub trait VoxtralFrontEndMeta {
fn sample_rate(&self) -> usize;
fn hop(&self) -> usize;
fn n_mels(&self) -> usize;
fn global_log_mel_max(&self) -> f64;
}
#[derive(Config, Debug)]
pub struct VoxtralFrontEndConfig {
#[config(default = "128")]
pub n_mels: usize,
#[config(default = "1.5")]
pub global_log_mel_max: f64,
#[config(default = "WhisperFrontEndConfig::new()")]
pub whisper: WhisperFrontEndConfig,
}
impl VoxtralFrontEndMeta for VoxtralFrontEndConfig {
fn sample_rate(&self) -> usize {
self.whisper.sample_rate
}
fn hop(&self) -> usize {
self.whisper.hop()
}
fn n_mels(&self) -> usize {
self.n_mels
}
fn global_log_mel_max(&self) -> f64 {
self.global_log_mel_max
}
}
impl ModuleInit<VoxtralFrontEnd> for VoxtralFrontEndConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<VoxtralFrontEnd> {
Ok(VoxtralFrontEnd {
converter: self.whisper.try_init_audio_converter(self.n_mels, device)?,
config: self.clone(),
})
}
}
#[derive(Module, Debug)]
pub struct VoxtralFrontEnd {
#[module(skip)]
pub config: VoxtralFrontEndConfig,
pub converter: PerceptiveAudioConverter,
}
impl VoxtralFrontEndMeta for VoxtralFrontEnd {
fn sample_rate(&self) -> usize {
self.converter.sample_rate()
}
fn hop(&self) -> usize {
self.converter.hop()
}
fn n_mels(&self) -> usize {
self.converter.n_mels()
}
fn global_log_mel_max(&self) -> f64 {
self.config.global_log_mel_max
}
}
impl VoxtralFrontEnd {
pub fn package(
&self,
window: Tensor<3>,
) -> Tensor<3> {
let [batch] = crate::contracts::unpack_shape_contract!(
["batch", "frames", "n_mels"],
&window.dims(),
&["batch"],
&[("n_mels", self.n_mels())]
);
let level = Tensor::<1>::full([batch], self.global_log_mel_max(), &window.device());
self.config.whisper.package_window(window, level)
}
pub fn silence(
&self,
frames: usize,
device: &Device,
) -> Tensor<3> {
let floor = self.global_log_mel_max() - self.config.whisper.range_clamp_db;
self.package(Tensor::full(
[1, frames, self.n_mels()],
floor - 1.0,
device,
))
}
pub fn forward(
&self,
samples: &[f32],
device: &Device,
) -> BunsenResult<Tensor<3>> {
let n = samples.len();
let wave = Tensor::<2>::from_data(TensorData::new(samples.to_vec(), [1, n]), device);
let (mels, context) = self.converter.new_context(1).transform(wave)?;
let joined = match context.finish() {
Some(tail) => Tensor::cat(vec![mels, tail], 1),
None => mels,
};
let features = self.package(drop_last_frame(joined));
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
[1, "n_mels", "frames"],
&features.dims(),
&[("n_mels", self.n_mels()), ("frames", n / self.hop())],
);
Ok(features)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta_floor_and_shapes() {
let device = cpu_device();
let config = VoxtralFrontEndConfig::new()
.with_n_mels(80)
.with_global_log_mel_max(2.0)
.with_whisper(WhisperFrontEndConfig::new().with_range_clamp_db(6.0));
let front: VoxtralFrontEnd = config.init(&device);
assert_eq!(front.sample_rate(), config.sample_rate());
assert_eq!(front.hop(), config.hop());
assert_eq!(front.n_mels(), config.n_mels());
assert_eq!(front.global_log_mel_max(), config.global_log_mel_max());
let features = front.forward(&vec![0.0; 1600], &device).unwrap();
assert_eq!(features.dims(), [1, 80, 10]);
let max = features.abs().max().into_scalar::<f32>();
assert!(max < 1e-6, "{max}");
let silence = front.silence(3, &device);
assert_eq!(silence.dims(), [1, 80, 3]);
assert!(silence.abs().max().into_scalar::<f32>() < 1e-6);
}
}