use std::path::PathBuf;
use burn::{
Tensor,
prelude::TensorData,
tensor::Tolerance,
};
use crate::{
burner::module::ModuleInit,
errors::WithOkOrPanic,
ops::signal::{
SamplingWindowBuilder,
perceptive_audio::{
AffineCompress,
FilterNorm,
MelScale,
PaddingMode,
PerceptiveAudioConverter,
PerceptiveAudioConverterMeta,
PerceptiveAudioConverterOptions,
RangeClamp,
},
},
support::testing::{
DeviceMemoryGuard,
assert_close_to_vec,
performance_device,
},
};
type F = f32;
fn fixture(name: &str) -> Vec<f64> {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("testdata/perceptive_audio")
.join(name);
let bytes =
std::fs::read(&path).unwrap_or_else(|e| panic!("reading fixture {}: {e}", path.display()));
assert_eq!(
bytes.len() % 4,
0,
"fixture {name} is not a whole number of f32",
);
bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]) as f64)
.collect()
}
fn assert_matches_fixture<const D: usize>(
actual: &Tensor<D>,
name: &str,
tolerance: Tolerance<F>,
) {
let expected = TensorData::new(fixture(name), actual.dims()).convert::<F>();
actual
.to_data_as::<F>()
.assert_approx_eq::<F>(&expected, tolerance);
}
fn signal_tensor(device: &burn::prelude::Device) -> (Tensor<2>, usize) {
let samples = fixture("signal_2s_16k.f32");
let n = samples.len();
assert_eq!(n, 32_000);
(
Tensor::from_data(TensorData::new(samples, [1, n]), device),
n,
)
}
fn logmel_tolerance() -> Tolerance<F> {
Tolerance::<F>::rel_abs(1e-4, 5e-5)
}
fn parity_options() -> PerceptiveAudioConverterOptions {
PerceptiveAudioConverterOptions::default()
}
#[test]
fn test_hann_window_matches_librosa() {
let opts = PerceptiveAudioConverterOptions::default();
assert_close_to_vec(
&opts.window.to_vec_window(opts.n_fft),
&fixture("hann_400_periodic.f32"),
1e-7,
);
}
#[test]
fn test_filterbank_matches_librosa() {
let opts = PerceptiveAudioConverterOptions::default();
assert_close_to_vec(
&opts.try_to_filterbank_vec().unwrap(),
&fixture("mel_fb_slaney_16k_400_80.f32"),
1e-8,
);
let htk = PerceptiveAudioConverterOptions::default()
.with_mel_scale(MelScale::Htk)
.with_filter_norm(FilterNorm::None);
assert_close_to_vec(
&htk.try_to_filterbank_vec().unwrap(),
&fixture("mel_fb_htk_16k_400_80_nonorm.f32"),
1e-6,
);
}
#[test]
#[serial_test::serial]
fn test_batch_logmel_matches_librosa_center_false() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let opts = parity_options().with_start_padding(PaddingMode::None);
let conv: PerceptiveAudioConverter = opts.try_init(&device).ok_or_panic();
let (x, samples) = signal_tensor(&device);
let mels = conv.forward(x);
assert_eq!(conv.frame_count(samples), 198);
assert_eq!(mels.dims(), [1, 198, opts.n_mels]);
assert_matches_fixture(&mels, "logmel_center_false.f32", logmel_tolerance());
}
#[test]
#[serial_test::serial]
fn test_streaming_logmel_matches_librosa_center_true() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let conv: PerceptiveAudioConverter = parity_options().try_init(&device).ok_or_panic();
let (x, _) = signal_tensor(&device);
let (mels, ctx) = conv.new_context(1).transform(x).unwrap();
let tail = ctx.finish().expect("reflect end padding yields a tail");
assert_eq!(mels.dims()[1], 199);
assert_eq!(tail.dims()[1], 2);
let joined: Tensor<3> = Tensor::cat(vec![mels, tail], 1);
assert_eq!(joined.dims(), [1, 201, conv.n_mels()]);
assert_matches_fixture(&joined, "logmel_center_true.f32", logmel_tolerance());
}
#[test]
#[serial_test::serial]
fn test_whisper_logmel_matches_reference() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let conv: PerceptiveAudioConverter = parity_options().try_init(&device).ok_or_panic();
let (x, _) = signal_tensor(&device);
let (mels, ctx) = conv.new_context(1).transform(x).unwrap();
let tail = ctx.finish().unwrap();
let joined: Tensor<3> = Tensor::cat(vec![mels, tail], 1);
assert_eq!(joined.dims(), [1, 201, conv.n_mels()]);
let cut = joined.clone().slice_dim(1, 0..200);
let max_all = joined.max().into_scalar::<f64>();
let max_cut = cut.clone().max().into_scalar::<f64>();
assert!(
(max_all - max_cut).abs() < 1e-6,
"the dropped frame holds the maximum ({max_all} vs {max_cut}); \
Whisper's clamp reference is taken after the drop",
);
let packaged =
AffineCompress::default().apply(RangeClamp::PerWindowClamp { db: 8.0 }.apply(cut));
assert_matches_fixture(&packaged, "whisper_logmel.f32", logmel_tolerance());
}
#[test]
#[serial_test::serial]
fn test_chunked_streaming_matches_librosa_center_true() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let conv: PerceptiveAudioConverter = parity_options().try_init(&device).ok_or_panic();
let samples = fixture("signal_2s_16k.f32");
let mut ctx = conv.new_context(1);
let mut pieces = Vec::new();
let mut at = 0;
for n in [3_200, 1_600, 12_800, 6_400, 8_000] {
let chunk = Tensor::from_data(
TensorData::new(samples[at..at + n].to_vec(), [1, n]),
&device,
);
let (mels, next) = ctx.transform(chunk).unwrap();
ctx = next;
pieces.push(mels);
at += n;
}
assert_eq!(at, samples.len());
pieces.push(ctx.finish().unwrap());
let joined: Tensor<3> = Tensor::cat(pieces, 1);
assert_eq!(joined.dims(), [1, 201, conv.n_mels()]);
assert_matches_fixture(&joined, "logmel_center_true.f32", logmel_tolerance());
}