use burn::{
Tensor,
config::Config,
module::Module,
prelude::Backend,
};
use crate::{
errors::{
BunsenError,
BunsenResult,
},
ops::signal::perceptive_audio::{
PaddingMode,
PerceptiveAudioConverter,
PerceptiveAudioConverterMeta,
},
};
#[derive(Config, Copy, Debug, PartialEq, Eq)]
pub enum StreamPhase {
Start,
Running,
}
#[derive(Module, Debug)]
pub struct PerceptiveAudioConversionContext<B: Backend> {
converter: PerceptiveAudioConverter<B>,
carry: Option<Tensor<B, 2>>,
#[module(skip)]
batch_size: usize,
#[module(skip)]
phase: StreamPhase,
}
impl<B: Backend> PerceptiveAudioConverterMeta for PerceptiveAudioConversionContext<B> {
fn sample_rate(&self) -> usize {
self.converter.sample_rate()
}
fn n_fft(&self) -> usize {
self.converter.n_fft()
}
fn hop(&self) -> usize {
self.converter.hop()
}
fn n_mels(&self) -> usize {
self.converter.n_mels()
}
fn pad_to_pow2(&self) -> bool {
self.converter.pad_to_pow2()
}
fn start_padding(&self) -> PaddingMode {
self.converter.start_padding()
}
fn end_padding(&self) -> PaddingMode {
self.converter.end_padding()
}
}
impl<B: Backend> PerceptiveAudioConversionContext<B> {
pub fn batch_size(&self) -> usize {
self.batch_size
}
pub fn phase(&self) -> StreamPhase {
self.phase
}
pub fn converter(&self) -> &PerceptiveAudioConverter<B> {
&self.converter
}
pub fn carry(&self) -> Option<&Tensor<B, 2>> {
self.carry.as_ref()
}
pub fn reset(&mut self) {
self.carry = None;
self.phase = StreamPhase::Start;
}
pub fn transform(
self,
waves: Tensor<B, 2>,
) -> BunsenResult<(Tensor<B, 3>, Self)> {
let (x, this) = self.t_stage_extend(waves)?;
let (x, this) = this.t_stage_preproc(x);
let (x, this) = this.t_stage_frame(x);
let (x, this) = this.t_stage_spectrum(x);
let (x, this) = this.t_stage_mel(x);
let (mels, this) = this.t_stage_compress(x);
Ok((mels, this))
}
pub fn finish(self) -> Option<Tensor<B, 3>> {
let carry = self.carry.clone()?;
let pad = self.end_padding().pad_len(self.n_fft());
if pad == 0 {
return None;
}
let len = carry.dims()[1];
let tail = match self.end_padding() {
PaddingMode::None => unreachable!("pad_len is zero for None"),
PaddingMode::Zero => Tensor::cat(
vec![
carry.clone(),
Tensor::zeros([self.batch_size, pad], &carry.device()),
],
1,
),
PaddingMode::Reflect => {
debug_assert!(
len > pad,
"carry ({len}) must exceed the reflect pad ({pad})",
);
let mirror = carry
.clone()
.slice_dim(1, (len - 1 - pad) as isize..(len - 1) as isize)
.flip([1]);
Tensor::cat(vec![carry, mirror], 1)
}
};
if self.converter.frame_count(tail.dims()[1]) == 0 {
return None;
}
Some(self.converter.forward(tail))
}
pub(crate) fn t_stage_extend(
mut self,
waves: Tensor<B, 2>,
) -> BunsenResult<(Tensor<B, 2>, Self)> {
let [batch, samples] = waves.dims();
if batch != self.batch_size {
return Err(BunsenError::Invalid(format!(
"MelConversionContext batch ({batch}) must match the context \
batch size ({})",
self.batch_size,
)));
}
let hop = self.hop();
if samples == 0 || samples % hop != 0 {
return Err(BunsenError::Invalid(format!(
"MelConversionContext samples ({samples}) must be a non-zero \
multiple of hop ({hop})",
)));
}
let extended = match self.phase {
StreamPhase::Running => match self.carry.take() {
Some(carry) => Tensor::cat(vec![carry, waves], 1),
None => waves,
},
StreamPhase::Start => {
let pad = self.start_padding().pad_len(self.n_fft());
match self.start_padding() {
PaddingMode::None => waves,
PaddingMode::Zero => {
Tensor::cat(vec![Tensor::zeros([batch, pad], &waves.device()), waves], 1)
}
PaddingMode::Reflect => {
if samples < self.min_first_chunk() {
return Err(BunsenError::Invalid(format!(
"MelConversionContext reflect start padding \
needs at least {} samples in the first chunk, \
got {samples}",
self.min_first_chunk(),
)));
}
let mirror = waves.clone().slice_dim(1, 1..(pad + 1) as isize).flip([1]);
Tensor::cat(vec![mirror, waves], 1)
}
}
}
};
let frames = self.converter.frame_count(extended.dims()[1]);
if frames == 0 {
return Err(BunsenError::Invalid(format!(
"MelConversionContext first chunk of {samples} samples is too \
short to fill a frame; with {:?} start padding it needs at \
least {} samples",
self.start_padding(),
self.n_fft() - self.start_padding().pad_len(self.n_fft()),
)));
}
let consumed = frames * hop;
self.carry = Some(extended.clone().slice_dim(1, consumed as isize..));
self.phase = StreamPhase::Running;
Ok((extended, self))
}
pub(crate) fn t_stage_preproc(
self,
x: Tensor<B, 2>,
) -> (Tensor<B, 2>, Self) {
(x, self)
}
pub(crate) fn t_stage_frame(
self,
x: Tensor<B, 2>,
) -> (Tensor<B, 3>, Self) {
let framed = self.converter.frame(x);
(framed, self)
}
pub(crate) fn t_stage_spectrum(
self,
x: Tensor<B, 3>,
) -> (Tensor<B, 3>, Self) {
let spectrum = self.converter.spectrum(x);
(spectrum, self)
}
pub(crate) fn t_stage_mel(
self,
x: Tensor<B, 3>,
) -> (Tensor<B, 3>, Self) {
let mels = self.converter.mel(x);
(mels, self)
}
pub(crate) fn t_stage_compress(
self,
x: Tensor<B, 3>,
) -> (Tensor<B, 3>, Self) {
let mels = self.converter.compress(x);
(mels, self)
}
}
impl<B: Backend> PerceptiveAudioConverter<B> {
pub fn new_context(
&self,
batch_size: usize,
) -> PerceptiveAudioConversionContext<B> {
assert_ne!(batch_size, 0, "MelConverter batch_size must be non-zero",);
PerceptiveAudioConversionContext {
converter: self.clone(),
carry: None,
batch_size,
phase: StreamPhase::Start,
}
}
}
#[cfg(test)]
mod tests {
use burn::tensor::Tolerance;
use super::*;
use crate::{
burner::{
module::ModuleInit,
tensor::TensorDataToVecAsExt,
},
errors::WithOkOrPanic,
ops::signal::perceptive_audio::PerceptiveAudioConverterOptions,
support::testing::{
PerformanceBackend,
assert_close_to_vec,
assert_tensor_close_to_vec,
assert_tensors_close,
},
};
type B = PerformanceBackend;
fn from_rows<B: Backend>(
rows: &[Vec<f64>],
device: &burn::prelude::Device<B>,
) -> Tensor<B, 2> {
let samples = rows[0].len();
Tensor::from_data(
burn::prelude::TensorData::new(rows.concat(), [rows.len(), samples]),
device,
)
}
fn sample(
row: usize,
i: usize,
) -> f64 {
(((row * 7919 + i * 104729) % 2003) as f64 / 1001.0) - 1.0
}
fn rows(
batch: usize,
samples: usize,
) -> Vec<Vec<f64>> {
(0..batch)
.map(|r| (0..samples).map(|i| sample(r, i)).collect())
.collect()
}
#[test]
fn test_context_meta_and_lifecycle() {
let device = Default::default();
let opts = PerceptiveAudioConverterOptions::default();
let conv: PerceptiveAudioConverter<B> = opts.try_init(&device).ok_or_panic();
let ctx = conv.new_context(2);
assert_eq!(ctx.batch_size(), 2);
assert_eq!(ctx.phase(), StreamPhase::Start);
assert!(ctx.carry().is_none());
assert_eq!(ctx.n_fft(), conv.n_fft());
assert_eq!(ctx.hop(), conv.hop());
assert_eq!(ctx.n_mels(), conv.n_mels());
assert_eq!(ctx.n_bins(), conv.n_bins());
assert_eq!(ctx.min_first_chunk(), conv.min_first_chunk());
let x = from_rows::<B>(&rows(2, 1600), &device);
let (_, ctx) = ctx.transform(x).unwrap();
assert_eq!(ctx.phase(), StreamPhase::Running);
assert!(ctx.carry().is_some());
let mut ctx = ctx;
ctx.reset();
assert_eq!(ctx.phase(), StreamPhase::Start);
assert!(ctx.carry().is_none());
}
#[test]
fn test_frame_accounting_over_a_30s_window() {
let device = Default::default();
let opts = PerceptiveAudioConverterOptions::default();
let conv: PerceptiveAudioConverter<B> = opts.try_init(&device).ok_or_panic();
let samples = 480_000;
let x = from_rows::<B>(&rows(1, samples), &device);
let (mels, ctx) = conv.new_context(1).transform(x).unwrap();
assert_eq!(mels.dims(), [1, 2999, opts.n_mels]);
assert_eq!(ctx.carry().unwrap().dims(), [1, 360]);
let tail = ctx.finish().expect("reflect end padding yields a tail");
assert_eq!(tail.dims(), [1, 2, opts.n_mels]);
assert_eq!(2999 + 2, 3001);
}
#[test]
fn test_running_frame_count_is_invariant() {
let device = Default::default();
let conv: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.try_init(&device)
.ok_or_panic();
let hop = conv.hop();
let mut ctx = conv.new_context(2);
let mut carry_len = None;
for (step, chunk) in [3200, 1600, 4800, 1600].into_iter().enumerate() {
let x = from_rows::<B>(&rows(2, chunk), &device);
let (mels, next) = ctx.transform(x).unwrap();
ctx = next;
if step > 0 {
assert_eq!(
mels.dims()[1],
chunk / hop,
"step {step}: running frame count should be samples / hop",
);
}
let len = ctx.carry().unwrap().dims()[1];
match carry_len {
None => carry_len = Some(len),
Some(prev) => assert_eq!(prev, len, "carry length drifted at step {step}"),
}
}
assert_eq!(carry_len, Some(360));
}
#[test]
fn test_chunked_transform_is_a_homomorphism() {
let device = Default::default();
let conv: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.try_init(&device)
.ok_or_panic();
let (batch, total) = (3, 32_000);
let host = rows(batch, total);
let whole = from_rows::<B>(&host, &device);
let (whole_mels, whole_ctx) = conv.new_context(batch).transform(whole).unwrap();
let whole_tail = whole_ctx.finish().unwrap();
for split in [
vec![32_000],
vec![16_000, 16_000],
vec![320, 31_680],
vec![6_400, 3_200, 12_800, 9_600],
] {
assert_eq!(split.iter().sum::<usize>(), total);
let mut ctx = conv.new_context(batch);
let mut pieces = Vec::new();
let mut at = 0;
for n in &split {
let chunk: Vec<Vec<f64>> = host.iter().map(|r| r[at..at + n].to_vec()).collect();
let (mels, next) = ctx.transform(from_rows::<B>(&chunk, &device)).unwrap();
ctx = next;
pieces.push(mels);
at += n;
}
let joined: Tensor<B, 3> = Tensor::cat(pieces, 1);
assert_eq!(
joined.dims(),
whole_mels.dims(),
"split {split:?} produced a different frame count",
);
assert_tensors_close(&joined, &whole_mels, Tolerance::absolute(1e-4));
let tail = ctx.finish().unwrap();
assert_tensors_close(&tail, &whole_tail, Tolerance::absolute(1e-4));
}
}
#[test]
fn test_batch_rows_are_independent() {
let device = Default::default();
let conv: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.try_init(&device)
.ok_or_panic();
let (batch, samples) = (3, 8_000);
let host = rows(batch, samples);
let (together, _) = conv
.new_context(batch)
.transform(from_rows::<B>(&host, &device))
.unwrap();
let dims = together.dims();
let per_row = dims[1] * dims[2];
let together = together.to_data().to_vec_as::<f64>().unwrap();
for row in 0..batch {
let (alone, _) = conv
.new_context(1)
.transform(from_rows::<B>(&host[row..row + 1], &device))
.unwrap();
assert_close_to_vec(
&alone.to_data().to_vec_as::<f64>().unwrap(),
&together[row * per_row..(row + 1) * per_row],
1e-4,
);
}
}
#[test]
fn test_transform_rejects_bad_input() {
let device = Default::default();
let conv: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.try_init(&device)
.ok_or_panic();
let ctx = conv.new_context(2);
let bad = from_rows::<B>(&rows(2, 1601), &device);
assert!(matches!(ctx.transform(bad), Err(BunsenError::Invalid(_)),));
let ctx = conv.new_context(2);
let bad = from_rows::<B>(&rows(3, 1600), &device);
assert!(matches!(ctx.transform(bad), Err(BunsenError::Invalid(_)),));
let ctx = conv.new_context(1);
let bad = from_rows::<B>(&rows(1, 160), &device);
assert!(matches!(ctx.transform(bad), Err(BunsenError::Invalid(_)),));
}
#[test]
fn test_finish_respects_end_padding() {
let device = Default::default();
let unpadded: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.with_end_padding(PaddingMode::None)
.try_init(&device)
.ok_or_panic();
let (_, ctx) = unpadded
.new_context(1)
.transform(from_rows::<B>(&rows(1, 3200), &device))
.unwrap();
assert!(ctx.finish().is_none());
assert!(unpadded.new_context(1).finish().is_none());
let zero: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.with_end_padding(PaddingMode::Zero)
.try_init(&device)
.ok_or_panic();
let (_, ctx) = zero
.new_context(1)
.transform(from_rows::<B>(&rows(1, 3200), &device))
.unwrap();
assert_eq!(ctx.finish().unwrap().dims()[1], 2);
}
#[test]
fn test_stage_stack_matches_transform() {
let device = Default::default();
let conv: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.try_init(&device)
.ok_or_panic();
let x = from_rows::<B>(&rows(2, 3200), &device);
let (folded, _) = conv.new_context(2).transform(x.clone()).unwrap();
let ctx = conv.new_context(2);
let (a, ctx) = ctx.t_stage_extend(x).unwrap();
let (a, ctx) = ctx.t_stage_preproc(a);
let (a, ctx) = ctx.t_stage_frame(a);
let (a, ctx) = ctx.t_stage_spectrum(a);
let (a, ctx) = ctx.t_stage_mel(a);
let (staged, _) = ctx.t_stage_compress(a);
assert_tensors_close(&staged, &folded, Tolerance::default());
}
#[test]
fn test_extend_stage_carry_contents() {
let device = Default::default();
let conv: PerceptiveAudioConverter<B> = PerceptiveAudioConverterOptions::default()
.try_init(&device)
.ok_or_panic();
let (n_fft, hop) = (conv.n_fft(), conv.hop());
let pad = n_fft / 2;
let host = rows(1, 1600);
let x = from_rows::<B>(&host, &device);
let (ext, ctx) = conv.new_context(1).t_stage_extend(x).unwrap();
let ext_host = ext.to_data().to_vec_as::<f64>().unwrap();
assert_eq!(ext_host.len(), pad + 1600);
let expected_prefix: Vec<f64> = (1..=pad).rev().map(|i| host[0][i]).collect();
assert_close_to_vec(&ext_host[..pad], &expected_prefix, 1e-6);
assert_close_to_vec(&ext_host[pad..], &host[0], 1e-6);
let frames = conv.frame_count(ext_host.len());
let consumed = frames * hop;
assert_tensor_close_to_vec(
&ctx.carry().unwrap(),
&ext_host[consumed..],
Tolerance::default(),
);
}
}