use burn::{
prelude::*,
tensor::{
ops::PadMode,
signal::{
StftOptions,
stft,
},
},
};
use crate::{
burner::{
module::ModuleInit,
tensor::TensorOpExt,
},
errors::{
BunsenError,
BunsenResult,
},
ops::signal::{
SamplingWindowBuilder,
StftWindowConfig,
},
};
pub trait SlidingStftMeta {
fn win_len(&self) -> usize;
fn hop_size(&self) -> usize;
fn fft_size(&self) -> usize;
fn n_bins(&self) -> usize {
self.fft_size() / 2 + 1
}
}
#[derive(Config, Debug)]
pub struct SlidingStftConfig {
#[config(default = "768")]
pub win_len: usize,
#[config(default = "256")]
pub hop_size: usize,
#[config(default = "1024")]
pub fft_size: usize,
#[config(default = "StftWindowConfig::Hann { periodic: true }")]
pub window: StftWindowConfig,
}
impl SlidingStftMeta for SlidingStftConfig {
fn win_len(&self) -> usize {
self.win_len
}
fn hop_size(&self) -> usize {
self.hop_size
}
fn fft_size(&self) -> usize {
self.fft_size
}
}
impl SlidingStftConfig {
pub fn validate(&self) -> BunsenResult<()> {
if self.hop_size == 0 {
return Err(BunsenError::Invalid(
"SlidingStft hop_size must be non-zero".to_string(),
));
}
if self.win_len < self.hop_size {
return Err(BunsenError::Invalid(format!(
"SlidingStft win_len ({}) must be >= hop_size ({})",
self.win_len, self.hop_size,
)));
}
if self.fft_size < self.win_len {
return Err(BunsenError::Invalid(format!(
"SlidingStft fft_size ({}) must be >= win_len ({})",
self.fft_size, self.win_len,
)));
}
if !self.fft_size.is_power_of_two() {
return Err(BunsenError::Invalid(format!(
"SlidingStft fft_size ({}) must be a power of two",
self.fft_size,
)));
}
Ok(())
}
}
impl<B: Backend> ModuleInit<B, SlidingStft<B>> for SlidingStftConfig {
fn try_init(
&self,
device: &B::Device,
) -> BunsenResult<SlidingStft<B>> {
self.validate()?;
let win_len = self.win_len;
let window = self.window.to_tensor_window(win_len, device);
let pad = self.fft_size - win_len;
let window = if pad > 0 {
window.pad([(0, pad)], PadMode::Constant(0.0))
} else {
window
};
Ok(SlidingStft {
hop_size: self.hop_size,
win_len,
window,
})
}
}
#[derive(Module, Debug)]
pub struct SlidingStft<B: Backend> {
hop_size: usize,
win_len: usize,
pub window: Tensor<B, 1>,
}
impl<B: Backend> SlidingStftMeta for SlidingStft<B> {
fn win_len(&self) -> usize {
self.win_len
}
fn hop_size(&self) -> usize {
self.hop_size
}
fn fft_size(&self) -> usize {
self.window.dims()[0]
}
}
impl<B: Backend> SlidingStft<B> {
pub fn init_state(
&self,
batch_size: usize,
) -> SlidingStftContext<B> {
assert_ne!(batch_size, 0, "SlidingStft batch_size must be non-zero");
let queue = self.zero_window(batch_size);
SlidingStftContext {
coef: self.clone(),
queue,
}
}
pub fn to_options(&self) -> StftOptions {
StftOptions {
n_fft: self.fft_size(),
hop_length: self.hop_size(),
win_length: None,
center: false,
onesided: true,
}
}
pub fn zero_window(
&self,
batch_size: usize,
) -> Tensor<B, 2> {
Tensor::zeros([batch_size, self.win_len()], &self.window.device())
}
pub fn analyze(
&self,
signal: Tensor<B, 2>,
) -> Tensor<B, 4> {
#[cfg(any(test, debug_assertions))]
let [batch, samples] = crate::contracts::unpack_shape_contract!(
["batch", "samples"],
&signal,
&["batch", "samples"],
);
#[cfg(any(test, debug_assertions))]
assert!(
samples >= self.win_len(),
"SlidingStft samples ({samples}) must be >= win_len ({})",
self.win_len(),
);
let n_samples = signal.dims()[1];
let frames = 1 + (n_samples - self.win_len()) / self.hop_size();
let pad = self.fft_size() - self.win_len();
let signal = if pad > 0 {
signal.pad([(0, pad)], PadMode::Constant(0.0))
} else {
signal
};
let covered = (frames - 1) * self.hop_size() + self.fft_size();
let signal = signal.slice_dim(1, 0..covered as isize);
let x = stft(signal, Some(self.window.clone()), self.to_options());
#[cfg(any(test, debug_assertions))]
crate::contracts::assert_shape_contract!(
["batch", "frames", "n_bins", 2],
&x,
&[
("batch", batch),
("frames", 1 + (samples - self.win_len()) / self.hop_size()),
("n_bins", self.n_bins())
],
);
x
}
}
#[derive(Module, Debug)]
pub struct SlidingStftContext<B: Backend> {
pub coef: SlidingStft<B>,
pub queue: Tensor<B, 2>,
}
impl<B: Backend> SlidingStftMeta for SlidingStftContext<B> {
fn win_len(&self) -> usize {
self.coef.win_len()
}
fn hop_size(&self) -> usize {
self.coef.hop_size()
}
fn fft_size(&self) -> usize {
self.coef.fft_size()
}
}
impl<B: Backend> SlidingStftContext<B> {
pub fn batch_size(&self) -> usize {
self.queue.dims()[0]
}
pub fn reset(&mut self) {
self.queue = Tensor::zeros_like(&self.queue);
}
pub fn forward(
&mut self,
hop: Tensor<B, 2>,
) -> Tensor<B, 3> {
#[cfg(any(test, debug_assertions))]
crate::contracts::assert_shape_contract!(
["batch", "hop_size"],
&hop,
&[("batch", self.batch_size()), ("hop_size", self.hop_size())],
);
let hop_size = self.hop_size();
if self.win_len() == hop_size {
self.queue = hop;
} else {
self.queue.inplace(|q| {
let keep = q.slice_dim(1, hop_size as isize..);
Tensor::cat(vec![keep, hop], 1)
});
}
let x = self.coef.analyze(self.queue.clone());
x.squeeze_dim(1)
}
pub fn forward_sequence(
&mut self,
hops: Tensor<B, 3>,
) -> Tensor<B, 4> {
#[cfg(any(test, debug_assertions))]
let [steps] = crate::contracts::unpack_shape_contract!(
["steps", "batch", "hop_size"],
&hops,
&["steps"],
&[("batch", self.batch_size()), ("hop_size", self.hop_size())],
);
let stream = hops.swap_dims(0, 1);
let stream = stream.flatten::<2>(1, 2);
let win_len = self.win_len();
let queue = self.queue.extract();
let ext = Tensor::cat(vec![queue, stream], 1);
self.queue = ext.clone().slice_dim(1, -(win_len as isize)..);
let x = self.coef.analyze(ext);
let x = x.slice_dim(1, 1..);
let x = x.swap_dims(0, 1);
#[cfg(any(test, debug_assertions))]
crate::contracts::assert_shape_contract!(
["steps", "batch", "n_bins", 2],
&x,
&[
("steps", steps),
("batch", self.batch_size()),
("n_bins", self.n_bins())
],
);
x
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
DType,
Distribution,
Tolerance,
backend::BackendTypes,
};
use super::*;
use crate::{
prelude::*,
support::testing::{
CpuBackend,
PerformanceBackend,
},
};
type B = CpuBackend;
type F = <B as BackendTypes>::FloatElem;
#[test]
fn test_config_meta() {
let cfg = SlidingStftConfig::new();
assert_eq!(cfg.win_len, 768);
assert_eq!(cfg.hop_size, 256);
assert_eq!(cfg.fft_size, 1024);
assert_eq!(cfg.window, StftWindowConfig::Hann { periodic: true });
assert_eq!(cfg.win_len(), 768);
assert_eq!(cfg.hop_size(), 256);
assert_eq!(cfg.fft_size(), 1024);
assert_eq!(cfg.n_bins(), 513);
cfg.validate().unwrap();
}
#[test]
fn test_validate_rejects_bad_geometry() {
for bad in [
SlidingStftConfig::new().with_hop_size(0),
SlidingStftConfig::new().with_hop_size(1024),
SlidingStftConfig::new().with_win_len(2048),
SlidingStftConfig::new()
.with_fft_size(1023)
.with_win_len(768),
SlidingStftConfig::new()
.with_fft_size(1536)
.with_win_len(768),
] {
assert!(
matches!(bad.validate(), Err(BunsenError::Invalid(_))),
"expected Invalid: {bad:?}",
);
}
}
#[test]
fn test_init_meta_matches_config() {
let device = Default::default();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft<B> = cfg.init(&device);
assert_eq!(coef.win_len(), cfg.win_len());
assert_eq!(coef.hop_size(), cfg.hop_size());
assert_eq!(coef.fft_size(), cfg.fft_size());
assert_eq!(coef.n_bins(), cfg.n_bins());
assert_eq!(coef.window.dims(), [64]);
let window: Vec<f64> = coef
.window
.clone()
.cast(DType::F64)
.to_data()
.to_vec()
.unwrap();
let host = cfg.window.to_vec_window(48);
for (n, (&w, &h)) in window.iter().zip(&host).enumerate() {
assert!((w - h).abs() <= 1e-6, "window[{n}]: {w} vs {h}");
}
assert!(window[48..].iter().all(|&v| v == 0.0));
let stft = coef.init_state(3);
assert_eq!(stft.win_len(), cfg.win_len());
assert_eq!(stft.hop_size(), cfg.hop_size());
assert_eq!(stft.fft_size(), cfg.fft_size());
assert_eq!(stft.n_bins(), cfg.n_bins());
assert_eq!(stft.batch_size(), 3);
assert_eq!(stft.queue.dims(), [3, 48]);
stft.queue
.to_data()
.assert_eq(&Tensor::<B, 2>::zeros([3, 48], &device).to_data(), true);
}
#[test]
fn test_module_semantics() {
let device = Default::default();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft<B> = cfg.init(&device);
let ctx = coef.clone().init_state(2);
assert_eq!(coef.devices(), vec![device]);
assert_eq!(ctx.devices(), vec![device]);
assert_eq!(coef.num_params(), 0);
assert_eq!(ctx.num_params(), 0);
let window_before = coef.window.to_data_as::<F>();
let queue_before = ctx.queue.to_data_as::<F>();
let moved = coef.clone().to_device(&device);
assert_eq!(moved.devices(), vec![device]);
moved
.window
.to_data_as::<F>()
.assert_eq(&window_before, true);
assert_eq!(moved.win_len(), coef.win_len());
assert_eq!(moved.hop_size(), coef.hop_size());
assert_eq!(moved.fft_size(), coef.fft_size());
let moved_ctx = ctx.clone().to_device(&device);
assert_eq!(moved_ctx.devices(), vec![device]);
moved_ctx
.queue
.to_data_as::<F>()
.assert_eq(&queue_before, true);
moved_ctx
.coef
.window
.to_data_as::<F>()
.assert_eq(&window_before, true);
assert_eq!(moved_ctx.batch_size(), ctx.batch_size());
}
#[test]
#[should_panic(expected = "batch_size must be non-zero")]
fn test_init_state_rejects_zero_batch() {
let device = Default::default();
let coef: SlidingStft<B> = SlidingStftConfig::new().init(&device);
coef.init_state(0);
}
struct HostStft {
win_len: usize,
hop_size: usize,
fft_size: usize,
window: Vec<f64>,
queue: Vec<f64>,
}
impl HostStft {
fn new(cfg: &SlidingStftConfig) -> Self {
Self {
win_len: cfg.win_len,
hop_size: cfg.hop_size,
fft_size: cfg.fft_size,
window: cfg.window.to_vec_window(cfg.win_len),
queue: vec![0.0; cfg.win_len],
}
}
fn push(
&mut self,
hop: &[f64],
) -> Vec<f64> {
assert_eq!(hop.len(), self.hop_size);
self.queue.drain(..self.hop_size);
self.queue.extend(hop.iter().map(|&v| v as f64));
let n_bins = self.fft_size / 2 + 1;
let mut out = Vec::with_capacity(2 * n_bins);
for k in 0..n_bins {
let (mut re, mut im) = (0.0f64, 0.0f64);
for n in 0..self.win_len {
let x = self.queue[n] * self.window[n] as f64;
let theta = core::f64::consts::TAU * ((n * k) % self.fft_size) as f64
/ self.fft_size as f64;
re += x * theta.cos();
im -= x * theta.sin();
}
out.push(re);
out.push(im);
}
out
}
}
fn sample(
batch: usize,
step: usize,
idx: usize,
) -> f64 {
((batch * 7919 + step * 104729 + idx * 1299709) % 1000) as f64 - 500.0
}
fn host_analyze(
row: &[f64],
cfg: &SlidingStftConfig,
) -> Vec<f64> {
let window = cfg.window.to_vec_window(cfg.win_len);
let frames = 1 + (row.len() - cfg.win_len) / cfg.hop_size;
let n_bins = cfg.n_bins();
let mut out = Vec::with_capacity(frames * n_bins * 2);
for f in 0..frames {
for k in 0..n_bins {
let (mut re, mut im) = (0.0f64, 0.0f64);
for n in 0..cfg.win_len {
let x = row[f * cfg.hop_size + n] * window[n];
let theta = core::f64::consts::TAU * ((n * k) % cfg.fft_size) as f64
/ cfg.fft_size as f64;
re += x * theta.cos();
im -= x * theta.sin();
}
out.push(re);
out.push(im);
}
}
out
}
#[test]
fn test_analyze_frame_count_and_shape() {
let device = Default::default();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft<B> = cfg.init(&device);
for (samples, frames) in [(48, 1), (64, 2), (80, 3), (88, 3), (95, 3), (96, 4)] {
let x = Tensor::<B, 2>::from_data(
TensorData::new(vec![0.0f64; 2 * samples], [2, samples]),
&device,
);
assert_eq!(
coef.analyze(x).dims(),
[2, frames, cfg.n_bins(), 2],
"samples = {samples}",
);
}
}
#[test]
fn test_analyze_matches_naive_dft() {
let device = Default::default();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft<B> = cfg.init(&device);
let batch = 2;
let samples = 88;
let rows: Vec<Vec<f64>> = (0..batch)
.map(|b| (0..samples).map(|i| sample(b, 0, i)).collect())
.collect();
let got = coef.analyze(Tensor::<B, 2>::from_data(
TensorData::new(rows.concat(), [batch, samples]),
&device,
));
let expected: Vec<f64> = rows.iter().flat_map(|r| host_analyze(r, &cfg)).collect();
let frames = 1 + (samples - cfg.win_len) / cfg.hop_size;
got.to_data_as::<F>().assert_approx_eq::<F>(
&TensorData::new(expected, [batch, frames, cfg.n_bins(), 2]).convert::<F>(),
Tolerance::permissive(),
);
}
#[test]
fn test_analyze_ragged_batch_matches_single_rows() {
type P = PerformanceBackend;
type PF = <P as BackendTypes>::FloatElem;
let device = Default::default();
let cfg = SlidingStftConfig::new();
let coef: SlidingStft<P> = cfg.init(&device);
let batch = 3;
let samples = cfg.win_len + 8;
assert_ne!(
(samples - cfg.win_len) % cfg.hop_size,
0,
"geometry must be ragged"
);
let rows: Vec<Vec<f64>> = (0..batch)
.map(|b| (0..samples).map(|i| sample(b, 0, i)).collect())
.collect();
let batched = coef.analyze(Tensor::<P, 2>::from_data(
TensorData::new(rows.concat(), [batch, samples]),
&device,
));
for (b, row) in rows.iter().enumerate() {
let alone = coef.analyze(Tensor::<P, 2>::from_data(
TensorData::new(row.clone(), [1, samples]),
&device,
));
let got = batched.clone().slice_dim(0, b as isize..b as isize + 1);
assert_eq!(got.dims(), alone.dims());
got.to_data_as::<PF>()
.assert_approx_eq::<PF>(&alone.to_data_as::<PF>(), Tolerance::permissive());
}
}
#[test]
fn test_forward_matches_naive_dft() {
let device = Default::default();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let batch = 2;
let n_bins = cfg.n_bins();
let coef: SlidingStft<B> = cfg.init(&device);
let mut stft = coef.init_state(batch);
let mut hosts: Vec<HostStft> = (0..batch).map(|_| HostStft::new(&cfg)).collect();
for step in 0..5 {
let rows: Vec<Vec<f64>> = (0..batch)
.map(|b| (0..cfg.hop_size).map(|i| sample(b, step, i)).collect())
.collect();
let hop = Tensor::<B, 2>::from_data(
TensorData::new(rows.concat(), [batch, cfg.hop_size]),
&device,
);
let out = stft.forward(hop);
assert_eq!(out.dims(), [batch, n_bins, 2]);
let expected: Vec<f64> = hosts
.iter_mut()
.zip(&rows)
.flat_map(|(host, row)| host.push(row))
.collect();
out.cast(DType::F64)
.to_data_as::<F>()
.assert_approx_eq::<F>(
&TensorData::new(expected, [batch, n_bins, 2]).convert::<F>(),
Tolerance::permissive(),
);
}
}
#[test]
fn test_forward_sequence_matches_stepwise() {
let device = Default::default();
let steps = 7;
let batch = 2;
for cfg in [
SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64),
SlidingStftConfig::new()
.with_win_len(16)
.with_hop_size(16)
.with_fft_size(16),
] {
let hops = Tensor::<B, 3>::random(
[steps, batch, cfg.hop_size],
Distribution::Default,
&device,
);
let coef: SlidingStft<B> = cfg.init(&device);
let mut seq_stft = coef.init_state(batch);
let mut step_stft = seq_stft.clone();
let seq_out = seq_stft.forward_sequence(hops.clone());
assert_eq!(seq_out.dims(), [steps, batch, cfg.n_bins(), 2]);
let mut step_outs = Vec::with_capacity(steps);
for step in 0..steps {
let hop = hops.clone().select_dim(0, step);
step_outs.push(step_stft.forward(hop));
}
let step_out: Tensor<B, 4> = Tensor::stack(step_outs, 0);
let tol = Tolerance::<F>::permissive();
seq_out
.to_data_as::<F>()
.assert_approx_eq::<F>(&step_out.to_data_as::<F>(), tol);
seq_stft
.queue
.to_data_as::<F>()
.assert_approx_eq::<F>(&step_stft.queue.to_data_as::<F>(), tol);
}
}
#[test]
fn test_reset() {
let device = Default::default();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft<B> = cfg.init(&device);
let mut stft = coef.init_state(1);
let hop = Tensor::random([1, 16], Distribution::Default, &device);
stft.forward(hop);
stft.reset();
stft.queue.to_data_as::<F>().assert_eq(
&Tensor::<B, 2>::zeros([1, 48], &device).to_data_as::<F>(),
true,
);
}
}