use alloc::string::ToString;
use burn::{
prelude::*,
tensor::{
ops::PadMode,
signal::{
StftOptions,
stft,
},
},
};
use crate::{
burner::module::ModuleInit,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::signal::{
SamplingWindowBuilder,
StftWindowConfig,
},
};
pub trait SlidingStftMeta {
fn win_len(&self) -> usize;
fn hop_size(&self) -> usize;
fn fft_size(&self) -> usize;
fn n_bins(&self) -> usize {
self.fft_size() / 2 + 1
}
}
#[derive(Config, Debug)]
pub struct SlidingStftConfig {
#[config(default = "768")]
pub win_len: usize,
#[config(default = "256")]
pub hop_size: usize,
#[config(default = "1024")]
pub fft_size: usize,
#[config(default = "StftWindowConfig::Hann { periodic: true }")]
pub window: StftWindowConfig,
}
impl SlidingStftMeta for SlidingStftConfig {
fn win_len(&self) -> usize {
self.win_len
}
fn hop_size(&self) -> usize {
self.hop_size
}
fn fft_size(&self) -> usize {
self.fft_size
}
}
impl SlidingStftConfig {
pub fn validate(&self) -> BunsenResult<()> {
const OWNER: &str = "SlidingStftConfig";
if self.hop_size == 0 {
return Err(ConstraintError::zero_or_empty(OWNER, "hop_size").into());
}
if self.win_len < self.hop_size {
return Err(ConstraintError::new(
OWNER,
"",
Rule::Relation {
lhs: ("win_len".into(), self.win_len.to_string()),
op: ">=",
rhs: ("hop_size".into(), self.hop_size.to_string()),
},
)
.into());
}
if self.fft_size < self.win_len {
return Err(ConstraintError::new(
OWNER,
"",
Rule::Relation {
lhs: ("fft_size".into(), self.fft_size.to_string()),
op: ">=",
rhs: ("win_len".into(), self.win_len.to_string()),
},
)
.into());
}
if !self.fft_size.is_power_of_two() {
return Err(ConstraintError::out_of_range(
OWNER,
"fft_size",
self.fft_size,
"the powers of two",
)
.into());
}
Ok(())
}
}
impl ModuleInit<SlidingStft> for SlidingStftConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<SlidingStft> {
self.validate()?;
let win_len = self.win_len;
let window = self.window.to_tensor_window(win_len, device);
let pad = self.fft_size - win_len;
let window = if pad > 0 {
window.pad([(0, pad)], PadMode::Constant(0.0))
} else {
window
};
Ok(SlidingStft {
hop_size: self.hop_size,
win_len,
window,
})
}
}
#[derive(Module, Debug)]
pub struct SlidingStft {
hop_size: usize,
win_len: usize,
pub window: Tensor<1>,
}
impl SlidingStftMeta for SlidingStft {
fn win_len(&self) -> usize {
self.win_len
}
fn hop_size(&self) -> usize {
self.hop_size
}
fn fft_size(&self) -> usize {
self.window.dims()[0]
}
}
impl SlidingStft {
pub fn init_state(
&self,
batch_size: usize,
) -> SlidingStftContext {
assert_ne!(batch_size, 0, "SlidingStft batch_size must be non-zero");
let queue = self.zero_window(batch_size);
SlidingStftContext {
coef: self.clone(),
queue,
}
}
pub fn to_options(&self) -> StftOptions {
StftOptions {
n_fft: self.fft_size(),
hop_length: self.hop_size(),
win_length: None,
center: false,
onesided: true,
}
}
pub fn zero_window(
&self,
batch_size: usize,
) -> Tensor<2> {
Tensor::zeros([batch_size, self.win_len()], &self.window.device())
}
pub fn analyze(
&self,
signal: Tensor<2>,
) -> Tensor<4> {
#[cfg(any(test, debug_assertions))]
let [batch, samples] = crate::contracts::unpack_shape_contract!(
["batch", "samples"],
&signal,
&["batch", "samples"],
);
#[cfg(any(test, debug_assertions))]
assert!(
samples >= self.win_len(),
"SlidingStft samples ({samples}) must be >= win_len ({})",
self.win_len(),
);
let n_samples = signal.dims()[1];
let frames = 1 + (n_samples - self.win_len()) / self.hop_size();
let pad = self.fft_size() - self.win_len();
let signal = if pad > 0 {
signal.pad([(0, pad)], PadMode::Constant(0.0))
} else {
signal
};
let covered = (frames - 1) * self.hop_size() + self.fft_size();
let signal = signal.slice_dim(1, 0..covered as isize);
let x = stft(signal, Some(self.window.clone()), self.to_options());
#[cfg(any(test, debug_assertions))]
crate::contracts::assert_shape_contract!(
["batch", "frames", "n_bins", 2],
&x,
&[
("batch", batch),
("frames", 1 + (samples - self.win_len()) / self.hop_size()),
("n_bins", self.n_bins())
],
);
x
}
}
#[derive(Module, Debug)]
pub struct SlidingStftContext {
pub coef: SlidingStft,
pub queue: Tensor<2>,
}
impl SlidingStftMeta for SlidingStftContext {
fn win_len(&self) -> usize {
self.coef.win_len()
}
fn hop_size(&self) -> usize {
self.coef.hop_size()
}
fn fft_size(&self) -> usize {
self.coef.fft_size()
}
}
impl SlidingStftContext {
pub fn batch_size(&self) -> usize {
self.queue.dims()[0]
}
pub fn reset(&mut self) {
self.queue = Tensor::zeros_like(&self.queue);
}
pub fn forward(
&mut self,
hop: Tensor<2>,
) -> Tensor<3> {
#[cfg(any(test, debug_assertions))]
crate::contracts::assert_shape_contract!(
["batch", "hop_size"],
&hop,
&[("batch", self.batch_size()), ("hop_size", self.hop_size())],
);
let hop_size = self.hop_size();
if self.win_len() == hop_size {
self.queue = hop;
} else {
self.queue.inplace(|q| {
let keep = q.slice_dim(1, hop_size as isize..);
Tensor::cat(vec![keep, hop], 1)
});
}
let x = self.coef.analyze(self.queue.clone());
x.squeeze_dim(1)
}
pub fn forward_sequence(
&mut self,
hops: Tensor<3>,
) -> Tensor<4> {
#[cfg(any(test, debug_assertions))]
let [steps] = crate::contracts::unpack_shape_contract!(
["steps", "batch", "hop_size"],
&hops,
&["steps"],
&[("batch", self.batch_size()), ("hop_size", self.hop_size())],
);
let stream = hops.swap_dims(0, 1);
let stream = stream.flatten::<2>(1, 2);
let win_len = self.win_len();
let queue = self.queue.extract();
let ext = Tensor::cat(vec![queue, stream], 1);
self.queue = ext.clone().slice_dim(1, -(win_len as isize)..);
let x = self.coef.analyze(ext);
let x = x.slice_dim(1, 1..);
let x = x.swap_dims(0, 1);
#[cfg(any(test, debug_assertions))]
crate::contracts::assert_shape_contract!(
["steps", "batch", "n_bins", 2],
&x,
&[
("steps", steps),
("batch", self.batch_size()),
("n_bins", self.n_bins())
],
);
x
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use super::*;
use crate::{
errors::{
BunsenErrorKind,
testing::ErrorMatcher,
},
support::testing::{
DeviceMemoryGuard,
cpu_device,
performance_device,
},
};
type F = f32;
#[test]
fn test_config_meta() {
let cfg = SlidingStftConfig::new();
assert_eq!(cfg.win_len, 768);
assert_eq!(cfg.hop_size, 256);
assert_eq!(cfg.fft_size, 1024);
assert_eq!(cfg.window, StftWindowConfig::Hann { periodic: true });
assert_eq!(cfg.win_len(), 768);
assert_eq!(cfg.hop_size(), 256);
assert_eq!(cfg.fft_size(), 1024);
assert_eq!(cfg.n_bins(), 513);
cfg.validate().unwrap();
}
#[test]
fn test_validate_rejects_bad_geometry() {
for bad in [
SlidingStftConfig::new().with_hop_size(0),
SlidingStftConfig::new().with_hop_size(1024),
SlidingStftConfig::new().with_win_len(2048),
SlidingStftConfig::new()
.with_fft_size(1023)
.with_win_len(768),
SlidingStftConfig::new()
.with_fft_size(1536)
.with_win_len(768),
] {
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.has_cause::<ConstraintError>()
.assert_err(&bad.validate());
}
}
#[test]
fn test_init_meta_matches_config() {
let device = cpu_device();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft = cfg.init(&device);
assert_eq!(coef.win_len(), cfg.win_len());
assert_eq!(coef.hop_size(), cfg.hop_size());
assert_eq!(coef.fft_size(), cfg.fft_size());
assert_eq!(coef.n_bins(), cfg.n_bins());
assert_eq!(coef.window.dims(), [64]);
let window: Vec<f64> = coef.window.to_data().try_into_vec_as::<f64>().unwrap();
let host = cfg.window.to_vec_window(48);
for (n, (&w, &h)) in window.iter().zip(&host).enumerate() {
assert!((w - h).abs() <= 1e-6, "window[{n}]: {w} vs {h}");
}
assert!(window[48..].iter().all(|&v| v == 0.0));
let stft = coef.init_state(3);
assert_eq!(stft.win_len(), cfg.win_len());
assert_eq!(stft.hop_size(), cfg.hop_size());
assert_eq!(stft.fft_size(), cfg.fft_size());
assert_eq!(stft.n_bins(), cfg.n_bins());
assert_eq!(stft.batch_size(), 3);
assert_eq!(stft.queue.dims(), [3, 48]);
stft.queue
.to_data()
.assert_eq(&Tensor::<2>::zeros([3, 48], &device).to_data(), true);
}
#[test]
fn test_module_semantics() {
let device = cpu_device();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft = cfg.init(&device);
let ctx = coef.clone().init_state(2);
assert_eq!(coef.devices(), vec![device.clone()]);
assert_eq!(ctx.devices(), vec![device.clone()]);
assert_eq!(coef.num_params(), 0);
assert_eq!(ctx.num_params(), 0);
let window_before = coef.window.to_data_as::<F>();
let queue_before = ctx.queue.to_data_as::<F>();
let moved = coef.clone().to_device(&device);
assert_eq!(moved.devices(), vec![device.clone()]);
moved
.window
.to_data_as::<F>()
.assert_eq(&window_before, true);
assert_eq!(moved.win_len(), coef.win_len());
assert_eq!(moved.hop_size(), coef.hop_size());
assert_eq!(moved.fft_size(), coef.fft_size());
let moved_ctx = ctx.clone().to_device(&device);
assert_eq!(moved_ctx.devices(), vec![device.clone()]);
moved_ctx
.queue
.to_data_as::<F>()
.assert_eq(&queue_before, true);
moved_ctx
.coef
.window
.to_data_as::<F>()
.assert_eq(&window_before, true);
assert_eq!(moved_ctx.batch_size(), ctx.batch_size());
}
#[test]
#[should_panic(expected = "batch_size must be non-zero")]
fn test_init_state_rejects_zero_batch() {
let device = cpu_device();
let coef: SlidingStft = SlidingStftConfig::new().init(&device);
coef.init_state(0);
}
struct HostStft {
win_len: usize,
hop_size: usize,
fft_size: usize,
window: Vec<f64>,
queue: Vec<f64>,
}
impl HostStft {
fn new(cfg: &SlidingStftConfig) -> Self {
Self {
win_len: cfg.win_len,
hop_size: cfg.hop_size,
fft_size: cfg.fft_size,
window: cfg.window.to_vec_window(cfg.win_len),
queue: vec![0.0; cfg.win_len],
}
}
fn push(
&mut self,
hop: &[f64],
) -> Vec<f64> {
assert_eq!(hop.len(), self.hop_size);
self.queue.drain(..self.hop_size);
self.queue.extend(hop.iter().map(|&v| v as f64));
let n_bins = self.fft_size / 2 + 1;
let mut out = Vec::with_capacity(2 * n_bins);
for k in 0..n_bins {
let (mut re, mut im) = (0.0f64, 0.0f64);
for n in 0..self.win_len {
let x = self.queue[n] * self.window[n] as f64;
let theta = core::f64::consts::TAU * ((n * k) % self.fft_size) as f64
/ self.fft_size as f64;
re += x * theta.cos();
im -= x * theta.sin();
}
out.push(re);
out.push(im);
}
out
}
}
fn sample(
batch: usize,
step: usize,
idx: usize,
) -> f64 {
((batch * 7919 + step * 104729 + idx * 1299709) % 1000) as f64 - 500.0
}
fn host_analyze(
row: &[f64],
cfg: &SlidingStftConfig,
) -> Vec<f64> {
let window = cfg.window.to_vec_window(cfg.win_len);
let frames = 1 + (row.len() - cfg.win_len) / cfg.hop_size;
let n_bins = cfg.n_bins();
let mut out = Vec::with_capacity(frames * n_bins * 2);
for f in 0..frames {
for k in 0..n_bins {
let (mut re, mut im) = (0.0f64, 0.0f64);
for n in 0..cfg.win_len {
let x = row[f * cfg.hop_size + n] * window[n];
let theta = core::f64::consts::TAU * ((n * k) % cfg.fft_size) as f64
/ cfg.fft_size as f64;
re += x * theta.cos();
im -= x * theta.sin();
}
out.push(re);
out.push(im);
}
}
out
}
#[test]
fn test_analyze_frame_count_and_shape() {
let device = cpu_device();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft = cfg.init(&device);
for (samples, frames) in [(48, 1), (64, 2), (80, 3), (88, 3), (95, 3), (96, 4)] {
let x = Tensor::<2>::from_data(
TensorData::new(vec![0.0f64; 2 * samples], [2, samples]),
&device,
);
assert_eq!(
coef.analyze(x).dims(),
[2, frames, cfg.n_bins(), 2],
"samples = {samples}",
);
}
}
#[test]
fn test_analyze_matches_naive_dft() {
let device = cpu_device();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft = cfg.init(&device);
let batch = 2;
let samples = 88;
let rows: Vec<Vec<f64>> = (0..batch)
.map(|b| (0..samples).map(|i| sample(b, 0, i)).collect())
.collect();
let got = coef.analyze(Tensor::<2>::from_data(
TensorData::new(rows.concat(), [batch, samples]),
&device,
));
let expected: Vec<f64> = rows.iter().flat_map(|r| host_analyze(r, &cfg)).collect();
let frames = 1 + (samples - cfg.win_len) / cfg.hop_size;
got.to_data_as::<F>().assert_approx_eq::<F>(
&TensorData::new(expected, [batch, frames, cfg.n_bins(), 2]).convert::<F>(),
Tolerance::permissive(),
);
}
#[test]
#[serial_test::serial]
fn test_analyze_ragged_batch_matches_single_rows() {
type PF = f32;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let cfg = SlidingStftConfig::new();
let coef: SlidingStft = cfg.init(&device);
let batch = 3;
let samples = cfg.win_len + 8;
assert_ne!(
(samples - cfg.win_len) % cfg.hop_size,
0,
"geometry must be ragged"
);
let rows: Vec<Vec<f64>> = (0..batch)
.map(|b| (0..samples).map(|i| sample(b, 0, i)).collect())
.collect();
let batched = coef.analyze(Tensor::<2>::from_data(
TensorData::new(rows.concat(), [batch, samples]),
&device,
));
for (b, row) in rows.iter().enumerate() {
let alone = coef.analyze(Tensor::<2>::from_data(
TensorData::new(row.clone(), [1, samples]),
&device,
));
let got = batched.clone().slice_dim(0, b as isize..b as isize + 1);
assert_eq!(got.dims(), alone.dims());
got.to_data_as::<PF>()
.assert_approx_eq::<PF>(&alone.to_data_as::<PF>(), Tolerance::permissive());
}
}
#[test]
fn test_forward_matches_naive_dft() {
let device = cpu_device();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let batch = 2;
let n_bins = cfg.n_bins();
let coef: SlidingStft = cfg.init(&device);
let mut stft = coef.init_state(batch);
let mut hosts: Vec<HostStft> = (0..batch).map(|_| HostStft::new(&cfg)).collect();
for step in 0..5 {
let rows: Vec<Vec<f64>> = (0..batch)
.map(|b| (0..cfg.hop_size).map(|i| sample(b, step, i)).collect())
.collect();
let hop = Tensor::<2>::from_data(
TensorData::new(rows.concat(), [batch, cfg.hop_size]),
&device,
);
let out = stft.forward(hop);
assert_eq!(out.dims(), [batch, n_bins, 2]);
let expected: Vec<f64> = hosts
.iter_mut()
.zip(&rows)
.flat_map(|(host, row)| host.push(row))
.collect();
out.to_data_as::<F>().assert_approx_eq::<F>(
&TensorData::new(expected, [batch, n_bins, 2]).convert::<F>(),
Tolerance::permissive(),
);
}
}
#[test]
fn test_forward_sequence_matches_stepwise() {
let device = cpu_device();
let steps = 7;
let batch = 2;
for cfg in [
SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64),
SlidingStftConfig::new()
.with_win_len(16)
.with_hop_size(16)
.with_fft_size(16),
] {
let hops =
Tensor::<3>::random([steps, batch, cfg.hop_size], Distribution::Default, &device);
let coef: SlidingStft = cfg.init(&device);
let mut seq_stft = coef.init_state(batch);
let mut step_stft = seq_stft.clone();
let seq_out = seq_stft.forward_sequence(hops.clone());
assert_eq!(seq_out.dims(), [steps, batch, cfg.n_bins(), 2]);
let mut step_outs = Vec::with_capacity(steps);
for step in 0..steps {
let hop = hops.clone().select_dim(0, step);
step_outs.push(step_stft.forward(hop));
}
let step_out: Tensor<4> = Tensor::stack(step_outs, 0);
let tol = Tolerance::<F>::permissive();
seq_out
.to_data_as::<F>()
.assert_approx_eq::<F>(&step_out.to_data_as::<F>(), tol);
seq_stft
.queue
.to_data_as::<F>()
.assert_approx_eq::<F>(&step_stft.queue.to_data_as::<F>(), tol);
}
}
#[test]
fn test_reset() {
let device = cpu_device();
let cfg = SlidingStftConfig::new()
.with_win_len(48)
.with_hop_size(16)
.with_fft_size(64);
let coef: SlidingStft = cfg.init(&device);
let mut stft = coef.init_state(1);
let hop = Tensor::random([1, 16], Distribution::Default, &device);
stft.forward(hop);
stft.reset();
stft.queue.to_data_as::<F>().assert_eq(
&Tensor::<2>::zeros([1, 48], &device).to_data_as::<F>(),
true,
);
}
}