use alloc::sync::Arc;
use burn::{
Tensor,
module::Module,
};
use super::{
SilenceFill,
SortformerStreamingConfig,
SpeakerCache,
blocks::{
Sortformer,
SortformerMeta,
},
};
use crate::{
burner::tensor::cat_optional,
contracts::unpack_shape_contract,
errors::{
BunsenError,
BunsenResult,
},
kits::speech::{
asr::driver::StreamingFeatures,
diarization::{
SpeakerActivity,
SpeakerActivityStream,
},
nemo::{
NemoMelFeatures,
NemoMelFrontEnd,
NemoMelFrontEndMeta,
},
},
};
#[derive(Clone, Debug)]
pub struct SortformerMelStream {
model: Arc<Sortformer>,
cache: SpeakerCache,
mels: Option<Tensor<2>>,
mels_start: usize,
next_chunk: usize,
mel_frames: usize,
emitted: usize,
finished: bool,
}
impl SortformerMelStream {
pub fn new(
model: Arc<Sortformer>,
config: SortformerStreamingConfig,
) -> BunsenResult<Self> {
let silence = match &model.silence {
Some(silence) => SilenceFill::Learned(silence.val()),
None => SilenceFill::running_mean(model.d_model(), &model.projection.weight.device()),
};
let cache = SpeakerCache::new(config, model.n_speakers(), silence)?;
Ok(Self {
model,
cache,
mels: None,
mels_start: 0,
next_chunk: 0,
mel_frames: 0,
emitted: 0,
finished: false,
})
}
pub fn model(&self) -> &Arc<Sortformer> {
&self.model
}
pub fn cache(&self) -> &SpeakerCache {
&self.cache
}
pub fn mels_per_output(&self) -> usize {
self.model.subsampling_factor() / self.model.output_upsampling()
}
pub fn push(
&mut self,
mels: Tensor<2>,
) -> BunsenResult<Vec<f32>> {
if self.finished {
return Err(BunsenError::illegal(
"the stream has ended; nothing more can be fed",
));
}
let [frames] = unpack_shape_contract!(
["frames", "n_mels"],
&mels.dims(),
&["frames"],
&[("n_mels", self.model.n_mels())],
);
if frames == 0 {
return Ok(Vec::new());
}
self.mel_frames += frames;
self.mels = cat_optional(self.mels.take(), Some(mels), 0);
let c = self.cache.config().clone();
let factor = self.model.subsampling_factor();
let (chunk, look_ahead) = (c.chunk_len * factor, c.right_context * factor);
let mut out = Vec::new();
while self.next_chunk + chunk + look_ahead <= self.mel_frames {
out.extend(self.step(self.next_chunk + chunk, look_ahead)?);
}
Ok(out)
}
pub fn finish(&mut self) -> BunsenResult<Vec<f32>> {
if self.finished {
return Ok(Vec::new());
}
self.finished = true;
let c = self.cache.config().clone();
let factor = self.model.subsampling_factor();
let mut out = Vec::new();
while self.next_chunk < self.mel_frames {
let end = (self.next_chunk + c.chunk_len * factor).min(self.mel_frames);
let look_ahead = (c.right_context * factor).min(self.mel_frames - end);
out.extend(self.step(end, look_ahead)?);
}
let total = self.mel_frames.div_ceil(self.mels_per_output());
let s = self.model.n_speakers();
let over = self.emitted.saturating_sub(total);
out.truncate(out.len() - over.min(out.len() / s) * s);
self.emitted -= over;
Ok(out)
}
pub fn emitted(&self) -> usize {
self.emitted
}
fn step(
&mut self,
end: usize,
look_ahead: usize,
) -> BunsenResult<Vec<f32>> {
let factor = self.model.subsampling_factor();
let left_context = self.cache.config().left_context * factor;
let start = self.next_chunk;
let left = left_context.min(start);
let mels = self.mels.clone().expect("mel frames are pending");
let window = mels.slice_dim(
0,
start - left - self.mels_start..end + look_ahead - self.mels_start,
);
let (embs, _) = self.model.pre_encode(window.unsqueeze_dim(0), None);
let embs = embs.squeeze_dim::<2>(0);
let n_embs = embs.dims()[0];
let (left_enc, right_enc) = (
(left as f64 / factor as f64).round() as usize,
look_ahead.div_ceil(factor),
);
let chunk = n_embs - left_enc - right_enc;
let chunk_embs = embs.clone().slice_dim(0, left_enc..left_enc + chunk);
let cached = self.cache.cached();
let n_cached = cached.as_ref().map_or(0, |c| c.dims()[0]);
let input = cat_optional(cached, Some(embs), 0).expect("the window is present");
let probs = self.model.forward_pre_encoded(input.unsqueeze_dim(0), None);
let up = self.model.output_upsampling();
let s = self.model.n_speakers();
let first = n_cached + left_enc;
let out: Vec<f32> = probs
.output
.slice_dim(1, first * up..(first + chunk) * up)
.into_data()
.try_into_vec_as::<f32>()
.expect("read back f32 probabilities");
let step: Vec<f32> = Tensor::cat(
vec![
probs.encoder_rate.clone().slice_dim(1, 0..n_cached),
probs.encoder_rate.slice_dim(1, first..first + chunk),
],
1,
)
.into_data()
.try_into_vec_as::<f32>()
.expect("read back f32 probabilities");
self.cache.update(&step, chunk_embs)?;
self.emitted += out.len() / s;
self.next_chunk = end;
let keep_from = end.saturating_sub(left_context).max(self.mels_start);
let n_mels = mels_len(&self.mels);
self.mels = (keep_from - self.mels_start < n_mels).then(|| {
self.mels
.take()
.expect("mel frames are pending")
.slice_dim(0, keep_from - self.mels_start..n_mels)
});
self.mels_start = keep_from;
Ok(out)
}
}
fn mels_len(mels: &Option<Tensor<2>>) -> usize {
mels.as_ref().map_or(0, |m| m.dims()[0])
}
#[derive(Clone, Debug)]
pub struct SortformerStream {
features: NemoMelFeatures,
hop: usize,
mels: SortformerMelStream,
samples: usize,
handed: usize,
held: Option<Tensor<2>>,
}
impl SortformerStream {
pub fn new(
model: Arc<Sortformer>,
front_end: NemoMelFrontEnd,
config: SortformerStreamingConfig,
) -> BunsenResult<Self> {
if front_end.n_mels() != model.n_mels() {
return Err(BunsenError::illegal(format!(
"SortformerStream: the front end makes {} mels, the model reads {}",
front_end.n_mels(),
model.n_mels()
)));
}
let device = model.devices()[0].clone();
let hop = front_end.hop();
Ok(Self {
features: NemoMelFeatures::new(front_end, device),
hop,
mels: SortformerMelStream::new(model, config)?,
samples: 0,
handed: 0,
held: None,
})
}
pub fn mel_stream(&self) -> &SortformerMelStream {
&self.mels
}
pub fn frame_samples(&self) -> usize {
self.hop * self.mels.mels_per_output()
}
fn activity(
&self,
probs: Vec<f32>,
) -> BunsenResult<SpeakerActivity> {
let s = self.mels.model.n_speakers();
let frames = probs.len() / s;
let start = (self.mels.emitted() - frames) * self.frame_samples();
SpeakerActivity::try_new(start, self.frame_samples(), s, probs)
}
fn hand_on(
&mut self,
limit: usize,
) -> BunsenResult<Vec<f32>> {
let Some(held) = self.held.take() else {
return Ok(Vec::new());
};
let n = held.dims()[0];
let take = limit.saturating_sub(self.handed).min(n);
if take < n {
self.held = Some(held.clone().slice_dim(0, take..n));
}
if take == 0 {
return Ok(Vec::new());
}
self.handed += take;
self.mels.push(held.slice_dim(0, 0..take))
}
fn hold(
&mut self,
frames: Option<Tensor<3>>,
) {
if let Some(frames) = frames {
self.held = cat_optional(self.held.take(), Some(frames.squeeze_dim(0)), 0);
}
}
}
impl SpeakerActivityStream for SortformerStream {
fn push(
&mut self,
samples: &[f32],
) -> BunsenResult<SpeakerActivity> {
self.samples += samples.len();
let frames = self.features.push(samples)?;
self.hold(frames);
let probs = self.hand_on(self.samples / self.hop)?;
self.activity(probs)
}
fn finish(&mut self) -> BunsenResult<SpeakerActivity> {
let frames = self.features.finish()?;
self.hold(frames);
let mut probs = self.hand_on(self.samples / self.hop)?;
self.held = None;
probs.extend(self.mels.finish()?);
self.activity(probs)
}
}