use alloc::sync::Arc;
use burn::tensor::Device;
use serde::{
Deserialize,
Serialize,
};
use super::{
SortformerFamily,
SortformerStream,
SortformerStreamingConfig,
blocks::{
Sortformer,
SortformerMeta,
},
};
use crate::{
burner::module::ModuleInit,
data::catalog::presets::PresetSelection,
errors::{
BunsenError,
BunsenResult,
ConstraintError,
Rule,
},
kits::speech::{
diarization::{
SpeakerActivity,
SpeakerActivityModel,
SpeakerActivityStream,
SpeakerTurn,
TurnFilterConfig,
},
nemo::{
NemoMelFrontEnd,
NemoMelFrontEndConfig,
NemoMelFrontEndMeta,
},
},
};
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct SortformerPolicy {
pub streaming: Option<SortformerStreamingConfig>,
pub turns: TurnFilterConfig,
}
#[derive(Debug)]
pub struct SortformerConstruct {
pub model: Arc<Sortformer>,
pub front_end: NemoMelFrontEnd,
pub family: SortformerFamily,
pub policy: SortformerPolicy,
pub presets: PresetSelection,
}
impl SortformerConstruct {
pub fn new(
model: Sortformer,
family: SortformerFamily,
front_end: &NemoMelFrontEndConfig,
device: &Device,
) -> BunsenResult<Self> {
if front_end.n_mels() != model.n_mels() {
return Err(ConstraintError::new(
"SortformerConstruct",
"front_end",
Rule::Relation {
lhs: ("front_end.n_mels".into(), front_end.n_mels().to_string()),
op: "==",
rhs: ("model.n_mels()".into(), model.n_mels().to_string()),
},
)
.into());
}
Ok(Self {
model: Arc::new(model),
front_end: front_end.try_init(device)?,
family,
policy: SortformerPolicy::default(),
presets: PresetSelection::default(),
})
}
pub fn with_policy(
mut self,
policy: SortformerPolicy,
presets: PresetSelection,
) -> Self {
self.policy = policy;
self.presets = presets;
self
}
pub fn streaming(&self) -> BunsenResult<SortformerStreamingConfig> {
self.policy.streaming.clone().ok_or_else(|| {
BunsenError::policy(format!(
"{} has no streaming latency: select a `latency` preset",
self.family
))
})
}
pub fn turns(&self) -> &TurnFilterConfig {
&self.policy.turns
}
pub fn sample_rate(&self) -> usize {
self.front_end.sample_rate()
}
pub fn diarizer(
&self,
streaming: SortformerStreamingConfig,
) -> BunsenResult<SortformerDiarizer> {
streaming.check(self.model.n_speakers())?;
Ok(SortformerDiarizer {
model: self.model.clone(),
front_end: self.front_end.clone(),
streaming,
})
}
pub fn diarize(
&self,
samples: &[f32],
streaming: SortformerStreamingConfig,
filter: &TurnFilterConfig,
) -> BunsenResult<Vec<SpeakerTurn>> {
let diarizer = self.diarizer(streaming)?;
let activity = diarizer.activity(samples)?;
let frame_seconds = activity.frame_samples as f64 / self.sample_rate() as f64;
Ok(filter.turns(&activity.probs, activity.n_speakers, frame_seconds))
}
}
impl core::fmt::Display for SortformerConstruct {
fn fmt(
&self,
f: &mut core::fmt::Formatter<'_>,
) -> core::fmt::Result {
let m = &self.model;
write!(
f,
"{}: {} speakers, {} mels, {}-wide encoder, {} ms frames",
self.family,
m.n_speakers(),
m.n_mels(),
m.d_model(),
1000 * self.front_end.hop() * m.subsampling_factor()
/ m.output_upsampling()
/ self.sample_rate()
)
}
}
#[derive(Clone, Debug)]
pub struct SortformerDiarizer {
model: Arc<Sortformer>,
front_end: NemoMelFrontEnd,
streaming: SortformerStreamingConfig,
}
impl SortformerDiarizer {
pub fn streaming(&self) -> &SortformerStreamingConfig {
&self.streaming
}
pub fn stream(&self) -> BunsenResult<SortformerStream> {
SortformerStream::new(
self.model.clone(),
self.front_end.clone(),
self.streaming.clone(),
)
}
pub fn activity(
&self,
samples: &[f32],
) -> BunsenResult<SpeakerActivity> {
let mut stream = self.stream()?;
let mut activity = stream.push(samples)?;
activity.append(stream.finish()?)?;
Ok(activity)
}
}
impl SpeakerActivityModel for SortformerDiarizer {
fn sample_rate(&self) -> usize {
self.front_end.sample_rate()
}
fn frame_samples(&self) -> usize {
self.front_end.hop() * self.model.subsampling_factor() / self.model.output_upsampling()
}
fn max_speakers(&self) -> usize {
self.model.n_speakers()
}
fn new_stream(&self) -> BunsenResult<Box<dyn SpeakerActivityStream>> {
Ok(Box::new(self.stream()?))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
kits::speech::diarization::sortformer::NemotronDiarizationConfig,
support::testing::cpu_device,
};
fn bundle() -> SortformerConstruct {
let model: Sortformer = NemotronDiarizationConfig::new()
.with_n_mels(24)
.with_d_model(16)
.with_n_layers(1)
.with_n_heads(2)
.with_d_hidden(8)
.with_n_speakers(3)
.init(&cpu_device());
SortformerConstruct::new(
model,
SortformerFamily::Nemotron,
&NemoMelFrontEndConfig::nemotron_diarization().with_n_mels(24),
&cpu_device(),
)
.unwrap()
}
#[test]
fn test_diarizer_is_a_speaker_activity_model() {
let bundle = bundle();
let diarizer = bundle
.diarizer(SortformerStreamingConfig::new(3, 1, 6, 12, 4))
.unwrap();
assert_eq!(diarizer.sample_rate(), 16_000);
assert_eq!(diarizer.frame_samples(), 160);
assert_eq!(diarizer.max_speakers(), 3);
assert!(
bundle
.diarizer(SortformerStreamingConfig::new(3, 1, 6, 5, 4))
.is_err()
);
assert!(bundle.to_string().contains("3 speakers"));
assert!(bundle.to_string().contains("10 ms"));
let samples = vec![0.0f32; 16_000];
let mut stream = diarizer.new_stream().unwrap();
let mut activity = stream.push(&samples).unwrap();
activity.append(stream.finish().unwrap()).unwrap();
assert_eq!(activity.frames(), 100);
assert_eq!(
activity.frames(),
diarizer.activity(&samples).unwrap().frames()
);
let turns = bundle
.diarize(
&samples,
SortformerStreamingConfig::new(3, 1, 6, 12, 4),
&TurnFilterConfig::none(),
)
.unwrap();
assert!(turns.iter().all(|t| t.end <= 1.0 + 1e-9));
}
#[test]
fn test_policy_and_front_end() {
let bundle = bundle();
assert!(bundle.to_string().starts_with("Nemotron-3-Diarization: "));
crate::errors::testing::ErrorMatcher::kind(crate::errors::BunsenErrorKind::Policy)
.message_contains("select a `latency` preset")
.assert_err(&bundle.streaming());
assert_eq!(bundle.turns(), &TurnFilterConfig::none());
let streaming = SortformerStreamingConfig::new(3, 1, 6, 12, 4);
let bundle = bundle.with_policy(
SortformerPolicy {
streaming: Some(streaming.clone()),
turns: TurnFilterConfig::none().with_onset(0.6),
},
PresetSelection::of(["low"]),
);
assert_eq!(bundle.streaming().unwrap(), streaming);
assert_eq!(bundle.turns().onset, 0.6);
assert_eq!(bundle.presets.presets, ["low"]);
let model = Arc::try_unwrap(bundle.model).unwrap();
let err = SortformerConstruct::new(
model,
SortformerFamily::Nemotron,
&NemoMelFrontEndConfig::nemotron_diarization(),
&cpu_device(),
)
.unwrap_err();
assert!(err.to_string().contains("front_end.n_mels"), "{err}");
}
}