use burn::config::Config;
use crate::errors::{
BunsenError,
BunsenResult,
ConstraintError,
Rule,
};
#[derive(Config, Debug, PartialEq)]
pub struct SortformerStreamingConfig {
pub chunk_len: usize,
pub right_context: usize,
#[config(default = "0")]
pub left_context: usize,
pub fifo_len: usize,
pub spkcache_len: usize,
pub update_period: usize,
#[config(default = "1")]
pub silence_frames_per_speaker: usize,
#[config(default = "0.25")]
pub pred_score_threshold: f64,
#[config(default = "0.05")]
pub latest_boost: f64,
#[config(default = "0.75")]
pub strong_boost_rate: f64,
#[config(default = "1.5")]
pub weak_boost_rate: f64,
#[config(default = "0.5")]
pub min_pos_scores_rate: f64,
#[config(default = "0.2")]
pub silence_threshold: f64,
}
impl SortformerStreamingConfig {
pub fn nemotron_low_latency() -> Self {
Self::new(9, 4, 264, 264, 222)
}
pub fn nemotron_very_low_latency() -> Self {
Self::new(6, 2, 264, 264, 222)
}
pub fn nemotron_ultra_low_latency() -> Self {
Self::new(3, 1, 264, 264, 222)
}
pub fn nemotron_offline() -> Self {
Self::new(340, 40, 40, 264, 300)
}
fn v2(
chunk_len: usize,
right_context: usize,
fifo_len: usize,
update_period: usize,
) -> Self {
Self::new(chunk_len, right_context, fifo_len, 188, update_period)
.with_left_context(1)
.with_silence_frames_per_speaker(3)
}
pub fn sortformer_v2_very_high_latency() -> Self {
Self::v2(340, 40, 40, 300)
}
pub fn sortformer_v2_high_latency() -> Self {
Self::v2(124, 1, 124, 124)
}
pub fn sortformer_v2_low_latency() -> Self {
Self::v2(6, 7, 188, 144)
}
pub fn sortformer_v2_ultra_low_latency() -> Self {
Self::v2(3, 1, 188, 144)
}
pub fn latency_frames(&self) -> usize {
self.chunk_len + self.right_context
}
pub fn speaker_budget(
&self,
n_speakers: usize,
) -> usize {
(self.spkcache_len / n_speakers).saturating_sub(self.silence_frames_per_speaker)
}
pub fn check(
&self,
n_speakers: usize,
) -> BunsenResult<()> {
for (field, value) in [
("chunk_len", self.chunk_len),
("update_period", self.update_period),
] {
if value == 0 {
return Err(
ConstraintError::zero_or_empty("SortformerStreamingConfig", field).into(),
);
}
}
let least = (1 + self.silence_frames_per_speaker) * n_speakers;
if self.spkcache_len < least {
return Err(ConstraintError::new(
"SortformerStreamingConfig",
"spkcache_len",
Rule::Relation {
lhs: ("spkcache_len".into(), self.spkcache_len.to_string()),
op: ">=",
rhs: (
"(1 + silence_frames_per_speaker) * n_speakers".into(),
least.to_string(),
),
},
)
.into());
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub enum SortformerLatency {
UltraLow,
VeryLow,
#[default]
Low,
High,
Offline,
}
impl SortformerLatency {
pub const ALL: [Self; 5] = [
Self::UltraLow,
Self::VeryLow,
Self::Low,
Self::High,
Self::Offline,
];
}
impl std::fmt::Display for SortformerLatency {
fn fmt(
&self,
f: &mut std::fmt::Formatter<'_>,
) -> std::fmt::Result {
f.write_str(match self {
Self::UltraLow => "ultra-low",
Self::VeryLow => "very-low",
Self::Low => "low",
Self::High => "high",
Self::Offline => "offline",
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum SortformerFamily {
Nemotron,
SortformerV2,
}
impl SortformerFamily {
pub fn preset(
self,
latency: SortformerLatency,
) -> BunsenResult<SortformerStreamingConfig> {
self.lookup(latency).ok_or_else(|| {
let has = self
.latencies()
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(", ");
BunsenError::unsupported(format!(
"{self} has no {latency}-latency preset; it has {has}"
))
})
}
pub fn latencies(self) -> Vec<SortformerLatency> {
SortformerLatency::ALL
.into_iter()
.filter(|&l| self.lookup(l).is_some())
.collect()
}
fn lookup(
self,
latency: SortformerLatency,
) -> Option<SortformerStreamingConfig> {
use SortformerLatency as L;
type C = SortformerStreamingConfig;
match (self, latency) {
(Self::Nemotron, L::UltraLow) => Some(C::nemotron_ultra_low_latency()),
(Self::Nemotron, L::VeryLow) => Some(C::nemotron_very_low_latency()),
(Self::Nemotron, L::Low) => Some(C::nemotron_low_latency()),
(Self::Nemotron, L::Offline) => Some(C::nemotron_offline()),
(Self::SortformerV2, L::UltraLow) => Some(C::sortformer_v2_ultra_low_latency()),
(Self::SortformerV2, L::Low) => Some(C::sortformer_v2_low_latency()),
(Self::SortformerV2, L::High) => Some(C::sortformer_v2_high_latency()),
(Self::SortformerV2, L::Offline) => Some(C::sortformer_v2_very_high_latency()),
(Self::Nemotron, L::High) | (Self::SortformerV2, L::VeryLow) => None,
}
}
}
impl std::fmt::Display for SortformerFamily {
fn fmt(
&self,
f: &mut std::fmt::Formatter<'_>,
) -> std::fmt::Result {
f.write_str(match self {
Self::Nemotron => "Nemotron-3-Diarization",
Self::SortformerV2 => "Sortformer v2",
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_presets() {
let low = SortformerStreamingConfig::nemotron_low_latency();
assert_eq!(low.latency_frames(), 13);
assert_eq!(low.speaker_budget(8), 32);
for preset in [
low,
SortformerStreamingConfig::nemotron_very_low_latency(),
SortformerStreamingConfig::nemotron_ultra_low_latency(),
SortformerStreamingConfig::nemotron_offline(),
] {
preset.check(8).unwrap();
}
assert_eq!(
SortformerStreamingConfig::nemotron_offline().latency_frames(),
380
);
assert_eq!(
SortformerStreamingConfig::nemotron_low_latency().left_context,
0
);
}
#[test]
fn test_v2_presets() {
let presets = [
(
SortformerStreamingConfig::sortformer_v2_very_high_latency(),
380,
),
(SortformerStreamingConfig::sortformer_v2_high_latency(), 125),
(SortformerStreamingConfig::sortformer_v2_low_latency(), 13),
(
SortformerStreamingConfig::sortformer_v2_ultra_low_latency(),
4,
),
];
for (preset, latency) in presets {
preset.check(4).unwrap();
assert_eq!(preset.latency_frames(), latency);
assert_eq!(preset.left_context, 1);
assert_eq!(preset.speaker_budget(4), 44);
}
}
#[test]
fn test_family_presets() {
use SortformerLatency as L;
type C = SortformerStreamingConfig;
let nemotron = SortformerFamily::Nemotron;
assert_eq!(
nemotron.latencies(),
[L::UltraLow, L::VeryLow, L::Low, L::Offline]
);
assert_eq!(nemotron.preset(L::Low).unwrap(), C::nemotron_low_latency());
assert_eq!(nemotron.preset(L::Offline).unwrap(), C::nemotron_offline());
let err = nemotron.preset(L::High).unwrap_err();
assert!(
err.to_string()
.contains("no high-latency preset; it has ultra-low, very-low, low, offline"),
"{err}"
);
let v2 = SortformerFamily::SortformerV2;
assert_eq!(v2.latencies(), [L::UltraLow, L::Low, L::High, L::Offline]);
assert_eq!(v2.preset(L::High).unwrap(), C::sortformer_v2_high_latency());
assert_eq!(
v2.preset(L::Offline).unwrap(),
C::sortformer_v2_very_high_latency()
);
assert!(v2.preset(L::VeryLow).is_err());
for latency in L::ALL {
let frames: Vec<usize> = [nemotron, v2]
.into_iter()
.filter_map(|f| f.preset(latency).ok())
.map(|c| c.latency_frames())
.collect();
assert!(frames.windows(2).all(|w| w[0] == w[1]), "{latency}");
}
}
#[test]
fn test_check() {
let c = SortformerStreamingConfig::new(3, 1, 6, 12, 4);
c.check(3).unwrap();
assert!(c.check(7).is_err());
let empty_chunk = SortformerStreamingConfig {
chunk_len: 0,
..c.clone()
};
assert!(empty_chunk.check(3).is_err());
let no_period = SortformerStreamingConfig {
update_period: 0,
..c
};
assert!(no_period.check(3).is_err());
}
}