use burn::config::Config;
#[derive(Clone, Debug, PartialEq)]
pub struct SpeakerTurn {
pub speaker: usize,
pub start: f64,
pub end: f64,
}
impl SpeakerTurn {
pub fn duration(&self) -> f64 {
self.end - self.start
}
}
#[derive(Config, Debug, PartialEq)]
pub struct TurnFilterConfig {
#[config(default = "0.5")]
pub onset: f64,
#[config(default = "0.5")]
pub offset: f64,
#[config(default = "0.0")]
pub pad_onset: f64,
#[config(default = "0.0")]
pub pad_offset: f64,
#[config(default = "0.0")]
pub min_duration_on: f64,
#[config(default = "0.0")]
pub min_duration_off: f64,
#[config(default = "true")]
pub filter_speech_first: bool,
}
impl Default for TurnFilterConfig {
fn default() -> Self {
Self::new()
}
}
impl TurnFilterConfig {
pub fn none() -> Self {
Self::new()
}
pub fn dihard3_v2() -> Self {
Self::new()
.with_onset(0.56)
.with_offset(1.0)
.with_pad_onset(0.063)
.with_pad_offset(0.002)
.with_min_duration_on(0.007)
.with_min_duration_off(0.151)
}
pub fn callhome_v2() -> Self {
Self::new()
.with_onset(0.641)
.with_offset(0.561)
.with_pad_onset(0.229)
.with_pad_offset(0.079)
.with_min_duration_on(0.511)
.with_min_duration_off(0.296)
}
pub fn init(
&self,
n_speakers: usize,
frame_seconds: f64,
) -> TurnFilter {
TurnFilter {
config: self.clone(),
frame_seconds,
frames: 0,
tracks: vec![Track::default(); n_speakers],
}
}
pub fn turns(
&self,
probs: &[f32],
n_speakers: usize,
frame_seconds: f64,
) -> Vec<SpeakerTurn> {
let mut filter = self.init(n_speakers, frame_seconds);
let mut turns = filter.push(probs);
turns.extend(filter.finish());
sort_turns(&mut turns);
turns
}
}
fn sort_turns(turns: &mut [SpeakerTurn]) {
turns.sort_by(|a, b| a.start.total_cmp(&b.start).then(a.speaker.cmp(&b.speaker)));
}
#[derive(Clone, Debug)]
pub struct TurnFilter {
config: TurnFilterConfig,
frame_seconds: f64,
frames: usize,
tracks: Vec<Track>,
}
#[derive(Clone, Debug, Default)]
struct Track {
above: bool,
run_start: Option<usize>,
stretch: Option<(f64, f64)>,
held: Option<(f64, f64)>,
}
impl TurnFilter {
pub fn config(&self) -> &TurnFilterConfig {
&self.config
}
pub fn frames_seen(&self) -> usize {
self.frames
}
pub fn is_speaking(
&self,
speaker: usize,
) -> bool {
self.tracks[speaker].above
}
pub fn push(
&mut self,
probs: &[f32],
) -> Vec<SpeakerTurn> {
let n = self.tracks.len();
assert!(
n > 0 && probs.len().is_multiple_of(n),
"TurnFilter::push: {} probabilities are not frames of {n} speakers",
probs.len()
);
let mut out = Vec::new();
for frame in probs.chunks_exact(n) {
for (speaker, &p) in frame.iter().enumerate() {
self.step(speaker, p, &mut out);
}
self.frames += 1;
}
for speaker in 0..n {
self.release(speaker, false, &mut out);
}
sort_turns(&mut out);
out
}
pub fn finish(&mut self) -> Vec<SpeakerTurn> {
let mut out = Vec::new();
for speaker in 0..self.tracks.len() {
if let Some(start) = self.tracks[speaker].run_start.take() {
self.close_run(speaker, start, self.frames, &mut out);
}
self.tracks[speaker].above = false;
self.release(speaker, true, &mut out);
}
sort_turns(&mut out);
out
}
fn step(
&mut self,
speaker: usize,
p: f32,
out: &mut Vec<SpeakerTurn>,
) {
let (onset, offset) = (self.config.onset as f32, self.config.offset as f32);
let track = &mut self.tracks[speaker];
let was = track.above;
let now = if onset >= offset {
if p > onset {
true
} else if p < offset {
false
} else {
was
}
} else if p >= offset {
true
} else if p <= onset {
false
} else {
!was
};
track.above = now;
let i = self.frames;
if now && !was {
track.run_start = Some(i);
} else if was && !now {
let start = track.run_start.take().expect("an open run");
self.close_run(speaker, start, i, out);
}
}
fn close_run(
&mut self,
speaker: usize,
start: usize,
end: usize,
out: &mut Vec<SpeakerTurn>,
) {
let s = self.padded_start(start);
let e = end as f64 * self.frame_seconds + self.config.pad_offset;
if e <= s {
return;
}
let track = &mut self.tracks[speaker];
match &mut track.stretch {
Some((_, open_end)) if *open_end >= s => *open_end = open_end.max(e),
_ => {
if let Some(done) = track.stretch.replace((s, e)) {
self.filter(speaker, done, out);
}
}
}
}
fn padded_start(
&self,
frame: usize,
) -> f64 {
(frame as f64 * self.frame_seconds - self.config.pad_onset).max(0.0)
}
fn next_run_start(
&self,
speaker: usize,
) -> f64 {
self.padded_start(self.tracks[speaker].run_start.unwrap_or(self.frames))
}
fn release(
&mut self,
speaker: usize,
all: bool,
out: &mut Vec<SpeakerTurn>,
) {
let next = self.next_run_start(speaker);
if let Some((_, e)) = self.tracks[speaker].stretch
&& (all || next > e)
{
let done = self.tracks[speaker].stretch.take().expect("present");
self.filter(speaker, done, out);
}
let next = self.tracks[speaker]
.stretch
.map_or(self.next_run_start(speaker), |(s, _)| s);
if let Some((_, e)) = self.tracks[speaker].held
&& (all || next - e >= self.config.min_duration_off)
{
let (s, e) = self.tracks[speaker].held.take().expect("present");
self.emit_filled(speaker, (s, e), out);
}
}
fn filter(
&mut self,
speaker: usize,
stretch: (f64, f64),
out: &mut Vec<SpeakerTurn>,
) {
if self.config.filter_speech_first && self.too_short(stretch) {
return;
}
if self.config.min_duration_off <= 0.0 {
self.emit_filled(speaker, stretch, out);
return;
}
let track = &mut self.tracks[speaker];
match &mut track.held {
Some((_, held_end)) if stretch.0 - *held_end < self.config.min_duration_off => {
*held_end = held_end.max(stretch.1)
}
_ => {
if let Some(done) = track.held.replace(stretch) {
self.emit_filled(speaker, done, out);
}
}
}
}
fn emit_filled(
&self,
speaker: usize,
(start, end): (f64, f64),
out: &mut Vec<SpeakerTurn>,
) {
if !self.config.filter_speech_first && self.too_short((start, end)) {
return;
}
out.push(SpeakerTurn {
speaker,
start,
end,
});
}
fn too_short(
&self,
(start, end): (f64, f64),
) -> bool {
self.config.min_duration_on > 0.0 && end - start < self.config.min_duration_on
}
}
#[cfg(test)]
mod tests {
use rand::{
RngExt,
SeedableRng,
rngs::StdRng,
};
use super::*;
fn binarize_reference(
track: &[f32],
c: &TurnFilterConfig,
frame: f64,
) -> Vec<(f64, f64)> {
let (onset, offset) = (c.onset as f32, c.offset as f32);
let mut above = Vec::with_capacity(track.len());
let mut state = false;
for &p in track {
state = if onset >= offset {
if p > onset {
true
} else if p < offset {
false
} else {
state
}
} else if p >= offset {
true
} else if p <= onset {
false
} else {
!state
};
above.push(state);
}
let mut segments = Vec::new();
let mut start = None;
for (i, &a) in above.iter().chain(std::iter::once(&false)).enumerate() {
match (start, a) {
(None, true) => start = Some(i),
(Some(s), false) => {
let s = (s as f64 * frame - c.pad_onset).max(0.0);
let e = i as f64 * frame + c.pad_offset;
if e > s {
segments.push((s, e));
}
start = None;
}
_ => {}
}
}
merge_reference(segments)
}
fn merge_reference(mut segments: Vec<(f64, f64)>) -> Vec<(f64, f64)> {
segments.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut merged: Vec<(f64, f64)> = Vec::new();
for (s, e) in segments {
match merged.last_mut() {
Some(last) if last.1 >= s => last.1 = last.1.max(e),
_ => merged.push((s, e)),
}
}
merged
}
fn filter_reference(
segments: Vec<(f64, f64)>,
c: &TurnFilterConfig,
) -> Vec<(f64, f64)> {
let drop_short = |s: Vec<(f64, f64)>| {
if c.min_duration_on > 0.0 {
s.into_iter()
.filter(|(a, b)| b - a >= c.min_duration_on)
.collect()
} else {
s
}
};
let fill_gaps = |s: Vec<(f64, f64)>| {
if c.min_duration_off > 0.0 && !s.is_empty() {
let mut all = s.clone();
for w in s.windows(2) {
if w[1].0 - w[0].1 < c.min_duration_off {
all.push((w[0].1, w[1].0));
}
}
merge_reference(all)
} else {
s
}
};
if c.filter_speech_first {
fill_gaps(drop_short(segments))
} else {
drop_short(fill_gaps(segments))
}
}
fn reference_turns(
probs: &[f32],
n_speakers: usize,
c: &TurnFilterConfig,
frame: f64,
) -> Vec<SpeakerTurn> {
let mut turns = Vec::new();
for speaker in 0..n_speakers {
let track: Vec<f32> = probs
.iter()
.skip(speaker)
.step_by(n_speakers)
.copied()
.collect();
for (start, end) in filter_reference(binarize_reference(&track, c, frame), c) {
turns.push(SpeakerTurn {
speaker,
start,
end,
});
}
}
sort_turns(&mut turns);
turns
}
fn tracks(
rng: &mut StdRng,
frames: usize,
n_speakers: usize,
) -> Vec<f32> {
let mut levels = vec![0.0f32; n_speakers];
let mut probs = Vec::with_capacity(frames * n_speakers);
for _ in 0..frames {
for level in levels.iter_mut() {
if rng.random::<f32>() < 0.08 {
*level = if *level > 0.5 { 0.1 } else { 0.9 };
}
let p = *level + (rng.random::<f32>() - 0.5) * 0.7;
probs.push(p.clamp(0.0, 1.0));
}
}
probs
}
fn configs() -> Vec<TurnFilterConfig> {
vec![
TurnFilterConfig::none(),
TurnFilterConfig::dihard3_v2(),
TurnFilterConfig::callhome_v2(),
TurnFilterConfig::callhome_v2().with_filter_speech_first(false),
TurnFilterConfig::new()
.with_onset(0.7)
.with_offset(0.3)
.with_pad_onset(0.15)
.with_pad_offset(0.25)
.with_min_duration_on(0.3)
.with_min_duration_off(0.4),
]
}
fn assert_turns_eq(
got: &[SpeakerTurn],
want: &[SpeakerTurn],
what: &str,
) {
assert_eq!(got.len(), want.len(), "{what}: {got:?} != {want:?}");
for (g, w) in got.iter().zip(want) {
assert_eq!(g.speaker, w.speaker, "{what}");
assert!(
(g.start - w.start).abs() < 1e-9 && (g.end - w.end).abs() < 1e-9,
"{what}: {g:?} != {w:?}"
);
}
}
#[test]
fn test_turns_are_nemo_post_processing() {
let mut rng = StdRng::seed_from_u64(7);
for (k, config) in configs().iter().enumerate() {
for trial in 0..20 {
let probs = tracks(&mut rng, 300, 3);
let got = config.turns(&probs, 3, 0.08);
let want = reference_turns(&probs, 3, config, 0.08);
assert_turns_eq(&got, &want, &format!("config {k}, trial {trial}"));
}
}
}
#[test]
fn test_streaming_is_the_whole_recording() {
let mut rng = StdRng::seed_from_u64(11);
for (k, config) in configs().iter().enumerate() {
for trial in 0..20 {
let probs = tracks(&mut rng, 300, 2);
let want = config.turns(&probs, 2, 0.01);
let mut filter = config.init(2, 0.01);
let mut got = Vec::new();
let mut at = 0;
while at < probs.len() / 2 {
let n = rng.random_range(1..=17).min(probs.len() / 2 - at);
for turn in filter.push(&probs[2 * at..2 * (at + n)]) {
let now = (at + n) as f64 * 0.01;
assert!(
turn.end <= now + config.pad_offset + 1e-9,
"{turn:?} at {now}"
);
got.push(turn);
}
at += n;
}
got.extend(filter.finish());
sort_turns(&mut got);
assert_turns_eq(&got, &want, &format!("config {k}, trial {trial}"));
}
}
}
#[derive(serde::Deserialize)]
struct NemoFixture {
turns: Vec<NemoTurnCase>,
}
#[derive(serde::Deserialize)]
struct NemoTurnCase {
config: String,
frame_seconds: f64,
n_speakers: usize,
probs: Vec<f32>,
turns: Vec<NemoTurn>,
}
#[derive(serde::Deserialize)]
struct NemoTurn {
speaker: usize,
start: f64,
end: f64,
}
fn fixture_config(name: &str) -> TurnFilterConfig {
match name {
"none" => TurnFilterConfig::none(),
"dihard3_v2" => TurnFilterConfig::dihard3_v2(),
"callhome_v2" => TurnFilterConfig::callhome_v2(),
"callhome_v2_fill_first" => {
TurnFilterConfig::callhome_v2().with_filter_speech_first(false)
}
"wide" => TurnFilterConfig::new()
.with_onset(0.7)
.with_offset(0.3)
.with_pad_onset(0.15)
.with_pad_offset(0.25)
.with_min_duration_on(0.3)
.with_min_duration_off(0.4),
other => panic!("no config named {other}"),
}
}
#[test]
fn test_turns_match_nemo() {
let path = concat!(
env!("CARGO_MANIFEST_DIR"),
"/testdata/diarization/nemo.json"
);
let fixture: NemoFixture =
serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();
assert!(!fixture.turns.is_empty());
for (k, case) in fixture.turns.iter().enumerate() {
let got = fixture_config(&case.config).turns(
&case.probs,
case.n_speakers,
case.frame_seconds,
);
let what = format!("case {k} ({})", case.config);
assert_eq!(got.len(), case.turns.len(), "{what}: {got:?}");
for (g, w) in got.iter().zip(&case.turns) {
assert_eq!(g.speaker, w.speaker, "{what}");
assert!(
(g.start - w.start).abs() < 1e-5 && (g.end - w.end).abs() < 1e-5,
"{what}: {g:?} != ({}, {}, {})",
w.speaker,
w.start,
w.end
);
}
}
}
#[test]
fn test_hysteresis() {
let keep = TurnFilterConfig::new().with_onset(0.6).with_offset(0.4);
let probs = [0.1, 0.7, 0.5, 0.5, 0.3, 0.5, 0.5];
let turns = keep.turns(&probs, 1, 1.0);
assert_eq!(
turns,
[SpeakerTurn {
speaker: 0,
start: 1.0,
end: 4.0
}]
);
let toggle = TurnFilterConfig::new().with_onset(0.4).with_offset(0.6);
let probs = [0.5, 0.5, 0.5, 0.1, 0.7, 0.7];
let turns = toggle.turns(&probs, 1, 1.0);
let spans: Vec<_> = turns.iter().map(|t| (t.start, t.end)).collect();
assert_eq!(spans, [(0.0, 1.0), (2.0, 3.0), (4.0, 6.0)]);
}
#[test]
fn test_padding_and_filters() {
let probs = [0.9, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.9];
let c = TurnFilterConfig::new()
.with_pad_onset(0.5)
.with_pad_offset(0.6);
let spans: Vec<_> = c
.turns(&probs, 1, 1.0)
.iter()
.map(|t| (t.start, t.end))
.collect();
assert_eq!(spans, [(0.0, 1.6), (2.5, 5.6), (9.5, 11.6)]);
let c = TurnFilterConfig::new().with_min_duration_on(1.5);
let spans: Vec<_> = c
.turns(&probs, 1, 1.0)
.iter()
.map(|t| (t.start, t.end))
.collect();
assert_eq!(spans, [(3.0, 5.0)]);
let c = TurnFilterConfig::new().with_min_duration_off(2.5);
let spans: Vec<_> = c
.turns(&probs, 1, 1.0)
.iter()
.map(|t| (t.start, t.end))
.collect();
assert_eq!(spans, [(0.0, 5.0), (10.0, 11.0)]);
}
#[test]
fn test_speaking_state() {
let mut filter = TurnFilterConfig::none().init(2, 0.08);
assert!(filter.push(&[0.9, 0.1]).is_empty());
assert!(filter.is_speaking(0));
assert!(!filter.is_speaking(1));
assert_eq!(filter.frames_seen(), 1);
let turns = filter.finish();
assert_eq!(turns.len(), 1);
assert!((turns[0].end - 0.08).abs() < 1e-12);
assert!(!filter.is_speaking(0));
}
}