1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
//! Where to cut a window at a change of speaker.
use burn::config::Config;
use crate::kits::speech::diarization::SpeakerActivity;
/// Config for [`cut`](Self::cut): when a change of speaker ends a window.
#[derive(Config, Debug, PartialEq)]
pub struct SpeakerCutConfig {
/// A window runs at least this long, in seconds, before a change of
/// speaker may end it.
#[config(default = "2.0")]
pub min_window_seconds: f64,
/// A speaker must hold this long, in seconds, to count as a turn: a
/// shorter stretch (a backchannel, a flicker) does not end the window.
#[config(default = "0.5")]
pub min_turn_seconds: f64,
/// A frame's dominant speaker must reach this probability; below it the
/// frame is nobody's.
#[config(default = "0.5")]
pub threshold: f64,
}
impl Default for SpeakerCutConfig {
fn default() -> Self {
Self::new()
}
}
impl SpeakerCutConfig {
/// Where a window is cut at a change of speaker, for an engine that
/// decodes a window whole: the first frame, past
/// [`min_window_seconds`](SpeakerCutConfig::min_window_seconds), that
/// starts a turn of a speaker other than the window's current one, a turn
/// being a speaker dominant for
/// [`min_turn_seconds`](SpeakerCutConfig::min_turn_seconds) running.
/// Decoded up to there, the window is one speaker's, as far as the turns
/// go; the rest is decoded with what follows.
///
/// A frame's dominant speaker is the one with the highest mean probability
/// over the activity frames the frame overlaps, when it reaches the
/// threshold; frames the activity does not cover are nobody's.
///
/// # Arguments
/// * `activity` - the diarizer's frames over the window.
/// * `start` - the stream sample the window's first frame starts at.
/// * `hop` - the samples in one of the window's frames.
/// * `frames` - the window's frames.
/// * `sample_rate` - in Hz.
///
/// # Returns
/// The frame to cut at, in `1..frames`; `None` when no turn ends the
/// window.
pub fn cut(
&self,
activity: &SpeakerActivity,
start: usize,
hop: usize,
frames: usize,
sample_rate: usize,
) -> Option<usize> {
let to_frames =
|seconds: f64| ((seconds * sample_rate as f64) / hop as f64).round() as usize;
let min_window = to_frames(self.min_window_seconds).max(1);
let min_turn = to_frames(self.min_turn_seconds).max(1);
// Runs of each frame's dominant speaker.
let mut runs: Vec<(Option<usize>, usize, usize)> = Vec::new();
for k in 0..frames {
let who = dominant(activity, start + k * hop, hop, self.threshold);
match runs.last_mut() {
Some((speaker, _, len)) if *speaker == who => *len += 1,
_ => runs.push((who, k, 1)),
}
}
let mut current = None;
for (speaker, at, len) in runs {
let Some(speaker) = speaker else {
continue;
};
if len < min_turn {
continue;
}
match current {
None => current = Some(speaker),
Some(c) if c == speaker => {}
Some(_) if at >= min_window => return Some(at),
Some(_) => current = Some(speaker),
}
}
None
}
}
/// The speaker dominant over samples `[start, start + len)`, if one reaches
/// `threshold`.
fn dominant(
activity: &SpeakerActivity,
start: usize,
len: usize,
threshold: f64,
) -> Option<usize> {
let window = activity.window(start, start + len);
if window.is_empty() {
return None;
}
let n = window.frames() as f32;
let mut means = vec![0.0f32; window.n_speakers];
for i in 0..window.frames() {
for (m, p) in means.iter_mut().zip(window.frame(i)) {
*m += p / n;
}
}
let (speaker, &best) = means.iter().enumerate().max_by(|a, b| a.1.total_cmp(b.1))?;
(best as f64 >= threshold).then_some(speaker)
}
#[cfg(test)]
mod tests {
use super::*;
/// Activity at 100 Hz frames of 10 samples (a 1 kHz toy rate), one
/// speaker per stretch: `(speaker, frames)`; `None` is silence.
fn activity(stretches: &[(Option<usize>, usize)]) -> SpeakerActivity {
let mut probs = Vec::new();
for &(who, n) in stretches {
for _ in 0..n {
probs.extend((0..2).map(|k| if who == Some(k) { 0.9 } else { 0.05 }));
}
}
SpeakerActivity::try_new(0, 10, 2, probs).unwrap()
}
fn config() -> SpeakerCutConfig {
// 2 s and 0.5 s at 1 kHz with 10-sample frames: 200 and 50 frames.
SpeakerCutConfig::new()
}
#[test]
fn test_cuts_at_the_first_turn_past_the_minimum() {
let a = activity(&[(Some(0), 300), (Some(1), 100), (Some(0), 100)]);
assert_eq!(config().cut(&a, 0, 10, 500, 1000), Some(300));
}
#[test]
fn test_short_stretches_and_silence_do_not_cut() {
// A backchannel of 30 frames, then silence, then the same speaker.
let a = activity(&[(Some(0), 250), (Some(1), 30), (None, 50), (Some(0), 170)]);
assert_eq!(config().cut(&a, 0, 10, 500, 1000), None);
}
#[test]
fn test_an_early_turn_moves_the_current_speaker() {
// Speaker 1 takes over before the minimum, and holds: no cut for that
// turn; speaker 0's return past it cuts.
let a = activity(&[(Some(0), 100), (Some(1), 200), (Some(0), 200)]);
assert_eq!(config().cut(&a, 0, 10, 500, 1000), Some(300));
}
#[test]
fn test_frames_past_the_activity_are_nobodys() {
let a = activity(&[(Some(0), 250)]);
assert_eq!(config().cut(&a, 0, 10, 500, 1000), None);
// Offset: the window starts 100 frames into the activity.
let a = activity(&[(Some(0), 300), (Some(1), 100)]);
assert_eq!(config().cut(&a, 1000, 10, 300, 1000), Some(200));
}
}