mod cache;
mod latency;
mod stability;
pub use cache::{ArrivalOrderSpeakerCache, AssignResult};
pub use latency::{LatencyPreset, LatencyPresetParseError, StreamingParams};
pub use stability::{label_flip_rate, prefer_current_speaker};
use crate::VadConfig;
use crate::embedder::{Embedder, EmbedderError};
use crate::types::{DiarizationConfig, SpeakerTurn, TimeRange};
use crate::vad::{VadError, VadEvent, VadStateMachine, VoiceActivityDetector};
use crate::window::WindowBuffer;
#[derive(Debug, thiserror::Error)]
pub enum StreamingError {
#[error("VAD error: {0}")]
Vad(#[from] VadError),
#[error("embedding error: {0}")]
Embedding(#[from] EmbedderError),
#[error(
"VAD returned {got} probabilities for one {frame_samples}-sample frame; \
StreamingPipeline requires exactly one probability per VadConfig::frame_size \
samples, so VadConfig::frame_size must equal the detector's native frame size"
)]
VadFrameMismatch { frame_samples: usize, got: usize },
#[error("invalid streaming params: {detail}")]
InvalidParams { detail: String },
}
impl StreamingError {
pub fn is_resource_exhausted(&self) -> bool {
match self {
Self::Embedding(e) => e.is_resource_exhausted(),
Self::Vad(_) | Self::VadFrameMismatch { .. } | Self::InvalidParams { .. } => false,
}
}
}
pub struct StreamingPipeline<V, E> {
vad: V,
extractor: E,
cache: ArrivalOrderSpeakerCache,
params: StreamingParams,
preset: Option<LatencyPreset>,
frame_size: usize,
sample_rate: u32,
vad_buffer: Vec<f32>,
vad_state: VadStateMachine,
window_buffer: WindowBuffer,
turns: Vec<SpeakerTurn>,
total_frames: usize,
}
impl<V, E> StreamingPipeline<V, E>
where
V: VoiceActivityDetector,
E: Embedder,
{
pub fn new(
vad: V,
extractor: E,
config: DiarizationConfig,
vad_config: VadConfig,
) -> Result<Self, StreamingError> {
let params = StreamingParams {
window_secs: config.window.window_secs,
hop_secs: config.window.hop_secs,
right_context_secs: 0.0,
speaker_cache_cap: config.cluster.max_speakers.max(1),
min_hits_to_stable: LatencyPreset::Balanced.params().min_hits_to_stable,
prefer_current_margin: LatencyPreset::Balanced.params().prefer_current_margin,
match_threshold: config.cluster.threshold,
};
Self::from_parts(vad, extractor, config, vad_config, params, None)
}
pub fn with_latency_preset(
vad: V,
extractor: E,
preset: LatencyPreset,
vad_config: VadConfig,
) -> Result<Self, StreamingError> {
let mut config = DiarizationConfig::default();
preset.apply(&mut config);
let params = preset.params();
Self::from_parts(vad, extractor, config, vad_config, params, Some(preset))
}
pub fn with_params(
vad: V,
extractor: E,
mut config: DiarizationConfig,
vad_config: VadConfig,
mut params: StreamingParams,
) -> Result<Self, StreamingError> {
params.speaker_cache_cap = params.speaker_cache_cap.max(1);
config.window.window_secs = params.window_secs;
config.window.hop_secs = params.hop_secs;
config.cluster.threshold = params.match_threshold;
config.cluster.max_speakers = params.speaker_cache_cap;
Self::from_parts(vad, extractor, config, vad_config, params, None)
}
fn from_parts(
vad: V,
extractor: E,
config: DiarizationConfig,
vad_config: VadConfig,
params: StreamingParams,
preset: Option<LatencyPreset>,
) -> Result<Self, StreamingError> {
let frame_size = vad_config.frame_size;
let sample_rate = config.window.sample_rate.get();
let geometry =
vad_config.frame_geometry(sample_rate, config.speech_filter.min_speech_secs)?;
Self::validate_window_geometry(&config, ¶ms)?;
let cache = ArrivalOrderSpeakerCache::new(
params.speaker_cache_cap,
params.match_threshold,
params.min_hits_to_stable,
params.prefer_current_margin,
);
let vad_state = VadStateMachine::new(
vad_config.threshold,
geometry.min_silence_frames,
geometry.min_speech_frames,
);
Ok(Self {
vad,
extractor,
cache,
params,
preset,
frame_size,
sample_rate,
vad_buffer: Vec::new(),
vad_state,
window_buffer: WindowBuffer::new(config.window_samples(), config.hop_samples()),
turns: Vec::new(),
total_frames: 0,
})
}
fn validate_window_geometry(
config: &DiarizationConfig,
params: &StreamingParams,
) -> Result<(), StreamingError> {
let window_secs = params.window_secs;
let hop_secs = params.hop_secs;
if !window_secs.is_finite() || window_secs <= 0.0 {
return Err(StreamingError::InvalidParams {
detail: format!("window_secs must be finite and > 0, got {window_secs}"),
});
}
if !hop_secs.is_finite() || hop_secs <= 0.0 {
return Err(StreamingError::InvalidParams {
detail: format!("hop_secs must be finite and > 0, got {hop_secs}"),
});
}
if hop_secs > window_secs {
return Err(StreamingError::InvalidParams {
detail: format!("hop_secs ({hop_secs}) must be <= window_secs ({window_secs})"),
});
}
if config.window_samples() == 0 || config.hop_samples() == 0 {
return Err(StreamingError::InvalidParams {
detail: format!(
"window_secs ({window_secs}) / hop_secs ({hop_secs}) must each yield at \
least one sample at sample_rate {}",
config.window.sample_rate.get()
),
});
}
Ok(())
}
pub fn params(&self) -> StreamingParams {
self.params
}
pub fn latency_preset(&self) -> Option<LatencyPreset> {
self.preset
}
pub fn speaker_cache_cap(&self) -> usize {
self.cache.cap()
}
pub fn cache_len(&self) -> usize {
self.cache.len()
}
pub fn feed(&mut self, samples: &[f32]) -> Result<Vec<SpeakerTurn>, StreamingError> {
let mut new_turns = Vec::new();
self.vad_buffer.extend_from_slice(samples);
let frame_size = self.frame_size;
while self.vad_buffer.len() >= frame_size {
let frame: Vec<f32> = self.vad_buffer.drain(..frame_size).collect();
let probs = self.vad.process(&frame)?;
if probs.len() != 1 {
return Err(StreamingError::VadFrameMismatch {
frame_samples: frame_size,
got: probs.len(),
});
}
let prob = probs[0];
let current_frame = self.total_frames;
self.total_frames += 1;
if let Some(event) = self.vad_state.advance(prob, current_frame) {
match event {
VadEvent::SpeechStart { start_frame } => {
self.window_buffer.clear();
self.window_buffer.set_next_start(start_frame * frame_size);
}
VadEvent::SpeechEnd {
start_frame,
end_frame,
} => {
let seg_end_sample = end_frame * frame_size;
if self
.vad_state
.meets_min_speech_duration(start_frame, end_frame)
{
new_turns.extend(self.flush_window_buffer(seg_end_sample)?);
} else {
self.window_buffer.clear();
}
}
}
}
if self.vad_state.in_speech() {
self.window_buffer.extend(&frame);
new_turns.extend(self.try_extract_windows()?);
}
}
self.turns.extend(new_turns.iter().cloned());
Ok(new_turns)
}
pub fn flush(&mut self) -> Result<Vec<SpeakerTurn>, StreamingError> {
let mut new_turns = Vec::new();
self.vad_buffer.clear();
if let Some(VadEvent::SpeechEnd {
start_frame,
end_frame,
}) = self.vad_state.flush(self.total_frames)
{
if self
.vad_state
.meets_min_speech_duration(start_frame, end_frame)
{
let seg_end_sample = end_frame * self.frame_size;
new_turns.extend(self.flush_window_buffer(seg_end_sample)?);
} else {
self.window_buffer.clear();
}
}
self.turns.extend(new_turns.iter().cloned());
Ok(new_turns)
}
pub fn num_speakers(&self) -> usize {
self.cache.len()
}
pub fn turns(&self) -> &[SpeakerTurn] {
&self.turns
}
fn try_extract_windows(&mut self) -> Result<Vec<SpeakerTurn>, StreamingError> {
let mut turns = Vec::new();
let sr_f = self.sample_rate as f64;
while let Some((start, chunk)) = self.window_buffer.try_pop() {
let embedding = self.extractor.embed(&chunk)?;
let assigned = self.cache.assign(&embedding);
debug_assert!(self.cache.len() <= self.cache.cap());
let end = start + chunk.len();
turns.push(SpeakerTurn::with_stability(
assigned.speaker,
TimeRange {
start: start as f64 / sr_f,
end: end as f64 / sr_f,
},
assigned.stable,
));
}
Ok(turns)
}
fn flush_window_buffer(
&mut self,
seg_end_sample: usize,
) -> Result<Vec<SpeakerTurn>, StreamingError> {
let mut turns = Vec::new();
let sr_f = self.sample_rate as f64;
if let Some((start, padded)) = self.window_buffer.flush() {
let embedding = self.extractor.embed(&padded)?;
let assigned = self.cache.assign(&embedding);
debug_assert!(self.cache.len() <= self.cache.cap());
let end = seg_end_sample.min(start + padded.len());
turns.push(SpeakerTurn::with_stability(
assigned.speaker,
TimeRange {
start: start as f64 / sr_f,
end: end as f64 / sr_f,
},
assigned.stable,
));
}
Ok(turns)
}
}
#[allow(clippy::unwrap_used)]
#[cfg(test)]
mod tests {
use super::*;
use crate::embedder::DummyExtractor;
use crate::types::SpeakerId;
use crate::{EnergyVad, VadConfig};
fn default_config() -> DiarizationConfig {
DiarizationConfig::default()
}
fn default_vad_config() -> VadConfig {
VadConfig::default()
}
fn pipeline() -> StreamingPipeline<EnergyVad, DummyExtractor> {
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
StreamingPipeline::new(vad, extractor, default_config(), default_vad_config()).unwrap()
}
fn pipeline_preset(preset: LatencyPreset) -> StreamingPipeline<EnergyVad, DummyExtractor> {
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
StreamingPipeline::with_latency_preset(vad, extractor, preset, default_vad_config())
.unwrap()
}
fn loud_samples(seconds: f32) -> Vec<f32> {
let n = (seconds * 16000.0) as usize;
vec![0.5f32; n]
}
fn silent_samples(seconds: f32) -> Vec<f32> {
let n = (seconds * 16000.0) as usize;
vec![0.0f32; n]
}
#[test]
fn streaming_pipeline_new_is_empty() {
let p = pipeline();
assert_eq!(p.num_speakers(), 0);
assert!(p.turns().is_empty());
assert_eq!(p.cache_len(), 0);
assert!(p.speaker_cache_cap() >= 1);
}
#[test]
fn feed_silence_returns_no_turns() {
let mut p = pipeline();
let turns = p.feed(&silent_samples(2.0)).unwrap();
assert!(turns.is_empty());
assert!(p.turns().is_empty());
}
#[test]
fn feed_loud_audio_returns_at_least_one_turn() {
let mut p = pipeline();
let turns = p.feed(&loud_samples(5.0)).unwrap();
assert!(
!turns.is_empty(),
"expected at least one turn for 5 s of speech"
);
}
#[test]
fn feed_rejects_vad_with_mismatched_native_frame() {
let vad = EnergyVad::new(-40.0, 16000, 256);
let extractor = DummyExtractor::new(256);
let mut p =
StreamingPipeline::new(vad, extractor, default_config(), default_vad_config()).unwrap();
let err = p.feed(&loud_samples(1.0)).unwrap_err();
assert!(!err.is_resource_exhausted());
match err {
StreamingError::VadFrameMismatch { frame_samples, got } => {
assert_eq!(frame_samples, 512);
assert_eq!(got, 2);
}
other => panic!("expected VadFrameMismatch, got {other:?}"),
}
}
#[test]
fn flush_after_speech_emits_remaining_turn() {
let mut p = pipeline();
let _ = p.feed(&loud_samples(1.0)).unwrap();
let turns = p.flush().unwrap();
assert!(
!turns.is_empty(),
"flush should emit the trailing partial window"
);
}
#[test]
fn turns_are_monotonically_ordered() {
let mut p = pipeline();
let mut emitted: Vec<SpeakerTurn> = Vec::new();
emitted.extend(p.feed(&loud_samples(5.0)).unwrap());
emitted.extend(p.flush().unwrap());
assert!(
!p.turns().is_empty(),
"turns() must be populated after feeding speech"
);
assert_eq!(
p.turns(),
emitted.as_slice(),
"turns() must equal the concatenation of feed()/flush() returns"
);
let turns = p.turns();
for i in 1..turns.len() {
assert!(
turns[i].time.start >= turns[i - 1].time.start,
"turns must be monotonically ordered"
);
}
}
#[test]
fn turns_accumulates_across_feed_and_flush() {
let mut p = pipeline();
let mut emitted: Vec<SpeakerTurn> = Vec::new();
emitted.extend(p.feed(&loud_samples(3.0)).unwrap());
emitted.extend(p.feed(&loud_samples(3.0)).unwrap());
emitted.extend(p.flush().unwrap());
assert!(
!emitted.is_empty(),
"expected turns across two feeds plus a flush"
);
assert_eq!(
p.turns(),
emitted.as_slice(),
"turns() must accumulate every feed()/flush() return in order"
);
}
#[test]
fn balanced_preset_matches_default_window() {
let p = pipeline_preset(LatencyPreset::Balanced);
let d = DiarizationConfig::default();
assert!((p.params().window_secs - d.window.window_secs).abs() < 1e-6);
assert!((p.params().hop_secs - d.window.hop_secs).abs() < 1e-6);
assert_eq!(p.latency_preset(), Some(LatencyPreset::Balanced));
}
#[test]
fn realtime_preset_has_shorter_window() {
let p = pipeline_preset(LatencyPreset::Realtime);
assert!((p.params().window_secs - 1.0).abs() < 1e-6);
assert_eq!(p.speaker_cache_cap(), 16);
}
#[test]
fn cache_never_exceeds_cap_under_long_feed() {
let params = StreamingParams {
window_secs: 1.0,
hop_secs: 0.5,
right_context_secs: 0.0,
speaker_cache_cap: 2,
min_hits_to_stable: 2,
prefer_current_margin: 0.05,
match_threshold: 0.45,
};
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let mut p = StreamingPipeline::with_params(
vad,
extractor,
DiarizationConfig::default(),
default_vad_config(),
params,
)
.unwrap();
let _ = p.feed(&loud_samples(20.0)).unwrap();
let _ = p.flush().unwrap();
assert!(
p.cache_len() <= p.speaker_cache_cap(),
"cache len {} > cap {}",
p.cache_len(),
p.speaker_cache_cap()
);
assert!(p.cache_len() <= 2);
}
#[test]
fn with_params_rejects_non_positive_window_secs() {
let params = StreamingParams {
window_secs: 0.0,
..LatencyPreset::Balanced.params()
};
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let res = StreamingPipeline::with_params(
vad,
extractor,
DiarizationConfig::default(),
default_vad_config(),
params,
);
assert!(matches!(res, Err(StreamingError::InvalidParams { .. })));
}
#[test]
fn with_params_rejects_hop_larger_than_window() {
let params = StreamingParams {
window_secs: 0.5,
hop_secs: 1.0,
..LatencyPreset::Balanced.params()
};
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let res = StreamingPipeline::with_params(
vad,
extractor,
DiarizationConfig::default(),
default_vad_config(),
params,
);
match res {
Err(StreamingError::InvalidParams { detail }) => {
assert!(detail.contains("hop_secs"), "got: {detail}");
}
Err(other) => panic!("expected InvalidParams, got {other:?}"),
Ok(_) => panic!("expected InvalidParams error"),
}
}
#[test]
fn with_params_rejects_sub_sample_window() {
let params = StreamingParams {
window_secs: 1e-9,
hop_secs: 1e-9,
..LatencyPreset::Balanced.params()
};
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let res = StreamingPipeline::with_params(
vad,
extractor,
DiarizationConfig::default(),
default_vad_config(),
params,
);
assert!(matches!(res, Err(StreamingError::InvalidParams { .. })));
}
#[test]
fn with_params_clamps_zero_speaker_cache_cap() {
let params = StreamingParams {
speaker_cache_cap: 0,
..LatencyPreset::Balanced.params()
};
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let p = StreamingPipeline::with_params(
vad,
extractor,
DiarizationConfig::default(),
default_vad_config(),
params,
)
.unwrap();
assert_eq!(p.speaker_cache_cap(), 1);
}
#[test]
fn new_rejects_zero_window_secs_in_config() {
let mut config = default_config();
config.window.window_secs = 0.0;
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let res = StreamingPipeline::new(vad, extractor, config, default_vad_config());
assert!(matches!(res, Err(StreamingError::InvalidParams { .. })));
}
#[test]
fn emitted_turns_carry_stable_flag() {
let params = StreamingParams {
window_secs: 1.0,
hop_secs: 0.5,
right_context_secs: 0.0,
speaker_cache_cap: 2,
min_hits_to_stable: 2,
prefer_current_margin: 0.05,
match_threshold: 0.99,
};
let vad = EnergyVad::new(-40.0, 16000, 512);
let extractor = DummyExtractor::new(256);
let mut p = StreamingPipeline::with_params(
vad,
extractor,
DiarizationConfig::default(),
default_vad_config(),
params,
)
.unwrap();
let turns = p.feed(&loud_samples(6.0)).unwrap();
assert!(!turns.is_empty());
let all = p.turns();
assert!(
all.iter().any(|t| !t.stable),
"first hits for a speaker are provisional"
);
assert!(
all.iter().any(|t| t.stable),
"expected at least one stable turn after repeated overflow hits"
);
}
#[test]
fn model_long_stream_cache_stays_bounded() {
let mut cache = ArrivalOrderSpeakerCache::new(8, 0.5, 3, 0.05);
let dim = 32;
for i in 0..5_000 {
let mut emb = vec![0.0f32; dim];
emb[i % dim] = 1.0;
emb[(i * 3) % dim] += 0.1;
crate::utils::l2_normalize(&mut emb);
cache.assign(&emb);
assert!(cache.len() <= cache.cap());
}
assert_eq!(cache.cap(), 8);
assert!(cache.len() <= 8);
}
#[test]
fn flip_rate_helper_exported() {
let first = [SpeakerId(0), SpeakerId(1)];
let final_ = [SpeakerId(0), SpeakerId(0)];
assert!((label_flip_rate(&first, &final_) - 0.5).abs() < 1e-6);
}
}