#[cfg(test)]
use crate::window::WindowRecognizer;
use crate::window::{AsrConfig, AsrUpdate, RecognizerRef, Transcript, WindowProcessor};
use crate::{
Cancellation, ComputeUnits, Error, Result, load_model, multi_array, output_array, predict,
read_floating_tensor,
};
use objc2::rc::{Retained, autoreleasepool};
use objc2_core_ml::MLModel;
use serde::Serialize;
use std::{collections::VecDeque, path::Path, time::Instant};
pub const VAD_CHUNK_SAMPLES: usize = 4096;
const BLOCK: usize = VAD_CHUNK_SAMPLES;
const RATE: u64 = 16000;
#[derive(Clone, Debug, PartialEq)]
pub struct VadState {
pub hidden_state: [f32; 128],
pub cell_state: [f32; 128],
pub context: [f32; 64],
}
impl VadState {
pub fn initial() -> Self {
Self {
hidden_state: [0.0; 128],
cell_state: [0.0; 128],
context: [0.0; 64],
}
}
fn validate(&self) -> Result<()> {
for (name, values) in [
("hidden_state", self.hidden_state.as_slice()),
("cell_state", self.cell_state.as_slice()),
("context", self.context.as_slice()),
] {
if let Some(index) = values.iter().position(|value| !value.is_finite()) {
return Err(Error::Invalid(format!(
"Non-finite Silero state: field={name}, index={index}"
)));
}
}
Ok(())
}
}
#[derive(Clone, Debug)]
pub struct VadResult {
pub probability: f32,
pub is_voice_active: bool,
pub processing_seconds: f64,
pub output_state: VadState,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct VadFrame {
pub start_sample: u64,
pub end_sample: u64,
pub probability: f32,
pub is_voice_active: bool,
pub processing_seconds: f64,
}
#[derive(Clone, Debug)]
pub struct VadAnalysis {
pub frames: Vec<VadFrame>,
pub audio_samples: u64,
pub output_state: VadState,
}
impl VadAnalysis {
pub fn segments(&self, config: &VadSegmentationConfig) -> Result<Vec<VadSegment>> {
let mut expected = 0;
for frame in &self.frames {
if frame.start_sample != expected
|| frame.end_sample
!= expected
.saturating_add(BLOCK as u64)
.min(self.audio_samples)
|| frame.end_sample <= expected
{
return Err(Error::Invalid(format!(
"Invalid VAD analysis coverage: expected_start={expected}, frame={frame:?}, audio_samples={}",
self.audio_samples
)));
}
expected = frame.end_sample;
}
if expected != self.audio_samples {
return Err(Error::Invalid(format!(
"Incomplete VAD analysis: covered={expected}, audio_samples={}",
self.audio_samples
)));
}
let probabilities: Vec<f32> = self.frames.iter().map(|frame| frame.probability).collect();
segment_probabilities(&probabilities, self.audio_samples, config)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
pub struct VadSegment {
pub start_sample: u64,
pub end_sample: u64,
}
impl VadSegment {
pub fn start_seconds(&self) -> f64 {
self.start_sample as f64 / RATE as f64
}
pub fn end_seconds(&self) -> f64 {
self.end_sample as f64 / RATE as f64
}
pub fn extract<'a>(&self, audio: &'a [f32]) -> Result<&'a [f32]> {
if self.start_sample > self.end_sample || self.end_sample > audio.len() as u64 {
return Err(Error::Invalid(format!(
"VAD segment outside input: segment={self:?}, input_samples={}",
audio.len()
)));
}
Ok(&audio[self.start_sample as usize..self.end_sample as usize])
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum VadSplitPolicy {
LongestSilence,
LatestSilence,
}
#[derive(Clone, Debug)]
pub struct VadSegmentationConfig {
pub positive_threshold: f32,
pub negative_threshold: f32,
pub minimum_speech_samples: u64,
pub minimum_silence_samples: u64,
pub maximum_speech_samples: Option<u64>,
pub speech_padding_samples: u64,
pub silence_threshold_for_split: f32,
pub minimum_split_silence_samples: u64,
pub split_policy: VadSplitPolicy,
}
impl VadSegmentationConfig {
pub fn standard() -> Self {
Self {
positive_threshold: 0.85,
negative_threshold: 0.85 - 0.15,
minimum_speech_samples: 2400,
minimum_silence_samples: 12000,
maximum_speech_samples: Some(224000),
speech_padding_samples: 1600,
silence_threshold_for_split: 0.3,
minimum_split_silence_samples: 1568,
split_policy: VadSplitPolicy::LongestSilence,
}
}
pub fn validate(&self) -> Result<()> {
validate_probability(self.positive_threshold, "positive threshold")?;
validate_probability(self.negative_threshold, "negative threshold")?;
validate_probability(self.silence_threshold_for_split, "silence split threshold")?;
if self.negative_threshold > self.positive_threshold
|| self.speech_padding_samples.checked_mul(2).is_none()
|| self
.maximum_speech_samples
.is_some_and(|limit| limit == 0 || self.minimum_speech_samples > limit)
{
return Err(Error::Invalid(format!(
"Invalid VAD segmentation configuration: {self:?}"
)));
}
Ok(())
}
}
fn validate_probability(value: f32, name: &str) -> Result<()> {
if !(0.0..=1.0).contains(&value) {
return Err(Error::Invalid(format!(
"Invalid VAD {name}: value={value}; expected finite0..1"
)));
}
Ok(())
}
fn validate_chunk(audio: &[f32]) -> Result<()> {
if audio.is_empty() || audio.len() > BLOCK || audio.iter().any(|sample| !sample.is_finite()) {
return Err(Error::Invalid(format!(
"Silero requires1..{BLOCK} finite samples: samples={}, non_finite={}",
audio.len(),
audio.iter().any(|sample| !sample.is_finite())
)));
}
Ok(())
}
pub struct SileroVad {
model: Retained<MLModel>,
compute: ComputeUnits,
}
impl SileroVad {
pub fn load(path: &Path) -> Result<Self> {
Self::load_with_compute(path, ComputeUnits::CpuOnly)
}
pub fn load_with_compute(path: &Path, compute: ComputeUnits) -> Result<Self> {
Ok(Self {
model: load_model(path, compute.native())?,
compute,
})
}
pub fn compute_units(&self) -> ComputeUnits {
self.compute
}
pub(crate) fn detector(&self) -> Box<dyn VoiceDetector + '_> {
Box::new(SileroDetector {
model: &self.model,
state: VadState::initial(),
})
}
pub fn process_chunk(
&self,
audio: &[f32],
state: &VadState,
threshold: f32,
cancellation: &Cancellation,
) -> Result<VadResult> {
cancellation.check()?;
validate_probability(threshold, "voice threshold")?;
validate_chunk(audio)?;
state.validate()?;
let started = Instant::now();
let mut padded = [audio[audio.len() - 1]; BLOCK];
padded[..audio.len()].copy_from_slice(audio);
let (probability, output_state) = infer(&self.model, &padded, state)?;
cancellation.check()?;
Ok(VadResult {
probability,
is_voice_active: probability >= threshold,
processing_seconds: started.elapsed().as_secs_f64(),
output_state,
})
}
pub fn process_samples(
&self,
audio: &[f32],
threshold: f32,
cancellation: &Cancellation,
) -> Result<VadAnalysis> {
validate_probability(threshold, "voice threshold")?;
cancellation.check()?;
let mut collector = VadCollector::new();
for chunk in audio.chunks(BLOCK) {
collector.push(self, chunk, threshold, cancellation)?;
}
collector.finish(audio.len() as u64)
}
pub fn process_wav(
&self,
path: &Path,
threshold: f32,
cancellation: &Cancellation,
) -> Result<VadAnalysis> {
validate_probability(threshold, "voice threshold")?;
cancellation.check()?;
let mut source = crate::audio::WavSource::open(path)?;
self.process_source(&mut source, threshold, cancellation)
}
pub fn process_file(
&self,
path: &Path,
threshold: f32,
cancellation: &Cancellation,
) -> Result<VadAnalysis> {
validate_probability(threshold, "voice threshold")?;
cancellation.check()?;
let mut source = crate::audio::AudioSource::open(path, cancellation)?;
self.process_source(&mut source, threshold, cancellation)
}
pub fn process_source(
&self,
source: &mut dyn crate::audio::AudioSampleSource,
threshold: f32,
cancellation: &Cancellation,
) -> Result<VadAnalysis> {
validate_probability(threshold, "voice threshold")?;
cancellation.check()?;
let total = source.total_samples();
let mut collector = VadCollector::new();
let mut offset = 0;
while offset < total {
cancellation.check()?;
let count = (total - offset).min(BLOCK as u64) as usize;
let chunk = source.read_window(offset, count)?;
if chunk.len() != count {
return Err(Error::Invalid(format!(
"Audio source returned an incomplete window: source={:?}, offset={offset}, requested={count}, received={}",
source.origin(),
chunk.len()
)));
}
collector.push(self, &chunk, threshold, cancellation)?;
offset += chunk.len() as u64;
}
collector.finish(total)
}
pub fn segment_samples(
&self,
audio: &[f32],
config: &VadSegmentationConfig,
cancellation: &Cancellation,
) -> Result<Vec<VadSegment>> {
config.validate()?;
self.process_samples(audio, config.positive_threshold, cancellation)?
.segments(config)
}
pub fn segment_wav(
&self,
path: &Path,
config: &VadSegmentationConfig,
cancellation: &Cancellation,
) -> Result<Vec<VadSegment>> {
config.validate()?;
self.process_wav(path, config.positive_threshold, cancellation)?
.segments(config)
}
pub fn segment_file(
&self,
path: &Path,
config: &VadSegmentationConfig,
cancellation: &Cancellation,
) -> Result<Vec<VadSegment>> {
config.validate()?;
self.process_file(path, config.positive_threshold, cancellation)?
.segments(config)
}
}
struct VadCollector {
state: VadState,
frames: Vec<VadFrame>,
samples: u64,
}
impl VadCollector {
fn new() -> Self {
Self {
state: VadState::initial(),
frames: Vec::new(),
samples: 0,
}
}
fn push(
&mut self,
vad: &SileroVad,
audio: &[f32],
threshold: f32,
cancellation: &Cancellation,
) -> Result<()> {
let end = self
.samples
.checked_add(audio.len() as u64)
.ok_or_else(|| Error::Invalid("VAD audio clock overflow".into()))?;
let result = vad.process_chunk(audio, &self.state, threshold, cancellation)?;
self.frames.push(VadFrame {
start_sample: self.samples,
end_sample: end,
probability: result.probability,
is_voice_active: result.is_voice_active,
processing_seconds: result.processing_seconds,
});
self.samples = end;
self.state = result.output_state;
Ok(())
}
fn finish(self, expected_samples: u64) -> Result<VadAnalysis> {
if self.samples != expected_samples {
return Err(Error::Invalid(format!(
"Incomplete VAD input: processed={}, expected={expected_samples}",
self.samples
)));
}
Ok(VadAnalysis {
frames: self.frames,
audio_samples: self.samples,
output_state: self.state,
})
}
}
pub(crate) trait VoiceDetector {
fn probability(&mut self, audio: &[f32]) -> Result<f32>;
}
struct SileroDetector<'a> {
model: &'a MLModel,
state: VadState,
}
impl VoiceDetector for SileroDetector<'_> {
fn probability(&mut self, audio: &[f32]) -> Result<f32> {
validate_chunk(audio)?;
let mut padded = [0.0; BLOCK];
padded[..audio.len()].copy_from_slice(audio);
let (probability, state) = infer(self.model, &padded, &self.state)?;
self.state = state;
Ok(probability)
}
}
fn infer(model: &MLModel, audio: &[f32; BLOCK], state: &VadState) -> Result<(f32, VadState)> {
autoreleasepool(|_| {
let waveform: Vec<f32> = state.context.iter().chain(audio.iter()).copied().collect();
let input = multi_array(&[1, 4160], &waveform)?;
let hidden = multi_array(&[1, 128], &state.hidden_state)?;
let cell = multi_array(&[1, 128], &state.cell_state)?;
let output = predict(
model,
&[
("audio_input", &input),
("hidden_state", &hidden),
("cell_state", &cell),
],
)?;
let probability = read_floating_tensor(&*output_array(&output, "vad_output")?)?;
let next_hidden = read_floating_tensor(&*output_array(&output, "new_hidden_state")?)?;
let next_cell = read_floating_tensor(&*output_array(&output, "new_cell_state")?)?;
if probability.len() != 1
|| !(0.0..=1.0).contains(&probability[0])
|| next_hidden.len() != 128
|| next_cell.len() != 128
|| next_hidden.iter().chain(&next_cell).any(|v| !v.is_finite())
{
return Err(Error::Invalid(
"Invalid Silero probability or recurrent state".into(),
));
}
let hidden_state = next_hidden.try_into().map_err(|values: Vec<f32>| {
Error::Invalid(format!("Silero hidden state length: {}", values.len()))
})?;
let cell_state = next_cell.try_into().map_err(|values: Vec<f32>| {
Error::Invalid(format!("Silero cell state length: {}", values.len()))
})?;
let mut context = [0.0; 64];
context.copy_from_slice(&audio[BLOCK - 64..]);
Ok((
probability[0],
VadState {
hidden_state,
cell_state,
context,
},
))
})
}
#[derive(Clone, Copy)]
struct CandidateSilence {
start: u64,
duration: u64,
minimum_probability: f32,
}
fn longest_silence(candidates: impl Iterator<Item = CandidateSilence>) -> Option<CandidateSilence> {
candidates.fold(None, |best, candidate| match best {
Some(previous) if previous.duration >= candidate.duration => Some(previous),
_ => Some(candidate),
})
}
fn select_silence(
candidates: &[CandidateSilence],
config: &VadSegmentationConfig,
) -> Option<CandidateSilence> {
let low_probability =
longest_silence(candidates.iter().copied().filter(|candidate| {
candidate.minimum_probability <= config.silence_threshold_for_split
}));
low_probability.or_else(|| match config.split_policy {
VadSplitPolicy::LongestSilence => longest_silence(candidates.iter().copied()),
VadSplitPolicy::LatestSilence => candidates.last().copied(),
})
}
fn raw_segment(
start: u64,
end: u64,
total: u64,
minimum: u64,
previous: Option<&VadSegment>,
) -> Option<VadSegment> {
let minimum = if previous.is_some_and(|segment| segment.end_sample == start) {
0
} else {
minimum
};
if end > start && end - start >= minimum {
Some(VadSegment {
start_sample: start,
end_sample: end.min(total),
})
} else {
None
}
}
pub fn segment_probabilities(
probabilities: &[f32],
audio_samples: u64,
config: &VadSegmentationConfig,
) -> Result<Vec<VadSegment>> {
config.validate()?;
let expected = audio_samples.div_ceil(BLOCK as u64);
if probabilities.len() as u64 != expected {
return Err(Error::Invalid(format!(
"VAD probability coverage mismatch: probabilities={}, expected={expected}, audio_samples={audio_samples}",
probabilities.len()
)));
}
for probability in probabilities {
validate_probability(*probability, "probability")?;
}
let maximum = config.maximum_speech_samples.map(|maximum| {
maximum
.saturating_sub(BLOCK as u64)
.saturating_sub(2 * config.speech_padding_samples)
});
let mut triggered = false;
let mut start = 0;
let mut temporary_end = None;
let mut minimum_probability: Option<f32> = None;
let mut candidates = Vec::new();
let mut segments = Vec::new();
for (index, probability) in probabilities.iter().copied().enumerate() {
let frame_start = index as u64 * BLOCK as u64;
if probability >= config.positive_threshold {
if let Some(end) = temporary_end {
let duration = frame_start - end;
if duration > config.minimum_split_silence_samples {
candidates.push(CandidateSilence {
start: end,
duration,
minimum_probability: minimum_probability.unwrap_or(1.0),
});
}
}
temporary_end = None;
minimum_probability = None;
if !triggered {
triggered = true;
start = frame_start;
continue;
}
}
if triggered && maximum.is_some_and(|maximum| frame_start - start > maximum) {
let split = select_silence(&candidates, config);
let end = split.map(|split| split.start).unwrap_or(frame_start);
segments.extend(raw_segment(
start,
end,
audio_samples,
config.minimum_speech_samples,
segments.last(),
));
triggered = match split {
Some(split) => {
start = split.start + split.duration;
true
}
None => {
start = frame_start;
probability >= config.negative_threshold
}
};
candidates.clear();
temporary_end = None;
minimum_probability = None;
if !triggered {
continue;
}
}
if probability < config.negative_threshold && triggered {
let end = *temporary_end.get_or_insert(frame_start);
minimum_probability = Some(
minimum_probability
.map(|previous| previous.min(probability))
.unwrap_or(probability),
);
if frame_start - end >= config.minimum_silence_samples {
segments.extend(raw_segment(
start,
end,
audio_samples,
config.minimum_speech_samples,
segments.last(),
));
triggered = false;
temporary_end = None;
minimum_probability = None;
candidates.clear();
}
}
}
if triggered {
segments.extend(raw_segment(
start,
audio_samples,
audio_samples,
config.minimum_speech_samples,
segments.last(),
));
}
let count = segments.len();
for index in 0..count {
if index == 0 {
segments[index].start_sample = segments[index]
.start_sample
.saturating_sub(config.speech_padding_samples);
}
if index + 1 < count {
let gap = segments[index + 1]
.start_sample
.checked_sub(segments[index].end_sample)
.ok_or_else(|| {
Error::Invalid("VAD segmentation produced overlapping raw intervals".into())
})?;
let padding = if gap < 2 * config.speech_padding_samples {
gap / 2
} else {
config.speech_padding_samples
};
segments[index].end_sample = segments[index]
.end_sample
.saturating_add(padding)
.min(audio_samples);
segments[index + 1].start_sample =
segments[index + 1].start_sample.saturating_sub(padding);
} else {
segments[index].end_sample = segments[index]
.end_sample
.saturating_add(config.speech_padding_samples)
.min(audio_samples);
}
}
segments.retain(|segment| segment.end_sample > segment.start_sample);
Ok(segments)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum VadStreamEventKind {
SpeechStart,
SpeechEnd,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
pub struct VadStreamEvent {
pub kind: VadStreamEventKind,
pub sample_index: u64,
}
impl VadStreamEvent {
pub fn seconds(&self) -> f64 {
self.sample_index as f64 / RATE as f64
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct VadStreamState {
pub model_state: VadState,
pub triggered: bool,
pub temporary_end_sample: Option<u64>,
pub processed_samples: u64,
finished: bool,
has_padded_tail: bool,
}
impl VadStreamState {
pub fn initial() -> Self {
Self {
model_state: VadState::initial(),
triggered: false,
temporary_end_sample: None,
processed_samples: 0,
finished: false,
has_padded_tail: false,
}
}
pub fn is_finished(&self) -> bool {
self.finished
}
fn validate(&self) -> Result<()> {
self.model_state.validate()?;
if self.finished
|| self
.temporary_end_sample
.is_some_and(|end| !self.triggered || end > self.processed_samples)
{
return Err(Error::Invalid(format!(
"Invalid or finished VAD stream: processed_samples={}, triggered={}, temporary_end={:?}, finished={}",
self.processed_samples, self.triggered, self.temporary_end_sample, self.finished
)));
}
Ok(())
}
fn require_input(&self) -> Result<()> {
self.validate()?;
if self.has_padded_tail {
return Err(Error::Invalid(format!(
"VAD stream already processed its short final chunk: processed_samples={}; finish this stream before starting another",
self.processed_samples
)));
}
Ok(())
}
pub fn finish(&self) -> Result<(Self, Option<VadStreamEvent>)> {
self.validate()?;
let mut next = self.clone();
let event = self.triggered.then_some(VadStreamEvent {
kind: VadStreamEventKind::SpeechEnd,
sample_index: self.processed_samples,
});
next.triggered = false;
next.temporary_end_sample = None;
next.finished = true;
Ok((next, event))
}
}
#[derive(Clone, Debug)]
pub struct VadStreamResult {
pub state: VadStreamState,
pub event: Option<VadStreamEvent>,
pub probability: f32,
pub processing_seconds: f64,
}
fn stream_transition(
probability: f32,
samples: usize,
mut state: VadStreamState,
config: &VadSegmentationConfig,
) -> Result<(VadStreamState, Option<VadStreamEvent>)> {
config.validate()?;
state.require_input()?;
validate_probability(probability, "probability")?;
if samples == 0 || samples > BLOCK {
return Err(Error::Invalid(format!(
"Invalid VAD stream block: samples={samples}, maximum={BLOCK}"
)));
}
let previous = state.processed_samples;
state.processed_samples = previous.checked_add(samples as u64).ok_or_else(|| {
Error::Invalid(format!(
"VAD stream clock overflow: previous={previous}, samples={samples}"
))
})?;
state.has_padded_tail = samples < BLOCK;
let mut event = None;
if probability >= config.positive_threshold {
state.temporary_end_sample = None;
if !state.triggered {
state.triggered = true;
event = Some(VadStreamEvent {
kind: VadStreamEventKind::SpeechStart,
sample_index: previous.saturating_sub(config.speech_padding_samples),
});
}
} else if probability < config.negative_threshold && state.triggered {
let end = *state
.temporary_end_sample
.get_or_insert(state.processed_samples);
if state.processed_samples - end >= config.minimum_silence_samples {
let padded = end
.checked_add(config.speech_padding_samples)
.ok_or_else(|| {
Error::Invalid(format!(
"VAD stream padding overflow: end={end}, padding={}",
config.speech_padding_samples
))
})?;
event = Some(VadStreamEvent {
kind: VadStreamEventKind::SpeechEnd,
sample_index: padded.saturating_sub(samples as u64),
});
state.triggered = false;
state.temporary_end_sample = None;
}
}
Ok((state, event))
}
impl SileroVad {
pub fn process_streaming_chunk(
&self,
audio: &[f32],
state: &VadStreamState,
config: &VadSegmentationConfig,
cancellation: &Cancellation,
) -> Result<VadStreamResult> {
config.validate()?;
state.require_input()?;
state
.processed_samples
.checked_add(audio.len() as u64)
.ok_or_else(|| Error::Invalid("VAD stream clock overflow".into()))?;
let result = self.process_chunk(
audio,
&state.model_state,
config.positive_threshold,
cancellation,
)?;
let mut next = state.clone();
next.model_state = result.output_state;
let (next, event) = stream_transition(result.probability, audio.len(), next, config)?;
Ok(VadStreamResult {
state: next,
event,
probability: result.probability,
processing_seconds: result.processing_seconds,
})
}
}
#[derive(Clone, Copy)]
pub(crate) enum SpeechBoundary {
Quiet,
DigitalSilence,
}
pub(crate) struct SpeechBoundaries {
speaking: bool,
quiet_blocks: usize,
zero_blocks: usize,
minimum_quiet_blocks: usize,
}
impl SpeechBoundaries {
pub(crate) fn new(minimum_quiet_samples: usize) -> Self {
Self {
speaking: false,
quiet_blocks: 0,
zero_blocks: 0,
minimum_quiet_blocks: minimum_quiet_samples / BLOCK,
}
}
pub(crate) fn observe(
&mut self,
probability: f32,
exact_silence: bool,
) -> Option<SpeechBoundary> {
if probability >= 0.5 {
self.speaking = true;
self.quiet_blocks = 0;
} else if self.speaking && probability < 0.35 {
self.quiet_blocks += 1;
} else {
self.quiet_blocks = 0;
}
self.zero_blocks = if exact_silence {
self.zero_blocks + 1
} else {
0
};
let boundary = if self.zero_blocks >= 2 {
Some(SpeechBoundary::DigitalSilence)
} else if self.speaking && self.quiet_blocks >= self.minimum_quiet_blocks {
Some(SpeechBoundary::Quiet)
} else {
None
};
if boundary.is_some() {
self.speaking = false;
self.quiet_blocks = 0;
self.zero_blocks = 0;
}
boundary
}
}
pub(crate) struct SpeechProcessor<'a> {
window: WindowProcessor<'a>,
detector: Box<dyn VoiceDetector + 'a>,
pending: VecDeque<f32>,
boundaries: SpeechBoundaries,
has_audio: bool,
partial_interval_samples: usize,
}
impl<'a> SpeechProcessor<'a> {
#[cfg(test)]
pub(crate) fn new(
recognizer: &'a dyn WindowRecognizer,
config: AsrConfig,
detector: Box<dyn VoiceDetector + 'a>,
) -> Result<Self> {
Self::from_recognizer(RecognizerRef::Borrowed(recognizer), config, detector)
}
pub(crate) fn from_recognizer(
recognizer: RecognizerRef<'a>,
config: AsrConfig,
detector: Box<dyn VoiceDetector + 'a>,
) -> Result<Self> {
config.validate_for(recognizer.input_limits())?;
let boundaries = SpeechBoundaries::new(config.vad_min_silence_samples);
Ok(Self {
window: WindowProcessor::from_recognizer(recognizer, config.window_config())?,
detector,
pending: VecDeque::new(),
boundaries,
has_audio: false,
partial_interval_samples: config.partial_interval_samples,
})
}
pub(crate) fn result(&self, transcript: &Transcript) -> Result<crate::result::AsrResult> {
self.window.result(transcript)
}
pub(crate) fn received(&self) -> u64 {
self.window.received() + self.pending.len() as u64
}
pub(crate) fn peak_buffer(&self) -> usize {
self.window.peak_buffer() + BLOCK
}
fn block(&mut self, audio: &[f32], c: &Cancellation) -> Result<Vec<AsrUpdate>> {
c.check()?;
let probability = self.detector.probability(audio)?;
let exact_silence = audio.iter().all(|v| *v == 0.0);
if exact_silence && !self.has_audio {
self.window.advance_quiet(audio)?;
return Ok(Vec::new());
}
self.has_audio = true;
let mut updates = self.window.feed(audio, c)?;
if let Some(boundary) = self.boundaries.observe(probability, exact_silence) {
let update = match boundary {
SpeechBoundary::DigitalSilence => self.window.utterance_boundary(c)?,
SpeechBoundary::Quiet => self.window.confirm_boundary(c)?,
};
updates.push(update);
self.has_audio = false;
}
Ok(updates)
}
pub(crate) fn feed(&mut self, audio: &[f32], c: &Cancellation) -> Result<Vec<AsrUpdate>> {
if audio.len() > 16000 || audio.iter().any(|v| !v.is_finite()) {
return Err(Error::Invalid(
"Speech processor requires <=one second of finite input per call".into(),
));
}
self.pending.extend(audio);
let mut updates = Vec::new();
while self.pending.len() >= BLOCK {
let block: Vec<_> = self.pending.drain(..BLOCK).collect();
updates.extend(self.block(&block, c)?);
}
Ok(updates)
}
pub(crate) fn partial(&mut self, c: &Cancellation) -> Result<Option<AsrUpdate>> {
if self.has_audio {
self.window.partial(self.partial_interval_samples, c)
} else {
Ok(None)
}
}
pub(crate) fn finish(&mut self, c: &Cancellation) -> Result<(Transcript, AsrUpdate)> {
c.check()?;
let tail: Vec<_> = self.pending.drain(..).collect();
self.has_audio |= tail.iter().any(|sample| *sample != 0.0);
if !self.has_audio {
self.window.advance_quiet(&tail)?;
self.window.finish_quiet();
return self.window.finish(c);
}
self.window.finish_with_audio(&tail, c)
}
pub(crate) fn clear(&mut self) {
self.pending.clear();
self.window.clear();
}
}
pub fn verify_probabilities(
model_path: &Path,
audio_path: &Path,
reference_path: &Path,
output_path: &Path,
) -> Result<()> {
#[derive(serde::Deserialize)]
struct Reference {
sample_rate: u32,
block_samples: usize,
audio_samples: u64,
tolerance: f32,
probabilities: Vec<f32>,
}
#[derive(serde::Serialize)]
struct Verification {
model: String,
reference: String,
tested_samples: u64,
tested_blocks: usize,
max_absolute_difference: f32,
mean_absolute_difference: f64,
start_threshold_disagreements: usize,
end_threshold_disagreements: usize,
tolerance: f32,
passed: bool,
}
let reference: Reference = crate::read_json(reference_path)?;
let mut source = crate::batch::WavSource::open(audio_path)?;
let expected_blocks = source.total_samples().div_ceil(BLOCK as u64) as usize;
if reference.sample_rate != 16000
|| reference.block_samples != BLOCK
|| reference.audio_samples != source.total_samples()
|| reference.probabilities.len() != expected_blocks
|| expected_blocks == 0
|| !reference.tolerance.is_finite()
|| reference.tolerance <= 0.0
|| reference.tolerance >= 0.01
|| reference
.probabilities
.iter()
.any(|p| !(0.0..=1.0).contains(p))
{
return Err(Error::Invalid(format!(
"Invalid VAD reference contract: reference={}, wav={}, audio_samples={}, expected_blocks={expected_blocks}",
reference_path.display(),
audio_path.display(),
source.total_samples()
)));
}
let model = SileroVad::load(model_path)?;
let mut detector = model.detector();
let (mut samples, mut count, mut maximum, mut sum, mut starts, mut ends) =
(0u64, 0usize, 0.0_f32, 0.0_f64, 0usize, 0usize);
loop {
let audio = source.read_block(BLOCK)?;
if audio.is_empty() {
break;
}
if count >= reference.probabilities.len() {
return Err(Error::Invalid(
"VAD input exceeds reference block count".into(),
));
}
let expected = reference.probabilities[count];
let actual = detector.probability(&audio)?;
let difference = (actual - expected).abs();
maximum = maximum.max(difference);
sum += difference as f64;
starts += usize::from((actual >= 0.5) != (expected >= 0.5));
ends += usize::from((actual >= 0.35) != (expected >= 0.35));
samples += audio.len() as u64;
count += 1;
}
let passed = count == expected_blocks
&& samples == reference.audio_samples
&& maximum <= reference.tolerance;
crate::write_json(
output_path,
&Verification {
model: model_path.display().to_string(),
reference: reference_path.display().to_string(),
tested_samples: samples,
tested_blocks: count,
max_absolute_difference: maximum,
mean_absolute_difference: sum / count.max(1) as f64,
start_threshold_disagreements: starts,
end_threshold_disagreements: ends,
tolerance: reference.tolerance,
passed,
},
)?;
if !passed {
return Err(Error::Invalid(format!(
"VAD probability parity failed: max_difference={maximum}, tolerance={}, report={}",
reference.tolerance,
output_path.display()
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::window::test_support::Markers;
#[test]
fn offline_segments_apply_padding_and_extract_original_samples_exactly() {
let mut probabilities = vec![0.0; 4];
probabilities.extend([1.0; 5]);
probabilities.extend([0.0; 4]);
let total = probabilities.len() as u64 * BLOCK as u64;
let segments =
segment_probabilities(&probabilities, total, &VadSegmentationConfig::standard())
.unwrap();
assert_eq!(
segments,
vec![VadSegment {
start_sample: 4 * BLOCK as u64 - 1600,
end_sample: 9 * BLOCK as u64 + 1600
}]
);
let audio: Vec<f32> = (0..total).map(|index| index as f32).collect();
let segment = segments[0];
let extracted = segment.extract(&audio).unwrap();
assert_eq!(
extracted.as_ptr(),
audio[segment.start_sample as usize..].as_ptr()
);
assert_eq!(
extracted.len() as u64,
segment.end_sample - segment.start_sample
);
assert!(
VadSegment {
start_sample: 3,
end_sample: 2
}
.extract(&audio)
.is_err()
);
assert!(
VadSegment {
start_sample: 0,
end_sample: total + 1
}
.extract(&audio)
.is_err()
);
}
#[test]
fn offline_hysteresis_merges_short_gaps_and_filters_short_speech() {
let mut config = VadSegmentationConfig::standard();
config.speech_padding_samples = 0;
let probabilities = [1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0];
assert_eq!(
segment_probabilities(
&probabilities,
(probabilities.len() * BLOCK) as u64,
&config
)
.unwrap(),
vec![VadSegment {
start_sample: 0,
end_sample: 6 * BLOCK as u64
}]
);
config.minimum_speech_samples = 2 * BLOCK as u64;
let probabilities = [1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0];
assert_eq!(
segment_probabilities(
&probabilities,
(probabilities.len() * BLOCK) as u64,
&config
)
.unwrap(),
vec![VadSegment {
start_sample: 5 * BLOCK as u64,
end_sample: 8 * BLOCK as u64
}]
);
}
#[test]
fn offline_tail_clamps_padding_to_real_audio_and_empty_input_is_empty() {
let mut config = VadSegmentationConfig::standard();
config.minimum_speech_samples = 0;
assert!(segment_probabilities(&[], 0, &config).unwrap().is_empty());
assert_eq!(
segment_probabilities(&[1.0], 17, &config).unwrap(),
vec![VadSegment {
start_sample: 0,
end_sample: 17
}]
);
assert!(
segment_probabilities(&[0.0; 8], (BLOCK * 8) as u64, &config)
.unwrap()
.is_empty()
);
}
#[test]
fn maximum_length_uses_silence_candidates_and_infinite_length_disables_splits() {
let mut config = VadSegmentationConfig::standard();
let probabilities = vec![1.0; 130];
let segments =
segment_probabilities(&probabilities, (130 * BLOCK) as u64, &config).unwrap();
assert!(segments.len() >= 2);
assert!(
segments
.windows(2)
.all(|pair| pair[0].end_sample <= pair[1].start_sample)
);
config.maximum_speech_samples = None;
assert_eq!(
segment_probabilities(&probabilities, (130 * BLOCK) as u64, &config).unwrap(),
vec![VadSegment {
start_sample: 0,
end_sample: (130 * BLOCK) as u64
}]
);
let candidates = [
CandidateSilence {
start: 0,
duration: 500,
minimum_probability: 0.5,
},
CandidateSilence {
start: 1000,
duration: 200,
minimum_probability: 0.2,
},
];
assert_eq!(select_silence(&candidates, &config).unwrap().start, 1000);
config.silence_threshold_for_split = 0.1;
assert_eq!(select_silence(&candidates, &config).unwrap().start, 0);
config.split_policy = VadSplitPolicy::LatestSilence;
assert_eq!(select_silence(&candidates, &config).unwrap().start, 1000);
}
#[test]
fn forced_maximum_splits_preserve_continuous_speech_without_dropping_frames() {
let config = VadSegmentationConfig::standard();
for probabilities in [
vec![1.0; 130],
std::iter::once(1.0)
.chain(std::iter::repeat_n(0.72, 129))
.collect(),
] {
let total = probabilities.len() as u64 * BLOCK as u64;
let segments = segment_probabilities(&probabilities, total, &config).unwrap();
assert!(segments.len() >= 2);
assert_eq!(segments.first().unwrap().start_sample, 0);
assert_eq!(segments.last().unwrap().end_sample, total);
assert!(
segments
.windows(2)
.all(|pair| pair[0].end_sample == pair[1].start_sample),
"continuous speech has gaps: {segments:?}"
);
assert_eq!(
segments
.iter()
.map(|segment| segment.end_sample - segment.start_sample)
.sum::<u64>(),
total
);
}
let total = 53 * BLOCK as u64 + 17;
let segments = segment_probabilities(&[1.0; 54], total, &config).unwrap();
assert_eq!(segments.first().unwrap().start_sample, 0);
assert_eq!(segments.last().unwrap().end_sample, total);
assert_eq!(
segments
.iter()
.map(|segment| segment.end_sample - segment.start_sample)
.sum::<u64>(),
total
);
}
#[test]
fn malformed_probabilities_analysis_and_configuration_are_explicit_errors() {
let standard = VadSegmentationConfig::standard();
assert!(segment_probabilities(&[], 1, &standard).is_err());
assert!(segment_probabilities(&[0.5], 0, &standard).is_err());
for probability in [f32::NAN, f32::INFINITY, -0.1, 1.1] {
assert!(segment_probabilities(&[probability], 1, &standard).is_err());
}
let mut invalid = standard.clone();
invalid.negative_threshold = 1.0;
assert!(invalid.validate().is_err());
invalid = standard.clone();
invalid.maximum_speech_samples = Some(0);
assert!(invalid.validate().is_err());
invalid = standard;
invalid.speech_padding_samples = u64::MAX;
assert!(invalid.validate().is_err());
let analysis = VadAnalysis {
frames: vec![],
audio_samples: 1,
output_state: VadState::initial(),
};
assert!(
analysis
.segments(&VadSegmentationConfig::standard())
.is_err()
);
assert!(validate_chunk(&[]).is_err());
assert!(validate_chunk(&vec![0.0; BLOCK + 1]).is_err());
assert!(validate_chunk(&[f32::NAN]).is_err());
let mut state = VadState::initial();
state.context[3] = f32::NAN;
assert!(state.validate().is_err());
}
#[test]
fn streaming_thresholds_are_inclusive_at_entry_and_strict_at_exit() {
let config = VadSegmentationConfig::standard();
let original = VadStreamState::initial();
let (state, event) =
stream_transition(config.positive_threshold, BLOCK, original.clone(), &config).unwrap();
assert_eq!(original, VadStreamState::initial());
assert_eq!(
event,
Some(VadStreamEvent {
kind: VadStreamEventKind::SpeechStart,
sample_index: 0
})
);
let (state, event) =
stream_transition(config.negative_threshold, BLOCK, state, &config).unwrap();
assert!(event.is_none() && state.temporary_end_sample.is_none());
let (mut state, event) = stream_transition(0.0, BLOCK, state, &config).unwrap();
assert!(event.is_none());
let silence_start = state.temporary_end_sample.unwrap();
for _ in 0..2 {
let (next, event) = stream_transition(0.0, BLOCK, state, &config).unwrap();
assert!(event.is_none());
state = next;
}
let (state, event) = stream_transition(0.0, BLOCK, state, &config).unwrap();
assert_eq!(
event,
Some(VadStreamEvent {
kind: VadStreamEventKind::SpeechEnd,
sample_index: silence_start + config.speech_padding_samples - BLOCK as u64
})
);
assert!(!state.triggered);
}
#[test]
fn streaming_finish_keeps_real_tail_clock_and_rejects_further_input() {
let config = VadSegmentationConfig::standard();
let (state, _) = stream_transition(1.0, 17, VadStreamState::initial(), &config).unwrap();
assert_eq!(state.processed_samples, 17);
assert!(stream_transition(1.0, BLOCK, state.clone(), &config).is_err());
let (finished, event) = state.finish().unwrap();
assert!(finished.is_finished());
assert_eq!(
event,
Some(VadStreamEvent {
kind: VadStreamEventKind::SpeechEnd,
sample_index: 17
})
);
assert!(finished.finish().is_err());
assert!(stream_transition(1.0, BLOCK, finished, &config).is_err());
let (_, event) = VadStreamState::initial().finish().unwrap();
assert!(event.is_none());
}
#[test]
fn streaming_overflow_and_invalid_state_do_not_mutate_the_input() {
let config = VadSegmentationConfig::standard();
let mut state = VadStreamState::initial();
state.processed_samples = u64::MAX;
assert!(stream_transition(0.0, 1, state.clone(), &config).is_err());
assert_eq!(state.processed_samples, u64::MAX);
state = VadStreamState::initial();
state.temporary_end_sample = Some(1);
assert!(state.validate().is_err());
}
struct MarkedVoice;
impl VoiceDetector for MarkedVoice {
fn probability(&mut self, audio: &[f32]) -> Result<f32> {
Ok(if audio.iter().any(|v| *v >= 1.0) {
1.0
} else {
0.0
})
}
}
#[test]
fn final_pending_vad_boundary_does_not_publish_an_invisible_revision() {
let recognizer = Markers::new();
let mut processor =
SpeechProcessor::new(&recognizer, AsrConfig::interactive(), Box::new(MarkedVoice))
.unwrap();
let mut audio = vec![0.0001; BLOCK * 8 + 17];
audio[512] = 7.0;
let cancellation = Cancellation::new();
let mut visible = Vec::new();
let mut sequence = 0;
for block in audio.chunks(BLOCK) {
let mut events = processor.feed(block, &cancellation).unwrap();
if let Some(update) = processor.partial(&cancellation).unwrap() {
events.push(update);
}
for event in events {
assert_eq!(event.sequence, sequence + 1);
sequence = event.sequence;
visible.truncate(event.revision.retained_token_count);
visible.extend(event.revision.appended_tokens);
}
}
let (transcript, final_update) = processor.finish(&cancellation).unwrap();
assert_eq!(final_update.sequence, sequence + 1);
assert_eq!(final_update.revision.previous_token_count, visible.len());
visible.truncate(final_update.revision.retained_token_count);
visible.extend(final_update.revision.appended_tokens);
assert_eq!(visible, transcript.tokens);
}
#[test]
fn independent_utterances_preserve_repeated_words_and_absolute_time() {
let engine = Markers::new();
let c = Cancellation::new();
let mut p =
SpeechProcessor::new(&engine, AsrConfig::interactive(), Box::new(MarkedVoice)).unwrap();
let mut audio = vec![0.0; BLOCK * 16];
audio[BLOCK * 3 + 512] = 7.0;
audio[BLOCK * 10 + 512] = 7.0;
for part in audio.chunks(137) {
p.feed(part, &c).unwrap();
p.partial(&c).unwrap();
}
let (result, _) = p.finish(&c).unwrap();
assert_eq!(result.text, "7 7");
assert_eq!(
result
.tokens
.iter()
.map(|t| t.start_sample)
.collect::<Vec<_>>(),
[(BLOCK * 3 + 512) as u64, (BLOCK * 10 + 512) as u64]
);
assert_eq!(result.audio_samples, audio.len() as u64);
}
struct LowConfidence;
impl VoiceDetector for LowConfidence {
fn probability(&mut self, _audio: &[f32]) -> Result<f32> {
Ok(0.02)
}
}
#[test]
fn low_vad_probability_does_not_discard_recognizable_speech() {
let engine = Markers::new();
let c = Cancellation::new();
let mut p =
SpeechProcessor::new(&engine, AsrConfig::interactive(), Box::new(LowConfidence))
.unwrap();
let mut audio = vec![0.0001; 16000];
audio[4000] = 7.0;
for part in audio.chunks(137) {
p.feed(part, &c).unwrap();
}
let (result, _) = p.finish(&c).unwrap();
assert_eq!(result.text, "7");
assert_eq!(result.tokens[0].start_sample, 4000);
assert_eq!(result.audio_samples, 16000);
}
struct TailSensitive;
impl WindowRecognizer for TailSensitive {
fn input_limits(&self) -> crate::InputLimits {
crate::window::test_support::FULL_WINDOW
}
fn infer(&self, audio: &[f32], _: &Cancellation) -> Result<Vec<crate::window::TimedToken>> {
if let Some(i) = audio.iter().position(|x| *x >= 1.0) {
Ok(vec![crate::window::TimedToken {
token_id: 7,
start_sample: i as u64,
confidence: 0.95,
duration_samples: crate::window::ENCODER_STEP as u64,
}])
} else if audio.iter().any(|x| *x != 0.0) {
Ok(vec![crate::window::TimedToken {
token_id: 99,
start_sample: 0,
confidence: 0.95,
duration_samples: crate::window::ENCODER_STEP as u64,
}])
} else {
Ok(Vec::new())
}
}
fn render(&self, t: &[crate::window::TimedToken]) -> Result<String> {
Ok(t.iter()
.map(|t| t.token_id.to_string())
.collect::<Vec<_>>()
.join(" "))
}
}
#[test]
fn trailing_breath_is_not_recognized_as_an_independent_utterance() {
let mut p = SpeechProcessor::new(
&TailSensitive,
AsrConfig::interactive(),
Box::new(MarkedVoice),
)
.unwrap();
let c = Cancellation::new();
let mut audio = vec![0.0001; BLOCK * 6];
audio[512] = 7.0;
audio.extend(vec![0.0; BLOCK * 6]);
for part in audio.chunks(137) {
p.feed(part, &c).unwrap();
}
assert_eq!(p.finish(&c).unwrap().0.text, "7");
}
#[test]
fn reverb_after_vad_confirmation_keeps_previous_speech_context() {
let mut p = SpeechProcessor::new(
&TailSensitive,
AsrConfig::interactive(),
Box::new(MarkedVoice),
)
.unwrap();
let c = Cancellation::new();
let mut audio = vec![0.0001; BLOCK * 14];
audio[512] = 7.0;
audio.extend(vec![0.0; BLOCK * 4]);
for part in audio.chunks(137) {
p.feed(part, &c).unwrap();
}
assert_eq!(p.finish(&c).unwrap().0.text, "7");
}
#[test]
fn continuous_silence_never_runs_haas_or_accumulates_audio() {
let engine = Markers::new();
let c = Cancellation::new();
let mut p =
SpeechProcessor::new(&engine, AsrConfig::interactive(), Box::new(MarkedVoice)).unwrap();
for _ in 0..180 {
p.feed(&[0.0; 16000], &c).unwrap();
assert!(p.partial(&c).unwrap().is_none());
}
let (result, _) = p.finish(&c).unwrap();
assert!(result.text.is_empty());
assert_eq!(result.audio_samples, 180 * 16000);
assert_eq!(engine.calls.get(), 0);
assert!(p.peak_buffer() <= BLOCK * 3);
}
}