use async_trait::async_trait;
use crate::error::Result;
use crate::processor::frame::{AudioFrame, Frame, StartParams, VadParams};
use crate::processor::{Envelope, FrameProcessor, Link, ProcessorSetup};
use crate::service::{VadAnalyzer, VadState};
pub const VAD_CONFIDENCE: f32 = 0.7;
pub const VAD_START_SECS: f32 = 0.2;
pub const VAD_STOP_SECS: f32 = 0.2;
pub const VAD_MIN_VOLUME: f32 = 0.6;
impl Default for VadParams {
fn default() -> Self {
Self {
confidence: VAD_CONFIDENCE,
start_secs: VAD_START_SECS,
stop_secs: VAD_STOP_SECS,
min_volume: VAD_MIN_VOLUME,
}
}
}
#[derive(Debug, Clone)]
pub struct VadStateMachine {
params: VadParams,
sample_rate: u32,
window_samples: usize,
start_frames: u32,
stop_frames: u32,
starting_count: u32,
stopping_count: u32,
state: VadState,
}
impl VadStateMachine {
pub fn new(sample_rate: u32, window_samples: usize, params: VadParams) -> Self {
let mut m = Self {
params,
sample_rate,
window_samples,
start_frames: 1,
stop_frames: 1,
starting_count: 0,
stopping_count: 0,
state: VadState::Quiet,
};
m.recompute_debounce();
m
}
fn recompute_debounce(&mut self) {
let window_secs = self.window_samples as f32 / self.sample_rate.max(1) as f32;
let per = |secs: f32| -> u32 {
if window_secs <= 0.0 {
1
} else {
(secs / window_secs).round().max(1.0) as u32
}
};
self.start_frames = per(self.params.start_secs);
self.stop_frames = per(self.params.stop_secs);
}
pub fn set_params(&mut self, params: VadParams) {
self.params = params;
self.starting_count = 0;
self.stopping_count = 0;
self.state = VadState::Quiet;
self.recompute_debounce();
}
pub fn state(&self) -> VadState {
self.state
}
pub fn advance(&mut self, confidence: f32, volume: f32) -> VadState {
let speaking = confidence >= self.params.confidence && volume >= self.params.min_volume;
if speaking {
match self.state {
VadState::Quiet => {
self.state = VadState::Starting;
self.starting_count = 1;
}
VadState::Starting => self.starting_count += 1,
VadState::Stopping => {
self.state = VadState::Speaking;
self.stopping_count = 0;
}
VadState::Speaking => {}
}
} else {
match self.state {
VadState::Starting => {
self.state = VadState::Quiet;
self.starting_count = 0;
}
VadState::Speaking => {
self.state = VadState::Stopping;
self.stopping_count = 1;
}
VadState::Stopping => self.stopping_count += 1,
VadState::Quiet => {}
}
}
if self.state == VadState::Starting && self.starting_count >= self.start_frames {
self.state = VadState::Speaking;
self.starting_count = 0;
}
if self.state == VadState::Stopping && self.stopping_count >= self.stop_frames {
self.state = VadState::Quiet;
self.stopping_count = 0;
}
self.state
}
}
pub fn window_volume(pcm: &[i16]) -> f32 {
if pcm.is_empty() {
return 0.0;
}
let sum_sq: f64 = pcm
.iter()
.map(|&s| {
let f = s as f64 / 32768.0;
f * f
})
.sum();
let rms = (sum_sq / pcm.len() as f64).sqrt();
if rms <= 0.0 {
return 0.0;
}
let db = 20.0 * rms.log10(); (((db + 60.0) / 60.0).clamp(0.0, 1.0)) as f32
}
#[derive(Debug, Clone, Default)]
pub struct VolumeSmoother {
prev: f32,
}
impl VolumeSmoother {
pub fn smooth(&mut self, volume: f32) -> f32 {
self.prev += 0.2 * (volume - self.prev);
self.prev
}
}
pub struct MockVad {
sm: VadStateMachine,
window_samples: usize,
script: Option<Vec<f32>>,
script_pos: usize,
}
impl MockVad {
pub fn new(sample_rate: u32, window_samples: usize) -> Self {
Self {
sm: VadStateMachine::new(sample_rate, window_samples, VadParams::default()),
window_samples,
script: None,
script_pos: 0,
}
}
pub fn with_params(sample_rate: u32, window_samples: usize, params: VadParams) -> Self {
Self {
sm: VadStateMachine::new(sample_rate, window_samples, params),
window_samples,
script: None,
script_pos: 0,
}
}
pub fn with_script(sample_rate: u32, window_samples: usize, script: Vec<f32>) -> Self {
Self::with_script_params(sample_rate, window_samples, script, VadParams::default())
}
pub fn with_script_params(
sample_rate: u32,
window_samples: usize,
script: Vec<f32>,
params: VadParams,
) -> Self {
Self {
sm: VadStateMachine::new(sample_rate, window_samples, params),
window_samples,
script: Some(script),
script_pos: 0,
}
}
pub fn window_samples(&self) -> usize {
self.window_samples
}
}
impl VadAnalyzer for MockVad {
fn sample_rate(&self) -> u32 {
self.sm.sample_rate
}
fn analyze(&mut self, audio: &AudioFrame) -> VadState {
let (confidence, gate_volume) = match &self.script {
Some(script) => {
let c = script
.get(self.script_pos)
.copied()
.or_else(|| script.last().copied())
.unwrap_or(0.0);
self.script_pos += 1;
(c, self.sm.params.min_volume)
}
None => {
let volume = window_volume(&audio.pcm);
let c = if volume >= self.sm.params.min_volume {
1.0
} else {
0.0
};
(c, volume)
}
};
self.sm.advance(confidence, gate_volume)
}
fn set_params(&mut self, params: VadParams) {
self.sm.set_params(params);
}
}
pub struct VadProcessor<V: VadAnalyzer> {
analyzer: V,
window_samples: usize,
buffer: Vec<i16>,
sample_rate: u32,
last_state: VadState,
bot_speaking: bool,
interrupt_on_barge_in: bool,
}
impl<V: VadAnalyzer> VadProcessor<V> {
pub fn new(analyzer: V, window_samples: usize) -> Self {
let sample_rate = analyzer.sample_rate();
Self {
analyzer,
window_samples: window_samples.max(1),
buffer: Vec::new(),
sample_rate,
last_state: VadState::Quiet,
bot_speaking: false,
interrupt_on_barge_in: true,
}
}
pub fn without_barge_in_interruption(mut self) -> Self {
self.interrupt_on_barge_in = false;
self
}
async fn drain_windows(&mut self, link: &Link) {
while self.buffer.len() >= self.window_samples {
let window: Vec<i16> = self.buffer.drain(..self.window_samples).collect();
let frame = AudioFrame::mono(window, self.sample_rate);
let new_state = self.analyzer.analyze(&frame);
let was_speaking = matches!(self.last_state, VadState::Speaking | VadState::Stopping);
let is_speaking = matches!(new_state, VadState::Speaking);
if is_speaking && !was_speaking {
let secs = self.window_samples as f32 / self.sample_rate.max(1) as f32;
link.push_down(Frame::VadUserStartedSpeaking { start_secs: secs })
.await;
link.push_down(Frame::UserStartedSpeaking).await;
if self.bot_speaking && self.interrupt_on_barge_in {
link.broadcast(Frame::Interruption).await;
}
}
else if was_speaking && matches!(new_state, VadState::Quiet) {
let secs = self.window_samples as f32 / self.sample_rate.max(1) as f32;
link.push_down(Frame::VadUserStoppedSpeaking { stop_secs: secs })
.await;
link.push_down(Frame::UserStoppedSpeaking).await;
}
self.last_state = new_state;
}
}
}
#[async_trait]
impl<V: VadAnalyzer + 'static> FrameProcessor for VadProcessor<V> {
fn name(&self) -> &str {
"VadProcessor"
}
async fn start(&mut self, _setup: &ProcessorSetup, params: &StartParams) -> Result<()> {
self.sample_rate = params.audio_in_sample_rate;
Ok(())
}
async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
match &env.frame {
Frame::BotStartedSpeaking => self.bot_speaking = true,
Frame::BotStoppedSpeaking => self.bot_speaking = false,
Frame::InputAudio(audio) => {
self.buffer.extend_from_slice(&audio.pcm);
self.drain_windows(link).await;
}
_ => {}
}
link.push(env.meta, env.frame, env.direction).await;
Ok(())
}
}
#[cfg(feature = "vad-ort")]
pub struct SileroVad {
sm: VadStateMachine,
sample_rate: u32,
window_samples: usize,
session: ort::session::Session,
state: ndarray::Array3<f32>,
context: Vec<f32>,
context_size: usize,
windows_since_reset: u32,
reset_every_windows: u32,
smoother: VolumeSmoother,
}
#[cfg(feature = "vad-ort")]
impl SileroVad {
pub fn from_model_path(
model_path: impl AsRef<std::path::Path>,
sample_rate: u32,
) -> Result<Self> {
Self::with_params(model_path, sample_rate, VadParams::default())
}
pub fn with_params(
model_path: impl AsRef<std::path::Path>,
sample_rate: u32,
params: VadParams,
) -> Result<Self> {
use crate::error::FlowcatError;
if sample_rate != 8000 && sample_rate != 16000 {
return Err(FlowcatError::Codec(format!(
"Silero VAD sample rate must be 8000 or 16000 (got {sample_rate})"
)));
}
let (window_samples, context_size) = if sample_rate == 16000 {
(512usize, 64usize)
} else {
(256usize, 32usize)
};
let session = ort::session::Session::builder()
.and_then(|mut b| b.commit_from_file(model_path.as_ref()))
.map_err(|e| FlowcatError::Realtime(format!("silero load: {e}")))?;
let window_secs = window_samples as f32 / sample_rate as f32;
let reset_every_windows = (5.0 / window_secs).round().max(1.0) as u32;
Ok(Self {
sm: VadStateMachine::new(sample_rate, window_samples, params),
sample_rate,
window_samples,
session,
state: ndarray::Array3::<f32>::zeros((2, 1, 128)),
context: vec![0.0; context_size],
context_size,
windows_since_reset: 0,
reset_every_windows,
smoother: VolumeSmoother::default(),
})
}
pub fn window_samples(&self) -> usize {
self.window_samples
}
fn reset_states(&mut self) {
self.state = ndarray::Array3::<f32>::zeros((2, 1, 128));
self.context = vec![0.0; self.context_size];
}
fn voice_confidence(&mut self, pcm: &[i16]) -> f32 {
use ndarray::{Array1, Array2};
let mut x: Vec<f32> = Vec::with_capacity(self.context_size + pcm.len());
x.extend_from_slice(&self.context);
x.extend(pcm.iter().map(|&s| s as f32 / 32768.0));
let input_len = x.len();
let input = match Array2::from_shape_vec((1, input_len), x.clone()) {
Ok(a) => a,
Err(_) => return 0.0,
};
let sr = Array1::from_vec(vec![self.sample_rate as i64]);
let run = (|| -> std::result::Result<f32, Box<dyn std::error::Error>> {
let input_val = ort::value::Tensor::from_array(input)?;
let state_val = ort::value::Tensor::from_array(self.state.clone())?;
let sr_val = ort::value::Tensor::from_array(sr)?;
let outputs = self.session.run(ort::inputs![
"input" => input_val,
"state" => state_val,
"sr" => sr_val,
])?;
let (_, prob) = outputs[0].try_extract_tensor::<f32>()?;
let confidence = prob.first().copied().unwrap_or(0.0);
let (_, new_state) = outputs[1].try_extract_tensor::<f32>()?;
if new_state.len() == self.state.len() {
self.state = ndarray::Array3::from_shape_vec((2, 1, 128), new_state.to_vec())?;
}
Ok(confidence)
})();
if input_len >= self.context_size {
self.context = x[input_len - self.context_size..].to_vec();
}
match run {
Ok(c) => c,
Err(e) => {
tracing::warn!("Silero VAD inference failed: {e}");
0.0
}
}
}
}
#[cfg(feature = "vad-ort")]
impl VadAnalyzer for SileroVad {
fn sample_rate(&self) -> u32 {
self.sample_rate
}
fn analyze(&mut self, audio: &AudioFrame) -> VadState {
self.windows_since_reset += 1;
if self.windows_since_reset >= self.reset_every_windows {
self.reset_states();
self.windows_since_reset = 0;
}
let confidence = self.voice_confidence(&audio.pcm);
let volume = self.smoother.smooth(window_volume(&audio.pcm));
self.sm.advance(confidence, volume)
}
fn set_params(&mut self, params: VadParams) {
self.sm.set_params(params);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::observer::{FrameEvent, FrameObserver};
use crate::pipeline::{Pipeline, PipelineTask, PipelineTaskParams};
use crate::processor::frame::Direction;
use crate::processor::Envelope;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
fn loud(n: usize, rate: u32) -> AudioFrame {
AudioFrame::mono(vec![20000i16; n], rate)
}
fn quiet(n: usize, rate: u32) -> AudioFrame {
AudioFrame::mono(vec![0i16; n], rate)
}
#[test]
fn state_machine_debounce_start_and_stop() {
let params = VadParams {
confidence: 0.5,
start_secs: 0.064,
stop_secs: 0.064,
min_volume: 0.0,
};
let mut sm = VadStateMachine::new(16000, 512, params);
assert_eq!(
sm.advance(1.0, 1.0),
VadState::Starting,
"1st speech window"
);
assert_eq!(sm.advance(1.0, 1.0), VadState::Speaking, "2nd => debounced");
assert_eq!(sm.advance(1.0, 1.0), VadState::Speaking);
assert_eq!(sm.advance(0.0, 0.0), VadState::Stopping);
assert_eq!(sm.advance(0.0, 0.0), VadState::Quiet);
}
#[test]
fn state_machine_brief_blip_does_not_reach_speaking() {
let params = VadParams {
confidence: 0.5,
start_secs: 0.1, stop_secs: 0.064,
min_volume: 0.0,
};
let mut sm = VadStateMachine::new(16000, 512, params);
assert_eq!(sm.advance(1.0, 1.0), VadState::Starting);
assert_eq!(sm.advance(0.0, 0.0), VadState::Quiet);
}
#[test]
fn mock_vad_volume_driven_reaches_speaking_then_quiet() {
let mut vad = MockVad::with_params(
16000,
512,
VadParams {
confidence: 0.5,
start_secs: 0.032, stop_secs: 0.032, min_volume: 0.3,
},
);
assert_eq!(vad.analyze(&loud(512, 16000)), VadState::Speaking);
assert_eq!(vad.analyze(&quiet(512, 16000)), VadState::Quiet);
}
#[test]
fn mock_vad_scripted_confidence() {
let mut vad = MockVad::with_script_params(
8000,
256,
vec![0.0, 1.0, 1.0, 0.0, 0.0],
VadParams {
confidence: 0.5,
start_secs: 0.032,
stop_secs: 0.032,
min_volume: 0.0,
},
);
let w = || AudioFrame::mono(vec![0i16; 256], 8000);
assert_eq!(vad.analyze(&w()), VadState::Quiet);
assert_eq!(vad.analyze(&w()), VadState::Speaking);
vad.analyze(&w());
assert_eq!(vad.analyze(&w()), VadState::Quiet);
}
#[derive(Default)]
struct EdgeTap {
started: AtomicUsize,
stopped: AtomicUsize,
interruptions: AtomicUsize,
names: Mutex<Vec<&'static str>>,
}
#[async_trait]
impl FrameObserver for EdgeTap {
async fn on_process(&self, e: &FrameEvent<'_>) {
self.names.lock().unwrap().push(e.frame.name());
match e.frame {
Frame::VadUserStartedSpeaking { .. } => {
self.started.fetch_add(1, Ordering::Relaxed);
}
Frame::VadUserStoppedSpeaking { .. } => {
self.stopped.fetch_add(1, Ordering::Relaxed);
}
Frame::Interruption => {
self.interruptions.fetch_add(1, Ordering::Relaxed);
}
_ => {}
}
}
}
#[tokio::test]
async fn vad_processor_emits_start_and_stop_edges() {
let vad = MockVad::with_params(
16000,
512,
VadParams {
confidence: 0.5,
start_secs: 0.032,
stop_secs: 0.032,
min_volume: 0.3,
},
);
let proc = VadProcessor::new(vad, 512);
let pipeline = Pipeline::new(vec![Box::new(proc)]);
let tap = Arc::new(EdgeTap::default());
let task = PipelineTask::new(
pipeline,
PipelineTaskParams::default(),
vec![tap.clone() as Arc<dyn FrameObserver>],
);
task.queue_frame(Frame::InputAudio(Arc::new(loud(512, 16000))))
.await;
task.queue_frame(Frame::InputAudio(Arc::new(loud(512, 16000))))
.await;
task.queue_frame(Frame::InputAudio(Arc::new(quiet(512, 16000))))
.await;
task.queue_frame(Frame::InputAudio(Arc::new(quiet(512, 16000))))
.await;
task.stop_when_done().await;
tokio::time::timeout(std::time::Duration::from_secs(5), task.run())
.await
.expect("vad pipeline timed out")
.expect("run ok");
assert_eq!(
tap.started.load(Ordering::Relaxed),
1,
"exactly one start edge; saw {:?}",
tap.names.lock().unwrap()
);
assert_eq!(
tap.stopped.load(Ordering::Relaxed),
1,
"exactly one stop edge"
);
}
fn capture_link() -> (
Link,
crate::processor::runtime::ProcessorRx,
crate::processor::runtime::ProcessorRx,
) {
use crate::processor::runtime::channel;
let (down_tx, down_rx) = channel(Arc::from("down"), 64);
let (up_tx, up_rx) = channel(Arc::from("up"), 64);
let link = Link {
next: Some(down_tx),
prev: Some(up_tx),
name: Arc::from("VadProcessor"),
clock: crate::processor::Clock::new(),
observer: None,
enable_metrics: false,
enable_usage_metrics: false,
ttfb_start: Arc::new(std::sync::atomic::AtomicI64::new(0)),
processing_start: Arc::new(std::sync::atomic::AtomicI64::new(0)),
};
(link, down_rx, up_rx)
}
fn drain(rx: &mut crate::processor::runtime::ProcessorRx) -> Vec<&'static str> {
let mut out = Vec::new();
while let Ok(e) = rx.system.try_recv() {
out.push(e.frame.name());
}
while let Ok(e) = rx.normal.try_recv() {
out.push(e.frame.name());
}
out
}
#[tokio::test]
async fn vad_processor_broadcasts_interruption_during_bot_speech() {
let vad = MockVad::with_params(
16000,
512,
VadParams {
confidence: 0.5,
start_secs: 0.032,
stop_secs: 0.032,
min_volume: 0.3,
},
);
let mut proc = VadProcessor::new(vad, 512);
let (link, mut down_rx, mut up_rx) = capture_link();
proc.process_frame(
Envelope::new(Frame::BotStartedSpeaking, Direction::Downstream),
&link,
)
.await
.unwrap();
let _ = drain(&mut down_rx);
proc.process_frame(
Envelope::new(
Frame::InputAudio(Arc::new(loud(512, 16000))),
Direction::Downstream,
),
&link,
)
.await
.unwrap();
let down = drain(&mut down_rx);
let up = drain(&mut up_rx);
assert!(
down.contains(&"VadUserStartedSpeaking") && down.contains(&"UserStartedSpeaking"),
"rising edge frames must be emitted downstream; saw {down:?}"
);
assert!(
down.iter().filter(|n| **n == "Interruption").count() == 1,
"exactly one downstream Interruption; saw {down:?}"
);
assert!(
up.iter().filter(|n| **n == "Interruption").count() == 1,
"exactly one upstream Interruption; saw {up:?}"
);
}
#[tokio::test]
async fn vad_processor_no_interruption_when_bot_silent() {
let vad = MockVad::with_params(
16000,
512,
VadParams {
confidence: 0.5,
start_secs: 0.032,
stop_secs: 0.032,
min_volume: 0.3,
},
);
let mut proc = VadProcessor::new(vad, 512);
let (link, mut down_rx, mut up_rx) = capture_link();
proc.process_frame(
Envelope::new(
Frame::InputAudio(Arc::new(loud(512, 16000))),
Direction::Downstream,
),
&link,
)
.await
.unwrap();
let down = drain(&mut down_rx);
let up = drain(&mut up_rx);
assert!(
down.contains(&"UserStartedSpeaking"),
"rising edge still fires; saw {down:?}"
);
assert!(
!down.contains(&"Interruption") && !up.contains(&"Interruption"),
"no interruption when the bot is not speaking; down={down:?} up={up:?}"
);
}
#[cfg(feature = "vad-ort")]
#[tokio::test]
#[ignore = "needs Silero ONNX model at $SILERO_VAD_ONNX"]
async fn silero_vad_smoke() {
let path = std::env::var("SILERO_VAD_ONNX")
.expect("set SILERO_VAD_ONNX to the silero_vad.onnx path");
let mut vad = SileroVad::from_model_path(&path, 16000).expect("load model");
for _ in 0..5 {
let st = vad.analyze(&AudioFrame::mono(vec![0i16; 512], 16000));
assert_eq!(st, VadState::Quiet);
}
}
}