Skip to main content

active_call/media/vad/
mod.rs

1use crate::event::{EventSender, SessionEvent};
2use crate::media::processor::Processor;
3use crate::media::{AudioFrame, PcmBuf, Samples};
4use anyhow::Result;
5use serde::{Deserialize, Serialize};
6use serde_with::skip_serializing_none;
7use std::any::Any;
8use std::collections::VecDeque;
9use tokio_util::sync::CancellationToken;
10
11pub(crate) mod simd;
12pub mod tiny_silero;
13pub(crate) mod utils;
14pub use tiny_silero::TinySilero;
15
16#[cfg(test)]
17mod benchmark_all;
18#[cfg(test)]
19mod tests;
20
21#[skip_serializing_none]
22#[derive(Clone, Debug, Deserialize, Serialize)]
23#[serde(rename_all = "camelCase")]
24#[serde(default)]
25pub struct VADOption {
26    pub r#type: VadType,
27    pub samplerate: u32,
28    /// Padding before speech detection (in ms)
29    pub speech_padding: u64,
30    /// Padding after silence detection (in ms)
31    pub silence_padding: u64,
32    pub ratio: f32,
33    pub voice_threshold: f32,
34    /// Audio quieter than this (RMS, dBFS) is never speech, whatever the model
35    /// says; None disables. Faint background talk - the call-centre ambience
36    /// some voice assistants play under themselves - otherwise reads as speech
37    /// once heard for a few seconds.
38    pub min_level_dbfs: Option<f32>,
39    pub max_buffer_duration_secs: u64,
40    /// Timeout duration for silence (in ms), None means disable this feature
41    pub silence_timeout: Option<u64>,
42    pub endpoint: Option<String>,
43    pub secret_key: Option<String>,
44    pub secret_id: Option<String>,
45    #[serde(skip)]
46    pub refer: Option<bool>,
47}
48
49impl Default for VADOption {
50    fn default() -> Self {
51        Self {
52            r#type: VadType::Silero,
53            samplerate: 16000,
54            speech_padding: 250, // min_speech_duration_ms (match silero_vad default)
55            silence_padding: 100, // min_silence_duration_ms
56            ratio: 0.5,
57            voice_threshold: 0.5,
58            min_level_dbfs: None,
59            max_buffer_duration_secs: 50,
60            silence_timeout: None,
61            endpoint: None,
62            secret_key: None,
63            secret_id: None,
64            refer: None,
65        }
66    }
67}
68
69#[derive(Clone, Debug, Serialize, Eq, Hash, PartialEq)]
70#[serde(rename_all = "lowercase")]
71pub enum VadType {
72    Silero,
73    Other(String),
74}
75
76impl<'de> Deserialize<'de> for VadType {
77    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
78    where
79        D: serde::Deserializer<'de>,
80    {
81        let value = String::deserialize(deserializer)?;
82        match value.as_str() {
83            "silero" => Ok(VadType::Silero),
84            _ => Ok(VadType::Other(value)),
85        }
86    }
87}
88
89impl std::fmt::Display for VadType {
90    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
91        match self {
92            VadType::Silero => write!(f, "silero"),
93            VadType::Other(provider) => write!(f, "{}", provider),
94        }
95    }
96}
97
98impl TryFrom<&String> for VadType {
99    type Error = String;
100
101    fn try_from(value: &String) -> std::result::Result<Self, Self::Error> {
102        match value.as_str() {
103            "silero" => Ok(VadType::Silero),
104            other => Ok(VadType::Other(other.to_string())),
105        }
106    }
107}
108/// Audio kept while idle, so an utterance can start a little before the
109/// frame that completed its first speech chunk (a chunk spans earlier frames).
110const IDLE_KEEP_MS: u64 = 500;
111
112/// Recent input, contiguous, so an utterance's samples can be cut out by
113/// timestamp. Engines report one decision per chunk of their own size (512
114/// samples for silero), which doesn't line up with the frames (e.g. 320
115/// samples at 20 ms ptime) - so audio is kept per sample, not per decision.
116struct SampleBuf {
117    samples: VecDeque<i16>,
118    /// absolute index (samples since the first frame) of `samples[0]`
119    start: u64,
120    /// timestamp of the first frame, which engine timestamps count from
121    base_ts: Option<u64>,
122    sample_rate: u64,
123}
124
125impl SampleBuf {
126    fn new() -> Self {
127        Self {
128            samples: VecDeque::new(),
129            start: 0,
130            base_ts: None,
131            sample_rate: 16000,
132        }
133    }
134
135    fn push(&mut self, frame_ts: u64, sample_rate: u32, samples: &[i16]) {
136        self.base_ts.get_or_insert(frame_ts);
137        if sample_rate > 0 {
138            self.sample_rate = sample_rate as u64;
139        }
140        self.samples.extend(samples);
141    }
142
143    /// Absolute sample index of an engine timestamp.
144    fn index(&self, ts: u64) -> u64 {
145        ts.saturating_sub(self.base_ts.unwrap_or(ts)) * self.sample_rate / 1000
146    }
147
148    /// Samples in [from_ts, to_ts), clamped to what is still buffered.
149    fn slice(&self, from_ts: u64, to_ts: u64) -> PcmBuf {
150        let end = self.start + self.samples.len() as u64;
151        let from = self.index(from_ts).clamp(self.start, end);
152        let to = self.index(to_ts).clamp(from, end);
153        self.samples
154            .range((from - self.start) as usize..(to - self.start) as usize)
155            .copied()
156            .collect()
157    }
158
159    /// Drops everything before `ts`.
160    fn trim_before(&mut self, ts: u64) {
161        let drop = self
162            .index(ts)
163            .saturating_sub(self.start)
164            .min(self.samples.len() as u64);
165        self.samples.drain(..drop as usize);
166        self.start += drop;
167    }
168
169    /// Drops all but the last `ms`.
170    fn keep_last(&mut self, ms: u64) {
171        let keep = (ms * self.sample_rate / 1000) as usize;
172        let drop = self.samples.len().saturating_sub(keep);
173        self.samples.drain(..drop);
174        self.start += drop as u64;
175    }
176}
177
178struct VadProcessorInner {
179    vad: Box<dyn VadEngine>,
180    event_sender: EventSender,
181    option: VADOption,
182    buf: SampleBuf,
183    triggered: bool,
184    triggered_event_sent: bool,
185    current_speech_start: Option<u64>,
186    temp_end: Option<u64>,
187    refer: Option<bool>,
188}
189pub struct VadProcessor {
190    inner: VadProcessorInner,
191}
192
193pub trait VadEngine: Send + Sync + Any {
194    fn process(&mut self, frame: &mut AudioFrame) -> Vec<(bool, u64)>;
195
196    /// Latest raw speech probability in [0, 1] from the underlying model.
197    /// Returns None for engines that don't produce a probability (e.g. NopVad).
198    fn last_probability(&self) -> Option<f32> {
199        None
200    }
201}
202
203impl VadProcessorInner {
204    pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
205        let samples = match &frame.samples {
206            Samples::PCM { samples } => samples,
207            _ => return Ok(()),
208        };
209
210        // every frame is kept, not just those completing an engine chunk:
211        // decisions are cut back out of this by timestamp
212        self.buf.push(frame.timestamp, frame.sample_rate, samples);
213        let results = self.vad.process(frame);
214        frame.speech_probability = self.vad.last_probability();
215        for (is_speaking, timestamp) in results {
216            self.process_vad_logic(is_speaking, timestamp, &frame.track_id)?;
217        }
218
219        if self.triggered {
220            let max_ms = self.option.max_buffer_duration_secs.max(1) * 1000;
221            if self.buf.samples.len() as u64 > max_ms * self.buf.sample_rate / 1000 {
222                // an utterance this long loses its beginning, not the whole buffer
223                self.buf.keep_last(max_ms);
224            }
225        } else {
226            self.buf.keep_last(IDLE_KEEP_MS);
227        }
228
229        Ok(())
230    }
231
232    fn process_vad_logic(
233        &mut self,
234        is_speaking: bool,
235        timestamp: u64,
236        track_id: &str,
237    ) -> Result<()> {
238        if is_speaking && !self.triggered {
239            self.triggered = true;
240            self.current_speech_start = Some(timestamp);
241            self.triggered_event_sent = false;
242        } else if is_speaking && self.triggered {
243            // Already triggered, check if we need to emit Speaking event for the first time
244            if let Some(start_time) = self.current_speech_start {
245                let duration = timestamp.saturating_sub(start_time);
246                // L1 filter: only emit Speaking if duration is enough (e.g. 200ms)
247                // Use speech_padding as min_speech_duration
248                if duration >= self.option.speech_padding && !self.triggered_event_sent {
249                    let event = SessionEvent::Speaking {
250                        track_id: track_id.to_string(),
251                        timestamp: crate::media::get_timestamp(),
252                        start_time,
253                        is_filler: None, // Will be enriched by MFCC or ASR
254                        confidence: Some(1.0),
255                        refer: self.refer,
256                    };
257                    self.event_sender.send(event).ok();
258                    self.triggered_event_sent = true;
259                }
260            }
261        } else if !is_speaking {
262            if self.temp_end.is_none() {
263                self.temp_end = Some(timestamp);
264            }
265
266            if let Some(temp_end) = self.temp_end {
267                // Use saturating_sub to handle timestamp wrapping or out-of-order frames
268                let silence_duration = timestamp.saturating_sub(temp_end);
269
270                // Process regular silence detection for speech segments
271                if self.triggered && silence_duration >= self.option.silence_padding {
272                    if let Some(start_time) = self.current_speech_start {
273                        // Use safe duration calculation
274                        let duration = temp_end.saturating_sub(start_time);
275                        if duration >= self.option.speech_padding {
276                            // temp_end is the first non-speech chunk's start
277                            let samples_vec = self.buf.slice(start_time, temp_end);
278                            self.buf.trim_before(temp_end);
279
280                            let event = SessionEvent::Silence {
281                                track_id: track_id.to_string(),
282                                timestamp: crate::media::get_timestamp(),
283                                start_time,
284                                duration,
285                                samples: Some(samples_vec),
286                                refer: self.refer,
287                            };
288                            self.event_sender.send(event).ok();
289                        }
290                    }
291                    self.triggered = false;
292                    self.triggered_event_sent = false;
293                    self.current_speech_start = None;
294                    self.temp_end = Some(timestamp); // Update temp_end for silence timeout tracking
295                }
296
297                // Process silence timeout if configured
298                if let Some(timeout) = self.option.silence_timeout {
299                    // Use same safe calculation for silence timeout
300                    let timeout_duration = timestamp.saturating_sub(temp_end);
301
302                    if timeout_duration >= timeout {
303                        let event = SessionEvent::Silence {
304                            track_id: track_id.to_string(),
305                            timestamp: crate::media::get_timestamp(),
306                            start_time: temp_end,
307                            duration: timeout_duration,
308                            samples: None,
309                            refer: self.refer,
310                        };
311                        self.event_sender.send(event).ok();
312                        self.temp_end = Some(timestamp);
313                    }
314                }
315            }
316        }
317
318        if is_speaking && self.temp_end.is_some() {
319            self.temp_end = None;
320        }
321
322        Ok(())
323    }
324}
325
326impl VadProcessor {
327    pub fn create(
328        _token: CancellationToken,
329        event_sender: EventSender,
330        option: VADOption,
331    ) -> Result<Box<dyn Processor>> {
332        let vad: Box<dyn VadEngine> = match option.r#type {
333            VadType::Silero => Box::new(tiny_silero::TinySilero::new(option.clone())?),
334            _ => Box::new(NopVad::new()?),
335        };
336        Ok(Box::new(VadProcessor::new(vad, event_sender, option)?))
337    }
338
339    pub fn create_nop(
340        _token: CancellationToken,
341        event_sender: EventSender,
342        option: VADOption,
343    ) -> Result<Box<dyn Processor>> {
344        let vad: Box<dyn VadEngine> = match option.r#type {
345            _ => Box::new(NopVad::new()?),
346        };
347        Ok(Box::new(VadProcessor::new(vad, event_sender, option)?))
348    }
349
350    pub fn new(
351        engine: Box<dyn VadEngine>,
352        event_sender: EventSender,
353        option: VADOption,
354    ) -> Result<Self> {
355        let refer = option.refer;
356        let inner = VadProcessorInner {
357            vad: engine,
358            event_sender,
359            option,
360            buf: SampleBuf::new(),
361            triggered_event_sent: false,
362            triggered: false,
363            current_speech_start: None,
364            temp_end: None,
365            refer,
366        };
367        Ok(Self { inner })
368    }
369}
370
371impl Processor for VadProcessor {
372    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
373        self.inner.process_frame(frame)
374    }
375}
376
377struct NopVad {}
378
379impl NopVad {
380    pub fn new() -> Result<Self> {
381        Ok(Self {})
382    }
383}
384
385impl VadEngine for NopVad {
386    fn process(&mut self, frame: &mut AudioFrame) -> Vec<(bool, u64)> {
387        let samples = match &frame.samples {
388            Samples::PCM { samples } => samples,
389            _ => return vec![(false, frame.timestamp)],
390        };
391        // Check if there are any non-zero samples
392        let has_speech = samples.iter().any(|&x| x != 0);
393        vec![(has_speech, frame.timestamp)]
394    }
395}