active_call/media/vad/
mod.rs

1use crate::event::{EventSender, SessionEvent};
2use crate::media::processor::Processor;
3use crate::media::{AudioFrame, PcmBuf, Samples};
4use anyhow::Result;
5use serde::{Deserialize, Serialize};
6use serde_with::skip_serializing_none;
7use std::any::Any;
8use tokio_util::sync::CancellationToken;
9
10pub(crate) mod simd;
11pub mod tiny_silero;
12pub(crate) mod utils;
13pub use tiny_silero::TinySilero;
14
15#[cfg(test)]
16mod benchmark_all;
17#[cfg(test)]
18mod tests;
19
20#[skip_serializing_none]
21#[derive(Clone, Debug, Deserialize, Serialize)]
22#[serde(rename_all = "camelCase")]
23#[serde(default)]
24pub struct VADOption {
25    pub r#type: VadType,
26    pub samplerate: u32,
27    /// Padding before speech detection (in ms)
28    pub speech_padding: u64,
29    /// Padding after silence detection (in ms)
30    pub silence_padding: u64,
31    pub ratio: f32,
32    pub voice_threshold: f32,
33    pub max_buffer_duration_secs: u64,
34    /// Timeout duration for silence (in ms), None means disable this feature
35    pub silence_timeout: Option<u64>,
36    pub endpoint: Option<String>,
37    pub secret_key: Option<String>,
38    pub secret_id: Option<String>,
39    #[serde(skip)]
40    pub refer: Option<bool>,
41}
42
43impl Default for VADOption {
44    fn default() -> Self {
45        Self {
46            r#type: VadType::Silero,
47            samplerate: 16000,
48            speech_padding: 250, // min_speech_duration_ms (match silero_vad default)
49            silence_padding: 100, // min_silence_duration_ms
50            ratio: 0.5,
51            voice_threshold: 0.5,
52            max_buffer_duration_secs: 50,
53            silence_timeout: None,
54            endpoint: None,
55            secret_key: None,
56            secret_id: None,
57            refer: None,
58        }
59    }
60}
61
62#[derive(Clone, Debug, Serialize, Eq, Hash, PartialEq)]
63#[serde(rename_all = "lowercase")]
64pub enum VadType {
65    Silero,
66    Other(String),
67}
68
69impl<'de> Deserialize<'de> for VadType {
70    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
71    where
72        D: serde::Deserializer<'de>,
73    {
74        let value = String::deserialize(deserializer)?;
75        match value.as_str() {
76            "silero" => Ok(VadType::Silero),
77            _ => Ok(VadType::Other(value)),
78        }
79    }
80}
81
82impl std::fmt::Display for VadType {
83    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
84        match self {
85            VadType::Silero => write!(f, "silero"),
86            VadType::Other(provider) => write!(f, "{}", provider),
87        }
88    }
89}
90
91impl TryFrom<&String> for VadType {
92    type Error = String;
93
94    fn try_from(value: &String) -> std::result::Result<Self, Self::Error> {
95        match value.as_str() {
96            "silero" => Ok(VadType::Silero),
97            other => Ok(VadType::Other(other.to_string())),
98        }
99    }
100}
101struct SpeechBuf {
102    samples: PcmBuf,
103    timestamp: u64,
104}
105
106struct VadProcessorInner {
107    vad: Box<dyn VadEngine>,
108    event_sender: EventSender,
109    option: VADOption,
110    window_bufs: Vec<SpeechBuf>,
111    triggered: bool,
112    triggered_event_sent: bool,
113    current_speech_start: Option<u64>,
114    temp_end: Option<u64>,
115    refer: Option<bool>,
116}
117pub struct VadProcessor {
118    inner: VadProcessorInner,
119}
120
121pub trait VadEngine: Send + Sync + Any {
122    fn process(&mut self, frame: &mut AudioFrame) -> Vec<(bool, u64)>;
123
124    /// Latest raw speech probability in [0, 1] from the underlying model.
125    /// Returns None for engines that don't produce a probability (e.g. NopVad).
126    fn last_probability(&self) -> Option<f32> {
127        None
128    }
129}
130
131impl VadProcessorInner {
132    pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
133        let samples = match &frame.samples {
134            Samples::PCM { samples } => samples,
135            _ => return Ok(()),
136        };
137
138        let samples_cloned = samples.to_owned();
139        let results = self.vad.process(frame);
140        frame.speech_probability = self.vad.last_probability();
141        for (is_speaking, timestamp) in results {
142            if is_speaking || self.triggered {
143                let current_buf = SpeechBuf {
144                    samples: samples_cloned.clone(),
145                    timestamp,
146                };
147                self.window_bufs.push(current_buf);
148            }
149            self.process_vad_logic(is_speaking, timestamp, &frame.track_id)?;
150
151            // Clean up old buffers periodically
152            if self.window_bufs.len() > 1000 || !self.triggered {
153                let cutoff = if self.triggered {
154                    timestamp.saturating_sub(5000)
155                } else {
156                    timestamp.saturating_sub(self.option.silence_padding)
157                };
158                self.window_bufs.retain(|buf| buf.timestamp > cutoff);
159            }
160        }
161
162        Ok(())
163    }
164
165    fn process_vad_logic(
166        &mut self,
167        is_speaking: bool,
168        timestamp: u64,
169        track_id: &str,
170    ) -> Result<()> {
171        if is_speaking && !self.triggered {
172            self.triggered = true;
173            self.current_speech_start = Some(timestamp);
174            self.triggered_event_sent = false;
175        } else if is_speaking && self.triggered {
176            // Already triggered, check if we need to emit Speaking event for the first time
177            if let Some(start_time) = self.current_speech_start {
178                let duration = timestamp.saturating_sub(start_time);
179                // L1 filter: only emit Speaking if duration is enough (e.g. 200ms)
180                // Use speech_padding as min_speech_duration
181                if duration >= self.option.speech_padding && !self.triggered_event_sent {
182                    let event = SessionEvent::Speaking {
183                        track_id: track_id.to_string(),
184                        timestamp: crate::media::get_timestamp(),
185                        start_time,
186                        is_filler: None, // Will be enriched by MFCC or ASR
187                        confidence: Some(1.0),
188                        refer: self.refer,
189                    };
190                    self.event_sender.send(event).ok();
191                    self.triggered_event_sent = true;
192                }
193            }
194        } else if !is_speaking {
195            if self.temp_end.is_none() {
196                self.temp_end = Some(timestamp);
197            }
198
199            if let Some(temp_end) = self.temp_end {
200                // Use saturating_sub to handle timestamp wrapping or out-of-order frames
201                let silence_duration = timestamp.saturating_sub(temp_end);
202
203                // Process regular silence detection for speech segments
204                if self.triggered && silence_duration >= self.option.silence_padding {
205                    if let Some(start_time) = self.current_speech_start {
206                        // Use safe duration calculation
207                        let duration = temp_end.saturating_sub(start_time);
208                        if duration >= self.option.speech_padding {
209                            let samples_vec = self
210                                .window_bufs
211                                .iter()
212                                .filter(|buf| {
213                                    buf.timestamp >= start_time && buf.timestamp <= temp_end
214                                })
215                                .flat_map(|buf| buf.samples.iter())
216                                .cloned()
217                                .collect();
218                            self.window_bufs.clear();
219
220                            let event = SessionEvent::Silence {
221                                track_id: track_id.to_string(),
222                                timestamp: crate::media::get_timestamp(),
223                                start_time,
224                                duration,
225                                samples: Some(samples_vec),
226                                refer: self.refer,
227                            };
228                            self.event_sender.send(event).ok();
229                        }
230                    }
231                    self.triggered = false;
232                    self.triggered_event_sent = false;
233                    self.current_speech_start = None;
234                    self.temp_end = Some(timestamp); // Update temp_end for silence timeout tracking
235                }
236
237                // Process silence timeout if configured
238                if let Some(timeout) = self.option.silence_timeout {
239                    // Use same safe calculation for silence timeout
240                    let timeout_duration = timestamp.saturating_sub(temp_end);
241
242                    if timeout_duration >= timeout {
243                        let event = SessionEvent::Silence {
244                            track_id: track_id.to_string(),
245                            timestamp: crate::media::get_timestamp(),
246                            start_time: temp_end,
247                            duration: timeout_duration,
248                            samples: None,
249                            refer: self.refer,
250                        };
251                        self.event_sender.send(event).ok();
252                        self.temp_end = Some(timestamp);
253                    }
254                }
255            }
256        }
257
258        if is_speaking && self.temp_end.is_some() {
259            self.temp_end = None;
260        }
261
262        Ok(())
263    }
264}
265
266impl VadProcessor {
267    pub fn create(
268        _token: CancellationToken,
269        event_sender: EventSender,
270        option: VADOption,
271    ) -> Result<Box<dyn Processor>> {
272        let vad: Box<dyn VadEngine> = match option.r#type {
273            VadType::Silero => Box::new(tiny_silero::TinySilero::new(option.clone())?),
274            _ => Box::new(NopVad::new()?),
275        };
276        Ok(Box::new(VadProcessor::new(vad, event_sender, option)?))
277    }
278
279    pub fn create_nop(
280        _token: CancellationToken,
281        event_sender: EventSender,
282        option: VADOption,
283    ) -> Result<Box<dyn Processor>> {
284        let vad: Box<dyn VadEngine> = match option.r#type {
285            _ => Box::new(NopVad::new()?),
286        };
287        Ok(Box::new(VadProcessor::new(vad, event_sender, option)?))
288    }
289
290    pub fn new(
291        engine: Box<dyn VadEngine>,
292        event_sender: EventSender,
293        option: VADOption,
294    ) -> Result<Self> {
295        let refer = option.refer;
296        let inner = VadProcessorInner {
297            vad: engine,
298            event_sender,
299            option,
300            window_bufs: Vec::new(),
301            triggered_event_sent: false,
302            triggered: false,
303            current_speech_start: None,
304            temp_end: None,
305            refer,
306        };
307        Ok(Self { inner })
308    }
309}
310
311impl Processor for VadProcessor {
312    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
313        self.inner.process_frame(frame)
314    }
315}
316
317struct NopVad {}
318
319impl NopVad {
320    pub fn new() -> Result<Self> {
321        Ok(Self {})
322    }
323}
324
325impl VadEngine for NopVad {
326    fn process(&mut self, frame: &mut AudioFrame) -> Vec<(bool, u64)> {
327        let samples = match &frame.samples {
328            Samples::PCM { samples } => samples,
329            _ => return vec![(false, frame.timestamp)],
330        };
331        // Check if there are any non-zero samples
332        let has_speech = samples.iter().any(|&x| x != 0);
333        vec![(has_speech, frame.timestamp)]
334    }
335}
active_call/media/vad/mod.rs

active_call/media/vad/
mod.rs
