1use crate::event::{EventSender, SessionEvent};
2use crate::media::processor::Processor;
3use crate::media::{AudioFrame, PcmBuf, Samples};
4use anyhow::Result;
5use serde::{Deserialize, Serialize};
6use serde_with::skip_serializing_none;
7use std::any::Any;
8use std::collections::VecDeque;
9use tokio_util::sync::CancellationToken;
10
11pub(crate) mod simd;
12pub mod tiny_silero;
13pub(crate) mod utils;
14pub use tiny_silero::TinySilero;
15
16#[cfg(test)]
17mod benchmark_all;
18#[cfg(test)]
19mod tests;
20
21#[skip_serializing_none]
22#[derive(Clone, Debug, Deserialize, Serialize)]
23#[serde(rename_all = "camelCase")]
24#[serde(default)]
25pub struct VADOption {
26 pub r#type: VadType,
27 pub samplerate: u32,
28 pub speech_padding: u64,
30 pub silence_padding: u64,
32 pub ratio: f32,
33 pub voice_threshold: f32,
34 pub min_level_dbfs: Option<f32>,
39 pub max_buffer_duration_secs: u64,
40 pub silence_timeout: Option<u64>,
42 pub endpoint: Option<String>,
43 pub secret_key: Option<String>,
44 pub secret_id: Option<String>,
45 #[serde(skip)]
46 pub refer: Option<bool>,
47}
48
49impl Default for VADOption {
50 fn default() -> Self {
51 Self {
52 r#type: VadType::Silero,
53 samplerate: 16000,
54 speech_padding: 250, silence_padding: 100, ratio: 0.5,
57 voice_threshold: 0.5,
58 min_level_dbfs: None,
59 max_buffer_duration_secs: 50,
60 silence_timeout: None,
61 endpoint: None,
62 secret_key: None,
63 secret_id: None,
64 refer: None,
65 }
66 }
67}
68
69#[derive(Clone, Debug, Serialize, Eq, Hash, PartialEq)]
70#[serde(rename_all = "lowercase")]
71pub enum VadType {
72 Silero,
73 Other(String),
74}
75
76impl<'de> Deserialize<'de> for VadType {
77 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
78 where
79 D: serde::Deserializer<'de>,
80 {
81 let value = String::deserialize(deserializer)?;
82 match value.as_str() {
83 "silero" => Ok(VadType::Silero),
84 _ => Ok(VadType::Other(value)),
85 }
86 }
87}
88
89impl std::fmt::Display for VadType {
90 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
91 match self {
92 VadType::Silero => write!(f, "silero"),
93 VadType::Other(provider) => write!(f, "{}", provider),
94 }
95 }
96}
97
98impl TryFrom<&String> for VadType {
99 type Error = String;
100
101 fn try_from(value: &String) -> std::result::Result<Self, Self::Error> {
102 match value.as_str() {
103 "silero" => Ok(VadType::Silero),
104 other => Ok(VadType::Other(other.to_string())),
105 }
106 }
107}
108const IDLE_KEEP_MS: u64 = 500;
111
112struct SampleBuf {
117 samples: VecDeque<i16>,
118 start: u64,
120 base_ts: Option<u64>,
122 sample_rate: u64,
123}
124
125impl SampleBuf {
126 fn new() -> Self {
127 Self {
128 samples: VecDeque::new(),
129 start: 0,
130 base_ts: None,
131 sample_rate: 16000,
132 }
133 }
134
135 fn push(&mut self, frame_ts: u64, sample_rate: u32, samples: &[i16]) {
136 self.base_ts.get_or_insert(frame_ts);
137 if sample_rate > 0 {
138 self.sample_rate = sample_rate as u64;
139 }
140 self.samples.extend(samples);
141 }
142
143 fn index(&self, ts: u64) -> u64 {
145 ts.saturating_sub(self.base_ts.unwrap_or(ts)) * self.sample_rate / 1000
146 }
147
148 fn slice(&self, from_ts: u64, to_ts: u64) -> PcmBuf {
150 let end = self.start + self.samples.len() as u64;
151 let from = self.index(from_ts).clamp(self.start, end);
152 let to = self.index(to_ts).clamp(from, end);
153 self.samples
154 .range((from - self.start) as usize..(to - self.start) as usize)
155 .copied()
156 .collect()
157 }
158
159 fn trim_before(&mut self, ts: u64) {
161 let drop = self
162 .index(ts)
163 .saturating_sub(self.start)
164 .min(self.samples.len() as u64);
165 self.samples.drain(..drop as usize);
166 self.start += drop;
167 }
168
169 fn keep_last(&mut self, ms: u64) {
171 let keep = (ms * self.sample_rate / 1000) as usize;
172 let drop = self.samples.len().saturating_sub(keep);
173 self.samples.drain(..drop);
174 self.start += drop as u64;
175 }
176}
177
178struct VadProcessorInner {
179 vad: Box<dyn VadEngine>,
180 event_sender: EventSender,
181 option: VADOption,
182 buf: SampleBuf,
183 triggered: bool,
184 triggered_event_sent: bool,
185 current_speech_start: Option<u64>,
186 temp_end: Option<u64>,
187 refer: Option<bool>,
188}
189pub struct VadProcessor {
190 inner: VadProcessorInner,
191}
192
193pub trait VadEngine: Send + Sync + Any {
194 fn process(&mut self, frame: &mut AudioFrame) -> Vec<(bool, u64)>;
195
196 fn last_probability(&self) -> Option<f32> {
199 None
200 }
201}
202
203impl VadProcessorInner {
204 pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
205 let samples = match &frame.samples {
206 Samples::PCM { samples } => samples,
207 _ => return Ok(()),
208 };
209
210 self.buf.push(frame.timestamp, frame.sample_rate, samples);
213 let results = self.vad.process(frame);
214 frame.speech_probability = self.vad.last_probability();
215 for (is_speaking, timestamp) in results {
216 self.process_vad_logic(is_speaking, timestamp, &frame.track_id)?;
217 }
218
219 if self.triggered {
220 let max_ms = self.option.max_buffer_duration_secs.max(1) * 1000;
221 if self.buf.samples.len() as u64 > max_ms * self.buf.sample_rate / 1000 {
222 self.buf.keep_last(max_ms);
224 }
225 } else {
226 self.buf.keep_last(IDLE_KEEP_MS);
227 }
228
229 Ok(())
230 }
231
232 fn process_vad_logic(
233 &mut self,
234 is_speaking: bool,
235 timestamp: u64,
236 track_id: &str,
237 ) -> Result<()> {
238 if is_speaking && !self.triggered {
239 self.triggered = true;
240 self.current_speech_start = Some(timestamp);
241 self.triggered_event_sent = false;
242 } else if is_speaking && self.triggered {
243 if let Some(start_time) = self.current_speech_start {
245 let duration = timestamp.saturating_sub(start_time);
246 if duration >= self.option.speech_padding && !self.triggered_event_sent {
249 let event = SessionEvent::Speaking {
250 track_id: track_id.to_string(),
251 timestamp: crate::media::get_timestamp(),
252 start_time,
253 is_filler: None, confidence: Some(1.0),
255 refer: self.refer,
256 };
257 self.event_sender.send(event).ok();
258 self.triggered_event_sent = true;
259 }
260 }
261 } else if !is_speaking {
262 if self.temp_end.is_none() {
263 self.temp_end = Some(timestamp);
264 }
265
266 if let Some(temp_end) = self.temp_end {
267 let silence_duration = timestamp.saturating_sub(temp_end);
269
270 if self.triggered && silence_duration >= self.option.silence_padding {
272 if let Some(start_time) = self.current_speech_start {
273 let duration = temp_end.saturating_sub(start_time);
275 if duration >= self.option.speech_padding {
276 let samples_vec = self.buf.slice(start_time, temp_end);
278 self.buf.trim_before(temp_end);
279
280 let event = SessionEvent::Silence {
281 track_id: track_id.to_string(),
282 timestamp: crate::media::get_timestamp(),
283 start_time,
284 duration,
285 samples: Some(samples_vec),
286 refer: self.refer,
287 };
288 self.event_sender.send(event).ok();
289 }
290 }
291 self.triggered = false;
292 self.triggered_event_sent = false;
293 self.current_speech_start = None;
294 self.temp_end = Some(timestamp); }
296
297 if let Some(timeout) = self.option.silence_timeout {
299 let timeout_duration = timestamp.saturating_sub(temp_end);
301
302 if timeout_duration >= timeout {
303 let event = SessionEvent::Silence {
304 track_id: track_id.to_string(),
305 timestamp: crate::media::get_timestamp(),
306 start_time: temp_end,
307 duration: timeout_duration,
308 samples: None,
309 refer: self.refer,
310 };
311 self.event_sender.send(event).ok();
312 self.temp_end = Some(timestamp);
313 }
314 }
315 }
316 }
317
318 if is_speaking && self.temp_end.is_some() {
319 self.temp_end = None;
320 }
321
322 Ok(())
323 }
324}
325
326impl VadProcessor {
327 pub fn create(
328 _token: CancellationToken,
329 event_sender: EventSender,
330 option: VADOption,
331 ) -> Result<Box<dyn Processor>> {
332 let vad: Box<dyn VadEngine> = match option.r#type {
333 VadType::Silero => Box::new(tiny_silero::TinySilero::new(option.clone())?),
334 _ => Box::new(NopVad::new()?),
335 };
336 Ok(Box::new(VadProcessor::new(vad, event_sender, option)?))
337 }
338
339 pub fn create_nop(
340 _token: CancellationToken,
341 event_sender: EventSender,
342 option: VADOption,
343 ) -> Result<Box<dyn Processor>> {
344 let vad: Box<dyn VadEngine> = match option.r#type {
345 _ => Box::new(NopVad::new()?),
346 };
347 Ok(Box::new(VadProcessor::new(vad, event_sender, option)?))
348 }
349
350 pub fn new(
351 engine: Box<dyn VadEngine>,
352 event_sender: EventSender,
353 option: VADOption,
354 ) -> Result<Self> {
355 let refer = option.refer;
356 let inner = VadProcessorInner {
357 vad: engine,
358 event_sender,
359 option,
360 buf: SampleBuf::new(),
361 triggered_event_sent: false,
362 triggered: false,
363 current_speech_start: None,
364 temp_end: None,
365 refer,
366 };
367 Ok(Self { inner })
368 }
369}
370
371impl Processor for VadProcessor {
372 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
373 self.inner.process_frame(frame)
374 }
375}
376
377struct NopVad {}
378
379impl NopVad {
380 pub fn new() -> Result<Self> {
381 Ok(Self {})
382 }
383}
384
385impl VadEngine for NopVad {
386 fn process(&mut self, frame: &mut AudioFrame) -> Vec<(bool, u64)> {
387 let samples = match &frame.samples {
388 Samples::PCM { samples } => samples,
389 _ => return vec![(false, frame.timestamp)],
390 };
391 let has_speech = samples.iter().any(|&x| x != 0);
393 vec![(has_speech, frame.timestamp)]
394 }
395}