1#![warn(missing_docs)]
32
33mod config;
34mod resample;
35mod segmenter;
36
37pub use config::VadConfig;
38pub use resample::PcmFormat;
39pub use segmenter::Segment;
40
41use ort::session::Session;
42use ort::value::Tensor;
43use resample::PcmConverter;
44use segmenter::Segmenter;
45
46static MODEL_BYTES: &[u8] = include_bytes!("../assets/silero_vad.onnx");
48
49const STATE_LEN: usize = 2 * 128;
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum VadEvent {
55 SpeechStart {
57 at_sample: u64,
59 },
60 SpeechEnd {
62 at_sample: u64,
64 },
65}
66
67#[derive(Debug, Clone, PartialEq, Eq)]
69pub enum VadError {
70 ModelLoad(String),
72 Inference(String),
74 InvalidConfig(String),
76}
77
78impl std::fmt::Display for VadError {
79 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
80 match self {
81 VadError::ModelLoad(m) => write!(f, "model load error: {m}"),
82 VadError::Inference(m) => write!(f, "inference error: {m}"),
83 VadError::InvalidConfig(m) => write!(f, "invalid config: {m}"),
84 }
85 }
86}
87
88impl std::error::Error for VadError {}
89
90pub struct Vad {
95 session: Session,
96 config: VadConfig,
97 segmenter: Segmenter,
98
99 state: Vec<f32>,
101 context: Vec<f32>,
103 pending: Vec<f32>,
105
106 last_probs: Vec<f32>,
108
109 converter: Option<PcmConverter>,
113}
114
115impl Vad {
116 pub fn new(config: VadConfig) -> Result<Vad, VadError> {
118 config.validate().map_err(VadError::InvalidConfig)?;
119
120 let session = Session::builder()
121 .map_err(|e| VadError::ModelLoad(e.to_string()))?
122 .commit_from_memory(MODEL_BYTES)
123 .map_err(|e| VadError::ModelLoad(e.to_string()))?;
124
125 let segmenter = Segmenter::new(&config);
126 let ctx_size = config.context_size();
127
128 Ok(Vad {
129 session,
130 config,
131 segmenter,
132 state: vec![0.0f32; STATE_LEN],
133 context: vec![0.0f32; ctx_size],
134 pending: Vec::new(),
135 last_probs: Vec::new(),
136 converter: None,
137 })
138 }
139
140 pub fn process(&mut self, samples: &[f32]) -> Vec<VadEvent> {
145 let frame_size = self.config.frame_size();
146 self.last_probs.clear();
147
148 let mut segments_out = Vec::new();
149 let mut events = Vec::new();
150
151 self.pending.extend_from_slice(samples);
153
154 let mut offset = 0;
155 let mut frame_buf = vec![0.0f32; frame_size];
158 while offset + frame_size <= self.pending.len() {
159 frame_buf.copy_from_slice(&self.pending[offset..offset + frame_size]);
160 let prob = self.infer_frame(&frame_buf).unwrap_or(0.0);
162 self.last_probs.push(prob);
163 self.segmenter.feed(prob, &mut segments_out);
164 offset += frame_size;
165 }
166 self.pending.drain(0..offset);
168
169 for seg in segments_out {
170 events.push(VadEvent::SpeechStart {
171 at_sample: seg.start_sample,
172 });
173 events.push(VadEvent::SpeechEnd {
174 at_sample: seg.end_sample,
175 });
176 }
177 events
178 }
179
180 pub fn process_pcm(
203 &mut self,
204 samples: &[f32],
205 input_sample_rate: u32,
206 input_channels: u16,
207 ) -> Vec<VadEvent> {
208 let target = self.config.sample_rate;
209 let format = PcmFormat {
210 sample_rate: input_sample_rate,
211 channels: input_channels,
212 };
213
214 if let Some(c) = &self.converter {
216 if !c.matches(format) {
217 self.converter = None;
218 }
219 }
220
221 if input_sample_rate == target && input_channels <= 1 {
223 return self.process(samples);
224 }
225
226 if self.converter.is_none() {
228 match PcmConverter::new(format, target) {
229 Ok(c) => self.converter = Some(c),
230 Err(_) => return Vec::new(),
231 }
232 }
233
234 let mut converted = Vec::new();
236 {
237 let conv = self.converter.as_mut().expect("converter は直前に用意済み");
238 if conv.convert(samples, &mut converted).is_err() {
239 return Vec::new();
240 }
241 }
242 self.process(&converted)
243 }
244
245 fn infer_frame(&mut self, frame: &[f32]) -> Result<f32, VadError> {
251 let ctx_size = self.config.context_size();
252 let frame_size = self.config.frame_size();
253 debug_assert_eq!(frame.len(), frame_size);
254 debug_assert_eq!(self.context.len(), ctx_size);
255
256 let input_len = ctx_size + frame_size;
258 let mut x = Vec::with_capacity(input_len);
259 x.extend_from_slice(&self.context);
260 x.extend_from_slice(frame);
261
262 let next_context: Vec<f32> = x[input_len - ctx_size..].to_vec();
265
266 let input_tensor = Tensor::from_array(([1_i64, input_len as i64], x))
268 .map_err(|e| VadError::Inference(e.to_string()))?;
269 let state_tensor = Tensor::from_array(([2_i64, 1, 128], self.state.clone()))
270 .map_err(|e| VadError::Inference(e.to_string()))?;
271 let sr_tensor =
273 Tensor::from_array((vec![] as Vec<i64>, vec![self.config.sample_rate as i64]))
274 .map_err(|e| VadError::Inference(e.to_string()))?;
275
276 let outputs = self
277 .session
278 .run(ort::inputs![
279 "input" => input_tensor,
280 "state" => state_tensor,
281 "sr" => sr_tensor,
282 ])
283 .map_err(|e| VadError::Inference(e.to_string()))?;
284
285 let (_pshape, prob_slice) = outputs["output"]
287 .try_extract_tensor::<f32>()
288 .map_err(|e| VadError::Inference(e.to_string()))?;
289 let prob = *prob_slice
290 .first()
291 .ok_or_else(|| VadError::Inference("empty output tensor".to_string()))?;
292
293 let (_sshape, state_slice) = outputs["stateN"]
294 .try_extract_tensor::<f32>()
295 .map_err(|e| VadError::Inference(e.to_string()))?;
296 if state_slice.len() != STATE_LEN {
297 return Err(VadError::Inference(format!(
298 "unexpected state length {} (expected {STATE_LEN})",
299 state_slice.len()
300 )));
301 }
302 let new_state = state_slice.to_vec();
303
304 drop(outputs);
306 self.state = new_state;
307 self.context = next_context;
308
309 Ok(prob)
310 }
311
312 pub fn last_frame_probabilities(&self) -> &[f32] {
316 &self.last_probs
317 }
318
319 pub fn reset(&mut self) {
322 self.state = vec![0.0f32; STATE_LEN];
323 self.context = vec![0.0f32; self.config.context_size()];
324 self.pending.clear();
325 self.last_probs.clear();
326 self.segmenter.reset();
327 self.converter = None;
330 }
331
332 pub fn config(&self) -> &VadConfig {
334 &self.config
335 }
336}
337
338pub fn get_speech_timestamps(
342 samples: &[f32],
343 config: &VadConfig,
344) -> Result<Vec<Segment>, VadError> {
345 let mut vad = Vad::new(config.clone())?;
346 let frame_size = config.frame_size();
347
348 let mut out = Vec::new();
349 let mut chunks = samples.chunks_exact(frame_size);
350 for frame in chunks.by_ref() {
351 let prob = vad.infer_frame(frame)?;
352 vad.last_probs.push(prob);
353 vad.segmenter.feed(prob, &mut out);
354 }
355 vad.segmenter.flush(&mut out);
357
358 Ok(out)
359}
360
361#[cfg(test)]
362mod tests {
363 use super::*;
364 use crate::config::VadConfig;
365 use crate::segmenter::{Segment, Segmenter};
366
367 fn run_probs(config: &VadConfig, probs: &[f32]) -> Vec<Segment> {
369 let mut seg = Segmenter::new(config);
370 let mut out = Vec::new();
371 for &p in probs {
372 seg.feed(p, &mut out);
373 }
374 seg.flush(&mut out);
375 out
376 }
377
378 fn base_config() -> VadConfig {
379 VadConfig {
381 threshold: 0.5,
382 neg_threshold: Some(0.35),
383 min_speech_ms: 0, min_silence_ms: 0,
385 speech_pad_ms: 0,
386 max_speech_ms: 0,
387 sample_rate: 16000,
388 }
389 }
390
391 #[test]
392 fn neg_threshold_default_formula() {
393 let mut c = VadConfig::default();
394 assert_eq!(c.resolved_neg_threshold(), (0.5 - 0.15_f32).max(0.01));
395 c.threshold = 0.1;
396 assert_eq!(c.resolved_neg_threshold(), 0.01); c.neg_threshold = Some(0.2);
398 assert_eq!(c.resolved_neg_threshold(), 0.2); }
400
401 #[test]
402 fn simple_speech_then_silence() {
403 let mut c = base_config();
405 c.min_silence_ms = 32; let mut probs = vec![0.9f32; 20];
408 probs.extend(vec![0.1f32; 30]);
409 let segs = run_probs(&c, &probs);
410 assert_eq!(segs.len(), 1, "exactly one segment");
411 let s = segs[0];
412 assert_eq!(s.start_sample, 0);
414 assert_eq!(s.end_sample, 20 * 512);
416 }
417
418 #[test]
419 fn min_speech_discards_short_segment() {
420 let mut c = base_config();
421 c.min_silence_ms = 32; c.min_speech_ms = 250; let mut probs = vec![0.9f32; 5];
425 probs.extend(vec![0.1f32; 10]);
426 let segs = run_probs(&c, &probs);
427 assert!(
428 segs.is_empty(),
429 "short segment must be discarded, got {segs:?}"
430 );
431
432 let mut probs2 = vec![0.9f32; 10];
434 probs2.extend(vec![0.1f32; 10]);
435 let segs2 = run_probs(&c, &probs2);
436 assert_eq!(segs2.len(), 1);
437 assert_eq!(segs2[0].len_samples(), 10 * 512);
438 }
439
440 #[test]
441 fn min_silence_boundary_keeps_segment_together() {
442 let mut c = base_config();
444 c.min_silence_ms = 192; let mut probs = vec![0.9f32; 10];
447 probs.extend(vec![0.1f32; 3]);
448 probs.extend(vec![0.9f32; 10]);
449 probs.extend(vec![0.1f32; 10]); let segs = run_probs(&c, &probs);
451 assert_eq!(segs.len(), 1, "short gap must NOT split, got {segs:?}");
452 assert_eq!(segs[0].start_sample, 0);
453 assert_eq!(segs[0].end_sample, 23 * 512);
455 }
456
457 #[test]
458 fn min_silence_just_over_splits() {
459 let mut c = base_config();
461 c.min_silence_ms = 64; let mut probs = vec![0.9f32; 5];
467 probs.extend(vec![0.1f32; 3]);
468 probs.extend(vec![0.9f32; 5]);
469 probs.extend(vec![0.1f32; 5]);
470 let segs = run_probs(&c, &probs);
471 assert_eq!(segs.len(), 2, "long gap must split into two, got {segs:?}");
472 assert_eq!(segs[0].start_sample, 0);
473 assert_eq!(segs[0].end_sample, 5 * 512); }
475
476 #[test]
477 fn speech_pad_extends_and_clamps() {
478 let mut c = base_config();
479 c.min_silence_ms = 32; c.speech_pad_ms = 32; let mut probs = vec![0.1f32; 2]; probs.extend(vec![0.9f32; 4]); probs.extend(vec![0.1f32; 5]); let segs = run_probs(&c, &probs);
486 assert_eq!(segs.len(), 1);
487 assert_eq!(segs[0].start_sample, 1024 - 512);
489 assert_eq!(segs[0].end_sample, 6 * 512 + 512);
491 }
492
493 #[test]
494 fn speech_pad_start_clamps_at_zero() {
495 let mut c = base_config();
497 c.min_silence_ms = 32;
498 c.speech_pad_ms = 64; let mut probs = vec![0.9f32; 5];
500 probs.extend(vec![0.1f32; 5]);
501 let segs = run_probs(&c, &probs);
502 assert_eq!(segs.len(), 1);
503 assert_eq!(segs[0].start_sample, 0, "start pad must clamp to 0");
504 }
505
506 #[test]
507 fn pad_does_not_overlap_previous_segment() {
508 let mut c = base_config();
510 c.min_silence_ms = 64; c.speech_pad_ms = 192; let mut probs = vec![0.9f32; 5];
514 probs.extend(vec![0.1f32; 3]);
515 probs.extend(vec![0.9f32; 5]);
516 probs.extend(vec![0.1f32; 5]);
517 let segs = run_probs(&c, &probs);
518 assert_eq!(segs.len(), 2);
519 assert!(
521 segs[1].start_sample >= segs[0].end_sample,
522 "seg1.start ({}) must be >= seg0.end ({})",
523 segs[1].start_sample,
524 segs[0].end_sample
525 );
526 }
527
528 #[test]
529 fn max_speech_forces_split_when_no_silence() {
530 let mut c = base_config();
532 c.max_speech_ms = 192; c.min_silence_ms = 32;
534 let probs = vec![0.9f32; 20];
536 let segs = run_probs(&c, &probs);
537 assert!(
538 segs.len() >= 2,
539 "max_speech must force at least one split, got {segs:?}"
540 );
541 for s in &segs {
543 assert!(
544 s.len_samples() <= c.ms_to_samples(c.max_speech_ms) + 512,
545 "segment {s:?} exceeds max_speech by more than one frame"
546 );
547 }
548 }
549
550 #[test]
551 fn gray_zone_keeps_triggered() {
552 let mut c = base_config(); c.min_silence_ms = 32;
555 let mut probs = vec![0.9f32; 5];
556 probs.extend(vec![0.4f32; 5]); probs.extend(vec![0.9f32; 5]);
558 probs.extend(vec![0.1f32; 5]); let segs = run_probs(&c, &probs);
560 assert_eq!(segs.len(), 1, "gray zone must not split, got {segs:?}");
561 assert_eq!(segs[0].end_sample, 15 * 512);
562 }
563
564 #[test]
567 fn vad_loads_model() {
568 let vad = Vad::new(VadConfig::default());
569 assert!(vad.is_ok(), "model load failed: {:?}", vad.err());
570 }
571
572 #[test]
573 fn zeros_produce_low_prob_no_speech() {
574 let mut vad = Vad::new(VadConfig::default()).unwrap();
575 let zeros = vec![0.0f32; 16000];
576 let events = vad.process(&zeros);
577 assert!(
579 !events
580 .iter()
581 .any(|e| matches!(e, VadEvent::SpeechStart { .. })),
582 "silence must not trigger SpeechStart, got {events:?}"
583 );
584 let probs = vad.last_frame_probabilities();
585 assert_eq!(probs.len(), 16000 / 512, "expected one prob per frame");
586 for &p in probs {
587 assert!((0.0..=1.0).contains(&p), "prob {p} out of [0,1]");
588 assert!(p < 0.5, "silence prob {p} unexpectedly high");
589 }
590 }
591
592 #[test]
593 fn process_streams_across_calls() {
594 let mut vad = Vad::new(VadConfig::default()).unwrap();
596 let total = 2048usize;
598 let chunk = vec![0.0f32; 300];
599 let mut fed = 0usize;
600 let mut total_probs = 0usize;
601 while fed < total {
602 let take = chunk.len().min(total - fed);
603 vad.process(&chunk[..take]);
604 total_probs += vad.last_frame_probabilities().len();
605 fed += take;
606 }
607 assert_eq!(
608 total_probs,
609 total / 512,
610 "all complete frames must be inferred"
611 );
612 }
613
614 #[test]
615 fn reset_clears_state() {
616 let mut vad = Vad::new(VadConfig::default()).unwrap();
617 vad.process(&vec![0.0f32; 1000]); vad.reset();
619 assert_eq!(vad.last_frame_probabilities().len(), 0);
620 assert_eq!(vad.config().sample_rate, 16000);
621 }
622
623 #[test]
624 fn invalid_sample_rate_rejected() {
625 let c = VadConfig {
626 sample_rate: 44100,
627 ..VadConfig::default()
628 };
629 let r = Vad::new(c);
630 assert!(matches!(r, Err(VadError::InvalidConfig(_))));
631 }
632
633 #[test]
634 fn batch_get_speech_timestamps_on_silence() {
635 let zeros = vec![0.0f32; 16000];
636 let segs = get_speech_timestamps(&zeros, &VadConfig::default()).unwrap();
637 assert!(segs.is_empty(), "silence yields no segments, got {segs:?}");
638 }
639
640 fn harmonics(rate: usize, n: usize) -> Vec<f32> {
646 (0..n)
647 .map(|i| {
648 let t = i as f32 / rate as f32;
649 let mut v = 0.0f32;
650 for (k, f) in [180.0f32, 540.0, 1200.0, 2600.0].iter().enumerate() {
651 v += (0.4 / (k as f32 + 1.0)) * (2.0 * std::f32::consts::PI * f * t).sin();
652 }
653 v * 0.5
654 })
655 .collect()
656 }
657
658 fn to_stereo(mono: &[f32]) -> Vec<f32> {
660 let mut s = Vec::with_capacity(mono.len() * 2);
661 for &v in mono {
662 s.push(v);
663 s.push(v);
664 }
665 s
666 }
667
668 #[test]
671 fn process_pcm_passthrough_matches_process() {
672 let sig = harmonics(16_000, 16_000);
673 let mut vad = Vad::new(VadConfig::default()).unwrap();
674
675 let e_pcm = vad.process_pcm(&sig, 16_000, 1);
676 let p_pcm = vad.last_frame_probabilities().to_vec();
677
678 vad.reset();
679 let e_proc = vad.process(&sig);
680 let p_proc = vad.last_frame_probabilities().to_vec();
681
682 assert_eq!(
683 e_pcm, e_proc,
684 "passthrough のイベント列が process と一致しない"
685 );
686 assert_eq!(
687 p_pcm, p_proc,
688 "passthrough の確率列が process とビット一致しない"
689 );
690 assert_eq!(p_pcm.len(), 16_000 / 512, "フレーム数が想定通りでない");
691 }
692
693 #[test]
697 fn process_pcm_split_matches_bulk() {
698 let cfg = VadConfig {
703 threshold: 0.0,
704 neg_threshold: Some(0.0),
705 min_speech_ms: 0,
706 min_silence_ms: 0,
707 speech_pad_ms: 0,
708 max_speech_ms: 200, sample_rate: 16_000,
710 };
711 let stereo = to_stereo(&harmonics(48_000, 48_000));
712
713 let mut vad = Vad::new(cfg).unwrap();
714
715 let bulk_events = vad.process_pcm(&stereo, 48_000, 2);
717 let bulk_probs = vad.last_frame_probabilities().to_vec();
718
719 vad.reset();
720
721 let mut split_events = Vec::new();
723 let mut split_probs = Vec::new();
724 for chunk in stereo.chunks(777) {
725 let evs = vad.process_pcm(chunk, 48_000, 2);
726 split_events.extend(evs);
727 split_probs.extend_from_slice(vad.last_frame_probabilities());
728 }
729
730 assert_eq!(
731 bulk_probs, split_probs,
732 "分割と一括で確率列が一致しない(継ぎ目が出ている)"
733 );
734 assert_eq!(
735 bulk_events, split_events,
736 "分割と一括でイベント列が一致しない"
737 );
738 assert!(
739 !bulk_events.is_empty(),
740 "この設定では発話イベントが出るはず(テストの実効性確認)"
741 );
742 }
743
744 #[test]
747 fn process_pcm_reset_is_deterministic() {
748 let stereo = to_stereo(&harmonics(48_000, 24_000));
749
750 let mut vad = Vad::new(VadConfig::default()).unwrap();
751 let e1 = vad.process_pcm(&stereo, 48_000, 2);
752 let p1 = vad.last_frame_probabilities().to_vec();
753
754 vad.reset();
755 let e2 = vad.process_pcm(&stereo, 48_000, 2);
756 let p2 = vad.last_frame_probabilities().to_vec();
757
758 assert_eq!(e1, e2, "reset 後に同一入力で同一イベントにならない");
759 assert_eq!(p1, p2, "reset 後に同一入力で同一確率にならない");
760 }
761
762 #[test]
765 fn process_pcm_48k_stereo_matches_direct_16k() {
766 let n16 = 16_000usize;
767 let ref16 = harmonics(16_000, n16);
768 let stereo48 = to_stereo(&harmonics(48_000, n16 * 3));
769
770 let mut ref_vad = Vad::new(VadConfig::default()).unwrap();
771 let ref_events = ref_vad.process(&ref16);
772 let ref_probs = ref_vad.last_frame_probabilities().to_vec();
773
774 let mut pcm_vad = Vad::new(VadConfig::default()).unwrap();
775 let pcm_events = pcm_vad.process_pcm(&stereo48, 48_000, 2);
776 let pcm_probs = pcm_vad.last_frame_probabilities().to_vec();
777
778 let n = ref_probs.len().min(pcm_probs.len());
780 assert!(n >= 8, "十分なフレーム数が必要: {n}");
781 for k in 2..(n - 2) {
782 let d = (ref_probs[k] - pcm_probs[k]).abs();
783 assert!(
784 d < 0.05,
785 "frame {k}: 直接16kと変換経路の確率差が大きい: {d} (ref={} pcm={})",
786 ref_probs[k],
787 pcm_probs[k]
788 );
789 }
790 assert_eq!(
792 ref_events, pcm_events,
793 "変換経路のイベント列が直接16kと一致しない"
794 );
795 }
796}