#![warn(missing_docs)]
mod config;
mod resample;
mod segmenter;
pub use config::VadConfig;
pub use resample::PcmFormat;
pub use segmenter::Segment;
use ort::session::Session;
use ort::value::Tensor;
use resample::PcmConverter;
use segmenter::Segmenter;
static MODEL_BYTES: &[u8] = include_bytes!("../assets/silero_vad.onnx");
const STATE_LEN: usize = 2 * 128;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VadEvent {
SpeechStart {
at_sample: u64,
},
SpeechEnd {
at_sample: u64,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VadError {
ModelLoad(String),
Inference(String),
InvalidConfig(String),
}
impl std::fmt::Display for VadError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VadError::ModelLoad(m) => write!(f, "model load error: {m}"),
VadError::Inference(m) => write!(f, "inference error: {m}"),
VadError::InvalidConfig(m) => write!(f, "invalid config: {m}"),
}
}
}
impl std::error::Error for VadError {}
pub struct Vad {
session: Session,
config: VadConfig,
segmenter: Segmenter,
state: Vec<f32>,
context: Vec<f32>,
pending: Vec<f32>,
last_probs: Vec<f32>,
converter: Option<PcmConverter>,
}
impl Vad {
pub fn new(config: VadConfig) -> Result<Vad, VadError> {
config.validate().map_err(VadError::InvalidConfig)?;
let session = Session::builder()
.map_err(|e| VadError::ModelLoad(e.to_string()))?
.commit_from_memory(MODEL_BYTES)
.map_err(|e| VadError::ModelLoad(e.to_string()))?;
let segmenter = Segmenter::new(&config);
let ctx_size = config.context_size();
Ok(Vad {
session,
config,
segmenter,
state: vec![0.0f32; STATE_LEN],
context: vec![0.0f32; ctx_size],
pending: Vec::new(),
last_probs: Vec::new(),
converter: None,
})
}
pub fn process(&mut self, samples: &[f32]) -> Vec<VadEvent> {
let frame_size = self.config.frame_size();
self.last_probs.clear();
let mut segments_out = Vec::new();
let mut events = Vec::new();
self.pending.extend_from_slice(samples);
let mut offset = 0;
let mut frame_buf = vec![0.0f32; frame_size];
while offset + frame_size <= self.pending.len() {
frame_buf.copy_from_slice(&self.pending[offset..offset + frame_size]);
let prob = self.infer_frame(&frame_buf).unwrap_or(0.0);
self.last_probs.push(prob);
self.segmenter.feed(prob, &mut segments_out);
offset += frame_size;
}
self.pending.drain(0..offset);
for seg in segments_out {
events.push(VadEvent::SpeechStart {
at_sample: seg.start_sample,
});
events.push(VadEvent::SpeechEnd {
at_sample: seg.end_sample,
});
}
events
}
pub fn process_pcm(
&mut self,
samples: &[f32],
input_sample_rate: u32,
input_channels: u16,
) -> Vec<VadEvent> {
let target = self.config.sample_rate;
let format = PcmFormat {
sample_rate: input_sample_rate,
channels: input_channels,
};
if let Some(c) = &self.converter {
if !c.matches(format) {
self.converter = None;
}
}
if input_sample_rate == target && input_channels <= 1 {
return self.process(samples);
}
if self.converter.is_none() {
match PcmConverter::new(format, target) {
Ok(c) => self.converter = Some(c),
Err(_) => return Vec::new(),
}
}
let mut converted = Vec::new();
{
let conv = self.converter.as_mut().expect("converter は直前に用意済み");
if conv.convert(samples, &mut converted).is_err() {
return Vec::new();
}
}
self.process(&converted)
}
fn infer_frame(&mut self, frame: &[f32]) -> Result<f32, VadError> {
let ctx_size = self.config.context_size();
let frame_size = self.config.frame_size();
debug_assert_eq!(frame.len(), frame_size);
debug_assert_eq!(self.context.len(), ctx_size);
let input_len = ctx_size + frame_size;
let mut x = Vec::with_capacity(input_len);
x.extend_from_slice(&self.context);
x.extend_from_slice(frame);
let next_context: Vec<f32> = x[input_len - ctx_size..].to_vec();
let input_tensor = Tensor::from_array(([1_i64, input_len as i64], x))
.map_err(|e| VadError::Inference(e.to_string()))?;
let state_tensor = Tensor::from_array(([2_i64, 1, 128], self.state.clone()))
.map_err(|e| VadError::Inference(e.to_string()))?;
let sr_tensor =
Tensor::from_array((vec![] as Vec<i64>, vec![self.config.sample_rate as i64]))
.map_err(|e| VadError::Inference(e.to_string()))?;
let outputs = self
.session
.run(ort::inputs![
"input" => input_tensor,
"state" => state_tensor,
"sr" => sr_tensor,
])
.map_err(|e| VadError::Inference(e.to_string()))?;
let (_pshape, prob_slice) = outputs["output"]
.try_extract_tensor::<f32>()
.map_err(|e| VadError::Inference(e.to_string()))?;
let prob = *prob_slice
.first()
.ok_or_else(|| VadError::Inference("empty output tensor".to_string()))?;
let (_sshape, state_slice) = outputs["stateN"]
.try_extract_tensor::<f32>()
.map_err(|e| VadError::Inference(e.to_string()))?;
if state_slice.len() != STATE_LEN {
return Err(VadError::Inference(format!(
"unexpected state length {} (expected {STATE_LEN})",
state_slice.len()
)));
}
let new_state = state_slice.to_vec();
drop(outputs);
self.state = new_state;
self.context = next_context;
Ok(prob)
}
pub fn last_frame_probabilities(&self) -> &[f32] {
&self.last_probs
}
pub fn reset(&mut self) {
self.state = vec![0.0f32; STATE_LEN];
self.context = vec![0.0f32; self.config.context_size()];
self.pending.clear();
self.last_probs.clear();
self.segmenter.reset();
self.converter = None;
}
pub fn config(&self) -> &VadConfig {
&self.config
}
}
pub fn get_speech_timestamps(
samples: &[f32],
config: &VadConfig,
) -> Result<Vec<Segment>, VadError> {
let mut vad = Vad::new(config.clone())?;
let frame_size = config.frame_size();
let mut out = Vec::new();
let mut chunks = samples.chunks_exact(frame_size);
for frame in chunks.by_ref() {
let prob = vad.infer_frame(frame)?;
vad.last_probs.push(prob);
vad.segmenter.feed(prob, &mut out);
}
vad.segmenter.flush(&mut out);
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::VadConfig;
use crate::segmenter::{Segment, Segmenter};
fn run_probs(config: &VadConfig, probs: &[f32]) -> Vec<Segment> {
let mut seg = Segmenter::new(config);
let mut out = Vec::new();
for &p in probs {
seg.feed(p, &mut out);
}
seg.flush(&mut out);
out
}
fn base_config() -> VadConfig {
VadConfig {
threshold: 0.5,
neg_threshold: Some(0.35),
min_speech_ms: 0, min_silence_ms: 0,
speech_pad_ms: 0,
max_speech_ms: 0,
sample_rate: 16000,
}
}
#[test]
fn neg_threshold_default_formula() {
let mut c = VadConfig::default();
assert_eq!(c.resolved_neg_threshold(), (0.5 - 0.15_f32).max(0.01));
c.threshold = 0.1;
assert_eq!(c.resolved_neg_threshold(), 0.01); c.neg_threshold = Some(0.2);
assert_eq!(c.resolved_neg_threshold(), 0.2); }
#[test]
fn simple_speech_then_silence() {
let mut c = base_config();
c.min_silence_ms = 32; let mut probs = vec![0.9f32; 20];
probs.extend(vec![0.1f32; 30]);
let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 1, "exactly one segment");
let s = segs[0];
assert_eq!(s.start_sample, 0);
assert_eq!(s.end_sample, 20 * 512);
}
#[test]
fn min_speech_discards_short_segment() {
let mut c = base_config();
c.min_silence_ms = 32; c.min_speech_ms = 250; let mut probs = vec![0.9f32; 5];
probs.extend(vec![0.1f32; 10]);
let segs = run_probs(&c, &probs);
assert!(
segs.is_empty(),
"short segment must be discarded, got {segs:?}"
);
let mut probs2 = vec![0.9f32; 10];
probs2.extend(vec![0.1f32; 10]);
let segs2 = run_probs(&c, &probs2);
assert_eq!(segs2.len(), 1);
assert_eq!(segs2[0].len_samples(), 10 * 512);
}
#[test]
fn min_silence_boundary_keeps_segment_together() {
let mut c = base_config();
c.min_silence_ms = 192; let mut probs = vec![0.9f32; 10];
probs.extend(vec![0.1f32; 3]);
probs.extend(vec![0.9f32; 10]);
probs.extend(vec![0.1f32; 10]); let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 1, "short gap must NOT split, got {segs:?}");
assert_eq!(segs[0].start_sample, 0);
assert_eq!(segs[0].end_sample, 23 * 512);
}
#[test]
fn min_silence_just_over_splits() {
let mut c = base_config();
c.min_silence_ms = 64; let mut probs = vec![0.9f32; 5];
probs.extend(vec![0.1f32; 3]);
probs.extend(vec![0.9f32; 5]);
probs.extend(vec![0.1f32; 5]);
let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 2, "long gap must split into two, got {segs:?}");
assert_eq!(segs[0].start_sample, 0);
assert_eq!(segs[0].end_sample, 5 * 512); }
#[test]
fn speech_pad_extends_and_clamps() {
let mut c = base_config();
c.min_silence_ms = 32; c.speech_pad_ms = 32; let mut probs = vec![0.1f32; 2]; probs.extend(vec![0.9f32; 4]); probs.extend(vec![0.1f32; 5]); let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 1);
assert_eq!(segs[0].start_sample, 1024 - 512);
assert_eq!(segs[0].end_sample, 6 * 512 + 512);
}
#[test]
fn speech_pad_start_clamps_at_zero() {
let mut c = base_config();
c.min_silence_ms = 32;
c.speech_pad_ms = 64; let mut probs = vec![0.9f32; 5];
probs.extend(vec![0.1f32; 5]);
let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 1);
assert_eq!(segs[0].start_sample, 0, "start pad must clamp to 0");
}
#[test]
fn pad_does_not_overlap_previous_segment() {
let mut c = base_config();
c.min_silence_ms = 64; c.speech_pad_ms = 192; let mut probs = vec![0.9f32; 5];
probs.extend(vec![0.1f32; 3]);
probs.extend(vec![0.9f32; 5]);
probs.extend(vec![0.1f32; 5]);
let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 2);
assert!(
segs[1].start_sample >= segs[0].end_sample,
"seg1.start ({}) must be >= seg0.end ({})",
segs[1].start_sample,
segs[0].end_sample
);
}
#[test]
fn max_speech_forces_split_when_no_silence() {
let mut c = base_config();
c.max_speech_ms = 192; c.min_silence_ms = 32;
let probs = vec![0.9f32; 20];
let segs = run_probs(&c, &probs);
assert!(
segs.len() >= 2,
"max_speech must force at least one split, got {segs:?}"
);
for s in &segs {
assert!(
s.len_samples() <= c.ms_to_samples(c.max_speech_ms) + 512,
"segment {s:?} exceeds max_speech by more than one frame"
);
}
}
#[test]
fn gray_zone_keeps_triggered() {
let mut c = base_config(); c.min_silence_ms = 32;
let mut probs = vec![0.9f32; 5];
probs.extend(vec![0.4f32; 5]); probs.extend(vec![0.9f32; 5]);
probs.extend(vec![0.1f32; 5]); let segs = run_probs(&c, &probs);
assert_eq!(segs.len(), 1, "gray zone must not split, got {segs:?}");
assert_eq!(segs[0].end_sample, 15 * 512);
}
#[test]
fn vad_loads_model() {
let vad = Vad::new(VadConfig::default());
assert!(vad.is_ok(), "model load failed: {:?}", vad.err());
}
#[test]
fn zeros_produce_low_prob_no_speech() {
let mut vad = Vad::new(VadConfig::default()).unwrap();
let zeros = vec![0.0f32; 16000];
let events = vad.process(&zeros);
assert!(
!events
.iter()
.any(|e| matches!(e, VadEvent::SpeechStart { .. })),
"silence must not trigger SpeechStart, got {events:?}"
);
let probs = vad.last_frame_probabilities();
assert_eq!(probs.len(), 16000 / 512, "expected one prob per frame");
for &p in probs {
assert!((0.0..=1.0).contains(&p), "prob {p} out of [0,1]");
assert!(p < 0.5, "silence prob {p} unexpectedly high");
}
}
#[test]
fn process_streams_across_calls() {
let mut vad = Vad::new(VadConfig::default()).unwrap();
let total = 2048usize;
let chunk = vec![0.0f32; 300];
let mut fed = 0usize;
let mut total_probs = 0usize;
while fed < total {
let take = chunk.len().min(total - fed);
vad.process(&chunk[..take]);
total_probs += vad.last_frame_probabilities().len();
fed += take;
}
assert_eq!(
total_probs,
total / 512,
"all complete frames must be inferred"
);
}
#[test]
fn reset_clears_state() {
let mut vad = Vad::new(VadConfig::default()).unwrap();
vad.process(&vec![0.0f32; 1000]); vad.reset();
assert_eq!(vad.last_frame_probabilities().len(), 0);
assert_eq!(vad.config().sample_rate, 16000);
}
#[test]
fn invalid_sample_rate_rejected() {
let c = VadConfig {
sample_rate: 44100,
..VadConfig::default()
};
let r = Vad::new(c);
assert!(matches!(r, Err(VadError::InvalidConfig(_))));
}
#[test]
fn batch_get_speech_timestamps_on_silence() {
let zeros = vec![0.0f32; 16000];
let segs = get_speech_timestamps(&zeros, &VadConfig::default()).unwrap();
assert!(segs.is_empty(), "silence yields no segments, got {segs:?}");
}
fn harmonics(rate: usize, n: usize) -> Vec<f32> {
(0..n)
.map(|i| {
let t = i as f32 / rate as f32;
let mut v = 0.0f32;
for (k, f) in [180.0f32, 540.0, 1200.0, 2600.0].iter().enumerate() {
v += (0.4 / (k as f32 + 1.0)) * (2.0 * std::f32::consts::PI * f * t).sin();
}
v * 0.5
})
.collect()
}
fn to_stereo(mono: &[f32]) -> Vec<f32> {
let mut s = Vec::with_capacity(mono.len() * 2);
for &v in mono {
s.push(v);
s.push(v);
}
s
}
#[test]
fn process_pcm_passthrough_matches_process() {
let sig = harmonics(16_000, 16_000);
let mut vad = Vad::new(VadConfig::default()).unwrap();
let e_pcm = vad.process_pcm(&sig, 16_000, 1);
let p_pcm = vad.last_frame_probabilities().to_vec();
vad.reset();
let e_proc = vad.process(&sig);
let p_proc = vad.last_frame_probabilities().to_vec();
assert_eq!(
e_pcm, e_proc,
"passthrough のイベント列が process と一致しない"
);
assert_eq!(
p_pcm, p_proc,
"passthrough の確率列が process とビット一致しない"
);
assert_eq!(p_pcm.len(), 16_000 / 512, "フレーム数が想定通りでない");
}
#[test]
fn process_pcm_split_matches_bulk() {
let cfg = VadConfig {
threshold: 0.0,
neg_threshold: Some(0.0),
min_speech_ms: 0,
min_silence_ms: 0,
speech_pad_ms: 0,
max_speech_ms: 200, sample_rate: 16_000,
};
let stereo = to_stereo(&harmonics(48_000, 48_000));
let mut vad = Vad::new(cfg).unwrap();
let bulk_events = vad.process_pcm(&stereo, 48_000, 2);
let bulk_probs = vad.last_frame_probabilities().to_vec();
vad.reset();
let mut split_events = Vec::new();
let mut split_probs = Vec::new();
for chunk in stereo.chunks(777) {
let evs = vad.process_pcm(chunk, 48_000, 2);
split_events.extend(evs);
split_probs.extend_from_slice(vad.last_frame_probabilities());
}
assert_eq!(
bulk_probs, split_probs,
"分割と一括で確率列が一致しない(継ぎ目が出ている)"
);
assert_eq!(
bulk_events, split_events,
"分割と一括でイベント列が一致しない"
);
assert!(
!bulk_events.is_empty(),
"この設定では発話イベントが出るはず(テストの実効性確認)"
);
}
#[test]
fn process_pcm_reset_is_deterministic() {
let stereo = to_stereo(&harmonics(48_000, 24_000));
let mut vad = Vad::new(VadConfig::default()).unwrap();
let e1 = vad.process_pcm(&stereo, 48_000, 2);
let p1 = vad.last_frame_probabilities().to_vec();
vad.reset();
let e2 = vad.process_pcm(&stereo, 48_000, 2);
let p2 = vad.last_frame_probabilities().to_vec();
assert_eq!(e1, e2, "reset 後に同一入力で同一イベントにならない");
assert_eq!(p1, p2, "reset 後に同一入力で同一確率にならない");
}
#[test]
fn process_pcm_48k_stereo_matches_direct_16k() {
let n16 = 16_000usize;
let ref16 = harmonics(16_000, n16);
let stereo48 = to_stereo(&harmonics(48_000, n16 * 3));
let mut ref_vad = Vad::new(VadConfig::default()).unwrap();
let ref_events = ref_vad.process(&ref16);
let ref_probs = ref_vad.last_frame_probabilities().to_vec();
let mut pcm_vad = Vad::new(VadConfig::default()).unwrap();
let pcm_events = pcm_vad.process_pcm(&stereo48, 48_000, 2);
let pcm_probs = pcm_vad.last_frame_probabilities().to_vec();
let n = ref_probs.len().min(pcm_probs.len());
assert!(n >= 8, "十分なフレーム数が必要: {n}");
for k in 2..(n - 2) {
let d = (ref_probs[k] - pcm_probs[k]).abs();
assert!(
d < 0.05,
"frame {k}: 直接16kと変換経路の確率差が大きい: {d} (ref={} pcm={})",
ref_probs[k],
pcm_probs[k]
);
}
assert_eq!(
ref_events, pcm_events,
"変換経路のイベント列が直接16kと一致しない"
);
}
}