use std::path::PathBuf;
use anyhow::Result;
use crate::{utils::VAD_SAMPLE_RATE, vad_iter, SpeechSegment, VadParams};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[non_exhaustive]
pub enum Model {
Silero,
PyAnnote,
#[default]
PulseVad,
Fsmn,
Ten,
MarbleNet,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[non_exhaustive]
pub enum Runtime {
#[default]
Candle,
OnnxRuntime,
}
pub struct DetectorBuilder {
model_path: PathBuf,
model: Model,
runtime: Runtime,
params: VadParams,
#[cfg(feature = "candle")]
candle_device: candle_core::Device,
#[cfg(feature = "onnxruntime")]
execution_providers: Vec<ort::ep::ExecutionProviderDispatch>,
}
impl DetectorBuilder {
#[must_use]
pub fn new(model_path: impl Into<PathBuf>) -> Self {
Self {
model_path: model_path.into(),
model: Model::default(),
runtime: Runtime::default(),
params: VadParams::default(),
#[cfg(feature = "candle")]
candle_device: candle_core::Device::Cpu,
#[cfg(feature = "onnxruntime")]
execution_providers: vec![ort::ep::CPU::default().build()],
}
}
#[must_use]
pub fn model(mut self, model: Model) -> Self {
self.model = model;
self
}
#[must_use]
pub fn runtime(mut self, runtime: Runtime) -> Self {
self.runtime = runtime;
self
}
#[must_use]
pub fn parameters(mut self, params: VadParams) -> Self {
self.params = params;
self
}
#[cfg(feature = "candle")]
#[must_use]
pub fn candle_device(mut self, device: candle_core::Device) -> Self {
self.candle_device = device;
self
}
#[cfg(feature = "onnxruntime")]
#[must_use]
pub fn execution_providers(
mut self,
execution_providers: Vec<ort::ep::ExecutionProviderDispatch>,
) -> Self {
self.execution_providers = execution_providers;
self
}
pub fn build(mut self) -> Result<Detector> {
if self.model == Model::Ten {
self.params.frame_size = ten_frame_size_ms();
}
validate_model_parameters(self.model, &self.params)?;
match self.runtime {
Runtime::Candle => self.build_candle(),
Runtime::OnnxRuntime => self.build_onnxruntime(),
}
}
#[cfg(feature = "candle")]
fn build_candle(self) -> Result<Detector> {
let backend = match self.model {
Model::Silero => Backend::SileroCandle(crate::vad_iter::VadIter::new(
crate::models::silero::candle::Silero::new(
self.params.clone(),
self.model_path,
self.candle_device,
)?,
self.params,
)),
Model::PulseVad => Backend::PulseVadCandle(crate::pulsevad_iter::PulseVadIter::new(
crate::pulsevad::PulseVad::new(
self.model_path,
self.candle_device,
self.params.debug,
)?,
self.params,
)),
Model::PyAnnote => {
Backend::PyAnnoteCandle(crate::pyannote_vad_iter::PyAnnoteVadIterator::new(
crate::pyannote_vad::PyAnnote::new(
self.params.clone(),
self.model_path,
self.candle_device,
)?,
self.params,
))
}
Model::Fsmn => Backend::FsmnCandle(crate::fsmn_vad_iter::FsmnVadIterator::new(
crate::fsmn_vad::FsmnVad::new(
self.model_path,
self.candle_device,
self.params.debug,
)?,
self.params,
)),
Model::MarbleNet => {
Backend::MarbleNetCandle(crate::marblenet_iter::MarbleNetIter::new(
crate::marblenet::MarbleNet::new(
self.model_path,
self.candle_device,
self.params.debug,
)?,
self.params,
))
}
Model::Ten => Backend::TenCandle(Box::new(crate::vad_iter::VadIter::new(
crate::ten_vad::TenVad::new(
self.model_path,
self.candle_device,
self.params.debug,
)?,
self.params,
))),
};
Ok(Detector { backend })
}
#[cfg(not(feature = "candle"))]
fn build_candle(self) -> Result<Detector> {
anyhow::bail!("Candle support is disabled; enable the `candle` Cargo feature")
}
#[cfg(feature = "onnxruntime")]
fn build_onnxruntime(self) -> Result<Detector> {
let backend = match self.model {
Model::Silero => Backend::SileroOnnx(Box::new(crate::vad_iter::VadIter::new(
crate::models::silero::onnx::Silero::new(
self.params.clone(),
self.execution_providers,
self.model_path,
)?,
self.params,
))),
Model::PyAnnote => {
Backend::PyAnnoteOnnx(crate::pyannote_vad_iter::PyAnnoteVadIterator::new(
crate::pyannote_vad_ort::PyAnnote::new(
self.params.clone(),
self.execution_providers,
self.model_path,
)?,
self.params,
))
}
Model::PulseVad => Backend::PulseVadOnnx(crate::pulsevad_iter::PulseVadIter::new(
crate::pulsevad_ort::PulseVad::new(
self.execution_providers,
self.model_path,
self.params.debug,
)?,
self.params,
)),
Model::Fsmn => Backend::FsmnOnnx(crate::fsmn_vad_iter::FsmnVadIterator::new(
crate::fsmn_vad_ort::FsmnVad::new(
self.execution_providers,
self.model_path,
self.params.debug,
)?,
self.params,
)),
Model::Ten => Backend::TenOnnx(Box::new(crate::vad_iter::VadIter::new(
crate::ten_vad_ort::TenVad::new(self.model_path, self.params.debug)?,
self.params,
))),
Model::MarbleNet => Backend::MarbleNetOnnx(crate::marblenet_iter::MarbleNetIter::new(
crate::marblenet_ort::MarbleNet::new(
self.execution_providers,
self.model_path,
self.params.debug,
)?,
self.params,
)),
};
Ok(Detector { backend })
}
#[cfg(not(feature = "onnxruntime"))]
fn build_onnxruntime(self) -> Result<Detector> {
anyhow::bail!("ONNX Runtime support is disabled; enable the `onnxruntime` Cargo feature")
}
}
pub struct Detector {
backend: Backend,
}
impl Detector {
#[must_use]
pub fn builder(model_path: impl Into<PathBuf>) -> DetectorBuilder {
DetectorBuilder::new(model_path)
}
pub fn detect(&mut self, samples: &[f32]) -> Result<Vec<SpeechSegment>> {
let segments: &[SpeechSegment] = match &mut self.backend {
#[cfg(feature = "candle")]
Backend::SileroCandle(iter) => iter.process(samples)?,
#[cfg(feature = "candle")]
Backend::PulseVadCandle(iter) => iter.process(samples)?,
#[cfg(feature = "candle")]
Backend::PyAnnoteCandle(iter) => iter.process(samples)?,
#[cfg(feature = "candle")]
Backend::FsmnCandle(iter) => iter.process(samples)?,
#[cfg(feature = "candle")]
Backend::TenCandle(iter) => iter.process(samples)?,
#[cfg(feature = "candle")]
Backend::MarbleNetCandle(iter) => iter.process(samples)?,
#[cfg(feature = "onnxruntime")]
Backend::SileroOnnx(iter) => iter.process(samples)?,
#[cfg(feature = "onnxruntime")]
Backend::PyAnnoteOnnx(iter) => iter.process(samples)?,
#[cfg(feature = "onnxruntime")]
Backend::PulseVadOnnx(iter) => iter.process(samples)?,
#[cfg(feature = "onnxruntime")]
Backend::FsmnOnnx(iter) => iter.process(samples)?,
#[cfg(feature = "onnxruntime")]
Backend::TenOnnx(iter) => iter.process(samples)?,
#[cfg(feature = "onnxruntime")]
Backend::MarbleNetOnnx(iter) => iter.process(samples)?,
#[cfg(not(any(feature = "candle", feature = "onnxruntime")))]
Backend::Disabled => {
anyhow::bail!("no inference runtime is enabled; enable `candle` or `onnxruntime`")
}
};
Ok(segments.to_vec())
}
}
#[allow(clippy::enum_variant_names)]
enum Backend {
#[cfg(feature = "candle")]
SileroCandle(crate::vad_iter::VadIter<crate::models::silero::candle::Silero>),
#[cfg(feature = "candle")]
PulseVadCandle(crate::pulsevad_iter::PulseVadIter<crate::pulsevad::PulseVad>),
#[cfg(feature = "candle")]
PyAnnoteCandle(crate::pyannote_vad_iter::PyAnnoteVadIterator<crate::pyannote_vad::PyAnnote>),
#[cfg(feature = "candle")]
FsmnCandle(crate::fsmn_vad_iter::FsmnVadIterator<crate::fsmn_vad::FsmnVad>),
#[cfg(feature = "candle")]
TenCandle(Box<crate::vad_iter::VadIter<crate::ten_vad::TenVad>>),
#[cfg(feature = "candle")]
MarbleNetCandle(crate::marblenet_iter::MarbleNetIter<crate::marblenet::MarbleNet>),
#[cfg(feature = "onnxruntime")]
SileroOnnx(Box<crate::vad_iter::VadIter<crate::models::silero::onnx::Silero>>),
#[cfg(feature = "onnxruntime")]
PyAnnoteOnnx(crate::pyannote_vad_iter::PyAnnoteVadIterator<crate::pyannote_vad_ort::PyAnnote>),
#[cfg(feature = "onnxruntime")]
PulseVadOnnx(crate::pulsevad_iter::PulseVadIter<crate::pulsevad_ort::PulseVad>),
#[cfg(feature = "onnxruntime")]
FsmnOnnx(crate::fsmn_vad_iter::FsmnVadIterator<crate::fsmn_vad_ort::FsmnVad>),
#[cfg(feature = "onnxruntime")]
TenOnnx(Box<crate::vad_iter::VadIter<crate::ten_vad_ort::TenVad>>),
#[cfg(feature = "onnxruntime")]
MarbleNetOnnx(crate::marblenet_iter::MarbleNetIter<crate::marblenet_ort::MarbleNet>),
#[cfg(not(any(feature = "candle", feature = "onnxruntime")))]
Disabled,
}
fn validate_model_parameters(model: Model, params: &VadParams) -> Result<()> {
vad_iter::validate_parameters(params)?;
if model == Model::Silero {
let frame_samples = vad_iter::milliseconds_to_samples(
params.frame_size,
params.sample_rate,
"Silero frame size",
)?;
let context_samples = if params.sample_rate == VAD_SAMPLE_RATE {
64
} else {
32
};
anyhow::ensure!(
frame_samples >= context_samples,
"Silero frame must contain at least {context_samples} samples"
);
}
if model == Model::PulseVad && params.max_speech_duration_s.is_finite() {
let total = vad_iter::seconds_to_samples(params.max_speech_duration_s, params.sample_rate)?;
let padding = vad_iter::milliseconds_to_samples(
params.speech_pad_ms,
params.sample_rate,
"speech padding",
)?
.checked_mul(2)
.ok_or_else(|| anyhow::anyhow!("speech padding exceeds usize"))?;
let unpadded = total.checked_sub(padding).ok_or_else(|| {
anyhow::anyhow!("maximum speech duration is shorter than its padding")
})?;
anyhow::ensure!(
unpadded >= crate::pulsevad_frontend::WINDOW_SAMPLES,
"PulseVAD maximum speech duration must contain at least one model window"
);
}
Ok(())
}
#[cfg(feature = "onnxruntime")]
pub fn init_onnx_runtime(
dynamic_library_path: impl AsRef<std::path::Path>,
execution_providers: Vec<ort::ep::ExecutionProviderDispatch>,
) -> Result<()> {
let path = dynamic_library_path
.as_ref()
.to_str()
.ok_or_else(|| anyhow::anyhow!("ONNX Runtime library path is not valid UTF-8"))?;
ort::init_from(path)?
.with_execution_providers(execution_providers)
.commit();
Ok(())
}
const fn ten_frame_size_ms() -> usize {
crate::models::ten::frontend::FRAME_SAMPLES * 1_000 / VAD_SAMPLE_RATE
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn pulsevad_is_the_default_model() {
assert_eq!(Model::default(), Model::PulseVad);
}
#[test]
fn rejects_non_vad_sample_rate() {
let params = VadParams {
sample_rate: 8_000,
..VadParams::default()
};
assert!(vad_iter::validate_parameters(¶ms).is_err());
}
#[test]
fn rejects_invalid_threshold() {
let params = VadParams {
threshold: f32::NAN,
..VadParams::default()
};
assert!(vad_iter::validate_parameters(¶ms).is_err());
}
#[test]
fn rejects_silero_frames_shorter_than_the_recurrent_context() {
let params = VadParams {
frame_size: 1,
..VadParams::default()
};
assert!(validate_model_parameters(Model::Silero, ¶ms).is_err());
}
}