extract-speech 0.9.1

Voice activity detection inference for Silero, PulseVAD, PyAnnote, FSMN, TEN VAD, and MarbleNet
Documentation
//! FSMN-VAD feature extraction.

use std::{fs, path::Path};

use anyhow::{Context, Result};
use kaldi_native_fbank::{
    fbank::{FbankComputer, FbankOptions},
    online::{FeatureComputer, OnlineFeature},
};

pub const FEATURE_DIM: usize = 400;
const N_MELS: usize = 80;
const LFR_M: usize = 5;
const SAMPLE_RATE: f32 = 16_000.0;

pub struct FsmnVadFrontend {
    means: Vec<f32>,
    scales: Vec<f32>,
}

impl FsmnVadFrontend {
    /// Loads the CMVN sidecar data located next to an FSMN model.
    ///
    /// # Errors
    ///
    /// Returns an error if the sidecar is missing, unreadable, or malformed.
    pub fn from_model_path(model_path: &Path) -> Result<Self> {
        let model_dir = model_path.parent().unwrap_or_else(|| Path::new("."));
        let cmvn_path = ["vad.mvn", "am.mvn"]
            .into_iter()
            .map(|name| model_dir.join(name))
            .find(|path| path.is_file())
            .with_context(|| {
                format!(
                    "FSMN-VAD requires vad.mvn (or am.mvn) next to {}",
                    model_path.display()
                )
            })?;
        let contents = fs::read_to_string(&cmvn_path)
            .with_context(|| format!("failed to read CMVN file {}", cmvn_path.display()))?;
        let means = parse_coefficients(&contents, "<AddShift>")?;
        let scales = parse_coefficients(&contents, "<Rescale>")?;
        anyhow::ensure!(
            means.len() == FEATURE_DIM && scales.len() == FEATURE_DIM,
            "FSMN-VAD CMVN file {} contains {} means and {} scales; expected {FEATURE_DIM} of each",
            cmvn_path.display(),
            means.len(),
            scales.len()
        );

        Ok(Self { means, scales })
    }

    /// Extracts normalized low-frame-rate filterbank features.
    ///
    /// # Errors
    ///
    /// Returns an error if the filterbank cannot be initialized or evaluated.
    pub fn extract(&self, waveform: &[f32]) -> Result<Vec<f32>> {
        let mut options = FbankOptions::default();
        options.frame_opts.samp_freq = SAMPLE_RATE;
        options.frame_opts.dither = 0.0;
        "hamming".clone_into(&mut options.frame_opts.window_type);
        options.frame_opts.frame_shift_ms = 10.0;
        options.frame_opts.frame_length_ms = 25.0;
        options.frame_opts.snip_edges = true;
        options.mel_opts.num_bins = N_MELS;
        options.mel_opts.debug_mel = false;
        options.use_energy = false;
        options.energy_floor = 0.0;

        let computer = FbankComputer::new(options).map_err(anyhow::Error::msg)?;
        let mut fbank = OnlineFeature::new(FeatureComputer::Fbank(computer));
        let scaled_waveform: Vec<f32> = waveform.iter().map(|sample| sample * 32_768.0).collect();
        fbank.accept_waveform(SAMPLE_RATE, &scaled_waveform);

        if fbank.features.is_empty() {
            return Ok(Vec::new());
        }

        let mut features = apply_lfr(&fbank.features);
        for frame in features.as_chunks_mut::<FEATURE_DIM>().0 {
            for (index, value) in frame.iter_mut().enumerate() {
                *value = (*value + self.means[index]) * self.scales[index];
            }
        }
        Ok(features)
    }
}

fn parse_coefficients(contents: &str, marker: &str) -> Result<Vec<f32>> {
    let mut lines = contents.lines();
    while let Some(line) = lines.next() {
        if line.split_whitespace().next() != Some(marker) {
            continue;
        }

        let coefficients = lines
            .next()
            .with_context(|| format!("CMVN marker {marker} has no coefficient line"))?;
        let mut values = Vec::new();
        let mut inside_brackets = false;
        for token in coefficients.split_whitespace() {
            if token == "[" {
                inside_brackets = true;
            } else if token == "]" {
                break;
            } else if inside_brackets {
                values.push(token.parse::<f32>().with_context(|| {
                    format!("invalid CMVN coefficient {token:?} after {marker}")
                })?);
            }
        }
        anyhow::ensure!(
            !values.is_empty(),
            "CMVN marker {marker} has no coefficients"
        );
        return Ok(values);
    }

    Err(anyhow::anyhow!("CMVN file does not contain {marker}"))
}

fn apply_lfr(inputs: &[Vec<f32>]) -> Vec<f32> {
    let frame_count = inputs.len();
    let mut output = Vec::with_capacity(frame_count * FEATURE_DIM);

    for output_frame in 0..frame_count {
        for lfr_index in 0..LFR_M {
            let source_index = output_frame
                .saturating_add(lfr_index)
                .saturating_sub((LFR_M - 1) / 2)
                .min(frame_count - 1);
            output.extend_from_slice(&inputs[source_index]);
        }
    }
    output
}

#[cfg(test)]
#[allow(clippy::float_cmp)] // These fixtures contain exactly representable values.
mod tests {
    use super::*;

    #[test]
    fn parses_kaldi_cmvn_coefficients() {
        let cmvn = "<AddShift> 2 2\n<LearnRateCoef> 0 [ -1.5 2.25 ]\n<Rescale> 2 2\n<LearnRateCoef> 0 [ 0.5 4 ]\n";

        assert_eq!(
            parse_coefficients(cmvn, "<AddShift>").unwrap(),
            [-1.5, 2.25]
        );
        assert_eq!(parse_coefficients(cmvn, "<Rescale>").unwrap(), [0.5, 4.0]);
    }

    #[test]
    fn lfr_repeats_edge_frames() {
        let inputs = vec![vec![1.0; N_MELS], vec![2.0; N_MELS], vec![3.0; N_MELS]];
        let output = apply_lfr(&inputs);

        assert_eq!(output.len(), inputs.len() * FEATURE_DIM);
        assert_eq!(output[0], 1.0);
        assert_eq!(output[N_MELS], 1.0);
        assert_eq!(output[2 * N_MELS], 1.0);
        assert_eq!(output[3 * N_MELS], 2.0);
        assert_eq!(output[4 * N_MELS], 3.0);
        assert_eq!(output[FEATURE_DIM + 4 * N_MELS], 3.0);
    }
}