bunsen 0.41.0

bunsen is a batteries included common library for burn
Documentation
//! Load pretrained models.
use std::path::Path;

use burn::{
    store::{
        BurnpackStore,
        KeyRemapper,
        ModuleSnapshot,
        burn_pack::Error as BurnpackError,
    },
    tensor::Device,
};

use crate::{
    blocks::{
        conv::ConvSeq1d,
        rnn::lstm::FusedLstm,
    },
    burner::module::{
        ModuleInit,
        Rekey,
    },
    errors::{
        BunsenError,
        BunsenErrorKind,
        BunsenResult,
        ResultContext,
        WithOkOrPanic,
        sys_at,
    },
    kits::speech::silero_vad::{
        SileroVad,
        SileroVadCollection,
        SileroVadSignalConfig,
    },
};

impl SileroVad {
    /// Load the 16khz model from pretrained burnpack bytes.
    /// Uses the upstream `silero_vad` keying.
    pub fn load_16khz_from_burnpack_bytes(
        bytes: burn::tensor::Bytes,
        device: &Device,
    ) -> BunsenResult<Self> {
        Self::load_from_burnpack(
            BurnpackStore::from_bytes(Some(bytes)),
            SileroVadSignalConfig::standard_16khz(),
            Self::pretrained_16khz_remapper(),
            device,
        )
    }

    /// Load the 16khz model from a burnpack file.
    /// Uses the upstream `silero_vad` keying.
    ///
    /// # Errors
    /// [`Lookup`](crate::errors::BunsenErrorKind::Lookup) when `path` is
    /// missing or forbidden; otherwise as
    /// [`load_from_burnpack`](Self::load_from_burnpack), under a frame
    /// naming the file.
    pub fn load_16khz_from_burnpack_file(
        path: impl AsRef<Path>,
        device: &Device,
    ) -> BunsenResult<Self> {
        Self::load_from_burnpack_path(
            path,
            SileroVadSignalConfig::standard_16khz(),
            Self::pretrained_16khz_remapper(),
            device,
        )
    }

    /// Load the 8khz model from pretrained burnpack bytes.
    /// Uses the upstream `silero_vad` keying.
    pub fn load_8khz_from_burnpack_bytes(
        bytes: burn::tensor::Bytes,
        device: &Device,
    ) -> BunsenResult<Self> {
        Self::load_from_burnpack(
            BurnpackStore::from_bytes(Some(bytes)),
            SileroVadSignalConfig::standard_8khz(),
            Self::pretrained_8khz_remapper(),
            device,
        )
    }

    /// Load the 8khz model from a burnpack file.
    /// Uses the upstream `silero_vad` keying.
    ///
    /// # Errors
    /// [`Lookup`](crate::errors::BunsenErrorKind::Lookup) when `path` is
    /// missing or forbidden; otherwise as
    /// [`load_from_burnpack`](Self::load_from_burnpack), under a frame
    /// naming the file.
    pub fn load_8khz_from_burnpack_file(
        path: impl AsRef<Path>,
        device: &Device,
    ) -> BunsenResult<Self> {
        Self::load_from_burnpack_path(
            path,
            SileroVadSignalConfig::standard_8khz(),
            Self::pretrained_8khz_remapper(),
            device,
        )
    }

    /// The key remapping for the 16khz model.
    pub fn pretrained_16khz_remapper() -> KeyRemapper {
        KeyRemapper::from_patterns(vec![
            ("conv1d37", "stft".to_string()),
            ("conv1d38", ConvSeq1d::block_conv_path("encoder", 0)),
            ("conv1d39", ConvSeq1d::block_conv_path("encoder", 1)),
            ("conv1d40", ConvSeq1d::block_conv_path("encoder", 2)),
            ("conv1d41", ConvSeq1d::block_conv_path("encoder", 3)),
            ("linear13", "lstm.hidden".to_string()),
            ("linear14", "lstm.features".to_string()),
            ("conv1d42", "decoder".to_string()),
        ])
        .ok_or_panic()
    }

    /// The key remapping for the 8khz model.
    pub fn pretrained_8khz_remapper() -> KeyRemapper {
        KeyRemapper::from_patterns(vec![
            ("conv1d43", "stft".to_string()),
            ("conv1d44", ConvSeq1d::block_conv_path("encoder", 0)),
            ("conv1d45", ConvSeq1d::block_conv_path("encoder", 1)),
            ("conv1d46", ConvSeq1d::block_conv_path("encoder", 2)),
            ("conv1d47", ConvSeq1d::block_conv_path("encoder", 3)),
            ("linear15", "lstm.hidden".to_string()),
            ("linear16", "lstm.features".to_string()),
            ("conv1d48", "decoder".to_string()),
        ])
        .ok_or_panic()
    }

    /// Load from a burnpack file, under a frame naming it.
    fn load_from_burnpack_path<C>(
        path: impl AsRef<Path>,
        cfg: C,
        remapper: KeyRemapper,
        device: &Device,
    ) -> BunsenResult<Self>
    where
        C: ModuleInit<Self>,
    {
        let path = path.as_ref();
        std::fs::metadata(path).map_err(sys_at("open", path))?;
        Self::load_from_burnpack(BurnpackStore::from_file(path), cfg, remapper, device)
            .with_context(|| format!("loading {}", path.display()))
    }

    /// Load from a burnpack store.
    ///
    /// # Errors
    /// - as `cfg`'s [`try_init`](ModuleInit::try_init);
    /// - [`InvalidResource`](crate::errors::BunsenErrorKind::InvalidResource),
    ///   with the [`BurnpackError`] as its cause, when the store is not a
    ///   burnpack or lacks a tensor the model needs;
    ///   [`Unsupported`](crate::errors::BunsenErrorKind::Unsupported) for a
    ///   burnpack version this build does not read;
    ///   [`Sys`](crate::errors::BunsenErrorKind::Sys) when reading it fails.
    pub fn load_from_burnpack<C>(
        store: BurnpackStore,
        cfg: C,
        remapper: KeyRemapper,
        device: &Device,
    ) -> BunsenResult<Self>
    where
        C: ModuleInit<Self>,
    {
        let mut store = store.remap(remapper);
        let mut module = cfg.try_init(device)?;
        module.load_from(&mut store).map_err(burnpack_load_error)?;

        Ok(module)
    }
}

/// Sorts a [`BurnpackError`] from loading a module by what it means.
fn burnpack_load_error(error: BurnpackError) -> BunsenError {
    let kind = match &error {
        BurnpackError::IoError(_) => BunsenErrorKind::Sys,
        BurnpackError::InvalidVersion => BunsenErrorKind::Unsupported,
        _ => BunsenErrorKind::InvalidResource,
    };
    BunsenError::from_cause(kind, error)
}

impl SileroVadCollection {
    pub(super) fn new_common_collection(
        vad_16: SileroVad,
        vad_8: SileroVad,
    ) -> Self {
        Self {
            branches: vec![(16000, vad_16), (8000, vad_8)],
        }
    }

    /// Load the standard 16khz/8khz pretrained models from burnpack bytes.
    /// Uses the upstream `silero_vad` keying.
    pub fn load_from_burnpack_bytes(
        bytes: burn::tensor::Bytes,
        device: &Device,
    ) -> BunsenResult<Self> {
        Ok(Self::new_common_collection(
            SileroVad::load_16khz_from_burnpack_bytes(bytes.clone(), device)?,
            SileroVad::load_8khz_from_burnpack_bytes(bytes, device)?,
        ))
    }

    /// Load the standard 16khz/8khz pretrained models from a burnpack file.
    /// Uses the upstream `silero_vad` keying.
    pub fn load_from_burnpack_file<P: AsRef<Path>>(
        path: P,
        device: &Device,
    ) -> BunsenResult<Self> {
        let path = path.as_ref();
        Ok(Self::new_common_collection(
            SileroVad::load_16khz_from_burnpack_file(path, device)?,
            SileroVad::load_8khz_from_burnpack_file(path, device)?,
        ))
    }
}

/// Upstream's `silero_vad_16k.safetensors` names (the 16 kHz model's
/// `PyTorch` state dict) onto a [`SileroVad`]'s module paths; anything else
/// is dropped.
///
/// | upstream | bunsen |
/// |---|---|
/// | `stft_conv` | `stft` |
/// | `conv1` … `conv4` | `encoder.blocks.0` … `3.conv`, by [`ConvSeq1d::one_based_rekey`] |
/// | `lstm_cell` (an `nn.LSTMCell`) | `lstm`, by [`FusedLstm::torch_cell_rekey`] |
/// | `final_conv` | `decoder` |
pub fn upstream_rekey() -> Rekey {
    Rekey::drop_rest()
        .rename("stft_conv.", "stft.")
        .rename("final_conv.", "decoder.")
        .mount("lstm_cell.", "lstm.", FusedLstm::torch_cell_rekey())
        .mount("conv", "encoder.", ConvSeq1d::one_based_rekey())
}

#[cfg(feature = "store_safetensors")]
impl SileroVad {
    /// Load the 16khz model from tensors named as upstream's
    /// `silero_vad_16k.safetensors` names them ([`upstream_rekey`]).
    ///
    /// # Errors
    /// As [`apply_rekeyed_tensors`](crate::data::pretrained::apply_rekeyed_tensors):
    /// [`InvalidResource`](crate::errors::BunsenErrorKind::InvalidResource)
    /// when a tensor does not fit, or a parameter is missing.
    pub fn load_16khz_from_upstream_tensors(
        tensors: impl IntoIterator<Item = burn::store::burn_pack::Tensor>,
        device: &Device,
    ) -> BunsenResult<Self> {
        let mut vad: Self = SileroVadSignalConfig::standard_16khz().try_init(device)?;
        crate::data::pretrained::apply_rekeyed_tensors(&mut vad, tensors, &upstream_rekey())?;
        Ok(vad)
    }

    /// Load the 16khz model from upstream's `silero_vad_16k.safetensors`.
    ///
    /// # Errors
    /// [`InvalidResource`](crate::errors::BunsenErrorKind::InvalidResource)
    /// when the file is not a safetensors file; otherwise as
    /// [`load_16khz_from_upstream_tensors`](Self::load_16khz_from_upstream_tensors),
    /// under a frame naming the file.
    pub fn load_16khz_from_safetensors_file(
        path: impl AsRef<Path>,
        device: &Device,
    ) -> BunsenResult<Self> {
        use burn::store::{
            ModuleStore,
            SafetensorsStore,
        };

        let path = path.as_ref();
        let read = || -> BunsenResult<Self> {
            std::fs::metadata(path).map_err(sys_at("open", path))?;
            let tensors: Vec<_> = SafetensorsStore::from_file(path)
                .get_all_tensors()
                .map_err(|e| {
                    BunsenError::invalid_resource("the store refused the checkpoint").with_cause(e)
                })?
                .values()
                .cloned()
                .collect();
            Self::load_16khz_from_upstream_tensors(tensors, device)
        };
        read().with_context(|| format!("loading {}", path.display()))
    }
}

impl SileroVadCollection {
    /// A collection of one 16khz model: what upstream's
    /// `silero_vad_16k.safetensors` holds.
    pub fn from_16khz(vad_16: SileroVad) -> Self {
        Self {
            branches: vec![(16000, vad_16)],
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_upstream_rekey() {
        let rekey = upstream_rekey();
        for (upstream, bunsen) in [
            ("stft_conv.weight", Some("stft.weight")),
            ("conv1.weight", Some("encoder.blocks.0.conv.weight")),
            ("conv4.bias", Some("encoder.blocks.3.conv.bias")),
            ("lstm_cell.weight_ih", Some("lstm.features.weight")),
            ("lstm_cell.bias_ih", Some("lstm.features.bias")),
            ("lstm_cell.weight_hh", Some("lstm.hidden.weight")),
            ("lstm_cell.bias_hh", Some("lstm.hidden.bias")),
            ("final_conv.bias", Some("decoder.bias")),
            ("conv0.weight", None),
            ("lstm_cell.weight_xx", None),
            ("lstm_cell.weight_ih_l0", None),
            ("unknown", None),
        ] {
            assert_eq!(rekey.apply(upstream).as_deref(), bunsen, "{upstream}");
        }
    }

    /// A state dict with upstream's names and shapes loads into every
    /// parameter, the LSTM's linears transposed as `PyTorch`'s are.
    #[cfg(feature = "store_safetensors")]
    #[test]
    fn test_load_upstream_tensors() {
        use burn::{
            prelude::TensorData,
            store::bridge,
        };

        use crate::support::testing::cpu_device;

        let shapes: [(&str, &[usize]); 15] = [
            ("stft_conv.weight", &[258, 1, 256]),
            ("conv1.weight", &[128, 129, 3]),
            ("conv1.bias", &[128]),
            ("conv2.weight", &[64, 128, 3]),
            ("conv2.bias", &[64]),
            ("conv3.weight", &[64, 64, 3]),
            ("conv3.bias", &[64]),
            ("conv4.weight", &[128, 64, 3]),
            ("conv4.bias", &[128]),
            ("lstm_cell.weight_ih", &[512, 128]),
            ("lstm_cell.weight_hh", &[512, 128]),
            ("lstm_cell.bias_ih", &[512]),
            ("lstm_cell.bias_hh", &[512]),
            ("final_conv.weight", &[1, 128, 1]),
            ("final_conv.bias", &[1]),
        ];
        let tensors = shapes.iter().map(|(name, shape)| {
            let n: usize = shape.iter().product();
            let values: Vec<f32> = (0..n).map(|k| k as f32 / n as f32).collect();
            bridge::from_data(
                TensorData::new(values, shape.to_vec()),
                name.to_string(),
                None,
            )
        });
        let vad = SileroVad::load_16khz_from_upstream_tensors(tensors, &cpu_device()).unwrap();

        // `weight_ih[1, 0]` is the 128th value; burn's linear holds its
        // transpose.
        let w: Vec<f32> = vad
            .lstm
            .features
            .weight
            .val()
            .slice([0..1, 1..2])
            .into_data()
            .try_into_vec_as()
            .unwrap();
        let n = 512 * 128;
        assert_eq!(w, [128.0 / n as f32]);
        let b: Vec<f32> = vad
            .decoder
            .bias
            .as_ref()
            .unwrap()
            .val()
            .into_data()
            .try_into_vec_as()
            .unwrap();
        assert_eq!(b, [0.0]);
        assert_eq!(SileroVadCollection::from_16khz(vad).branches.len(), 1);
    }
}