use std::path::Path;
use burn::{
store::{
BurnpackStore,
KeyRemapper,
ModuleSnapshot,
burn_pack::Error as BurnpackError,
},
tensor::Device,
};
use crate::{
blocks::{
conv::ConvSeq1d,
rnn::lstm::FusedLstm,
},
burner::module::{
ModuleInit,
Rekey,
},
errors::{
BunsenError,
BunsenErrorKind,
BunsenResult,
ResultContext,
WithOkOrPanic,
sys_at,
},
kits::speech::silero_vad::{
SileroVad,
SileroVadCollection,
SileroVadSignalConfig,
},
};
impl SileroVad {
pub fn load_16khz_from_burnpack_bytes(
bytes: burn::tensor::Bytes,
device: &Device,
) -> BunsenResult<Self> {
Self::load_from_burnpack(
BurnpackStore::from_bytes(Some(bytes)),
SileroVadSignalConfig::standard_16khz(),
Self::pretrained_16khz_remapper(),
device,
)
}
pub fn load_16khz_from_burnpack_file(
path: impl AsRef<Path>,
device: &Device,
) -> BunsenResult<Self> {
Self::load_from_burnpack_path(
path,
SileroVadSignalConfig::standard_16khz(),
Self::pretrained_16khz_remapper(),
device,
)
}
pub fn load_8khz_from_burnpack_bytes(
bytes: burn::tensor::Bytes,
device: &Device,
) -> BunsenResult<Self> {
Self::load_from_burnpack(
BurnpackStore::from_bytes(Some(bytes)),
SileroVadSignalConfig::standard_8khz(),
Self::pretrained_8khz_remapper(),
device,
)
}
pub fn load_8khz_from_burnpack_file(
path: impl AsRef<Path>,
device: &Device,
) -> BunsenResult<Self> {
Self::load_from_burnpack_path(
path,
SileroVadSignalConfig::standard_8khz(),
Self::pretrained_8khz_remapper(),
device,
)
}
pub fn pretrained_16khz_remapper() -> KeyRemapper {
KeyRemapper::from_patterns(vec![
("conv1d37", "stft".to_string()),
("conv1d38", ConvSeq1d::block_conv_path("encoder", 0)),
("conv1d39", ConvSeq1d::block_conv_path("encoder", 1)),
("conv1d40", ConvSeq1d::block_conv_path("encoder", 2)),
("conv1d41", ConvSeq1d::block_conv_path("encoder", 3)),
("linear13", "lstm.hidden".to_string()),
("linear14", "lstm.features".to_string()),
("conv1d42", "decoder".to_string()),
])
.ok_or_panic()
}
pub fn pretrained_8khz_remapper() -> KeyRemapper {
KeyRemapper::from_patterns(vec![
("conv1d43", "stft".to_string()),
("conv1d44", ConvSeq1d::block_conv_path("encoder", 0)),
("conv1d45", ConvSeq1d::block_conv_path("encoder", 1)),
("conv1d46", ConvSeq1d::block_conv_path("encoder", 2)),
("conv1d47", ConvSeq1d::block_conv_path("encoder", 3)),
("linear15", "lstm.hidden".to_string()),
("linear16", "lstm.features".to_string()),
("conv1d48", "decoder".to_string()),
])
.ok_or_panic()
}
fn load_from_burnpack_path<C>(
path: impl AsRef<Path>,
cfg: C,
remapper: KeyRemapper,
device: &Device,
) -> BunsenResult<Self>
where
C: ModuleInit<Self>,
{
let path = path.as_ref();
std::fs::metadata(path).map_err(sys_at("open", path))?;
Self::load_from_burnpack(BurnpackStore::from_file(path), cfg, remapper, device)
.with_context(|| format!("loading {}", path.display()))
}
pub fn load_from_burnpack<C>(
store: BurnpackStore,
cfg: C,
remapper: KeyRemapper,
device: &Device,
) -> BunsenResult<Self>
where
C: ModuleInit<Self>,
{
let mut store = store.remap(remapper);
let mut module = cfg.try_init(device)?;
module.load_from(&mut store).map_err(burnpack_load_error)?;
Ok(module)
}
}
fn burnpack_load_error(error: BurnpackError) -> BunsenError {
let kind = match &error {
BurnpackError::IoError(_) => BunsenErrorKind::Sys,
BurnpackError::InvalidVersion => BunsenErrorKind::Unsupported,
_ => BunsenErrorKind::InvalidResource,
};
BunsenError::from_cause(kind, error)
}
impl SileroVadCollection {
pub(super) fn new_common_collection(
vad_16: SileroVad,
vad_8: SileroVad,
) -> Self {
Self {
branches: vec![(16000, vad_16), (8000, vad_8)],
}
}
pub fn load_from_burnpack_bytes(
bytes: burn::tensor::Bytes,
device: &Device,
) -> BunsenResult<Self> {
Ok(Self::new_common_collection(
SileroVad::load_16khz_from_burnpack_bytes(bytes.clone(), device)?,
SileroVad::load_8khz_from_burnpack_bytes(bytes, device)?,
))
}
pub fn load_from_burnpack_file<P: AsRef<Path>>(
path: P,
device: &Device,
) -> BunsenResult<Self> {
let path = path.as_ref();
Ok(Self::new_common_collection(
SileroVad::load_16khz_from_burnpack_file(path, device)?,
SileroVad::load_8khz_from_burnpack_file(path, device)?,
))
}
}
pub fn upstream_rekey() -> Rekey {
Rekey::drop_rest()
.rename("stft_conv.", "stft.")
.rename("final_conv.", "decoder.")
.mount("lstm_cell.", "lstm.", FusedLstm::torch_cell_rekey())
.mount("conv", "encoder.", ConvSeq1d::one_based_rekey())
}
#[cfg(feature = "store_safetensors")]
impl SileroVad {
pub fn load_16khz_from_upstream_tensors(
tensors: impl IntoIterator<Item = burn::store::burn_pack::Tensor>,
device: &Device,
) -> BunsenResult<Self> {
let mut vad: Self = SileroVadSignalConfig::standard_16khz().try_init(device)?;
crate::data::pretrained::apply_rekeyed_tensors(&mut vad, tensors, &upstream_rekey())?;
Ok(vad)
}
pub fn load_16khz_from_safetensors_file(
path: impl AsRef<Path>,
device: &Device,
) -> BunsenResult<Self> {
use burn::store::{
ModuleStore,
SafetensorsStore,
};
let path = path.as_ref();
let read = || -> BunsenResult<Self> {
std::fs::metadata(path).map_err(sys_at("open", path))?;
let tensors: Vec<_> = SafetensorsStore::from_file(path)
.get_all_tensors()
.map_err(|e| {
BunsenError::invalid_resource("the store refused the checkpoint").with_cause(e)
})?
.values()
.cloned()
.collect();
Self::load_16khz_from_upstream_tensors(tensors, device)
};
read().with_context(|| format!("loading {}", path.display()))
}
}
impl SileroVadCollection {
pub fn from_16khz(vad_16: SileroVad) -> Self {
Self {
branches: vec![(16000, vad_16)],
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_upstream_rekey() {
let rekey = upstream_rekey();
for (upstream, bunsen) in [
("stft_conv.weight", Some("stft.weight")),
("conv1.weight", Some("encoder.blocks.0.conv.weight")),
("conv4.bias", Some("encoder.blocks.3.conv.bias")),
("lstm_cell.weight_ih", Some("lstm.features.weight")),
("lstm_cell.bias_ih", Some("lstm.features.bias")),
("lstm_cell.weight_hh", Some("lstm.hidden.weight")),
("lstm_cell.bias_hh", Some("lstm.hidden.bias")),
("final_conv.bias", Some("decoder.bias")),
("conv0.weight", None),
("lstm_cell.weight_xx", None),
("lstm_cell.weight_ih_l0", None),
("unknown", None),
] {
assert_eq!(rekey.apply(upstream).as_deref(), bunsen, "{upstream}");
}
}
#[cfg(feature = "store_safetensors")]
#[test]
fn test_load_upstream_tensors() {
use burn::{
prelude::TensorData,
store::bridge,
};
use crate::support::testing::cpu_device;
let shapes: [(&str, &[usize]); 15] = [
("stft_conv.weight", &[258, 1, 256]),
("conv1.weight", &[128, 129, 3]),
("conv1.bias", &[128]),
("conv2.weight", &[64, 128, 3]),
("conv2.bias", &[64]),
("conv3.weight", &[64, 64, 3]),
("conv3.bias", &[64]),
("conv4.weight", &[128, 64, 3]),
("conv4.bias", &[128]),
("lstm_cell.weight_ih", &[512, 128]),
("lstm_cell.weight_hh", &[512, 128]),
("lstm_cell.bias_ih", &[512]),
("lstm_cell.bias_hh", &[512]),
("final_conv.weight", &[1, 128, 1]),
("final_conv.bias", &[1]),
];
let tensors = shapes.iter().map(|(name, shape)| {
let n: usize = shape.iter().product();
let values: Vec<f32> = (0..n).map(|k| k as f32 / n as f32).collect();
bridge::from_data(
TensorData::new(values, shape.to_vec()),
name.to_string(),
None,
)
});
let vad = SileroVad::load_16khz_from_upstream_tensors(tensors, &cpu_device()).unwrap();
let w: Vec<f32> = vad
.lstm
.features
.weight
.val()
.slice([0..1, 1..2])
.into_data()
.try_into_vec_as()
.unwrap();
let n = 512 * 128;
assert_eq!(w, [128.0 / n as f32]);
let b: Vec<f32> = vad
.decoder
.bias
.as_ref()
.unwrap()
.val()
.into_data()
.try_into_vec_as()
.unwrap();
assert_eq!(b, [0.0]);
assert_eq!(SileroVadCollection::from_16khz(vad).branches.len(), 1);
}
}