use burn::store::burn_pack::Tensor as PackTensor;
use super::blocks::{
ParakeetTdt,
ParakeetTdtConfig,
};
use crate::{
blocks::conformer::{
ConformerBlockConfig,
ConformerEncoderConfig,
DwStridingSubsamplingConfig,
DwStridingSubsamplingMeta,
},
data::pretrained::{
HfConfigJson,
NamedTensorsApplied,
apply_named_tensors,
},
errors::BunsenResult,
};
pub fn hf_to_bunsen_path(
name: &str,
n_stages: usize,
) -> Option<String> {
if name.ends_with(".num_batches_tracked") {
return None;
}
if let Some(rest) = name.strip_prefix("encoder.subsampling.layers.") {
let (k, param) = rest.split_once('.')?;
let k: usize = k.parse().ok()?;
let module = match k {
0 => "stem.conv".to_string(),
k if k >= 2 && (k - 2) / 3 + 1 < n_stages => match (k - 2) % 3 {
0 => format!("stages.{}.depthwise.conv", (k - 2) / 3),
1 => format!("stages.{}.pointwise.conv", (k - 2) / 3),
_ => return None,
},
_ => return None,
};
return Some(format!("encoder.subsampling.{module}.{param}"));
}
if let Some(rest) = name.strip_prefix("decoder.lstm.") {
let (param, layer) = rest.rsplit_once("_l")?;
let (proj, kind) = match param {
"weight_ih" => ("features", "weight"),
"weight_hh" => ("hidden", "weight"),
"bias_ih" => ("features", "bias"),
"bias_hh" => ("hidden", "bias"),
_ => return None,
};
return Some(format!("decoder.lstm.{layer}.{proj}.{kind}"));
}
const RENAMES: &[(&str, &str)] = &[
(".self_attn.q_proj.", ".self_attn.query."),
(".self_attn.k_proj.", ".self_attn.key."),
(".self_attn.v_proj.", ".self_attn.value."),
(".self_attn.o_proj.", ".self_attn.output."),
(".self_attn.relative_k_proj.", ".self_attn.pos."),
(".self_attn.bias_u", ".self_attn.pos_bias_u"),
(".self_attn.bias_v", ".self_attn.pos_bias_v"),
(".conv.depthwise_conv.", ".conv.depthwise.conv."),
(".conv.norm.", ".conv.depthwise.norm."),
("decoder.decoder_projector.", "decoder.projector."),
];
let mut path = name.to_string();
for (from, to) in RENAMES {
if path.contains(from) {
path = path.replacen(from, to, 1);
break;
}
}
Some(path)
}
pub fn load_hf_tensors(
model: &mut ParakeetTdt,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
let n_stages = model.encoder.subsampling.n_stages();
apply_named_tensors(model, tensors, |name| hf_to_bunsen_path(name, n_stages))
}
pub fn parakeet_config_from_hf(json: &str) -> BunsenResult<ParakeetTdtConfig> {
let c = HfConfigJson::parse(json)?;
c.expect_str("model_type", "parakeet_tdt")?;
let e = c.section("encoder_config")?;
e.expect_eq("num_mel_bins", e.usize("num_mel_bins")?, 128)?;
e.expect_eq(
"subsampling_conv_kernel_size",
e.usize("subsampling_conv_kernel_size")?,
3,
)?;
e.expect_eq(
"subsampling_conv_stride",
e.usize("subsampling_conv_stride")?,
2,
)?;
let factor = e.usize("subsampling_factor")?;
if !factor.is_power_of_two() || factor < 2 {
return Err(crate::errors::BunsenError::unsupported(format!(
"config.json: `encoder_config.subsampling_factor` is {factor}, not a power of two"
)));
}
let d_model = e.usize("hidden_size")?;
let encoder = ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(128, e.usize("subsampling_conv_channels")?, d_model)
.with_n_stages(factor.trailing_zeros() as usize),
ConformerBlockConfig::new(
d_model,
e.usize("num_attention_heads")?,
e.usize("intermediate_size")?,
e.usize("conv_kernel_size")?,
)
.with_bias(e.bool_or("attention_bias", true)?)
.with_conv_bias(e.bool_or("convolution_bias", true)?),
e.usize("num_hidden_layers")?,
)
.with_scale_input(e.bool_or("scale_input", false)?);
Ok(ParakeetTdtConfig::new(
encoder,
c.usize("vocab_size")?,
c.usize("decoder_hidden_size")?,
)
.with_n_pred_layers(c.usize("num_decoder_layers")?)
.with_durations(c.usizes("durations")?))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_hf_to_bunsen_path() {
for (hf, bunsen) in [
(
"encoder.subsampling.layers.0.weight",
Some("encoder.subsampling.stem.conv.weight"),
),
(
"encoder.subsampling.layers.2.bias",
Some("encoder.subsampling.stages.0.depthwise.conv.bias"),
),
(
"encoder.subsampling.layers.3.weight",
Some("encoder.subsampling.stages.0.pointwise.conv.weight"),
),
(
"encoder.subsampling.layers.5.weight",
Some("encoder.subsampling.stages.1.depthwise.conv.weight"),
),
(
"encoder.subsampling.layers.6.bias",
Some("encoder.subsampling.stages.1.pointwise.conv.bias"),
),
("encoder.subsampling.layers.8.weight", None),
(
"encoder.subsampling.linear.weight",
Some("encoder.subsampling.linear.weight"),
),
(
"encoder.layers.3.self_attn.q_proj.weight",
Some("encoder.layers.3.self_attn.query.weight"),
),
(
"encoder.layers.3.self_attn.relative_k_proj.weight",
Some("encoder.layers.3.self_attn.pos.weight"),
),
(
"encoder.layers.3.self_attn.bias_u",
Some("encoder.layers.3.self_attn.pos_bias_u"),
),
(
"encoder.layers.3.conv.depthwise_conv.weight",
Some("encoder.layers.3.conv.depthwise.conv.weight"),
),
(
"encoder.layers.3.conv.norm.running_var",
Some("encoder.layers.3.conv.depthwise.norm.running_var"),
),
("encoder.layers.3.conv.norm.num_batches_tracked", None),
(
"encoder.layers.3.conv.pointwise_conv1.weight",
Some("encoder.layers.3.conv.pointwise_conv1.weight"),
),
(
"encoder.layers.3.norm_out.weight",
Some("encoder.layers.3.norm_out.weight"),
),
(
"decoder.lstm.weight_ih_l1",
Some("decoder.lstm.1.features.weight"),
),
(
"decoder.lstm.bias_hh_l0",
Some("decoder.lstm.0.hidden.bias"),
),
(
"decoder.decoder_projector.bias",
Some("decoder.projector.bias"),
),
("joint.head.weight", Some("joint.head.weight")),
("encoder_projector.weight", Some("encoder_projector.weight")),
] {
assert_eq!(hf_to_bunsen_path(hf, 3).as_deref(), bunsen, "{hf}");
}
}
#[test]
fn test_parakeet_config_from_hf() {
const V3: &str = r#"{"architectures": ["ParakeetForTDT"], "blank_token_id": 8192,
"decoder_hidden_size": 640, "durations": [0, 1, 2, 3, 4],
"encoder_config": {"attention_bias": false, "conv_kernel_size": 9,
"convolution_bias": false, "hidden_size": 1024, "intermediate_size": 4096,
"model_type": "parakeet_encoder", "num_attention_heads": 8,
"num_hidden_layers": 24, "num_mel_bins": 128, "scale_input": false,
"subsampling_conv_channels": 256, "subsampling_conv_kernel_size": 3,
"subsampling_conv_stride": 2, "subsampling_factor": 8},
"model_type": "parakeet_tdt", "num_decoder_layers": 2, "vocab_size": 8193}"#;
let config = parakeet_config_from_hf(V3).unwrap();
assert_eq!(
format!("{config:?}"),
format!("{:?}", ParakeetTdtConfig::tdt_0_6b_v3())
);
let ctc = V3.replace("parakeet_tdt", "parakeet_ctc");
crate::errors::testing::ErrorMatcher::kind(crate::errors::BunsenErrorKind::Unsupported)
.message_contains("`model_type`")
.assert_err(¶keet_config_from_hf(&ctc));
let mels = V3.replace("\"num_mel_bins\": 128", "\"num_mel_bins\": 80");
crate::errors::testing::ErrorMatcher::kind(crate::errors::BunsenErrorKind::Unsupported)
.message_contains("`encoder_config.num_mel_bins` is 80")
.assert_err(¶keet_config_from_hf(&mels));
}
}