use burn::store::burn_pack::Tensor as PackTensor;
use super::blocks::{
ParakeetTdt,
ParakeetTdtConfig,
};
use crate::{
blocks::{
rnn::lstm::FusedLstm,
signals::{
DwStridingSubsamplingConfig,
DwStridingSubsamplingMeta,
FlattenOrder,
},
transformers::encoders::conformer::{
ConformerBlockConfig,
ConformerEncoderConfig,
},
},
burner::module::Rekey,
data::pretrained::{
HfConfigJson,
NamedTensorsApplied,
apply_rekeyed_tensors,
},
errors::{
BunsenError,
BunsenResult,
},
};
pub fn subsampling_rekey(striding: &impl DwStridingSubsamplingMeta) -> Rekey {
striding.reflatten_from(
FlattenOrder::ChannelsFreqs,
striding.sequential_rekey("layers."),
)
}
pub fn encoder_block_rekey() -> Rekey {
Rekey::keep_rest()
.rename("self_attn.q_proj.", "attn.query.")
.rename("self_attn.k_proj.", "attn.key.")
.rename("self_attn.v_proj.", "attn.value.")
.rename("self_attn.o_proj.", "attn.output.")
.rename("self_attn.relative_k_proj.", "attn.pos.")
.exact("self_attn.bias_u", "attn.pos_bias_u")
.exact("self_attn.bias_v", "attn.pos_bias_v")
.rename("conv.depthwise_conv.", "conv.depthwise.conv.")
.rename("conv.norm.", "conv.depthwise.norm.")
.rename("norm_feed_forward1.", "feed_forward1_norm.")
.rename("norm_self_att.", "attn_norm.")
.rename("norm_conv.", "conv_norm.")
.rename("norm_feed_forward2.", "feed_forward2_norm.")
.rename("norm_out.", "out_norm.")
.rename("self_attn.", "attn.")
}
pub fn encoder_rekey(striding: &impl DwStridingSubsamplingMeta) -> Rekey {
Rekey::keep_rest()
.drop_suffix(".num_batches_tracked")
.mount("subsampling.", "subsampling.", subsampling_rekey(striding))
.mount_indexed("layers.", "transformer.layers.", encoder_block_rekey())
}
pub fn hf_rekey(striding: &impl DwStridingSubsamplingMeta) -> Rekey {
Rekey::keep_rest()
.drop_suffix(".num_batches_tracked")
.mount("encoder.", "encoder.", encoder_rekey(striding))
.mount(
"decoder.lstm.",
"decoder.lstm.",
FusedLstm::torch_stack_rekey(),
)
.rename("decoder.decoder_projector.", "decoder.projector.")
}
pub fn load_hf_tensors(
model: &mut ParakeetTdt,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
let striding = model.encoder.subsampling.dw_striding().ok_or_else(|| {
BunsenError::invalid_resource("a Parakeet encoder subsamples by depthwise striding")
})?;
let rekey = hf_rekey(striding);
apply_rekeyed_tensors(model, tensors, &rekey)
}
pub fn parakeet_config_from_hf(json: &str) -> BunsenResult<ParakeetTdtConfig> {
let c = HfConfigJson::parse(json)?;
c.expect_str("model_type", "parakeet_tdt")?;
let e = c.section("encoder_config")?;
e.expect_eq("num_mel_bins", e.usize("num_mel_bins")?, 128)?;
e.expect_eq(
"subsampling_conv_kernel_size",
e.usize("subsampling_conv_kernel_size")?,
3,
)?;
e.expect_eq(
"subsampling_conv_stride",
e.usize("subsampling_conv_stride")?,
2,
)?;
let factor = e.usize("subsampling_factor")?;
if !factor.is_power_of_two() || factor < 2 {
return Err(crate::errors::BunsenError::unsupported(format!(
"config.json: `encoder_config.subsampling_factor` is {factor}, not a power of two"
)));
}
let d_model = e.usize("hidden_size")?;
let encoder = ConformerEncoderConfig::new(
DwStridingSubsamplingConfig::new(128, e.usize("subsampling_conv_channels")?, d_model)
.with_n_stages(factor.trailing_zeros() as usize),
ConformerBlockConfig::new(
d_model,
e.usize("num_attention_heads")?,
e.usize("intermediate_size")?,
e.usize("conv_kernel_size")?,
)
.with_bias(e.bool_or("attention_bias", true)?)
.with_conv_bias(e.bool_or("convolution_bias", true)?),
e.usize("num_hidden_layers")?,
)
.with_scale_input(e.bool_or("scale_input", false)?);
Ok(ParakeetTdtConfig::new(
encoder,
c.usize("vocab_size")?,
c.usize("decoder_hidden_size")?,
)
.with_n_pred_layers(c.usize("num_decoder_layers")?)
.with_durations(c.usizes("durations")?))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_hf_rekey() {
for (hf, bunsen) in [
(
"encoder.subsampling.layers.0.weight",
Some("encoder.subsampling.stem.conv.weight"),
),
(
"encoder.subsampling.layers.2.bias",
Some("encoder.subsampling.stages.0.depthwise.conv.bias"),
),
(
"encoder.subsampling.layers.3.weight",
Some("encoder.subsampling.stages.0.pointwise.conv.weight"),
),
(
"encoder.subsampling.layers.5.weight",
Some("encoder.subsampling.stages.1.depthwise.conv.weight"),
),
(
"encoder.subsampling.layers.6.bias",
Some("encoder.subsampling.stages.1.pointwise.conv.bias"),
),
("encoder.subsampling.layers.8.weight", None),
(
"encoder.subsampling.linear.weight",
Some("encoder.subsampling.linear.weight"),
),
(
"encoder.layers.3.self_attn.q_proj.weight",
Some("encoder.transformer.layers.3.attn.query.weight"),
),
(
"encoder.layers.3.self_attn.relative_k_proj.weight",
Some("encoder.transformer.layers.3.attn.pos.weight"),
),
(
"encoder.layers.3.self_attn.bias_u",
Some("encoder.transformer.layers.3.attn.pos_bias_u"),
),
(
"encoder.layers.3.conv.depthwise_conv.weight",
Some("encoder.transformer.layers.3.conv.depthwise.conv.weight"),
),
(
"encoder.layers.3.conv.norm.running_var",
Some("encoder.transformer.layers.3.conv.depthwise.norm.running_var"),
),
("encoder.layers.3.conv.norm.num_batches_tracked", None),
(
"encoder.layers.3.conv.pointwise_conv1.weight",
Some("encoder.transformer.layers.3.conv.pointwise_conv1.weight"),
),
(
"encoder.layers.3.norm_out.weight",
Some("encoder.transformer.layers.3.out_norm.weight"),
),
(
"decoder.lstm.weight_ih_l1",
Some("decoder.lstm.1.features.weight"),
),
(
"decoder.lstm.bias_hh_l0",
Some("decoder.lstm.0.hidden.bias"),
),
(
"decoder.decoder_projector.bias",
Some("decoder.projector.bias"),
),
("joint.head.weight", Some("joint.head.weight")),
("encoder_projector.weight", Some("encoder_projector.weight")),
] {
assert_eq!(
hf_rekey(&DwStridingSubsamplingConfig::new(128, 256, 1024))
.apply(hf)
.as_deref(),
bunsen,
"{hf}"
);
}
}
#[test]
fn test_parakeet_config_from_hf() {
const V3: &str = r#"{"architectures": ["ParakeetForTDT"], "blank_token_id": 8192,
"decoder_hidden_size": 640, "durations": [0, 1, 2, 3, 4],
"encoder_config": {"attention_bias": false, "conv_kernel_size": 9,
"convolution_bias": false, "hidden_size": 1024, "intermediate_size": 4096,
"model_type": "parakeet_encoder", "num_attention_heads": 8,
"num_hidden_layers": 24, "num_mel_bins": 128, "scale_input": false,
"subsampling_conv_channels": 256, "subsampling_conv_kernel_size": 3,
"subsampling_conv_stride": 2, "subsampling_factor": 8},
"model_type": "parakeet_tdt", "num_decoder_layers": 2, "vocab_size": 8193}"#;
let config = parakeet_config_from_hf(V3).unwrap();
assert_eq!(
format!("{config:?}"),
format!("{:?}", ParakeetTdtConfig::tdt_0_6b_v3())
);
let ctc = V3.replace("parakeet_tdt", "parakeet_ctc");
crate::errors::testing::ErrorMatcher::kind(crate::errors::BunsenErrorKind::Unsupported)
.message_contains("`model_type`")
.assert_err(¶keet_config_from_hf(&ctc));
let mels = V3.replace("\"num_mel_bins\": 128", "\"num_mel_bins\": 80");
crate::errors::testing::ErrorMatcher::kind(crate::errors::BunsenErrorKind::Unsupported)
.message_contains("`encoder_config.num_mel_bins` is 80")
.assert_err(¶keet_config_from_hf(&mels));
}
}