use burn::store::burn_pack::Tensor as PackTensor;
use super::blocks::{
NemotronDiarizationConfig,
Sortformer,
};
use crate::{
blocks::signals::{
DwStridingSubsamplingMeta,
FeatureStackingMeta,
FeatureStackingOrder,
},
burner::module::{
CanonicalKeys,
Rekey,
},
data::formats::NamedTensorsApplied,
errors::{
BunsenError,
BunsenResult,
},
kits::{
speech::nemo::{
conformer_encoder_rekey,
transformer_encoder_rekey,
},
support::model_config::TransformersConfigJson,
},
};
pub fn embedder_rekey(stacking: &impl FeatureStackingMeta) -> Rekey {
stacking.restack_from(
FeatureStackingOrder::FramesFeatures,
Rekey::drop_rest().rename("projection.", "projection."),
)
}
pub fn encoder_layer_rekey() -> Rekey {
Rekey::drop_rest()
.rename("layer_norm1.", "attn_norm.")
.rename("layer_norm2.", "mlp_norm.")
.rename("self_attn.q_proj.", "attn.c_q.")
.rename("self_attn.k_proj.", "attn.c_k.")
.rename("self_attn.v_proj.", "attn.c_v.")
.rename("self_attn.o_proj.", "attn.c_proj.")
.rename("mlp.fc1.", "mlp.linear1.")
.rename("mlp.fc2.", "mlp.linear2.")
}
pub fn encoder_rekey() -> Rekey {
Rekey::drop_rest()
.rename("input_layer_norm.", "input_norm.")
.rename("layer_norm.", "norm.")
.mount_indexed("layers.", "layers.", encoder_layer_rekey())
}
pub fn nemotron_transformers_rekey(stacking: &impl FeatureStackingMeta) -> Rekey {
Rekey::drop_rest()
.exact("silence_embeds", "silence")
.mount(
"model.audio_tower.embedder.",
"encoder.subsampling.",
embedder_rekey(stacking),
)
.mount(
"model.audio_tower.",
"encoder.transformer.",
encoder_rekey(),
)
.rename("model.proj.", "projection.")
.rename("model.upsampler.conv.", "upsample.conv.")
.rename("classifier.dense.", "head.dense.")
.rename("classifier.out_proj.", "head.out.")
}
pub fn load_nemotron_transformers_tensors(
model: &mut Sortformer,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
let stacking = model.encoder.subsampling.stacking().ok_or_else(|| {
BunsenError::invalid_resource("a Nemotron encoder subsamples by feature stacking")
})?;
let rekey = nemotron_transformers_rekey(stacking);
let rekey = rekey.then(model.canonical_rekey());
rekey.apply_to(model, tensors)
}
pub fn sortformer_v2_nemo_rekey(striding: &impl DwStridingSubsamplingMeta) -> Rekey {
Rekey::drop_rest()
.mount("encoder.", "encoder.", conformer_encoder_rekey(striding))
.mount(
"transformer_encoder.",
"transformer.",
transformer_encoder_rekey(),
)
.rename("sortformer_modules.encoder_proj.", "projection.")
.rename("sortformer_modules.first_hidden_to_hidden.", "head.dense.")
.rename("sortformer_modules.single_hidden_to_spks.", "head.out.")
}
pub fn load_sortformer_v2_nemo_tensors(
model: &mut Sortformer,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
let striding = model.encoder.subsampling.dw_striding().ok_or_else(|| {
BunsenError::invalid_resource("a Sortformer v2 encoder subsamples by depthwise striding")
})?;
let rekey = sortformer_v2_nemo_rekey(striding);
let rekey = rekey.then(model.canonical_rekey());
rekey.apply_to(model, tensors)
}
pub fn nemotron_config_from_transformers(json: &str) -> BunsenResult<NemotronDiarizationConfig> {
let c = TransformersConfigJson::parse(json)?;
c.expect_str("model_type", "nemotron3_diarization")?;
let a = c.section("audio_config")?;
a.expect_str("hidden_act", "gelu")?;
let d_model = a.usize("hidden_size")?;
let intermediate = a.usize("intermediate_size")?;
if d_model == 0 || !intermediate.is_multiple_of(d_model) {
return Err(BunsenError::unsupported(format!(
"config.json: `audio_config.intermediate_size` {intermediate} is not a whole \
multiple of `hidden_size` {d_model}"
)));
}
let rope = a.section("rope_parameters")?;
a.expect_eq(
"partial_rotary_factor",
a.f64("partial_rotary_factor")?,
1.0,
)?;
let theta = rope.f64("rope_theta")?;
if theta.fract() != 0.0 || theta < 1.0 {
return Err(BunsenError::unsupported(format!(
"config.json: `audio_config.rope_parameters.rope_theta` {theta} is not a whole number"
)));
}
let factor = a.usize("subsampling_factor")?;
let h = c.section("head_config")?;
h.expect_eq("audio_hidden_size", h.usize("audio_hidden_size")?, d_model)?;
h.expect_eq("subsampling_factor", h.usize("subsampling_factor")?, factor)?;
Ok(NemotronDiarizationConfig::new()
.with_n_mels(a.usize("num_mel_bins")?)
.with_d_model(d_model)
.with_n_layers(a.usize("num_hidden_layers")?)
.with_n_heads(a.usize("num_attention_heads")?)
.with_ff_expansion(intermediate / d_model)
.with_subsampling_factor(factor)
.with_rope_base(theta as usize)
.with_d_hidden(h.usize("hidden_size")?)
.with_n_speakers(h.usize("num_speakers")?))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_paths() {
for (hf, bunsen) in [
("silence_embeds", "silence"),
(
"model.audio_tower.embedder.projection.weight",
"encoder.subsampling.projection.weight",
),
(
"model.audio_tower.input_layer_norm.bias",
"encoder.transformer.input_norm.bias",
),
(
"model.audio_tower.layer_norm.weight",
"encoder.transformer.norm.weight",
),
(
"model.audio_tower.layers.30.self_attn.o_proj.bias",
"encoder.transformer.layers.30.attn.c_proj.bias",
),
(
"model.audio_tower.layers.0.mlp.fc2.weight",
"encoder.transformer.layers.0.mlp.linear2.weight",
),
("model.upsampler.conv.weight", "upsample.conv.weight"),
("classifier.out_proj.bias", "head.out.bias"),
] {
assert_eq!(
nemotron_transformers_rekey(&crate::blocks::signals::FeatureStackingConfig::new(
128, 512
))
.apply(hf)
.as_deref(),
Some(bunsen),
"{hf}"
);
}
assert_eq!(
nemotron_transformers_rekey(&crate::blocks::signals::FeatureStackingConfig::new(
128, 512
))
.apply("model.audio_tower.rotary_emb.inv_freq"),
None
);
}
#[test]
fn test_v2_paths() {
for (nemo, bunsen) in [
(
"encoder.pre_encode.conv.0.weight",
Some("encoder.subsampling.stem.conv.weight"),
),
(
"encoder.layers.16.self_attn.linear_q.bias",
Some("encoder.transformer.layers.16.attn.query.bias"),
),
("encoder.layers.0.conv.batch_norm.num_batches_tracked", None),
(
"transformer_encoder.layers.17.first_sub_layer.out_projection.weight",
Some("transformer.layers.17.attn.output.weight"),
),
(
"sortformer_modules.encoder_proj.weight",
Some("projection.weight"),
),
(
"sortformer_modules.first_hidden_to_hidden.bias",
Some("head.dense.bias"),
),
(
"sortformer_modules.single_hidden_to_spks.weight",
Some("head.out.weight"),
),
("sortformer_modules.hidden_to_spks.weight", None),
("preprocessor.featurizer.fb", None),
] {
assert_eq!(
sortformer_v2_nemo_rekey(
&crate::blocks::signals::DwStridingSubsamplingConfig::new(128, 256, 512)
)
.apply(nemo)
.as_deref(),
bunsen,
"{nemo}"
);
}
}
#[test]
fn test_nemotron_config_from_transformers() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/testdata/sortformer/nemotron_config.json"
));
let config = nemotron_config_from_transformers(json).unwrap();
let want = NemotronDiarizationConfig::nemotron_3_diarization();
assert_eq!(config.n_mels, want.n_mels);
assert_eq!(config.d_model, want.d_model);
assert_eq!(config.n_layers, want.n_layers);
assert_eq!(config.n_heads, want.n_heads);
assert_eq!(config.ff_expansion, want.ff_expansion);
assert_eq!(config.subsampling_factor, want.subsampling_factor);
assert_eq!(config.rope_base, want.rope_base);
assert_eq!(config.d_hidden, want.d_hidden);
assert_eq!(config.n_speakers, want.n_speakers);
let other = json.replace("\"nemotron3_diarization\"", "\"something_else\"");
assert!(nemotron_config_from_transformers(&other).is_err());
let relu = json.replace("\"gelu\"", "\"relu\"");
assert!(nemotron_config_from_transformers(&relu).is_err());
}
}