use burn::store::burn_pack::Tensor as PackTensor;
use super::blocks::{
VoxtralAudioEncoderConfig,
VoxtralRealtime,
VoxtralRealtimeConfig,
VoxtralTextConfig,
};
use crate::{
blocks::{
conv::ConvSeq1d,
transformers::decoders::causal::{
CausalDecoderBlock,
DecoderLayerKeys,
},
},
burner::module::{
CanonicalKeys,
Rekey,
},
data::formats::NamedTensorsApplied,
errors::BunsenResult,
kits::{
language::qwen3::weights::text_layer_rekey,
support::model_config::{
TransformersConfigJson,
TransformersDecoderSection,
},
},
};
pub fn audio_layer_rekey() -> Rekey {
CausalDecoderBlock::layer_rekey(&DecoderLayerKeys::llama(
"self_attn_layer_norm",
"final_layer_norm",
))
}
pub fn text_layer_rekey_voxtral() -> Rekey {
text_layer_rekey().rename("ada_rms_norm.", "mlp_scale.")
}
pub fn transformers_rekey() -> Rekey {
Rekey::drop_rest()
.mount(
"audio_tower.",
"audio_tower.",
Rekey::keep_rest()
.mount("embedder.conv", "stem.", ConvSeq1d::one_based_rekey())
.mount_indexed("layers.", "stack.layers.", audio_layer_rekey())
.rename("norm.", "stack.norm."),
)
.mount(
"multi_modal_projector.",
"multi_modal_projector.",
Rekey::keep_rest().rename("linear_1.", "stacking.projection."),
)
.mount(
"language_model.model.",
"language_model.lm.",
Rekey::keep_rest()
.mount_indexed("layers.", "stack.layers.", text_layer_rekey_voxtral())
.rename("embed_tokens.", "embedding.")
.rename("norm.", "stack.norm."),
)
}
pub fn load_transformers_tensors(
model: &mut VoxtralRealtime,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
let rekey = transformers_rekey().then(model.canonical_rekey());
rekey.load(model, tensors, None)
}
pub fn load_transformers_tensors_as(
model: &mut VoxtralRealtime,
tensors: impl IntoIterator<Item = PackTensor>,
dtype: burn::tensor::FloatDType,
) -> BunsenResult<NamedTensorsApplied> {
let rekey = transformers_rekey().then(model.canonical_rekey());
rekey.load(model, tensors, Some(dtype))
}
pub fn voxtral_config_from_transformers(json: &str) -> BunsenResult<VoxtralRealtimeConfig> {
let c = TransformersConfigJson::parse(json)?;
c.expect_str("model_type", "voxtral_realtime")?;
c.expect_str("projector_hidden_act", "gelu")?;
let a = c.section("audio_config")?;
a.expect_str("activation_function", "gelu")?;
let ad = TransformersDecoderSection::read(&a)?;
a.expect_eq("hidden_act", ad.hidden_act.as_str(), "silu")?;
a.expect_eq(
"num_key_value_heads",
ad.num_key_value_heads,
ad.num_attention_heads,
)?;
let audio = VoxtralAudioEncoderConfig::new(
a.usize("num_mel_bins")?,
ad.hidden_size,
ad.num_attention_heads,
ad.head_dim,
ad.intermediate_size,
ad.num_hidden_layers,
)
.with_rms_eps(ad.rms_norm_eps)
.with_rope_theta(ad.rope_theta as usize)
.with_sliding_window(ad.sliding_window);
let t = c.section("text_config")?;
let td = TransformersDecoderSection::read(&t)?;
t.expect_eq("hidden_act", td.hidden_act.as_str(), "silu")?;
t.expect_eq(
"tie_word_embeddings",
t.bool_or(
"tie_word_embeddings",
c.bool_or("tie_word_embeddings", true)?,
)?,
true,
)?;
let text = VoxtralTextConfig::new(
t.usize("vocab_size")?,
td.hidden_size,
td.num_hidden_layers,
td.num_attention_heads,
td.num_key_value_heads,
td.head_dim,
td.intermediate_size,
)
.with_rms_eps(td.rms_norm_eps)
.with_rope_theta(td.rope_theta as usize)
.with_sliding_window(td.sliding_window);
Ok(VoxtralRealtimeConfig::new(audio, text).with_downsample(c.usize("downsample_factor")?))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
blocks::transformers::models::language_decoder::LmDecoderMeta,
errors::{
BunsenErrorKind,
testing::ErrorMatcher,
},
kits::speech::asr::voxtral::blocks::{
VoxtralAudioEncoderMeta,
VoxtralRealtimeMeta,
},
};
#[test]
fn test_transformers_rekey() {
let rekey = transformers_rekey();
for (hf, bunsen) in [
(
"audio_tower.embedder.conv1.weight",
Some("audio_tower.stem.blocks.0.conv.weight"),
),
(
"audio_tower.embedder.conv2.bias",
Some("audio_tower.stem.blocks.1.conv.bias"),
),
(
"audio_tower.layers.31.self_attn_layer_norm.weight",
Some("audio_tower.stack.layers.31.attn_norm.weight"),
),
(
"audio_tower.layers.0.final_layer_norm.weight",
Some("audio_tower.stack.layers.0.mlp_norm.weight"),
),
(
"audio_tower.layers.0.self_attn.o_proj.bias",
Some("audio_tower.stack.layers.0.attn.c_proj.bias"),
),
(
"audio_tower.layers.0.mlp.down_proj.bias",
Some("audio_tower.stack.layers.0.mlp.down_proj.bias"),
),
(
"audio_tower.norm.weight",
Some("audio_tower.stack.norm.weight"),
),
(
"multi_modal_projector.linear_1.weight",
Some("multi_modal_projector.stacking.projection.weight"),
),
(
"multi_modal_projector.linear_2.weight",
Some("multi_modal_projector.linear_2.weight"),
),
(
"language_model.model.embed_tokens.weight",
Some("language_model.lm.embedding.weight"),
),
(
"language_model.model.layers.25.ada_rms_norm.linear1.weight",
Some("language_model.lm.stack.layers.25.mlp_scale.linear1.weight"),
),
(
"language_model.model.layers.3.self_attn.k_proj.weight",
Some("language_model.lm.stack.layers.3.attn.c_k.weight"),
),
(
"language_model.model.layers.3.post_attention_layernorm.weight",
Some("language_model.lm.stack.layers.3.mlp_norm.weight"),
),
(
"language_model.model.norm.weight",
Some("language_model.lm.stack.norm.weight"),
),
("lm_head.weight", None),
] {
assert_eq!(rekey.apply(hf).as_deref(), bunsen, "{hf}");
}
}
const CONFIG_JSON: &str = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/testdata/voxtral/voxtral-mini-4b-realtime-2602.config.json"
));
#[test]
fn test_config_from_transformers() {
let config = voxtral_config_from_transformers(CONFIG_JSON).unwrap();
let expected = VoxtralRealtimeConfig::voxtral_mini_4b_realtime();
assert_eq!(config.audio.n_layers(), expected.audio.n_layers());
assert_eq!(config.audio.d_model(), expected.audio.d_model());
assert_eq!(config.audio.head_dim(), expected.audio.head_dim());
assert_eq!(config.audio.d_ff, expected.audio.d_ff);
assert_eq!(config.audio.sliding_window(), Some(750));
assert_eq!(config.audio.rope_theta, 1_000_000);
assert_eq!(config.text.n_vocab(), expected.text.n_vocab());
assert_eq!(config.text.n_layers(), expected.text.n_layers());
assert_eq!(config.text.n_kv_heads(), expected.text.n_kv_heads());
assert_eq!(config.text.head_dim(), expected.text.head_dim());
assert_eq!(config.text.d_ff, expected.text.d_ff);
assert_eq!(config.text.sliding_window(), Some(8192));
assert_eq!(config.downsample(), 4);
assert_eq!(config.mel_frames_per_position(), 8);
let untied = CONFIG_JSON.replace(
"\"tie_word_embeddings\": true",
"\"tie_word_embeddings\": false",
);
ErrorMatcher::kind(BunsenErrorKind::Unsupported)
.message_contains("tie_word_embeddings")
.assert_err(&voxtral_config_from_transformers(&untied));
ErrorMatcher::kind(BunsenErrorKind::Unsupported)
.message_contains("model_type")
.assert_err(&voxtral_config_from_transformers(
&CONFIG_JSON.replace("\"voxtral_realtime\"", "\"voxtral\""),
));
}
}