bunsen 0.42.0

bunsen is a batteries included common library for burn
Documentation
//! Loading `transformers`-named tensors into a [`VoxtralRealtime`].
use burn::store::burn_pack::Tensor as PackTensor;

use super::blocks::{
    VoxtralAudioEncoderConfig,
    VoxtralRealtime,
    VoxtralRealtimeConfig,
    VoxtralTextConfig,
};
use crate::{
    blocks::{
        conv::ConvSeq1d,
        transformers::decoders::causal::{
            CausalDecoderBlock,
            DecoderLayerKeys,
        },
    },
    burner::module::{
        CanonicalKeys,
        Rekey,
    },
    data::formats::NamedTensorsApplied,
    errors::BunsenResult,
    kits::{
        language::qwen3::weights::text_layer_rekey,
        support::model_config::{
            TransformersConfigJson,
            TransformersDecoderSection,
        },
    },
};

/// `transformers`' Voxtral Realtime audio-encoder layer names onto a
/// [`CausalDecoderBlock`]'s,
/// relative to it: `self_attn_layer_norm` the attention's norm,
/// `final_layer_norm` the feed-forward's, `self_attn`'s `{q,k,v,o}_proj` the
/// attention's projections, and the `SwiGLU`'s projections as they are.
pub fn audio_layer_rekey() -> Rekey {
    CausalDecoderBlock::layer_rekey(&DecoderLayerKeys::llama(
        "self_attn_layer_norm",
        "final_layer_norm",
    ))
}

/// `transformers`' Voxtral Realtime text-layer names onto a
/// [`CausalDecoderBlock`]'s:
/// Mistral's, which are Qwen3's ([`text_layer_rekey`]), and `ada_rms_norm`
/// the block's
/// [`mlp_scale`](crate::blocks::transformers::decoders::causal::CausalDecoderBlock::mlp_scale).
pub fn text_layer_rekey_voxtral() -> Rekey {
    text_layer_rekey().rename("ada_rms_norm.", "mlp_scale.")
}

/// The names of `mistralai/Voxtral-Mini-4B-Realtime-2602`'s
/// `model.safetensors` onto a [`VoxtralRealtime`]'s module paths: the audio
/// tower's `embedder.conv1` and `conv2` onto its stem's 0-based
/// [`ConvSeq1d`] blocks ([`ConvSeq1d::one_based_rekey`]), its layers by
/// [`audio_layer_rekey`]; the projector's `linear_1` onto its stacking's
/// projection; the text model, under `language_model.model.`, onto its
/// language model (`language_model.lm.`): its layers by
/// [`text_layer_rekey_voxtral`], its embedding and final norm; and everything
/// else as it is. The head is
/// tied, so nothing else is read.
///
/// Normalization parameters keep their `weight` spelling, which
/// [`Rekey::load`] resolves to `gamma`.
pub fn transformers_rekey() -> Rekey {
    Rekey::drop_rest()
        .mount(
            "audio_tower.",
            "audio_tower.",
            Rekey::keep_rest()
                .mount("embedder.conv", "stem.", ConvSeq1d::one_based_rekey())
                .mount_indexed("layers.", "stack.layers.", audio_layer_rekey())
                .rename("norm.", "stack.norm."),
        )
        .mount(
            "multi_modal_projector.",
            "multi_modal_projector.",
            Rekey::keep_rest().rename("linear_1.", "stacking.projection."),
        )
        .mount(
            "language_model.model.",
            "language_model.lm.",
            Rekey::keep_rest()
                .mount_indexed("layers.", "stack.layers.", text_layer_rekey_voxtral())
                .rename("embed_tokens.", "embedding.")
                .rename("norm.", "stack.norm."),
        )
}

/// Loads tensors named as `mistralai/Voxtral-Mini-4B-Realtime-2602`'s
/// `model.safetensors` names them into `model`: [`Rekey::load`] under
/// [`transformers_rekey`], in the dtype `model` was built in. The checkpoint
/// ships `bf16`, which not every backend computes in.
///
/// # Errors
/// As [`Rekey::load`].
pub fn load_transformers_tensors(
    model: &mut VoxtralRealtime,
    tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
    let rekey = transformers_rekey().then(model.canonical_rekey());
    rekey.load(model, tensors, None)
}

/// [`load_transformers_tensors`], every floating tensor placed in `dtype` (a
/// load policy's resolution) rather than the dtype the model was built in.
///
/// # Errors
/// As [`Rekey::load`].
pub fn load_transformers_tensors_as(
    model: &mut VoxtralRealtime,
    tensors: impl IntoIterator<Item = PackTensor>,
    dtype: burn::tensor::FloatDType,
) -> BunsenResult<NamedTensorsApplied> {
    let rekey = transformers_rekey().then(model.canonical_rekey());
    rekey.load(model, tensors, Some(dtype))
}

/// The [`VoxtralRealtimeConfig`] of a `transformers` `VoxtralRealtime`
/// `config.json`.
///
/// The audio encoder's and the text model's sizes, their windows, norm
/// epsilons and rotary bases, and the downsampling come from the file. What
/// bunsen's model fixes is checked: a GELU stem and a `SiLU` feed-forward in
/// both stacks, an audio encoder with as many key/value heads as query
/// heads, a projector with a GELU, and a head tied to the embedding.
///
/// # Errors
/// [`InvalidResource`](crate::errors::BunsenErrorKind::InvalidResource),
/// naming the field, when the file lacks one or does not parse;
/// [`Unsupported`](crate::errors::BunsenErrorKind::Unsupported), naming the
/// field and its value, for a model bunsen's Voxtral Realtime is not.
pub fn voxtral_config_from_transformers(json: &str) -> BunsenResult<VoxtralRealtimeConfig> {
    let c = TransformersConfigJson::parse(json)?;
    c.expect_str("model_type", "voxtral_realtime")?;
    c.expect_str("projector_hidden_act", "gelu")?;

    let a = c.section("audio_config")?;
    a.expect_str("activation_function", "gelu")?;
    let ad = TransformersDecoderSection::read(&a)?;
    a.expect_eq("hidden_act", ad.hidden_act.as_str(), "silu")?;
    a.expect_eq(
        "num_key_value_heads",
        ad.num_key_value_heads,
        ad.num_attention_heads,
    )?;
    let audio = VoxtralAudioEncoderConfig::new(
        a.usize("num_mel_bins")?,
        ad.hidden_size,
        ad.num_attention_heads,
        ad.head_dim,
        ad.intermediate_size,
        ad.num_hidden_layers,
    )
    .with_rms_eps(ad.rms_norm_eps)
    .with_rope_theta(ad.rope_theta as usize)
    .with_sliding_window(ad.sliding_window);

    let t = c.section("text_config")?;
    let td = TransformersDecoderSection::read(&t)?;
    t.expect_eq("hidden_act", td.hidden_act.as_str(), "silu")?;
    t.expect_eq(
        "tie_word_embeddings",
        t.bool_or(
            "tie_word_embeddings",
            c.bool_or("tie_word_embeddings", true)?,
        )?,
        true,
    )?;
    let text = VoxtralTextConfig::new(
        t.usize("vocab_size")?,
        td.hidden_size,
        td.num_hidden_layers,
        td.num_attention_heads,
        td.num_key_value_heads,
        td.head_dim,
        td.intermediate_size,
    )
    .with_rms_eps(td.rms_norm_eps)
    .with_rope_theta(td.rope_theta as usize)
    .with_sliding_window(td.sliding_window);

    Ok(VoxtralRealtimeConfig::new(audio, text).with_downsample(c.usize("downsample_factor")?))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{
        blocks::transformers::models::language_decoder::LmDecoderMeta,
        errors::{
            BunsenErrorKind,
            testing::ErrorMatcher,
        },
        kits::speech::asr::voxtral::blocks::{
            VoxtralAudioEncoderMeta,
            VoxtralRealtimeMeta,
        },
    };

    #[test]
    fn test_transformers_rekey() {
        let rekey = transformers_rekey();
        for (hf, bunsen) in [
            (
                "audio_tower.embedder.conv1.weight",
                Some("audio_tower.stem.blocks.0.conv.weight"),
            ),
            (
                "audio_tower.embedder.conv2.bias",
                Some("audio_tower.stem.blocks.1.conv.bias"),
            ),
            (
                "audio_tower.layers.31.self_attn_layer_norm.weight",
                Some("audio_tower.stack.layers.31.attn_norm.weight"),
            ),
            (
                "audio_tower.layers.0.final_layer_norm.weight",
                Some("audio_tower.stack.layers.0.mlp_norm.weight"),
            ),
            (
                "audio_tower.layers.0.self_attn.o_proj.bias",
                Some("audio_tower.stack.layers.0.attn.c_proj.bias"),
            ),
            (
                "audio_tower.layers.0.mlp.down_proj.bias",
                Some("audio_tower.stack.layers.0.mlp.down_proj.bias"),
            ),
            (
                "audio_tower.norm.weight",
                Some("audio_tower.stack.norm.weight"),
            ),
            (
                "multi_modal_projector.linear_1.weight",
                Some("multi_modal_projector.stacking.projection.weight"),
            ),
            (
                "multi_modal_projector.linear_2.weight",
                Some("multi_modal_projector.linear_2.weight"),
            ),
            (
                "language_model.model.embed_tokens.weight",
                Some("language_model.lm.embedding.weight"),
            ),
            (
                "language_model.model.layers.25.ada_rms_norm.linear1.weight",
                Some("language_model.lm.stack.layers.25.mlp_scale.linear1.weight"),
            ),
            (
                "language_model.model.layers.3.self_attn.k_proj.weight",
                Some("language_model.lm.stack.layers.3.attn.c_k.weight"),
            ),
            (
                "language_model.model.layers.3.post_attention_layernorm.weight",
                Some("language_model.lm.stack.layers.3.mlp_norm.weight"),
            ),
            (
                "language_model.model.norm.weight",
                Some("language_model.lm.stack.norm.weight"),
            ),
            ("lm_head.weight", None),
        ] {
            assert_eq!(rekey.apply(hf).as_deref(), bunsen, "{hf}");
        }
    }

    const CONFIG_JSON: &str = include_str!(concat!(
        env!("CARGO_MANIFEST_DIR"),
        "/testdata/voxtral/voxtral-mini-4b-realtime-2602.config.json"
    ));

    /// The published `config.json` is the published geometry.
    #[test]
    fn test_config_from_transformers() {
        let config = voxtral_config_from_transformers(CONFIG_JSON).unwrap();
        let expected = VoxtralRealtimeConfig::voxtral_mini_4b_realtime();
        assert_eq!(config.audio.n_layers(), expected.audio.n_layers());
        assert_eq!(config.audio.d_model(), expected.audio.d_model());
        assert_eq!(config.audio.head_dim(), expected.audio.head_dim());
        assert_eq!(config.audio.d_ff, expected.audio.d_ff);
        assert_eq!(config.audio.sliding_window(), Some(750));
        assert_eq!(config.audio.rope_theta, 1_000_000);
        assert_eq!(config.text.n_vocab(), expected.text.n_vocab());
        assert_eq!(config.text.n_layers(), expected.text.n_layers());
        assert_eq!(config.text.n_kv_heads(), expected.text.n_kv_heads());
        assert_eq!(config.text.head_dim(), expected.text.head_dim());
        assert_eq!(config.text.d_ff, expected.text.d_ff);
        assert_eq!(config.text.sliding_window(), Some(8192));
        assert_eq!(config.downsample(), 4);
        assert_eq!(config.mel_frames_per_position(), 8);

        let untied = CONFIG_JSON.replace(
            "\"tie_word_embeddings\": true",
            "\"tie_word_embeddings\": false",
        );
        ErrorMatcher::kind(BunsenErrorKind::Unsupported)
            .message_contains("tie_word_embeddings")
            .assert_err(&voxtral_config_from_transformers(&untied));
        ErrorMatcher::kind(BunsenErrorKind::Unsupported)
            .message_contains("model_type")
            .assert_err(&voxtral_config_from_transformers(
                &CONFIG_JSON.replace("\"voxtral_realtime\"", "\"voxtral\""),
            ));
    }
}