use burn::store::burn_pack::Tensor as PackTensor;
use super::{
Qwen3Config,
Qwen3Model,
};
use crate::{
blocks::transformers::decoders::causal::{
CausalDecoderBlock,
DecoderLayerKeys,
},
burner::module::Rekey,
data::formats::NamedTensorsApplied,
errors::BunsenResult,
kits::support::{
construct::load_lm_rekeyed,
model_config::{
TransformersConfigJson,
TransformersDecoderSection,
},
},
};
pub fn text_layer_rekey() -> Rekey {
CausalDecoderBlock::layer_rekey(&DecoderLayerKeys::llama(
"input_layernorm",
"post_attention_layernorm",
))
}
pub fn transformers_rekey() -> Rekey {
Rekey::drop_rest()
.mount(
"model.",
"",
Rekey::keep_rest()
.mount_indexed("layers.", "stack.layers.", text_layer_rekey())
.rename("embed_tokens.", "embedding.")
.rename("norm.", "stack.norm."),
)
.exact("lm_head.weight", "head.proj.weight")
}
pub fn load_transformers_tensors(
model: &mut Qwen3Model,
tensors: impl IntoIterator<Item = PackTensor>,
) -> BunsenResult<NamedTensorsApplied> {
load_lm_rekeyed(model, tensors, transformers_rekey(), None)
}
pub fn load_transformers_tensors_as(
model: &mut Qwen3Model,
tensors: impl IntoIterator<Item = PackTensor>,
dtype: burn::tensor::FloatDType,
) -> BunsenResult<NamedTensorsApplied> {
load_lm_rekeyed(model, tensors, transformers_rekey(), Some(dtype))
}
pub fn qwen3_config_from_transformers(json: &str) -> BunsenResult<Qwen3Config> {
qwen3_config_from_transformers_section(&TransformersConfigJson::parse(json)?, false)
}
pub fn qwen3_config_from_transformers_section(
t: &TransformersConfigJson,
tie_default: bool,
) -> BunsenResult<Qwen3Config> {
t.expect_str("model_type", "qwen3")?;
let d = TransformersDecoderSection::read(t)?;
t.expect_eq("hidden_act", d.hidden_act.as_str(), "silu")?;
t.expect_eq("attention_bias", d.attention_bias, false)?;
t.expect_eq("sliding_window", d.sliding_window, None)?;
t.expect_eq("rope_parameters.rope_type", d.rope_type.as_str(), "default")?;
Ok(Qwen3Config::new(
t.usize("vocab_size")?,
d.hidden_size,
d.num_hidden_layers,
d.num_attention_heads,
d.num_key_value_heads,
d.head_dim,
d.intermediate_size,
)
.with_rms_eps(d.rms_norm_eps)
.with_rope_theta(d.rope_theta as usize)
.with_tied_head(t.bool_or("tie_word_embeddings", tie_default)?))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::errors::{
BunsenErrorKind,
testing::ErrorMatcher,
};
#[test]
fn test_text_layer_rekey() {
let rekey = text_layer_rekey();
for (hf, bunsen) in [
("input_layernorm.weight", "attn_norm.weight"),
("post_attention_layernorm.weight", "mlp_norm.weight"),
("self_attn.q_proj.weight", "attn.c_q.weight"),
("self_attn.k_proj.weight", "attn.c_k.weight"),
("self_attn.v_proj.weight", "attn.c_v.weight"),
("self_attn.o_proj.weight", "attn.c_proj.weight"),
("self_attn.q_norm.weight", "attn.q_norm.weight"),
("self_attn.k_norm.weight", "attn.k_norm.weight"),
("mlp.gate_proj.weight", "mlp.gate_proj.weight"),
("mlp.down_proj.weight", "mlp.down_proj.weight"),
] {
assert_eq!(rekey.apply(hf).as_deref(), Some(bunsen), "{hf}");
}
}
#[test]
fn test_transformers_rekey() {
let rekey = transformers_rekey();
for (hf, bunsen) in [
("model.embed_tokens.weight", Some("embedding.weight")),
(
"model.layers.0.self_attn.o_proj.weight",
Some("stack.layers.0.attn.c_proj.weight"),
),
(
"model.layers.27.self_attn.k_norm.weight",
Some("stack.layers.27.attn.k_norm.weight"),
),
(
"model.layers.3.mlp.up_proj.weight",
Some("stack.layers.3.mlp.up_proj.weight"),
),
("model.norm.weight", Some("stack.norm.weight")),
("lm_head.weight", Some("head.proj.weight")),
("lm_head.bias", None),
] {
assert_eq!(rekey.apply(hf).as_deref(), bunsen, "{hf}");
}
}
macro_rules! config_json {
($name:literal) => {
include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/testdata/qwen3/",
$name,
".config.json"
))
};
}
#[test]
fn test_qwen3_config_from_transformers() {
for (name, json, expected) in [
(
"qwen3-0.6b",
config_json!("qwen3-0.6b"),
Qwen3Config::qwen3_0_6b(),
),
(
"qwen3-0.6b-base",
config_json!("qwen3-0.6b-base"),
Qwen3Config::qwen3_0_6b(),
),
(
"qwen3-1.7b",
config_json!("qwen3-1.7b"),
Qwen3Config::qwen3_1_7b(),
),
(
"qwen3-1.7b-base",
config_json!("qwen3-1.7b-base"),
Qwen3Config::qwen3_1_7b(),
),
(
"qwen3-4b",
config_json!("qwen3-4b"),
Qwen3Config::qwen3_4b(),
),
(
"qwen3-4b-base",
config_json!("qwen3-4b-base"),
Qwen3Config::qwen3_4b(),
),
(
"qwen3-4b-instruct-2507",
config_json!("qwen3-4b-instruct-2507"),
Qwen3Config::qwen3_4b_2507(),
),
(
"qwen3-4b-thinking-2507",
config_json!("qwen3-4b-thinking-2507"),
Qwen3Config::qwen3_4b_2507(),
),
(
"qwen3-8b",
config_json!("qwen3-8b"),
Qwen3Config::qwen3_8b(),
),
(
"qwen3-8b-base",
config_json!("qwen3-8b-base"),
Qwen3Config::qwen3_8b(),
),
(
"qwen3-4b-saferl",
config_json!("qwen3-4b-saferl"),
Qwen3Config::qwen3_4b(),
),
(
"qwen3guard-gen-0.6b",
config_json!("qwen3guard-gen-0.6b"),
Qwen3Config::qwen3_0_6b(),
),
(
"qwen3guard-gen-4b",
config_json!("qwen3guard-gen-4b"),
Qwen3Config::qwen3_4b(),
),
(
"qwen3guard-gen-8b",
config_json!("qwen3guard-gen-8b"),
Qwen3Config::qwen3_8b(),
),
] {
let config = qwen3_config_from_transformers(json).unwrap();
assert_eq!(format!("{config:?}"), format!("{expected:?}"), "{name}");
}
let small = config_json!("qwen3-0.6b");
for (from, to, field) in [
(
"\"model_type\": \"qwen3\"",
"\"model_type\": \"qwen3_moe\"",
"model_type",
),
(
"\"rope_scaling\": null",
"\"rope_scaling\": {\"rope_type\": \"yarn\", \"factor\": 4.0}",
"rope_type",
),
] {
assert!(small.contains(from), "{from}");
ErrorMatcher::kind(BunsenErrorKind::Unsupported)
.message_contains(field)
.assert_err(&qwen3_config_from_transformers(&small.replace(from, to)));
}
let windowed = small.replace("\"sliding_window\": null", "\"sliding_window\": 4096");
assert!(
qwen3_config_from_transformers(&windowed).is_ok(),
"switched off"
);
ErrorMatcher::kind(BunsenErrorKind::Unsupported)
.message_contains("sliding_window")
.assert_err(&qwen3_config_from_transformers(&windowed.replace(
"\"use_sliding_window\": false",
"\"use_sliding_window\": true",
)));
let untied = small.replace(
"\"tie_word_embeddings\": true",
"\"tie_word_embeddings\": false",
);
let unsaid = small.replace("\"tie_word_embeddings\": true,", "");
for json in [untied, unsaid] {
let config = qwen3_config_from_transformers(&json).unwrap();
assert!(!config.tied_head);
assert_eq!(config.d_model, 1024, "the rest as it was");
}
}
}