use burn::store::burn_pack::Tensor as PackTensor;
use super::{
ModernBertConfig,
ModernBertLayerKind,
ModernBertModel,
};
use crate::{
blocks::transformers::decoders::causal::{
CausalDecoderBlock,
DecoderLayerKeys,
},
burner::module::{
CanonicalKeys,
Rekey,
TensorLoadRowSplit,
},
data::formats::{
NamedTensorsApplied,
split_rows,
},
errors::{
BunsenError,
BunsenResult,
},
kits::support::model_config::TransformersConfigJson,
};
pub fn transformers_row_splits() -> [TensorLoadRowSplit; 2] {
[
TensorLoadRowSplit::equal(
".attn.Wqkv.weight",
[".attn.Wq.weight", ".attn.Wk.weight", ".attn.Wv.weight"],
),
TensorLoadRowSplit::equal(
".mlp.Wi.weight",
[".mlp.Wi_gate.weight", ".mlp.Wi_up.weight"],
),
]
}
pub fn transformers_layer_keys() -> DecoderLayerKeys {
DecoderLayerKeys {
attn: "attn".to_string(),
attn_projs: ["Wq", "Wk", "Wv", "Wo"].map(String::from),
..DecoderLayerKeys::llama("attn_norm", "mlp_norm")
}
}
pub fn transformers_rekey() -> Rekey {
let layer = CausalDecoderBlock::layer_rekey(&transformers_layer_keys())
.rename("mlp.Wi_gate.", "mlp.gate_proj.")
.rename("mlp.Wi_up.", "mlp.up_proj.")
.rename("mlp.Wo.", "mlp.down_proj.");
Rekey::drop_rest()
.rename("embeddings.tok_embeddings.", "embeddings.")
.rename("embeddings.norm.", "embeddings.norm.")
.mount_indexed("layers.", "stack.layers.", layer)
.rename("final_norm.", "stack.norm.")
}
pub fn transformers_prefix(tensors: &[PackTensor]) -> &'static str {
if tensors.iter().any(|t| t.name.starts_with("model.layers.")) {
"model."
} else {
""
}
}
pub fn load_transformers_tensors(
model: &mut ModernBertModel,
tensors: impl IntoIterator<Item = PackTensor>,
dtype: Option<burn::tensor::FloatDType>,
) -> BunsenResult<NamedTensorsApplied> {
let tensors: Vec<PackTensor> = tensors.into_iter().collect();
let prefix = transformers_prefix(&tensors);
let tensors = split_rows(tensors, &transformers_row_splits())?;
let rekey = Rekey::drop_rest().mount(prefix, "", transformers_rekey());
let rekey = rekey.then(model.canonical_rekey());
rekey.load(model, tensors, dtype)
}
pub fn modernbert_config_from_transformers(json: &str) -> BunsenResult<ModernBertConfig> {
let t = TransformersConfigJson::parse(json)?;
t.expect_str("model_type", "modernbert")?;
t.expect_eq(
"hidden_activation",
t.str_opt("hidden_activation")?.unwrap_or("gelu"),
"gelu",
)?;
for field in ["attention_bias", "mlp_bias", "norm_bias"] {
t.expect_eq(field, t.bool_or(field, false)?, false)?;
}
let n_layers = t.usize("num_hidden_layers")?;
let layer_kinds = match t.strs_opt("layer_types")? {
Some(types) => types
.iter()
.enumerate()
.map(|(i, kind)| match kind.as_str() {
"full_attention" => Ok(ModernBertLayerKind::Global),
"sliding_attention" => Ok(ModernBertLayerKind::Local),
other => Err(BunsenError::unsupported(format!(
"config.json: `layer_types[{i}]` is {other:?}"
))),
})
.collect::<BunsenResult<Vec<_>>>()?,
None => {
let every = t.usize_opt("global_attn_every_n_layers")?.unwrap_or(3);
(0..n_layers)
.map(|i| {
if i.is_multiple_of(every) {
ModernBertLayerKind::Global
} else {
ModernBertLayerKind::Local
}
})
.collect()
}
};
let (global_theta, local_theta) = match t.section_opt("rope_parameters")? {
Some(rope) => {
let theta = |kind: &str| -> BunsenResult<f64> {
let params = rope.section(kind)?;
params.expect_eq(
"rope_type",
params.str_opt("rope_type")?.unwrap_or("default"),
"default",
)?;
params.f64("rope_theta")
};
(theta("full_attention")?, theta("sliding_attention")?)
}
None => (
t.f64_opt("global_rope_theta")?.unwrap_or(160_000.0),
t.f64_opt("local_rope_theta")?.unwrap_or(10_000.0),
),
};
Ok(ModernBertConfig::new(
t.usize("vocab_size")?,
t.usize("hidden_size")?,
n_layers,
t.usize("num_attention_heads")?,
t.usize("intermediate_size")?,
)
.with_norm_eps(t.f64_opt("norm_eps")?.unwrap_or(1e-5))
.with_layer_kinds(layer_kinds)
.with_local_radius(t.usize_opt("local_attention")?.unwrap_or(128) / 2)
.with_global_rope_theta(global_theta)
.with_local_rope_theta(local_theta))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_transformers_rekey() {
let rekey = Rekey::drop_rest().mount("model.", "", transformers_rekey());
for (stored, bunsen) in [
(
"model.embeddings.tok_embeddings.weight",
"embeddings.weight",
),
("model.embeddings.norm.weight", "embeddings.norm.weight"),
(
"model.layers.1.attn_norm.weight",
"stack.layers.1.attn_norm.weight",
),
(
"model.layers.0.attn.Wq.weight",
"stack.layers.0.attn.c_q.weight",
),
(
"model.layers.0.attn.Wk.weight",
"stack.layers.0.attn.c_k.weight",
),
(
"model.layers.0.attn.Wv.weight",
"stack.layers.0.attn.c_v.weight",
),
(
"model.layers.0.attn.Wo.weight",
"stack.layers.0.attn.c_proj.weight",
),
(
"model.layers.0.mlp_norm.weight",
"stack.layers.0.mlp_norm.weight",
),
(
"model.layers.2.mlp.Wi_gate.weight",
"stack.layers.2.mlp.gate_proj.weight",
),
(
"model.layers.2.mlp.Wi_up.weight",
"stack.layers.2.mlp.up_proj.weight",
),
(
"model.layers.2.mlp.Wo.weight",
"stack.layers.2.mlp.down_proj.weight",
),
("model.final_norm.weight", "stack.norm.weight"),
] {
assert_eq!(rekey.apply(stored).as_deref(), Some(bunsen), "{stored}");
}
assert_eq!(rekey.apply("head.dense.weight"), None);
assert_eq!(rekey.apply("model.head.norm.weight"), None);
}
#[test]
fn test_config_from_transformers() {
let json = r#"{
"model_type": "modernbert", "vocab_size": 50368, "hidden_size": 1024,
"num_hidden_layers": 6, "num_attention_heads": 16, "intermediate_size": 2624,
"hidden_activation": "gelu", "norm_eps": 1e-05, "norm_bias": false,
"attention_bias": false, "mlp_bias": false, "local_attention": 128,
"layer_types": ["full_attention", "sliding_attention", "sliding_attention",
"full_attention", "sliding_attention", "sliding_attention"],
"rope_parameters": {
"full_attention": {"rope_theta": 160000.0, "rope_type": "default"},
"sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}
}
}"#;
let config = modernbert_config_from_transformers(json).unwrap();
assert_eq!(
(
config.n_vocab,
config.d_model,
config.n_layers,
config.n_heads,
config.d_ff
),
(50368, 1024, 6, 16, 2624)
);
assert_eq!(config.local_radius, 64);
assert_eq!(
(config.global_rope_theta, config.local_rope_theta),
(160000.0, 10000.0)
);
use ModernBertLayerKind::*;
assert_eq!(
config.layer_kinds,
[Global, Local, Local, Global, Local, Local]
);
let older = r#"{
"model_type": "modernbert", "vocab_size": 8, "hidden_size": 8,
"num_hidden_layers": 4, "num_attention_heads": 2, "intermediate_size": 12,
"global_attn_every_n_layers": 2, "global_rope_theta": 1000.0,
"local_rope_theta": 100.0, "local_attention": 8
}"#;
let config = modernbert_config_from_transformers(older).unwrap();
assert_eq!(config.layer_kinds, [Global, Local, Global, Local]);
assert_eq!(
(
config.global_rope_theta,
config.local_rope_theta,
config.local_radius
),
(1000.0, 100.0, 4)
);
let biased = json.replace(r#""mlp_bias": false"#, r#""mlp_bias": true"#);
assert!(modernbert_config_from_transformers(&biased).is_err());
}
}