use burn::{
Tensor,
config::Config,
module::Module,
nn::LayerNormConfig,
prelude::Int,
tensor::Device,
};
use crate::{
blocks::{
activation::BunsenActivationConfig,
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::{
AttentionMask,
CausalSelfAttentionConfig,
},
decoders::{
causal::CausalDecoderBlockConfig,
stack::{
DecoderStack,
DecoderStackMeta,
DecoderStackStructureConfig,
PatternedStackConfig,
},
},
embedding::{
RotaryDirection,
RotaryEmbeddingConfig,
TokenEmbedding,
TokenEmbeddingConfig,
TokenEmbeddingMeta,
},
mlp::TransformerGatedMlpConfig,
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
ToStructureConfig,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
},
};
pub trait ModernBertMeta {
fn n_vocab(&self) -> usize;
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
}
#[derive(Config, Copy, Debug, PartialEq, Eq)]
pub enum ModernBertLayerKind {
Global,
Local,
}
#[derive(Config, Debug)]
pub struct ModernBertConfig {
pub n_vocab: usize,
pub d_model: usize,
pub n_layers: usize,
pub n_heads: usize,
pub d_ff: usize,
#[config(default = "1e-5")]
pub norm_eps: f64,
#[config(default = "Vec::new()")]
pub layer_kinds: Vec<ModernBertLayerKind>,
#[config(default = "64")]
pub local_radius: usize,
#[config(default = "160000.0")]
pub global_rope_theta: f64,
#[config(default = "10000.0")]
pub local_rope_theta: f64,
}
impl ModernBertMeta for ModernBertConfig {
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
}
impl ModernBertConfig {
pub fn modernbert_base() -> Self {
Self::new(50368, 768, 22, 12, 1152)
}
pub fn modernbert_large() -> Self {
Self::new(50368, 1024, 28, 16, 2624)
}
pub fn layer_kind(
&self,
i: usize,
) -> ModernBertLayerKind {
match self.layer_kinds.get(i) {
Some(&kind) => kind,
None if self.layer_kinds.is_empty() && !i.is_multiple_of(3) => {
ModernBertLayerKind::Local
}
None => ModernBertLayerKind::Global,
}
}
fn norm(&self) -> BunsenNormalizationConfig {
BunsenNormalizationConfig::Layer(
LayerNormConfig::new(0)
.with_epsilon(self.norm_eps)
.with_bias(false),
)
}
pub fn embedding(&self) -> TokenEmbeddingConfig {
TokenEmbeddingConfig::new(self.n_vocab, self.d_model).with_norm(Some(self.norm()))
}
fn layer(
&self,
kind: ModernBertLayerKind,
attn_norm: bool,
) -> CausalDecoderBlockConfig {
let window = match kind {
ModernBertLayerKind::Global => None,
ModernBertLayerKind::Local => Some(self.local_radius + 1),
};
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(self.n_heads, self.n_heads, self.d_model)
.with_norm(None)
.with_mask(AttentionMask::Bidirectional)
.with_sliding_window(window)
.into(),
TransformerGatedMlpConfig::new(self.d_model, self.d_ff)
.with_activation(BunsenActivationConfig::Gelu)
.into(),
)
.with_norm(self.norm())
.with_attn_norm(attn_norm)
}
fn rotary(
&self,
theta: f64,
) -> RotaryEmbeddingConfig {
RotaryEmbeddingConfig::new(0, self.d_model / self.n_heads)
.with_base(theta as usize)
.with_direction(RotaryDirection::Forward)
}
pub fn stack(&self) -> BunsenResult<PatternedStackConfig> {
if !self.layer_kinds.is_empty() && self.layer_kinds.len() != self.n_layers {
return Err(ConstraintError::new(
"ModernBertConfig",
"layer_kinds",
Rule::Relation {
lhs: ("layer_kinds.len".into(), self.layer_kinds.len().to_string()),
op: "==",
rhs: ("n_layers".into(), self.n_layers.to_string()),
},
)
.into());
}
if self.n_heads == 0 || !self.d_model.is_multiple_of(self.n_heads) {
return Err(ConstraintError::new(
"ModernBertConfig",
"n_heads",
Rule::Relation {
lhs: ("n_heads".into(), self.n_heads.to_string()),
op: "divides",
rhs: ("d_model".into(), self.d_model.to_string()),
},
)
.into());
}
let first = self.layer_kind(0);
let kinds = vec![
self.layer(first, false),
self.layer(ModernBertLayerKind::Global, true),
self.layer(ModernBertLayerKind::Local, true),
];
let layer_kinds = (0..self.n_layers)
.map(|i| match (i, self.layer_kind(i)) {
(0, _) => 0,
(_, ModernBertLayerKind::Global) => 1,
(_, ModernBertLayerKind::Local) => 2,
})
.collect();
let local = Some(0);
Ok(PatternedStackConfig::new(
kinds,
layer_kinds,
self.norm(),
self.rotary(self.global_rope_theta),
)
.with_rotary_alts(vec![self.rotary(self.local_rope_theta)])
.with_kind_rotary(vec![
(first == ModernBertLayerKind::Local).then_some(0),
None,
local,
]))
}
}
impl ToStructureConfig for ModernBertConfig {
type Structure = ModernBertStructureConfig;
fn try_to_structure(&self) -> BunsenResult<ModernBertStructureConfig> {
Ok(ModernBertStructureConfig::new(
self.embedding(),
self.stack()?.try_to_structure()?,
))
}
}
#[derive(Config, Debug)]
pub struct ModernBertStructureConfig {
pub embedding: TokenEmbeddingConfig,
pub stack: DecoderStackStructureConfig,
}
impl ModuleInit<ModernBertModel> for ModernBertStructureConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<ModernBertModel> {
Ok(ModernBertModel {
embeddings: self.embedding.try_init(device)?,
stack: self.stack.try_init(device)?,
})
}
}
#[derive(Module, Debug)]
pub struct ModernBertModel {
pub embeddings: TokenEmbedding,
pub stack: DecoderStack,
}
impl ModernBertMeta for ModernBertModel {
fn n_vocab(&self) -> usize {
self.embeddings.n_vocab()
}
fn d_model(&self) -> usize {
self.embeddings.d_model()
}
fn n_layers(&self) -> usize {
self.stack.n_layers()
}
}
impl ModernBertModel {
pub fn forward(
&self,
ids: Tensor<2, Int>,
lengths: Option<&[usize]>,
) -> Tensor<3> {
self.stack
.encoder_forward(self.embeddings.forward(ids), lengths)
}
}
impl CanonicalKeys for ModernBertModel {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("stack.", &self.stack)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_layer_pattern() {
let config = ModernBertConfig::new(32, 16, 5, 2, 24).with_local_radius(3);
let kinds: Vec<_> = (0..5).map(|i| config.layer_kind(i)).collect();
use ModernBertLayerKind::*;
assert_eq!(kinds, [Global, Local, Local, Global, Local]);
let model: ModernBertModel = config.init(&cpu_device());
let windows = model.stack.layer_windows();
assert_eq!(windows, [None, Some(4), Some(4), None, Some(4)]);
let norms: Vec<bool> = model
.stack
.layers
.iter()
.map(|l| l.attn.pre.is_some())
.collect();
assert_eq!(norms, [false, true, true, true, true]);
assert_eq!(
model.stack.layer_rotary,
[None, Some(0), Some(0), None, Some(0)]
);
assert!(model.embeddings.norm.is_some());
}
#[test]
fn test_layer_kinds_must_cover_the_layers() {
let config = ModernBertConfig::new(32, 16, 3, 2, 24)
.with_layer_kinds(vec![ModernBertLayerKind::Global]);
assert!(config.stack().is_err());
assert!(
ModernBertConfig::new(32, 16, 3, 3, 24).stack().is_err(),
"3 heads over 16"
);
}
}