use burn::{
config::Config,
module::Module,
nn::norm::RmsNormConfig,
tensor::FloatDType,
};
use crate::{
blocks::{
activation::BunsenActivationConfig,
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::{
CausalSelfAttentionConfig,
QkNormPlacement,
},
decoders::{
causal::CausalDecoderBlockConfig,
stack::PatternedStackConfig,
},
embedding::{
RopeScaling,
RotaryDirection,
RotaryEmbeddingConfig,
TokenEmbeddingConfig,
},
head::LmHeadConfig,
mlp::TransformerGatedMlpConfig,
models::language_decoder::{
LmAttentionOnlyKind,
LmDecoderHolder,
LmDecoderMeta,
LmDecoderModel,
LmDecoderVariantConfig,
LmDecoderWrapper,
},
},
},
burner::module::ToStructureConfig,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
};
pub const GEMMA_KV_CACHE_PRE_SIZE: usize = 1024;
#[derive(Config, Copy, Debug, PartialEq, Eq)]
pub enum GemmaLayerKind {
Sliding,
Full,
}
#[derive(Config, Debug)]
pub struct GemmaConfig {
pub n_vocab: usize,
pub d_model: usize,
pub n_layers: usize,
pub n_heads: usize,
pub n_kv_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
#[config(default = "1e-6")]
pub rms_eps: f64,
#[config(default = "None")]
pub query_pre_attn_scalar: Option<f64>,
#[config(default = "false")]
pub post_norms: bool,
#[config(default = "false")]
pub qk_norm: bool,
#[config(default = "Vec::new()")]
pub layer_kinds: Vec<GemmaLayerKind>,
#[config(default = "None")]
pub sliding_window: Option<usize>,
#[config(default = "10000.0")]
pub rope_theta: f64,
#[config(default = "RopeScaling::None")]
pub rope_scaling: RopeScaling,
#[config(default = "None")]
pub rope_local_theta: Option<f64>,
#[config(default = "None")]
pub attn_softcap: Option<f64>,
#[config(default = "None")]
pub final_softcap: Option<f64>,
#[config(default = "Some(FloatDType::F32)")]
pub softmax_dtype: Option<FloatDType>,
#[config(default = "Some(FloatDType::F32)")]
pub logits_dtype: Option<FloatDType>,
}
impl LmDecoderMeta for GemmaConfig {
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn n_kv_heads(&self) -> usize {
self.n_kv_heads
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl GemmaConfig {
pub fn every_nth_full(
n: usize,
full_every: usize,
) -> Vec<GemmaLayerKind> {
(0..n)
.map(|i| {
if (i + 1) % full_every == 0 {
GemmaLayerKind::Full
} else {
GemmaLayerKind::Sliding
}
})
.collect()
}
fn gemma3(
n_vocab: usize,
d_model: usize,
n_layers: usize,
heads: [usize; 3],
d_ff: usize,
window: usize,
) -> Self {
let [n_heads, n_kv_heads, head_dim] = heads;
Self::new(
n_vocab, d_model, n_layers, n_heads, n_kv_heads, head_dim, d_ff,
)
.with_post_norms(true)
.with_qk_norm(true)
.with_query_pre_attn_scalar(Some(256.0))
.with_layer_kinds(Self::every_nth_full(n_layers, 6))
.with_sliding_window(Some(window))
.with_rope_theta(1_000_000.0)
.with_rope_local_theta(Some(10_000.0))
}
fn gemma2(
d_model: usize,
n_layers: usize,
heads: [usize; 3],
d_ff: usize,
) -> Self {
let [n_heads, n_kv_heads, head_dim] = heads;
Self::new(
256_000, d_model, n_layers, n_heads, n_kv_heads, head_dim, d_ff,
)
.with_post_norms(true)
.with_query_pre_attn_scalar(Some(256.0))
.with_layer_kinds(Self::every_nth_full(n_layers, 2))
.with_sliding_window(Some(4096))
.with_attn_softcap(Some(50.0))
.with_final_softcap(Some(30.0))
}
pub fn gemma_3_270m() -> Self {
Self::gemma3(262_144, 640, 18, [4, 1, 256], 2048, 512)
}
pub fn gemma_3_1b() -> Self {
Self::gemma3(262_144, 1152, 26, [4, 1, 256], 6912, 512)
}
pub fn gemma_3_4b() -> Self {
Self::gemma3(262_208, 2560, 34, [8, 4, 256], 10240, 1024)
.with_rope_scaling(RopeScaling::Linear { factor: 8.0 })
}
pub fn gemma_2_2b() -> Self {
Self::gemma2(2304, 26, [8, 4, 256], 9216)
}
pub fn gemma_2_9b() -> Self {
Self::gemma2(3584, 42, [16, 8, 256], 14336)
}
pub fn gemma_2b() -> Self {
Self::new(256_000, 2048, 18, 8, 1, 256, 16384)
}
pub fn gemma_7b() -> Self {
Self::new(256_000, 3072, 28, 16, 16, 256, 24576)
}
pub fn layer_kind(
&self,
i: usize,
) -> GemmaLayerKind {
self.layer_kinds
.get(i)
.copied()
.unwrap_or(GemmaLayerKind::Full)
}
pub fn embedding(&self) -> TokenEmbeddingConfig {
TokenEmbeddingConfig::new(self.n_vocab, self.d_model)
.with_scale(Some((self.d_model as f64).sqrt()))
}
pub fn head(&self) -> LmHeadConfig {
LmHeadConfig::new(self.d_model, self.n_vocab)
.with_softcap(self.final_softcap)
.with_logits_dtype(self.logits_dtype)
}
fn layer(
&self,
kind: GemmaLayerKind,
) -> CausalDecoderBlockConfig {
let norm =
|n| BunsenNormalizationConfig::Rms(RmsNormConfig::new(n).with_epsilon(self.rms_eps));
let window = match kind {
GemmaLayerKind::Sliding => self.sliding_window,
GemmaLayerKind::Full => None,
};
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(self.n_heads, self.n_kv_heads, self.d_model)
.with_head_dim(Some(self.head_dim))
.with_norm(self.qk_norm.then(|| norm(self.head_dim)))
.with_qk_norm(QkNormPlacement::BeforeRope)
.with_sliding_window(window)
.with_score_scale(self.query_pre_attn_scalar.map(|s| s.powf(-0.5)))
.with_attn_softcap(self.attn_softcap)
.with_softmax_dtype(self.softmax_dtype)
.into(),
TransformerGatedMlpConfig::new(self.d_model, self.d_ff)
.with_activation(BunsenActivationConfig::GeluApproximate)
.into(),
)
.with_norm(norm(0))
.with_post_norms(self.post_norms)
}
fn rotary(
&self,
theta: f64,
scaling: RopeScaling,
) -> RotaryEmbeddingConfig {
RotaryEmbeddingConfig::new(0, self.head_dim)
.with_base(theta as usize)
.with_scaling(scaling)
.with_direction(RotaryDirection::Forward)
}
pub fn stack(&self) -> BunsenResult<PatternedStackConfig> {
if !self.layer_kinds.is_empty() && self.layer_kinds.len() != self.n_layers {
return Err(ConstraintError::new(
"GemmaConfig",
"layer_kinds",
Rule::Relation {
lhs: ("layer_kinds.len".into(), self.layer_kinds.len().to_string()),
op: "==",
rhs: ("n_layers".into(), self.n_layers.to_string()),
},
)
.into());
}
let sliding = self.layer_kinds.contains(&GemmaLayerKind::Sliding);
if sliding && self.sliding_window.is_none() {
return Err(ConstraintError::zero_or_empty("GemmaConfig", "sliding_window").into());
}
let global = self.rotary(self.rope_theta, self.rope_scaling);
let (alts, sliding_rotary) = match self.rope_local_theta {
Some(theta) => (vec![self.rotary(theta, RopeScaling::None)], Some(0)),
None => (Vec::new(), None),
};
let norm = BunsenNormalizationConfig::Rms(RmsNormConfig::new(0).with_epsilon(self.rms_eps));
Ok(PatternedStackConfig::new(
vec![
self.layer(GemmaLayerKind::Full),
self.layer(GemmaLayerKind::Sliding),
],
(0..self.n_layers)
.map(|i| match self.layer_kind(i) {
GemmaLayerKind::Full => 0,
GemmaLayerKind::Sliding => 1,
})
.collect(),
norm,
global,
)
.with_rotary_alts(alts)
.with_kind_rotary(vec![None, sliding_rotary]))
}
}
impl ToStructureConfig for GemmaConfig {
type Structure = LmDecoderVariantConfig;
fn try_to_structure(&self) -> BunsenResult<LmDecoderVariantConfig> {
Ok(
LmDecoderVariantConfig::new(self.embedding(), self.head(), self.stack()?.into())
.with_kv_cache_pre_size(GEMMA_KV_CACHE_PRE_SIZE),
)
}
}
#[derive(Module, Debug)]
pub struct GemmaModel {
pub lm: LmDecoderModel,
}
impl LmDecoderHolder for GemmaModel {
type Kind = LmAttentionOnlyKind;
fn lm(&self) -> &LmDecoderModel {
&self.lm
}
fn lm_mut(&mut self) -> &mut LmDecoderModel {
&mut self.lm
}
fn from_lm(lm: LmDecoderModel) -> Self {
Self { lm }
}
}
impl LmDecoderWrapper for GemmaModel {}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
burner::module::ModuleInit,
ops::transformers::attention::KVCacheModel,
support::testing::cpu_device,
};
fn config() -> GemmaConfig {
GemmaConfig::new(40, 12, 3, 2, 1, 8, 24)
.with_post_norms(true)
.with_qk_norm(true)
.with_layer_kinds(vec![
GemmaLayerKind::Sliding,
GemmaLayerKind::Full,
GemmaLayerKind::Sliding,
])
.with_sliding_window(Some(4))
.with_rope_theta(1_000_000.0)
.with_rope_scaling(RopeScaling::Linear { factor: 2.0 })
.with_rope_local_theta(Some(10_000.0))
.with_query_pre_attn_scalar(Some(16.0))
}
#[test]
fn test_config_and_meta() {
let config = config();
let stack = config.stack().unwrap();
assert_eq!(stack.layer_kinds, [1, 0, 1]);
assert_eq!(stack.kinds[0].attn.attention().sliding_window, None);
assert_eq!(stack.kinds[1].attn.attention().sliding_window, Some(4));
assert_eq!(stack.kinds[0].attn.attention().score_scale, Some(0.25));
assert!(stack.kinds[0].attn.attention().norm.is_some() && stack.kinds[0].post_norms);
assert_eq!(stack.rotary.base, 1_000_000);
assert_eq!(stack.rotary.scaling, RopeScaling::Linear { factor: 2.0 });
assert_eq!(stack.rotary_alts[0].base, 10_000);
assert_eq!(stack.kind_rotary, [None, Some(0)]);
let device = cpu_device();
let model: GemmaModel = config.init(&device);
assert_eq!(
(
model.n_vocab(),
model.d_model(),
model.n_layers(),
model.n_kv_heads(),
model.head_dim()
),
(40, 12, 3, 1, 8)
);
assert_eq!(model.lm.embedding.scale, Some(12f64.sqrt()));
assert!(model.lm.head.proj.is_none(), "tied");
assert_eq!(
model.kv_cache_config().layer_windows,
[Some(4), None, Some(4)]
);
}
#[test]
fn test_refusals() {
use crate::errors::{
BunsenErrorKind,
testing::ErrorMatcher,
};
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("layer_kinds")
.assert_err(
&config()
.with_layer_kinds(vec![GemmaLayerKind::Full])
.stack(),
);
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("sliding_window")
.assert_err(&config().with_sliding_window(None).stack());
ErrorMatcher::kind(BunsenErrorKind::Illegal)
.message_contains("sliding_window")
.assert_err(&config().with_sliding_window(None).try_to_structure());
}
}