use burn::{
config::Config,
module::Module,
nn::norm::RmsNormConfig,
tensor::FloatDType,
};
use crate::{
blocks::{
norm::{
BunsenNormalizationConfig,
GatedNormConfig,
},
transformers::{
attention::csa::{
CausalSelfAttentionConfig,
QkNormPlacement,
},
decoders::{
causal::CausalDecoderBlockConfig,
stack::PatternedStackConfig,
},
embedding::{
RotaryDirection,
RotaryEmbeddingConfig,
TokenEmbeddingConfig,
},
head::LmHeadConfig,
mixers::GatedDeltaNetConfig,
mlp::TransformerGatedMlpConfig,
models::language_decoder::{
LmDecoderHolder,
LmDecoderMeta,
LmDecoderModel,
LmDecoderVariantConfig,
LmDecoderWrapper,
LmHybridKind,
},
},
},
burner::module::ToStructureConfig,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
};
#[derive(Config, Copy, Debug, PartialEq, Eq)]
pub enum Qwen3_5LayerKind {
Linear,
Full,
}
#[derive(Config, Debug)]
pub struct Qwen3_5Config {
pub n_vocab: usize,
pub d_model: usize,
pub n_layers: usize,
pub n_heads: usize,
pub n_kv_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
pub linear_key_heads: usize,
pub linear_value_heads: usize,
pub linear_key_dim: usize,
pub linear_value_dim: usize,
#[config(default = "4")]
pub linear_conv_kernel: usize,
#[config(default = "1e-6")]
pub rms_eps: f64,
#[config(default = "10_000_000")]
pub rope_theta: usize,
#[config(default = "0.25")]
pub partial_rotary_factor: f64,
#[config(default = "true")]
pub attn_output_gate: bool,
#[config(default = "Vec::new()")]
pub layer_kinds: Vec<Qwen3_5LayerKind>,
#[config(default = "false")]
pub tied_head: bool,
#[config(default = "Some(FloatDType::F32)")]
pub softmax_dtype: Option<FloatDType>,
#[config(default = "Some(FloatDType::F32)")]
pub logits_dtype: Option<FloatDType>,
}
impl LmDecoderMeta for Qwen3_5Config {
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn n_kv_heads(&self) -> usize {
self.n_kv_heads
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl Qwen3_5Config {
fn published(
d_model: usize,
n_layers: usize,
heads: [usize; 2],
d_ff: usize,
linear_value_heads: usize,
tied_head: bool,
) -> Self {
let [n_heads, n_kv_heads] = heads;
Self::new(
248_320,
d_model,
n_layers,
n_heads,
n_kv_heads,
256,
d_ff,
16,
linear_value_heads,
128,
128,
)
.with_layer_kinds(Self::every_nth_full(n_layers, 4))
.with_tied_head(tied_head)
}
pub fn qwen3_5_0_8b() -> Self {
Self::published(1024, 24, [8, 2], 3584, 16, true)
}
pub fn qwen3_5_2b() -> Self {
Self::published(2048, 24, [8, 2], 6144, 16, true)
}
pub fn qwen3_5_4b() -> Self {
Self::published(2560, 32, [16, 4], 9216, 32, true)
}
pub fn qwen3_5_9b() -> Self {
Self::published(4096, 32, [16, 4], 12288, 32, false)
}
pub fn qwen3_5_27b() -> Self {
Self::published(5120, 64, [24, 4], 17408, 48, false)
}
pub fn every_nth_full(
n: usize,
interval: usize,
) -> Vec<Qwen3_5LayerKind> {
(0..n)
.map(|i| {
if (i + 1).is_multiple_of(interval) {
Qwen3_5LayerKind::Full
} else {
Qwen3_5LayerKind::Linear
}
})
.collect()
}
pub fn layer_kind(
&self,
i: usize,
) -> Qwen3_5LayerKind {
match self.layer_kinds.get(i) {
Some(kind) => *kind,
None if (i + 1).is_multiple_of(4) => Qwen3_5LayerKind::Full,
None => Qwen3_5LayerKind::Linear,
}
}
pub fn rotary_dim(&self) -> usize {
(self.head_dim as f64 * self.partial_rotary_factor) as usize
}
pub fn embedding(&self) -> TokenEmbeddingConfig {
TokenEmbeddingConfig::new(self.n_vocab, self.d_model)
}
pub fn head(&self) -> LmHeadConfig {
LmHeadConfig::new(self.d_model, self.n_vocab)
.with_tied(self.tied_head)
.with_logits_dtype(self.logits_dtype)
}
pub fn rotary(&self) -> RotaryEmbeddingConfig {
RotaryEmbeddingConfig::new(0, self.head_dim)
.with_base(self.rope_theta)
.with_rotary_dim(Some(self.rotary_dim()))
.with_direction(RotaryDirection::Forward)
}
fn layer(
&self,
kind: Qwen3_5LayerKind,
) -> CausalDecoderBlockConfig {
let norm =
|n| BunsenNormalizationConfig::Rms(RmsNormConfig::new(n).with_epsilon(self.rms_eps));
let mixer = match kind {
Qwen3_5LayerKind::Full => {
CausalSelfAttentionConfig::new(self.n_heads, self.n_kv_heads, self.d_model)
.with_head_dim(Some(self.head_dim))
.with_norm(Some(norm(self.head_dim)))
.with_qk_norm(QkNormPlacement::BeforeRope)
.with_output_gate(self.attn_output_gate)
.with_softmax_dtype(self.softmax_dtype)
.into()
}
Qwen3_5LayerKind::Linear => GatedDeltaNetConfig::new(
self.d_model,
self.linear_key_heads,
self.linear_value_heads,
self.linear_key_dim,
self.linear_value_dim,
)
.with_conv_kernel(self.linear_conv_kernel)
.with_norm(GatedNormConfig::rms(0, self.rms_eps))
.into(),
};
CausalDecoderBlockConfig::new(
mixer,
TransformerGatedMlpConfig::new(self.d_model, self.d_ff).into(),
)
.with_norm(norm(0))
}
pub fn stack(&self) -> BunsenResult<PatternedStackConfig> {
if !self.layer_kinds.is_empty() && self.layer_kinds.len() != self.n_layers {
return Err(ConstraintError::new(
"Qwen3_5Config",
"layer_kinds",
Rule::Relation {
lhs: ("layer_kinds.len".into(), self.layer_kinds.len().to_string()),
op: "==",
rhs: ("n_layers".into(), self.n_layers.to_string()),
},
)
.into());
}
let norm = BunsenNormalizationConfig::Rms(RmsNormConfig::new(0).with_epsilon(self.rms_eps));
Ok(PatternedStackConfig::new(
vec![
self.layer(Qwen3_5LayerKind::Full),
self.layer(Qwen3_5LayerKind::Linear),
],
(0..self.n_layers)
.map(|i| match self.layer_kind(i) {
Qwen3_5LayerKind::Full => 0,
Qwen3_5LayerKind::Linear => 1,
})
.collect(),
norm,
self.rotary(),
))
}
}
impl ToStructureConfig for Qwen3_5Config {
type Structure = LmDecoderVariantConfig;
fn try_to_structure(&self) -> BunsenResult<LmDecoderVariantConfig> {
Ok(LmDecoderVariantConfig::new(
self.embedding(),
self.head(),
self.stack()?.into(),
))
}
}
#[derive(Module, Debug)]
pub struct Qwen3_5Model {
pub lm: LmDecoderModel<LmHybridKind>,
}
impl LmDecoderHolder for Qwen3_5Model {
type Kind = LmHybridKind;
fn lm(&self) -> &LmDecoderModel<LmHybridKind> {
&self.lm
}
fn lm_mut(&mut self) -> &mut LmDecoderModel<LmHybridKind> {
&mut self.lm
}
fn from_lm(lm: LmDecoderModel<LmHybridKind>) -> Self {
Self { lm }
}
}
impl LmDecoderWrapper for Qwen3_5Model {}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
blocks::transformers::{
attention::csa::CausalSelfAttentionMeta,
decoders::stack::DecoderStackMeta,
mixers::{
GatedDeltaNetMeta,
TokenMixer,
},
models::language_decoder::LmDecoderOps,
},
burner::module::ModuleInit,
support::testing::cpu_device,
};
fn config() -> Qwen3_5Config {
Qwen3_5Config::new(40, 16, 4, 4, 2, 8, 24, 2, 4, 4, 3)
.with_partial_rotary_factor(0.5)
.with_layer_kinds(vec![
Qwen3_5LayerKind::Linear,
Qwen3_5LayerKind::Full,
Qwen3_5LayerKind::Linear,
Qwen3_5LayerKind::Full,
])
}
#[test]
fn test_config_and_meta() {
let device = cpu_device();
let config = config();
assert_eq!(config.rotary_dim(), 4);
let model: Qwen3_5Model = config.init(&device);
assert_eq!(model.n_vocab(), 40);
assert_eq!(model.d_model(), 16);
assert_eq!(model.n_layers(), 4);
assert_eq!(model.n_kv_heads(), 2);
assert_eq!(model.head_dim(), 8);
assert_eq!(
model.lm.stack.layer_attends(),
[false, true, false, true],
"the layer kinds"
);
match &model.lm.stack.layers[1].attn.branch {
TokenMixer::Attention(attn) => {
assert!(attn.output_gate && attn.q_norm.is_some());
assert_eq!(attn.n_head(), 4);
}
other => panic!("a full layer attends: {other:?}"),
}
match &model.lm.stack.layers[0].attn.branch {
TokenMixer::GatedDeltaNet(net) => {
assert_eq!(
(
net.n_key_heads(),
net.n_value_heads(),
net.key_dim(),
net.value_dim()
),
(2, 4, 4, 3)
);
}
other => panic!("a linear layer is a DeltaNet: {other:?}"),
}
assert_eq!(model.rotary(5, &device).tables[0].cos.dims(), [1, 5, 1, 2]);
let default = Qwen3_5Config::new(40, 16, 8, 4, 2, 8, 24, 2, 4, 4, 3);
assert_eq!(
(0..8).map(|i| default.layer_kind(i)).collect::<Vec<_>>(),
Qwen3_5Config::every_nth_full(8, 4),
"every fourth layer full by default"
);
assert!(
config
.with_layer_kinds(vec![Qwen3_5LayerKind::Full])
.stack()
.is_err()
);
}
}