use std::ops::Range;
use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Embedding,
EmbeddingConfig,
norm::{
NormalizationConfig,
RmsNorm,
RmsNormConfig,
},
},
prelude::{
Int,
s,
},
tensor::Device,
};
use crate::{
blocks::transformers::{
attention::csa::{
CausalSelfAttention,
CausalSelfAttentionConfig,
QkNormPlacement,
},
embedding::{
RotaryDirection,
RotaryEmbedding,
RotaryEmbeddingConfig,
},
mlp::{
GatedMlp,
GatedMlpConfig,
},
},
burner::module::ModuleInit,
errors::BunsenResult,
ops::{
embedding::unembed,
transformers::attention::{
KVCache,
KVCacheConfig,
},
},
};
pub trait Qwen3Meta {
fn n_vocab(&self) -> usize;
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
fn n_kv_heads(&self) -> usize;
fn head_dim(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct Qwen3Config {
pub n_vocab: usize,
pub d_model: usize,
pub n_layers: usize,
pub n_heads: usize,
pub n_kv_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
#[config(default = "1e-6")]
pub rms_eps: f64,
#[config(default = "1_000_000")]
pub rope_theta: usize,
}
impl Qwen3Config {
pub fn qwen3_0_6b() -> Self {
Self::new(151_936, 1024, 28, 16, 8, 128, 3072)
}
pub fn qwen3_1_7b() -> Self {
Self::new(151_936, 2048, 28, 16, 8, 128, 6144)
}
pub fn rotary(
&self,
seq_len: usize,
) -> RotaryEmbeddingConfig {
RotaryEmbeddingConfig::new(seq_len, self.head_dim)
.with_base(self.rope_theta)
.with_direction(RotaryDirection::Forward)
}
}
impl Qwen3Meta for Qwen3Config {
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn d_model(&self) -> usize {
self.d_model
}
fn n_layers(&self) -> usize {
self.n_layers
}
fn n_kv_heads(&self) -> usize {
self.n_kv_heads
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl ModuleInit<Qwen3Model> for Qwen3Config {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<Qwen3Model> {
let norm = |n| RmsNormConfig::new(n).with_epsilon(self.rms_eps);
let layers = (0..self.n_layers)
.map(|i| {
Ok(Qwen3Layer {
input_layernorm: norm(self.d_model).init(device),
self_attn: CausalSelfAttentionConfig::new(
self.n_heads,
self.n_kv_heads,
self.d_model,
)
.with_head_dim(Some(self.head_dim))
.with_norm(NormalizationConfig::Rms(norm(self.head_dim)))
.with_qk_norm(QkNormPlacement::BeforeRope)
.try_init(i, device)?,
post_attention_layernorm: norm(self.d_model).init(device),
mlp: GatedMlpConfig::new(self.d_model, self.d_ff).try_init(device)?,
})
})
.collect::<BunsenResult<_>>()?;
Ok(Qwen3Model {
embed_tokens: EmbeddingConfig::new(self.n_vocab, self.d_model).init(device),
layers,
norm: norm(self.d_model).init(device),
rope_theta: self.rope_theta,
})
}
}
#[derive(Module, Debug)]
pub struct Qwen3Layer {
pub input_layernorm: RmsNorm,
pub self_attn: CausalSelfAttention,
pub post_attention_layernorm: RmsNorm,
pub mlp: GatedMlp,
}
impl Qwen3Layer {
pub fn forward(
&self,
x: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
let x = x.clone()
+ self
.self_attn
.forward(self.input_layernorm.forward(x), r_emb, kv_cache);
x.clone() + self.mlp.forward(self.post_attention_layernorm.forward(x))
}
}
#[derive(Module, Debug)]
pub struct Qwen3Model {
pub embed_tokens: Embedding,
pub layers: Vec<Qwen3Layer>,
pub norm: RmsNorm,
pub rope_theta: usize,
}
impl Qwen3Meta for Qwen3Model {
fn n_vocab(&self) -> usize {
self.embed_tokens.weight.dims()[0]
}
fn d_model(&self) -> usize {
self.embed_tokens.weight.dims()[1]
}
fn n_layers(&self) -> usize {
self.layers.len()
}
fn n_kv_heads(&self) -> usize {
use crate::blocks::transformers::attention::csa::CausalSelfAttentionMeta;
self.layers[0].self_attn.n_kv_head()
}
fn head_dim(&self) -> usize {
self.layers[0].self_attn.head_dim
}
}
impl Qwen3Model {
pub fn rotary(
&self,
seq_len: usize,
device: &Device,
) -> RotaryEmbedding {
RotaryEmbeddingConfig::new(seq_len, self.head_dim())
.with_base(self.rope_theta)
.with_direction(RotaryDirection::Forward)
.init(device)
}
pub fn new_cache(
&self,
batch: usize,
seq_len: usize,
) -> KVCache {
KVCacheConfig::new(
batch,
self.n_kv_heads(),
seq_len,
self.head_dim(),
self.n_layers(),
)
.init()
}
pub fn embed(
&self,
tokens: Tensor<2, Int>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, seq] =
crate::contracts::unpack_shape_contract!(["batch", "seq"], &tokens.dims());
let y = self.embed_tokens.forward(tokens);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "d_model"],
&y.dims(),
&[("batch", batch), ("seq", seq), ("d_model", self.d_model())],
);
y
}
pub fn forward_hidden(
&self,
embeds: Tensor<3>,
r_emb: &RotaryEmbedding,
mut kv_cache: Option<&mut KVCache>,
) -> Tensor<3> {
let [batch, seq] = crate::contracts::unpack_shape_contract!(
["batch", "seq", "d_model"],
&embeds.dims(),
&["batch", "seq"],
&[("d_model", self.d_model())]
);
let start = kv_cache.as_ref().map(|c| c.pos()).unwrap_or(0);
let r_emb = r_emb.clip_range(start..start + seq);
let mut x = embeds;
for layer in &self.layers {
x = layer.forward(x, &r_emb, &mut kv_cache);
}
let y = self.norm.forward(x);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "d_model"],
&y.dims(),
&[("batch", batch), ("seq", seq), ("d_model", self.d_model())],
);
#[cfg(not(any(debug_assertions, test)))]
let _ = batch;
y
}
pub fn attention_weights(
&self,
embeds: Tensor<3>,
r_emb: &RotaryEmbedding,
layers: &[usize],
rows: Range<usize>,
cols: Range<usize>,
) -> Vec<Tensor<4>> {
let [_, seq] = crate::contracts::unpack_shape_contract!(
["batch", "seq", "d_model"],
&embeds.dims(),
&["batch", "seq"],
&[("d_model", self.d_model())]
);
assert!(
layers.iter().all(|&l| l < self.layers.len()),
"layers {layers:?} of {}",
self.layers.len()
);
let r_emb = r_emb.clip_range(0..seq);
let deepest = layers.iter().copied().max().map_or(0, |l| l + 1);
let mut weights: Vec<Option<Tensor<4>>> = vec![None; layers.len()];
let mut x = embeds;
for (index, layer) in self.layers.iter().enumerate().take(deepest) {
for (slot, _) in layers.iter().enumerate().filter(|&(_, &l)| l == index) {
let w = layer
.self_attn
.attention_weights(layer.input_layernorm.forward(x.clone()), &r_emb);
weights[slot] = Some(w.slice(s![.., .., rows.clone(), cols.clone()]));
}
x = layer.forward(x, &r_emb, &mut None);
}
weights
.into_iter()
.map(|w| w.expect("every layer visited"))
.collect()
}
pub fn logits(
&self,
hidden: Tensor<3>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, seq] = crate::contracts::unpack_shape_contract!(
["batch", "seq", "d_model"],
&hidden.dims(),
&["batch", "seq"],
&[("d_model", self.d_model())],
);
let y = unembed(&self.embed_tokens, hidden);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "n_vocab"],
&y.dims(),
&[("batch", batch), ("seq", seq), ("n_vocab", self.n_vocab())],
);
y
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_meta_and_cache() {
let config = Qwen3Config::new(40, 12, 2, 3, 1, 6, 20)
.with_rms_eps(1e-5)
.with_rope_theta(500);
let device = cpu_device();
let model: Qwen3Model = config.init(&device);
assert_eq!(model.n_vocab(), config.n_vocab());
assert_eq!(model.d_model(), config.d_model());
assert_eq!(model.n_layers(), config.n_layers());
assert_eq!(model.n_kv_heads(), config.n_kv_heads());
assert_eq!(model.head_dim(), config.head_dim());
assert_eq!(model.rope_theta, config.rope_theta);
assert_eq!(model.layers[0].self_attn.c_q.weight.dims(), [12, 18]);
let tokens = Tensor::<2, Int>::from_data([[1, 5, 7, 2, 9]], &device);
let embeds = model.embed(tokens);
let r_emb = model.rotary(8, &device);
let whole = model.forward_hidden(embeds.clone(), &r_emb, None);
let mut cache = model.new_cache(1, 8);
let head =
model.forward_hidden(embeds.clone().slice_dim(1, 0..3), &r_emb, Some(&mut cache));
let tail = model.forward_hidden(embeds.slice_dim(1, 3..5), &r_emb, Some(&mut cache));
let stepped = Tensor::cat(vec![head, tail], 1);
stepped.to_data_as::<f32>().assert_approx_eq::<f32>(
&whole.to_data_as::<f32>(),
burn::tensor::Tolerance::rel_abs(1e-4, 1e-5),
);
}
#[test]
fn test_attention_weights() {
let device = cpu_device();
let model: Qwen3Model = Qwen3Config::new(40, 12, 3, 3, 1, 6, 20).init(&device);
let embeds = model.embed(Tensor::<2, Int>::from_data([[1, 5, 7, 2, 9]], &device));
let r_emb = model.rotary(5, &device);
let weights = model.attention_weights(embeds.clone(), &r_emb, &[1, 0], 2..5, 0..3);
assert_eq!(weights.len(), 2);
assert_eq!(weights[0].dims(), [1, 3, 3, 3]);
let first = &model.layers[0];
let x0 = embeds.clone();
let expected0 = first
.self_attn
.attention_weights(first.input_layernorm.forward(x0.clone()), &r_emb)
.slice(s![.., .., 2..5, 0..3]);
let second = &model.layers[1];
let x1 = first.forward(x0, &r_emb, &mut None);
let expected1 = second
.self_attn
.attention_weights(second.input_layernorm.forward(x1), &r_emb)
.slice(s![.., .., 2..5, 0..3]);
crate::support::testing::assert_tensors_close(
&weights[0],
&expected1,
burn::tensor::Tolerance::default(),
);
crate::support::testing::assert_tensors_close(
&weights[1],
&expected0,
burn::tensor::Tolerance::default(),
);
}
}