use burn::{
Tensor,
module::{
Module,
Param,
},
nn::{
Embedding,
EmbeddingConfig,
Initializer,
Linear,
LinearConfig,
norm::RmsNormConfig,
},
prelude::{
Config,
Int,
s,
},
tensor::{
Device,
FloatDType,
activation::sigmoid,
},
};
use crate::{
blocks::{
activation::BunsenActivationConfig,
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::{
CausalSelfAttentionConfig,
CausalSelfAttentionMeta,
ValueResidualConfig,
},
decoders::{
causal::CausalDecoderBlockConfig,
stack::{
DecoderStackMeta,
DecoderStackResidualMix,
DecoderStackSides,
DecoderStackStructureConfig,
},
},
embedding::{
RotaryEmbedding,
RotaryEmbeddingConfig,
RotaryEmbeddingMeta,
RotarySet,
TokenEmbeddingConfig,
TokenEmbeddingMeta,
},
head::{
LmHeadConfig,
LmHeadMeta,
},
mlp::TransformerMlpConfig,
models::language_decoder::{
LmDecoderModel,
LmDecoderStructureConfig,
},
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
ToStructureConfig,
},
contracts::{
assert_shape_contract_periodically,
unpack_shape_contract,
},
errors::{
BunsenError,
BunsenResult,
},
ops::transformers::attention::{
KVCache,
KVCacheConfig,
KVCacheModel,
},
};
fn kv_cache_config(meta: &impl NanoChatGptMeta) -> KVCacheConfig {
KVCacheConfig::for_decode(
meta.n_kv_head(),
meta.init_seq_len(),
meta.head_dim(),
meta.n_layer(),
)
.with_max_seq_len(Some(meta.max_seq_len()))
}
impl KVCacheModel for NanoChatGptContractConfig {
fn kv_cache_config(&self) -> KVCacheConfig {
kv_cache_config(self)
}
}
impl KVCacheModel for NanoChatGptStructureConfig {
fn kv_cache_config(&self) -> KVCacheConfig {
kv_cache_config(self)
}
}
impl KVCacheModel for NanoChatGpt {
fn kv_cache_config(&self) -> KVCacheConfig {
kv_cache_config(self)
}
}
pub trait NanoChatGptMeta {
fn n_embed(&self) -> usize;
fn n_head(&self) -> usize;
fn n_kv_head(&self) -> usize;
fn head_dim(&self) -> usize {
self.n_embed() / self.n_head()
}
fn init_seq_len(&self) -> usize;
fn max_seq_len(&self) -> usize;
fn n_layer(&self) -> usize;
fn vocab_size(&self) -> usize;
}
pub fn has_value_embed(
layer: usize,
n_layer: usize,
) -> bool {
layer % 2 == (n_layer - 1) % 2
}
pub fn window_pattern_windows(
pattern: &str,
seq_len: usize,
n_layer: usize,
) -> BunsenResult<Vec<usize>> {
let pattern = pattern.to_ascii_uppercase();
if pattern.is_empty() || !pattern.chars().all(|c| c == 'S' || c == 'L') {
return Err(BunsenError::illegal(format!(
"a window pattern is letters S and L; got {pattern:?}"
)));
}
let long = seq_len;
let short = seq_len.div_ceil(4).div_ceil(128) * 128;
let letters: Vec<char> = pattern.chars().collect();
let mut windows: Vec<usize> = (0..n_layer)
.map(|i| match letters[i % letters.len()] {
'S' => short + 1,
_ => long + 1,
})
.collect();
if let Some(last) = windows.last_mut() {
*last = long + 1;
}
Ok(windows)
}
#[derive(Config, Debug)]
pub struct NanoChatGptContractConfig {
#[config(default = "1024")]
pub init_seq_len: usize,
#[config(default = "10")]
pub max_seq_len_factor: usize,
#[config(default = "50304")]
pub vocab_size: usize,
#[config(default = "12")]
pub n_layer: usize,
#[config(default = "6")]
pub n_head: usize,
#[config(default = "6")]
pub n_kv_head: usize,
#[config(default = "768")]
pub n_embed: usize,
#[config(default = "15.0")]
pub softcap: f64,
#[config(default = "4")]
pub expansion_factor: usize,
#[config(default = "BunsenActivationConfig::Relu")]
pub activation: BunsenActivationConfig,
#[config(default = "BunsenNormalizationConfig::Rms(RmsNormConfig::new(0).with_epsilon(1e-6))")]
pub norm: BunsenNormalizationConfig,
#[config(default = "1e-6")]
pub norm_eps: f64,
#[config(default = "10000")]
pub rope_base: usize,
#[config(default = "true")]
pub rope_bf16_tables: bool,
#[config(default = "false")]
pub head_bias: bool,
#[config(default = "None")]
pub pad_vocab_to: Option<usize>,
#[config(default = "false")]
pub residual_lambdas: bool,
#[config(default = "false")]
pub value_embeds: bool,
#[config(default = "None")]
pub smear_channels: Option<usize>,
#[config(default = "false")]
pub backout: bool,
#[config(default = "None")]
pub qk_gain: Option<f64>,
#[config(default = "None")]
pub window_pattern: Option<String>,
}
impl NanoChatGptMeta for NanoChatGptContractConfig {
fn n_embed(&self) -> usize {
self.n_embed
}
fn n_head(&self) -> usize {
self.n_head
}
fn n_kv_head(&self) -> usize {
self.n_kv_head
}
fn init_seq_len(&self) -> usize {
self.init_seq_len
}
fn max_seq_len(&self) -> usize {
self.init_seq_len() * self.max_seq_len_factor
}
fn n_layer(&self) -> usize {
self.n_layer
}
fn vocab_size(&self) -> usize {
self.vocab_size
}
}
impl NanoChatGptContractConfig {
pub fn with_upstream_2026(
self,
window_pattern: impl Into<String>,
) -> Self {
self.with_rope_base(100000)
.with_rope_bf16_tables(false)
.with_pad_vocab_to(Some(64))
.with_residual_lambdas(true)
.with_value_embeds(true)
.with_smear_channels(Some(24))
.with_backout(true)
.with_qk_gain(Some(1.2))
.with_window_pattern(Some(window_pattern.into()))
}
pub fn padded_vocab_size(&self) -> usize {
match self.pad_vocab_to {
Some(m) if m > 0 => self.vocab_size.div_ceil(m) * m,
_ => self.vocab_size,
}
}
pub fn block_config(&self) -> CausalDecoderBlockConfig {
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(self.n_head, self.n_kv_head, self.n_embed)
.with_norm(Some(self.norm.clone()))
.with_score_scale(
self.qk_gain
.map(|g| g * g / (self.head_dim() as f64).sqrt()),
)
.into(),
TransformerMlpConfig::new(self.n_embed)
.with_expansion_factor(self.expansion_factor)
.with_activation(BunsenActivationConfig::Chain(vec![
self.activation.clone(),
BunsenActivationConfig::Pow(2.0),
]))
.into(),
)
.with_norm(self.norm.clone())
}
}
impl ToStructureConfig for NanoChatGptContractConfig {
type Structure = NanoChatGptStructureConfig;
fn try_to_structure(&self) -> BunsenResult<NanoChatGptStructureConfig> {
let windows = self
.window_pattern
.as_deref()
.map(|p| window_pattern_windows(p, self.init_seq_len, self.n_layer))
.transpose()?;
let h = (0..self.n_layer)
.map(|i| {
let mut block = self.block_config();
block.attn.attention_mut().sliding_window = windows.as_ref().map(|w| w[i]);
if self.value_embeds && has_value_embed(i, self.n_layer) {
block.attn.attention_mut().value_residual = Some(ValueResidualConfig::new());
}
block
})
.collect();
let vocab = self.padded_vocab_size();
let kv_width = self.n_kv_head * self.head_dim();
Ok(NanoChatGptStructureConfig {
lm: LmDecoderStructureConfig::new(
TokenEmbeddingConfig::new(vocab, self.n_embed).with_norm_eps(Some(self.norm_eps)),
LmHeadConfig::new(self.n_embed, vocab)
.with_tied(false)
.with_bias(self.head_bias)
.with_softcap(Some(self.softcap))
.with_vocab(Some(self.vocab_size)),
DecoderStackStructureConfig::new(
h,
self.norm.clone(),
RotaryEmbeddingConfig::new(self.max_seq_len(), self.head_dim())
.with_base(self.rope_base)
.with_table_dtype(self.rope_bf16_tables.then_some(FloatDType::BF16)),
),
)
.with_kv_cache_pre_size(self.init_seq_len),
residual_lambdas: self.residual_lambdas,
value_embeds: (0..self.n_layer)
.map(|i| {
(self.value_embeds && has_value_embed(i, self.n_layer))
.then(|| EmbeddingConfig::new(vocab, kv_width))
})
.collect(),
smear_channels: self.smear_channels,
backout: self.backout,
})
}
}
#[derive(Config, Debug)]
pub struct NanoChatGptStructureConfig {
pub lm: LmDecoderStructureConfig,
#[config(default = "false")]
pub residual_lambdas: bool,
#[config(default = "Vec::new()")]
pub value_embeds: Vec<Option<EmbeddingConfig>>,
#[config(default = "None")]
pub smear_channels: Option<usize>,
#[config(default = "false")]
pub backout: bool,
}
impl NanoChatGptMeta for NanoChatGptStructureConfig {
fn n_embed(&self) -> usize {
self.lm.embedding.d_model
}
fn n_head(&self) -> usize {
self.lm.stack.layers[0].attn.attention().n_head()
}
fn n_kv_head(&self) -> usize {
self.lm.stack.n_kv_heads()
}
fn head_dim(&self) -> usize {
self.lm.stack.head_dim()
}
fn init_seq_len(&self) -> usize {
self.lm.kv_cache_pre_size
}
fn max_seq_len(&self) -> usize {
self.lm.stack.rotary.seq_len()
}
fn n_layer(&self) -> usize {
self.lm.stack.n_layers()
}
fn vocab_size(&self) -> usize {
self.lm.head.n_vocab()
}
}
impl ModuleInit<NanoChatGpt> for NanoChatGptStructureConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<NanoChatGpt> {
let n_layer = self.lm.stack.layers.len();
if !self.value_embeds.is_empty() && self.value_embeds.len() != n_layer {
return Err(BunsenError::illegal(format!(
"value_embeds has {} entries for {n_layer} layers",
self.value_embeds.len()
)));
}
let ramp = |start: f64, drop: f64| {
let values: Vec<f32> = (0..n_layer)
.map(|i| (start - drop * i as f64 / (n_layer.max(2) - 1) as f64) as f32)
.collect();
Param::from_tensor(Tensor::<1>::from_floats(values.as_slice(), device))
};
let scalar = |v: f32| Param::from_tensor(Tensor::<1>::from_floats([v], device));
Ok(NanoChatGpt {
lm: self.lm.try_init(device)?,
r_emb: self.lm.stack.rotary.try_init(device)?,
resid_lambdas: self.residual_lambdas.then(|| ramp(1.15, 0.10)),
x0_lambdas: self.residual_lambdas.then(|| ramp(0.20, 0.15)),
value_embeds: self
.value_embeds
.iter()
.map(|c| c.as_ref().map(|c| c.init(device)))
.collect(),
smear_gate: self.smear_channels.map(|channels| {
LinearConfig::new(channels, 1)
.with_bias(false)
.with_initializer(Initializer::Uniform {
min: 0.0,
max: 0.02,
})
.init(device)
}),
smear_lambda: self.smear_channels.map(|_| scalar(0.0)),
backout_lambda: self.backout.then(|| scalar(0.2)),
})
}
}
#[derive(Module, Debug)]
pub struct NanoChatGpt {
lm: LmDecoderModel,
r_emb: RotaryEmbedding,
resid_lambdas: Option<Param<Tensor<1>>>,
x0_lambdas: Option<Param<Tensor<1>>>,
value_embeds: Vec<Option<Embedding>>,
smear_gate: Option<Linear>,
smear_lambda: Option<Param<Tensor<1>>>,
backout_lambda: Option<Param<Tensor<1>>>,
}
impl NanoChatGptMeta for NanoChatGpt {
fn n_embed(&self) -> usize {
self.lm.embedding.d_model()
}
fn n_head(&self) -> usize {
self.lm.stack.layers[0].attn.branch.attention().n_head()
}
fn n_kv_head(&self) -> usize {
self.lm.stack.n_kv_heads()
}
fn head_dim(&self) -> usize {
self.lm.stack.head_dim()
}
fn init_seq_len(&self) -> usize {
self.lm.kv_cache_pre_size()
}
fn max_seq_len(&self) -> usize {
self.r_emb.seq_len()
}
fn n_layer(&self) -> usize {
self.lm.stack.n_layers()
}
fn vocab_size(&self) -> usize {
self.lm.head.n_vocab()
}
}
impl NanoChatGpt {
pub fn float_dtype(&self) -> burn::tensor::DType {
self.lm.float_dtype()
}
#[cfg(test)]
pub(crate) fn set_smear_lambda_for_tests(
&mut self,
lambda: f32,
device: &Device,
) {
self.smear_lambda = Some(Param::from_tensor(Tensor::from_floats([lambda], device)));
}
pub fn is_smeared(&self) -> bool {
self.smear_gate.is_some()
}
pub fn forward(
&self,
idx: Tensor<2, Int>,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
self.forward_after(idx, None, kv_cache)
}
pub fn forward_after(
&self,
idx: Tensor<2, Int>,
prev: Option<Tensor<2, Int>>,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
self.logits(self.hidden_after(idx, prev, kv_cache))
}
pub fn hidden_after(
&self,
idx: Tensor<2, Int>,
prev: Option<Tensor<2, Int>>,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
let [b, t] = unpack_shape_contract!(["B", "T"], &idx.dims());
let t0 = match kv_cache {
Some(kv_cache) => kv_cache.pos(),
None => 0,
};
assert!(
t0 + t <= self.r_emb.seq_len(),
"Sequence position grew beyond the rotary embeddings table: {t0} + {t} > {}",
self.r_emb.seq_len()
);
let mut x = self.lm.embedding.forward(idx.clone());
if let (Some(gate), Some(lambda)) = (&self.smear_gate, &self.smear_lambda) {
x = self.smear(x, prev, t0, gate, lambda);
}
let sides = DecoderStackSides {
value_embeds: self
.value_embeds
.iter()
.map(|ve| ve.as_ref().map(|ve| ve.forward(idx.clone())))
.collect(),
mix: DecoderStackResidualMix {
resid_x0: self
.resid_lambdas
.as_ref()
.zip(self.x0_lambdas.as_ref())
.map(|(resid, x0)| (resid.val(), x0.val())),
backout: self
.backout_lambda
.as_ref()
.map(|lambda| (self.lm.stack.layers.len() / 2, lambda.val())),
},
..Default::default()
};
let hidden = self.lm.stack.forward_with(
x,
&RotarySet::from(self.r_emb.clone()),
kv_cache.as_deref_mut(),
sides,
);
assert_shape_contract_periodically!(
["B", "T", "D"],
&hidden.dims(),
&[("B", b), ("T", t), ("D", self.n_embed())]
);
hidden
}
pub fn logits(
&self,
hidden: Tensor<3>,
) -> Tensor<3> {
let [b, t] = unpack_shape_contract!(
["B", "T", "D"],
&hidden.dims(),
&["B", "T"],
&[("D", self.n_embed())]
);
let logits = self.lm.head.forward(hidden, &self.lm.embedding.weight);
assert_shape_contract_periodically!(
["B", "T", "V"],
&logits.dims(),
&[("B", b), ("T", t), ("V", self.vocab_size())]
);
logits
}
fn smear(
&self,
x: Tensor<3>,
prev: Option<Tensor<2, Int>>,
t0: usize,
gate: &Linear,
lambda: &Param<Tensor<1>>,
) -> Tensor<3> {
let [b, t, d] = x.dims();
let channels = gate.weight.dims()[0];
let g = sigmoid(gate.forward(x.clone().slice(s![.., .., ..channels])))
* lambda.val().unsqueeze::<3>();
let before = match (t0, prev) {
(_, Some(prev)) => {
let p = self.lm.embedding.forward(prev);
Some(Tensor::cat(vec![p, x.clone().slice(s![.., ..t - 1])], 1))
}
(0, None) => None,
(_, None) => panic!(
"a smeared model continuing a decode at position {t0} needs the token before it"
),
};
match before {
Some(before) => x + g * before,
None if t == 1 => x,
None => {
let rest = x.clone().slice(s![.., 1..])
+ g.slice(s![.., 1..]) * x.clone().slice(s![.., ..t - 1]);
let out = Tensor::cat(vec![x.slice(s![.., ..1]), rest], 1);
debug_assert_eq!(out.dims(), [b, t, d]);
out
}
}
}
pub fn estimate_flops_per_token(&self) -> usize {
let nparams = self.num_params();
let nparams_embedding = self.lm.embedding.num_params()
+ self
.value_embeds
.iter()
.flatten()
.map(|ve| ve.num_params())
.sum::<usize>();
let nparams = nparams - nparams_embedding;
let l = self.n_layer();
let h = self.n_head();
let q = self.head_dim();
let t = self.init_seq_len();
6 * nparams + 12 * l * h * q * t
}
}
impl crate::blocks::transformers::models::language_decoder::LmDecoderOps for NanoChatGpt {
fn float_dtype(&self) -> burn::tensor::DType {
NanoChatGpt::float_dtype(self)
}
fn rotary(
&self,
seq_len: usize,
device: &Device,
) -> RotarySet {
self.lm.rotary(seq_len, device)
}
fn embed(
&self,
tokens: Tensor<2, Int>,
) -> Tensor<3> {
self.lm.embed(tokens)
}
fn logits(
&self,
hidden: Tensor<3>,
) -> Tensor<3> {
NanoChatGpt::logits(self, hidden)
}
}
impl crate::blocks::transformers::models::language_decoder::LmDecoderMeta for NanoChatGpt {
fn n_vocab(&self) -> usize {
self.lm.n_vocab()
}
fn n_logits(&self) -> usize {
self.lm.n_logits()
}
fn d_model(&self) -> usize {
self.lm.d_model()
}
fn n_layers(&self) -> usize {
self.lm.n_layers()
}
fn n_kv_heads(&self) -> usize {
self.lm.n_kv_heads()
}
fn head_dim(&self) -> usize {
self.lm.head_dim()
}
}
impl CanonicalKeys for NanoChatGpt {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("lm.", &self.lm)
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Tolerance,
},
};
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
cpu_device,
performance_device,
seeded_tensor,
},
};
#[test]
fn test_gpt_config() {
let cfg = NanoChatGptContractConfig::new();
assert_eq!(cfg.init_seq_len, 1024);
assert_eq!(cfg.vocab_size, 50304);
assert_eq!(cfg.n_layer, 12);
assert_eq!(cfg.n_head, 6);
assert_eq!(cfg.n_kv_head, 6);
assert_eq!(cfg.n_embed, 768);
assert_eq!(cfg.expansion_factor, 4);
assert_eq!(cfg.n_embed(), 768);
}
#[test]
fn test_defaults_are_upstream_2025() {
let device = cpu_device();
let gpt: NanoChatGpt = NanoChatGptContractConfig::new()
.with_vocab_size(64)
.with_n_layer(2)
.with_n_head(2)
.with_n_kv_head(2)
.with_n_embed(16)
.with_init_seq_len(8)
.init(&device);
assert!(gpt.lm.head.proj.as_ref().is_some_and(|p| p.bias.is_none()));
assert_eq!(gpt.lm.embedding.norm_eps, Some(1e-6));
assert!(gpt.resid_lambdas.is_none() && gpt.x0_lambdas.is_none());
assert!(gpt.value_embeds.iter().all(Option::is_none));
assert!(!gpt.is_smeared() && gpt.backout_lambda.is_none());
assert!(gpt.lm.stack.layers.iter().all(|b| {
b.attn.branch.attention().score_scale.is_none()
&& b.attn.branch.attention().sliding_window.is_none()
&& b.attn.branch.attention().value_gate.is_none()
}));
let structure = NanoChatGptContractConfig::new().to_structure();
assert_eq!(structure.lm.stack.rotary.base, 10000);
assert_eq!(
structure.lm.stack.rotary.table_dtype,
Some(FloatDType::BF16)
);
assert_eq!(structure.vocab_size(), 50304);
}
#[test]
fn test_has_value_embed() {
let layers = |n: usize| {
(0..n)
.filter(|&i| has_value_embed(i, n))
.collect::<Vec<_>>()
};
assert_eq!(layers(18), vec![1, 3, 5, 7, 9, 11, 13, 15, 17]);
assert_eq!(layers(5), vec![0, 2, 4]);
assert_eq!(layers(1), vec![0]);
}
#[test]
fn test_window_pattern_windows() {
assert_eq!(
window_pattern_windows("SSSL", 2048, 6).unwrap(),
vec![513, 513, 513, 2049, 513, 2049]
);
assert_eq!(window_pattern_windows("l", 2048, 3).unwrap(), vec![2049; 3]);
assert_eq!(window_pattern_windows("S", 4, 2).unwrap(), vec![129, 5]);
assert!(window_pattern_windows("", 2048, 2).is_err());
assert!(window_pattern_windows("SXL", 2048, 2).is_err());
}
#[test]
fn test_upstream_2026_structure() {
let policy = NanoChatGptContractConfig::new()
.with_vocab_size(60)
.with_n_layer(3)
.with_n_head(2)
.with_n_kv_head(1)
.with_n_embed(32)
.with_init_seq_len(4)
.with_upstream_2026("SL");
assert_eq!(policy.padded_vocab_size(), 64);
let structure = policy.to_structure();
assert_eq!(structure.lm.stack.rotary.base, 100000);
assert_eq!(structure.lm.stack.rotary.table_dtype, None);
assert_eq!(structure.lm.embedding.n_vocab, 64);
assert_eq!(structure.vocab_size(), 60);
assert_eq!(
structure
.lm
.stack
.layers
.iter()
.map(|b| b.attn.attention().sliding_window)
.collect::<Vec<_>>(),
vec![Some(129), Some(5), Some(5)]
);
assert_eq!(
structure
.value_embeds
.iter()
.map(|ve| ve.as_ref().map(|c| (c.n_embedding, c.d_model)))
.collect::<Vec<_>>(),
vec![Some((64, 16)), None, Some((64, 16))]
);
assert!(
structure
.lm
.stack
.layers
.iter()
.all(|b| b.attn.attention().score_scale
== Some(1.2 * 1.2 / (structure.head_dim() as f64).sqrt()))
);
assert_eq!(structure.smear_channels, Some(24));
assert!(structure.residual_lambdas && structure.backout);
}
fn tiny_2026(device: &Device) -> NanoChatGpt {
let mut gpt: NanoChatGpt = NanoChatGptContractConfig::new()
.with_vocab_size(30)
.with_n_layer(3)
.with_n_head(4)
.with_n_kv_head(2)
.with_n_embed(32)
.with_init_seq_len(4)
.with_max_seq_len_factor(4)
.with_upstream_2026("SL")
.init(device);
gpt.smear_lambda = Some(Param::from_tensor(Tensor::from_floats([0.7], device)));
gpt.backout_lambda = Some(Param::from_tensor(Tensor::from_floats([0.3], device)));
for block in gpt.lm.stack.layers.iter_mut() {
if let Some(gate) = block.attn.branch.attention_mut().value_gate.as_mut() {
gate.weight = Param::from_tensor(seeded_tensor(
7,
gate.weight.dims(),
Distribution::Normal(0.0, 1.0),
device,
));
}
}
if let Some(gate) = gpt.smear_gate.as_mut() {
gate.weight = Param::from_tensor(seeded_tensor(
11,
gate.weight.dims(),
Distribution::Normal(0.0, 1.0),
device,
));
}
gpt
}
#[test]
#[serial]
fn test_upstream_2026_cached_decode_matches_full_pass() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_2026(&device);
assert!(gpt.is_smeared());
let ids: Tensor<2, Int> =
Tensor::from_data([[3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8]], &device);
let n = ids.dims()[1];
let full = gpt.forward(ids.clone(), &mut None);
assert_eq!(full.dims(), [1, n, 30]);
let prefill = 3;
let mut cache = gpt.new_kv_cache(1);
let mut steps =
vec![gpt.forward(ids.clone().slice(s![.., ..prefill]), &mut Some(&mut cache))];
for i in prefill..n {
steps.push(gpt.forward_after(
ids.clone().slice(s![.., i..i + 1]),
Some(ids.clone().slice(s![.., i - 1..i])),
&mut Some(&mut cache),
));
}
let cached = Tensor::cat(steps, 1);
cached
.into_data()
.assert_approx_eq::<f32>(&full.into_data(), Tolerance::rel_abs(1e-3, 1e-4));
}
#[test]
#[serial]
fn test_bpk_round_trip() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_2026(&device);
let ids: Tensor<2, Int> = Tensor::from_data([[3, 1, 4, 1, 5, 9]], &device);
let expected = gpt.forward(ids.clone(), &mut None);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("model.bpk");
gpt.save_file(&path).unwrap();
device.seed(99);
let fresh = tiny_2026(&device);
assert!(
!fresh
.forward(ids.clone(), &mut None)
.all_close(expected.clone(), None, None),
"a fresh model computes otherwise"
);
let loaded = fresh.load_file(&path);
loaded
.forward(ids, &mut None)
.into_data()
.assert_approx_eq::<f32>(&expected.into_data(), Tolerance::rel_abs(1e-5, 1e-6));
}
#[test]
#[serial]
fn test_upstream_2026_cached_chunk_matches_full_pass() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_2026(&device);
let ids: Tensor<2, Int> = Tensor::from_data([[7, 2, 9, 4, 4, 1, 0, 3]], &device);
let full = gpt.forward(ids.clone(), &mut None);
let mut cache = gpt.new_kv_cache(1);
let head = gpt.forward(ids.clone().slice(s![.., ..2]), &mut Some(&mut cache));
let tail = gpt.forward_after(
ids.clone().slice(s![.., 2..]),
Some(ids.clone().slice(s![.., 1..2])),
&mut Some(&mut cache),
);
Tensor::cat(vec![head, tail], 1)
.into_data()
.assert_approx_eq::<f32>(&full.into_data(), Tolerance::rel_abs(1e-3, 1e-4));
}
#[test]
#[serial]
#[should_panic(expected = "needs the token before it")]
fn test_smeared_continuation_needs_the_previous_token() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_2026(&device);
let mut cache = gpt.new_kv_cache(1);
gpt.forward(Tensor::zeros([1, 2], &device), &mut Some(&mut cache));
gpt.forward(Tensor::zeros([1, 1], &device), &mut Some(&mut cache));
}
#[test]
#[serial]
fn test_upstream_2026_options_bind() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let ids: Tensor<2, Int> = Tensor::from_data([[3, 1, 4, 1, 5, 9, 2, 6, 5, 3]], &device);
let gpt = tiny_2026(&device);
let base = gpt.forward(ids.clone(), &mut None);
let differs = |other: &NanoChatGpt, what: &str| {
let diff = (other.forward(ids.clone(), &mut None) - base.clone())
.abs()
.max()
.into_scalar::<f32>();
assert!(diff > 1e-4, "{what} changed nothing ({diff})");
};
let mut g = tiny_2026(&device);
g.smear_lambda = Some(Param::from_tensor(Tensor::from_floats([0.0], &device)));
differs(&g, "the smear");
let mut g = tiny_2026(&device);
g.backout_lambda = Some(Param::from_tensor(Tensor::from_floats([0.0], &device)));
differs(&g, "the backout");
let mut g = tiny_2026(&device);
g.x0_lambdas = Some(Param::from_tensor(Tensor::zeros([3], &device)));
differs(&g, "the x0 lambdas");
let mut g = tiny_2026(&device);
for block in g.lm.stack.layers.iter_mut() {
block.attn.branch.attention_mut().score_scale = None;
}
differs(&g, "the qk gain");
let mut g = tiny_2026(&device);
for block in g.lm.stack.layers.iter_mut() {
block.attn.branch.attention_mut().sliding_window = None;
}
differs(&g, "the windows");
let mut g = tiny_2026(&device);
for block in g.lm.stack.layers.iter_mut() {
if let Some(gate) = block.attn.branch.attention_mut().value_gate.as_mut() {
gate.weight = Param::from_tensor(gate.weight.val().zeros_like().sub_scalar(30.0));
}
}
differs(&g, "the value embeddings");
}
#[test]
#[serial]
fn test_gpt_forward() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let batch_size = 1;
let seq_len = 100;
let n_layer = 4;
let n_embed = 36;
let vocab_size = 1000;
let cfg = NanoChatGptContractConfig::new()
.with_vocab_size(vocab_size)
.with_n_embed(n_embed)
.with_n_layer(n_layer);
let gpt: NanoChatGpt = cfg.init(&device);
let mut kv_cache = gpt.new_kv_cache(batch_size);
let input_tokens = Tensor::<2>::random(
[batch_size, seq_len],
Distribution::Uniform(0.0, vocab_size as f64),
&device,
)
.int();
let logits = gpt.forward(input_tokens, &mut Some(&mut kv_cache));
assert_shape_contract!(
["B", "T", "V"],
&logits.dims(),
&[("B", batch_size), ("T", seq_len), ("V", vocab_size)]
);
assert_eq!(gpt.n_embed(), n_embed);
}
fn assert_meta_agrees(
a: &impl NanoChatGptMeta,
b: &impl NanoChatGptMeta,
) {
assert_eq!(a.n_embed(), b.n_embed());
assert_eq!(a.n_head(), b.n_head());
assert_eq!(a.n_kv_head(), b.n_kv_head());
assert_eq!(a.head_dim(), b.head_dim());
assert_eq!(a.init_seq_len(), b.init_seq_len());
assert_eq!(a.max_seq_len(), b.max_seq_len());
assert_eq!(a.n_layer(), b.n_layer());
}
#[test]
#[serial]
fn test_policy_pathways_agree() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let policy = NanoChatGptContractConfig::new()
.with_init_seq_len(16)
.with_max_seq_len_factor(2)
.with_vocab_size(64)
.with_n_layer(2)
.with_n_head(4)
.with_n_kv_head(2)
.with_n_embed(32);
let structure = policy.to_structure();
assert_meta_agrees(&policy, &structure);
assert_eq!(structure.max_seq_len(), 32);
assert_eq!(structure.head_dim(), 8);
let lowered: NanoChatGpt = structure.init(&device);
let direct: NanoChatGpt = policy.init(&device);
assert_meta_agrees(&direct, &lowered);
assert_meta_agrees(&structure, &lowered);
assert_eq!(lowered.n_embed(), 32);
}
fn tiny_gpt(device: &Device) -> NanoChatGpt {
let gpt: NanoChatGpt = NanoChatGptContractConfig::new()
.with_init_seq_len(4)
.with_max_seq_len_factor(2)
.with_vocab_size(16)
.with_n_layer(2)
.with_n_head(2)
.with_n_kv_head(2)
.with_n_embed(16)
.init(device);
assert_eq!(gpt.max_seq_len(), 8);
gpt
}
#[test]
#[serial]
fn test_cached_decode_fills_rotary_table() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_gpt(&device);
let mut cache = gpt.new_kv_cache(1);
gpt.forward(Tensor::zeros([1, 7], &device), &mut Some(&mut cache));
let logits = gpt.forward(Tensor::zeros([1, 1], &device), &mut Some(&mut cache));
assert_eq!(logits.dims(), [1, 1, 16]);
assert_eq!(cache.pos(), 8);
}
#[test]
#[serial]
#[should_panic(expected = "beyond the rotary embeddings table: 8 + 1 > 8")]
fn test_cached_decode_past_rotary_table_panics() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_gpt(&device);
let mut cache = gpt.new_kv_cache(1);
gpt.forward(Tensor::zeros([1, 8], &device), &mut Some(&mut cache));
gpt.forward(Tensor::zeros([1, 1], &device), &mut Some(&mut cache));
}
#[test]
#[serial]
fn test_forward_normalizes_the_embedding() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let mut gpt = tiny_gpt(&device);
let table: Tensor<2> = Tensor::random([16, 16], Distribution::Normal(0.0, 1.0), &device);
let tokens: Tensor<2, Int> = Tensor::from_data([[3, 1, 4, 1, 5, 9]], &device);
gpt.lm.embedding.weight = Param::from_tensor(table.clone());
let logits = gpt.forward(tokens.clone(), &mut None);
gpt.lm.embedding.weight = Param::from_tensor(table.mul_scalar(8.0));
let scaled = gpt.forward(tokens, &mut None);
scaled
.into_data()
.assert_approx_eq::<f32>(&logits.into_data(), Tolerance::rel_abs(1e-3, 1e-4));
}
#[test]
#[serial]
fn test_lm_decoder_ops() {
use crate::blocks::transformers::models::language_decoder::{
LmDecoderMeta,
LmDecoderOps,
};
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let gpt = tiny_2026(&device);
let ids: Tensor<2, Int> = Tensor::from_data([[3, 1, 4, 1, 5]], &device);
let hidden = gpt.hidden_after(ids.clone(), None, &mut None);
crate::support::testing::assert_tensors_close(
&LmDecoderOps::logits(&gpt, hidden),
&gpt.forward(ids.clone(), &mut None),
burn::tensor::Tolerance::default(),
);
crate::support::testing::assert_tensors_close(
&LmDecoderOps::embed(&gpt, ids.clone()),
&gpt.lm.embedding.forward(ids),
burn::tensor::Tolerance::default(),
);
assert_eq!(LmDecoderMeta::n_vocab(&gpt), 64, "the padded table");
assert_eq!(LmDecoderMeta::n_logits(&gpt), 30, "the head's slice");
assert_eq!(gpt.vocab_size(), 30);
assert_eq!(
LmDecoderOps::rotary(&gpt, 5, &device).tables[0].cos.dims()[1],
5
);
}
}