use burn::{
Tensor,
module::Module,
nn::{
Embedding,
EmbeddingConfig,
Linear,
LinearConfig,
activation::ActivationConfig,
norm::{
Normalization,
NormalizationConfig,
RmsNormConfig,
},
},
prelude::{
Backend,
Config,
Int,
},
};
use crate::{
blocks::transformers::{
attention::csa::{
CausalSelfAttentionConfig,
CausalSelfAttentionMeta,
},
embedding::{
RotaryEmbedding,
RotaryEmbeddingConfig,
RotaryEmbeddingMeta,
},
mlp::MlpConfig,
},
burner::module::{
ModuleInit,
ToStructureConfig,
},
contracts::{
assert_shape_contract_periodically,
unpack_shape_contract,
},
errors::BunsenResult,
kits::gpts::nanochat::blocks::{
NanoChatGptBlock,
NanoChatGptBlockConfig,
},
ops::{
norm::{
RmsNormOptions,
rms_norm,
},
transformers::attention::{
KVCache,
KVCacheConfig,
},
},
};
pub trait NanoChatGptMeta {
fn n_embed(&self) -> usize;
fn n_head(&self) -> usize;
fn n_kv_head(&self) -> usize;
fn head_dim(&self) -> usize {
self.n_embed() / self.n_head()
}
fn init_seq_len(&self) -> usize;
fn max_seq_len(&self) -> usize;
fn n_layer(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct NanoChatGptContractConfig {
#[config(default = "1024")]
pub init_seq_len: usize,
#[config(default = "10")]
pub max_seq_len_factor: usize,
#[config(default = "50304")]
pub vocab_size: usize,
#[config(default = "12")]
pub n_layer: usize,
#[config(default = "6")]
pub n_head: usize,
#[config(default = "6")]
pub n_kv_head: usize,
#[config(default = "768")]
pub n_embed: usize,
#[config(default = "15.0")]
pub softcap: f64,
#[config(default = "4")]
pub expansion_factor: usize,
#[config(default = "ActivationConfig::Relu")]
pub activation: ActivationConfig,
#[config(default = "NormalizationConfig::Rms(RmsNormConfig::new(0))")]
pub norm: NormalizationConfig,
}
impl NanoChatGptMeta for NanoChatGptContractConfig {
fn n_embed(&self) -> usize {
self.n_embed
}
fn n_head(&self) -> usize {
self.n_head
}
fn n_kv_head(&self) -> usize {
self.n_kv_head
}
fn init_seq_len(&self) -> usize {
self.init_seq_len
}
fn max_seq_len(&self) -> usize {
self.init_seq_len() * self.max_seq_len_factor
}
fn n_layer(&self) -> usize {
self.n_layer
}
}
impl NanoChatGptContractConfig {
pub fn block_config(&self) -> NanoChatGptBlockConfig {
NanoChatGptBlockConfig::new(
CausalSelfAttentionConfig::new(self.n_head, self.n_kv_head, self.n_embed)
.with_norm(self.norm.clone()),
MlpConfig::new(self.n_embed)
.with_expansion_factor(self.expansion_factor)
.with_activation(self.activation.clone())
.with_act_exponent(Some(2.0)),
)
.with_norm(self.norm.clone())
}
}
impl ToStructureConfig for NanoChatGptContractConfig {
type Structure = NanoChatGptStructureConfig;
fn try_to_structure(&self) -> BunsenResult<NanoChatGptStructureConfig> {
let block_config = self.block_config();
Ok(NanoChatGptStructureConfig {
wte: EmbeddingConfig::new(self.vocab_size, self.n_embed),
h: (0..self.n_layer).map(|_| block_config.clone()).collect(),
lm_head: LinearConfig::new(self.n_embed, self.vocab_size),
r_emb: RotaryEmbeddingConfig::new(self.max_seq_len(), self.head_dim()),
norm: self.norm.clone(),
init_seq_len: self.init_seq_len,
softcap: self.softcap,
})
}
}
#[derive(Config, Debug)]
pub struct NanoChatGptStructureConfig {
pub wte: EmbeddingConfig,
pub h: Vec<NanoChatGptBlockConfig>,
pub lm_head: LinearConfig,
pub r_emb: RotaryEmbeddingConfig,
#[config(default = "1024")]
pub init_seq_len: usize,
#[config(default = "15.0")]
pub softcap: f64,
#[config(default = "NormalizationConfig::Rms(RmsNormConfig::new(0))")]
pub norm: NormalizationConfig,
}
impl NanoChatGptMeta for NanoChatGptStructureConfig {
fn n_embed(&self) -> usize {
self.wte.d_model
}
fn n_head(&self) -> usize {
self.h[0].attn.n_head()
}
fn n_kv_head(&self) -> usize {
self.h[0].attn.n_kv_head()
}
fn head_dim(&self) -> usize {
self.h[0].attn.head_dim()
}
fn init_seq_len(&self) -> usize {
self.init_seq_len
}
fn max_seq_len(&self) -> usize {
self.r_emb.seq_len()
}
fn n_layer(&self) -> usize {
self.h.len()
}
}
impl<B: Backend> ModuleInit<B, NanoChatGpt<B>> for NanoChatGptStructureConfig {
fn try_init(
&self,
device: &B::Device,
) -> BunsenResult<NanoChatGpt<B>> {
let n_embed = self.n_embed();
Ok(NanoChatGpt {
wte: self.wte.init(device),
h: self
.h
.iter()
.enumerate()
.map(|(layer_idx, c)| c.try_init(layer_idx, device))
.collect::<BunsenResult<Vec<NanoChatGptBlock<B>>>>()?,
h_norm: self.norm.clone().with_num_features(n_embed).init(device),
lm_head: self.lm_head.init(device),
r_emb: self.r_emb.try_init(device)?,
init_seq_len: self.init_seq_len,
softcap: self.softcap,
})
}
}
#[derive(Module, Debug)]
pub struct NanoChatGpt<B: Backend> {
wte: Embedding<B>,
h: Vec<NanoChatGptBlock<B>>,
h_norm: Normalization<B>,
lm_head: Linear<B>,
r_emb: RotaryEmbedding<B>,
init_seq_len: usize,
softcap: f64,
}
impl<B: Backend> NanoChatGptMeta for NanoChatGpt<B> {
fn n_embed(&self) -> usize {
self.wte.weight.dims()[1]
}
fn n_head(&self) -> usize {
self.h[0].attn.n_head()
}
fn n_kv_head(&self) -> usize {
self.h[0].attn.n_kv_head()
}
fn head_dim(&self) -> usize {
self.h[0].attn.head_dim()
}
fn init_seq_len(&self) -> usize {
self.init_seq_len
}
fn max_seq_len(&self) -> usize {
self.r_emb.seq_len()
}
fn n_layer(&self) -> usize {
self.h.len()
}
}
impl<B: Backend> NanoChatGpt<B> {
pub fn forward(
&self,
idx: Tensor<B, 2, Int>,
kv_cache: &mut Option<&mut KVCache<B>>,
) -> Tensor<B, 3> {
let [b, t] = unpack_shape_contract!(["B", "T"], &idx.dims());
let t0 = match kv_cache {
Some(kv_cache) => kv_cache.pos(),
None => 0,
};
assert!(
t0 + t <= self.r_emb.seq_len(),
"Sequence position grew beyond the rotary embeddings table: {t0} + {t} > {}",
self.r_emb.seq_len()
);
let r_emb = self.r_emb.clip_range(t0..t0 + t);
let mut x = rms_norm(self.wte.forward(idx), &RmsNormOptions::default());
for block in &self.h {
x = block.forward(x, &r_emb, kv_cache);
}
x = self.h_norm.forward(x);
let logits = self
.lm_head
.forward(x)
.div_scalar(self.softcap)
.tanh()
.mul_scalar(self.softcap);
assert_shape_contract_periodically!(
["B", "T", "V"],
&logits.dims(),
&[("B", b), ("T", t), ("V", self.wte.weight.dims()[0])]
);
logits
}
pub fn new_kv_cache(
&self,
batch_size: usize,
) -> KVCache<B> {
KVCacheConfig {
batch_size,
num_heads: self.n_kv_head(),
seq_len: self.init_seq_len(),
head_dim: self.head_dim(),
num_layers: self.n_layer(),
}
.init()
}
pub fn estimate_flops_per_token(&self) -> usize {
let nparams = self.num_params();
let nparams_embedding = self.wte.num_params();
let nparams = nparams - nparams_embedding;
let l = self.n_layer();
let h = self.n_head();
let q = self.head_dim();
let t = self.init_seq_len;
6 * nparams + 12 * l * h * q * t
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Tolerance,
},
};
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
PerformanceBackend,
default_device,
},
};
#[test]
fn test_gpt_config() {
let cfg = NanoChatGptContractConfig::new();
assert_eq!(cfg.init_seq_len, 1024);
assert_eq!(cfg.vocab_size, 50304);
assert_eq!(cfg.n_layer, 12);
assert_eq!(cfg.n_head, 6);
assert_eq!(cfg.n_kv_head, 6);
assert_eq!(cfg.n_embed, 768);
assert_eq!(cfg.expansion_factor, 4);
assert_eq!(cfg.n_embed(), 768);
}
#[test]
#[serial]
fn test_gpt_forward() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let batch_size = 1;
let seq_len = 100;
let n_layer = 4;
let n_embed = 36;
let vocab_size = 1000;
let cfg = NanoChatGptContractConfig::new()
.with_vocab_size(vocab_size)
.with_n_embed(n_embed)
.with_n_layer(n_layer);
let gpt: NanoChatGpt<B> = cfg.init(&device);
let mut kv_cache = gpt.new_kv_cache(batch_size);
let input_tokens = Tensor::<B, 2>::random(
[batch_size, seq_len],
Distribution::Uniform(0.0, vocab_size as f64),
&device,
)
.int();
let logits = gpt.forward(input_tokens, &mut Some(&mut kv_cache));
assert_shape_contract!(
["B", "T", "V"],
&logits.dims(),
&[("B", batch_size), ("T", seq_len), ("V", vocab_size)]
);
assert_eq!(gpt.n_embed(), n_embed);
}
fn assert_meta_agrees(
a: &impl NanoChatGptMeta,
b: &impl NanoChatGptMeta,
) {
assert_eq!(a.n_embed(), b.n_embed());
assert_eq!(a.n_head(), b.n_head());
assert_eq!(a.n_kv_head(), b.n_kv_head());
assert_eq!(a.head_dim(), b.head_dim());
assert_eq!(a.init_seq_len(), b.init_seq_len());
assert_eq!(a.max_seq_len(), b.max_seq_len());
assert_eq!(a.n_layer(), b.n_layer());
}
#[test]
#[serial]
fn test_policy_pathways_agree() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let policy = NanoChatGptContractConfig::new()
.with_init_seq_len(16)
.with_max_seq_len_factor(2)
.with_vocab_size(64)
.with_n_layer(2)
.with_n_head(4)
.with_n_kv_head(2)
.with_n_embed(32);
let structure = policy.to_structure();
assert_meta_agrees(&policy, &structure);
assert_eq!(structure.max_seq_len(), 32);
assert_eq!(structure.head_dim(), 8);
let lowered: NanoChatGpt<B> = structure.init(&device);
let direct: NanoChatGpt<B> = policy.init(&device);
assert_meta_agrees(&direct, &lowered);
assert_meta_agrees(&structure, &lowered);
assert_eq!(lowered.n_embed(), 32);
}
fn tiny_gpt<B: Backend>(device: &B::Device) -> NanoChatGpt<B> {
let gpt: NanoChatGpt<B> = NanoChatGptContractConfig::new()
.with_init_seq_len(4)
.with_max_seq_len_factor(2)
.with_vocab_size(16)
.with_n_layer(2)
.with_n_head(2)
.with_n_kv_head(2)
.with_n_embed(16)
.init(device);
assert_eq!(gpt.max_seq_len(), 8);
gpt
}
#[test]
#[serial]
fn test_cached_decode_fills_rotary_table() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let gpt = tiny_gpt::<B>(&device);
let mut cache = gpt.new_kv_cache(1);
gpt.forward(Tensor::zeros([1, 7], &device), &mut Some(&mut cache));
let logits = gpt.forward(Tensor::zeros([1, 1], &device), &mut Some(&mut cache));
assert_eq!(logits.dims(), [1, 1, 16]);
assert_eq!(cache.pos(), 8);
}
#[test]
#[serial]
#[should_panic(expected = "beyond the rotary embeddings table: 8 + 1 > 8")]
fn test_cached_decode_past_rotary_table_panics() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let gpt = tiny_gpt::<B>(&device);
let mut cache = gpt.new_kv_cache(1);
gpt.forward(Tensor::zeros([1, 8], &device), &mut Some(&mut cache));
gpt.forward(Tensor::zeros([1, 1], &device), &mut Some(&mut cache));
}
#[test]
#[serial]
fn test_forward_normalizes_the_embedding() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let mut gpt = tiny_gpt::<B>(&device);
let table: Tensor<B, 2> = Tensor::random([16, 16], Distribution::Normal(0.0, 1.0), &device);
let tokens: Tensor<B, 2, Int> = Tensor::from_data([[3, 1, 4, 1, 5, 9]], &device);
gpt.wte.weight = Param::from_tensor(table.clone());
let logits = gpt.forward(tokens.clone(), &mut None);
gpt.wte.weight = Param::from_tensor(table.mul_scalar(8.0));
let scaled = gpt.forward(tokens, &mut None);
scaled
.into_data()
.assert_approx_eq::<f32>(&logits.into_data(), Tolerance::rel_abs(1e-3, 1e-4));
}
}