use crate::attention::KvCache;
use crate::batch::TokenBatch;
use crate::causal_lm_loss::{TotalLoss, causal_lm_loss_batch};
use crate::embedding::Embedding;
use crate::matrix::Matrix;
use crate::moe::{DEFAULT_AUX_LOSS_WEIGHT, DEFAULT_ROUTER_Z_LOSS_WEIGHT, MoeConfig};
use crate::network::NetworkError;
use crate::norm::RmsNorm;
use crate::optimizers::Optimizer;
use crate::param::{Linear, Param};
use crate::rope::Rope;
use crate::transformer_block::{FeedForward, TransformerBlock, TransformerBlockCache};
use rand::{SeedableRng, rngs::StdRng};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::path::Path;
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct TransformerConfig {
pub vocab_size: usize,
pub d_model: usize,
pub n_layers: usize,
pub n_heads: usize,
pub n_kv_heads: usize,
pub head_dim: usize,
pub d_ff: usize,
pub moe_d_ff: usize,
pub num_experts: usize,
pub experts_per_token: usize,
pub moe_layers: Vec<usize>,
pub shared_expert: bool,
pub max_seq_len: usize,
pub rope_base: f32,
pub rmsnorm_eps: f32,
pub tie_embeddings: bool,
pub aux_loss_weight: f32,
pub router_z_loss_weight: f32,
}
impl Default for TransformerConfig {
fn default() -> Self {
Self {
vocab_size: 32_000,
d_model: 512,
n_layers: 8,
n_heads: 8,
n_kv_heads: 2,
head_dim: 64,
d_ff: 1408,
moe_d_ff: 352,
num_experts: 8,
experts_per_token: 2,
moe_layers: (2..8).collect(),
shared_expert: true,
max_seq_len: 2048,
rope_base: 10_000.0,
rmsnorm_eps: 1e-6,
tie_embeddings: true,
aux_loss_weight: DEFAULT_AUX_LOSS_WEIGHT,
router_z_loss_weight: DEFAULT_ROUTER_Z_LOSS_WEIGHT,
}
}
}
impl TransformerConfig {
pub fn validate(&self) -> Result<(), NetworkError> {
for (name, value) in [
("vocab_size", self.vocab_size),
("d_model", self.d_model),
("n_layers", self.n_layers),
("n_heads", self.n_heads),
("n_kv_heads", self.n_kv_heads),
("head_dim", self.head_dim),
("d_ff", self.d_ff),
("max_seq_len", self.max_seq_len),
] {
if value == 0 {
return Err(NetworkError::InvalidConfig(format!("{name} must be non-zero")));
}
}
if self.n_heads % self.n_kv_heads != 0 || self.n_kv_heads > self.n_heads {
return Err(NetworkError::InvalidConfig(format!(
"n_heads ({}) must be a multiple of n_kv_heads ({})",
self.n_heads, self.n_kv_heads
)));
}
if self.head_dim % 2 != 0 {
return Err(NetworkError::InvalidConfig(format!(
"head_dim must be even for rotary embeddings, got {}",
self.head_dim
)));
}
if let Some(&layer) = self.moe_layers.iter().find(|&&l| l >= self.n_layers) {
return Err(NetworkError::InvalidConfig(format!(
"moe_layers names layer {layer}, but the model has {} layers",
self.n_layers
)));
}
if !self.moe_layers.is_empty() {
self.moe_config().validate()?;
}
Ok(())
}
pub fn moe_config(&self) -> MoeConfig {
MoeConfig {
num_experts: self.num_experts,
experts_per_token: self.experts_per_token,
d_ff: self.moe_d_ff,
shared_expert: self.shared_expert,
aux_loss_weight: self.aux_loss_weight,
router_z_loss_weight: self.router_z_loss_weight,
}
}
pub fn is_moe_layer(&self, layer: usize) -> bool {
self.moe_layers.contains(&layer)
}
pub fn parameter_counts(&self) -> ParameterCounts {
let query = self.d_model * self.n_heads * self.head_dim;
let key_value = 2 * self.d_model * self.n_kv_heads * self.head_dim;
let attention = 2 * query + key_value;
let norms = 2 * self.d_model;
let dense_ffn = 3 * self.d_model * self.d_ff;
let expert = 3 * self.d_model * self.moe_d_ff;
let shared = if self.shared_expert { expert } else { 0 };
let router = self.num_experts * self.d_model;
let moe_total = router + self.num_experts * expert + shared;
let moe_active = router + self.experts_per_token * expert + shared;
let mut total = self.vocab_size * self.d_model + self.d_model;
if !self.tie_embeddings {
total += self.vocab_size * self.d_model;
}
let mut active = total;
for layer in 0..self.n_layers {
total += attention + norms;
active += attention + norms;
if self.is_moe_layer(layer) {
total += moe_total;
active += moe_active;
} else {
total += dense_ffn;
active += dense_ffn;
}
}
ParameterCounts { total, active }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ParameterCounts {
pub total: usize,
pub active: usize,
}
impl ParameterCounts {
pub fn sparsity_ratio(&self) -> f32 {
if self.active == 0 {
return 0.0;
}
self.total as f32 / self.active as f32
}
}
impl std::fmt::Display for ParameterCounts {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
formatter,
"{:.1}M total / {:.1}M active ({:.2}x)",
self.total as f64 / 1e6,
self.active as f64 / 1e6,
self.sparsity_ratio()
)
}
}
#[derive(Clone, Debug)]
pub struct TransformerBuilder {
config: TransformerConfig,
optimizer: Optimizer,
seed: Option<u64>,
mixed_precision: bool,
}
impl TransformerBuilder {
pub fn new() -> Self {
Self {
config: TransformerConfig::default(),
optimizer: Optimizer::adam(3e-4),
seed: None,
mixed_precision: false,
}
}
pub fn config(mut self, config: TransformerConfig) -> Self {
self.config = config;
self
}
pub fn vocab_size(mut self, vocab_size: usize) -> Self {
self.config.vocab_size = vocab_size;
self
}
pub fn d_model(mut self, d_model: usize) -> Self {
self.config.d_model = d_model;
self
}
pub fn n_layers(mut self, n_layers: usize) -> Self {
self.config.n_layers = n_layers;
self
}
pub fn heads(mut self, n_heads: usize, n_kv_heads: usize, head_dim: usize) -> Self {
self.config.n_heads = n_heads;
self.config.n_kv_heads = n_kv_heads;
self.config.head_dim = head_dim;
self
}
pub fn d_ff(mut self, d_ff: usize) -> Self {
self.config.d_ff = d_ff;
self
}
pub fn moe_d_ff(mut self, moe_d_ff: usize) -> Self {
self.config.moe_d_ff = moe_d_ff;
self
}
pub fn experts(mut self, num_experts: usize, experts_per_token: usize) -> Self {
self.config.num_experts = num_experts;
self.config.experts_per_token = experts_per_token;
self
}
pub fn moe_layers(mut self, layers: impl IntoIterator<Item = usize>) -> Self {
self.config.moe_layers = layers.into_iter().collect();
self
}
pub fn shared_expert(mut self, shared_expert: bool) -> Self {
self.config.shared_expert = shared_expert;
self
}
pub fn max_seq_len(mut self, max_seq_len: usize) -> Self {
self.config.max_seq_len = max_seq_len;
self
}
pub fn rope_base(mut self, rope_base: f32) -> Self {
self.config.rope_base = rope_base;
self
}
pub fn rmsnorm_eps(mut self, rmsnorm_eps: f32) -> Self {
self.config.rmsnorm_eps = rmsnorm_eps;
self
}
pub fn tie_embeddings(mut self, tie_embeddings: bool) -> Self {
self.config.tie_embeddings = tie_embeddings;
self
}
pub fn aux_loss_weight(mut self, weight: f32) -> Self {
self.config.aux_loss_weight = weight;
self
}
pub fn router_z_loss_weight(mut self, weight: f32) -> Self {
self.config.router_z_loss_weight = weight;
self
}
pub fn optimizer(mut self, optimizer: Optimizer) -> Self {
self.optimizer = optimizer;
self
}
pub fn seed(mut self, seed: u64) -> Self {
self.seed = Some(seed);
self
}
pub fn mixed_precision(mut self, mixed_precision: bool) -> Self {
self.mixed_precision = mixed_precision;
self
}
pub fn parameter_counts(&self) -> ParameterCounts {
self.config.parameter_counts()
}
pub fn build(self) -> Result<TransformerLm, NetworkError> {
TransformerLm::from_builder(self)
}
}
impl Default for TransformerBuilder {
fn default() -> Self {
Self::new()
}
}
#[derive(Clone, Debug)]
pub struct TransformerCache {
batch: TokenBatch,
blocks: Vec<TransformerBlockCache>,
final_input: Matrix,
final_output: Matrix,
auxiliary_loss: f32,
#[cfg(feature = "cuda")]
device: Option<std::sync::Arc<crate::gpu_model::GpuCache>>,
}
impl TransformerCache {
pub fn auxiliary_loss(&self) -> f32 {
self.auxiliary_loss
}
pub fn batch(&self) -> &TokenBatch {
&self.batch
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TransformerLm {
pub config: TransformerConfig,
pub embedding: Embedding,
pub blocks: Vec<TransformerBlock>,
pub final_norm: RmsNorm,
pub lm_head: Option<Linear>,
pub optimizer: Optimizer,
optimizer_step: usize,
#[cfg(feature = "cuda")]
#[serde(skip)]
device: Option<std::sync::Arc<crate::gpu_transformer::GpuContext>>,
#[serde(skip)]
#[cfg_attr(not(feature = "cuda"), allow(dead_code))]
mixed_precision: bool,
}
impl PartialEq for TransformerLm {
fn eq(&self, other: &Self) -> bool {
self.config == other.config
&& self.embedding == other.embedding
&& self.blocks == other.blocks
&& self.final_norm == other.final_norm
&& self.lm_head == other.lm_head
&& self.optimizer == other.optimizer
}
}
impl TransformerLm {
pub fn builder() -> TransformerBuilder {
TransformerBuilder::new()
}
fn from_builder(builder: TransformerBuilder) -> Result<Self, NetworkError> {
let config = builder.config;
config.validate()?;
let mut rng = builder
.seed
.map_or_else(StdRng::from_entropy, StdRng::seed_from_u64);
let rope = Rope::new(config.head_dim, config.max_seq_len, config.rope_base)?;
let embedding = Embedding::new(config.vocab_size, config.d_model, &mut rng);
let mut blocks = Vec::with_capacity(config.n_layers);
for layer in 0..config.n_layers {
let feed_forward = if config.is_moe_layer(layer) {
FeedForward::moe(config.d_model, config.moe_config(), &mut rng)?
} else {
FeedForward::swiglu(config.d_model, config.d_ff, &mut rng)
};
blocks.push(TransformerBlock::new(
config.d_model,
config.n_heads,
config.n_kv_heads,
config.head_dim,
rope.clone(),
feed_forward,
config.rmsnorm_eps,
&mut rng,
)?);
}
let lm_head = (!config.tie_embeddings)
.then(|| Linear::new(config.d_model, config.vocab_size, &mut rng));
Ok(Self {
final_norm: RmsNorm::new(config.d_model, config.rmsnorm_eps),
config,
embedding,
blocks,
lm_head,
optimizer: builder.optimizer,
optimizer_step: 0,
#[cfg(feature = "cuda")]
device: None,
mixed_precision: builder.mixed_precision,
})
}
pub fn parameter_counts(&self) -> ParameterCounts {
let head = self.lm_head.as_ref().map_or(0, |head| head.weight.len());
let base = self.embedding.weight.len() + self.final_norm.weight.len() + head;
ParameterCounts {
total: base
+ self
.blocks
.iter()
.map(TransformerBlock::num_parameters)
.sum::<usize>(),
active: base
+ self
.blocks
.iter()
.map(TransformerBlock::active_parameters)
.sum::<usize>(),
}
}
pub fn new_kv_caches(&self) -> Vec<KvCache> {
(0..self.config.n_layers)
.map(|_| {
KvCache::with_capacity(
self.config.n_kv_heads,
self.config.head_dim,
self.config.max_seq_len,
)
})
.collect()
}
pub fn forward_train<S: AsRef<[u32]>>(
&self,
sequences: &[S],
) -> Result<(Matrix, TransformerCache), NetworkError> {
self.forward_batch(&TokenBatch::new(sequences)?)
}
pub fn forward_batch(
&self,
batch: &TokenBatch,
) -> Result<(Matrix, TransformerCache), NetworkError> {
self.check_length(batch.seq_len(), 0)?;
#[cfg(feature = "cuda")]
if let Some(context) = &self.device {
let (logits, cache) = crate::gpu_model::forward(self, context, batch)?;
self.check_device()?;
return Ok((
logits,
TransformerCache {
batch: batch.clone(),
blocks: Vec::new(),
final_input: Matrix::new(0, 0),
final_output: Matrix::new(0, 0),
auxiliary_loss: cache.auxiliary_loss(),
device: Some(std::sync::Arc::new(cache)),
},
));
}
let layout = batch.layout();
let mut hidden = self.embedding.forward(batch.ids())?;
let mut blocks = Vec::with_capacity(self.blocks.len());
let mut auxiliary_loss = 0.0;
for block in &self.blocks {
let (output, cache) = block.forward_train(&hidden, layout)?;
auxiliary_loss += cache.auxiliary_loss();
blocks.push(cache);
hidden = output;
}
let final_output = self.final_norm.forward(&hidden);
let logits = match &self.lm_head {
Some(head) => head.forward(&final_output),
None => self.embedding.unembed(&final_output),
};
self.check_device()?;
Ok((
logits,
TransformerCache {
batch: batch.clone(),
blocks,
final_input: hidden,
final_output,
auxiliary_loss,
#[cfg(feature = "cuda")]
device: None,
},
))
}
pub fn forward_cached(
&self,
ids: &[u32],
caches: &mut [KvCache],
) -> Result<Matrix, NetworkError> {
if caches.len() != self.blocks.len() {
return Err(NetworkError::InvalidConfig(format!(
"expected one kv cache per layer ({}), got {}",
self.blocks.len(),
caches.len()
)));
}
self.check_length(ids.len(), caches.first().map_or(0, KvCache::len))?;
let mut hidden = self.embedding.forward(ids)?;
for (block, cache) in self.blocks.iter().zip(caches.iter_mut()) {
hidden = block.forward_cached(&hidden, cache)?;
}
let hidden = self.final_norm.forward(&hidden);
let logits = match &self.lm_head {
Some(head) => head.forward(&hidden),
None => self.embedding.unembed(&hidden),
};
self.check_device()?;
Ok(logits)
}
pub fn backward(
&mut self,
cache: &TransformerCache,
grad_logits: &Matrix,
) -> Result<(), NetworkError> {
#[cfg(feature = "cuda")]
if let Some(device) = &cache.device {
let context = self
.device
.clone()
.ok_or_else(|| NetworkError::Cuda("the model left the device mid-step".into()))?;
crate::gpu_model::backward(self, &context, device, grad_logits)?;
return self.check_device();
}
let mut grad_hidden = match &mut self.lm_head {
Some(head) => head.backward(&cache.final_output, grad_logits),
None => self
.embedding
.unembed_backward(&cache.final_output, grad_logits),
};
grad_hidden = self
.final_norm
.backward(&cache.final_input, &grad_hidden);
for (block, block_cache) in self.blocks.iter_mut().zip(&cache.blocks).rev() {
grad_hidden = block.backward(block_cache, &grad_hidden)?;
}
self.embedding.backward(cache.batch.ids(), &grad_hidden)?;
self.check_device()
}
#[cfg(feature = "cuda")]
pub fn to_cuda(&mut self, device: usize, memory_budget_mib: usize) -> Result<(), NetworkError> {
let counts = self.config.parameter_counts();
let estimated_mib = (counts.total * 4 * 4).div_ceil(1024 * 1024);
if memory_budget_mib > 0 && estimated_mib > memory_budget_mib {
return Err(NetworkError::CudaMemoryBudget {
estimated_mib,
budget_mib: memory_budget_mib,
});
}
let context =
crate::gpu_transformer::GpuContext::with_precision(device, self.mixed_precision)?;
self.embedding.weight.move_to_cuda(&context)?;
for block in &mut self.blocks {
for linear in block.linears_mut() {
linear.weight.move_to_cuda(&context)?;
}
}
if let Some(head) = &mut self.lm_head {
head.weight.move_to_cuda(&context)?;
}
self.device = Some(context);
Ok(())
}
#[cfg(feature = "cuda")]
pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
for param in self.params_mut() {
param.move_to_cpu()?;
}
if let Some(context) = self.device.take() {
context.check()?;
}
Ok(())
}
#[cfg(feature = "cuda")]
pub fn sync_from_device(&mut self) -> Result<(), NetworkError> {
for param in self.params_mut() {
param.sync_from_device()?;
}
self.check_device()
}
pub fn synchronize(&self) -> Result<(), NetworkError> {
#[cfg(feature = "cuda")]
if let Some(context) = &self.device {
return context.synchronize();
}
Ok(())
}
pub fn on_device(&self) -> bool {
#[cfg(feature = "cuda")]
{
self.device.is_some()
}
#[cfg(not(feature = "cuda"))]
{
false
}
}
fn check_device(&self) -> Result<(), NetworkError> {
#[cfg(feature = "cuda")]
if let Some(context) = &self.device {
context.check()?;
}
Ok(())
}
pub fn train_step<S: AsRef<[u32]>>(
&mut self,
sequences: &[S],
) -> Result<TotalLoss, NetworkError> {
self.train_step_batch(&TokenBatch::new(sequences)?)
}
pub fn train_step_batch(&mut self, batch: &TokenBatch) -> Result<TotalLoss, NetworkError> {
#[cfg(feature = "cuda")]
if let Some(context) = self.device.clone() {
self.check_length(batch.seq_len(), 0)?;
self.zero_grad();
let (lm_loss, auxiliary_loss) =
crate::gpu_model::train_step(self, &context, batch)?;
self.step(1.0);
self.check_device()?;
return Ok(TotalLoss {
lm_loss,
auxiliary_loss,
});
}
let (logits, cache) = self.forward_batch(batch)?;
let loss = causal_lm_loss_batch(&logits, batch)?;
self.zero_grad();
self.backward(&cache, &loss.grad_logits)?;
self.step(1.0);
self.check_device()?;
Ok(TotalLoss {
lm_loss: loss.loss,
auxiliary_loss: cache.auxiliary_loss(),
})
}
pub fn step(&mut self, scale: f32) {
self.optimizer_step += 1;
let step = self.optimizer_step;
let optimizer = self.optimizer.clone();
if self.on_device() {
for param in self.params_mut() {
param.step(&optimizer, step, scale);
}
return;
}
self.params_mut()
.par_iter_mut()
.for_each(|param| param.step(&optimizer, step, scale));
}
pub fn zero_grad(&mut self) {
if self.on_device() {
for param in self.params_mut() {
param.zero_grad();
}
return;
}
self.params_mut()
.par_iter_mut()
.for_each(|param| param.zero_grad());
}
pub fn optimizer_step(&self) -> usize {
self.optimizer_step
}
pub fn params_mut(&mut self) -> Vec<&mut Param> {
let mut params = self.embedding.params_mut();
for block in &mut self.blocks {
params.extend(block.params_mut());
}
params.extend(self.final_norm.params_mut());
if let Some(head) = &mut self.lm_head {
params.extend(head.params_mut());
}
params
}
pub fn save_json<P: AsRef<Path>>(&self, path: P) -> Result<(), NetworkError> {
let file = std::fs::File::create(path)?;
let mut writer = std::io::BufWriter::new(file);
serde_json::to_writer(&mut writer, self)?;
std::io::Write::flush(&mut writer)?;
Ok(())
}
pub fn load_json<P: AsRef<Path>>(path: P) -> Result<Self, NetworkError> {
let file = std::fs::File::open(path)?;
let model: Self = serde_json::from_reader(std::io::BufReader::new(file))?;
model.config.validate()?;
Ok(model)
}
fn check_length(&self, new_tokens: usize, already_cached: usize) -> Result<(), NetworkError> {
let length = new_tokens + already_cached;
if length > self.config.max_seq_len {
return Err(NetworkError::SequenceTooLong {
length,
max_seq_len: self.config.max_seq_len,
});
}
if new_tokens == 0 {
return Err(NetworkError::EmptyDataset);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::causal_lm_loss::causal_lm_loss;
fn tiny() -> TransformerBuilder {
TransformerLm::builder()
.vocab_size(24)
.d_model(16)
.n_layers(3)
.heads(4, 2, 4)
.d_ff(32)
.moe_d_ff(12)
.experts(4, 2)
.moe_layers([1, 2])
.shared_expert(true)
.max_seq_len(32)
.seed(1234)
}
#[test]
fn forward_produces_one_logit_row_per_token() {
let model = tiny().build().unwrap();
let (logits, _) = model.forward_train(&[[1, 2, 3, 4]]).unwrap();
assert_eq!((logits.rows, logits.cols), (4, 24));
}
#[test]
fn the_configured_layers_are_the_moe_layers() {
let model = tiny().build().unwrap();
assert!(!model.blocks[0].feed_forward.is_moe());
assert!(model.blocks[1].feed_forward.is_moe());
assert!(model.blocks[2].feed_forward.is_moe());
}
#[test]
fn module_and_config_parameter_counts_agree() {
for tied in [true, false] {
let builder = tiny().tie_embeddings(tied);
let predicted = builder.parameter_counts();
let model = builder.build().unwrap();
assert_eq!(model.parameter_counts(), predicted, "tied: {tied}");
}
}
#[test]
fn a_sparse_model_has_more_total_than_active_parameters() {
let counts = tiny().experts(16, 2).parameter_counts();
assert!(counts.total > counts.active);
assert!(counts.sparsity_ratio() > 1.0);
}
#[test]
fn a_fully_dense_model_has_no_inactive_parameters() {
let counts = tiny().moe_layers([]).parameter_counts();
assert_eq!(counts.total, counts.active);
assert!((counts.sparsity_ratio() - 1.0).abs() < 1e-6);
}
#[test]
fn the_default_configuration_lands_in_the_fifty_million_range() {
let counts = TransformerConfig::default().parameter_counts();
assert!((40e6..80e6).contains(&(counts.total as f64)), "{counts}");
assert!((20e6..60e6).contains(&(counts.active as f64)), "{counts}");
}
#[test]
fn tying_embeddings_removes_the_output_projection() {
let tied = tiny().tie_embeddings(true).build().unwrap();
let untied = tiny().tie_embeddings(false).build().unwrap();
assert!(tied.lm_head.is_none());
assert!(untied.lm_head.is_some());
assert_eq!(
untied.parameter_counts().total - tied.parameter_counts().total,
24 * 16
);
}
#[test]
fn a_cached_decode_reproduces_the_full_sequence_forward() {
let model = tiny().build().unwrap();
let ids = [5u32, 9, 2, 17, 3];
let (expected, _) = model.forward_train(&[&ids[..]]).unwrap();
let mut caches = model.new_kv_caches();
for (position, &id) in ids.iter().enumerate() {
let logits = model.forward_cached(&[id], &mut caches).unwrap();
for (actual, expected) in logits.data.iter().zip(expected.row(position)) {
assert!(
(actual - expected).abs() < 1e-3,
"position {position}: {actual} vs {expected}"
);
}
}
}
#[test]
fn a_prefill_then_decode_reproduces_the_full_sequence_forward() {
let model = tiny().build().unwrap();
let ids = [7u32, 1, 12, 4];
let (expected, _) = model.forward_train(&[&ids[..]]).unwrap();
let mut caches = model.new_kv_caches();
model.forward_cached(&ids[..2], &mut caches).unwrap();
let logits = model.forward_cached(&ids[2..3], &mut caches).unwrap();
for (actual, expected) in logits.data.iter().zip(expected.row(2)) {
assert!((actual - expected).abs() < 1e-3);
}
}
#[test]
fn end_to_end_backward_matches_finite_differences_on_the_embedding() {
let mut model = tiny()
.moe_layers([1])
.aux_loss_weight(0.0)
.router_z_loss_weight(0.0)
.build()
.unwrap();
let ids = [3u32, 8, 1, 5];
let (logits, cache) = model.forward_train(&[&ids[..]]).unwrap();
let loss = causal_lm_loss(&logits, &ids).unwrap();
model.zero_grad();
model.backward(&cache, &loss.grad_logits).unwrap();
let analytic = model.embedding.weight.grad.data.clone();
let epsilon = 1e-4;
let row = 8usize;
for column in 0..model.config.d_model {
let index = row * model.config.d_model + column;
let mut probe = model.clone();
probe.embedding.weight.value.data[index] += epsilon;
let high = causal_lm_loss(&probe.forward_train(&[&ids[..]]).unwrap().0, &ids)
.unwrap()
.loss;
probe.embedding.weight.value.data[index] -= 2.0 * epsilon;
let low = causal_lm_loss(&probe.forward_train(&[&ids[..]]).unwrap().0, &ids)
.unwrap()
.loss;
let numeric = (high - low) / (2.0 * epsilon);
assert!(
(analytic[index] - numeric).abs() < 1e-2,
"column {column}: {} vs {numeric}",
analytic[index]
);
}
}
#[test]
fn an_unused_embedding_row_gets_no_gradient_when_untied() {
let mut model = tiny().tie_embeddings(false).build().unwrap();
let ids = [2u32, 4, 6];
let (logits, cache) = model.forward_train(&[&ids[..]]).unwrap();
let loss = causal_lm_loss(&logits, &ids).unwrap();
model.zero_grad();
model.backward(&cache, &loss.grad_logits).unwrap();
assert!(model.embedding.weight.grad.row(23).iter().all(|&g| g == 0.0));
assert!(model.embedding.weight.grad.row(2).iter().any(|&g| g != 0.0));
}
#[test]
fn tied_embeddings_collect_gradients_from_both_ends() {
let mut model = tiny().tie_embeddings(true).build().unwrap();
let ids = [2u32, 4, 6];
let (logits, cache) = model.forward_train(&[&ids[..]]).unwrap();
let loss = causal_lm_loss(&logits, &ids).unwrap();
model.zero_grad();
model.backward(&cache, &loss.grad_logits).unwrap();
assert!(model.embedding.weight.grad.row(23).iter().any(|&g| g != 0.0));
}
#[test]
fn a_batched_forward_matches_one_forward_per_sequence() {
let model = tiny().build().unwrap();
let sequences: [&[u32]; 2] = [&[3, 8, 1, 5], &[7, 2]];
let (batched, _) = model.forward_train(&sequences).unwrap();
let seq_len = 4;
for (sequence, ids) in sequences.iter().enumerate() {
let (alone, _) = model.forward_train(&[*ids]).unwrap();
for position in 0..ids.len() {
let packed = batched.row((sequence * seq_len) + position);
for (left, right) in packed.iter().zip(alone.row(position)) {
assert!((left - right).abs() < 1e-4, "{left} vs {right}");
}
}
}
}
#[test]
fn a_padded_batch_loss_is_the_mean_over_every_predicted_position() {
let model = tiny()
.aux_loss_weight(0.0)
.router_z_loss_weight(0.0)
.build()
.unwrap();
let sequences: [&[u32]; 2] = [&[3, 8, 1, 5], &[7, 2]];
let batch = TokenBatch::new(&sequences).unwrap();
let (logits, _) = model.forward_batch(&batch).unwrap();
let batched = causal_lm_loss_batch(&logits, &batch).unwrap();
let mut total = 0.0;
for ids in sequences {
let (alone, _) = model.forward_train(&[ids]).unwrap();
total += causal_lm_loss(&alone, ids).unwrap().loss * (ids.len() - 1) as f32;
}
let expected = total / batch.predicted() as f32;
assert!(
(batched.loss - expected).abs() < 1e-4,
"{} vs {expected}",
batched.loss
);
}
#[test]
fn a_batched_gradient_is_the_mean_of_the_per_sequence_gradients() {
let mut model = tiny()
.aux_loss_weight(0.0)
.router_z_loss_weight(0.0)
.build()
.unwrap();
let sequences: [&[u32]; 2] = [&[3, 8, 1, 5], &[7, 2, 9, 4]];
let batch = TokenBatch::new(&sequences).unwrap();
let (logits, cache) = model.forward_batch(&batch).unwrap();
let loss = causal_lm_loss_batch(&logits, &batch).unwrap();
model.zero_grad();
model.backward(&cache, &loss.grad_logits).unwrap();
let batched = model.embedding.weight.grad.data.clone();
let mut summed = vec![0.0f32; batched.len()];
for ids in sequences {
let (alone, cache) = model.forward_train(&[ids]).unwrap();
let loss = causal_lm_loss(&alone, ids).unwrap();
model.zero_grad();
model.backward(&cache, &loss.grad_logits).unwrap();
for (total, one) in summed.iter_mut().zip(&model.embedding.weight.grad.data) {
*total += one / sequences.len() as f32;
}
}
for (left, right) in batched.iter().zip(&summed) {
assert!((left - right).abs() < 1e-5, "{left} vs {right}");
}
}
#[test]
fn a_training_step_lowers_the_loss_on_a_repeated_sequence() {
let mut model = tiny()
.optimizer(crate::optimizers::Optimizer::adam(1e-2))
.build()
.unwrap();
let ids = [1u32, 2, 3, 4, 5, 6];
let first = model.train_step(&[&ids[..]]).unwrap();
let mut last = first;
for _ in 0..20 {
last = model.train_step(&[&ids[..]]).unwrap();
}
assert!(
last.lm_loss < first.lm_loss,
"loss went from {} to {}",
first.lm_loss,
last.lm_loss
);
assert!(last.total() >= last.lm_loss);
}
#[test]
fn identical_seeds_produce_identical_models() {
let left = tiny().seed(77).build().unwrap();
let right = tiny().seed(77).build().unwrap();
assert_eq!(left, right);
assert_eq!(
left.forward_train(&[[1, 2, 3]]).unwrap().0,
right.forward_train(&[[1, 2, 3]]).unwrap().0
);
}
#[test]
fn a_snapshot_round_trips_and_predicts_identically() {
let model = tiny().build().unwrap();
let path = std::env::temp_dir().join("rusting_brain_transformer_round_trip.json");
model.save_json(&path).unwrap();
let restored = TransformerLm::load_json(&path).unwrap();
std::fs::remove_file(&path).ok();
assert_eq!(restored.config, model.config);
assert_eq!(restored, model);
assert_eq!(
restored.forward_train(&[[4, 11, 2]]).unwrap().0,
model.forward_train(&[[4, 11, 2]]).unwrap().0
);
}
#[test]
fn a_snapshot_survives_a_training_step() {
let mut model = tiny().build().unwrap();
model.train_step(&[[1, 2, 3, 4]]).unwrap();
let json = serde_json::to_string(&model).unwrap();
let restored: TransformerLm = serde_json::from_str(&json).unwrap();
assert_eq!(restored, model);
}
#[test]
fn a_sequence_past_the_maximum_is_rejected() {
let model = tiny().max_seq_len(4).build().unwrap();
assert!(matches!(
model.forward_train(&[[1, 2, 3, 4, 5]]),
Err(NetworkError::SequenceTooLong { length: 5, .. })
));
}
#[test]
fn a_cached_decode_past_the_maximum_is_rejected() {
let model = tiny().max_seq_len(3).build().unwrap();
let mut caches = model.new_kv_caches();
model.forward_cached(&[1, 2, 3], &mut caches).unwrap();
assert!(matches!(
model.forward_cached(&[4], &mut caches),
Err(NetworkError::SequenceTooLong { length: 4, .. })
));
}
#[test]
fn an_invalid_configuration_is_rejected_before_allocation() {
assert!(tiny().heads(4, 3, 4).build().is_err());
assert!(tiny().heads(4, 2, 5).build().is_err());
assert!(tiny().moe_layers([9]).build().is_err());
assert!(tiny().experts(2, 4).build().is_err());
}
}