use crate::attention::AttentionKvCache;
use crate::gpu::{GpuContext, GpuError};
use crate::layers::{Embedding, Initializer, Linear, WeightQuantization};
use crate::norm::RmsNorm;
use crate::profile;
use crate::rope::RopeScaling;
use crate::transformer::TransformerBlock;
use crate::{IndexTensor, Tensor, TensorError};
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::fmt;
use std::str::FromStr;
pub const DEFAULT_INITIALIZER_STD: f32 = 0.02;
fn default_initializer_std() -> f32 {
DEFAULT_INITIALIZER_STD
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ArchitecturePreset {
LlamaTiny,
LlamaMicro,
LlamaSmall,
GptTiny,
GptMicro,
GptSmall,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ArchitectureSpec {
pub hidden_size: usize,
pub num_layers: usize,
pub num_heads: usize,
pub num_key_value_heads: usize,
pub intermediate_size: usize,
pub max_sequence_length: usize,
pub use_bias: bool,
pub tie_word_embeddings: bool,
}
impl ArchitecturePreset {
pub fn all() -> &'static [Self] {
&[
Self::LlamaTiny,
Self::LlamaMicro,
Self::LlamaSmall,
Self::GptTiny,
Self::GptMicro,
Self::GptSmall,
]
}
pub fn as_str(self) -> &'static str {
match self {
Self::LlamaTiny => "llama-tiny",
Self::LlamaMicro => "llama-micro",
Self::LlamaSmall => "llama-small",
Self::GptTiny => "gpt-tiny",
Self::GptMicro => "gpt-micro",
Self::GptSmall => "gpt-small",
}
}
pub fn spec(self) -> ArchitectureSpec {
match self {
Self::LlamaTiny => ArchitectureSpec {
hidden_size: 128,
num_layers: 4,
num_heads: 4,
num_key_value_heads: 2,
intermediate_size: 384,
max_sequence_length: 256,
use_bias: false,
tie_word_embeddings: true,
},
Self::LlamaMicro => ArchitectureSpec {
hidden_size: 256,
num_layers: 6,
num_heads: 8,
num_key_value_heads: 4,
intermediate_size: 768,
max_sequence_length: 512,
use_bias: false,
tie_word_embeddings: true,
},
Self::LlamaSmall => ArchitectureSpec {
hidden_size: 512,
num_layers: 8,
num_heads: 8,
num_key_value_heads: 4,
intermediate_size: 1536,
max_sequence_length: 1024,
use_bias: false,
tie_word_embeddings: false,
},
Self::GptTiny => ArchitectureSpec {
hidden_size: 128,
num_layers: 4,
num_heads: 4,
num_key_value_heads: 4,
intermediate_size: 512,
max_sequence_length: 256,
use_bias: true,
tie_word_embeddings: true,
},
Self::GptMicro => ArchitectureSpec {
hidden_size: 256,
num_layers: 6,
num_heads: 8,
num_key_value_heads: 8,
intermediate_size: 1024,
max_sequence_length: 512,
use_bias: true,
tie_word_embeddings: true,
},
Self::GptSmall => ArchitectureSpec {
hidden_size: 512,
num_layers: 8,
num_heads: 8,
num_key_value_heads: 8,
intermediate_size: 2048,
max_sequence_length: 1024,
use_bias: true,
tie_word_embeddings: true,
},
}
}
pub fn causal_lm_config(
self,
vocab_size: usize,
eos_token_id: usize,
seed: u64,
) -> CausalLmConfig {
let spec = self.spec();
CausalLmConfig {
vocab_size,
hidden_size: spec.hidden_size,
num_layers: spec.num_layers,
num_heads: spec.num_heads,
num_key_value_heads: spec.num_key_value_heads,
intermediate_size: spec.intermediate_size,
max_sequence_length: spec.max_sequence_length,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id,
seed,
initializer_std: DEFAULT_INITIALIZER_STD,
tie_word_embeddings: spec.tie_word_embeddings,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: spec.use_bias,
}
}
}
impl fmt::Display for ArchitecturePreset {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
impl FromStr for ArchitecturePreset {
type Err = ModelConfigError;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"llama-tiny" => Ok(Self::LlamaTiny),
"llama-micro" => Ok(Self::LlamaMicro),
"llama-small" => Ok(Self::LlamaSmall),
"gpt-tiny" => Ok(Self::GptTiny),
"gpt-micro" => Ok(Self::GptMicro),
"gpt-small" => Ok(Self::GptSmall),
other => Err(ModelConfigError(format!(
"unknown architecture preset '{other}'; expected one of: {}",
Self::all()
.iter()
.map(|preset| preset.as_str())
.collect::<Vec<_>>()
.join(", ")
))),
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CausalLmConfig {
pub vocab_size: usize,
pub hidden_size: usize,
pub num_layers: usize,
pub num_heads: usize,
#[serde(default)]
pub num_key_value_heads: usize,
pub intermediate_size: usize,
pub max_sequence_length: usize,
pub rms_norm_epsilon: f32,
pub rope_theta: f32,
#[serde(default)]
pub rope_scaling: Option<RopeScaling>,
pub eos_token_id: usize,
pub seed: u64,
#[serde(default = "default_initializer_std")]
pub initializer_std: f32,
#[serde(default)]
pub tie_word_embeddings: bool,
#[serde(default)]
pub attention_dropout: f32,
#[serde(default)]
pub residual_dropout: f32,
#[serde(default)]
pub embedding_dropout: f32,
#[serde(default)]
pub use_bias: bool,
}
#[derive(Debug)]
pub struct ModelConfigError(pub String);
impl fmt::Display for ModelConfigError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "invalid causal LM configuration: {}", self.0)
}
}
impl Error for ModelConfigError {}
#[derive(Debug)]
pub enum ModelError {
InvalidInput(String),
Tensor(TensorError),
Gpu(GpuError),
Peft(crate::peft::PeftError),
}
impl fmt::Display for ModelError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidInput(message) => write!(formatter, "model input error: {message}"),
Self::Tensor(error) => error.fmt(formatter),
Self::Gpu(error) => error.fmt(formatter),
Self::Peft(error) => error.fmt(formatter),
}
}
}
impl Error for ModelError {}
impl From<TensorError> for ModelError {
fn from(error: TensorError) -> Self {
Self::Tensor(error)
}
}
impl From<GpuError> for ModelError {
fn from(error: GpuError) -> Self {
Self::Gpu(error)
}
}
impl From<crate::peft::PeftError> for ModelError {
fn from(error: crate::peft::PeftError) -> Self {
Self::Peft(error)
}
}
#[cfg(not(feature = "gpu"))]
fn gpu_disabled_error() -> GpuError {
GpuError::Initialization(
"GPU support is not compiled in; rebuild with `--features gpu`".into(),
)
}
#[derive(Clone)]
pub struct CausalLM {
pub config: CausalLmConfig,
pub tok_embeddings: Embedding,
pub layers: Vec<TransformerBlock>,
pub norm: RmsNorm,
pub lm_head: Linear,
}
#[derive(Clone)]
pub struct KvCache {
pub layers: Vec<Option<AttentionKvCache>>,
position: usize,
}
impl KvCache {
pub fn new(num_layers: usize) -> Self {
Self {
layers: vec![None; num_layers],
position: 0,
}
}
pub fn position(&self) -> usize {
self.position
}
pub fn sequence_length(&self) -> usize {
self.layers
.first()
.and_then(Option::as_ref)
.map(AttentionKvCache::sequence_length)
.unwrap_or(0)
}
pub fn batch_size(&self) -> Result<usize, ModelError> {
let mut batch_size = None;
for layer in &self.layers {
let Some(layer) = layer else {
continue;
};
let layer_batch = layer.batch_size();
if let Some(expected) = batch_size {
if expected != layer_batch {
return Err(ModelError::InvalidInput(
"KV cache layer batch sizes differ".into(),
));
}
} else {
batch_size = Some(layer_batch);
}
}
Ok(batch_size.unwrap_or(0))
}
pub fn concat_batch(caches: &[&Self]) -> Result<Self, ModelError> {
if caches.is_empty() {
return Err(ModelError::InvalidInput(
"cannot concat an empty KV cache batch".into(),
));
}
let num_layers = caches[0].layers.len();
let position = caches[0].position;
for cache in caches {
if cache.layers.len() != num_layers {
return Err(ModelError::InvalidInput(
"KV cache layer counts differ during concat".into(),
));
}
if cache.position != position {
return Err(ModelError::InvalidInput(
"KV cache positions differ during concat".into(),
));
}
if cache.sequence_length() != caches[0].sequence_length() {
return Err(ModelError::InvalidInput(
"KV cache sequence lengths differ during concat".into(),
));
}
}
let mut layers = Vec::with_capacity(num_layers);
for layer_index in 0..num_layers {
let populated = caches
.iter()
.any(|cache| cache.layers[layer_index].is_some());
if !populated {
layers.push(None);
continue;
}
let mut layer_refs = Vec::with_capacity(caches.len());
for cache in caches {
let Some(layer) = cache.layers[layer_index].as_ref() else {
return Err(ModelError::InvalidInput(
"KV cache layer occupancy differs during concat".into(),
));
};
layer_refs.push(layer);
}
layers.push(Some(AttentionKvCache::concat_batch(&layer_refs)?));
}
Ok(Self { layers, position })
}
pub fn split_batch(&self) -> Result<Vec<Self>, ModelError> {
let batch_size = self.batch_size()?;
if batch_size == 0 {
return Ok(Vec::new());
}
let mut split_layers: Vec<Vec<Option<AttentionKvCache>>> =
(0..batch_size).map(|_| vec![None; self.layers.len()]).collect();
for (layer_index, layer) in self.layers.iter().enumerate() {
let Some(layer) = layer else {
continue;
};
let parts = layer.split_batch()?;
if parts.len() != batch_size {
return Err(ModelError::InvalidInput(
"KV cache split produced unexpected batch size".into(),
));
}
for (batch_index, part) in parts.into_iter().enumerate() {
split_layers[batch_index][layer_index] = Some(part);
}
}
Ok(split_layers
.into_iter()
.map(|layers| Self {
layers,
position: self.position,
})
.collect())
}
pub fn clear(&mut self) {
self.layers.fill(None);
self.position = 0;
}
pub fn truncate(&mut self, sequence_length: usize) -> Result<(), ModelError> {
let current = self.sequence_length();
if sequence_length > current {
return Err(ModelError::InvalidInput(
"cannot truncate KV cache past its sequence length".into(),
));
}
if sequence_length == 0 {
self.clear();
return Ok(());
}
for layer in &mut self.layers {
if let Some(layer) = layer.as_mut() {
layer.truncate(sequence_length)?;
}
}
self.position = sequence_length;
Ok(())
}
}
impl CausalLmConfig {
pub fn validate(&self) -> Result<(), ModelConfigError> {
validate_config(self)
}
}
impl CausalLM {
pub fn new(mut config: CausalLmConfig) -> Result<Self, ModelConfigError> {
if config.num_key_value_heads == 0 {
config.num_key_value_heads = config.num_heads
}
if let Some(scaling) = &mut config.rope_scaling {
scaling.infer_original_context(config.max_sequence_length);
}
config.validate()?;
let mut initializer = Initializer::normal(config.seed, config.initializer_std);
let residual_scale = 1.0 / (2.0 * config.num_layers as f32).sqrt();
let tok_embeddings =
Embedding::from_initializer(config.vocab_size, config.hidden_size, &mut initializer);
let layers = (0..config.num_layers)
.map(|_| {
TransformerBlock::from_initializer(
config.hidden_size,
config.num_heads,
config.num_key_value_heads,
config.intermediate_size,
config.rms_norm_epsilon,
config.rope_theta,
config.rope_scaling.clone(),
&mut initializer,
residual_scale,
config.use_bias,
)
})
.collect();
let norm = RmsNorm::new(config.hidden_size, config.rms_norm_epsilon);
let lm_head = if config.tie_word_embeddings {
Linear::from_transposed_weight(
tok_embeddings.weight.clone(),
config.hidden_size,
config.vocab_size,
)
} else {
Linear::from_initializer(
config.hidden_size,
config.vocab_size,
&mut initializer,
config.use_bias,
)
};
Ok(Self {
config,
tok_embeddings,
layers,
norm,
lm_head,
})
}
pub fn forward(&self, input_ids: &IndexTensor) -> Result<Tensor, ModelError> {
self.forward_with_optional_attention_mask(input_ids, None)
}
pub fn forward_with_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: &IndexTensor,
) -> Result<Tensor, ModelError> {
self.forward_with_optional_attention_mask(input_ids, Some(attention_mask))
}
fn forward_with_optional_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: Option<&IndexTensor>,
) -> Result<Tensor, ModelError> {
self.validate_input_ids(input_ids)?;
if let Some(mask) = attention_mask {
self.validate_attention_mask(input_ids, mask)?;
}
let mut hidden_states = self.tok_embeddings.forward(input_ids);
for (index, layer) in self.layers.iter().enumerate() {
let _span = profile::layer(index);
hidden_states = match attention_mask {
Some(mask) => layer.forward_with_attention_mask(&hidden_states, mask),
None => layer.forward(&hidden_states),
};
}
Ok(self.lm_head.forward(&self.norm.forward(&hidden_states)))
}
pub fn forward_training(
&self,
input_ids: &IndexTensor,
seed: u64,
) -> Result<Tensor, ModelError> {
self.forward_training_with_optional_attention_mask(input_ids, None, seed, false)
}
pub fn forward_training_with_checkpointing(
&self,
input_ids: &IndexTensor,
seed: u64,
checkpoint_activations: bool,
) -> Result<Tensor, ModelError> {
self.forward_training_with_low_memory(
input_ids,
None,
seed,
checkpoint_activations,
1,
crate::trainer::SelectiveCheckpoint::Full,
0,
)
}
pub fn forward_training_with_low_memory(
&self,
input_ids: &IndexTensor,
attention_mask: Option<&IndexTensor>,
seed: u64,
checkpoint_activations: bool,
checkpoint_interval: usize,
selective: crate::trainer::SelectiveCheckpoint,
attention_chunk_size: usize,
) -> Result<Tensor, ModelError> {
self.validate_input_ids(input_ids)?;
if let Some(mask) = attention_mask {
self.validate_attention_mask(input_ids, mask)?;
}
let interval = checkpoint_interval.max(1);
let mut hidden_states = self
.tok_embeddings
.forward(input_ids)
.dropout(self.config.embedding_dropout, crate::derive_seed(seed, 0));
for (index, layer) in self.layers.iter().enumerate() {
let _span = profile::layer(index);
let layer_seed = crate::derive_seed(seed, index as u64 + 1);
let attention_dropout = self.config.attention_dropout;
let residual_dropout = self.config.residual_dropout;
let checkpoint_layer =
checkpoint_activations && ((index + 1) % interval == 0 || self.layers.len() == 1);
hidden_states = if checkpoint_layer
&& selective == crate::trainer::SelectiveCheckpoint::Full
{
let layer = layer.clone();
let mask = attention_mask.cloned();
crate::checkpoint(&hidden_states, move |input| {
layer.forward_training_selective(
input,
mask.as_ref(),
attention_dropout,
residual_dropout,
layer_seed,
crate::trainer::SelectiveCheckpoint::Full,
attention_chunk_size,
)
})
} else {
layer.forward_training_selective(
&hidden_states,
attention_mask,
attention_dropout,
residual_dropout,
layer_seed,
if checkpoint_layer {
selective
} else {
crate::trainer::SelectiveCheckpoint::Full
},
attention_chunk_size,
)
};
}
Ok(self.lm_head.forward(&self.norm.forward(&hidden_states)))
}
pub fn forward_training_with_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: &IndexTensor,
seed: u64,
) -> Result<Tensor, ModelError> {
self.forward_training_with_optional_attention_mask(
input_ids,
Some(attention_mask),
seed,
false,
)
}
fn forward_training_with_optional_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: Option<&IndexTensor>,
seed: u64,
checkpoint_activations: bool,
) -> Result<Tensor, ModelError> {
self.forward_training_with_low_memory(
input_ids,
attention_mask,
seed,
checkpoint_activations,
1,
crate::trainer::SelectiveCheckpoint::Full,
0,
)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(
&self,
input_ids: &IndexTensor,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.forward_gpu_with_optional_attention_mask(input_ids, None, gpu)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu_with_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: &IndexTensor,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.forward_gpu_with_optional_attention_mask(input_ids, Some(attention_mask), gpu)
}
#[cfg(feature = "gpu")]
fn forward_gpu_with_optional_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: Option<&IndexTensor>,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.validate_input_ids(input_ids)?;
if let Some(mask) = attention_mask {
self.validate_attention_mask(input_ids, mask)?;
}
self.prepare_gpu(gpu)?;
let mut hidden_states = self.tok_embeddings.forward_gpu(input_ids, gpu)?;
for (index, layer) in self.layers.iter().enumerate() {
let _span = profile::layer(index);
hidden_states = match attention_mask {
Some(mask) => layer.forward_gpu_with_attention_mask(&hidden_states, mask, gpu)?,
None => layer.forward_gpu(&hidden_states, gpu)?,
};
}
let normalized = self.norm.forward_gpu(&hidden_states, gpu)?;
Ok(self.lm_head.forward_gpu(&normalized, gpu)?)
}
#[cfg(feature = "gpu")]
pub fn forward_training_gpu(
&self,
input_ids: &IndexTensor,
seed: u64,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.forward_training_gpu_with_optional_attention_mask(input_ids, None, seed, false, gpu)
}
#[cfg(feature = "gpu")]
pub fn forward_training_gpu_with_checkpointing(
&self,
input_ids: &IndexTensor,
seed: u64,
checkpoint_activations: bool,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.forward_training_gpu_with_optional_attention_mask(
input_ids,
None,
seed,
checkpoint_activations,
gpu,
)
}
#[cfg(feature = "gpu")]
pub fn forward_training_gpu_with_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: &IndexTensor,
seed: u64,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.forward_training_gpu_with_optional_attention_mask(
input_ids,
Some(attention_mask),
seed,
false,
gpu,
)
}
#[cfg(feature = "gpu")]
fn forward_training_gpu_with_optional_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: Option<&IndexTensor>,
seed: u64,
checkpoint_activations: bool,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.validate_input_ids(input_ids)?;
if let Some(mask) = attention_mask {
self.validate_attention_mask(input_ids, mask)?;
}
self.prepare_gpu(gpu)?;
let embeddings = self.tok_embeddings.forward_gpu(input_ids, gpu)?;
let mut hidden_states = embeddings.gpu_dropout(
self.config.embedding_dropout,
crate::derive_seed(seed, 0),
gpu,
)?;
for (index, layer) in self.layers.iter().enumerate() {
let _span = profile::layer(index);
let layer_seed = crate::derive_seed(seed, index as u64 + 1);
let attention_dropout = self.config.attention_dropout;
let residual_dropout = self.config.residual_dropout;
hidden_states = if checkpoint_activations {
let layer = layer.clone();
let mask = attention_mask.cloned();
crate::checkpoint_gpu(&hidden_states, gpu, move |input, gpu| {
layer.forward_training_gpu(
input,
mask.as_ref(),
attention_dropout,
residual_dropout,
layer_seed,
gpu,
)
})?
} else {
layer.forward_training_gpu(
&hidden_states,
attention_mask,
attention_dropout,
residual_dropout,
layer_seed,
gpu,
)?
};
}
let normalized = self.norm.forward_gpu(&hidden_states, gpu)?;
Ok(self.lm_head.forward_gpu(&normalized, gpu)?)
}
pub fn forward_cached(
&self,
input_ids: &IndexTensor,
cache: &mut KvCache,
) -> Result<Tensor, ModelError> {
let shape = self.validate_input_ids(input_ids)?;
self.validate_cache(cache)?;
let position = cache.position;
let mut hidden_states = self.tok_embeddings.forward(input_ids);
for (index, (layer, layer_cache)) in
self.layers.iter().zip(cache.layers.iter_mut()).enumerate()
{
let _span = profile::layer(index);
hidden_states = layer.forward_cached(
&hidden_states,
layer_cache,
position,
self.config.max_sequence_length,
);
}
cache.position += shape[1];
Ok(self.lm_head.forward(&self.norm.forward(&hidden_states)))
}
#[cfg(feature = "gpu")]
pub fn forward_cached_gpu(
&self,
input_ids: &IndexTensor,
cache: &mut KvCache,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
let shape = self.validate_input_ids(input_ids)?;
self.validate_cache(cache)?;
self.prepare_gpu(gpu)?;
let position = cache.position;
let mut hidden_states = self.tok_embeddings.forward_gpu(input_ids, gpu)?;
for (index, (layer, layer_cache)) in
self.layers.iter().zip(cache.layers.iter_mut()).enumerate()
{
let _span = profile::layer(index);
hidden_states = layer.forward_cached_gpu(
&hidden_states,
layer_cache,
position,
self.config.max_sequence_length,
gpu,
)?;
}
let normalized = self.norm.forward_gpu(&hidden_states, gpu)?;
let logits = self.lm_head.forward_gpu(&normalized, gpu)?;
cache.position += shape[1];
Ok(logits)
}
pub fn compute_loss(
&self,
logits: &Tensor,
target_ids: &IndexTensor,
) -> Result<Tensor, ModelError> {
self.compute_loss_ignore_index(logits, target_ids, crate::tokenizer::PAD_ID)
}
pub fn compute_loss_ignore_index(
&self,
logits: &Tensor,
target_ids: &IndexTensor,
ignore_index: usize,
) -> Result<Tensor, ModelError> {
self.validate_logits(logits)?;
Ok(logits.try_cross_entropy_ignore_index(target_ids, ignore_index)?)
}
#[cfg(feature = "gpu")]
pub fn compute_loss_gpu(
&self,
logits: &Tensor,
target_ids: &IndexTensor,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.compute_loss_gpu_ignore_index(logits, target_ids, crate::tokenizer::PAD_ID, gpu)
}
#[cfg(feature = "gpu")]
pub fn compute_loss_gpu_ignore_index(
&self,
logits: &Tensor,
target_ids: &IndexTensor,
ignore_index: usize,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.validate_logits(logits)?;
Ok(gpu.cross_entropy_tensor_ignore_index(logits, target_ids, ignore_index)?)
}
pub fn generate(
&self,
prompt: &[usize],
max_new_tokens: usize,
) -> Result<Vec<usize>, crate::generate::GeneratorError> {
self.generate_with_temperature(prompt, max_new_tokens, 0.0, self.config.seed)
}
pub fn generate_with_temperature(
&self,
prompt: &[usize],
max_new_tokens: usize,
temperature: f32,
seed: u64,
) -> Result<Vec<usize>, crate::generate::GeneratorError> {
let config = crate::generate::SamplingConfig {
max_new_tokens,
temperature,
top_k: None,
top_p: 1.0,
repetition_penalty: 1.0,
seed,
return_logprobs: false,
top_logprobs: 0,
..Default::default()
};
self.generate_with_sampling_config(prompt, &config)
}
pub fn parameters(&self) -> Vec<Tensor> {
self.named_parameters()
.into_iter()
.map(|(_, parameter)| parameter)
.collect()
}
pub fn named_parameters(&self) -> Vec<(String, Tensor)> {
let mut parameters = Vec::new();
parameters.push((
"tok_embeddings.weight".to_string(),
self.tok_embeddings.weight.clone(),
));
for (layer_index, layer) in self.layers.iter().enumerate() {
let prefix = format!("layers.{layer_index}");
parameters.push((
format!("{prefix}.attention_norm.weight"),
layer.attention_norm.weight.clone(),
));
push_named_linear(
&mut parameters,
&format!("{prefix}.attention.q_proj"),
&layer.attention.q_proj,
);
push_named_linear(
&mut parameters,
&format!("{prefix}.attention.k_proj"),
&layer.attention.k_proj,
);
push_named_linear(
&mut parameters,
&format!("{prefix}.attention.v_proj"),
&layer.attention.v_proj,
);
push_named_linear(
&mut parameters,
&format!("{prefix}.attention.o_proj"),
&layer.attention.o_proj,
);
parameters.push((
format!("{prefix}.ffn_norm.weight"),
layer.ffn_norm.weight.clone(),
));
push_named_linear(
&mut parameters,
&format!("{prefix}.feed_forward.gate_proj"),
&layer.feed_forward.gate_proj,
);
push_named_linear(
&mut parameters,
&format!("{prefix}.feed_forward.up_proj"),
&layer.feed_forward.up_proj,
);
push_named_linear(
&mut parameters,
&format!("{prefix}.feed_forward.down_proj"),
&layer.feed_forward.down_proj,
);
}
parameters.push(("norm.weight".to_string(), self.norm.weight.clone()));
if !self.config.tie_word_embeddings {
push_named_linear(&mut parameters, "lm_head", &self.lm_head);
}
parameters
}
#[cfg(feature = "gpu")]
pub fn prepare_gpu(&self, gpu: &GpuContext) -> Result<(), GpuError> {
let _ = self.tok_embeddings.weight.to_gpu(gpu)?;
for layer in &self.layers {
Self::prepare_layer_gpu(layer, gpu)?;
}
let _ = self.norm.weight.to_gpu(gpu)?;
self.lm_head.prepare_gpu(gpu)?;
Ok(())
}
#[cfg(feature = "gpu")]
fn prepare_layer_gpu(layer: &TransformerBlock, gpu: &GpuContext) -> Result<(), GpuError> {
let _ = layer.attention_norm.weight.to_gpu(gpu)?;
layer.attention.q_proj.prepare_gpu(gpu)?;
layer.attention.k_proj.prepare_gpu(gpu)?;
layer.attention.v_proj.prepare_gpu(gpu)?;
layer.attention.o_proj.prepare_gpu(gpu)?;
let _ = layer.ffn_norm.weight.to_gpu(gpu)?;
layer.feed_forward.gate_proj.prepare_gpu(gpu)?;
layer.feed_forward.up_proj.prepare_gpu(gpu)?;
layer.feed_forward.down_proj.prepare_gpu(gpu)?;
Ok(())
}
#[cfg(feature = "gpu")]
fn offload_layer_gpu(layer: &TransformerBlock) {
for parameter in layer.parameters() {
parameter.offload_gpu_cache();
}
}
#[cfg(feature = "gpu")]
pub fn forward_training_gpu_with_layer_offload(
&self,
input_ids: &IndexTensor,
seed: u64,
checkpoint_activations: bool,
gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
self.validate_input_ids(input_ids)?;
let _ = self.tok_embeddings.weight.to_gpu(gpu)?;
let embeddings = self.tok_embeddings.forward_gpu(input_ids, gpu)?;
self.tok_embeddings.weight.offload_gpu_cache();
let mut hidden_states = embeddings.gpu_dropout(
self.config.embedding_dropout,
crate::derive_seed(seed, 0),
gpu,
)?;
for (index, layer) in self.layers.iter().enumerate() {
let _span = profile::layer(index);
Self::prepare_layer_gpu(layer, gpu)?;
let layer_seed = crate::derive_seed(seed, index as u64 + 1);
let attention_dropout = self.config.attention_dropout;
let residual_dropout = self.config.residual_dropout;
hidden_states = if checkpoint_activations {
let layer = layer.clone();
crate::checkpoint_gpu(&hidden_states, gpu, move |input, gpu| {
Self::prepare_layer_gpu(&layer, gpu)?;
let output = layer.forward_training_gpu(
input,
None,
attention_dropout,
residual_dropout,
layer_seed,
gpu,
)?;
Self::offload_layer_gpu(&layer);
Ok(output)
})?
} else {
layer.forward_training_gpu(
&hidden_states,
None,
attention_dropout,
residual_dropout,
layer_seed,
gpu,
)?
};
Self::offload_layer_gpu(layer);
let _ = gpu.trim_dispatch_caches();
}
let _ = self.norm.weight.to_gpu(gpu)?;
self.lm_head.prepare_gpu(gpu)?;
let normalized = self.norm.forward_gpu(&hidden_states, gpu)?;
let logits = self.lm_head.forward_gpu(&normalized, gpu)?;
self.norm.weight.offload_gpu_cache();
for parameter in self.lm_head.parameters() {
parameter.offload_gpu_cache();
}
Ok(logits)
}
#[cfg(not(feature = "gpu"))]
pub fn forward_gpu(
&self,
_input_ids: &IndexTensor,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn forward_gpu_with_attention_mask(
&self,
_input_ids: &IndexTensor,
_attention_mask: &IndexTensor,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn forward_training_gpu(
&self,
_input_ids: &IndexTensor,
_seed: u64,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn forward_training_gpu_with_checkpointing(
&self,
_input_ids: &IndexTensor,
_seed: u64,
_checkpoint_activations: bool,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn forward_training_gpu_with_attention_mask(
&self,
_input_ids: &IndexTensor,
_attention_mask: &IndexTensor,
_seed: u64,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn forward_cached_gpu(
&self,
_input_ids: &IndexTensor,
_cache: &mut KvCache,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn compute_loss_gpu(
&self,
_logits: &Tensor,
_target_ids: &IndexTensor,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn compute_loss_gpu_ignore_index(
&self,
_logits: &Tensor,
_target_ids: &IndexTensor,
_ignore_index: usize,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
#[cfg(not(feature = "gpu"))]
pub fn prepare_gpu(&self, _gpu: &GpuContext) -> Result<(), GpuError> {
Err(gpu_disabled_error())
}
#[cfg(not(feature = "gpu"))]
pub fn forward_training_gpu_with_layer_offload(
&self,
_input_ids: &IndexTensor,
_seed: u64,
_checkpoint_activations: bool,
_gpu: &GpuContext,
) -> Result<Tensor, ModelError> {
Err(ModelError::Gpu(gpu_disabled_error()))
}
pub fn quantize_linear_weights(&mut self, format: WeightQuantization) {
self.quantize_linear_weights_where(format, |_| true);
}
pub fn quantize_linear_weights_where(
&mut self,
format: WeightQuantization,
mut select: impl FnMut(&str) -> bool,
) {
for (name, linear) in self.named_linears_mut() {
if select(&name) {
linear.quantize_weight(format);
}
}
}
pub fn enable_activation_quant_where(
&mut self,
config: crate::act_quant::ActQuantConfig,
mut select: impl FnMut(&str) -> bool,
) -> Result<(), String> {
for (name, linear) in self.named_linears_mut() {
if select(&name) {
linear.enable_activation_quant(config.clone())?;
}
}
Ok(())
}
pub fn enable_kv_cache_quant(&mut self, scheme: crate::kv_quant::KvQuantScheme) {
for layer in &mut self.layers {
layer.attention.enable_kv_quant(scheme);
}
}
pub fn clear_kv_cache_quant(&mut self) {
for layer in &mut self.layers {
layer.attention.clear_kv_quant();
}
}
pub fn kv_cache_quant(&self) -> Option<crate::kv_quant::KvQuantScheme> {
self.layers.first().and_then(|layer| layer.attention.kv_quant())
}
pub fn named_linears_mut(&mut self) -> Vec<(String, &mut Linear)> {
let mut linears = Vec::new();
for (layer_index, layer) in self.layers.iter_mut().enumerate() {
let prefix = format!("layers.{layer_index}");
linears.push((
format!("{prefix}.attention.q_proj"),
&mut layer.attention.q_proj,
));
linears.push((
format!("{prefix}.attention.k_proj"),
&mut layer.attention.k_proj,
));
linears.push((
format!("{prefix}.attention.v_proj"),
&mut layer.attention.v_proj,
));
linears.push((
format!("{prefix}.attention.o_proj"),
&mut layer.attention.o_proj,
));
linears.push((
format!("{prefix}.feed_forward.gate_proj"),
&mut layer.feed_forward.gate_proj,
));
linears.push((
format!("{prefix}.feed_forward.up_proj"),
&mut layer.feed_forward.up_proj,
));
linears.push((
format!("{prefix}.feed_forward.down_proj"),
&mut layer.feed_forward.down_proj,
));
}
if !self.config.tie_word_embeddings {
linears.push(("lm_head".into(), &mut self.lm_head));
}
linears
}
pub fn named_linears(&self) -> Vec<(String, &Linear)> {
let mut linears = Vec::new();
for (layer_index, layer) in self.layers.iter().enumerate() {
let prefix = format!("layers.{layer_index}");
linears.push((
format!("{prefix}.attention.q_proj"),
&layer.attention.q_proj,
));
linears.push((
format!("{prefix}.attention.k_proj"),
&layer.attention.k_proj,
));
linears.push((
format!("{prefix}.attention.v_proj"),
&layer.attention.v_proj,
));
linears.push((
format!("{prefix}.attention.o_proj"),
&layer.attention.o_proj,
));
linears.push((
format!("{prefix}.feed_forward.gate_proj"),
&layer.feed_forward.gate_proj,
));
linears.push((
format!("{prefix}.feed_forward.up_proj"),
&layer.feed_forward.up_proj,
));
linears.push((
format!("{prefix}.feed_forward.down_proj"),
&layer.feed_forward.down_proj,
));
}
if !self.config.tie_word_embeddings {
linears.push(("lm_head".into(), &self.lm_head));
}
linears
}
pub fn set_quantized_linear(
&mut self,
module_name: &str,
weight: crate::layers::QuantizedLinearWeight,
) -> Result<(), String> {
for (name, linear) in self.named_linears_mut() {
if name == module_name {
return linear.set_quantized_weight(weight);
}
}
Err(format!("unknown linear module '{module_name}'"))
}
pub fn clear_quantized_linear_weights(&mut self) {
for (_, linear) in self.named_linears_mut() {
linear.clear_quantized_weight();
}
}
pub fn enable_qat(&mut self, format: WeightQuantization) {
self.enable_qat_with(crate::qat::QatConfig::from_format(format))
.expect("default QAT format is valid");
}
pub fn enable_qat_with(&mut self, config: crate::qat::QatConfig) -> Result<(), ModelError> {
for layer in &mut self.layers {
layer
.attention
.q_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
layer
.attention
.k_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
layer
.attention
.v_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
layer
.attention
.o_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
layer
.feed_forward
.gate_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
layer
.feed_forward
.up_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
layer
.feed_forward
.down_proj
.enable_qat_with(config.clone())
.map_err(ModelError::InvalidInput)?;
}
self.lm_head
.enable_qat_with(config)
.map_err(ModelError::InvalidInput)?;
Ok(())
}
pub fn clear_qat(&mut self) {
for layer in &mut self.layers {
layer.attention.q_proj.clear_qat();
layer.attention.k_proj.clear_qat();
layer.attention.v_proj.clear_qat();
layer.attention.o_proj.clear_qat();
layer.feed_forward.gate_proj.clear_qat();
layer.feed_forward.down_proj.clear_qat();
layer.feed_forward.up_proj.clear_qat();
}
self.lm_head.clear_qat()
}
pub fn export_qat_packed(&mut self) -> Result<(), ModelError> {
for (_, linear) in self.named_linears_mut() {
if linear.qat_format().is_some() {
linear
.export_qat_packed()
.map_err(ModelError::InvalidInput)?;
}
}
Ok(())
}
pub fn install_lora(&mut self, config: &crate::peft::LoraConfig) -> Result<(), ModelError> {
if let Some(format) = config.base_quant {
let scheme = crate::weight_quant::QuantScheme {
format,
granularity: match config.base_group_size {
Some(size) => crate::weight_quant::QuantGranularity::PerGroup { size },
None if matches!(
format,
WeightQuantization::Nf4 | WeightQuantization::Int4
) =>
{
crate::weight_quant::QuantGranularity::PerGroup { size: 64 }
}
None => crate::weight_quant::QuantGranularity::PerChannel,
},
symmetric: true,
scale_method: crate::weight_quant::ScaleMethod::AbsMax,
percentile: 99.99,
};
scheme.validate().map_err(ModelError::InvalidInput)?;
for (_, linear) in self.named_linears_mut() {
linear.quantize_weight_with(scheme);
linear.weight.set_requires_grad(false);
if let Some(bias) = &linear.bias {
bias.set_requires_grad(false);
}
}
}
let attached = crate::peft::attach_lora_to_model(self, config)?;
let freeze_patterns: Vec<String> = attached
.iter()
.flat_map(|(name, _)| [format!("{name}.weight"), format!("{name}.bias")])
.collect();
crate::peft::freeze_parameters(&self.named_parameters(), &freeze_patterns);
Ok(())
}
pub fn zero_grad(&self) {
for parameter in self.parameters() {
parameter.zero_grad();
}
}
pub fn sgd_step(&self, learning_rate: f32) {
for parameter in self.parameters() {
parameter.sgd_step(learning_rate);
}
}
fn generate_with_sampling_config(
&self,
prompt: &[usize],
config: &crate::generate::SamplingConfig,
) -> Result<Vec<usize>, crate::generate::GeneratorError> {
let mut generated = prompt.to_vec();
generated.extend(
crate::generate::generate_tokens(self, prompt, config, None)?.token_ids,
);
Ok(generated)
}
fn validate_input_ids(&self, input_ids: &IndexTensor) -> Result<Vec<usize>, ModelError> {
let shape = input_ids.shape();
if shape.len() != 2 {
return Err(ModelError::InvalidInput(
"input IDs must have shape [B, S]".into(),
));
}
if shape[1] > self.config.max_sequence_length {
return Err(ModelError::InvalidInput(
"sequence exceeds configured context length".into(),
));
}
input_ids.try_validate_upper_bound(self.config.vocab_size, "input token ID")?;
Ok(shape)
}
fn validate_attention_mask(
&self,
input_ids: &IndexTensor,
attention_mask: &IndexTensor,
) -> Result<(), ModelError> {
if attention_mask.shape() != input_ids.shape() {
return Err(ModelError::InvalidInput(
"attention mask must match input IDs shape [B, S]".into(),
));
}
if attention_mask.data().iter().any(|value| *value > 1) {
return Err(ModelError::InvalidInput(
"attention mask values must be zero or one".into(),
));
}
Ok(())
}
fn validate_cache(&self, cache: &KvCache) -> Result<(), ModelError> {
if cache.layers.len() != self.layers.len() {
return Err(ModelError::InvalidInput(
"cache layer count mismatch".into(),
));
}
Ok(())
}
fn validate_logits(&self, logits: &Tensor) -> Result<(), ModelError> {
if logits.shape().last() != Some(&self.config.vocab_size) {
return Err(ModelError::InvalidInput(
"logit vocabulary width mismatch".into(),
));
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::{ArchitecturePreset, CausalLM, CausalLmConfig, ModelError};
use crate::IndexTensor;
fn tiny_config() -> CausalLmConfig {
CausalLmConfig {
vocab_size: 32,
hidden_size: 16,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 2,
intermediate_size: 32,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 2,
seed: 0,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
}
}
#[test]
fn forward_rejects_bad_input_shapes() {
let model = CausalLM::new(tiny_config()).unwrap();
let wrong_rank = IndexTensor::new(vec![1, 2, 3], vec![3]);
assert!(matches!(
model.forward(&wrong_rank),
Err(ModelError::InvalidInput(_))
));
let too_long = IndexTensor::new(vec![1; 9], vec![1, 9]);
assert!(matches!(
model.forward(&too_long),
Err(ModelError::InvalidInput(_))
));
let bad_token = IndexTensor::new(vec![99], vec![1, 1]);
assert!(matches!(
model.forward(&bad_token),
Err(ModelError::Tensor(_))
));
}
#[test]
fn generate_rejects_empty_prompt() {
let model = CausalLM::new(tiny_config()).unwrap();
assert!(model.generate(&[], 1).is_err());
}
#[test]
fn bias_preset_allocates_linear_biases() {
let mut config = ArchitecturePreset::GptTiny.causal_lm_config(32, 2, 0);
config.max_sequence_length = 8;
let model = CausalLM::new(config).unwrap();
assert!(model.layers[0].attention.q_proj.bias.is_some());
assert!(model.layers[0].feed_forward.down_proj.bias.is_some());
assert!(model.lm_head.bias.is_none()); }
#[test]
fn llama_preset_has_no_biases_and_uses_gqa() {
let spec = ArchitecturePreset::LlamaTiny.spec();
assert!(!spec.use_bias);
assert!(spec.num_key_value_heads < spec.num_heads);
let mut config = ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0);
config.max_sequence_length = 8;
let model = CausalLM::new(config).unwrap();
assert!(model.layers[0].attention.q_proj.bias.is_none());
}
#[test]
fn micro_presets_sit_between_tiny_and_small() {
let llama_tiny = ArchitecturePreset::LlamaTiny.spec();
let llama_micro = ArchitecturePreset::LlamaMicro.spec();
let llama_small = ArchitecturePreset::LlamaSmall.spec();
assert!(llama_tiny.hidden_size < llama_micro.hidden_size);
assert!(llama_micro.hidden_size < llama_small.hidden_size);
assert!(llama_tiny.num_layers < llama_micro.num_layers);
assert!(llama_micro.num_layers <= llama_small.num_layers);
assert!(!llama_micro.use_bias);
assert!(llama_micro.num_key_value_heads < llama_micro.num_heads);
let gpt_tiny = ArchitecturePreset::GptTiny.spec();
let gpt_micro = ArchitecturePreset::GptMicro.spec();
let gpt_small = ArchitecturePreset::GptSmall.spec();
assert!(gpt_tiny.hidden_size < gpt_micro.hidden_size);
assert!(gpt_micro.hidden_size < gpt_small.hidden_size);
assert!(gpt_micro.use_bias);
assert_eq!(gpt_micro.num_key_value_heads, gpt_micro.num_heads);
}
}
fn push_named_linear(
parameters: &mut Vec<(String, Tensor)>,
prefix: &str,
linear: &Linear,
) {
parameters.push((format!("{prefix}.weight"), linear.weight.clone()));
if let Some(bias) = &linear.bias {
parameters.push((format!("{prefix}.bias"), bias.clone()));
}
if let Some(scales) = linear.qat_scales() {
parameters.push((format!("{prefix}.qat_scale"), scales.clone()));
}
if let Some(lora) = &linear.lora {
parameters.extend(lora.named_parameters(&format!("{prefix}.lora")));
}
}
fn validate_config(config: &CausalLmConfig) -> Result<(), ModelConfigError> {
if config.vocab_size == 0 {
return Err(ModelConfigError("vocabulary must not be empty".into()));
}
if config.hidden_size == 0 || config.num_heads == 0 {
return Err(ModelConfigError(
"hidden size and head count must be positive".into(),
));
}
if config.hidden_size % config.num_heads != 0 {
return Err(ModelConfigError(
"head count must divide hidden size".into(),
));
}
let num_key_value_heads = if config.num_key_value_heads == 0 {
config.num_heads
} else {
config.num_key_value_heads
};
if num_key_value_heads > config.num_heads || config.num_heads % num_key_value_heads != 0 {
return Err(ModelConfigError(
"key/value head count must divide query head count".into(),
));
}
if (config.hidden_size / config.num_heads) % 2 != 0 {
return Err(ModelConfigError("RoPE head width must be even".into()));
}
if config.num_layers == 0 || config.intermediate_size == 0 || config.max_sequence_length == 0 {
return Err(ModelConfigError(
"layer count, FFN size, and context length must be positive".into(),
));
}
if config.rms_norm_epsilon <= 0.0 || config.rope_theta <= 0.0 {
return Err(ModelConfigError(
"normalization epsilon and RoPE theta must be positive".into(),
));
}
if let Some(scaling) = &config.rope_scaling {
scaling.validate().map_err(ModelConfigError)?;
let supported_context =
scaling.original_max_position_embeddings as f64 * scaling.factor as f64;
if config.max_sequence_length as f64 > supported_context {
return Err(ModelConfigError(format!(
"context length {} exceeds RoPE-scaled limit {}",
config.max_sequence_length, supported_context as usize
)));
}
}
if !config.initializer_std.is_finite() || config.initializer_std <= 0.0 {
return Err(ModelConfigError(
"initializer standard deviation must be positive and finite".into(),
));
}
for (name, probability) in [
("attention", config.attention_dropout),
("residual", config.residual_dropout),
("embedding", config.embedding_dropout),
] {
if !probability.is_finite() || !(0.0..1.0).contains(&probability) {
return Err(ModelConfigError(format!(
"{name} dropout probability must be finite and in [0, 1)"
)));
}
}
if config.eos_token_id >= config.vocab_size {
return Err(ModelConfigError(
"EOS token ID is outside vocabulary".into(),
));
}
Ok(())
}