use crate::attention::{AttentionKvCache, MultiHeadAttention};
#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::layers::{Initializer, Linear};
use crate::norm::RmsNorm;
use crate::rope::RopeScaling;
use crate::{IndexTensor, Tensor};
#[derive(Clone)]
pub struct FeedForward {
pub gate_proj: Linear,
pub up_proj: Linear,
pub down_proj: Linear,
}
impl FeedForward {
pub fn new(hidden_size: usize, intermediate_size: usize, seed: u64) -> Self {
let mut initializer = Initializer::new(seed);
Self::from_initializer(hidden_size, intermediate_size, &mut initializer, 1.0, false)
}
pub(crate) fn from_initializer(
hidden_size: usize,
intermediate_size: usize,
initializer: &mut Initializer,
residual_scale: f32,
use_bias: bool,
) -> Self {
Self {
gate_proj: Linear::from_initializer(
hidden_size,
intermediate_size,
initializer,
use_bias,
),
up_proj: Linear::from_initializer(
hidden_size,
intermediate_size,
initializer,
use_bias,
),
down_proj: Linear::from_initializer_scaled(
intermediate_size,
hidden_size,
initializer,
residual_scale,
use_bias,
),
}
}
pub fn forward(&self, input: &Tensor) -> Tensor {
let gate = self.gate_proj.forward(input);
let activated_gate = &gate * &gate.sigmoid();
let up = self.up_proj.forward(input);
self.down_proj.forward(&(&activated_gate * &up))
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
let gate = self.gate_proj.forward_gpu(input, gpu)?;
let up = self.up_proj.forward_gpu(input, gpu)?;
let fused = gate.gpu_swiglu(&up, gpu)?;
self.down_proj.forward_gpu(&fused, gpu)
}
pub fn parameters(&self) -> Vec<Tensor> {
let mut parameters = Vec::new();
parameters.extend(self.gate_proj.parameters());
parameters.extend(self.up_proj.parameters());
parameters.extend(self.down_proj.parameters());
parameters
}
}
#[derive(Clone)]
pub struct TransformerBlock {
pub attention_norm: RmsNorm,
pub attention: MultiHeadAttention,
pub ffn_norm: RmsNorm,
pub feed_forward: FeedForward,
}
impl TransformerBlock {
pub fn new(
hidden_size: usize,
num_heads: usize,
intermediate_size: usize,
rms_norm_epsilon: f32,
rope_theta: f32,
seed: u64,
) -> Self {
let mut initializer = Initializer::new(seed);
Self::from_initializer(
hidden_size,
num_heads,
num_heads,
intermediate_size,
rms_norm_epsilon,
rope_theta,
None,
&mut initializer,
1.0,
false,
)
}
pub(crate) fn from_initializer(
hidden_size: usize,
num_heads: usize,
num_key_value_heads: usize,
intermediate_size: usize,
rms_norm_epsilon: f32,
rope_theta: f32,
rope_scaling: Option<RopeScaling>,
initializer: &mut Initializer,
residual_scale: f32,
use_bias: bool,
) -> Self {
Self {
attention_norm: RmsNorm::new(hidden_size, rms_norm_epsilon),
attention: MultiHeadAttention::from_initializer(
hidden_size,
num_heads,
num_key_value_heads,
rope_theta,
rope_scaling,
initializer,
residual_scale,
use_bias,
),
ffn_norm: RmsNorm::new(hidden_size, rms_norm_epsilon),
feed_forward: FeedForward::from_initializer(
hidden_size,
intermediate_size,
initializer,
residual_scale,
use_bias,
),
}
}
pub fn forward(&self, input: &Tensor) -> Tensor {
self.forward_with_optional_attention_mask(input, None)
}
pub fn forward_with_attention_mask(
&self,
input: &Tensor,
attention_mask: &IndexTensor,
) -> Tensor {
self.forward_with_optional_attention_mask(input, Some(attention_mask))
}
fn forward_with_optional_attention_mask(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
) -> Tensor {
self.forward_training(input, attention_mask, 0.0, 0.0, 0)
}
pub(crate) fn forward_training(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
attention_dropout: f32,
residual_dropout: f32,
seed: u64,
) -> Tensor {
self.forward_training_selective(
input,
attention_mask,
attention_dropout,
residual_dropout,
seed,
crate::trainer::SelectiveCheckpoint::Full,
0,
)
}
pub(crate) fn forward_training_selective(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
attention_dropout: f32,
residual_dropout: f32,
seed: u64,
selective: crate::trainer::SelectiveCheckpoint,
attention_chunk_size: usize,
) -> Tensor {
let attention_norm = self.attention_norm.clone();
let attention = self.attention.clone();
let ffn_norm = self.ffn_norm.clone();
let feed_forward = self.feed_forward.clone();
let mask = attention_mask.cloned();
let attn_seed = crate::derive_seed(seed, 0) as u32;
let drop_seed_1 = crate::derive_seed(seed, 1);
let drop_seed_2 = crate::derive_seed(seed, 2);
let run_attention = move |input: &Tensor| {
let normalized = attention_norm.forward(input);
let attention_output = attention.forward_training_chunked(
&normalized,
mask.as_ref(),
attention_dropout,
attn_seed,
attention_chunk_size,
);
attention_output.dropout(residual_dropout, drop_seed_1)
};
let run_ffn = move |after_attention: &Tensor| {
let ffn_output = feed_forward.forward(&ffn_norm.forward(after_attention));
ffn_output.dropout(residual_dropout, drop_seed_2)
};
match selective {
crate::trainer::SelectiveCheckpoint::Full => {
let attention_output = run_attention(input);
let after_attention = input + &attention_output;
&after_attention + &run_ffn(&after_attention)
}
crate::trainer::SelectiveCheckpoint::Attention => {
let attention_output = crate::checkpoint(input, run_attention);
let after_attention = input + &attention_output;
&after_attention + &run_ffn(&after_attention)
}
crate::trainer::SelectiveCheckpoint::Ffn => {
let attention_output = run_attention(input);
let after_attention = input + &attention_output;
let ffn_output = crate::checkpoint(&after_attention, run_ffn);
&after_attention + &ffn_output
}
}
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
self.forward_gpu_with_optional_attention_mask(input, None, gpu)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu_with_attention_mask(
&self,
input: &Tensor,
attention_mask: &IndexTensor,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
self.forward_gpu_with_optional_attention_mask(input, Some(attention_mask), gpu)
}
#[cfg(feature = "gpu")]
fn forward_gpu_with_optional_attention_mask(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
self.forward_training_gpu(input, attention_mask, 0.0, 0.0, 0, gpu)
}
#[cfg(feature = "gpu")]
pub(crate) fn forward_training_gpu(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
attention_dropout: f32,
residual_dropout: f32,
seed: u64,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
let normalized = self.attention_norm.forward_gpu(input, gpu)?;
let attention_output = self.attention.forward_training_gpu(
&normalized,
attention_mask,
attention_dropout,
crate::derive_seed(seed, 0) as u32,
gpu,
)?;
let attention_output =
attention_output.gpu_dropout(residual_dropout, crate::derive_seed(seed, 1), gpu)?;
let (after_attention, normalized_ffn) = input.gpu_residual_rms_norm(
&attention_output,
&self.ffn_norm.weight.to_gpu(gpu)?,
self.ffn_norm.epsilon,
gpu,
)?;
let ffn_output = self.feed_forward.forward_gpu(&normalized_ffn, gpu)?;
let ffn_output =
ffn_output.gpu_dropout(residual_dropout, crate::derive_seed(seed, 2), gpu)?;
after_attention.gpu_add(&ffn_output, gpu)
}
pub fn forward_cached(
&self,
input: &Tensor,
cache: &mut Option<AttentionKvCache>,
position_offset: usize,
max_sequence_length: usize,
) -> Tensor {
let attention_output = self.attention.forward_cached(
&self.attention_norm.forward(input),
cache,
position_offset,
max_sequence_length,
);
let after_attention = input + &attention_output;
let ffn_output = self
.feed_forward
.forward(&self.ffn_norm.forward(&after_attention));
&after_attention + &ffn_output
}
#[cfg(feature = "gpu")]
pub fn forward_cached_gpu(
&self,
input: &Tensor,
cache: &mut Option<AttentionKvCache>,
position_offset: usize,
max_sequence_length: usize,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
let normalized = self.attention_norm.forward_gpu(input, gpu)?;
let attention_output = self.attention.forward_cached_gpu(
&normalized,
cache,
position_offset,
max_sequence_length,
gpu,
)?;
let (after_attention, normalized_ffn) = input.gpu_residual_rms_norm(
&attention_output,
&self.ffn_norm.weight.to_gpu(gpu)?,
self.ffn_norm.epsilon,
gpu,
)?;
let ffn_output = self.feed_forward.forward_gpu(&normalized_ffn, gpu)?;
after_attention.gpu_add(&ffn_output, gpu)
}
pub fn parameters(&self) -> Vec<Tensor> {
let mut parameters = Vec::new();
parameters.extend(self.attention_norm.parameters());
parameters.extend(self.attention.parameters());
parameters.extend(self.ffn_norm.parameters());
parameters.extend(self.feed_forward.parameters());
parameters
}
}
#[cfg(test)]
mod tests {
use super::TransformerBlock;
use crate::Tensor;
#[test]
fn transformer_block_preserves_shape() {
let block = TransformerBlock::new(64, 8, 128, 1e-5, 10_000.0, 7);
let input = Tensor::new(vec![0.01; 8 * 64], vec![1, 8, 64], false);
assert_eq!(block.forward(&input).shape(), vec![1, 8, 64]);
}
}