oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
use crate::attention::{AttentionKvCache, MultiHeadAttention};
#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::layers::{Initializer, Linear};
use crate::norm::RmsNorm;
use crate::rope::RopeScaling;
use crate::{IndexTensor, Tensor};

#[derive(Clone)]
pub struct FeedForward {
    pub gate_proj: Linear,
    pub up_proj: Linear,
    pub down_proj: Linear,
}

impl FeedForward {
    pub fn new(hidden_size: usize, intermediate_size: usize, seed: u64) -> Self {
        let mut initializer = Initializer::new(seed);
        Self::from_initializer(hidden_size, intermediate_size, &mut initializer, 1.0, false)
    }

    pub(crate) fn from_initializer(
        hidden_size: usize,
        intermediate_size: usize,
        initializer: &mut Initializer,
        residual_scale: f32,
        use_bias: bool,
    ) -> Self {
        Self {
            gate_proj: Linear::from_initializer(
                hidden_size,
                intermediate_size,
                initializer,
                use_bias,
            ),
            up_proj: Linear::from_initializer(
                hidden_size,
                intermediate_size,
                initializer,
                use_bias,
            ),
            down_proj: Linear::from_initializer_scaled(
                intermediate_size,
                hidden_size,
                initializer,
                residual_scale,
                use_bias,
            ),
        }
    }

    pub fn forward(&self, input: &Tensor) -> Tensor {
        let gate = self.gate_proj.forward(input);
        let activated_gate = &gate * &gate.sigmoid();
        let up = self.up_proj.forward(input);
        self.down_proj.forward(&(&activated_gate * &up))
    }

    #[cfg(feature = "gpu")]
    pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
        let gate = self.gate_proj.forward_gpu(input, gpu)?;
        let up = self.up_proj.forward_gpu(input, gpu)?;
        let fused = gate.gpu_swiglu(&up, gpu)?;
        self.down_proj.forward_gpu(&fused, gpu)
    }

    pub fn parameters(&self) -> Vec<Tensor> {
        let mut parameters = Vec::new();
        parameters.extend(self.gate_proj.parameters());
        parameters.extend(self.up_proj.parameters());
        parameters.extend(self.down_proj.parameters());
        parameters
    }
}

#[derive(Clone)]
pub struct TransformerBlock {
    pub attention_norm: RmsNorm,
    pub attention: MultiHeadAttention,
    pub ffn_norm: RmsNorm,
    pub feed_forward: FeedForward,
}

impl TransformerBlock {
    pub fn new(
        hidden_size: usize,
        num_heads: usize,
        intermediate_size: usize,
        rms_norm_epsilon: f32,
        rope_theta: f32,
        seed: u64,
    ) -> Self {
        let mut initializer = Initializer::new(seed);
        Self::from_initializer(
            hidden_size,
            num_heads,
            num_heads,
            intermediate_size,
            rms_norm_epsilon,
            rope_theta,
            None,
            &mut initializer,
            1.0,
            false,
        )
    }

    pub(crate) fn from_initializer(
        hidden_size: usize,
        num_heads: usize,
        num_key_value_heads: usize,
        intermediate_size: usize,
        rms_norm_epsilon: f32,
        rope_theta: f32,
        rope_scaling: Option<RopeScaling>,
        initializer: &mut Initializer,
        residual_scale: f32,
        use_bias: bool,
    ) -> Self {
        Self {
            attention_norm: RmsNorm::new(hidden_size, rms_norm_epsilon),
            attention: MultiHeadAttention::from_initializer(
                hidden_size,
                num_heads,
                num_key_value_heads,
                rope_theta,
                rope_scaling,
                initializer,
                residual_scale,
                use_bias,
            ),
            ffn_norm: RmsNorm::new(hidden_size, rms_norm_epsilon),
            feed_forward: FeedForward::from_initializer(
                hidden_size,
                intermediate_size,
                initializer,
                residual_scale,
                use_bias,
            ),
        }
    }

    pub fn forward(&self, input: &Tensor) -> Tensor {
        self.forward_with_optional_attention_mask(input, None)
    }

    pub fn forward_with_attention_mask(
        &self,
        input: &Tensor,
        attention_mask: &IndexTensor,
    ) -> Tensor {
        self.forward_with_optional_attention_mask(input, Some(attention_mask))
    }

    fn forward_with_optional_attention_mask(
        &self,
        input: &Tensor,
        attention_mask: Option<&IndexTensor>,
    ) -> Tensor {
        self.forward_training(input, attention_mask, 0.0, 0.0, 0)
    }

    pub(crate) fn forward_training(
        &self,
        input: &Tensor,
        attention_mask: Option<&IndexTensor>,
        attention_dropout: f32,
        residual_dropout: f32,
        seed: u64,
    ) -> Tensor {
        self.forward_training_selective(
            input,
            attention_mask,
            attention_dropout,
            residual_dropout,
            seed,
            crate::trainer::SelectiveCheckpoint::Full,
            0,
        )
    }

    /// Training forward with optional selective activation checkpointing and
    /// sequence-chunked attention (`attention_chunk_size == 0` disables chunking).
    pub(crate) fn forward_training_selective(
        &self,
        input: &Tensor,
        attention_mask: Option<&IndexTensor>,
        attention_dropout: f32,
        residual_dropout: f32,
        seed: u64,
        selective: crate::trainer::SelectiveCheckpoint,
        attention_chunk_size: usize,
    ) -> Tensor {
        let attention_norm = self.attention_norm.clone();
        let attention = self.attention.clone();
        let ffn_norm = self.ffn_norm.clone();
        let feed_forward = self.feed_forward.clone();
        let mask = attention_mask.cloned();
        let attn_seed = crate::derive_seed(seed, 0) as u32;
        let drop_seed_1 = crate::derive_seed(seed, 1);
        let drop_seed_2 = crate::derive_seed(seed, 2);

        let run_attention = move |input: &Tensor| {
            let normalized = attention_norm.forward(input);
            let attention_output = attention.forward_training_chunked(
                &normalized,
                mask.as_ref(),
                attention_dropout,
                attn_seed,
                attention_chunk_size,
            );
            attention_output.dropout(residual_dropout, drop_seed_1)
        };
        let run_ffn = move |after_attention: &Tensor| {
            let ffn_output = feed_forward.forward(&ffn_norm.forward(after_attention));
            ffn_output.dropout(residual_dropout, drop_seed_2)
        };

        match selective {
            crate::trainer::SelectiveCheckpoint::Full => {
                // Caller wraps the whole block when using Full + outer checkpoint.
                let attention_output = run_attention(input);
                let after_attention = input + &attention_output;
                &after_attention + &run_ffn(&after_attention)
            }
            crate::trainer::SelectiveCheckpoint::Attention => {
                let attention_output = crate::checkpoint(input, run_attention);
                let after_attention = input + &attention_output;
                &after_attention + &run_ffn(&after_attention)
            }
            crate::trainer::SelectiveCheckpoint::Ffn => {
                let attention_output = run_attention(input);
                let after_attention = input + &attention_output;
                let ffn_output = crate::checkpoint(&after_attention, run_ffn);
                &after_attention + &ffn_output
            }
        }
    }

    #[cfg(feature = "gpu")]
    pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
        self.forward_gpu_with_optional_attention_mask(input, None, gpu)
    }

    #[cfg(feature = "gpu")]
    pub fn forward_gpu_with_attention_mask(
        &self,
        input: &Tensor,
        attention_mask: &IndexTensor,
        gpu: &GpuContext,
    ) -> Result<Tensor, GpuError> {
        self.forward_gpu_with_optional_attention_mask(input, Some(attention_mask), gpu)
    }

    #[cfg(feature = "gpu")]
    fn forward_gpu_with_optional_attention_mask(
        &self,
        input: &Tensor,
        attention_mask: Option<&IndexTensor>,
        gpu: &GpuContext,
    ) -> Result<Tensor, GpuError> {
        self.forward_training_gpu(input, attention_mask, 0.0, 0.0, 0, gpu)
    }

    #[cfg(feature = "gpu")]
    pub(crate) fn forward_training_gpu(
        &self,
        input: &Tensor,
        attention_mask: Option<&IndexTensor>,
        attention_dropout: f32,
        residual_dropout: f32,
        seed: u64,
        gpu: &GpuContext,
    ) -> Result<Tensor, GpuError> {
        let normalized = self.attention_norm.forward_gpu(input, gpu)?;
        let attention_output = self.attention.forward_training_gpu(
            &normalized,
            attention_mask,
            attention_dropout,
            crate::derive_seed(seed, 0) as u32,
            gpu,
        )?;
        let attention_output =
            attention_output.gpu_dropout(residual_dropout, crate::derive_seed(seed, 1), gpu)?;
        let (after_attention, normalized_ffn) = input.gpu_residual_rms_norm(
            &attention_output,
            &self.ffn_norm.weight.to_gpu(gpu)?,
            self.ffn_norm.epsilon,
            gpu,
        )?;
        let ffn_output = self.feed_forward.forward_gpu(&normalized_ffn, gpu)?;
        let ffn_output =
            ffn_output.gpu_dropout(residual_dropout, crate::derive_seed(seed, 2), gpu)?;
        after_attention.gpu_add(&ffn_output, gpu)
    }

    pub fn forward_cached(
        &self,
        input: &Tensor,
        cache: &mut Option<AttentionKvCache>,
        position_offset: usize,
        max_sequence_length: usize,
    ) -> Tensor {
        let attention_output = self.attention.forward_cached(
            &self.attention_norm.forward(input),
            cache,
            position_offset,
            max_sequence_length,
        );
        let after_attention = input + &attention_output;
        let ffn_output = self
            .feed_forward
            .forward(&self.ffn_norm.forward(&after_attention));
        &after_attention + &ffn_output
    }

    #[cfg(feature = "gpu")]
    pub fn forward_cached_gpu(
        &self,
        input: &Tensor,
        cache: &mut Option<AttentionKvCache>,
        position_offset: usize,
        max_sequence_length: usize,
        gpu: &GpuContext,
    ) -> Result<Tensor, GpuError> {
        let normalized = self.attention_norm.forward_gpu(input, gpu)?;
        let attention_output = self.attention.forward_cached_gpu(
            &normalized,
            cache,
            position_offset,
            max_sequence_length,
            gpu,
        )?;
        let (after_attention, normalized_ffn) = input.gpu_residual_rms_norm(
            &attention_output,
            &self.ffn_norm.weight.to_gpu(gpu)?,
            self.ffn_norm.epsilon,
            gpu,
        )?;
        let ffn_output = self.feed_forward.forward_gpu(&normalized_ffn, gpu)?;
        after_attention.gpu_add(&ffn_output, gpu)
    }

    pub fn parameters(&self) -> Vec<Tensor> {
        let mut parameters = Vec::new();
        parameters.extend(self.attention_norm.parameters());
        parameters.extend(self.attention.parameters());
        parameters.extend(self.ffn_norm.parameters());
        parameters.extend(self.feed_forward.parameters());
        parameters
    }
}

#[cfg(test)]
mod tests {
    use super::TransformerBlock;
    use crate::Tensor;

    #[test]
    fn transformer_block_preserves_shape() {
        let block = TransformerBlock::new(64, 8, 128, 1e-5, 10_000.0, 7);
        let input = Tensor::new(vec![0.01; 8 * 64], vec![1, 8, 64], false);
        assert_eq!(block.forward(&input).shape(), vec![1, 8, 64]);
    }
}