oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::Tensor;

#[derive(Clone)]
pub struct RmsNorm {
    pub weight: Tensor,
    pub hidden_size: usize,
    pub epsilon: f32,
}

impl RmsNorm {
    pub fn new(hidden_size: usize, epsilon: f32) -> Self {
        assert!(hidden_size > 0, "RMSNorm width must be positive");
        assert!(epsilon > 0.0, "RMSNorm epsilon must be positive");
        Self {
            weight: Tensor::ones(vec![hidden_size], true),
            hidden_size,
            epsilon,
        }
    }

    pub fn forward(&self, input: &Tensor) -> Tensor {
        let shape = input.shape();
        assert_eq!(
            shape.last(),
            Some(&self.hidden_size),
            "RMSNorm input width mismatch"
        );
        let feature_dim = shape.len() - 1;
        let squared = input * input;
        let mean_square = squared.mean_dim(feature_dim, true);
        let epsilon = Tensor::scalar(self.epsilon, false);
        let inverse_rms = (&mean_square + &epsilon).powf(-0.5);
        let normalized = input * &inverse_rms;
        &normalized * &self.weight
    }

    #[cfg(feature = "gpu")]
    pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
        input
            .to_gpu(gpu)?
            .gpu_rms_norm(&self.weight.to_gpu(gpu)?, self.epsilon, gpu)
    }

    pub fn parameters(&self) -> Vec<Tensor> {
        vec![self.weight.clone()]
    }
}