#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::Tensor;
#[derive(Clone)]
pub struct RmsNorm {
pub weight: Tensor,
pub hidden_size: usize,
pub epsilon: f32,
}
impl RmsNorm {
pub fn new(hidden_size: usize, epsilon: f32) -> Self {
assert!(hidden_size > 0, "RMSNorm width must be positive");
assert!(epsilon > 0.0, "RMSNorm epsilon must be positive");
Self {
weight: Tensor::ones(vec![hidden_size], true),
hidden_size,
epsilon,
}
}
pub fn forward(&self, input: &Tensor) -> Tensor {
let shape = input.shape();
assert_eq!(
shape.last(),
Some(&self.hidden_size),
"RMSNorm input width mismatch"
);
let feature_dim = shape.len() - 1;
let squared = input * input;
let mean_square = squared.mean_dim(feature_dim, true);
let epsilon = Tensor::scalar(self.epsilon, false);
let inverse_rms = (&mean_square + &epsilon).powf(-0.5);
let normalized = input * &inverse_rms;
&normalized * &self.weight
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
input
.to_gpu(gpu)?
.gpu_rms_norm(&self.weight.to_gpu(gpu)?, self.epsilon, gpu)
}
pub fn parameters(&self) -> Vec<Tensor> {
vec![self.weight.clone()]
}
}