use burn::{
Tensor,
tensor::FloatDType,
};
use crate::burner::tensor::Widening;
#[derive(Clone, Copy, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
pub struct RmsNormOptions {
pub eps: f32,
#[serde(default)]
pub widen_input: Option<FloatDType>,
#[serde(default = "default_widen_sum")]
pub widen_sum: Option<FloatDType>,
}
fn default_widen_sum() -> Option<FloatDType> {
Some(FloatDType::F32)
}
impl Default for RmsNormOptions {
fn default() -> Self {
Self {
eps: 1e-5,
widen_input: None,
widen_sum: default_widen_sum(),
}
}
}
impl RmsNormOptions {
pub fn with_eps(
mut self,
eps: f32,
) -> Self {
self.eps = eps;
self
}
pub fn with_widen_input(
mut self,
widen_input: Option<FloatDType>,
) -> Self {
self.widen_input = widen_input;
self
}
pub fn with_widen_sum(
mut self,
widen_sum: Option<FloatDType>,
) -> Self {
self.widen_sum = widen_sum;
self
}
pub fn norm<const R: usize>(
&self,
x: Tensor<R>,
) -> Tensor<R> {
Widening::from(self.widen_input).around(x.dtype(), |input| {
let x = input.widen(x);
let rms = Widening::from(self.widen_sum).around(x.dtype(), |sum| {
sum.widen(x.clone())
.square()
.mean_dim(R - 1)
.add_scalar(self.eps)
.sqrt()
});
x / rms
})
}
}
#[cfg(test)]
mod tests {
use burn::{
Tensor,
tensor::Distribution,
};
use serial_test::serial;
use super::*;
use crate::support::testing::{
DeviceMemoryGuard,
performance_device,
};
#[test]
#[serial]
fn test_rms_norm() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let x: Tensor<3> = Tensor::random([2, 3, 4], Distribution::Default, &device);
let options = RmsNormOptions::default();
let y = options.norm(x.clone());
let x_rms = x
.clone()
.square()
.mean_dim(-1)
.add_scalar(options.eps)
.sqrt();
let expected = x / x_rms;
y.to_data().assert_eq(&expected.to_data(), true);
}
#[test]
fn test_widenings() {
use burn::tensor::DType;
let device = crate::support::testing::cpu_device();
let x: Tensor<2> = Tensor::from_data(
[[0.5f32, -1.25, 2.0, 3.5], [-0.75, 4.0, 1.5, -2.25]],
&device,
);
let eps = 1e-5;
let rms = |t: Tensor<2>| t.square().mean_dim(1).add_scalar(eps).sqrt();
let half = x.clone().cast(DType::BF16);
let cases = [
(
RmsNormOptions::default(),
half.clone() / rms(half.clone().cast(DType::F32)).cast(DType::BF16),
),
(
RmsNormOptions::default()
.with_widen_input(Some(FloatDType::F32))
.with_widen_sum(None),
(half.clone().cast(DType::F32) / rms(half.clone().cast(DType::F32)))
.cast(DType::BF16),
),
(
RmsNormOptions::default().with_widen_sum(None),
half.clone() / rms(half.clone()),
),
];
for (options, expected) in cases {
let got = options.norm(half.clone());
assert_eq!(got.dtype(), DType::BF16, "{options:?}");
got.into_data().assert_eq(&expected.into_data(), true);
options.norm(x.clone()).into_data().assert_approx_eq::<f32>(
&RmsNormOptions::default().norm(x.clone()).into_data(),
burn::tensor::Tolerance::absolute(1e-6),
);
}
}
#[test]
fn test_options_serde_defaults() {
let options: RmsNormOptions = serde_json::from_str(r#"{"eps":1e-6}"#).unwrap();
assert_eq!(options, RmsNormOptions::default().with_eps(1e-6));
}
}