use std::ops::Range;
use burn::{
Tensor,
config::Config,
module::Module,
prelude::s,
tensor::{
DType,
Device,
},
};
use crate::{
burner::module::ModuleInit,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::embedding::positional_frequency_table,
};
pub trait RotaryEmbeddingMeta {
fn seq_len(&self) -> usize;
fn head_dim(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct RotaryEmbeddingConfig {
pub seq_len: usize,
pub head_dim: usize,
#[config(default = 10000)]
pub base: usize,
#[config(default = "RotaryDirection::Reverse")]
pub direction: RotaryDirection,
}
#[derive(Config, Copy, Debug, PartialEq, Eq)]
pub enum RotaryDirection {
Reverse,
Forward,
}
impl RotaryDirection {
pub fn rotate(
self,
x1: Tensor<4>,
x2: Tensor<4>,
cos: Tensor<4>,
sin: Tensor<4>,
) -> (Tensor<4>, Tensor<4>) {
match self {
Self::Reverse => (
x1.clone() * cos.clone() + x2.clone() * sin.clone(),
x2 * cos - x1 * sin,
),
Self::Forward => (
x1.clone() * cos.clone() - x2.clone() * sin.clone(),
x2 * cos + x1 * sin,
),
}
}
}
impl RotaryEmbeddingMeta for RotaryEmbeddingConfig {
fn seq_len(&self) -> usize {
self.seq_len
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl ModuleInit<RotaryEmbedding> for RotaryEmbeddingConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<RotaryEmbedding> {
if !self.head_dim.is_multiple_of(2) {
return Err(ConstraintError::new(
"RotaryEmbeddingConfig",
"head_dim",
Rule::NotMultiple {
value: self.head_dim,
of: "2".into(),
},
)
.into());
}
let freq_matrix =
positional_frequency_table(self.seq_len, self.base, self.head_dim, device);
let cos = freq_matrix
.clone()
.cos()
.set_require_grad(false)
.unsqueeze_dim::<3>(1)
.unsqueeze_dim(0);
let sin = freq_matrix
.sin()
.set_require_grad(false)
.unsqueeze_dim::<3>(1)
.unsqueeze_dim(0);
Ok(RotaryEmbedding {
head_dim: self.head_dim,
direction: self.direction,
cos,
sin,
})
}
}
#[derive(Module, Debug)]
pub struct RotaryEmbedding {
pub head_dim: usize,
#[module(skip)]
pub direction: RotaryDirection,
pub cos: Tensor<4>,
pub sin: Tensor<4>,
}
impl RotaryEmbeddingMeta for RotaryEmbedding {
fn seq_len(&self) -> usize {
self.cos.dims()[1]
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl RotaryEmbedding {
pub fn cast(
self,
dtype: DType,
) -> Self {
Self {
cos: self.cos.cast(dtype),
sin: self.sin.cast(dtype),
..self
}
}
pub fn clip_range(
&self,
range: Range<usize>,
) -> Self {
Self {
head_dim: self.head_dim,
direction: self.direction,
cos: self.cos.clone().slice_dim(1, range.clone()),
sin: self.sin.clone().slice_dim(1, range),
}
}
pub fn apply(
&self,
input: Tensor<4>,
) -> Tensor<4> {
#[cfg(debug_assertions)]
let [b, h] = crate::contracts::unpack_shape_contract!(
["B", "T", "H", "D"],
&input.dims(),
&["B", "H"],
&[("T", self.seq_len()), ("D", self.head_dim())]
);
let pivot = self.head_dim() / 2;
let x1 = input.clone().slice_dim(3, s![..pivot]);
let x2 = input.clone().slice_dim(3, s![pivot..]);
let (y1, y2) = self
.direction
.rotate(x1, x2, self.cos.clone(), self.sin.clone());
let output = Tensor::cat(vec![y1, y2], 3);
#[cfg(debug_assertions)]
crate::contracts::assert_shape_contract_periodically!(
["B", "T", "H", "D"],
&output.dims(),
&[
("B", b),
("T", self.seq_len()),
("H", h),
("D", self.head_dim())
]
);
output
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
performance_device,
},
};
#[test]
#[serial]
fn test_clip_range() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let config = RotaryEmbeddingConfig::new(1024, 64);
let re: RotaryEmbedding = config.init(&device);
assert_eq!(re.seq_len(), 1024);
assert_eq!(re.head_dim(), 64);
let clip_re = re.clip_range(10..20);
assert_eq!(clip_re.seq_len(), 10);
clip_re
.sin
.clone()
.to_data()
.assert_eq(&re.sin.clone().slice_dim(1, 10..20).to_data(), true);
clip_re
.cos
.clone()
.to_data()
.assert_eq(&re.cos.clone().slice_dim(1, 10..20).to_data(), true);
}
#[test]
#[serial]
fn test_rotary_embedding() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let batch = 1;
let heads = 2;
let seq_len = 1024;
let head_dim = 64;
let config = RotaryEmbeddingConfig::new(seq_len, head_dim);
assert_eq!(config.seq_len(), seq_len);
assert_eq!(config.head_dim(), head_dim);
assert_eq!(config.base, 10000);
let re: RotaryEmbedding = config.init(&device);
assert_eq!(re.seq_len(), seq_len);
assert_eq!(re.head_dim(), head_dim);
let input: Tensor<4> = Tensor::random(
[batch, seq_len, heads, head_dim],
Distribution::Default,
&device,
);
let output = re.apply(input.clone());
assert_shape_contract!(
["B", "T", "H", "D"],
&output.dims(),
&[("B", batch), ("T", seq_len), ("H", heads), ("D", head_dim)]
);
let x1 = input.clone().slice_dim(3, s![..head_dim / 2]);
let x2 = input.clone().slice_dim(3, s![head_dim / 2..]);
let y1 = x1.clone() * re.cos.clone() + x2.clone() * re.sin.clone();
let y2 = x1 * (-re.sin.clone()) + x2 * re.cos.clone();
let expected = Tensor::cat(vec![y1, y2], 3);
expected
.to_data()
.assert_approx_eq(&output.to_data(), Tolerance::<f32>::default());
}
#[test]
fn test_directions() {
use crate::support::testing::cpu_device;
let device = cpu_device();
let x = Tensor::<4>::from_data([[[[0.0f32, 0.0]]], [[[1.0, 0.0]]]], &device)
.reshape([1, 2, 1, 2]);
for (direction, sign) in [
(RotaryDirection::Forward, 1.0f32),
(RotaryDirection::Reverse, -1.0),
] {
let re: RotaryEmbedding = RotaryEmbeddingConfig::new(2, 2)
.with_direction(direction)
.init(&device);
let y: Vec<f32> = re
.apply(x.clone())
.into_data()
.try_into_vec_as::<f32>()
.unwrap();
assert!((y[2] - 1f32.cos()).abs() < 1e-6, "{direction:?}: {y:?}");
assert!(
(y[3] - sign * 1f32.sin()).abs() < 1e-6,
"{direction:?}: {y:?}"
);
}
}
}