use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::{Float, MoiraiBackend, Scalar};
use coeus_tensor::Tensor;
#[derive(Clone)]
pub struct RotaryEmbedding<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
pub cos: Tensor<T, B>,
pub sin: Tensor<T, B>,
pub max_len: usize,
pub d_head: usize,
}
impl<T: Float + coeus_core::FloatElement, B: coeus_ops::BackendOps<T> + Default>
RotaryEmbedding<T, B>
{
pub fn new(
max_len: usize,
d_head: usize,
base: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>> {
assert!(
d_head.is_multiple_of(2),
"RotaryEmbedding: d_head must be even, got {d_head}"
);
let backend = B::default();
let mut cos_values = vec![T::ZERO; max_len * d_head];
let mut sin_values = vec![T::ZERO; max_len * d_head];
let half_dim = d_head / 2;
for pos in 0..max_len {
for i in 0..half_dim {
let theta = base.powf(
-2.0 * <f64 as coeus_core::FloatElement>::from_count(i)
/ <f64 as coeus_core::FloatElement>::from_count(d_head),
);
let angle = <f64 as coeus_core::FloatElement>::from_count(pos) * theta;
let c = <T as coeus_core::FloatElement>::from_f64(angle.cos());
let s = <T as coeus_core::FloatElement>::from_f64(angle.sin());
cos_values[pos * d_head + i] = c;
cos_values[pos * d_head + i + half_dim] = c;
sin_values[pos * d_head + i] = s;
sin_values[pos * d_head + i + half_dim] = s;
}
}
let cos_table = Tensor::from_slice_on([max_len, d_head], &cos_values, &backend)?;
let sin_table = Tensor::from_slice_on([max_len, d_head], &sin_values, &backend)?;
Ok(Self {
cos: cos_table,
sin: sin_table,
max_len,
d_head,
})
}
pub fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>>
where
B: coeus_ops::RotateHalfOps<T>,
T: coeus_cpu::RealScalar,
{
let shape = x.tensor.shape();
let ndim = shape.len();
if ndim < 2 {
return Err(ModuleError::InvalidRank {
module: "RotaryEmbedding",
expected: "at least 2",
actual: ndim,
});
}
let seq_len = shape[1];
let d_head = shape[ndim - 1];
if d_head != self.d_head {
return Err(ModuleError::ShapeMismatch {
module: "RotaryEmbedding",
parameter: "input last dimension",
expected: vec![self.d_head],
actual: vec![d_head],
});
}
if seq_len > self.max_len {
return Err(ModuleError::ShapeMismatch {
module: "RotaryEmbedding",
parameter: "input sequence length",
expected: vec![self.max_len],
actual: vec![seq_len],
});
}
let cos_slice = extract_pe_slice(&self.cos, seq_len, self.d_head);
let sin_slice = extract_pe_slice(&self.sin, seq_len, self.d_head);
let mut broadcast_shape = vec![1; ndim];
broadcast_shape[1] = seq_len;
broadcast_shape[ndim - 1] = d_head;
let cos_var = Var::new(cos_slice?.reshape(broadcast_shape.clone()), false)?;
let sin_var = Var::new(sin_slice?.reshape(broadcast_shape), false)?;
let x_cos = coeus_autograd::mul(x, &cos_var)?;
let rx = coeus_autograd::rotate_half(x).map_err(|source| ModuleError::Backend {
module: "RotaryEmbedding",
source,
})?;
let rx_sin = coeus_autograd::mul(&rx, &sin_var)?;
Ok(coeus_autograd::add(&x_cos, &rx_sin)?)
}
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
for RotaryEmbedding<T, B>
where
B: coeus_ops::RotateHalfOps<T>,
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.forward(input)
}
}
fn extract_pe_slice<T: Float, B: coeus_ops::BackendOps<T> + Default>(
table: &Tensor<T, B>,
seq_len: usize,
d_model: usize,
) -> Result<Tensor<T, B>, B::Error> {
Ok(table.slice(&[(0, seq_len), (0, d_model)]))
}