#[derive(Debug, Clone)]
pub struct RopeConfig {
pub head_dim: usize,
pub base: f32,
pub max_seq_len: usize,
}
impl RopeConfig {
pub fn llama() -> Self {
Self {
head_dim: 128,
base: 10_000.0,
max_seq_len: 4096,
}
}
pub fn llama3() -> Self {
Self {
head_dim: 128,
base: 500_000.0,
max_seq_len: 8192,
}
}
pub fn mistral() -> Self {
Self {
head_dim: 128,
base: 10_000.0,
max_seq_len: 32768,
}
}
pub fn qwen2() -> Self {
Self {
head_dim: 128,
base: 1_000_000.0,
max_seq_len: 32768,
}
}
pub fn phi3() -> Self {
Self {
head_dim: 96,
base: 10_000.0,
max_seq_len: 4096,
}
}
pub fn gemma2() -> Self {
Self {
head_dim: 256,
base: 10_000.0,
max_seq_len: 8192,
}
}
}
pub(crate) fn rotate_slice(x: &mut [f32], pos: i64, cfg: &RopeConfig) {
let hd = cfg.head_dim;
if hd == 0 {
return;
}
let half = hd / 2;
for chunk in x.chunks_mut(hd) {
for i in 0..half {
let idx1 = 2 * i;
let idx2 = 2 * i + 1;
if idx2 >= chunk.len() {
break;
}
let exponent = 2.0_f64 * i as f64 / hd as f64;
let freq = pos as f64 / (cfg.base as f64).powf(exponent);
let cos_v = freq.cos() as f32;
let sin_v = freq.sin() as f32;
let x0 = chunk[idx1];
let x1 = chunk[idx2];
chunk[idx1] = x0 * cos_v - x1 * sin_v;
chunk[idx2] = x0 * sin_v + x1 * cos_v;
}
}
}
pub fn apply_rope(q: &mut [f32], k: &mut [f32], pos: usize, cfg: &RopeConfig) {
rotate_slice(q, pos as i64, cfg);
rotate_slice(k, pos as i64, cfg);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_rope_identity_at_pos_0() {
let cfg = RopeConfig::llama3();
let original: Vec<f32> = (0..cfg.head_dim).map(|i| i as f32 * 0.1 + 1.0).collect();
let mut q = original.clone();
let mut k = original.clone();
apply_rope(&mut q, &mut k, 0, &cfg);
for (i, (got, &want)) in q.iter().zip(original.iter()).enumerate() {
assert!(
(got - want).abs() < 1e-6,
"q[{i}] mismatch at pos=0: got {got}, expected {want}"
);
}
for (i, (got, &want)) in k.iter().zip(original.iter()).enumerate() {
assert!(
(got - want).abs() < 1e-6,
"k[{i}] mismatch at pos=0: got {got}, expected {want}"
);
}
}
#[test]
fn test_rope_rotates_pos_1() {
let cfg = RopeConfig {
head_dim: 2,
base: 10_000.0,
max_seq_len: 4096,
};
let mut q = vec![1.0f32, 0.0];
let mut k = vec![0.0f32, 1.0];
apply_rope(&mut q, &mut k, 1, &cfg);
let cos1 = (1.0_f64).cos() as f32;
let sin1 = (1.0_f64).sin() as f32;
assert!(
(q[0] - cos1).abs() < 1e-5,
"q[0] = {}, expected cos(1) ≈ {cos1}",
q[0]
);
assert!(
(q[1] - sin1).abs() < 1e-5,
"q[1] = {}, expected sin(1) ≈ {sin1}",
q[1]
);
assert!(
(k[0] - (-sin1)).abs() < 1e-5,
"k[0] = {}, expected -sin(1) ≈ {}",
k[0],
-sin1
);
assert!(
(k[1] - cos1).abs() < 1e-5,
"k[1] = {}, expected cos(1) ≈ {cos1}",
k[1]
);
}
#[test]
fn test_rope_round_trip() {
let cfg = RopeConfig {
head_dim: 4,
base: 10_000.0,
max_seq_len: 4096,
};
let original = vec![1.0f32, 2.0, 3.0, 4.0];
let mut q = original.clone();
let mut k = original.clone();
rotate_slice(&mut q, 7, &cfg);
rotate_slice(&mut k, 7, &cfg);
rotate_slice(&mut q, -7, &cfg);
rotate_slice(&mut k, -7, &cfg);
for (i, (got, &want)) in q.iter().zip(original.iter()).enumerate() {
assert!(
(got - want).abs() < 1e-5,
"q[{i}] round-trip failed: got {got}, expected {want}"
);
}
for (i, (got, &want)) in k.iter().zip(original.iter()).enumerate() {
assert!(
(got - want).abs() < 1e-5,
"k[{i}] round-trip failed: got {got}, expected {want}"
);
}
}
}