Skip to main content

coeus_nn/positional/
rope.rs

1// ── Rotary Positional Embedding (RoPE) ──
2//
3// Applies a rotation to the input tensor (assumed shape [batch, seq_len, num_heads, d_head])
4// using precomputed cos and sin tables.
5//
6// x_rotated = x * cos + rotate_half(x) * sin
7//
8// where rotate_half([x1, x2]) = [-x2, x1].
9
10use crate::module::{Module, ModuleError};
11use coeus_autograd::Var;
12use coeus_core::{Float, MoiraiBackend, Scalar};
13use coeus_tensor::Tensor;
14
15/// Rotary Positional Embedding (RoPE) layer.
16///
17/// Precomputes `cos` and `sin` tables of shape `[max_len, d_head]`,
18/// and applies them to query/key tensors of shape `[batch, seq_len, num_heads, d_head]`.
19#[derive(Clone)]
20pub struct RotaryEmbedding<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
21    /// Precomputed cos table: `[max_len, d_head]`.
22    pub cos: Tensor<T, B>,
23    /// Precomputed sin table: `[max_len, d_head]`.
24    pub sin: Tensor<T, B>,
25    /// Maximum sequence length supported by the precomputed tables.
26    pub max_len: usize,
27    /// Dimension per attention head (must be even).
28    pub d_head: usize,
29}
30
31impl<T: Float + coeus_core::FloatElement, B: coeus_ops::BackendOps<T> + Default>
32    RotaryEmbedding<T, B>
33{
34    /// Create a new Rotary Embedding layer.
35    ///
36    /// - `max_len`: maximum sequence length.
37    /// - `d_head`: dimension per attention head (must be even).
38    /// - `base`: base value for theta (typically 10000.0).
39    pub fn new(
40        max_len: usize,
41        d_head: usize,
42        base: f64,
43    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
44        assert!(
45            d_head.is_multiple_of(2),
46            "RotaryEmbedding: d_head must be even, got {d_head}"
47        );
48        let backend = B::default();
49
50        let mut cos_values = vec![T::ZERO; max_len * d_head];
51        let mut sin_values = vec![T::ZERO; max_len * d_head];
52        let half_dim = d_head / 2;
53        for pos in 0..max_len {
54            for i in 0..half_dim {
55                let theta = base.powf(
56                    -2.0 * <f64 as coeus_core::FloatElement>::from_count(i)
57                        / <f64 as coeus_core::FloatElement>::from_count(d_head),
58                );
59                let angle = <f64 as coeus_core::FloatElement>::from_count(pos) * theta;
60                let c = <T as coeus_core::FloatElement>::from_f64(angle.cos());
61                let s = <T as coeus_core::FloatElement>::from_f64(angle.sin());
62
63                // GPT-NeoX/LLaMA stores each frequency in both half-vectors.
64                cos_values[pos * d_head + i] = c;
65                cos_values[pos * d_head + i + half_dim] = c;
66                sin_values[pos * d_head + i] = s;
67                sin_values[pos * d_head + i + half_dim] = s;
68            }
69        }
70        let cos_table = Tensor::from_slice_on([max_len, d_head], &cos_values, &backend)?;
71        let sin_table = Tensor::from_slice_on([max_len, d_head], &sin_values, &backend)?;
72
73        Ok(Self {
74            cos: cos_table,
75            sin: sin_table,
76            max_len,
77            d_head,
78        })
79    }
80
81    /// Forward pass applying RoPE to `x`.
82    ///
83    /// `x` is assumed to have shape `[batch, seq_len, num_heads, d_head]` or any shape
84    /// where dimension 1 is `seq_len` and the last dimension is `d_head`.
85    pub fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>>
86    where
87        B: coeus_ops::RotateHalfOps<T>,
88        T: coeus_cpu::RealScalar,
89    {
90        let shape = x.tensor.shape();
91        let ndim = shape.len();
92        if ndim < 2 {
93            return Err(ModuleError::InvalidRank {
94                module: "RotaryEmbedding",
95                expected: "at least 2",
96                actual: ndim,
97            });
98        }
99        let seq_len = shape[1];
100        let d_head = shape[ndim - 1];
101        if d_head != self.d_head {
102            return Err(ModuleError::ShapeMismatch {
103                module: "RotaryEmbedding",
104                parameter: "input last dimension",
105                expected: vec![self.d_head],
106                actual: vec![d_head],
107            });
108        }
109        if seq_len > self.max_len {
110            return Err(ModuleError::ShapeMismatch {
111                module: "RotaryEmbedding",
112                parameter: "input sequence length",
113                expected: vec![self.max_len],
114                actual: vec![seq_len],
115            });
116        }
117
118        // Extract the top `seq_len` rows from the PE tables.
119        let cos_slice = extract_pe_slice(&self.cos, seq_len, self.d_head);
120        let sin_slice = extract_pe_slice(&self.sin, seq_len, self.d_head);
121
122        // Reshape cos and sin slices to [1, seq_len, 1, ..., d_head] for broadcasting
123        let mut broadcast_shape = vec![1; ndim];
124        broadcast_shape[1] = seq_len;
125        broadcast_shape[ndim - 1] = d_head;
126
127        let cos_var = Var::new(cos_slice?.reshape(broadcast_shape.clone()), false)?;
128        let sin_var = Var::new(sin_slice?.reshape(broadcast_shape), false)?;
129
130        // x_rot = x * cos + rotate_half(x) * sin
131        let x_cos = coeus_autograd::mul(x, &cos_var)?;
132        let rx = coeus_autograd::rotate_half(x).map_err(|source| ModuleError::Backend {
133            module: "RotaryEmbedding",
134            source,
135        })?;
136        let rx_sin = coeus_autograd::mul(&rx, &sin_var)?;
137
138        Ok(coeus_autograd::add(&x_cos, &rx_sin)?)
139    }
140}
141
142impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
143    for RotaryEmbedding<T, B>
144where
145    B: coeus_ops::RotateHalfOps<T>,
146{
147    fn parameters(&self) -> Vec<Var<T, B>> {
148        vec![]
149    }
150
151    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
152        self.forward(input)
153    }
154}
155
156/// Helper function to extract the first `seq_len` rows from a table.
157fn extract_pe_slice<T: Float, B: coeus_ops::BackendOps<T> + Default>(
158    table: &Tensor<T, B>,
159    seq_len: usize,
160    d_model: usize,
161) -> Result<Tensor<T, B>, B::Error> {
162    Ok(table.slice(&[(0, seq_len), (0, d_model)]))
163}