coeus_nn/positional/
rope.rs1use crate::module::{Module, ModuleError};
11use coeus_autograd::Var;
12use coeus_core::{Float, MoiraiBackend, Scalar};
13use coeus_tensor::Tensor;
14
15#[derive(Clone)]
20pub struct RotaryEmbedding<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
21 pub cos: Tensor<T, B>,
23 pub sin: Tensor<T, B>,
25 pub max_len: usize,
27 pub d_head: usize,
29}
30
31impl<T: Float + coeus_core::FloatElement, B: coeus_ops::BackendOps<T> + Default>
32 RotaryEmbedding<T, B>
33{
34 pub fn new(
40 max_len: usize,
41 d_head: usize,
42 base: f64,
43 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
44 assert!(
45 d_head.is_multiple_of(2),
46 "RotaryEmbedding: d_head must be even, got {d_head}"
47 );
48 let backend = B::default();
49
50 let mut cos_values = vec![T::ZERO; max_len * d_head];
51 let mut sin_values = vec![T::ZERO; max_len * d_head];
52 let half_dim = d_head / 2;
53 for pos in 0..max_len {
54 for i in 0..half_dim {
55 let theta = base.powf(
56 -2.0 * <f64 as coeus_core::FloatElement>::from_count(i)
57 / <f64 as coeus_core::FloatElement>::from_count(d_head),
58 );
59 let angle = <f64 as coeus_core::FloatElement>::from_count(pos) * theta;
60 let c = <T as coeus_core::FloatElement>::from_f64(angle.cos());
61 let s = <T as coeus_core::FloatElement>::from_f64(angle.sin());
62
63 cos_values[pos * d_head + i] = c;
65 cos_values[pos * d_head + i + half_dim] = c;
66 sin_values[pos * d_head + i] = s;
67 sin_values[pos * d_head + i + half_dim] = s;
68 }
69 }
70 let cos_table = Tensor::from_slice_on([max_len, d_head], &cos_values, &backend)?;
71 let sin_table = Tensor::from_slice_on([max_len, d_head], &sin_values, &backend)?;
72
73 Ok(Self {
74 cos: cos_table,
75 sin: sin_table,
76 max_len,
77 d_head,
78 })
79 }
80
81 pub fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>>
86 where
87 B: coeus_ops::RotateHalfOps<T>,
88 T: coeus_cpu::RealScalar,
89 {
90 let shape = x.tensor.shape();
91 let ndim = shape.len();
92 if ndim < 2 {
93 return Err(ModuleError::InvalidRank {
94 module: "RotaryEmbedding",
95 expected: "at least 2",
96 actual: ndim,
97 });
98 }
99 let seq_len = shape[1];
100 let d_head = shape[ndim - 1];
101 if d_head != self.d_head {
102 return Err(ModuleError::ShapeMismatch {
103 module: "RotaryEmbedding",
104 parameter: "input last dimension",
105 expected: vec![self.d_head],
106 actual: vec![d_head],
107 });
108 }
109 if seq_len > self.max_len {
110 return Err(ModuleError::ShapeMismatch {
111 module: "RotaryEmbedding",
112 parameter: "input sequence length",
113 expected: vec![self.max_len],
114 actual: vec![seq_len],
115 });
116 }
117
118 let cos_slice = extract_pe_slice(&self.cos, seq_len, self.d_head);
120 let sin_slice = extract_pe_slice(&self.sin, seq_len, self.d_head);
121
122 let mut broadcast_shape = vec![1; ndim];
124 broadcast_shape[1] = seq_len;
125 broadcast_shape[ndim - 1] = d_head;
126
127 let cos_var = Var::new(cos_slice?.reshape(broadcast_shape.clone()), false)?;
128 let sin_var = Var::new(sin_slice?.reshape(broadcast_shape), false)?;
129
130 let x_cos = coeus_autograd::mul(x, &cos_var)?;
132 let rx = coeus_autograd::rotate_half(x).map_err(|source| ModuleError::Backend {
133 module: "RotaryEmbedding",
134 source,
135 })?;
136 let rx_sin = coeus_autograd::mul(&rx, &sin_var)?;
137
138 Ok(coeus_autograd::add(&x_cos, &rx_sin)?)
139 }
140}
141
142impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
143 for RotaryEmbedding<T, B>
144where
145 B: coeus_ops::RotateHalfOps<T>,
146{
147 fn parameters(&self) -> Vec<Var<T, B>> {
148 vec![]
149 }
150
151 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
152 self.forward(input)
153 }
154}
155
156fn extract_pe_slice<T: Float, B: coeus_ops::BackendOps<T> + Default>(
158 table: &Tensor<T, B>,
159 seq_len: usize,
160 d_model: usize,
161) -> Result<Tensor<T, B>, B::Error> {
162 Ok(table.slice(&[(0, seq_len), (0, d_model)]))
163}