coeus_nn/positional/
sinusoidal.rs1use crate::module::{Module, ModuleError};
10use coeus_autograd::Var;
11use coeus_core::{ComputeBackend, Float, FloatElement, MoiraiBackend};
12use coeus_tensor::Tensor;
13
14pub struct SinusoidalEncoding<T: coeus_core::Scalar, B: ComputeBackend + Default = MoiraiBackend> {
19 pub table: Tensor<T, B>,
21 pub max_len: usize,
23 pub d_model: usize,
25}
26
27impl<T: Float + FloatElement, B: ComputeBackend + Default> SinusoidalEncoding<T, B> {
28 pub fn new(
33 max_len: usize,
34 d_model: usize,
35 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
36 assert!(
37 d_model.is_multiple_of(2),
38 "SinusoidalEncoding: d_model must be even, got {d_model}"
39 );
40 let backend = B::default();
41 let table_len = max_len
42 .checked_mul(d_model)
43 .expect("SinusoidalEncoding: table element count overflows usize");
44 let mut values = Vec::with_capacity(table_len);
45 let base = T::from_count(10_000);
46 let dimension = T::from_count(d_model);
47 for pos in 0..max_len {
48 let position = T::from_count(pos);
49 for i in 0..(d_model / 2) {
50 let exponent = T::from_count(
51 i.checked_mul(2)
52 .expect("SinusoidalEncoding: frequency index overflows usize"),
53 ) / dimension;
54 let angle = position / Float::powf(base, exponent);
55 values.push(Float::sin(angle));
56 values.push(Float::cos(angle));
57 }
58 }
59 let table = Tensor::from_slice_on([max_len, d_model], &values, &backend)?;
60 Ok(Self {
61 table,
62 max_len,
63 d_model,
64 })
65 }
66}
67
68impl<
69 T: Float + coeus_cpu::RealScalar,
70 B: coeus_ops::ElementwiseOps<T> + coeus_ops::ReductionOps<T> + Default,
71 > Module<T, B> for SinusoidalEncoding<T, B>
72{
73 fn parameters(&self) -> Vec<Var<T, B>> {
74 vec![] }
76
77 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
83 let shape = input.tensor.shape();
84 if shape.len() != 3 {
85 return Err(ModuleError::InvalidRank {
86 module: "SinusoidalEncoding",
87 expected: "3",
88 actual: shape.len(),
89 });
90 }
91 if shape[2] != self.d_model {
92 return Err(ModuleError::ShapeMismatch {
93 module: "SinusoidalEncoding",
94 parameter: "input last dimension",
95 expected: vec![self.d_model],
96 actual: vec![shape[2]],
97 });
98 }
99 let seq_len = shape[1];
100 if seq_len > self.max_len {
101 return Err(ModuleError::ShapeMismatch {
102 module: "SinusoidalEncoding",
103 parameter: "input sequence length",
104 expected: vec![self.max_len],
105 actual: vec![seq_len],
106 });
107 }
108
109 let pe_slice = prefix_view(&self.table, seq_len, self.d_model);
113 let pe_var = Var::new(pe_slice?, false)?;
114
115 Ok(coeus_autograd::add(input, &pe_var)?)
118 }
119}
120
121#[inline]
123fn prefix_view<T: coeus_core::Scalar, B: ComputeBackend>(
124 table: &Tensor<T, B>,
125 seq_len: usize,
126 d_model: usize,
127) -> Result<Tensor<T, B>, B::Error> {
128 Ok(table.slice(&[(0, seq_len), (0, d_model)]))
129}
130
131#[cfg(test)]
132mod tests {
133 use super::prefix_view;
134 use coeus_core::{SequentialBackend, Storage};
135 use coeus_tensor::Tensor;
136
137 #[test]
138 fn prefix_view_shares_cpu_storage() -> Result<(), Box<dyn std::error::Error>> {
139 let backend = SequentialBackend;
140 let table = Tensor::from_slice_on([4, 6], &[0.0_f32; 24], &backend)?;
141 let prefix = prefix_view(&table, 2, 6)?;
142
143 let table_ptr = table
144 .storage()
145 .try_as_slice()
146 .expect("SequentialBackend storage is CPU-addressable")
147 .as_ptr();
148 let prefix_ptr = prefix
149 .storage()
150 .try_as_slice()
151 .expect("SequentialBackend storage is CPU-addressable")
152 .as_ptr();
153 assert_eq!(prefix_ptr, table_ptr);
154 assert_eq!(prefix.shape(), &[2, 6]);
155 Ok(())
156 }
157}