Skip to main content

coeus_nn/positional/
sinusoidal.rs

1// ── Sinusoidal Positional Encoding ──
2//
3// PE(pos, 2i)   = sin(pos / 10000^(2i / d_model))
4// PE(pos, 2i+1) = cos(pos / 10000^(2i / d_model))
5//
6// The full `[max_len, d_model]` table is precomputed at construction.
7// `forward` adds a non-owning slice view over the selected backend allocation.
8
9use crate::module::{Module, ModuleError};
10use coeus_autograd::Var;
11use coeus_core::{ComputeBackend, Float, FloatElement, MoiraiBackend};
12use coeus_tensor::Tensor;
13
14/// Sinusoidal (non-learnable) positional encoding layer.
15///
16/// Precomputes a `[max_len, d_model]` table and adds a row-slice of it to
17/// the input at forward time.
18pub struct SinusoidalEncoding<T: coeus_core::Scalar, B: ComputeBackend + Default = MoiraiBackend> {
19    /// Precomputed PE table: `[max_len, d_model]`.
20    pub table: Tensor<T, B>,
21    /// Maximum sequence length the precomputed table covers.
22    pub max_len: usize,
23    /// Embedding dimension (must be even).
24    pub d_model: usize,
25}
26
27impl<T: Float + FloatElement, B: ComputeBackend + Default> SinusoidalEncoding<T, B> {
28    /// Build the encoding table.
29    ///
30    /// - `max_len`: maximum sequence length supported.
31    /// - `d_model`: embedding dimension (must be even).
32    pub fn new(
33        max_len: usize,
34        d_model: usize,
35    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
36        assert!(
37            d_model.is_multiple_of(2),
38            "SinusoidalEncoding: d_model must be even, got {d_model}"
39        );
40        let backend = B::default();
41        let table_len = max_len
42            .checked_mul(d_model)
43            .expect("SinusoidalEncoding: table element count overflows usize");
44        let mut values = Vec::with_capacity(table_len);
45        let base = T::from_count(10_000);
46        let dimension = T::from_count(d_model);
47        for pos in 0..max_len {
48            let position = T::from_count(pos);
49            for i in 0..(d_model / 2) {
50                let exponent = T::from_count(
51                    i.checked_mul(2)
52                        .expect("SinusoidalEncoding: frequency index overflows usize"),
53                ) / dimension;
54                let angle = position / Float::powf(base, exponent);
55                values.push(Float::sin(angle));
56                values.push(Float::cos(angle));
57            }
58        }
59        let table = Tensor::from_slice_on([max_len, d_model], &values, &backend)?;
60        Ok(Self {
61            table,
62            max_len,
63            d_model,
64        })
65    }
66}
67
68impl<
69        T: Float + coeus_cpu::RealScalar,
70        B: coeus_ops::ElementwiseOps<T> + coeus_ops::ReductionOps<T> + Default,
71    > Module<T, B> for SinusoidalEncoding<T, B>
72{
73    fn parameters(&self) -> Vec<Var<T, B>> {
74        vec![] // non-learnable
75    }
76
77    /// Add positional encoding to `input`.
78    ///
79    /// - `input`: `[batch, seq_len, d_model]`
80    ///
81    /// Returns `[batch, seq_len, d_model]`.
82    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
83        let shape = input.tensor.shape();
84        if shape.len() != 3 {
85            return Err(ModuleError::InvalidRank {
86                module: "SinusoidalEncoding",
87                expected: "3",
88                actual: shape.len(),
89            });
90        }
91        if shape[2] != self.d_model {
92            return Err(ModuleError::ShapeMismatch {
93                module: "SinusoidalEncoding",
94                parameter: "input last dimension",
95                expected: vec![self.d_model],
96                actual: vec![shape[2]],
97            });
98        }
99        let seq_len = shape[1];
100        if seq_len > self.max_len {
101            return Err(ModuleError::ShapeMismatch {
102                module: "SinusoidalEncoding",
103                parameter: "input sequence length",
104                expected: vec![self.max_len],
105                actual: vec![seq_len],
106            });
107        }
108
109        // The view keeps the selected backend allocation. The elementwise
110        // provider consumes the view layout directly, so forward never stages
111        // or downloads the precomputed table.
112        let pe_slice = prefix_view(&self.table, seq_len, self.d_model);
113        let pe_var = Var::new(pe_slice?, false)?;
114
115        // Broadcast add: input [B, seq, d] + pe [seq, d] via autograd add.
116        // autograd::add handles the broadcast accumulation.
117        Ok(coeus_autograd::add(input, &pe_var)?)
118    }
119}
120
121/// Borrow the active sequence prefix without changing its backend storage.
122#[inline]
123fn prefix_view<T: coeus_core::Scalar, B: ComputeBackend>(
124    table: &Tensor<T, B>,
125    seq_len: usize,
126    d_model: usize,
127) -> Result<Tensor<T, B>, B::Error> {
128    Ok(table.slice(&[(0, seq_len), (0, d_model)]))
129}
130
131#[cfg(test)]
132mod tests {
133    use super::prefix_view;
134    use coeus_core::{SequentialBackend, Storage};
135    use coeus_tensor::Tensor;
136
137    #[test]
138    fn prefix_view_shares_cpu_storage() -> Result<(), Box<dyn std::error::Error>> {
139        let backend = SequentialBackend;
140        let table = Tensor::from_slice_on([4, 6], &[0.0_f32; 24], &backend)?;
141        let prefix = prefix_view(&table, 2, 6)?;
142
143        let table_ptr = table
144            .storage()
145            .try_as_slice()
146            .expect("SequentialBackend storage is CPU-addressable")
147            .as_ptr();
148        let prefix_ptr = prefix
149            .storage()
150            .try_as_slice()
151            .expect("SequentialBackend storage is CPU-addressable")
152            .as_ptr();
153        assert_eq!(prefix_ptr, table_ptr);
154        assert_eq!(prefix.shape(), &[2, 6]);
155        Ok(())
156    }
157}