use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::{ComputeBackend, Float, FloatElement, MoiraiBackend};
use coeus_tensor::Tensor;
pub struct SinusoidalEncoding<T: coeus_core::Scalar, B: ComputeBackend + Default = MoiraiBackend> {
pub table: Tensor<T, B>,
pub max_len: usize,
pub d_model: usize,
}
impl<T: Float + FloatElement, B: ComputeBackend + Default> SinusoidalEncoding<T, B> {
pub fn new(
max_len: usize,
d_model: usize,
) -> Result<Self, crate::init::InitializationError<B::Error>> {
assert!(
d_model.is_multiple_of(2),
"SinusoidalEncoding: d_model must be even, got {d_model}"
);
let backend = B::default();
let table_len = max_len
.checked_mul(d_model)
.expect("SinusoidalEncoding: table element count overflows usize");
let mut values = Vec::with_capacity(table_len);
let base = T::from_count(10_000);
let dimension = T::from_count(d_model);
for pos in 0..max_len {
let position = T::from_count(pos);
for i in 0..(d_model / 2) {
let exponent = T::from_count(
i.checked_mul(2)
.expect("SinusoidalEncoding: frequency index overflows usize"),
) / dimension;
let angle = position / Float::powf(base, exponent);
values.push(Float::sin(angle));
values.push(Float::cos(angle));
}
}
let table = Tensor::from_slice_on([max_len, d_model], &values, &backend)?;
Ok(Self {
table,
max_len,
d_model,
})
}
}
impl<
T: Float + coeus_cpu::RealScalar,
B: coeus_ops::ElementwiseOps<T> + coeus_ops::ReductionOps<T> + Default,
> Module<T, B> for SinusoidalEncoding<T, B>
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![] }
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let shape = input.tensor.shape();
if shape.len() != 3 {
return Err(ModuleError::InvalidRank {
module: "SinusoidalEncoding",
expected: "3",
actual: shape.len(),
});
}
if shape[2] != self.d_model {
return Err(ModuleError::ShapeMismatch {
module: "SinusoidalEncoding",
parameter: "input last dimension",
expected: vec![self.d_model],
actual: vec![shape[2]],
});
}
let seq_len = shape[1];
if seq_len > self.max_len {
return Err(ModuleError::ShapeMismatch {
module: "SinusoidalEncoding",
parameter: "input sequence length",
expected: vec![self.max_len],
actual: vec![seq_len],
});
}
let pe_slice = prefix_view(&self.table, seq_len, self.d_model);
let pe_var = Var::new(pe_slice?, false)?;
Ok(coeus_autograd::add(input, &pe_var)?)
}
}
#[inline]
fn prefix_view<T: coeus_core::Scalar, B: ComputeBackend>(
table: &Tensor<T, B>,
seq_len: usize,
d_model: usize,
) -> Result<Tensor<T, B>, B::Error> {
Ok(table.slice(&[(0, seq_len), (0, d_model)]))
}
#[cfg(test)]
mod tests {
use super::prefix_view;
use coeus_core::{SequentialBackend, Storage};
use coeus_tensor::Tensor;
#[test]
fn prefix_view_shares_cpu_storage() -> Result<(), Box<dyn std::error::Error>> {
let backend = SequentialBackend;
let table = Tensor::from_slice_on([4, 6], &[0.0_f32; 24], &backend)?;
let prefix = prefix_view(&table, 2, 6)?;
let table_ptr = table
.storage()
.try_as_slice()
.expect("SequentialBackend storage is CPU-addressable")
.as_ptr();
let prefix_ptr = prefix
.storage()
.try_as_slice()
.expect("SequentialBackend storage is CPU-addressable")
.as_ptr();
assert_eq!(prefix_ptr, table_ptr);
assert_eq!(prefix.shape(), &[2, 6]);
Ok(())
}
}