Skip to main content

coeus_nn/
embedding.rs

1// ── Embedding layer module ──
2
3use crate::module::{Module, ModuleError};
4use coeus_autograd::Var;
5use coeus_core::{MoiraiBackend, Scalar};
6use coeus_tensor::Tensor;
7
8/// Embedding layer mapping discrete token indices to dense vectors.
9#[derive(Clone)]
10pub struct Embedding<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
11    /// Weight matrix: [num_embeddings, embedding_dim]
12    pub weight: Var<T, B>,
13    /// Number of embeddings (vocabulary size)
14    pub num_embeddings: usize,
15    /// Dimension of each embedding vector
16    pub embedding_dim: usize,
17    /// Optional index whose row is forced to all-zeros and whose gradient is zeroed.
18    pub padding_idx: Option<usize>,
19}
20
21impl<T: Scalar, B: coeus_ops::BackendOps<T> + Default> Embedding<T, B> {
22    /// Create an Embedding layer with weights initialized to ones.
23    pub fn new(
24        num_embeddings: usize,
25        embedding_dim: usize,
26    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
27        let backend = B::default();
28        let w_tensor = Tensor::ones_on([num_embeddings, embedding_dim], &backend)?;
29        let weight = Var::new(w_tensor, true)?;
30        Ok(Self {
31            weight,
32            num_embeddings,
33            embedding_dim,
34            padding_idx: None,
35        })
36    }
37
38    /// Create with explicit `padding_idx`.
39    ///
40    /// Row `padding_idx` in the weight matrix is zeroed on construction and
41    /// its gradient is zeroed by the autograd embedding backward node.
42    pub fn with_padding_idx(
43        num_embeddings: usize,
44        embedding_dim: usize,
45        padding_idx: usize,
46    ) -> Result<Self, crate::init::InitializationError<B::Error>>
47    where
48        B::DeviceBuffer<T>:
49            coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
50    {
51        assert!(
52            padding_idx < num_embeddings,
53            "Embedding::with_padding_idx: padding_idx {padding_idx} out of bounds [0, {num_embeddings})"
54        );
55        let backend = B::default();
56        let mut w_data = vec![T::ONE; num_embeddings * embedding_dim];
57        let start = padding_idx * embedding_dim;
58        for v in &mut w_data[start..start + embedding_dim] {
59            *v = T::ZERO;
60        }
61        let w_tensor =
62            Tensor::from_slice_on(vec![num_embeddings, embedding_dim], &w_data, &backend)?;
63        let weight = Var::new(w_tensor, true)?;
64        Ok(Self {
65            weight,
66            num_embeddings,
67            embedding_dim,
68            padding_idx: Some(padding_idx),
69        })
70    }
71
72    /// Forward pass using an explicit integer index tensor.
73    ///
74    /// # Errors
75    ///
76    /// Returns [`ModuleError::ShapeMismatch`] when an index is negative or
77    /// outside the configured embedding vocabulary.
78    pub fn forward_indices<I: Scalar + 'static>(
79        &self,
80        indices: &Tensor<I, B>,
81    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
82        validate_indices(indices, self.num_embeddings, "Embedding")?;
83        Ok(coeus_autograd::embedding_with_padding_idx(
84            &self.weight,
85            indices,
86            self.padding_idx,
87        )?)
88    }
89}
90
91impl<T: Scalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Embedding<T, B> {
92    fn parameters(&self) -> Vec<Var<T, B>> {
93        vec![self.weight.clone()]
94    }
95
96    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
97        validate_indices(&input.tensor, self.num_embeddings, "Embedding")?;
98        Ok(coeus_autograd::embedding_with_padding_idx(
99            &self.weight,
100            &input.tensor,
101            self.padding_idx,
102        )?)
103    }
104}
105
106fn validate_indices<I: Scalar, B: coeus_core::ComputeBackend + Default>(
107    indices: &Tensor<I, B>,
108    num_embeddings: usize,
109    module: &'static str,
110) -> Result<(), ModuleError<B::Error>> {
111    let backend = B::default();
112    for (position, &index) in indices.host_cow_on(&backend)?.iter().enumerate() {
113        let value = <I as coeus_core::NumericElement>::to_f64(index);
114        if !value.is_finite()
115            || value < 0.0
116            || value.trunc() != value
117            || value >= coeus_core::FloatElement::from_count(num_embeddings)
118        {
119            return Err(ModuleError::ShapeMismatch {
120                module,
121                parameter: "indices must be finite integers within the embedding vocabulary",
122                expected: vec![num_embeddings],
123                actual: vec![position],
124            });
125        }
126    }
127
128    Ok(())
129}