Skip to main content

coeus_nn/
embeddingbag.rs

1use crate::module::{Module, ModuleError};
2use coeus_autograd::{cat, embedding, max_axis, mean_axis, slice, sum_axis, Var};
3use coeus_core::{CpuAddressableStorage, CpuAddressableStorageMut, Float, MoiraiBackend, Scalar};
4use coeus_tensor::Tensor;
5
6/// Aggregation mode for EmbeddingBag.
7#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub enum EmbeddingBagMode {
9    /// Sum of embedding vectors in each bag.
10    Sum,
11    /// Mean of embedding vectors in each bag.
12    Mean,
13    /// Element-wise maximum over embedding vectors in each bag.
14    Max,
15}
16
17impl EmbeddingBagMode {
18    /// Parse from a string ("sum", "mean", "max").
19    pub fn parse(s: &str) -> Option<Self> {
20        s.parse().ok()
21    }
22}
23
24impl core::str::FromStr for EmbeddingBagMode {
25    type Err = &'static str;
26
27    fn from_str(s: &str) -> Result<Self, Self::Err> {
28        match s {
29            "sum" => Ok(Self::Sum),
30            "mean" => Ok(Self::Mean),
31            "max" => Ok(Self::Max),
32            _ => Err("mode must be one of: sum, mean, max"),
33        }
34    }
35}
36
37/// EmbeddingBag layer: aggregates embedding rows by bag using sum, mean, or max.
38#[derive(Clone)]
39pub struct EmbeddingBag<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
40    /// Weight matrix: `[num_embeddings, embedding_dim]`.
41    pub weight: Var<T, B>,
42    /// Number of rows in the weight matrix.
43    pub num_embeddings: usize,
44    /// Embedding vector dimension.
45    pub embedding_dim: usize,
46    /// Aggregation mode (sum / mean / max).
47    pub mode: EmbeddingBagMode,
48}
49
50impl<
51        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
52        B: coeus_ops::BackendOps<T> + Default,
53    > EmbeddingBag<T, B>
54where
55    B::DeviceBuffer<T>: CpuAddressableStorage<T> + CpuAddressableStorageMut<T>,
56{
57    /// Create an EmbeddingBag with unit weight matrix.
58    pub fn new(
59        num_embeddings: usize,
60        embedding_dim: usize,
61        mode: EmbeddingBagMode,
62    ) -> Result<Self, crate::init::InitializationError<B::Error>> {
63        let backend = B::default();
64        let w_tensor = Tensor::ones_on([num_embeddings, embedding_dim], &backend)?;
65        let weight = Var::new(w_tensor, true)?;
66        Ok(Self {
67            weight,
68            num_embeddings,
69            embedding_dim,
70            mode,
71        })
72    }
73
74    fn bag_starts(
75        indices_len: usize,
76        offsets: Option<&[usize]>,
77    ) -> Result<Vec<usize>, ModuleError<B::Error>> {
78        match offsets {
79            Some(offs) => {
80                if offs.is_empty()
81                    || !offs.windows(2).all(|window| window[0] <= window[1])
82                    || !offs.iter().all(|&offset| offset <= indices_len)
83                {
84                    return Err(ModuleError::ShapeMismatch {
85                        module: "EmbeddingBag",
86                        parameter: "offsets must be non-empty, ordered, and within indices",
87                        expected: vec![indices_len],
88                        actual: offs.to_vec(),
89                    });
90                }
91                Ok(offs.to_vec())
92            }
93            None => Ok(vec![0]),
94        }
95    }
96
97    fn reduce_one_bag(
98        &self,
99        embeddings: &Var<T, B>,
100        start: usize,
101        end: usize,
102    ) -> Result<Var<T, B>, B::Error> {
103        let backend = B::default();
104        let d = self.embedding_dim;
105
106        if start == end {
107            return Var::new(Tensor::zeros_on([1, d], &backend)?, false);
108        }
109
110        let bag = slice(embeddings, &[(start, end), (0, d)])?;
111        match self.mode {
112            EmbeddingBagMode::Sum => sum_axis(&bag, 0),
113            EmbeddingBagMode::Mean => mean_axis(&bag, 0),
114            EmbeddingBagMode::Max => max_axis(&bag, 0),
115        }
116    }
117
118    /// Forward pass with explicit bag offsets.
119    ///
120    /// `indices` is a flattened list of token ids and `offsets` are bag start
121    /// positions into that list. If `offsets` is `None`, `indices` is treated as one bag.
122    ///
123    /// Returns a tensor of shape `[num_bags, embedding_dim]`.
124    ///
125    /// # Errors
126    ///
127    /// Returns [`ModuleError::ShapeMismatch`] when an index is outside the
128    /// vocabulary or offsets are empty, unordered, or outside `indices`.
129    pub fn forward_with_offsets(
130        &self,
131        indices: &[usize],
132        offsets: Option<&[usize]>,
133    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
134        for (position, &index) in indices.iter().enumerate() {
135            if index >= self.num_embeddings || i64::try_from(index).is_err() {
136                return Err(ModuleError::ShapeMismatch {
137                    module: "EmbeddingBag",
138                    parameter: "indices must be within the embedding vocabulary",
139                    expected: vec![self.num_embeddings],
140                    actual: vec![position],
141                });
142            }
143        }
144
145        let backend = B::default();
146        let starts = Self::bag_starts(indices.len(), offsets)?;
147        let idx_data: Vec<i64> = indices
148            .iter()
149            .map(|&index| {
150                i64::try_from(index)
151                    .expect("invariant: embedding index range was validated before conversion")
152            })
153            .collect();
154        let idx_tensor = Tensor::from_slice_on([indices.len()], &idx_data, &backend)?;
155        let embeddings = embedding(&self.weight, &idx_tensor)?;
156
157        let rows: Vec<Var<T, B>> = starts
158            .iter()
159            .enumerate()
160            .map(|(bag, &start)| {
161                let end = starts.get(bag + 1).copied().unwrap_or(indices.len());
162                self.reduce_one_bag(&embeddings, start, end)
163            })
164            .collect::<Result<_, _>>()?;
165
166        let row_refs: Vec<&Var<T, B>> = rows.iter().collect();
167        Ok(cat(&row_refs, 0)?)
168    }
169}
170
171impl<
172        T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
173        B: coeus_ops::BackendOps<T> + Default,
174    > Module<T, B> for EmbeddingBag<T, B>
175where
176    B::DeviceBuffer<T>: CpuAddressableStorage<T> + CpuAddressableStorageMut<T>,
177{
178    fn parameters(&self) -> Vec<Var<T, B>> {
179        vec![self.weight.clone()]
180    }
181
182    /// Forward pass treating `input` as flat index tensor for a single bag.
183    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
184        let mut indices = Vec::with_capacity(input.tensor.numel());
185        for (position, &index) in input.tensor.as_slice().iter().enumerate() {
186            let value = <T as coeus_core::NumericElement>::to_f64(index);
187            if !value.is_finite()
188                || value < 0.0
189                || value.trunc() != value
190                || value >= coeus_core::FloatElement::from_count(self.num_embeddings)
191                || value > i64::MAX as f64
192            {
193                return Err(ModuleError::ShapeMismatch {
194                    module: "EmbeddingBag",
195                    parameter: "indices must be finite integers within the embedding vocabulary",
196                    expected: vec![self.num_embeddings],
197                    actual: vec![position],
198                });
199            }
200            // SAFETY CONTRACT: the finite, integral, non-negative vocabulary
201            // bounds above make this float-to-index conversion lossless.
202            indices.push(value as usize);
203        }
204        self.forward_with_offsets(&indices, None)
205    }
206}