1use crate::module::{Module, ModuleError};
2use coeus_autograd::{cat, embedding, max_axis, mean_axis, slice, sum_axis, Var};
3use coeus_core::{CpuAddressableStorage, CpuAddressableStorageMut, Float, MoiraiBackend, Scalar};
4use coeus_tensor::Tensor;
5
6#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub enum EmbeddingBagMode {
9 Sum,
11 Mean,
13 Max,
15}
16
17impl EmbeddingBagMode {
18 pub fn parse(s: &str) -> Option<Self> {
20 s.parse().ok()
21 }
22}
23
24impl core::str::FromStr for EmbeddingBagMode {
25 type Err = &'static str;
26
27 fn from_str(s: &str) -> Result<Self, Self::Err> {
28 match s {
29 "sum" => Ok(Self::Sum),
30 "mean" => Ok(Self::Mean),
31 "max" => Ok(Self::Max),
32 _ => Err("mode must be one of: sum, mean, max"),
33 }
34 }
35}
36
37#[derive(Clone)]
39pub struct EmbeddingBag<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
40 pub weight: Var<T, B>,
42 pub num_embeddings: usize,
44 pub embedding_dim: usize,
46 pub mode: EmbeddingBagMode,
48}
49
50impl<
51 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
52 B: coeus_ops::BackendOps<T> + Default,
53 > EmbeddingBag<T, B>
54where
55 B::DeviceBuffer<T>: CpuAddressableStorage<T> + CpuAddressableStorageMut<T>,
56{
57 pub fn new(
59 num_embeddings: usize,
60 embedding_dim: usize,
61 mode: EmbeddingBagMode,
62 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
63 let backend = B::default();
64 let w_tensor = Tensor::ones_on([num_embeddings, embedding_dim], &backend)?;
65 let weight = Var::new(w_tensor, true)?;
66 Ok(Self {
67 weight,
68 num_embeddings,
69 embedding_dim,
70 mode,
71 })
72 }
73
74 fn bag_starts(
75 indices_len: usize,
76 offsets: Option<&[usize]>,
77 ) -> Result<Vec<usize>, ModuleError<B::Error>> {
78 match offsets {
79 Some(offs) => {
80 if offs.is_empty()
81 || !offs.windows(2).all(|window| window[0] <= window[1])
82 || !offs.iter().all(|&offset| offset <= indices_len)
83 {
84 return Err(ModuleError::ShapeMismatch {
85 module: "EmbeddingBag",
86 parameter: "offsets must be non-empty, ordered, and within indices",
87 expected: vec![indices_len],
88 actual: offs.to_vec(),
89 });
90 }
91 Ok(offs.to_vec())
92 }
93 None => Ok(vec![0]),
94 }
95 }
96
97 fn reduce_one_bag(
98 &self,
99 embeddings: &Var<T, B>,
100 start: usize,
101 end: usize,
102 ) -> Result<Var<T, B>, B::Error> {
103 let backend = B::default();
104 let d = self.embedding_dim;
105
106 if start == end {
107 return Var::new(Tensor::zeros_on([1, d], &backend)?, false);
108 }
109
110 let bag = slice(embeddings, &[(start, end), (0, d)])?;
111 match self.mode {
112 EmbeddingBagMode::Sum => sum_axis(&bag, 0),
113 EmbeddingBagMode::Mean => mean_axis(&bag, 0),
114 EmbeddingBagMode::Max => max_axis(&bag, 0),
115 }
116 }
117
118 pub fn forward_with_offsets(
130 &self,
131 indices: &[usize],
132 offsets: Option<&[usize]>,
133 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
134 for (position, &index) in indices.iter().enumerate() {
135 if index >= self.num_embeddings || i64::try_from(index).is_err() {
136 return Err(ModuleError::ShapeMismatch {
137 module: "EmbeddingBag",
138 parameter: "indices must be within the embedding vocabulary",
139 expected: vec![self.num_embeddings],
140 actual: vec![position],
141 });
142 }
143 }
144
145 let backend = B::default();
146 let starts = Self::bag_starts(indices.len(), offsets)?;
147 let idx_data: Vec<i64> = indices
148 .iter()
149 .map(|&index| {
150 i64::try_from(index)
151 .expect("invariant: embedding index range was validated before conversion")
152 })
153 .collect();
154 let idx_tensor = Tensor::from_slice_on([indices.len()], &idx_data, &backend)?;
155 let embeddings = embedding(&self.weight, &idx_tensor)?;
156
157 let rows: Vec<Var<T, B>> = starts
158 .iter()
159 .enumerate()
160 .map(|(bag, &start)| {
161 let end = starts.get(bag + 1).copied().unwrap_or(indices.len());
162 self.reduce_one_bag(&embeddings, start, end)
163 })
164 .collect::<Result<_, _>>()?;
165
166 let row_refs: Vec<&Var<T, B>> = rows.iter().collect();
167 Ok(cat(&row_refs, 0)?)
168 }
169}
170
171impl<
172 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
173 B: coeus_ops::BackendOps<T> + Default,
174 > Module<T, B> for EmbeddingBag<T, B>
175where
176 B::DeviceBuffer<T>: CpuAddressableStorage<T> + CpuAddressableStorageMut<T>,
177{
178 fn parameters(&self) -> Vec<Var<T, B>> {
179 vec![self.weight.clone()]
180 }
181
182 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
184 let mut indices = Vec::with_capacity(input.tensor.numel());
185 for (position, &index) in input.tensor.as_slice().iter().enumerate() {
186 let value = <T as coeus_core::NumericElement>::to_f64(index);
187 if !value.is_finite()
188 || value < 0.0
189 || value.trunc() != value
190 || value >= coeus_core::FloatElement::from_count(self.num_embeddings)
191 || value > i64::MAX as f64
192 {
193 return Err(ModuleError::ShapeMismatch {
194 module: "EmbeddingBag",
195 parameter: "indices must be finite integers within the embedding vocabulary",
196 expected: vec![self.num_embeddings],
197 actual: vec![position],
198 });
199 }
200 indices.push(value as usize);
203 }
204 self.forward_with_offsets(&indices, None)
205 }
206}