1use crate::module::{Module, ModuleError};
4use coeus_autograd::Var;
5use coeus_core::{MoiraiBackend, Scalar};
6use coeus_tensor::Tensor;
7
8#[derive(Clone)]
10pub struct Embedding<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
11 pub weight: Var<T, B>,
13 pub num_embeddings: usize,
15 pub embedding_dim: usize,
17 pub padding_idx: Option<usize>,
19}
20
21impl<T: Scalar, B: coeus_ops::BackendOps<T> + Default> Embedding<T, B> {
22 pub fn new(
24 num_embeddings: usize,
25 embedding_dim: usize,
26 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
27 let backend = B::default();
28 let w_tensor = Tensor::ones_on([num_embeddings, embedding_dim], &backend)?;
29 let weight = Var::new(w_tensor, true)?;
30 Ok(Self {
31 weight,
32 num_embeddings,
33 embedding_dim,
34 padding_idx: None,
35 })
36 }
37
38 pub fn with_padding_idx(
43 num_embeddings: usize,
44 embedding_dim: usize,
45 padding_idx: usize,
46 ) -> Result<Self, crate::init::InitializationError<B::Error>>
47 where
48 B::DeviceBuffer<T>:
49 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
50 {
51 assert!(
52 padding_idx < num_embeddings,
53 "Embedding::with_padding_idx: padding_idx {padding_idx} out of bounds [0, {num_embeddings})"
54 );
55 let backend = B::default();
56 let mut w_data = vec![T::ONE; num_embeddings * embedding_dim];
57 let start = padding_idx * embedding_dim;
58 for v in &mut w_data[start..start + embedding_dim] {
59 *v = T::ZERO;
60 }
61 let w_tensor =
62 Tensor::from_slice_on(vec![num_embeddings, embedding_dim], &w_data, &backend)?;
63 let weight = Var::new(w_tensor, true)?;
64 Ok(Self {
65 weight,
66 num_embeddings,
67 embedding_dim,
68 padding_idx: Some(padding_idx),
69 })
70 }
71
72 pub fn forward_indices<I: Scalar + 'static>(
79 &self,
80 indices: &Tensor<I, B>,
81 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
82 validate_indices(indices, self.num_embeddings, "Embedding")?;
83 Ok(coeus_autograd::embedding_with_padding_idx(
84 &self.weight,
85 indices,
86 self.padding_idx,
87 )?)
88 }
89}
90
91impl<T: Scalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Embedding<T, B> {
92 fn parameters(&self) -> Vec<Var<T, B>> {
93 vec![self.weight.clone()]
94 }
95
96 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
97 validate_indices(&input.tensor, self.num_embeddings, "Embedding")?;
98 Ok(coeus_autograd::embedding_with_padding_idx(
99 &self.weight,
100 &input.tensor,
101 self.padding_idx,
102 )?)
103 }
104}
105
106fn validate_indices<I: Scalar, B: coeus_core::ComputeBackend + Default>(
107 indices: &Tensor<I, B>,
108 num_embeddings: usize,
109 module: &'static str,
110) -> Result<(), ModuleError<B::Error>> {
111 let backend = B::default();
112 for (position, &index) in indices.host_cow_on(&backend)?.iter().enumerate() {
113 let value = <I as coeus_core::NumericElement>::to_f64(index);
114 if !value.is_finite()
115 || value < 0.0
116 || value.trunc() != value
117 || value >= coeus_core::FloatElement::from_count(num_embeddings)
118 {
119 return Err(ModuleError::ShapeMismatch {
120 module,
121 parameter: "indices must be finite integers within the embedding vocabulary",
122 expected: vec![num_embeddings],
123 actual: vec![position],
124 });
125 }
126 }
127
128 Ok(())
129}