Skip to main content

incin_core/nn/
embedding.rs

1use crate::backend_authoring::SupportsDType;
2use crate::dist::placement::Local;
3use crate::err::{Error, Result};
4use crate::exec::catalog::{NoAttributes, op};
5use crate::exec::context::ExecutionContext;
6use crate::exec::dispatch;
7use crate::exec::request::TensorHandle;
8use crate::nn::{Module, Param};
9use crate::shapes::error::OperationKind;
10use crate::shapes::shape::shape_buf_from_dims;
11use crate::shapes::{AppendDim, Dim, Dyn, DynShape, Shape, ShapeValue};
12use crate::tensor::backend::Execute;
13use crate::tensor::base::Tensor;
14use crate::tensor::device::Device;
15use crate::tensor::dtype::DType;
16
17/// A shape marker trait specifying an [`Embedding`] layer's vocabulary size
18/// and embedding dimension, analogous to [`crate::nn::linear::LinearShape`].
19/// The typical usage is `(Vocab, Embed)` for a static layer, or `Dyn` for
20/// runtime-determined sizes.
21pub trait EmbeddingShape: Shape + DynShape {
22    /// The vocabulary size (number of distinct token ids).
23    type Vocab: Dim;
24    /// The embedding dimension (length of each row vector).
25    type Embed: Dim;
26    /// The shape argument type used to construct the weight tensor.
27    type BuildArg: crate::tensor::arg_into::NotUnit;
28    /// The static shape type of the embedding weight parameter.
29    type WeightShape: Shape<Arg = Self::BuildArg> + DynShape;
30
31    /// Converts the target arguments into concrete shape args for weight and bias tensors.
32    fn build_args(target: (<Self::Vocab as Dim>::Arg, <Self::Embed as Dim>::Arg))
33    -> Self::BuildArg;
34}
35
36impl<V: Dim, E: Dim> EmbeddingShape
37    for crate::shapes::shape::DimCons<
38        V,
39        crate::shapes::shape::DimCons<E, crate::shapes::shape::Nil>,
40    >
41{
42    type Vocab = V;
43    type Embed = E;
44    type BuildArg = (<V as Dim>::Arg, (<E as Dim>::Arg, ()));
45    type WeightShape = crate::shapes::shape::DimCons<
46        V,
47        crate::shapes::shape::DimCons<E, crate::shapes::shape::Nil>,
48    >;
49
50    fn build_args(
51        target: (<Self::Vocab as Dim>::Arg, <Self::Embed as Dim>::Arg),
52    ) -> Self::BuildArg {
53        (target.0, (target.1, ()))
54    }
55}
56
57impl EmbeddingShape for Dyn {
58    /// The vocabulary size, resolved at runtime.
59    type Vocab = usize;
60    /// The embedding dimension, resolved at runtime.
61    type Embed = usize;
62    /// The weight tensor's shape as a `Vec<usize>`.
63    type BuildArg = alloc::vec::Vec<usize>;
64    type WeightShape = Dyn;
65
66    /// Converts the target arguments into concrete shape args for weight and bias tensors.
67    fn build_args(
68        target: (<Self::Vocab as Dim>::Arg, <Self::Embed as Dim>::Arg),
69    ) -> Self::BuildArg {
70        alloc::vec![target.0, target.1]
71    }
72}
73
74use crate::nn::param::{Frozen, TrainState, Trainable};
75use core::marker::PhantomData;
76
77#[derive(Debug, Clone)]
78#[incin_macros::module(internal)]
79/// An embedding table: maps integer token ids to dense vectors via row
80/// lookup in a learnable `[vocab_size, embed_dim]` weight matrix.
81pub struct Embedding<
82    S: EmbeddingShape,
83    B: crate::tensor::backend::VariableBackend,
84    K: DType = f32,
85    Train: TrainState = Trainable,
86> {
87    /// The learnable weight matrix parameter.
88    pub weight: Param<S::WeightShape, B, K, Train>,
89}
90
91impl<S: EmbeddingShape, B: crate::tensor::backend::VariableBackend, K: DType, Train: TrainState>
92    Embedding<S, B, K, Train>
93{
94    /// Constructs an Embedding from a raw weight parameter.
95    pub fn from_raw_parts(weight: Param<S::WeightShape, B, K, Train>) -> Self {
96        Self { weight }
97    }
98
99    /// Freezes this layer's parameters.
100    pub fn freeze(self) -> Embedding<S, B, K, Frozen> {
101        Embedding {
102            weight: self.weight.freeze(),
103        }
104    }
105
106    /// Unfreezes this layer's parameters.
107    pub fn unfreeze(self) -> Embedding<S, B, K, Trainable> {
108        Embedding {
109            weight: self.weight.unfreeze(),
110        }
111    }
112}
113
114/// A builder for constructing an [`Embedding`] layer with a target.
115#[derive(Debug, Clone)]
116pub struct EmbeddingBuilder<S: EmbeddingShape, Train: TrainState = Trainable> {
117    /// (vocab, embed) table shape.
118    pub shape: ShapeValue<S>,
119    /// Initialization scheme for the table.
120    pub weight_init: crate::nn::init::Init,
121    /// Train-state marker.
122    pub _train: PhantomData<Train>,
123}
124
125/// Creates a new builder for an [`Embedding`] layer with shape `shape`.
126pub fn embedding<S: EmbeddingShape>(shape: ShapeValue<S>) -> EmbeddingBuilder<S> {
127    EmbeddingBuilder {
128        shape,
129        weight_init: crate::nn::init::Init::Randn,
130        _train: PhantomData,
131    }
132}
133
134impl<S: EmbeddingShape, Train: TrainState> EmbeddingBuilder<S, Train> {
135    /// Configures weight initialization.
136    pub fn weight_init(mut self, init: crate::nn::init::Init) -> Self {
137        self.weight_init = init;
138        self
139    }
140
141    /// Marks the resulting layer as frozen (non-trainable).
142    pub fn frozen(self) -> EmbeddingBuilder<S, Frozen> {
143        EmbeddingBuilder {
144            shape: self.shape,
145            weight_init: self.weight_init,
146            _train: PhantomData,
147        }
148    }
149}
150
151impl<
152    S: EmbeddingShape,
153    B: crate::tensor::backend::VariableBackend
154        + crate::tensor::backend::SupportsDType<K>
155        + crate::nn::param::ParameterInit<K>,
156    K: DType,
157> Embedding<S, B, K, Trainable>
158where
159    B: SupportsDType<K>,
160{
161    /// Builds the embedding from vocab/embed arguments.
162    pub fn build<A>(args: A) -> Result<Self>
163    where
164        A: crate::tensor::arg_into::LayerArgInto<(
165                <S::Vocab as Dim>::Arg,
166                <S::Embed as Dim>::Arg,
167                <K as DType>::Arg,
168                <B::Device as Device>::Arg,
169            )>,
170    {
171        let (vocab, embed, dtype, device) = args.into_layer_arg();
172        let shape = S::build_args((vocab, embed));
173        let weight = Param::<S::WeightShape, B, K, Trainable>::new_init_raw(
174            crate::tensor::arg_into::TensorArgsData {
175                shape,
176                dtype,
177                device,
178                grad: (),
179            },
180            crate::nn::init::Init::Randn,
181        )?;
182        Ok(Self { weight })
183    }
184}
185
186impl<
187    S: EmbeddingShape,
188    InS: Shape + DynShape + AppendDim<S::Embed>,
189    B,
190    InK: DType,
191    K: DType,
192    Train: TrainState,
193> Module<Tensor<InS, B, InK>> for Embedding<S, B, K, Train>
194where
195    B: crate::tensor::backend::VariableBackend
196        + crate::exec::Capabilities
197        + Execute<op::EmbeddingExact>,
198    <B as Execute<op::EmbeddingExact>>::Output: Into<B::Storage<K>>,
199{
200    /// The output tensor type produced by this module's forward pass.
201    type Output = Tensor<<InS as AppendDim<S::Embed>>::Output, B, K>;
202    /// The error type returned if the forward pass fails.
203    type Error = Error;
204
205    #[inline]
206    /// Runs the forward pass of this module on the given input.
207    fn forward(&self, x: Tensor<InS, B, InK>) -> core::result::Result<Self::Output, Error> {
208        let weight = self.weight.as_tensor()?;
209        let x_handle = TensorHandle::from_storage::<B, InK, Local>(x.inner());
210        let weight_handle = TensorHandle::from_storage::<B, K, Local>(weight.inner());
211        let context = ExecutionContext::from_scope(B::default());
212        let mut dims = x.shape_buf().as_ref().to_vec();
213        // Read the validated weight extent so named static axes and runtime
214        // axes share the same execution path.
215        dims.push(weight.shape_buf()[1]);
216        let shape = shape_buf_from_dims::<<InS as AppendDim<S::Embed>>::Output>(
217            OperationKind::Embedding,
218            &dims,
219        )?;
220        let output_shape =
221            crate::shapes::ShapeValue::<<InS as AppendDim<S::Embed>>::Output>::try_new(shape)
222                .map_err(crate::err::Error::Shape)?;
223        let out = dispatch::execute_shaped::<
224            op::EmbeddingExact,
225            B,
226            <InS as AppendDim<S::Embed>>::Output,
227        >(
228            &context,
229            NoAttributes,
230            &[x_handle, weight_handle],
231            &output_shape,
232        )
233        .map_err(crate::err::Error::from)?;
234
235        Tensor::from_shape_value(
236            out.into(),
237            output_shape,
238            weight._dtype.clone(),
239            x._device.clone(),
240            core::marker::PhantomData,
241        )
242    }
243}