use burn::{
Tensor,
nn::Embedding,
};
pub trait EmbeddingArg {
fn weight(&self) -> Tensor<2>;
}
impl EmbeddingArg for &Embedding {
fn weight(&self) -> Tensor<2> {
self.weight.val()
}
}
impl EmbeddingArg for Tensor<2> {
fn weight(&self) -> Tensor<2> {
self.clone()
}
}
pub fn unembed<A: EmbeddingArg>(
emb: A,
x: Tensor<3>,
) -> Tensor<3> {
x.matmul(emb.weight().transpose().unsqueeze::<3>())
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Int,
},
};
use serial_test::serial;
use super::*;
use crate::support::testing::{
DeviceMemoryGuard,
performance_device,
};
#[test]
#[serial]
fn test_embedding_inverse_to_logits() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let n_embedding = 10;
let embedding = Embedding {
weight: Param::from_tensor(Tensor::<2>::eye(n_embedding, &device)),
};
let batch = 2;
let seq_len = 20;
let x: Tensor<2, Int> = Tensor::random(
[batch, seq_len],
Distribution::Uniform(0.0, n_embedding as f64),
&device,
);
let e = embedding.forward(x.clone());
let logits = unembed(&embedding, e.clone());
let logits2 = unembed(embedding.weight.val(), e.clone());
logits
.clone()
.into_data()
.assert_eq(&logits2.into_data(), true);
let y: Tensor<2, Int> = logits.argmax(2).squeeze_dim(2);
y.into_data().assert_eq(&x.into_data(), true);
}
}