use burn::{
Tensor,
config::Config,
module::{
Module,
Param,
},
nn::Initializer,
prelude::Int,
tensor::Device,
};
use crate::{
blocks::norm::{
BunsenNormalization,
BunsenNormalizationConfig,
},
burner::{
module::ModuleInit,
tensor::round_scalar_to_dtype,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::norm::RmsNormOptions,
};
pub trait TokenEmbeddingMeta {
fn n_vocab(&self) -> usize;
fn d_model(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct TokenEmbeddingConfig {
pub n_vocab: usize,
pub d_model: usize,
#[config(default = "None")]
pub norm_eps: Option<f64>,
#[config(default = "None")]
pub norm: Option<BunsenNormalizationConfig>,
#[config(default = "None")]
pub scale: Option<f64>,
#[config(default = "Initializer::Normal { mean: 0.0, std: 1.0 }")]
pub initializer: Initializer,
}
impl TokenEmbeddingMeta for TokenEmbeddingConfig {
fn n_vocab(&self) -> usize {
self.n_vocab
}
fn d_model(&self) -> usize {
self.d_model
}
}
impl ModuleInit<TokenEmbedding> for TokenEmbeddingConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<TokenEmbedding> {
if let (Some(eps), Some(norm)) = (self.norm_eps, &self.norm) {
return Err(ConstraintError::new(
"TokenEmbeddingConfig",
"norm",
Rule::Custom(format!(
"at most one of norm_eps ({eps}) and norm ({norm:?})"
)),
)
.into());
}
Ok(TokenEmbedding {
weight: self.initializer.init([self.n_vocab, self.d_model], device),
norm_eps: self.norm_eps,
norm: self
.norm
.clone()
.map(|norm| norm.with_num_features(self.d_model).init(device)),
scale: self.scale,
})
}
}
#[derive(Module, Debug)]
pub struct TokenEmbedding {
pub weight: Param<Tensor<2>>,
#[module(skip)]
pub norm_eps: Option<f64>,
pub norm: Option<BunsenNormalization>,
#[module(skip)]
pub scale: Option<f64>,
}
impl TokenEmbeddingMeta for TokenEmbedding {
fn n_vocab(&self) -> usize {
self.weight.dims()[0]
}
fn d_model(&self) -> usize {
self.weight.dims()[1]
}
}
impl TokenEmbedding {
pub fn applied_scale(&self) -> Option<f64> {
let dtype = crate::burner::tensor::param_float_dtype(&self.weight);
self.scale.map(|scale| round_scalar_to_dtype(scale, dtype))
}
pub fn forward(
&self,
ids: Tensor<2, Int>,
) -> Tensor<3> {
let x = burn::tensor::module::embedding(self.weight.val(), ids);
let x = match self.norm_eps {
Some(eps) => RmsNormOptions::default().with_eps(eps as f32).norm(x),
None => x,
};
let x = match &self.norm {
Some(norm) => norm.forward(x),
None => x,
};
match self.applied_scale() {
Some(scale) => x.mul_scalar(scale),
None => x,
}
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
TensorData,
Tolerance,
};
use super::*;
use crate::support::testing::cpu_device;
fn table(device: &Device) -> TokenEmbedding {
let mut embedding: TokenEmbedding = TokenEmbeddingConfig::new(3, 2).init(device);
embedding.weight = Param::from_tensor(Tensor::from_floats(
[[1.0, 2.0], [3.0, -4.0], [0.5, 0.5]],
device,
));
embedding
}
#[test]
fn test_lookup() {
let device = cpu_device();
let embedding = table(&device);
assert_eq!((embedding.n_vocab(), embedding.d_model()), (3, 2));
let ids = Tensor::<2, Int>::from_ints([[1, 0, 1]], &device);
embedding.forward(ids).into_data().assert_eq(
&TensorData::from([[[3.0f32, -4.0], [1.0, 2.0], [3.0, -4.0]]]),
false,
);
}
#[test]
fn test_norm_then_scale() {
let device = cpu_device();
let mut embedding = table(&device);
embedding.norm_eps = Some(0.0);
embedding.scale = Some(2.0);
let ids = Tensor::<2, Int>::from_ints([[1, 2]], &device);
let r = (12.5f32).sqrt();
embedding.forward(ids).into_data().assert_approx_eq::<f32>(
&TensorData::from([[[6.0 / r, -8.0 / r], [2.0, 2.0]]]),
Tolerance::default(),
);
}
#[test]
fn test_scale_rounds_to_the_table_dtype() {
let s = 1152f64.sqrt();
let device = cpu_device();
let mut embedding = table(&device);
embedding.scale = Some(s);
assert_eq!(embedding.applied_scale(), Some(s as f32 as f64));
}
#[test]
fn test_norm_with_parameters() {
use burn::nn::LayerNormConfig;
let device = cpu_device();
let config = TokenEmbeddingConfig::new(3, 2).with_norm(Some(
BunsenNormalizationConfig::Layer(LayerNormConfig::new(0).with_bias(false)),
));
let mut embedding: TokenEmbedding = config.init(&device);
embedding.weight = table(&device).weight;
let Some(BunsenNormalization::Layer(norm)) = &mut embedding.norm else {
panic!("a layer norm");
};
assert!(norm.beta.is_none(), "no bias");
norm.gamma = Param::from_tensor(Tensor::from_floats([2.0, 0.5], &device));
#[cfg(feature = "store")]
{
use burn::store::ModuleSnapshot;
let names: Vec<String> = embedding
.collect(None, None, false)
.into_iter()
.map(|t| t.name)
.collect();
assert_eq!(names, ["weight", "norm.Layer.gamma"]);
}
let ids = Tensor::<2, Int>::from_ints([[1]], &device);
embedding.forward(ids).into_data().assert_approx_eq::<f32>(
&TensorData::from([[[2.0f32, -0.5]]]),
Tolerance::rel_abs(1e-4, 1e-4),
);
let both = TokenEmbeddingConfig::new(3, 2)
.with_norm_eps(Some(1e-6))
.with_norm(Some(BunsenNormalizationConfig::Identity));
assert!(
ModuleInit::<TokenEmbedding>::try_init(&both, &device)
.unwrap_err()
.to_string()
.contains("at most one of norm_eps")
);
}
}