use burn::{
Tensor,
config::Config,
module::Module,
tensor::Device,
};
use crate::{
blocks::transformers::encoders::{
conformer::{
ConformerTransformer,
ConformerTransformerConfig,
ConformerTransformerMeta,
},
residual::{
ResidualEncoderTransformer,
ResidualEncoderTransformerConfig,
ResidualEncoderTransformerMeta,
},
rope::{
RopeTransformerEncoder,
RopeTransformerEncoderConfig,
RopeTransformerEncoderMeta,
},
},
burner::module::ModuleInit,
errors::BunsenResult,
};
pub trait EncoderTransformerMeta {
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
}
#[derive(Config, Debug)]
pub enum EncoderTransformerConfig {
Residual(ResidualEncoderTransformerConfig),
Rope(RopeTransformerEncoderConfig),
Conformer(ConformerTransformerConfig),
}
impl From<ResidualEncoderTransformerConfig> for EncoderTransformerConfig {
fn from(config: ResidualEncoderTransformerConfig) -> Self {
Self::Residual(config)
}
}
impl From<RopeTransformerEncoderConfig> for EncoderTransformerConfig {
fn from(config: RopeTransformerEncoderConfig) -> Self {
Self::Rope(config)
}
}
impl From<ConformerTransformerConfig> for EncoderTransformerConfig {
fn from(config: ConformerTransformerConfig) -> Self {
Self::Conformer(config)
}
}
impl EncoderTransformerMeta for EncoderTransformerConfig {
fn d_model(&self) -> usize {
match self {
Self::Residual(c) => c.d_model(),
Self::Rope(c) => c.d_model(),
Self::Conformer(c) => c.d_model(),
}
}
fn n_layers(&self) -> usize {
match self {
Self::Residual(c) => c.n_layers(),
Self::Rope(c) => c.n_layers(),
Self::Conformer(c) => c.n_layers(),
}
}
}
impl ModuleInit<EncoderTransformer> for EncoderTransformerConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<EncoderTransformer> {
Ok(match self {
Self::Residual(c) => EncoderTransformer::Residual(c.try_init(device)?),
Self::Rope(c) => EncoderTransformer::Rope(c.try_init(device)?),
Self::Conformer(c) => EncoderTransformer::Conformer(c.try_init(device)?),
})
}
}
#[derive(Module, Debug)]
#[allow(clippy::large_enum_variant)]
pub enum EncoderTransformer {
Residual(ResidualEncoderTransformer),
Rope(RopeTransformerEncoder),
Conformer(ConformerTransformer),
}
impl From<ResidualEncoderTransformer> for EncoderTransformer {
fn from(module: ResidualEncoderTransformer) -> Self {
Self::Residual(module)
}
}
impl From<RopeTransformerEncoder> for EncoderTransformer {
fn from(module: RopeTransformerEncoder) -> Self {
Self::Rope(module)
}
}
impl From<ConformerTransformer> for EncoderTransformer {
fn from(module: ConformerTransformer) -> Self {
Self::Conformer(module)
}
}
impl EncoderTransformerMeta for EncoderTransformer {
fn d_model(&self) -> usize {
match self {
Self::Residual(m) => m.d_model(),
Self::Rope(m) => m.d_model(),
Self::Conformer(m) => m.d_model(),
}
}
fn n_layers(&self) -> usize {
match self {
Self::Residual(m) => m.n_layers(),
Self::Rope(m) => m.n_layers(),
Self::Conformer(m) => m.n_layers(),
}
}
}
impl EncoderTransformer {
pub fn forward(
&self,
x: Tensor<3>,
lengths: Option<&[usize]>,
) -> Tensor<3> {
match self {
Self::Residual(m) => m.forward(x, lengths),
Self::Rope(m) => m.forward(x, lengths),
Self::Conformer(m) => m.forward(x, lengths),
}
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use super::*;
use crate::{
blocks::transformers::encoders::{
conformer::ConformerBlockConfig,
residual::ResidualEncoderAttentionBlockConfig,
rope::RopeEncoderBlockConfig,
},
support::testing::cpu_device,
};
fn configs() -> Vec<EncoderTransformerConfig> {
vec![
ResidualEncoderTransformerConfig::new(
ResidualEncoderAttentionBlockConfig::new(8).with_d_head(4),
2,
)
.into(),
RopeTransformerEncoderConfig::new(RopeEncoderBlockConfig::new(8, 2), 2).into(),
ConformerTransformerConfig::new(ConformerBlockConfig::new(8, 2, 16, 3), 2).into(),
]
}
#[test]
#[serial_test::serial]
fn test_each_variant_is_its_stack() {
let device = cpu_device();
for config in configs() {
device.seed(3);
let encoder: EncoderTransformer = config.init(&device);
assert_eq!(encoder.d_model(), config.d_model());
assert_eq!(encoder.n_layers(), 2);
let x = Tensor::<3>::random([2, 6, 8], Distribution::Default, &device);
let direct = match &encoder {
EncoderTransformer::Residual(m) => m.forward(x.clone(), Some(&[6, 4])),
EncoderTransformer::Rope(m) => m.forward(x.clone(), Some(&[6, 4])),
EncoderTransformer::Conformer(m) => m.forward(x.clone(), Some(&[6, 4])),
};
encoder
.forward(x, Some(&[6, 4]))
.to_data()
.assert_approx_eq::<f32>(&direct.to_data(), Tolerance::absolute(1e-6));
}
}
#[test]
#[serial_test::serial]
#[cfg(feature = "store")]
fn test_the_variant_is_not_in_the_loaded_paths() {
use burn::store::ModuleSnapshot;
let device = cpu_device();
for config in configs() {
let encoder: EncoderTransformer = config.init(&device);
let inner: Vec<String> = match &encoder {
EncoderTransformer::Residual(m) => m.collect(None, None, true),
EncoderTransformer::Rope(m) => m.collect(None, None, true),
EncoderTransformer::Conformer(m) => m.collect(None, None, true),
}
.into_iter()
.map(|t| t.name)
.collect();
let skipped: Vec<String> = encoder
.collect(None, None, true)
.into_iter()
.map(|t| t.name)
.collect();
assert_eq!(skipped, inner);
assert!(!inner.is_empty());
let native: Vec<String> = encoder
.collect(None, None, false)
.into_iter()
.map(|t| t.name)
.collect();
assert_ne!(native, inner, "a native record names the variant");
}
}
}