use burn::{
Tensor,
config::Config,
module::Module,
tensor::Device,
};
use crate::{
blocks::transformers::{
decoders::stack::{
DecoderStack,
DecoderStackMeta,
DecoderStackStructureConfig,
},
encoders::{
conformer::{
ConformerTransformer,
ConformerTransformerConfig,
ConformerTransformerMeta,
},
residual::{
ResidualEncoderTransformer,
ResidualEncoderTransformerConfig,
ResidualEncoderTransformerMeta,
},
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::BunsenResult,
};
pub trait EncoderTransformerMeta {
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
}
#[derive(Config, Debug)]
pub enum EncoderTransformerConfig {
Residual(ResidualEncoderTransformerConfig),
Stack(DecoderStackStructureConfig),
Conformer(ConformerTransformerConfig),
}
impl From<ResidualEncoderTransformerConfig> for EncoderTransformerConfig {
fn from(config: ResidualEncoderTransformerConfig) -> Self {
Self::Residual(config)
}
}
impl From<DecoderStackStructureConfig> for EncoderTransformerConfig {
fn from(config: DecoderStackStructureConfig) -> Self {
Self::Stack(config)
}
}
impl From<ConformerTransformerConfig> for EncoderTransformerConfig {
fn from(config: ConformerTransformerConfig) -> Self {
Self::Conformer(config)
}
}
impl EncoderTransformerMeta for EncoderTransformerConfig {
fn d_model(&self) -> usize {
match self {
Self::Residual(c) => c.d_model(),
Self::Stack(c) => c.d_model(),
Self::Conformer(c) => c.d_model(),
}
}
fn n_layers(&self) -> usize {
match self {
Self::Residual(c) => c.n_layers(),
Self::Stack(c) => c.n_layers(),
Self::Conformer(c) => c.n_layers(),
}
}
}
impl ModuleInit<EncoderTransformer> for EncoderTransformerConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<EncoderTransformer> {
Ok(match self {
Self::Residual(c) => EncoderTransformer::Residual(c.try_init(device)?),
Self::Stack(c) => EncoderTransformer::Stack(c.try_init(device)?),
Self::Conformer(c) => EncoderTransformer::Conformer(c.try_init(device)?),
})
}
}
#[derive(Module, Debug)]
#[allow(clippy::large_enum_variant)]
pub enum EncoderTransformer {
Residual(ResidualEncoderTransformer),
Stack(DecoderStack),
Conformer(ConformerTransformer),
}
impl From<ResidualEncoderTransformer> for EncoderTransformer {
fn from(module: ResidualEncoderTransformer) -> Self {
Self::Residual(module)
}
}
impl From<DecoderStack> for EncoderTransformer {
fn from(module: DecoderStack) -> Self {
Self::Stack(module)
}
}
impl From<ConformerTransformer> for EncoderTransformer {
fn from(module: ConformerTransformer) -> Self {
Self::Conformer(module)
}
}
impl EncoderTransformerMeta for EncoderTransformer {
fn d_model(&self) -> usize {
match self {
Self::Residual(m) => m.d_model(),
Self::Stack(m) => m.d_model(),
Self::Conformer(m) => m.d_model(),
}
}
fn n_layers(&self) -> usize {
match self {
Self::Residual(m) => m.n_layers(),
Self::Stack(m) => m.n_layers(),
Self::Conformer(m) => m.n_layers(),
}
}
}
impl EncoderTransformer {
pub fn forward(
&self,
x: Tensor<3>,
lengths: Option<&[usize]>,
) -> Tensor<3> {
match self {
Self::Residual(m) => m.forward(x, lengths),
Self::Stack(m) => m.encoder_forward(x, lengths),
Self::Conformer(m) => m.forward(x, lengths),
}
}
}
impl CanonicalKeys for EncoderTransformer {
fn canonical_rekey(&self) -> Rekey {
match self {
Self::Residual(t) => t.canonical_rekey(),
Self::Stack(t) => t.canonical_rekey(),
Self::Conformer(t) => t.canonical_rekey(),
}
}
}
#[cfg(test)]
pub(crate) fn test_rope_stack(
d_model: usize,
heads: usize,
n_layers: usize,
) -> DecoderStackStructureConfig {
use burn::nn::LayerNormConfig;
use crate::blocks::{
activation::BunsenActivationConfig,
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::{
AttentionMask,
CausalSelfAttentionConfig,
},
decoders::{
causal::CausalDecoderBlockConfig,
stack::UniformStackConfig,
},
embedding::{
RotaryDirection,
RotaryEmbeddingConfig,
},
mlp::TransformerMlpConfig,
},
};
let norm = BunsenNormalizationConfig::Layer(LayerNormConfig::new(0));
crate::burner::module::ToStructureConfig::to_structure(
&UniformStackConfig::new(
n_layers,
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(heads, heads, d_model)
.with_norm(None)
.with_output_bias(true)
.with_mask(AttentionMask::Bidirectional)
.into(),
TransformerMlpConfig::new(d_model)
.with_activation(BunsenActivationConfig::Gelu)
.with_bias(true)
.into(),
)
.with_norm(norm.clone()),
norm.clone(),
RotaryEmbeddingConfig::new(0, d_model / heads).with_direction(RotaryDirection::Forward),
)
.with_input_norm(Some(norm)),
)
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use super::*;
use crate::{
blocks::transformers::encoders::{
conformer::ConformerBlockConfig,
residual::ResidualEncoderAttentionBlockConfig,
},
support::testing::cpu_device,
};
fn configs() -> Vec<EncoderTransformerConfig> {
vec![
ResidualEncoderTransformerConfig::new(
ResidualEncoderAttentionBlockConfig::new(8).with_d_head(4),
2,
)
.into(),
test_rope_stack(8, 2, 2).into(),
ConformerTransformerConfig::new(ConformerBlockConfig::new(8, 2, 16, 3), 2).into(),
]
}
#[test]
#[serial_test::serial]
fn test_each_variant_is_its_stack() {
let device = cpu_device();
for config in configs() {
device.seed(3);
let encoder: EncoderTransformer = config.init(&device);
assert_eq!(encoder.d_model(), config.d_model());
assert_eq!(encoder.n_layers(), 2);
let x = Tensor::<3>::random([2, 6, 8], Distribution::Default, &device);
let direct = match &encoder {
EncoderTransformer::Residual(m) => m.forward(x.clone(), Some(&[6, 4])),
EncoderTransformer::Stack(m) => m.encoder_forward(x.clone(), Some(&[6, 4])),
EncoderTransformer::Conformer(m) => m.forward(x.clone(), Some(&[6, 4])),
};
encoder
.forward(x, Some(&[6, 4]))
.to_data()
.assert_approx_eq::<f32>(&direct.to_data(), Tolerance::absolute(1e-6));
}
}
#[test]
#[serial_test::serial]
#[cfg(feature = "store")]
fn test_the_variant_is_not_in_the_loaded_paths() {
use burn::store::ModuleSnapshot;
let device = cpu_device();
for config in configs() {
let encoder: EncoderTransformer = config.init(&device);
let inner: Vec<String> = match &encoder {
EncoderTransformer::Residual(m) => m.collect(None, None, true),
EncoderTransformer::Stack(m) => m.collect(None, None, true),
EncoderTransformer::Conformer(m) => m.collect(None, None, true),
}
.into_iter()
.map(|t| t.name)
.collect();
let skipped: Vec<String> = encoder
.collect(None, None, true)
.into_iter()
.map(|t| t.name)
.collect();
assert_eq!(skipped, inner);
assert!(!inner.is_empty());
let native: Vec<String> = encoder
.collect(None, None, false)
.into_iter()
.map(|t| t.name)
.collect();
assert_ne!(native, inner, "a native record names the variant");
}
}
}