use burn::{
Tensor,
config::Config,
module::Module,
nn::norm::{
Normalization,
NormalizationConfig,
RmsNormConfig,
},
tensor::Device,
};
use crate::{
blocks::transformers::{
attention::csa::{
CausalSelfAttention,
CausalSelfAttentionConfig,
CausalSelfAttentionMeta,
},
embedding::RotaryEmbedding,
mlp::{
FeedForward,
FeedForwardConfig,
FeedForwardMeta,
},
},
burner::module::ModuleInit,
errors::{
BunsenResult,
ConstraintError,
Rule,
WithOkOrPanic,
},
ops::transformers::attention::KVCache,
};
pub trait CausalDecoderBlockMeta {
fn n_embed(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct CausalDecoderBlockConfig {
pub attn: CausalSelfAttentionConfig,
pub mlp: FeedForwardConfig,
#[config(default = "NormalizationConfig::Rms(RmsNormConfig::new(0))")]
pub norm: NormalizationConfig,
}
impl CausalDecoderBlockMeta for CausalDecoderBlockConfig {
fn n_embed(&self) -> usize {
self.attn.n_embed()
}
}
impl CausalDecoderBlockConfig {
pub fn init(
&self,
layer_index: usize,
device: &Device,
) -> CausalDecoderBlock {
self.try_init(layer_index, device).ok_or_panic()
}
pub fn try_init(
&self,
layer_index: usize,
device: &Device,
) -> BunsenResult<CausalDecoderBlock> {
if self.attn.n_embed() != self.mlp.n_embed() {
return Err(ConstraintError::new(
"CausalDecoderBlockConfig",
"",
Rule::Relation {
lhs: ("attn.n_embed".into(), self.attn.n_embed().to_string()),
op: "==",
rhs: ("mlp.n_embed".into(), self.mlp.n_embed().to_string()),
},
)
.into());
}
let n_embed = self.n_embed();
Ok(CausalDecoderBlock {
attn_norm: self.norm.clone().with_num_features(n_embed).init(device),
attn: self.attn.try_init(layer_index, device)?,
mlp_norm: self.norm.clone().with_num_features(n_embed).init(device),
mlp: self.mlp.try_init(device)?,
})
}
}
#[derive(Module, Debug)]
pub struct CausalDecoderBlock {
pub attn_norm: Normalization,
pub attn: CausalSelfAttention,
pub mlp_norm: Normalization,
pub mlp: FeedForward,
}
impl CausalDecoderBlockMeta for CausalDecoderBlock {
fn n_embed(&self) -> usize {
self.attn.n_embed()
}
}
impl CausalDecoderBlock {
pub fn forward(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
let x = input.clone()
+ self
.attn
.forward(self.attn_norm.forward(input), r_emb, kv_cache);
x.clone() + self.mlp.forward(self.mlp_norm.forward(x))
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Tolerance,
},
};
use serial_test::serial;
use super::*;
use crate::{
blocks::transformers::{
embedding::RotaryEmbeddingConfig,
mlp::MlpConfig,
},
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
assert_tensors_close,
performance_device,
seeded_tensor,
},
};
#[test]
#[serial]
fn test_gpt_block_config() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let n_embed = 1024;
let n_head = 128;
let n_kv_head = 64;
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed),
MlpConfig::new(n_embed).with_act_exponent(Some(2.0)).into(),
);
assert_eq!(config.n_embed(), n_embed);
assert_eq!(config.attn.n_embed(), n_embed);
assert_eq!(config.attn.n_head(), n_head);
assert_eq!(config.attn.n_kv_head(), n_kv_head);
assert_eq!(FeedForwardMeta::n_embed(&config.mlp), n_embed);
let layer_index = 12;
let block: CausalDecoderBlock = config.init(layer_index, &device);
assert_eq!(block.n_embed(), n_embed);
}
#[test]
#[serial]
fn test_gpt_block_forward() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let batch = 2;
let seq_len = 10;
let n_embed = 1024;
let n_head = 128;
let n_kv_head = 64;
let layer_index = 12;
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed),
MlpConfig::new(n_embed).with_act_exponent(Some(2.0)).into(),
);
let block: CausalDecoderBlock = config.init(layer_index, &device);
let input = Tensor::random([batch, seq_len, n_embed], Distribution::Default, &device);
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.head_dim()).init(&device);
let mut kv_cache: Option<&mut KVCache> = None;
let output = block.forward(input.clone(), &r_emb, &mut kv_cache);
assert_shape_contract!(
["B", "T", "D"],
&output.dims(),
&[("B", batch), ("T", seq_len), ("D", n_embed)]
);
}
#[test]
#[serial]
fn test_gpt_block_adds_sublayers_to_residual() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let [batch, seq_len, n_embed] = [2, 5, 32];
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(4, 2, n_embed),
MlpConfig::new(n_embed).with_act_exponent(Some(2.0)).into(),
);
let mut block: CausalDecoderBlock = config.init(0, &device);
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.head_dim()).init(&device);
let input =
seeded_tensor::<3>(1, [batch, seq_len, n_embed], Distribution::Default, &device);
let FeedForward::Mlp(mlp) = &mut block.mlp else {
panic!("an Mlp feed-forward");
};
mlp.linear2.weight = Param::from_tensor(mlp.linear2.weight.val().zeros_like());
let attn = block
.attn
.forward(block.attn_norm.forward(input.clone()), &r_emb, &mut None);
let output = block.forward(input.clone(), &r_emb, &mut None);
assert_tensors_close(&output, &(input.clone() + attn), Tolerance::default());
block.attn.c_proj.weight = Param::from_tensor(block.attn.c_proj.weight.val().zeros_like());
let output = block.forward(input.clone(), &r_emb, &mut None);
assert_tensors_close(&output, &input, Tolerance::default());
}
}