use alloc::string::{
String,
ToString,
};
use burn::{
Tensor,
config::Config,
module::Module,
nn::norm::RmsNormConfig,
prelude::Bool,
tensor::Device,
};
use crate::{
blocks::{
norm::BunsenNormalizationConfig,
transformers::{
attention::csa::AttentionSides,
conditioning::{
AdaptiveScale,
AdaptiveScaleConfig,
AdaptiveScaleMeta,
},
embedding::RotaryEmbedding,
mixers::{
TokenMixer,
TokenMixerConfig,
},
mlp::{
TransformerFeedForward,
TransformerFeedForwardConfig,
TransformerFeedForwardMeta,
},
sublayer::{
ResidualSublayer,
ResidualSublayerConfig,
},
},
},
burner::module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
errors::{
BunsenResult,
ConstraintError,
Rule,
WithOkOrPanic,
},
ops::transformers::{
attention::KVCache,
linear_attention::DeltaNetCache,
},
};
#[derive(Debug, Default)]
pub struct DecoderSideInputs {
pub cond: Option<Tensor<3>>,
pub value_embed: Option<Tensor<3>>,
pub key_valid: Option<Tensor<2, Bool>>,
}
pub trait CausalDecoderBlockMeta {
fn n_embed(&self) -> usize;
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DecoderLayerKeys {
pub attn_norm: String,
pub mlp_norm: String,
pub post_norms: Option<[String; 2]>,
pub attn: String,
pub attn_projs: [String; 4],
}
impl DecoderLayerKeys {
pub fn llama(
attn_norm: impl Into<String>,
mlp_norm: impl Into<String>,
) -> Self {
Self {
attn_norm: attn_norm.into(),
mlp_norm: mlp_norm.into(),
post_norms: None,
attn: "self_attn".to_string(),
attn_projs: ["q_proj", "k_proj", "v_proj", "o_proj"].map(String::from),
}
}
pub fn with_post_norms(
self,
attn_post_norm: impl Into<String>,
mlp_post_norm: impl Into<String>,
) -> Self {
Self {
post_norms: Some([attn_post_norm.into(), mlp_post_norm.into()]),
..self
}
}
}
#[derive(Config, Debug)]
pub struct CausalDecoderBlockConfig {
pub attn: TokenMixerConfig,
pub mlp: TransformerFeedForwardConfig,
#[config(default = "BunsenNormalizationConfig::Rms(RmsNormConfig::new(0))")]
pub norm: BunsenNormalizationConfig,
#[config(default = true)]
pub attn_norm: bool,
#[config(default = false)]
pub post_norms: bool,
#[config(default = "None")]
pub mlp_scale: Option<AdaptiveScaleConfig>,
}
impl CausalDecoderBlockMeta for CausalDecoderBlockConfig {
fn n_embed(&self) -> usize {
self.attn.n_embed()
}
}
impl CausalDecoderBlockConfig {
pub fn init(
&self,
layer_index: usize,
device: &Device,
) -> CausalDecoderBlock {
self.try_init(layer_index, device).ok_or_panic()
}
pub fn try_init(
&self,
layer_index: usize,
device: &Device,
) -> BunsenResult<CausalDecoderBlock> {
if self.attn.n_embed() != self.mlp.n_embed() {
return Err(ConstraintError::new(
"CausalDecoderBlockConfig",
"",
Rule::Relation {
lhs: ("attn.n_embed".into(), self.attn.n_embed().to_string()),
op: "==",
rhs: ("mlp.n_embed".into(), self.mlp.n_embed().to_string()),
},
)
.into());
}
let n_embed = self.n_embed();
if let Some(scale) = &self.mlp_scale
&& scale.d_model() != n_embed
{
return Err(ConstraintError::new(
"CausalDecoderBlockConfig",
"",
Rule::Relation {
lhs: ("mlp_scale.d_model".into(), scale.d_model().to_string()),
op: "==",
rhs: ("n_embed".into(), n_embed.to_string()),
},
)
.into());
}
Ok(CausalDecoderBlock {
attn: ResidualSublayerConfig::pre_norm(self.attn_norm.then(|| self.norm.clone()))
.with_post(self.post_norms.then(|| self.norm.clone()))
.init(self.attn.try_init(layer_index, device)?, n_embed, device),
mlp: ResidualSublayerConfig::pre_norm(Some(self.norm.clone()))
.with_post(self.post_norms.then(|| self.norm.clone()))
.init(self.mlp.try_init(device)?, n_embed, device),
mlp_scale: self
.mlp_scale
.as_ref()
.map(|c| c.try_init(device))
.transpose()?,
})
}
}
#[derive(Module, Debug)]
pub struct CausalDecoderBlock {
pub attn: ResidualSublayer<TokenMixer>,
pub mlp: ResidualSublayer<TransformerFeedForward>,
pub mlp_scale: Option<AdaptiveScale>,
}
impl CausalDecoderBlockMeta for CausalDecoderBlock {
fn n_embed(&self) -> usize {
self.attn.branch.n_embed()
}
}
impl crate::burner::module::InputDTypes for CausalDecoderBlock {
fn preferred_dtype(&self) -> burn::tensor::FloatDType {
crate::burner::module::InputDTypes::preferred_dtype(&self.attn.branch)
}
}
impl crate::burner::module::ResultDType for CausalDecoderBlock {
fn result_dtype(
&self,
_input: burn::tensor::FloatDType,
) -> burn::tensor::FloatDType {
crate::burner::module::InputDTypes::preferred_dtype(self)
}
}
impl CausalDecoderBlock {
pub fn layer_rekey(keys: &DecoderLayerKeys) -> Rekey {
let attn = &keys.attn;
let [q, k, v, o] = &keys.attn_projs;
let mut rekey = Rekey::keep_rest()
.rename(format!("{}.", keys.attn_norm), "attn_norm.")
.rename(format!("{}.", keys.mlp_norm), "mlp_norm.");
if let Some([attn_post, mlp_post]) = &keys.post_norms {
rekey = rekey
.rename(format!("{attn_post}."), "attn_post_norm.")
.rename(format!("{mlp_post}."), "mlp_post_norm.");
}
rekey
.rename(format!("{attn}.{q}."), "attn.c_q.")
.rename(format!("{attn}.{k}."), "attn.c_k.")
.rename(format!("{attn}.{v}."), "attn.c_v.")
.rename(format!("{attn}.{o}."), "attn.c_proj.")
.rename(format!("{attn}."), "attn.")
}
pub fn attn_input(
&self,
x: Tensor<3>,
) -> Tensor<3> {
self.attn.branch_input(x)
}
pub fn forward(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
self.forward_conditioned(input, r_emb, kv_cache, None)
}
pub fn forward_conditioned(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
cond: Option<Tensor<3>>,
) -> Tensor<3> {
self.forward_with(
input,
r_emb,
kv_cache,
DecoderSideInputs {
cond,
..Default::default()
},
)
}
pub fn forward_with(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
sides: DecoderSideInputs,
) -> Tensor<3> {
self.forward_with_caches(input, r_emb, kv_cache, None, sides)
}
pub fn forward_with_caches(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
linear: Option<&mut DeltaNetCache>,
sides: DecoderSideInputs,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
crate::burner::module::assert_input_dtype(self, input.dtype(), "causal decoder block");
let DecoderSideInputs {
cond,
value_embed,
key_valid,
} = sides;
let x = self
.attn
.residual_update_with(input, |mixer, z| match mixer {
TokenMixer::Attention(attn) => attn.forward_with(
z,
r_emb,
kv_cache,
AttentionSides {
value_embed,
key_valid,
},
),
TokenMixer::GatedDeltaNet(net) => {
assert!(
value_embed.is_none() && key_valid.is_none(),
"a linear-attention block takes no value embedding or key padding"
);
assert!(
kv_cache.is_none() || linear.is_some(),
"a linear-attention block decodes through a DeltaNetCache"
);
net.forward(z, linear.map(|cache| cache.layer_mut(net.layer_index)))
}
});
self.mlp.residual_update_with(x, |mlp, h| {
let h = match (&self.mlp_scale, cond) {
(None, None) => h,
(Some(scale), Some(cond)) => scale.forward(h, cond),
(None, Some(_)) => panic!("a condition for a block without an mlp_scale"),
(Some(_), None) => panic!("a block with an mlp_scale needs its condition"),
};
mlp.forward(h)
})
}
}
impl CanonicalKeys for CausalDecoderBlock {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest()
.rename("attn_norm.", "attn.pre.")
.rename("attn_post_norm.", "attn.post.")
.rename("attn.", "attn.branch.")
.rename("mlp_norm.", "mlp.pre.")
.rename("mlp_post_norm.", "mlp.post.")
.rename("mlp.", "mlp.branch.")
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Tolerance,
},
};
use serial_test::serial;
use super::*;
use crate::{
blocks::{
norm::BunsenNormalization,
transformers::{
attention::csa::{
CausalSelfAttentionConfig,
CausalSelfAttentionMeta,
},
embedding::RotaryEmbeddingConfig,
mlp::TransformerMlpConfig,
},
},
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
assert_tensors_close,
performance_device,
seeded_tensor,
},
};
#[test]
#[serial]
fn test_gpt_block_config() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let n_embed = 1024;
let n_head = 128;
let n_kv_head = 64;
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed).into(),
TransformerMlpConfig::new(n_embed).into(),
);
assert_eq!(config.n_embed(), n_embed);
assert_eq!(config.attn.n_embed(), n_embed);
assert_eq!(config.attn.attention().n_head(), n_head);
assert_eq!(config.attn.attention().n_kv_head(), n_kv_head);
assert_eq!(TransformerFeedForwardMeta::n_embed(&config.mlp), n_embed);
let layer_index = 12;
let block: CausalDecoderBlock = config.init(layer_index, &device);
assert_eq!(block.n_embed(), n_embed);
}
#[test]
#[serial]
fn test_gpt_block_forward() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let batch = 2;
let seq_len = 10;
let n_embed = 1024;
let n_head = 128;
let n_kv_head = 64;
let layer_index = 12;
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed).into(),
TransformerMlpConfig::new(n_embed).into(),
);
let block: CausalDecoderBlock = config.init(layer_index, &device);
let input = Tensor::random([batch, seq_len, n_embed], Distribution::Default, &device);
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.branch.attention().head_dim())
.init(&device);
let mut kv_cache: Option<&mut KVCache> = None;
let output = block.forward(input.clone(), &r_emb, &mut kv_cache);
assert_shape_contract!(
["B", "T", "D"],
&output.dims(),
&[("B", batch), ("T", seq_len), ("D", n_embed)]
);
}
#[test]
fn test_block_without_an_attn_norm() {
let device = crate::support::testing::cpu_device();
let [batch, seq_len, n_embed] = [2, 5, 16];
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(4, 2, n_embed).into(),
TransformerMlpConfig::new(n_embed).into(),
);
assert!(config.init(0, &device).attn.pre.is_some());
let block: CausalDecoderBlock = config.with_attn_norm(false).init(0, &device);
assert!(block.attn.pre.is_none());
#[cfg(feature = "store")]
{
use burn::store::ModuleSnapshot;
let paths: Vec<String> = block
.collect(None, None, false)
.into_iter()
.map(|t| t.name)
.collect();
assert!(
paths.iter().all(|p| !p.starts_with("attn_norm")),
"{paths:?}"
);
assert!(paths.iter().any(|p| p.starts_with("mlp.pre")), "{paths:?}");
}
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.branch.attention().head_dim())
.init(&device);
let input =
seeded_tensor::<3>(2, [batch, seq_len, n_embed], Distribution::Default, &device);
let x = input.clone()
+ block
.attn
.branch
.attention()
.forward(input.clone(), &r_emb, &mut None);
let expected = x.clone()
+ block
.mlp
.branch
.forward(block.mlp.pre.as_ref().unwrap().forward(x));
let output = block.forward(input, &r_emb, &mut None);
assert_tensors_close(&output, &expected, Tolerance::default());
}
#[test]
#[serial]
fn test_gpt_block_adds_sublayers_to_residual() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let [batch, seq_len, n_embed] = [2, 5, 32];
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(4, 2, n_embed).into(),
TransformerMlpConfig::new(n_embed).into(),
);
let mut block: CausalDecoderBlock = config.init(0, &device);
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.branch.attention().head_dim())
.init(&device);
let input =
seeded_tensor::<3>(1, [batch, seq_len, n_embed], Distribution::Default, &device);
let TransformerFeedForward::Mlp(mlp) = &mut block.mlp.branch else {
panic!("an Mlp feed-forward");
};
mlp.linear2.weight = Param::from_tensor(mlp.linear2.weight.val().zeros_like());
let attn = block.attn.branch.attention().forward(
block.attn_input(input.clone()),
&r_emb,
&mut None,
);
let output = block.forward(input.clone(), &r_emb, &mut None);
assert_tensors_close(&output, &(input.clone() + attn), Tolerance::default());
block.attn.branch.attention_mut().c_proj.weight = Param::from_tensor(
block
.attn
.branch
.attention()
.c_proj
.weight
.val()
.zeros_like(),
);
let output = block.forward(input.clone(), &r_emb, &mut None);
assert_tensors_close(&output, &input, Tolerance::default());
}
#[test]
fn test_post_norms_sandwich_each_sublayer() {
let device = crate::support::testing::cpu_device();
let [batch, seq_len, n_embed] = [2, 5, 16];
let config = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(4, 2, n_embed).into(),
crate::blocks::transformers::mlp::TransformerGatedMlpConfig::new(n_embed, 24).into(),
);
assert!(config.init(0, &device).attn.post.is_none());
assert!(config.init(0, &device).mlp.post.is_none());
let mut block: CausalDecoderBlock = config.with_post_norms(true).init(0, &device);
for (seed, norm) in [
(3, block.attn.pre.as_mut().expect("an attention norm")),
(4, block.attn.post.as_mut().expect("a post-norm")),
(5, block.mlp.pre.as_mut().expect("a pre-norm")),
(6, block.mlp.post.as_mut().expect("a post-norm")),
] {
let BunsenNormalization::Rms(rms) = norm else {
panic!("an RMS norm");
};
rms.gamma = Param::from_tensor(seeded_tensor::<1>(
seed,
[n_embed],
Distribution::Uniform(0.5, 1.5),
&device,
));
}
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.branch.attention().head_dim())
.init(&device);
let input =
seeded_tensor::<3>(1, [batch, seq_len, n_embed], Distribution::Default, &device);
let post =
|norm: &Option<BunsenNormalization>, x| norm.as_ref().expect("a post-norm").forward(x);
let x = input.clone()
+ post(
&block.attn.post,
block.attn.branch.attention().forward(
block.attn_input(input.clone()),
&r_emb,
&mut None,
),
);
let expected = x.clone()
+ post(
&block.mlp.post,
block
.mlp
.branch
.forward(block.mlp.pre.as_ref().unwrap().forward(x)),
);
let output = block.forward(input, &r_emb, &mut None);
assert_tensors_close(&output, &expected, Tolerance::default());
}
#[test]
fn test_layer_rekey_with_post_norms() {
let rekey = CausalDecoderBlock::layer_rekey(
&DecoderLayerKeys::llama("input_layernorm", "pre_feedforward_layernorm")
.with_post_norms("post_attention_layernorm", "post_feedforward_layernorm"),
);
for (stored, bunsen) in [
("input_layernorm.weight", "attn_norm.weight"),
("post_attention_layernorm.weight", "attn_post_norm.weight"),
("pre_feedforward_layernorm.weight", "mlp_norm.weight"),
("post_feedforward_layernorm.weight", "mlp_post_norm.weight"),
("self_attn.q_proj.weight", "attn.c_q.weight"),
("self_attn.q_norm.weight", "attn.q_norm.weight"),
("mlp.gate_proj.weight", "mlp.gate_proj.weight"),
] {
assert_eq!(rekey.apply(stored).as_deref(), Some(bunsen), "{stored}");
}
}
fn conditioned(n_embed: usize) -> CausalDecoderBlockConfig {
CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(4, 2, n_embed)
.with_norm(None)
.into(),
crate::blocks::transformers::mlp::TransformerGatedMlpConfig::new(n_embed, 24).into(),
)
.with_mlp_scale(Some(AdaptiveScaleConfig::new(n_embed, 4)))
}
#[test]
fn test_conditioned_block_scales_the_mlp_input() {
let device = crate::support::testing::cpu_device();
let [batch, seq_len, n_embed] = [2, 5, 16];
let block: CausalDecoderBlock = conditioned(n_embed).init(0, &device);
let scale = block.mlp_scale.as_ref().expect("a conditioned block");
let r_emb = RotaryEmbeddingConfig::new(seq_len, block.attn.branch.attention().head_dim())
.init(&device);
let input =
seeded_tensor::<3>(1, [batch, seq_len, n_embed], Distribution::Default, &device);
let cond = seeded_tensor::<3>(2, [1, 1, n_embed], Distribution::Default, &device);
let x = input.clone()
+ block.attn.branch.attention().forward(
block.attn_input(input.clone()),
&r_emb,
&mut None,
);
let expected = x.clone()
+ block
.mlp
.branch
.forward(scale.forward(block.mlp.pre.as_ref().unwrap().forward(x), cond.clone()));
let output = block.forward_conditioned(input, &r_emb, &mut None, Some(cond));
assert_tensors_close(&output, &expected, Tolerance::default());
}
#[test]
#[should_panic(expected = "needs its condition")]
fn test_conditioned_block_needs_its_condition() {
let device = crate::support::testing::cpu_device();
let block: CausalDecoderBlock = conditioned(16).init(0, &device);
let r_emb =
RotaryEmbeddingConfig::new(3, block.attn.branch.attention().head_dim()).init(&device);
let _ = block.forward(Tensor::zeros([1, 3, 16], &device), &r_emb, &mut None);
}
#[test]
fn test_conditioned_block_width() {
let device = crate::support::testing::cpu_device();
crate::errors::testing::ErrorMatcher::kind(crate::errors::BunsenErrorKind::Illegal)
.message_contains("mlp_scale.d_model")
.assert_err(
&conditioned(16)
.with_mlp_scale(Some(AdaptiveScaleConfig::new(8, 4)))
.try_init(0, &device),
);
}
}