use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Initializer,
Linear,
LinearConfig,
norm::RmsNormConfig,
},
prelude::{
Bool,
Int,
s,
},
tensor::{
Device,
FloatDType,
},
};
use crate::{
blocks::{
norm::{
BunsenNormalization,
BunsenNormalizationConfig,
},
transformers::{
embedding::RotaryEmbedding,
sublayer::{
ResidualSublayer,
SublayerBranch,
},
},
},
contracts::{
assert_shape_contract_periodically,
unpack_shape_contract,
},
errors::{
BunsenResult,
WithOkOrPanic,
},
ops::transformers::attention::{
KVCache,
ScaledDotProductAttentionConfig,
causal_window_visible,
scaled_dot_product_attention,
sdpa_attn_weight,
two_sided_window_visible,
},
};
pub trait CausalSelfAttentionMeta {
fn n_embed(&self) -> usize;
fn n_head(&self) -> usize;
fn n_kv_head(&self) -> usize;
fn head_dim(&self) -> usize {
self.n_embed() / self.n_head()
}
fn gqa_enabled(&self) -> bool {
self.n_head() != self.n_kv_head()
}
}
#[derive(Config, Copy, Debug, PartialEq, Eq)]
pub enum AttentionMask {
Causal,
Bidirectional,
}
#[derive(Debug, Default)]
pub struct AttentionSides {
pub value_embed: Option<Tensor<3>>,
pub key_valid: Option<Tensor<2, Bool>>,
}
#[derive(Config, Debug)]
pub struct CausalSelfAttentionConfig {
pub n_head: usize,
pub n_kv_head: usize,
pub n_embed: usize,
#[config(default = "Some(BunsenNormalizationConfig::Rms(RmsNormConfig::new(0)))")]
pub norm: Option<BunsenNormalizationConfig>,
#[config(default = "None")]
pub head_dim: Option<usize>,
#[config(default = "QkNormPlacement::AfterRope")]
pub qk_norm: QkNormPlacement,
#[config(default = "false")]
pub qkv_bias: bool,
#[config(default = "None")]
pub key_bias: Option<bool>,
#[config(default = "false")]
pub output_bias: bool,
#[config(default = "None")]
pub sliding_window: Option<usize>,
#[config(default = "AttentionMask::Causal")]
pub mask: AttentionMask,
#[config(default = "None")]
pub softmax_dtype: Option<FloatDType>,
#[config(default = "None")]
pub score_scale: Option<f64>,
#[config(default = "None")]
pub attn_softcap: Option<f64>,
#[config(default = "None")]
pub value_residual: Option<ValueResidualConfig>,
#[config(default = "false")]
pub output_gate: bool,
}
#[derive(Config, Debug)]
pub struct ValueResidualConfig {
#[config(default = "12")]
pub gate_channels: usize,
#[config(default = "3.0")]
pub gate_scale: f64,
}
#[derive(Config, Debug, Copy, PartialEq, Eq)]
pub enum QkNormPlacement {
AfterRope,
BeforeRope,
}
impl QkNormPlacement {
pub fn apply(
self,
x: Tensor<4>,
norm: Option<&BunsenNormalization>,
r_emb: &RotaryEmbedding,
) -> Tensor<4> {
let Some(norm) = norm else {
return r_emb.apply(x);
};
match self {
Self::AfterRope => norm.forward(r_emb.apply(x)),
Self::BeforeRope => r_emb.apply(norm.forward(x)),
}
}
}
impl CausalSelfAttentionMeta for CausalSelfAttentionConfig {
fn n_embed(&self) -> usize {
self.n_embed
}
fn n_head(&self) -> usize {
self.n_head
}
fn n_kv_head(&self) -> usize {
self.n_kv_head
}
fn head_dim(&self) -> usize {
self.head_dim.unwrap_or(self.n_embed / self.n_head.max(1))
}
}
impl CausalSelfAttentionConfig {
pub fn key_bias(&self) -> bool {
self.key_bias.unwrap_or(self.qkv_bias)
}
pub fn validate(&self) {
assert!(
self.n_head > 0,
"n_head must be > 0; got n_head={}",
self.n_head
);
assert!(
self.head_dim.is_some() || self.n_embed.is_multiple_of(self.n_head),
"n_embed ({}) must be multiple of n_head ({})",
self.n_embed,
self.n_head
);
assert!(
self.head_dim.is_some() || self.n_embed.is_multiple_of(self.n_kv_head),
"n_embed ({}) must be multiple of n_kv_head ({})",
self.n_embed,
self.n_kv_head
);
assert!(
self.head_dim() > 0,
"head_dim must be > 0; got n_embed={}/n_head={}",
self.n_embed,
self.n_head
);
assert!(
self.n_kv_head <= self.n_head,
"n_kv_head must be <= n_head; got n_kv_head={}, n_head={}",
self.n_kv_head,
self.n_head
);
assert!(
self.n_head.is_multiple_of(self.n_kv_head),
"n_head must be divisible by n_kv_head; got n_head={}, n_kv_head={}",
self.n_head,
self.n_kv_head
);
assert!(
self.sliding_window != Some(0),
"sliding_window must be > 0 when set"
);
if let Some(vr) = &self.value_residual {
assert!(
vr.gate_channels > 0 && vr.gate_channels <= self.n_embed,
"value_residual.gate_channels must be in 1..=n_embed ({}); got {}",
self.n_embed,
vr.gate_channels
);
}
}
}
impl CausalSelfAttentionConfig {
pub fn try_init(
&self,
layer_index: usize,
device: &Device,
) -> BunsenResult<CausalSelfAttention> {
self.validate();
let head_dim = self.head_dim();
let norm = || {
self.norm
.clone()
.map(|n| n.with_num_features(head_dim).init(device))
};
let linear =
|d_in, d_out, bias| LinearConfig::new(d_in, d_out).with_bias(bias).init(device);
Ok(CausalSelfAttention {
layer_index,
head_dim,
qk_norm: self.qk_norm,
sliding_window: self.sliding_window,
mask: self.mask,
softmax_dtype: self.softmax_dtype,
output_gate: self.output_gate,
c_q: linear(
self.n_embed,
self.n_head * head_dim * if self.output_gate { 2 } else { 1 },
self.qkv_bias,
),
q_norm: norm(),
c_k: linear(self.n_embed, self.n_kv_head * head_dim, self.key_bias()),
k_norm: norm(),
c_v: linear(self.n_embed, self.n_kv_head * head_dim, self.qkv_bias),
c_proj: linear(self.n_head * head_dim, self.n_embed, self.output_bias),
score_scale: self.score_scale,
attn_softcap: self.attn_softcap,
value_gate: self.value_residual.as_ref().map(|vr| {
LinearConfig::new(vr.gate_channels, self.n_kv_head)
.with_bias(false)
.with_initializer(Initializer::Uniform {
min: 0.0,
max: 0.02,
})
.init(device)
}),
value_gate_scale: self.value_residual.as_ref().map(|vr| vr.gate_scale),
})
}
pub fn init(
&self,
layer_index: usize,
device: &Device,
) -> CausalSelfAttention {
self.try_init(layer_index, device).ok_or_panic()
}
}
#[derive(Module, Debug)]
pub struct CausalSelfAttention {
pub layer_index: usize,
pub head_dim: usize,
#[module(skip)]
pub qk_norm: QkNormPlacement,
#[module(skip)]
pub sliding_window: Option<usize>,
#[module(skip)]
pub mask: AttentionMask,
#[module(skip)]
pub softmax_dtype: Option<FloatDType>,
#[module(skip)]
pub output_gate: bool,
pub c_q: Linear,
pub q_norm: Option<BunsenNormalization>,
pub c_k: Linear,
pub k_norm: Option<BunsenNormalization>,
pub c_v: Linear,
pub c_proj: Linear,
#[module(skip)]
pub score_scale: Option<f64>,
#[module(skip)]
pub attn_softcap: Option<f64>,
pub value_gate: Option<Linear>,
#[module(skip)]
pub value_gate_scale: Option<f64>,
}
impl CausalSelfAttentionMeta for CausalSelfAttention {
fn n_embed(&self) -> usize {
self.c_q.weight.dims()[0]
}
fn n_head(&self) -> usize {
self.c_proj.weight.dims()[0] / self.head_dim()
}
fn n_kv_head(&self) -> usize {
self.c_k.weight.dims()[1] / self.head_dim()
}
fn head_dim(&self) -> usize {
self.head_dim
}
}
impl crate::burner::module::InputDTypes for CausalSelfAttention {
fn preferred_dtype(&self) -> FloatDType {
FloatDType::from(self.c_q.weight.val().dtype())
}
}
impl crate::burner::module::ResultDType for CausalSelfAttention {
fn result_dtype(
&self,
_input: FloatDType,
) -> FloatDType {
crate::burner::module::InputDTypes::preferred_dtype(self)
}
}
impl CausalSelfAttention {
fn queries_keys(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
) -> (Tensor<4>, Tensor<4>, Option<Tensor<3>>) {
let [b, t] = unpack_shape_contract!(["B", "T", "D"], &input.dims(), &["B", "T"]);
let (h, d) = (self.n_head(), self.head_dim());
let q = self.c_q.forward(input.clone());
let (q, gate) = if self.output_gate {
let q = q.reshape([b, t, h, 2 * d]);
let gate = q.clone().slice(s![.., .., .., d..]).reshape([b, t, h * d]);
(q.slice(s![.., .., .., ..d]), Some(gate))
} else {
(q.reshape([b, t, h, d]), None)
};
let q = self
.qk_norm
.apply(q, self.q_norm.as_ref(), r_emb)
.swap_dims(1, 2);
let k = self
.c_k
.forward(input)
.reshape([b, t, self.n_kv_head(), self.head_dim()]);
let k = self
.qk_norm
.apply(k, self.k_norm.as_ref(), r_emb)
.swap_dims(1, 2);
(q, k, gate)
}
fn sdpa_config(&self) -> ScaledDotProductAttentionConfig {
ScaledDotProductAttentionConfig::new()
.with_enable_gqa(self.gqa_enabled())
.with_softmax_dtype(self.softmax_dtype)
.with_scale(self.score_scale)
.with_softcap(self.attn_softcap)
}
fn values(
&self,
input: Tensor<3>,
value_embed: Option<Tensor<3>>,
) -> Tensor<4> {
let [b, t] = unpack_shape_contract!(["B", "T", "D"], &input.dims(), &["B", "T"]);
let shape = [b, t, self.n_kv_head(), self.head_dim()];
let v = self.c_v.forward(input.clone()).reshape(shape);
let v = match (&self.value_gate, value_embed) {
(None, None) => v,
(Some(gate), Some(ve)) => {
assert_shape_contract_periodically!(
["B", "T", "V"],
&ve.dims(),
&[
("B", b),
("T", t),
("V", self.n_kv_head() * self.head_dim())
]
);
let channels = gate.weight.dims()[0];
let scale = self.value_gate_scale.unwrap_or(1.0);
let g = burn::tensor::activation::sigmoid(gate.forward(input.slice(s![
..,
..,
..channels
])))
.mul_scalar(scale)
.unsqueeze_dim::<4>(3);
v + g * ve.reshape(shape)
}
(None, Some(_)) => {
panic!("a value embedding for an attention without a value residual")
}
(Some(_), None) => {
panic!("an attention with a value residual needs its value embedding")
}
};
v.swap_dims(1, 2)
}
pub fn attention_weights(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
) -> Tensor<4> {
let [b, t] = unpack_shape_contract!(
["B", "T", "D"],
&input.dims(),
&["B", "T"],
&[("D", self.n_embed())]
);
let (q, k, _) = self.queries_keys(input, r_emb);
let device = q.device();
let window = self.sliding_window.filter(|&w| w < t);
let visible = window.map(|w| match self.mask {
AttentionMask::Causal => causal_window_visible(t, t, Some(w), &device),
AttentionMask::Bidirectional => two_sided_window_visible(t, w, &device),
});
let weights = sdpa_attn_weight(
q,
k,
None,
visible,
None,
self.sdpa_config()
.with_is_causal(window.is_none() && self.mask == AttentionMask::Causal),
);
assert_shape_contract_periodically!(
["B", "H", "T", "T"],
&weights.dims(),
&[("B", b), ("H", self.n_head()), ("T", t)]
);
weights
}
pub fn forward(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
) -> Tensor<3> {
self.forward_with(input, r_emb, kv_cache, AttentionSides::default())
}
pub fn forward_with(
&self,
input: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
sides: AttentionSides,
) -> Tensor<3> {
let AttentionSides {
value_embed,
key_valid,
} = sides;
let [b, t_q] = unpack_shape_contract!(
["B", "T", "D"],
&input.dims(),
&["B", "T"],
&[("D", self.n_embed())]
);
#[cfg(any(debug_assertions, test))]
crate::burner::module::assert_input_dtype(self, input.dtype(), "causal self-attention");
let (q, k, gate) = self.queries_keys(input.clone(), r_emb);
let v = self.values(input, value_embed);
let mut k = k;
let mut v = v;
if let Some(kvc) = kv_cache {
(k, v) = (*kvc).insert_kv(self.layer_index, k.clone(), v.clone());
}
let mut t_kv = k.dims()[2];
let window = self.sliding_window.filter(|&w| w < t_kv);
if let (Some(w), 1) = (window, t_q) {
k = k.slice(s![.., .., t_kv - w..]);
v = v.slice(s![.., .., t_kv - w..]);
t_kv = w;
}
let attn_mask: Option<Tensor<2, Bool>>;
let is_causal: bool;
if self.mask == AttentionMask::Bidirectional {
assert!(
kv_cache.is_none(),
"a bidirectional attention takes no KV cache"
);
attn_mask = window.map(|w| two_sided_window_visible(t_q, w, &q.device()));
is_causal = false;
} else if let (Some(w), true) = (window, t_q > 1) {
attn_mask = Some(causal_window_visible(t_q, t_kv, Some(w), &q.device()));
is_causal = false;
} else if kv_cache.is_none() || t_q == t_kv {
attn_mask = None;
is_causal = true;
} else if t_q == 1 {
attn_mask = None;
is_causal = false;
} else {
let device = q.device();
let mut mask = Tensor::<2, Bool>::empty([t_q, t_kv], &device);
let prefix_len = t_kv - t_q;
if prefix_len > 0 {
mask = mask.slice_fill(s![.., ..prefix_len], true);
}
let fill = Tensor::<2, Int>::ones([t_q, t_q], &device).tril(0).bool();
mask = mask.slice_assign(s![.., prefix_len..], fill);
attn_mask = Some(mask);
is_causal = false;
};
let cfg = self.sdpa_config().with_is_causal(is_causal);
let y = scaled_dot_product_attention(q, k, v, None, attn_mask, key_valid, cfg)
.swap_dims(1, 2)
.reshape([
b as i32,
t_q as i32,
(self.n_head() * self.head_dim()) as i32,
]);
let y = match gate {
Some(gate) => y * burn::tensor::activation::sigmoid(gate),
None => y,
};
let y = self.c_proj.forward(y);
assert_shape_contract_periodically!(
["B", "T", "D"],
&y.dims(),
&[("B", b), ("T", t_q), ("D", self.n_embed())]
);
y
}
}
#[derive(Debug)]
pub struct CausalAttnCall<'a, 'c> {
pub r_emb: &'a RotaryEmbedding,
pub kv_cache: &'a mut Option<&'c mut KVCache>,
pub sides: AttentionSides,
}
impl SublayerBranch<CausalAttnCall<'_, '_>, 3> for CausalSelfAttention {
type Output = Tensor<3>;
fn call_branch(
&self,
z: Tensor<3>,
ctx: CausalAttnCall<'_, '_>,
) -> Tensor<3> {
self.forward_with(z, ctx.r_emb, ctx.kv_cache, ctx.sides)
}
}
pub trait CausalAttnSublayer {
fn causal_attn(
&self,
x: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
sides: AttentionSides,
) -> Tensor<3>;
}
impl<M> CausalAttnSublayer for ResidualSublayer<M>
where
M: Module + for<'a, 'c> SublayerBranch<CausalAttnCall<'a, 'c>, 3, Output = Tensor<3>>,
{
fn causal_attn(
&self,
x: Tensor<3>,
r_emb: &RotaryEmbedding,
kv_cache: &mut Option<&mut KVCache>,
sides: AttentionSides,
) -> Tensor<3> {
self.residual_update(
x,
CausalAttnCall {
r_emb,
kv_cache,
sides,
},
)
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Tolerance,
},
};
use serial_test::serial;
use super::*;
use crate::{
blocks::transformers::embedding::{
RotaryEmbedding,
RotaryEmbeddingConfig,
},
burner::module::ModuleInit,
contracts::assert_shape_contract,
ops::transformers::attention::KVCacheConfig,
support::testing::{
DeviceMemoryGuard,
assert_tensors_close,
cpu_device,
performance_device,
seeded_tensor,
},
};
#[test]
fn test_csa_config() {
let cfg = CausalSelfAttentionConfig::new(3, 2, 4);
assert_eq!(cfg.n_embed, 4);
assert_eq!(cfg.n_head, 3);
assert_eq!(cfg.n_kv_head, 2);
assert_eq!(cfg.n_embed(), 4);
assert_eq!(cfg.n_head(), 3);
assert_eq!(cfg.n_kv_head(), 2);
assert_eq!(cfg.head_dim(), 1);
}
#[test]
#[allow(unused)]
fn test_csa_forward() {
let device = cpu_device();
let batch = 1;
let seq_len = 10;
let n_embed = 1024;
let n_head = 128;
let n_kv_head = 64;
let layer_index = 12;
let cfg = CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed);
let csa: CausalSelfAttention = cfg.init(layer_index, &device);
let head_dim = csa.head_dim();
let re_cfg = RotaryEmbeddingConfig::new(seq_len, csa.head_dim());
let r_emb: RotaryEmbedding = re_cfg.init(&device);
let input: Tensor<3> =
Tensor::random([batch, seq_len, n_embed], Distribution::Default, &device);
let mut kv_cache: Option<&mut KVCache> = None;
let output = csa.forward(input.clone(), &r_emb, &mut kv_cache);
assert_shape_contract!(
["B", "T", "D"],
&output.dims(),
&[("B", batch), ("T", seq_len), ("D", n_embed)]
);
}
#[test]
fn test_attention_weights_are_the_forwards() {
let device = cpu_device();
let (batch, seq_len, n_embed, n_head, n_kv_head) = (2, 6, 32, 4, 2);
let csa: CausalSelfAttention =
CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed).init(0, &device);
let head_dim = csa.head_dim();
let r_emb: RotaryEmbedding = RotaryEmbeddingConfig::new(seq_len, head_dim).init(&device);
let input: Tensor<3> =
Tensor::random([batch, seq_len, n_embed], Distribution::Default, &device);
let weights = csa.attention_weights(input.clone(), &r_emb);
assert_eq!(weights.dims(), [batch, n_head, seq_len, seq_len]);
let sums = weights.clone().sum_dim(3);
assert_tensors_close(&sums, &Tensor::ones_like(&sums), Tolerance::default());
let upper = weights.clone().triu(1).abs().max().into_scalar::<f32>();
assert_eq!(upper, 0.0);
let v = csa
.c_v
.forward(input.clone())
.reshape([batch, seq_len, n_kv_head, head_dim])
.swap_dims(1, 2);
let v = crate::ops::repeat::repeat_interleave::<4, 5, _>(v, n_head / n_kv_head, 1);
let applied = csa
.c_proj
.forward(weights.matmul(v).swap_dims(1, 2).reshape([
batch,
seq_len,
n_head * head_dim,
]));
let forward = csa.forward(input, &r_emb, &mut None);
assert_tensors_close(&applied, &forward, Tolerance::default());
}
#[test]
fn test_bidirectional_with_padding() {
let device = cpu_device();
let (seq_len, n_embed, n_head) = (5, 16, 2);
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(n_head, n_head, n_embed)
.with_norm(None)
.with_mask(AttentionMask::Bidirectional)
.init(0, &device);
let r_emb: RotaryEmbedding =
RotaryEmbeddingConfig::new(seq_len, csa.head_dim()).init(&device);
let input = seeded_tensor::<3>(4, [2, seq_len, n_embed], Distribution::Default, &device);
let weights = csa.attention_weights(input.clone(), &r_emb);
let upper = weights.triu(1).abs().max().into_scalar::<f32>();
assert!(upper > 0.0, "a query sees the keys after it");
let valid =
crate::ops::transformers::attention::length_mask(&[seq_len, 3], seq_len, &device);
let forward = |x: Tensor<3>| {
csa.forward_with(
x,
&r_emb,
&mut None,
AttentionSides {
key_valid: Some(valid.clone()),
..Default::default()
},
)
};
let noise = seeded_tensor::<3>(5, [1, 2, n_embed], Distribution::Default, &device);
let changed = input.clone().slice_assign([1..2, 3..5, 0..n_embed], noise);
assert_tensors_close(
&forward(input.clone()).slice([1..2, 0..3, 0..n_embed]),
&forward(changed.clone()).slice([1..2, 0..3, 0..n_embed]),
Tolerance::default(),
);
let changed0 = input.clone().slice_assign(
[0..1, 3..5, 0..n_embed],
seeded_tensor::<3>(6, [1, 2, n_embed], Distribution::Default, &device),
);
assert!(!forward(input).slice([0..1, 0..3, 0..n_embed]).all_close(
forward(changed0).slice([0..1, 0..3, 0..n_embed]),
None,
None
));
}
#[test]
fn test_two_sided_window_is_the_window_alone() {
let device = cpu_device();
let (seq_len, window) = (9, 3);
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(4, 2, 16)
.with_norm(None)
.with_mask(AttentionMask::Bidirectional)
.with_sliding_window(Some(window))
.init(0, &device);
let r_emb = RotaryEmbeddingConfig::new(seq_len, csa.head_dim()).init(&device);
let input = seeded_tensor::<3>(7, [2, seq_len, 16], Distribution::Default, &device);
let full = csa.forward(input.clone(), &r_emb, &mut None);
let mut unwindowed = csa.clone();
unwindowed.sliding_window = None;
for i in 0..seq_len {
let (start, end) = ((i + 1).saturating_sub(window), (i + window).min(seq_len));
let alone = unwindowed.forward(
input.clone().slice(s![.., start..end]),
&r_emb.clip_range(start..end),
&mut None,
);
assert_tensors_close(
&full.clone().slice(s![.., i..=i]),
&alone.slice(s![.., i - start..=i - start]),
Tolerance::default(),
);
}
let weights = csa.attention_weights(input, &r_emb);
let sums = weights.clone().sum_dim(3);
assert_tensors_close(&sums, &Tensor::ones_like(&sums), Tolerance::default());
let outside = weights
.clone()
.triu(window as i64)
.abs()
.max()
.into_scalar::<f32>()
+ weights
.clone()
.tril(-(window as i64))
.abs()
.max()
.into_scalar::<f32>();
assert_eq!(outside, 0.0);
let after = weights.triu(1).abs().max().into_scalar::<f32>();
assert!(after > 0.0, "a query sees the keys after it");
}
#[test]
fn test_two_sided_window_with_padding() {
let device = cpu_device();
let (seq_len, window, valid_len) = (7, 3, 5);
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(2, 2, 16)
.with_norm(None)
.with_mask(AttentionMask::Bidirectional)
.with_sliding_window(Some(window))
.init(0, &device);
let r_emb = RotaryEmbeddingConfig::new(seq_len, csa.head_dim()).init(&device);
let input = seeded_tensor::<3>(3, [1, seq_len, 16], Distribution::Default, &device);
let valid =
crate::ops::transformers::attention::length_mask(&[valid_len], seq_len, &device);
let padded = csa.forward_with(
input.clone(),
&r_emb,
&mut None,
AttentionSides {
key_valid: Some(valid),
..Default::default()
},
);
let mut unwindowed = csa.clone();
unwindowed.sliding_window = None;
for i in 0..valid_len {
let (start, end) = ((i + 1).saturating_sub(window), (i + window).min(valid_len));
let alone = unwindowed.forward(
input.clone().slice(s![.., start..end]),
&r_emb.clip_range(start..end),
&mut None,
);
assert_tensors_close(
&padded.clone().slice(s![.., i..=i]),
&alone.slice(s![.., i - start..=i - start]),
Tolerance::default(),
);
}
}
#[test]
#[should_panic(expected = "takes no KV cache")]
fn test_bidirectional_takes_no_cache() {
let device = cpu_device();
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(2, 2, 16)
.with_mask(AttentionMask::Bidirectional)
.init(0, &device);
let r_emb: RotaryEmbedding = RotaryEmbeddingConfig::new(3, csa.head_dim()).init(&device);
let mut cache = KVCacheConfig::new(1, 2, 3, csa.head_dim(), 1).init();
let _ = csa.forward(
Tensor::zeros([1, 3, 16], &device),
&r_emb,
&mut Some(&mut cache),
);
}
#[test]
fn test_score_scale_and_softcap() {
let device = cpu_device();
let (batch, seq_len, n_embed, n_head, n_kv_head) = (1, 5, 16, 2, 1);
let (scale, cap) = (0.9, 0.7);
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed)
.with_score_scale(Some(scale))
.with_attn_softcap(Some(cap))
.init(0, &device);
let r_emb: RotaryEmbedding =
RotaryEmbeddingConfig::new(seq_len, csa.head_dim()).init(&device);
let input =
seeded_tensor::<3>(3, [batch, seq_len, n_embed], Distribution::Default, &device);
let (q, k, _) = csa.queries_keys(input.clone(), &r_emb);
let k = crate::ops::repeat::repeat_interleave::<4, 5, _>(k, n_head / n_kv_head, 1);
let scores = q.matmul(k.swap_dims(2, 3)).mul_scalar(scale);
assert!(
scores.clone().abs().max().into_scalar::<f32>() > cap as f32,
"the cap binds"
);
let capped = scores.div_scalar(cap).tanh().mul_scalar(cap);
let visible = Tensor::<2, Int>::ones([seq_len, seq_len], &device)
.tril(0)
.bool();
let expected = burn::tensor::activation::softmax(
capped.mask_fill(visible.bool_not().unsqueeze::<4>(), f32::NEG_INFINITY),
3,
);
assert_tensors_close(
&csa.attention_weights(input.clone(), &r_emb),
&expected,
Tolerance::default(),
);
let plain: CausalSelfAttention =
CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed).init(0, &device);
assert!(plain.score_scale.is_none() && plain.attn_softcap.is_none());
}
#[test]
#[serial]
fn test_csa_cached_chunk_matches_full_pass() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let [batch, seq_len, n_embed] = [2, 6, 16];
let prefix = 3;
for output_gate in [false, true] {
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(4, 2, n_embed)
.with_output_gate(output_gate)
.init(0, &device);
let r_emb: RotaryEmbedding =
RotaryEmbeddingConfig::new(seq_len, csa.head_dim()).init(&device);
let input =
seeded_tensor::<3>(1, [batch, seq_len, n_embed], Distribution::Default, &device);
let full = csa.forward(input.clone(), &r_emb, &mut None);
let mut cache: KVCache =
KVCacheConfig::new(batch, csa.n_kv_head(), seq_len, csa.head_dim(), 1).init();
let head = csa.forward(
input.clone().slice(s![.., ..prefix]),
&r_emb.clip_range(0..prefix),
&mut Some(&mut cache),
);
let chunk = csa.forward(
input.slice(s![.., prefix..]),
&r_emb.clip_range(prefix..seq_len),
&mut Some(&mut cache),
);
assert_tensors_close(
&head,
&full.clone().slice(s![.., ..prefix]),
Tolerance::default(),
);
assert_tensors_close(&chunk, &full.slice(s![.., prefix..]), Tolerance::default());
}
}
#[test]
fn test_explicit_head_dim() {
use crate::{
blocks::transformers::embedding::RotaryEmbeddingConfig,
burner::module::ModuleInit,
support::testing::cpu_device,
};
let device = cpu_device();
for placement in [QkNormPlacement::AfterRope, QkNormPlacement::BeforeRope] {
let cfg = CausalSelfAttentionConfig::new(2, 1, 12)
.with_head_dim(Some(8))
.with_qk_norm(placement);
assert_eq!(cfg.head_dim(), 8);
let attn = cfg.init(0, &device);
assert_eq!(attn.n_head(), 2);
assert_eq!(attn.n_kv_head(), 1);
assert_eq!(attn.c_proj.weight.dims(), [16, 12]);
let r_emb = RotaryEmbeddingConfig::new(5, 8).init(&device);
let y = attn.forward(Tensor::zeros([2, 5, 12], &device), &r_emb, &mut None);
assert_eq!(y.dims(), [2, 5, 12]);
}
}
#[test]
fn test_meta_and_options() {
let device = cpu_device();
let cfg = CausalSelfAttentionConfig::new(4, 2, 12)
.with_head_dim(Some(5))
.with_norm(None)
.with_qk_norm(QkNormPlacement::BeforeRope)
.with_qkv_bias(true)
.with_key_bias(Some(false))
.with_output_bias(true)
.with_sliding_window(Some(3));
let csa: CausalSelfAttention = cfg.init(1, &device);
assert_eq!(csa.n_embed(), cfg.n_embed());
assert_eq!(csa.n_head(), cfg.n_head());
assert_eq!(csa.n_kv_head(), cfg.n_kv_head());
assert_eq!(csa.head_dim(), cfg.head_dim());
assert_eq!(csa.qk_norm, cfg.qk_norm);
assert_eq!(csa.sliding_window, cfg.sliding_window);
assert!(csa.q_norm.is_none() && csa.k_norm.is_none());
assert!(csa.c_q.bias.is_some());
assert!(!cfg.key_bias());
assert!(csa.c_k.bias.is_none());
assert!(csa.c_v.bias.is_some());
assert!(csa.c_proj.bias.is_some());
let defaults: CausalSelfAttention =
CausalSelfAttentionConfig::new(4, 2, 12).init(0, &device);
assert!(defaults.q_norm.is_some() && defaults.k_norm.is_some());
for linear in [
&defaults.c_q,
&defaults.c_k,
&defaults.c_v,
&defaults.c_proj,
] {
assert!(linear.bias.is_none());
}
assert_eq!(defaults.sliding_window, None);
}
#[test]
fn test_output_gate() {
let device = cpu_device();
let (batch, seq_len, n_embed, n_head, n_kv_head) = (2, 5, 16, 4, 2);
let cfg = CausalSelfAttentionConfig::new(n_head, n_kv_head, n_embed);
let mut gated: CausalSelfAttention = cfg.clone().with_output_gate(true).init(0, &device);
let d = gated.head_dim();
assert_eq!(gated.c_q.weight.dims(), [n_embed, 2 * n_head * d]);
assert_eq!(gated.n_head(), n_head);
assert_eq!(gated.n_kv_head(), n_kv_head);
let identity = Param::from_tensor(Tensor::<2>::eye(n_embed, &device));
gated.c_proj.weight = identity.clone();
let w = gated.c_q.weight.val().reshape([n_embed, n_head, 2 * d]);
let query_cols = w
.clone()
.slice(s![.., .., ..d])
.reshape([n_embed, n_head * d]);
let gate_cols = w.slice(s![.., .., d..]).reshape([n_embed, n_head * d]);
let mut plain: CausalSelfAttention = cfg.init(0, &device);
plain.c_q.weight = Param::from_tensor(query_cols);
plain.c_k = gated.c_k.clone();
plain.c_v = gated.c_v.clone();
plain.q_norm = gated.q_norm.clone();
plain.k_norm = gated.k_norm.clone();
plain.c_proj.weight = identity;
let r_emb = RotaryEmbeddingConfig::new(seq_len, d).init(&device);
let input =
seeded_tensor::<3>(8, [batch, seq_len, n_embed], Distribution::Default, &device);
let gate = burn::tensor::activation::sigmoid(input.clone().matmul(gate_cols.unsqueeze()));
let expected = plain.forward(input.clone(), &r_emb, &mut None) * gate;
let actual = gated.forward(input.clone(), &r_emb, &mut None);
assert_tensors_close(&actual, &expected, Tolerance::default());
assert_tensors_close(
&gated.attention_weights(input.clone(), &r_emb),
&plain.attention_weights(input, &r_emb),
Tolerance::default(),
);
}
fn windowed(window: usize) -> (CausalSelfAttention, RotaryEmbedding, Tensor<3>) {
let device = cpu_device();
let csa: CausalSelfAttention = CausalSelfAttentionConfig::new(4, 2, 16)
.with_norm(None)
.with_qkv_bias(true)
.with_key_bias(Some(false))
.with_output_bias(true)
.with_sliding_window(Some(window))
.init(0, &device);
let r_emb = RotaryEmbeddingConfig::new(9, csa.head_dim()).init(&device);
let input = seeded_tensor::<3>(7, [2, 9, 16], Distribution::Default, &device);
(csa, r_emb, input)
}
#[test]
fn test_sliding_window_is_the_window_alone() {
let window = 3;
let (csa, r_emb, input) = windowed(window);
let full = csa.forward(input.clone(), &r_emb, &mut None);
let mut unwindowed = csa.clone();
unwindowed.sliding_window = None;
for i in 0..9 {
let start = (i + 1usize).saturating_sub(window);
let alone = unwindowed.forward(
input.clone().slice(s![.., start..=i]),
&r_emb.clip_range(start..i + 1),
&mut None,
);
assert_tensors_close(
&full.clone().slice(s![.., i..=i]),
&alone.slice(s![.., i - start..=i - start]),
Tolerance::default(),
);
}
let weights = csa.attention_weights(input, &r_emb);
let sums = weights.clone().sum_dim(3);
assert_tensors_close(&sums, &Tensor::ones_like(&sums), Tolerance::default());
let outside = weights.clone().triu(1).abs().max().into_scalar::<f32>()
+ weights
.tril(-(window as i64))
.abs()
.max()
.into_scalar::<f32>();
assert_eq!(outside, 0.0);
}
#[test]
fn test_sliding_window_through_the_cache() {
let (csa, r_emb, input) = windowed(3);
let full = csa.forward(input.clone(), &r_emb, &mut None);
let mut cache: KVCache = KVCacheConfig::new(2, 2, 9, csa.head_dim(), 1).init();
let steps: Vec<Tensor<3>> = (0..9)
.map(|i| {
csa.forward(
input.clone().slice(s![.., i..=i]),
&r_emb.clip_range(i..i + 1),
&mut Some(&mut cache),
)
})
.collect();
assert_tensors_close(&Tensor::cat(steps, 1), &full, Tolerance::default());
let mut cache: KVCache = KVCacheConfig::new(2, 2, 9, csa.head_dim(), 1).init();
let head = csa.forward(
input.clone().slice(s![.., ..4]),
&r_emb.clip_range(0..4),
&mut Some(&mut cache),
);
let chunk = csa.forward(
input.slice(s![.., 4..]),
&r_emb.clip_range(4..9),
&mut Some(&mut cache),
);
assert_tensors_close(
&Tensor::cat(vec![head, chunk], 1),
&full,
Tolerance::default(),
);
}
}