use burn::{
Tensor,
nn::attention::{
MhaInput,
MhaOutput,
MultiHeadAttention,
},
prelude::Bool,
tensor::activation::{
quiet_softmax,
softmax,
},
};
#[cfg(not(feature = "std"))]
#[allow(unused_imports)]
use num_traits::Float;
use super::KVCache;
pub fn project_kv(
mha: &MultiHeadAttention,
x: Tensor<3>,
) -> (Tensor<4>, Tensor<4>) {
(
split_heads(mha, mha.key.forward(x.clone())),
split_heads(mha, mha.value.forward(x)),
)
}
pub fn split_heads(
mha: &MultiHeadAttention,
x: Tensor<3>,
) -> Tensor<4> {
let [batch, seq, _] = x.dims();
x.reshape([batch, seq, mha.n_heads, mha.d_k])
.swap_dims(1, 2)
}
pub fn self_attn(
mh_attn: &MultiHeadAttention,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
) -> MhaOutput {
#[cfg(any(debug_assertions, test))]
{
use crate::contracts::*;
let d_model = mh_attn.d_model;
let [batch, seq_len] = unpack_shape_contract!(
["batch", "seq_len", "d_model"],
&x,
&["batch", "seq_len"],
&[("d_model", d_model)]
);
if let Some(mask) = &mask {
let [mask_batch] = unpack_shape_contract!(
["mask_batch", "seq_len", "seq_len"],
mask,
&["mask_batch"],
&[("seq_len", seq_len)]
);
if mask_batch != 1 {
assert_eq!(
mask_batch, batch,
"batch sizes not broadcastable {batch} vs {mask_batch}"
);
}
}
}
let input = MhaInput::self_attn(x);
let input = match mask {
Some(mask) => input.mask_attn(mask),
None => input,
};
mh_attn.forward(input)
}
pub fn cross_attn(
mh_attn: &MultiHeadAttention,
x: Tensor<3>,
xa: Tensor<3>,
) -> MhaOutput {
#[cfg(any(debug_assertions, test))]
{
crate::contracts::define_shape_contract!(CONTRACT, ["batch", "seq_len", "d_model"]);
let d_model = mh_attn.d_model;
let [batch, _seq_len] =
CONTRACT.unpack_shape(&x, &["batch", "seq_len"], &[("d_model", d_model)]);
CONTRACT.assert_shape(&xa, &[("batch", batch), ("d_model", d_model)]);
};
mh_attn.forward(MhaInput::new(x, xa.clone(), xa))
}
pub fn attend_q_kv_mask(
mha: &MultiHeadAttention,
q: Tensor<4>,
k: Tensor<4>,
v: Tensor<4>,
mask: Option<Tensor<3, Bool>>,
) -> Tensor<3> {
let [batch, _, seq_new, _] = q.dims();
let scores = q.matmul(k.transpose()).div_scalar((mha.d_k as f32).sqrt());
let scores = match mask {
Some(mask) => {
let [mask_batch, seq_q, seq_k] = mask.dims();
scores.mask_fill(mask.reshape([mask_batch, 1, seq_q, seq_k]), mha.min_float)
}
None => scores,
};
let weights = if mha.quiet_softmax {
quiet_softmax(scores, 3)
} else {
softmax(scores, 3)
};
let context = weights
.matmul(v)
.swap_dims(1, 2)
.reshape([batch, seq_new, mha.d_model]);
mha.output.forward(context)
}
pub fn self_attn_w_kv_cache(
mha: &MultiHeadAttention,
x: Tensor<3>,
mask: Option<Tensor<3, Bool>>,
cache: &mut KVCache,
layer: usize,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract!(
["batch", "seq_new", "d_model"],
&x,
&[("d_model", mha.d_model)],
);
let q = split_heads(mha, mha.query.forward(x.clone()));
let (k, v) = project_kv(mha, x);
let (k, v) = cache.insert_kv(layer, k, v);
attend_q_kv_mask(mha, q, k, v, mask)
}
pub fn cross_attn_w_kv_cache(
mha: &MultiHeadAttention,
x: Tensor<3>,
cache: &KVCache,
layer: usize,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract!(
["batch", "seq_new", "d_model"],
&x,
&[("d_model", mha.d_model)],
);
let (k, v) = cache
.cross_kv(layer)
.expect("the layer's cross-attention keys and values are set");
let group = cache.group();
let [rows, seq_new, d_model] = x.dims();
let encoded = k.dims()[0];
assert_eq!(
rows,
encoded * group,
"query rows ({rows}) are not {group} per encoder row ({encoded})",
);
let q = split_heads(mha, mha.query.forward(x));
if group == 1 {
return attend_q_kv_mask(mha, q, k, v, None);
}
let [_, heads, _, d_k] = q.dims();
let q = q
.reshape([encoded, group, heads, seq_new, d_k])
.swap_dims(1, 2)
.reshape([encoded, heads, group * seq_new, d_k]);
attend_q_kv_mask(mha, q, k, v, None).reshape([rows, seq_new, d_model])
}
#[cfg(test)]
mod tests {
use burn::{
nn::{
LayerNormConfig,
attention::{
MhaInput,
MultiHeadAttentionConfig,
},
},
tensor::{
Distribution,
Tolerance,
},
};
use super::{
super::causal_mask,
*,
};
use crate::support::testing::cpu_device;
type F = f32;
#[test]
fn test_cross_attn_accepts_a_different_cross_length() {
let device = cpu_device();
let (batch, d_model, n_heads) = (2, 32, 4);
let (seq_len, cross_len) = (3, 17);
let attn = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([batch, seq_len, d_model], Distribution::Default, &device);
let xa = Tensor::<3>::random([batch, cross_len, d_model], Distribution::Default, &device);
let out = cross_attn(&attn, ln.forward(x), xa);
assert_eq!(out.context.dims(), [batch, seq_len, d_model]);
assert_eq!(out.weights.dims(), [batch, n_heads, seq_len, cross_len]);
}
#[test]
fn test_cross_attn_accepts_equal_lengths() {
let device = cpu_device();
let (batch, d_model, n_heads, seq_len) = (1, 16, 2, 5);
let attn = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([batch, seq_len, d_model], Distribution::Default, &device);
let out = cross_attn(&attn, ln.forward(x.clone()), x);
assert_eq!(out.context.dims(), [batch, seq_len, d_model]);
}
#[test]
#[should_panic(expected = "Shape Error")]
fn test_cross_attn_rejects_a_d_model_mismatch() {
let device = cpu_device();
let (batch, d_model, n_heads) = (1, 16, 2);
let attn = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([batch, 4, d_model], Distribution::Default, &device);
let xa = Tensor::<3>::random([batch, 9, d_model * 2], Distribution::Default, &device);
cross_attn(&attn, ln.forward(x), xa);
}
fn cache_for(
mha: &MultiHeadAttention,
batch: usize,
seq: usize,
) -> KVCache {
super::super::KVCacheConfig::new(batch, mha.n_heads, seq, mha.d_k, 1).init()
}
#[test]
fn test_cached_self_attn_matches_uncached() {
let device = cpu_device();
let (batch, d_model, n_heads, seq) = (2, 32, 4, 5);
let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([batch, seq, d_model], Distribution::Default, &device);
let mask = causal_mask(seq, 0, &device);
let whole = mha
.forward(MhaInput::self_attn(ln.forward(x.clone())).mask_attn(mask))
.context;
let mut cache = cache_for(&mha, batch, seq);
let mut steps = Vec::with_capacity(seq);
for t in 0..seq {
let step = x.clone().slice_dim(1, t as isize..(t + 1) as isize);
steps.push(self_attn_w_kv_cache(
&mha,
ln.forward(step),
None,
&mut cache,
0,
));
}
assert_eq!(cache.pos(), seq);
let stepped: Tensor<3> = Tensor::cat(steps, 1);
stepped
.to_data_as::<F>()
.assert_approx_eq::<F>(&whole.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
fn test_cached_self_attn_prefill_matches_stepping() {
let device = cpu_device();
let (batch, d_model, n_heads, seq) = (1, 32, 4, 6);
let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([batch, seq, d_model], Distribution::Default, &device);
let split = 4;
let mut cache = cache_for(&mha, batch, seq);
let prefill = self_attn_w_kv_cache(
&mha,
ln.forward(x.clone().slice_dim(1, 0..split as isize)),
Some(causal_mask(split, 0, &device)),
&mut cache,
0,
);
let mut parts = vec![prefill];
for t in split..seq {
let step = x.clone().slice_dim(1, t as isize..(t + 1) as isize);
parts.push(self_attn_w_kv_cache(
&mha,
ln.forward(step),
None,
&mut cache,
0,
));
}
let mixed: Tensor<3> = Tensor::cat(parts, 1);
let mut cache2 = cache_for(&mha, batch, seq);
let at_once = self_attn_w_kv_cache(
&mha,
ln.forward(x),
Some(causal_mask(seq, 0, &device)),
&mut cache2,
0,
);
mixed
.to_data_as::<F>()
.assert_approx_eq::<F>(&at_once.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
fn test_cached_cross_attn_matches_uncached() {
let device = cpu_device();
let (batch, d_model, n_heads) = (2, 32, 4);
let (seq, cross_len) = (3, 17);
let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([batch, seq, d_model], Distribution::Default, &device);
let xa = Tensor::<3>::random([batch, cross_len, d_model], Distribution::Default, &device);
let uncached = mha
.forward(MhaInput::new(ln.forward(x.clone()), xa.clone(), xa.clone()))
.context;
let (k, v) = project_kv(&mha, xa);
assert_eq!(k.dims()[2], cross_len);
let mut cache = cache_for(&mha, batch, seq);
cache.set_cross_kv(0, k, v);
let cached = cross_attn_w_kv_cache(&mha, ln.forward(x), &cache, 0);
cached
.to_data_as::<F>()
.assert_approx_eq::<F>(&uncached.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
fn test_grouped_cross_attn_matches_repeated() {
let device = cpu_device();
let (audio, group, d_model, n_heads, seq, cross_len) = (2, 3, 32, 4, 2, 7);
let rows = audio * group;
let mha = MultiHeadAttentionConfig::new(d_model, n_heads).init(&device);
let ln =
crate::blocks::norm::BunsenNormalizationConfig::from(LayerNormConfig::new(d_model))
.init(&device);
let x = Tensor::<3>::random([rows, seq, d_model], Distribution::Default, &device);
let xa = Tensor::<3>::random([audio, cross_len, d_model], Distribution::Default, &device);
let mut shared = cache_for(&mha, rows, seq);
let (k, v) = project_kv(&mha, xa.clone());
shared.set_cross_kv(0, k, v);
assert_eq!(shared.group(), group);
let mut repeated = cache_for(&mha, rows, seq);
let indices = Tensor::<1, burn::tensor::Int>::from_data(
burn::tensor::TensorData::new(
(0..rows).map(|r| (r / group) as i64).collect::<Vec<_>>(),
[rows],
),
&device,
);
let (k, v) = project_kv(&mha, xa.select(0, indices));
repeated.set_cross_kv(0, k, v);
assert_eq!(repeated.group(), 1);
let a = cross_attn_w_kv_cache(&mha, ln.forward(x.clone()), &shared, 0);
let b = cross_attn_w_kv_cache(&mha, ln.forward(x), &repeated, 0);
a.to_data_as::<F>()
.assert_approx_eq::<F>(&b.to_data_as::<F>(), Tolerance::permissive());
}
}