use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
},
prelude::Bool,
tensor::{
Device,
activation::softmax,
},
};
use crate::{
blocks::transformers::embedding::{
RotaryEmbedding,
RotaryEmbeddingMeta,
},
burner::module::ModuleInit,
contracts::unpack_shape_contract,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
};
pub trait RopeSelfAttentionMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn head_dim(&self) -> usize {
self.d_model() / self.n_heads()
}
}
#[derive(Config, Debug)]
pub struct RopeSelfAttentionConfig {
pub d_model: usize,
pub n_heads: usize,
#[config(default = "false")]
pub qkv_bias: bool,
#[config(default = "true")]
pub output_bias: bool,
}
impl RopeSelfAttentionMeta for RopeSelfAttentionConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.n_heads
}
}
impl ModuleInit<RopeSelfAttention> for RopeSelfAttentionConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<RopeSelfAttention> {
if self.n_heads == 0 {
return Err(
ConstraintError::zero_or_empty("RopeSelfAttentionConfig", "n_heads").into(),
);
}
if !self.d_model.is_multiple_of(2 * self.n_heads) {
return Err(ConstraintError::new(
"RopeSelfAttentionConfig",
"d_model",
Rule::NotMultiple {
value: self.d_model,
of: format!("2 * n_heads ({})", 2 * self.n_heads),
},
)
.into());
}
let projection = |bias| {
LinearConfig::new(self.d_model, self.d_model)
.with_bias(bias)
.init(device)
};
Ok(RopeSelfAttention {
query: projection(self.qkv_bias),
key: projection(self.qkv_bias),
value: projection(self.qkv_bias),
output: projection(self.output_bias),
n_heads: self.n_heads,
})
}
}
#[derive(Module, Debug)]
pub struct RopeSelfAttention {
pub query: Linear,
pub key: Linear,
pub value: Linear,
pub output: Linear,
pub n_heads: usize,
}
impl RopeSelfAttentionMeta for RopeSelfAttention {
fn d_model(&self) -> usize {
self.query.weight.dims()[0]
}
fn n_heads(&self) -> usize {
self.n_heads
}
}
impl RopeSelfAttention {
pub fn forward(
&self,
x: Tensor<3>,
rope: &RotaryEmbedding,
valid: Option<Tensor<2, Bool>>,
) -> Tensor<3> {
let [batch, time] = unpack_shape_contract!(
["batch", "time", "d_model"],
&x.dims(),
&["batch", "time"],
&[("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
{
crate::contracts::assert_shape_contract_periodically!(
[1, "time", 1, "half_head"],
&rope.cos.dims(),
&[("time", time), ("half_head", self.head_dim() / 2)],
);
if let Some(valid) = &valid {
crate::contracts::assert_shape_contract_periodically!(
["batch", "time"],
&valid.dims(),
&[("batch", batch), ("time", time)],
);
}
}
debug_assert_eq!(rope.head_dim(), self.head_dim());
let (heads, head_dim) = (self.n_heads, self.head_dim());
let split = |t: Tensor<3>| t.reshape([batch, time, heads, head_dim]);
let q = rope
.apply(split(self.query.forward(x.clone())))
.swap_dims(1, 2);
let k = rope
.apply(split(self.key.forward(x.clone())))
.swap_dims(1, 2);
let v = split(self.value.forward(x)).swap_dims(1, 2);
let mut scores = q
.matmul(k.swap_dims(2, 3))
.div_scalar((head_dim as f64).sqrt());
if let Some(valid) = valid {
let padding = valid.bool_not().reshape([batch, 1, 1, time]);
scores = scores.mask_fill(padding, f32::NEG_INFINITY);
}
let y =
softmax(scores, 3)
.matmul(v)
.swap_dims(1, 2)
.reshape([batch, time, heads * head_dim]);
let y = self.output.forward(y);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[
("batch", batch),
("time", time),
("d_model", self.d_model())
],
);
y
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
Distribution,
Tolerance,
};
use super::*;
use crate::{
blocks::transformers::embedding::{
RotaryDirection,
RotaryEmbeddingConfig,
},
ops::transformers::attention::length_mask,
support::testing::{
cpu_device,
seeded_tensor,
},
};
fn rope(
time: usize,
head_dim: usize,
) -> RotaryEmbedding {
RotaryEmbeddingConfig::new(time, head_dim)
.with_direction(RotaryDirection::Forward)
.init(&cpu_device())
}
#[test]
fn test_meta() {
let config = RopeSelfAttentionConfig::new(24, 3)
.with_qkv_bias(true)
.with_output_bias(false);
let module: RopeSelfAttention = config.init(&cpu_device());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_heads(), config.n_heads());
assert_eq!(module.head_dim(), 8);
assert!(module.query.bias.is_some());
assert!(module.output.bias.is_none());
}
#[test]
fn test_refuses_odd_heads() {
let device = cpu_device();
for config in [
RopeSelfAttentionConfig::new(24, 0),
RopeSelfAttentionConfig::new(24, 5),
RopeSelfAttentionConfig::new(12, 4),
] {
assert!(ModuleInit::<RopeSelfAttention>::try_init(&config, &device).is_err());
}
}
#[test]
fn test_matches_a_head_by_head_reference() {
let device = cpu_device();
let module: RopeSelfAttention = RopeSelfAttentionConfig::new(16, 2)
.with_qkv_bias(true)
.init(&device);
let x = seeded_tensor::<3>(3, [1, 5, 16], Distribution::Default, &device);
let rope = rope(5, 8);
let got = module.forward(x.clone(), &rope, None);
let mut heads = Vec::new();
for h in 0..2 {
let head = |proj: &Linear| {
proj.forward(x.clone())
.slice_dim(2, h * 8..(h + 1) * 8)
.reshape([1, 5, 1, 8])
};
let q = rope.apply(head(&module.query)).reshape([5, 8]);
let k = rope.apply(head(&module.key)).reshape([5, 8]);
let v = head(&module.value).reshape([5, 8]);
let w = softmax(q.matmul(k.transpose()).div_scalar(8f64.sqrt()), 1);
heads.push(w.matmul(v));
}
let want = module
.output
.forward(Tensor::cat(heads, 1).reshape([1, 5, 16]));
got.into_data()
.assert_approx_eq::<f32>(&want.into_data(), Tolerance::absolute(1e-5));
}
#[test]
fn test_is_bidirectional() {
let device = cpu_device();
let module: RopeSelfAttention = RopeSelfAttentionConfig::new(8, 2).init(&device);
let x = seeded_tensor::<3>(5, [1, 4, 8], Distribution::Default, &device);
let changed = x.clone().slice_assign(
[0..1, 3..4, 0..8],
seeded_tensor::<3>(6, [1, 1, 8], Distribution::Default, &device),
);
let a = module.forward(x, &rope(4, 4), None).slice_dim(1, 0..1);
let b = module
.forward(changed, &rope(4, 4), None)
.slice_dim(1, 0..1);
assert!((a - b).abs().max().into_scalar::<f32>() > 1e-4);
}
#[test]
fn test_padding_is_ignored() {
let device = cpu_device();
let module: RopeSelfAttention = RopeSelfAttentionConfig::new(8, 2).init(&device);
let x = seeded_tensor::<3>(9, [1, 6, 8], Distribution::Default, &device);
let junk = seeded_tensor::<3>(10, [1, 3, 8], Distribution::Default, &device);
let padded = Tensor::cat(vec![x.clone(), junk], 1);
let alone = module.forward(x, &rope(6, 4), None);
let valid = length_mask(&[6], 9, &device);
let batched = module
.forward(padded, &rope(9, 4), Some(valid))
.slice_dim(1, 0..6);
batched
.into_data()
.assert_approx_eq::<f32>(&alone.into_data(), Tolerance::absolute(1e-5));
}
}