use burn::{
Tensor,
config::Config,
module::{
Module,
Param,
},
nn::{
Linear,
LinearConfig,
},
prelude::Bool,
tensor::{
Device,
activation::softmax,
},
};
#[cfg(not(feature = "std"))]
#[allow(unused_imports)]
use num_traits::Float;
use crate::{
blocks::transformers::sublayer::{
ResidualSublayer,
SublayerBranch,
},
burner::module::ModuleInit,
errors::{
BunsenResult,
ConstraintError,
Rule,
},
ops::transformers::attention::rel_shift,
};
pub trait RelPosSelfAttentionMeta {
fn d_model(&self) -> usize;
fn n_heads(&self) -> usize;
fn d_head(&self) -> usize {
self.d_model() / self.n_heads()
}
}
#[derive(Config, Debug)]
pub struct RelPosSelfAttentionConfig {
pub d_model: usize,
pub n_heads: usize,
#[config(default = "true")]
pub bias: bool,
}
impl RelPosSelfAttentionMeta for RelPosSelfAttentionConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn n_heads(&self) -> usize {
self.n_heads
}
}
impl ModuleInit<RelPosSelfAttention> for RelPosSelfAttentionConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<RelPosSelfAttention> {
if self.n_heads == 0 {
return Err(ConstraintError::new(
"RelPosSelfAttentionConfig",
"n_heads",
Rule::ZeroOrEmpty,
)
.into());
}
if !self.d_model.is_multiple_of(self.n_heads) {
return Err(ConstraintError::new(
"RelPosSelfAttentionConfig",
"d_model",
Rule::NotMultiple {
value: self.d_model,
of: format!("n_heads ({})", self.n_heads),
},
)
.into());
}
let d = self.d_model;
let linear = |bias| LinearConfig::new(d, d).with_bias(bias).init(device);
Ok(RelPosSelfAttention {
query: linear(self.bias),
key: linear(self.bias),
value: linear(self.bias),
output: linear(self.bias),
pos: linear(false),
pos_bias_u: Param::from_tensor(Tensor::zeros([self.n_heads, self.d_head()], device)),
pos_bias_v: Param::from_tensor(Tensor::zeros([self.n_heads, self.d_head()], device)),
n_heads: self.n_heads,
})
}
}
#[derive(Module, Debug)]
pub struct RelPosSelfAttention {
pub query: Linear,
pub key: Linear,
pub value: Linear,
pub output: Linear,
pub pos: Linear,
pub pos_bias_u: Param<Tensor<2>>,
pub pos_bias_v: Param<Tensor<2>>,
pub n_heads: usize,
}
impl RelPosSelfAttentionMeta for RelPosSelfAttention {
fn d_model(&self) -> usize {
self.query.weight.dims()[0]
}
fn n_heads(&self) -> usize {
self.n_heads
}
}
impl RelPosSelfAttention {
pub fn forward(
&self,
x: Tensor<3>,
positions: Tensor<3>,
valid: Option<Tensor<2, Bool>>,
) -> Tensor<3> {
let [batch, time] = crate::contracts::unpack_shape_contract!(
["batch", "time", "d_model"],
&x.dims(),
&["batch", "time"],
&[("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
[1, 2 * "time" - 1, "d_model"],
&positions.dims(),
&[("time", time), ("d_model", self.d_model())],
);
#[cfg(any(debug_assertions, test))]
if let Some(valid) = &valid {
crate::contracts::assert_shape_contract_periodically!(
["batch", "time"],
&valid.dims(),
&[("batch", batch), ("time", time)],
);
}
let d_model = self.d_model();
let (h, dk) = (self.n_heads, self.d_head());
let heads = |t: Tensor<3>| {
t.reshape([batch as i32, -1, h as i32, dk as i32])
.swap_dims(1, 2)
};
let q = heads(self.query.forward(x.clone()));
let k = heads(self.key.forward(x.clone()));
let v = heads(self.value.forward(x));
let p = self
.pos
.forward(positions)
.reshape([1, 2 * time - 1, h, dk])
.swap_dims(1, 2);
let bias = |b: &Param<Tensor<2>>| b.val().reshape([1, h, 1, dk]);
let q_u = q.clone() + bias(&self.pos_bias_u);
let q_v = q + bias(&self.pos_bias_v);
let content = q_u.matmul(k.swap_dims(2, 3));
let position = rel_shift(q_v.matmul(p.swap_dims(2, 3)));
let mut scores = (content + position).mul_scalar(1.0 / (dk as f64).sqrt());
if let Some(valid) = valid {
let masked = valid.bool_not().reshape([batch, 1, 1, time]);
scores = scores.mask_fill(masked, f32::NEG_INFINITY);
}
let context = softmax(scores, 3)
.matmul(v)
.swap_dims(1, 2)
.reshape([batch, time, d_model]);
let y = self.output.forward(context);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "time", "d_model"],
&y.dims(),
&[("batch", batch), ("time", time), ("d_model", d_model)],
);
y
}
}
#[derive(Debug, Clone)]
pub struct RelPosAttnCall {
pub positions: Tensor<3>,
pub valid: Option<Tensor<2, Bool>>,
}
impl SublayerBranch<RelPosAttnCall, 3> for RelPosSelfAttention {
type Output = Tensor<3>;
fn call_branch(
&self,
z: Tensor<3>,
ctx: RelPosAttnCall,
) -> Tensor<3> {
self.forward(z, ctx.positions, ctx.valid)
}
}
pub trait RelPosAttnSublayer {
fn rel_pos_attn(
&self,
x: Tensor<3>,
positions: Tensor<3>,
valid: Option<Tensor<2, Bool>>,
) -> Tensor<3>;
}
impl<M> RelPosAttnSublayer for ResidualSublayer<M>
where
M: Module + SublayerBranch<RelPosAttnCall, 3, Output = Tensor<3>>,
{
fn rel_pos_attn(
&self,
x: Tensor<3>,
positions: Tensor<3>,
valid: Option<Tensor<2, Bool>>,
) -> Tensor<3> {
self.residual_update(x, RelPosAttnCall { positions, valid })
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
ops::transformers::attention::rel_position_table,
support::testing::cpu_device,
};
#[test]
fn test_meta() {
let config = RelPosSelfAttentionConfig::new(12, 3).with_bias(false);
let module: RelPosSelfAttention = config.init(&cpu_device());
assert_eq!(module.d_model(), config.d_model());
assert_eq!(module.n_heads(), config.n_heads());
assert_eq!(module.d_head(), 4);
assert!(module.query.bias.is_none());
assert!(module.pos.bias.is_none());
let device = cpu_device();
let y = module.forward(
Tensor::zeros([2, 5, 12], &device),
rel_position_table(5, 12, &device),
None,
);
assert_eq!(y.dims(), [2, 5, 12]);
}
#[test]
fn test_heads_must_divide_width() {
let config = RelPosSelfAttentionConfig::new(10, 3);
assert!(ModuleInit::<RelPosSelfAttention>::try_init(&config, &cpu_device()).is_err());
}
}