use crate::init::kaiming_uniform;
use crate::module::{Module, ModuleError};
use coeus_autograd::{AttentionMask, Var};
use coeus_core::{Float, MoiraiBackend, Scalar};
use std::marker::PhantomData;
#[derive(Clone)]
pub struct MultiHeadAttention<
T: Scalar,
B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
const H: usize = 8,
M: AttentionMask = coeus_autograd::NullMask,
> {
pub w_q: Var<T, B>,
pub b_q: Option<Var<T, B>>,
pub w_k: Var<T, B>,
pub b_k: Option<Var<T, B>>,
pub w_v: Var<T, B>,
pub b_v: Option<Var<T, B>>,
pub w_o: Var<T, B>,
pub b_o: Option<Var<T, B>>,
pub d_model: usize,
_mask: PhantomData<M>,
}
pub struct MhaProjectionParams<'a, T: Scalar, B: coeus_ops::BackendOps<T> + Default> {
pub w_q: &'a Var<T, B>,
pub b_q: Option<&'a Var<T, B>>,
pub w_k: &'a Var<T, B>,
pub b_k: Option<&'a Var<T, B>>,
pub w_v: &'a Var<T, B>,
pub b_v: Option<&'a Var<T, B>>,
pub w_o: &'a Var<T, B>,
pub b_o: Option<&'a Var<T, B>>,
}
pub fn multi_head_attention_cross<
T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
>(
query: &Var<T, B>,
key: &Var<T, B>,
value: &Var<T, B>,
params: MhaProjectionParams<'_, T, B>,
key_padding_mask: Option<&Var<T, B>>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let wq_shape = params.w_q.tensor.shape_cloned();
if wq_shape.len() != 2 || wq_shape[0] != wq_shape[1] || wq_shape[0] == 0 {
return Err(super::validation::shape_mismatch(
"MultiHeadAttention",
"query projection weight",
&[1, 1],
&wq_shape,
));
}
let d_model = wq_shape[0];
if H == 0 || !d_model.is_multiple_of(H) {
return Err(super::validation::shape_mismatch(
"MultiHeadAttention",
"attention heads",
&[d_model],
&[H],
));
}
let dimensions = super::validation::sdp_dimensions(
"MultiHeadAttention",
query,
key,
value,
key_padding_mask,
H,
)?;
for (parameter, actual) in [
("query feature", dimensions.d_k),
("key feature", key.tensor.shape()[2]),
("value feature", dimensions.d_v),
] {
if actual != d_model {
return Err(super::validation::shape_mismatch(
"MultiHeadAttention",
parameter,
&[d_model],
&[actual],
));
}
}
for (parameter, weight, bias) in [
("query projection", params.w_q, params.b_q),
("key projection", params.w_k, params.b_k),
("value projection", params.w_v, params.b_v),
("output projection", params.w_o, params.b_o),
] {
super::validation::projection("MultiHeadAttention", parameter, weight, bias, d_model)?;
}
let d_head = d_model / H;
let d_head_f64 = <f64 as coeus_core::FloatElement>::from_count(d_head);
let scale = T::ONE / <T as coeus_core::FloatElement>::from_f64(d_head_f64.sqrt());
let batch = dimensions.batch;
let seq_q = dimensions.seq_q;
let seq_k = dimensions.seq_k;
let q_proj = project_3d(query, params.w_q, params.b_q, batch, seq_q, d_model)?;
let k_proj = project_3d(key, params.w_k, params.b_k, batch, seq_k, d_model)?;
let v_proj = project_3d(value, params.w_v, params.b_v, batch, seq_k, d_model)?;
let q_split = coeus_autograd::reshape(&q_proj, [batch, seq_q, H, d_head])?;
let k_split = coeus_autograd::reshape(&k_proj, [batch, seq_k, H, d_head])?;
let v_split = coeus_autograd::reshape(&v_proj, [batch, seq_k, H, d_head])?;
let q_perm = coeus_autograd::permute(&q_split, &[0, 2, 1, 3])?;
let k_perm = coeus_autograd::permute(&k_split, &[0, 2, 1, 3])?;
let v_perm = coeus_autograd::permute(&v_split, &[0, 2, 1, 3])?;
let q_heads = coeus_autograd::reshape(&q_perm, [batch * H, seq_q, d_head])?;
let k_heads = coeus_autograd::reshape(&k_perm, [batch * H, seq_k, d_head])?;
let v_heads = coeus_autograd::reshape(&v_perm, [batch * H, seq_k, d_head])?;
let (attn_out, _aw) = coeus_autograd::sdp_attention::<T, B, M>(
&q_heads,
&k_heads,
&v_heads,
key_padding_mask,
scale,
)
.map_err(|source| ModuleError::Backend {
module: "MultiHeadAttention",
source,
})?;
let merged_split = coeus_autograd::reshape(&attn_out, [batch, H, seq_q, d_head])?;
let merged_perm = coeus_autograd::permute(&merged_split, &[0, 2, 1, 3])?;
let merged = coeus_autograd::reshape(&merged_perm, [batch, seq_q, d_model])?;
Ok(project_3d(
&merged, params.w_o, params.b_o, batch, seq_q, d_model,
)?)
}
fn project_3d<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
x: &Var<T, B>,
w: &Var<T, B>,
bias: Option<&Var<T, B>>,
batch: usize,
seq: usize,
d_model: usize,
) -> Result<Var<T, B>, B::Error> {
let flat = coeus_autograd::reshape(x, [batch * seq, d_model])?;
let w_t = coeus_autograd::transpose_2d(w)?;
let out_flat = coeus_autograd::matmul(&flat, &w_t)?;
let out_flat = if let Some(b) = bias {
coeus_autograd::add(&out_flat, b)?
} else {
out_flat
};
coeus_autograd::reshape(&out_flat, [batch, seq, d_model])
}
impl<
T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
> MultiHeadAttention<T, B, H, M>
{
pub fn new(
d_model: usize,
bias: bool,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
T: coeus_cpu::RealScalar,
B: coeus_ops::RandomInitOps<T>,
{
if H == 0 || d_model == 0 || !d_model.is_multiple_of(H) {
return Err(crate::init::InitializationError::InvalidHeadConfiguration {
d_model,
heads: H,
});
}
let make_weight = || -> Result<Var<T, B>, crate::init::InitializationError<B::Error>> {
let mut v = Var::new(
coeus_tensor::Tensor::zeros_on([d_model, d_model], &B::default())?,
true,
)?;
kaiming_uniform(&mut v, d_model)?;
Ok(v)
};
let make_bias = || -> Result<Var<T, B>, B::Error> {
Var::new(
coeus_tensor::Tensor::zeros_on([d_model], &B::default())?,
true,
)
};
Ok(Self {
w_q: make_weight()?,
b_q: if bias { Some(make_bias()?) } else { None },
w_k: make_weight()?,
b_k: if bias { Some(make_bias()?) } else { None },
w_v: make_weight()?,
b_v: if bias { Some(make_bias()?) } else { None },
w_o: make_weight()?,
b_o: if bias { Some(make_bias()?) } else { None },
d_model,
_mask: PhantomData,
})
}
pub fn forward_cross(
&self,
query: &Var<T, B>,
key: &Var<T, B>,
value: &Var<T, B>,
key_padding_mask: Option<&Var<T, B>>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
multi_head_attention_cross::<T, B, H, M>(
query,
key,
value,
MhaProjectionParams {
w_q: &self.w_q,
b_q: self.b_q.as_ref(),
w_k: &self.w_k,
b_k: self.b_k.as_ref(),
w_v: &self.w_v,
b_v: self.b_v.as_ref(),
w_o: &self.w_o,
b_o: self.b_o.as_ref(),
},
key_padding_mask,
)
}
}
impl<
T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
const H: usize,
M: AttentionMask,
> Module<T, B> for MultiHeadAttention<T, B, H, M>
{
fn parameters(&self) -> Vec<Var<T, B>> {
let mut p = vec![
self.w_q.clone(),
self.w_k.clone(),
self.w_v.clone(),
self.w_o.clone(),
];
if let Some(ref b) = self.b_q {
p.push(b.clone());
}
if let Some(ref b) = self.b_k {
p.push(b.clone());
}
if let Some(ref b) = self.b_v {
p.push(b.clone());
}
if let Some(ref b) = self.b_o {
p.push(b.clone());
}
p
}
fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
let names = [
"query.weight",
"key.weight",
"value.weight",
"output.weight",
"query.bias",
"key.bias",
"value.bias",
"output.bias",
];
self.parameters()
.into_iter()
.zip(names)
.map(|(parameter, name)| coeus_autograd::Parameter::new(parameter, name))
.collect()
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.forward_cross(input, input, input, None)
}
}