use crate::module::{Module, ModuleError};
use coeus_autograd::{AttentionMask, Var};
use coeus_core::MoiraiBackend;
use std::marker::PhantomData;
#[derive(Clone, Copy)]
pub struct ScaledDotProductAttention<
T: coeus_core::Scalar,
B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
M: AttentionMask = coeus_autograd::NullMask,
> {
_marker: PhantomData<(T, B, M)>,
}
impl<T: coeus_core::Scalar, B: coeus_ops::BackendOps<T> + Default, M: AttentionMask> Default
for ScaledDotProductAttention<T, B, M>
{
fn default() -> Self {
Self {
_marker: PhantomData,
}
}
}
impl<
T: coeus_ops::AttentionScalar + coeus_core::FloatElement,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
M: AttentionMask,
> ScaledDotProductAttention<T, B, M>
{
pub fn new() -> Self {
Self::default()
}
pub fn forward(
&self,
query: &Var<T, B>,
key: &Var<T, B>,
value: &Var<T, B>,
key_padding_mask: Option<&Var<T, B>>,
scale: Option<T>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let dimensions = super::validation::sdp_dimensions(
"ScaledDotProductAttention",
query,
key,
value,
key_padding_mask,
1,
)?;
let scale = scale.unwrap_or_else(|| {
let d_k = <f64 as coeus_core::FloatElement>::from_count(dimensions.d_k);
T::ONE / <T as coeus_core::FloatElement>::from_f64(d_k.sqrt())
});
let (out, _attn_weights) =
coeus_autograd::sdp_attention::<T, B, M>(query, key, value, key_padding_mask, scale)
.map_err(|source| ModuleError::Backend {
module: "ScaledDotProductAttention",
source,
})?;
Ok(out)
}
}
impl<
T: coeus_ops::AttentionScalar + coeus_core::FloatElement,
B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
M: AttentionMask,
> Module<T, B> for ScaledDotProductAttention<T, B, M>
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
self.forward(input, input, input, None, None)
}
}