coeus_nn/attention/
sdp.rs1use crate::module::{Module, ModuleError};
4use coeus_autograd::{AttentionMask, Var};
5use coeus_core::MoiraiBackend;
6use std::marker::PhantomData;
7
8#[derive(Clone, Copy)]
17pub struct ScaledDotProductAttention<
18 T: coeus_core::Scalar,
19 B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
20 M: AttentionMask = coeus_autograd::NullMask,
21> {
22 _marker: PhantomData<(T, B, M)>,
23}
24
25impl<T: coeus_core::Scalar, B: coeus_ops::BackendOps<T> + Default, M: AttentionMask> Default
26 for ScaledDotProductAttention<T, B, M>
27{
28 fn default() -> Self {
29 Self {
30 _marker: PhantomData,
31 }
32 }
33}
34
35impl<
36 T: coeus_ops::AttentionScalar + coeus_core::FloatElement,
37 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
38 M: AttentionMask,
39 > ScaledDotProductAttention<T, B, M>
40{
41 pub fn new() -> Self {
43 Self::default()
44 }
45
46 pub fn forward(
48 &self,
49 query: &Var<T, B>,
50 key: &Var<T, B>,
51 value: &Var<T, B>,
52 key_padding_mask: Option<&Var<T, B>>,
53 scale: Option<T>,
54 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
55 let dimensions = super::validation::sdp_dimensions(
56 "ScaledDotProductAttention",
57 query,
58 key,
59 value,
60 key_padding_mask,
61 1,
62 )?;
63 let scale = scale.unwrap_or_else(|| {
64 let d_k = <f64 as coeus_core::FloatElement>::from_count(dimensions.d_k);
68 T::ONE / <T as coeus_core::FloatElement>::from_f64(d_k.sqrt())
69 });
70 let (out, _attn_weights) =
71 coeus_autograd::sdp_attention::<T, B, M>(query, key, value, key_padding_mask, scale)
72 .map_err(|source| ModuleError::Backend {
73 module: "ScaledDotProductAttention",
74 source,
75 })?;
76 Ok(out)
77 }
78}
79
80impl<
82 T: coeus_ops::AttentionScalar + coeus_core::FloatElement,
83 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
84 M: AttentionMask,
85 > Module<T, B> for ScaledDotProductAttention<T, B, M>
86{
87 fn parameters(&self) -> Vec<Var<T, B>> {
88 vec![]
89 }
90
91 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
92 self.forward(input, input, input, None, None)
93 }
94}