Skip to main content

coeus_nn/attention/
sdp.rs

1// ── Scaled Dot-Product Attention module ──
2
3use crate::module::{Module, ModuleError};
4use coeus_autograd::{AttentionMask, Var};
5use coeus_core::MoiraiBackend;
6use std::marker::PhantomData;
7
8/// Scaled dot-product attention layer.
9///
10/// `M` is a zero-sized mask strategy type sourced from `coeus_autograd`.
11/// The active masking path is selected at compile time via `M::IS_CAUSAL`.
12///
13/// # Forward shape
14/// - query: `[batch, seq_q, d_k]`, key: `[batch, seq_k, d_k]`, value: `[batch, seq_k, d_v]`
15/// - output: `[batch, seq_q, d_v]`
16#[derive(Clone, Copy)]
17pub struct ScaledDotProductAttention<
18    T: coeus_core::Scalar,
19    B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
20    M: AttentionMask = coeus_autograd::NullMask,
21> {
22    _marker: PhantomData<(T, B, M)>,
23}
24
25impl<T: coeus_core::Scalar, B: coeus_ops::BackendOps<T> + Default, M: AttentionMask> Default
26    for ScaledDotProductAttention<T, B, M>
27{
28    fn default() -> Self {
29        Self {
30            _marker: PhantomData,
31        }
32    }
33}
34
35impl<
36        T: coeus_ops::AttentionScalar + coeus_core::FloatElement,
37        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
38        M: AttentionMask,
39    > ScaledDotProductAttention<T, B, M>
40{
41    /// Create a new `ScaledDotProductAttention` layer (stateless, no parameters).
42    pub fn new() -> Self {
43        Self::default()
44    }
45
46    /// Forward pass with explicit Q, K, V.
47    pub fn forward(
48        &self,
49        query: &Var<T, B>,
50        key: &Var<T, B>,
51        value: &Var<T, B>,
52        key_padding_mask: Option<&Var<T, B>>,
53        scale: Option<T>,
54    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
55        let dimensions = super::validation::sdp_dimensions(
56            "ScaledDotProductAttention",
57            query,
58            key,
59            value,
60            key_padding_mask,
61            1,
62        )?;
63        let scale = scale.unwrap_or_else(|| {
64            // The scale is `1/sqrt(d_k)`: the count is exact at f64 precision and
65            // the root is rounded once into the element type, so a narrow format
66            // never squares a count it could not represent.
67            let d_k = <f64 as coeus_core::FloatElement>::from_count(dimensions.d_k);
68            T::ONE / <T as coeus_core::FloatElement>::from_f64(d_k.sqrt())
69        });
70        let (out, _attn_weights) =
71            coeus_autograd::sdp_attention::<T, B, M>(query, key, value, key_padding_mask, scale)
72                .map_err(|source| ModuleError::Backend {
73                    module: "ScaledDotProductAttention",
74                    source,
75                })?;
76        Ok(out)
77    }
78}
79
80/// Convenience `Module` impl — self-attention: Q = K = V = input.
81impl<
82        T: coeus_ops::AttentionScalar + coeus_core::FloatElement,
83        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
84        M: AttentionMask,
85    > Module<T, B> for ScaledDotProductAttention<T, B, M>
86{
87    fn parameters(&self) -> Vec<Var<T, B>> {
88        vec![]
89    }
90
91    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
92        self.forward(input, input, input, None, None)
93    }
94}