Skip to main content

coeus_autograd/ops/nn/
attention.rs

1// ── Autograd node: Scaled Dot-Product Attention ──
2//
3// Monomorphized struct-based BackwardNode following the Conv/Pool pattern.
4// `M: AttentionMask` is a ZST; the causal branch is selected at compile time
5// by DCE on `M::IS_CAUSAL`.
6
7use crate::grad_buffer::GradBuffer;
8use crate::node::BackwardNode;
9use crate::var::Var;
10use coeus_core::Scalar;
11use coeus_tensor::Tensor;
12use std::sync::Arc;
13
14/// ZST marker trait for attention mask strategies.
15///
16/// Implementations carry only a compile-time `IS_CAUSAL` constant.
17/// DCE eliminates the inactive branch in `forward`; no runtime branching occurs.
18pub trait AttentionMask: 'static + Copy + Clone + Send + Sync {
19    /// `true` iff the mask is a lower-triangular causal mask.
20    const IS_CAUSAL: bool;
21}
22
23/// Causal (lower-triangular) attention mask — positions j > i are masked to −∞.
24#[derive(Debug, Clone, Copy, Default)]
25pub struct CausalMask;
26impl AttentionMask for CausalMask {
27    const IS_CAUSAL: bool = true;
28}
29
30/// No masking (full attention).
31#[derive(Debug, Clone, Copy, Default)]
32pub struct NullMask;
33impl AttentionMask for NullMask {
34    const IS_CAUSAL: bool = false;
35}
36
37// ── Backward Node ──────────────────────────────────────────────────────────
38
39/// Autograd node for scaled dot-product attention.
40///
41/// Stores the forward inputs and the post-softmax attention weights needed for
42/// computing gradients w.r.t. Q, K, and V.
43pub struct ScaledDotProductAttnNode<
44    T: Scalar,
45    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
46    M: AttentionMask,
47> {
48    /// Accumulated gradient buffer for the output of this node.
49    pub output_grad: Arc<GradBuffer<T, B>>,
50    /// [Q_var, K_var, V_var]
51    pub inputs: Vec<Var<T, B>>,
52    /// Saved query tensor for backward.
53    pub q_clone: Tensor<T, B>,
54    /// Saved key tensor for backward.
55    pub k_clone: Tensor<T, B>,
56    /// Saved value tensor for backward.
57    pub v_clone: Tensor<T, B>,
58    /// Post-softmax attention weight matrix `[batch, seq_q, seq_k]`.
59    pub attn_weights: Tensor<T, B>,
60    /// Scaling factor `1/sqrt(head_dim)`.
61    pub scale: T,
62    /// Zero-sized phantom to bind the mask type parameter.
63    pub _mask: std::marker::PhantomData<M>,
64}
65
66impl<
67        T: coeus_ops::AttentionScalar,
68        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
69        M: AttentionMask,
70    > BackwardNode<T, B> for ScaledDotProductAttnNode<T, B, M>
71{
72    #[inline]
73    fn op_name(&self) -> &'static str {
74        "sdp_attention"
75    }
76
77    #[inline]
78    fn output_grad(&self) -> &Arc<GradBuffer<T, B>> {
79        &self.output_grad
80    }
81
82    #[inline]
83    fn inputs(&self) -> &[Var<T, B>] {
84        &self.inputs
85    }
86
87    fn backward(
88        &self,
89        grad_out: &Tensor<T, B>,
90        input_grads: &[Option<Arc<GradBuffer<T, B>>>],
91    ) -> Result<(), B::Error> {
92        let backend = B::default();
93
94        let need_gq = input_grads.first().and_then(|g| g.as_ref()).is_some();
95        let need_gk = input_grads.get(1).and_then(|g| g.as_ref()).is_some();
96        let need_gv = input_grads.get(2).and_then(|g| g.as_ref()).is_some();
97
98        if !need_gq && !need_gk && !need_gv {
99            return Ok(());
100        }
101
102        let mut grad_q = if need_gq {
103            Some(Tensor::zeros_on(self.q_clone.shape_cloned(), &backend)?)
104        } else {
105            None
106        };
107        let mut grad_k = if need_gk {
108            Some(Tensor::zeros_on(self.k_clone.shape_cloned(), &backend)?)
109        } else {
110            None
111        };
112        let mut grad_v = if need_gv {
113            Some(Tensor::zeros_on(self.v_clone.shape_cloned(), &backend)?)
114        } else {
115            None
116        };
117
118        // All six borrow paths handled in coeus_ops::scaled_dot_product_attention_backward.
119        coeus_ops::scaled_dot_product_attention_backward(
120            grad_out,
121            &self.q_clone,
122            &self.k_clone,
123            &self.v_clone,
124            &self.attn_weights,
125            self.scale,
126            grad_q.as_mut(),
127            grad_k.as_mut(),
128            grad_v.as_mut(),
129            &backend,
130        )?;
131
132        if let (Some(acc), Some(gq)) = (input_grads.first().and_then(|g| g.as_ref()), grad_q) {
133            let lock = acc.write();
134            coeus_ops::add_assign(lock, &gq, &backend)?;
135        }
136        if let (Some(acc), Some(gk)) = (input_grads.get(1).and_then(|g| g.as_ref()), grad_k) {
137            let lock = acc.write();
138            coeus_ops::add_assign(lock, &gk, &backend)?;
139        }
140        if let (Some(acc), Some(gv)) = (input_grads.get(2).and_then(|g| g.as_ref()), grad_v) {
141            let lock = acc.write();
142            coeus_ops::add_assign(lock, &gv, &backend)?;
143        }
144
145        Ok(())
146    }
147}
148
149// ── Public tracked function ────────────────────────────────────────────────
150
151/// The attention output and the weights that produced it, the latter kept for
152/// the backward pass rather than recomputed.
153type AttentionOutput<T, B> = (Var<T, B>, Tensor<T, B>);
154
155/// Tracked scaled dot-product attention.
156///
157/// `M::IS_CAUSAL` selects the masking strategy; dead code is eliminated at
158/// monomorphization time — no runtime branch.
159///
160/// Returns `(attn_output, attn_weights)`. `attn_weights` is detached (no
161/// further gradient tracking needed for the basic use-case; MHA uses it
162/// as an intermediate).
163///
164/// # Errors
165///
166/// Returns the backend's typed attention validation or dispatch failure.
167pub fn sdp_attention<
168    T: coeus_ops::AttentionScalar,
169    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
170    M: AttentionMask,
171>(
172    query: &Var<T, B>,
173    key: &Var<T, B>,
174    value: &Var<T, B>,
175    key_padding_mask: Option<&Var<T, B>>,
176    scale: T,
177) -> Result<AttentionOutput<T, B>, B::Error> {
178    let backend = B::default();
179
180    let (out_tensor, attn_weights) = coeus_ops::scaled_dot_product_attention(
181        &query.tensor,
182        &key.tensor,
183        &value.tensor,
184        key_padding_mask.map(|m| &m.tensor),
185        M::IS_CAUSAL,
186        scale,
187        &backend,
188    )?;
189
190    let requires_grad = crate::grad_mode::should_track_var(query)
191        || crate::grad_mode::should_track_var(key)
192        || crate::grad_mode::should_track_var(value);
193
194    if !requires_grad {
195        return Ok((Var::new(out_tensor, false)?, attn_weights));
196    }
197
198    let out_var = Var::from_tracked_op(out_tensor, requires_grad, &backend, |output_grad| {
199        ScaledDotProductAttnNode::<T, B, M> {
200            output_grad,
201            inputs: vec![query.clone(), key.clone(), value.clone()],
202            q_clone: query.tensor.clone(),
203            k_clone: key.tensor.clone(),
204            v_clone: value.tensor.clone(),
205            attn_weights: attn_weights.clone(),
206            scale,
207            _mask: std::marker::PhantomData,
208        }
209    })?;
210    Ok((out_var, attn_weights))
211}