Skip to main content

coeus_nn/attention/
mha.rs

1// ── Multi-Head Attention ──
2//
3// Uses separate W_Q, W_K, W_V, W_O weight matrices (no fused QKV) so that
4// projection is a generic tracked matmul with no CpuAddressableStorageMut requirement.
5// Head split/merge use coeus_autograd::reshape (tracked) to preserve gradient flow.
6
7use crate::init::kaiming_uniform;
8use crate::module::{Module, ModuleError};
9use coeus_autograd::{AttentionMask, Var};
10use coeus_core::{Float, MoiraiBackend, Scalar};
11use std::marker::PhantomData;
12
13/// Multi-head self/cross-attention.
14///
15/// # Type parameters
16/// - `H` — number of attention heads (const generic); `d_model % H == 0` is asserted at construction.
17/// - `M` — masking strategy ZST from `coeus_autograd` (`NullMask` or `CausalMask`)
18///
19/// Uses four separate projection weights W_Q, W_K, W_V, W_O, each shape `[d_model, d_model]`,
20/// projected via tracked matmul. Head reshape uses `coeus_autograd::reshape` for full gradient flow.
21#[derive(Clone)]
22pub struct MultiHeadAttention<
23    T: Scalar,
24    B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
25    const H: usize = 8,
26    M: AttentionMask = coeus_autograd::NullMask,
27> {
28    /// Query projection weight: `[d_model, d_model]`.
29    pub w_q: Var<T, B>,
30    /// Query projection bias: `[d_model]`.
31    pub b_q: Option<Var<T, B>>,
32    /// Key projection weight: `[d_model, d_model]`.
33    pub w_k: Var<T, B>,
34    /// Key projection bias: `[d_model]`.
35    pub b_k: Option<Var<T, B>>,
36    /// Value projection weight: `[d_model, d_model]`.
37    pub w_v: Var<T, B>,
38    /// Value projection bias: `[d_model]`.
39    pub b_v: Option<Var<T, B>>,
40    /// Output projection weight: `[d_model, d_model]`.
41    pub w_o: Var<T, B>,
42    /// Output projection bias: `[d_model]`.
43    pub b_o: Option<Var<T, B>>,
44    /// Model embedding dimension (must be divisible by `H`).
45    pub d_model: usize,
46    _mask: PhantomData<M>,
47}
48
49/// Borrowed projection parameters for functional multi-head attention.
50pub struct MhaProjectionParams<'a, T: Scalar, B: coeus_ops::BackendOps<T> + Default> {
51    /// Query projection weight `[d_model, d_model]`.
52    pub w_q: &'a Var<T, B>,
53    /// Optional query projection bias `[d_model]`.
54    pub b_q: Option<&'a Var<T, B>>,
55    /// Key projection weight `[d_model, d_model]`.
56    pub w_k: &'a Var<T, B>,
57    /// Optional key projection bias `[d_model]`.
58    pub b_k: Option<&'a Var<T, B>>,
59    /// Value projection weight `[d_model, d_model]`.
60    pub w_v: &'a Var<T, B>,
61    /// Optional value projection bias `[d_model]`.
62    pub b_v: Option<&'a Var<T, B>>,
63    /// Output projection weight `[d_model, d_model]`.
64    pub w_o: &'a Var<T, B>,
65    /// Optional output projection bias `[d_model]`.
66    pub b_o: Option<&'a Var<T, B>>,
67}
68
69/// Functional cross-attention helper shared by module and bindings.
70///
71/// Applies projections and scaled dot-product attention with explicit weights/biases
72/// and returns output shape `[batch, seq_q, d_model]`.
73///
74/// # Errors
75///
76/// Returns a typed contract failure before evaluating invalid Q/K/V,
77/// projection, head-count, bias, or mask shapes.
78pub fn multi_head_attention_cross<
79    T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
80    B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
81    const H: usize,
82    M: AttentionMask,
83>(
84    query: &Var<T, B>,
85    key: &Var<T, B>,
86    value: &Var<T, B>,
87    params: MhaProjectionParams<'_, T, B>,
88    key_padding_mask: Option<&Var<T, B>>,
89) -> Result<Var<T, B>, ModuleError<B::Error>> {
90    let wq_shape = params.w_q.tensor.shape_cloned();
91    if wq_shape.len() != 2 || wq_shape[0] != wq_shape[1] || wq_shape[0] == 0 {
92        return Err(super::validation::shape_mismatch(
93            "MultiHeadAttention",
94            "query projection weight",
95            &[1, 1],
96            &wq_shape,
97        ));
98    }
99    let d_model = wq_shape[0];
100    if H == 0 || !d_model.is_multiple_of(H) {
101        return Err(super::validation::shape_mismatch(
102            "MultiHeadAttention",
103            "attention heads",
104            &[d_model],
105            &[H],
106        ));
107    }
108
109    let dimensions = super::validation::sdp_dimensions(
110        "MultiHeadAttention",
111        query,
112        key,
113        value,
114        key_padding_mask,
115        H,
116    )?;
117    for (parameter, actual) in [
118        ("query feature", dimensions.d_k),
119        ("key feature", key.tensor.shape()[2]),
120        ("value feature", dimensions.d_v),
121    ] {
122        if actual != d_model {
123            return Err(super::validation::shape_mismatch(
124                "MultiHeadAttention",
125                parameter,
126                &[d_model],
127                &[actual],
128            ));
129        }
130    }
131    for (parameter, weight, bias) in [
132        ("query projection", params.w_q, params.b_q),
133        ("key projection", params.w_k, params.b_k),
134        ("value projection", params.w_v, params.b_v),
135        ("output projection", params.w_o, params.b_o),
136    ] {
137        super::validation::projection("MultiHeadAttention", parameter, weight, bias, d_model)?;
138    }
139
140    let d_head = d_model / H;
141    // The scale is `1/sqrt(d_head)`: the count is exact at f64 precision and the
142    // root is rounded once into the element type.
143    let d_head_f64 = <f64 as coeus_core::FloatElement>::from_count(d_head);
144    let scale = T::ONE / <T as coeus_core::FloatElement>::from_f64(d_head_f64.sqrt());
145
146    let batch = dimensions.batch;
147    let seq_q = dimensions.seq_q;
148    let seq_k = dimensions.seq_k;
149
150    let q_proj = project_3d(query, params.w_q, params.b_q, batch, seq_q, d_model)?;
151    let k_proj = project_3d(key, params.w_k, params.b_k, batch, seq_k, d_model)?;
152    let v_proj = project_3d(value, params.w_v, params.b_v, batch, seq_k, d_model)?;
153
154    let q_split = coeus_autograd::reshape(&q_proj, [batch, seq_q, H, d_head])?;
155    let k_split = coeus_autograd::reshape(&k_proj, [batch, seq_k, H, d_head])?;
156    let v_split = coeus_autograd::reshape(&v_proj, [batch, seq_k, H, d_head])?;
157
158    let q_perm = coeus_autograd::permute(&q_split, &[0, 2, 1, 3])?;
159    let k_perm = coeus_autograd::permute(&k_split, &[0, 2, 1, 3])?;
160    let v_perm = coeus_autograd::permute(&v_split, &[0, 2, 1, 3])?;
161
162    let q_heads = coeus_autograd::reshape(&q_perm, [batch * H, seq_q, d_head])?;
163    let k_heads = coeus_autograd::reshape(&k_perm, [batch * H, seq_k, d_head])?;
164    let v_heads = coeus_autograd::reshape(&v_perm, [batch * H, seq_k, d_head])?;
165
166    let (attn_out, _aw) = coeus_autograd::sdp_attention::<T, B, M>(
167        &q_heads,
168        &k_heads,
169        &v_heads,
170        key_padding_mask,
171        scale,
172    )
173    .map_err(|source| ModuleError::Backend {
174        module: "MultiHeadAttention",
175        source,
176    })?;
177
178    let merged_split = coeus_autograd::reshape(&attn_out, [batch, H, seq_q, d_head])?;
179    let merged_perm = coeus_autograd::permute(&merged_split, &[0, 2, 1, 3])?;
180    let merged = coeus_autograd::reshape(&merged_perm, [batch, seq_q, d_model])?;
181
182    Ok(project_3d(
183        &merged, params.w_o, params.b_o, batch, seq_q, d_model,
184    )?)
185}
186
187/// Project a 3D `[batch, seq, d_model]` var via a `[d_model, d_model]` weight:
188/// - Reshape to `[batch*seq, d_model]` (tracked)
189/// - Matmul with `w^T` (tracked)
190/// - Reshape back to `[batch, seq, d_model]` (tracked)
191fn project_3d<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
192    x: &Var<T, B>,
193    w: &Var<T, B>,
194    bias: Option<&Var<T, B>>,
195    batch: usize,
196    seq: usize,
197    d_model: usize,
198) -> Result<Var<T, B>, B::Error> {
199    let flat = coeus_autograd::reshape(x, [batch * seq, d_model])?;
200    let w_t = coeus_autograd::transpose_2d(w)?;
201    let out_flat = coeus_autograd::matmul(&flat, &w_t)?;
202    let out_flat = if let Some(b) = bias {
203        coeus_autograd::add(&out_flat, b)?
204    } else {
205        out_flat
206    };
207    coeus_autograd::reshape(&out_flat, [batch, seq, d_model])
208}
209
210impl<
211        T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
212        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
213        const H: usize,
214        M: AttentionMask,
215    > MultiHeadAttention<T, B, H, M>
216{
217    /// Construct a MHA layer with Kaiming-uniform initialized projection weights.
218    ///
219    /// # Errors
220    ///
221    /// Returns an initialization error when the model width cannot be divided
222    /// across `H` heads or the selected backend cannot initialize a weight.
223    pub fn new(
224        d_model: usize,
225        bias: bool,
226    ) -> Result<Self, crate::init::InitializationError<B::Error>>
227    where
228        T: coeus_cpu::RealScalar,
229        B: coeus_ops::RandomInitOps<T>,
230    {
231        if H == 0 || d_model == 0 || !d_model.is_multiple_of(H) {
232            return Err(crate::init::InitializationError::InvalidHeadConfiguration {
233                d_model,
234                heads: H,
235            });
236        }
237        let make_weight = || -> Result<Var<T, B>, crate::init::InitializationError<B::Error>> {
238            let mut v = Var::new(
239                coeus_tensor::Tensor::zeros_on([d_model, d_model], &B::default())?,
240                true,
241            )?;
242            kaiming_uniform(&mut v, d_model)?;
243            Ok(v)
244        };
245        let make_bias = || -> Result<Var<T, B>, B::Error> {
246            Var::new(
247                coeus_tensor::Tensor::zeros_on([d_model], &B::default())?,
248                true,
249            )
250        };
251        Ok(Self {
252            w_q: make_weight()?,
253            b_q: if bias { Some(make_bias()?) } else { None },
254            w_k: make_weight()?,
255            b_k: if bias { Some(make_bias()?) } else { None },
256            w_v: make_weight()?,
257            b_v: if bias { Some(make_bias()?) } else { None },
258            w_o: make_weight()?,
259            b_o: if bias { Some(make_bias()?) } else { None },
260            d_model,
261            _mask: PhantomData,
262        })
263    }
264
265    /// Cross-attention forward.
266    ///
267    /// - `query`: `[batch, seq_q, d_model]`
268    /// - `key`:   `[batch, seq_k, d_model]`
269    /// - `value`: `[batch, seq_k, d_model]`
270    ///
271    /// Returns `[batch, seq_q, d_model]`.
272    pub fn forward_cross(
273        &self,
274        query: &Var<T, B>,
275        key: &Var<T, B>,
276        value: &Var<T, B>,
277        key_padding_mask: Option<&Var<T, B>>,
278    ) -> Result<Var<T, B>, ModuleError<B::Error>> {
279        multi_head_attention_cross::<T, B, H, M>(
280            query,
281            key,
282            value,
283            MhaProjectionParams {
284                w_q: &self.w_q,
285                b_q: self.b_q.as_ref(),
286                w_k: &self.w_k,
287                b_k: self.b_k.as_ref(),
288                w_v: &self.w_v,
289                b_v: self.b_v.as_ref(),
290                w_o: &self.w_o,
291                b_o: self.b_o.as_ref(),
292            },
293            key_padding_mask,
294        )
295    }
296}
297
298/// Self-attention `Module` impl (Q = K = V = input).
299impl<
300        T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
301        B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
302        const H: usize,
303        M: AttentionMask,
304    > Module<T, B> for MultiHeadAttention<T, B, H, M>
305{
306    fn parameters(&self) -> Vec<Var<T, B>> {
307        let mut p = vec![
308            self.w_q.clone(),
309            self.w_k.clone(),
310            self.w_v.clone(),
311            self.w_o.clone(),
312        ];
313        if let Some(ref b) = self.b_q {
314            p.push(b.clone());
315        }
316        if let Some(ref b) = self.b_k {
317            p.push(b.clone());
318        }
319        if let Some(ref b) = self.b_v {
320            p.push(b.clone());
321        }
322        if let Some(ref b) = self.b_o {
323            p.push(b.clone());
324        }
325        p
326    }
327
328    fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
329        let names = [
330            "query.weight",
331            "key.weight",
332            "value.weight",
333            "output.weight",
334            "query.bias",
335            "key.bias",
336            "value.bias",
337            "output.bias",
338        ];
339        self.parameters()
340            .into_iter()
341            .zip(names)
342            .map(|(parameter, name)| coeus_autograd::Parameter::new(parameter, name))
343            .collect()
344    }
345
346    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
347        self.forward_cross(input, input, input, None)
348    }
349}