1use crate::init::kaiming_uniform;
8use crate::module::{Module, ModuleError};
9use coeus_autograd::{AttentionMask, Var};
10use coeus_core::{Float, MoiraiBackend, Scalar};
11use std::marker::PhantomData;
12
13#[derive(Clone)]
22pub struct MultiHeadAttention<
23 T: Scalar,
24 B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
25 const H: usize = 8,
26 M: AttentionMask = coeus_autograd::NullMask,
27> {
28 pub w_q: Var<T, B>,
30 pub b_q: Option<Var<T, B>>,
32 pub w_k: Var<T, B>,
34 pub b_k: Option<Var<T, B>>,
36 pub w_v: Var<T, B>,
38 pub b_v: Option<Var<T, B>>,
40 pub w_o: Var<T, B>,
42 pub b_o: Option<Var<T, B>>,
44 pub d_model: usize,
46 _mask: PhantomData<M>,
47}
48
49pub struct MhaProjectionParams<'a, T: Scalar, B: coeus_ops::BackendOps<T> + Default> {
51 pub w_q: &'a Var<T, B>,
53 pub b_q: Option<&'a Var<T, B>>,
55 pub w_k: &'a Var<T, B>,
57 pub b_k: Option<&'a Var<T, B>>,
59 pub w_v: &'a Var<T, B>,
61 pub b_v: Option<&'a Var<T, B>>,
63 pub w_o: &'a Var<T, B>,
65 pub b_o: Option<&'a Var<T, B>>,
67}
68
69pub fn multi_head_attention_cross<
79 T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
80 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
81 const H: usize,
82 M: AttentionMask,
83>(
84 query: &Var<T, B>,
85 key: &Var<T, B>,
86 value: &Var<T, B>,
87 params: MhaProjectionParams<'_, T, B>,
88 key_padding_mask: Option<&Var<T, B>>,
89) -> Result<Var<T, B>, ModuleError<B::Error>> {
90 let wq_shape = params.w_q.tensor.shape_cloned();
91 if wq_shape.len() != 2 || wq_shape[0] != wq_shape[1] || wq_shape[0] == 0 {
92 return Err(super::validation::shape_mismatch(
93 "MultiHeadAttention",
94 "query projection weight",
95 &[1, 1],
96 &wq_shape,
97 ));
98 }
99 let d_model = wq_shape[0];
100 if H == 0 || !d_model.is_multiple_of(H) {
101 return Err(super::validation::shape_mismatch(
102 "MultiHeadAttention",
103 "attention heads",
104 &[d_model],
105 &[H],
106 ));
107 }
108
109 let dimensions = super::validation::sdp_dimensions(
110 "MultiHeadAttention",
111 query,
112 key,
113 value,
114 key_padding_mask,
115 H,
116 )?;
117 for (parameter, actual) in [
118 ("query feature", dimensions.d_k),
119 ("key feature", key.tensor.shape()[2]),
120 ("value feature", dimensions.d_v),
121 ] {
122 if actual != d_model {
123 return Err(super::validation::shape_mismatch(
124 "MultiHeadAttention",
125 parameter,
126 &[d_model],
127 &[actual],
128 ));
129 }
130 }
131 for (parameter, weight, bias) in [
132 ("query projection", params.w_q, params.b_q),
133 ("key projection", params.w_k, params.b_k),
134 ("value projection", params.w_v, params.b_v),
135 ("output projection", params.w_o, params.b_o),
136 ] {
137 super::validation::projection("MultiHeadAttention", parameter, weight, bias, d_model)?;
138 }
139
140 let d_head = d_model / H;
141 let d_head_f64 = <f64 as coeus_core::FloatElement>::from_count(d_head);
144 let scale = T::ONE / <T as coeus_core::FloatElement>::from_f64(d_head_f64.sqrt());
145
146 let batch = dimensions.batch;
147 let seq_q = dimensions.seq_q;
148 let seq_k = dimensions.seq_k;
149
150 let q_proj = project_3d(query, params.w_q, params.b_q, batch, seq_q, d_model)?;
151 let k_proj = project_3d(key, params.w_k, params.b_k, batch, seq_k, d_model)?;
152 let v_proj = project_3d(value, params.w_v, params.b_v, batch, seq_k, d_model)?;
153
154 let q_split = coeus_autograd::reshape(&q_proj, [batch, seq_q, H, d_head])?;
155 let k_split = coeus_autograd::reshape(&k_proj, [batch, seq_k, H, d_head])?;
156 let v_split = coeus_autograd::reshape(&v_proj, [batch, seq_k, H, d_head])?;
157
158 let q_perm = coeus_autograd::permute(&q_split, &[0, 2, 1, 3])?;
159 let k_perm = coeus_autograd::permute(&k_split, &[0, 2, 1, 3])?;
160 let v_perm = coeus_autograd::permute(&v_split, &[0, 2, 1, 3])?;
161
162 let q_heads = coeus_autograd::reshape(&q_perm, [batch * H, seq_q, d_head])?;
163 let k_heads = coeus_autograd::reshape(&k_perm, [batch * H, seq_k, d_head])?;
164 let v_heads = coeus_autograd::reshape(&v_perm, [batch * H, seq_k, d_head])?;
165
166 let (attn_out, _aw) = coeus_autograd::sdp_attention::<T, B, M>(
167 &q_heads,
168 &k_heads,
169 &v_heads,
170 key_padding_mask,
171 scale,
172 )
173 .map_err(|source| ModuleError::Backend {
174 module: "MultiHeadAttention",
175 source,
176 })?;
177
178 let merged_split = coeus_autograd::reshape(&attn_out, [batch, H, seq_q, d_head])?;
179 let merged_perm = coeus_autograd::permute(&merged_split, &[0, 2, 1, 3])?;
180 let merged = coeus_autograd::reshape(&merged_perm, [batch, seq_q, d_model])?;
181
182 Ok(project_3d(
183 &merged, params.w_o, params.b_o, batch, seq_q, d_model,
184 )?)
185}
186
187fn project_3d<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
192 x: &Var<T, B>,
193 w: &Var<T, B>,
194 bias: Option<&Var<T, B>>,
195 batch: usize,
196 seq: usize,
197 d_model: usize,
198) -> Result<Var<T, B>, B::Error> {
199 let flat = coeus_autograd::reshape(x, [batch * seq, d_model])?;
200 let w_t = coeus_autograd::transpose_2d(w)?;
201 let out_flat = coeus_autograd::matmul(&flat, &w_t)?;
202 let out_flat = if let Some(b) = bias {
203 coeus_autograd::add(&out_flat, b)?
204 } else {
205 out_flat
206 };
207 coeus_autograd::reshape(&out_flat, [batch, seq, d_model])
208}
209
210impl<
211 T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
212 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
213 const H: usize,
214 M: AttentionMask,
215 > MultiHeadAttention<T, B, H, M>
216{
217 pub fn new(
224 d_model: usize,
225 bias: bool,
226 ) -> Result<Self, crate::init::InitializationError<B::Error>>
227 where
228 T: coeus_cpu::RealScalar,
229 B: coeus_ops::RandomInitOps<T>,
230 {
231 if H == 0 || d_model == 0 || !d_model.is_multiple_of(H) {
232 return Err(crate::init::InitializationError::InvalidHeadConfiguration {
233 d_model,
234 heads: H,
235 });
236 }
237 let make_weight = || -> Result<Var<T, B>, crate::init::InitializationError<B::Error>> {
238 let mut v = Var::new(
239 coeus_tensor::Tensor::zeros_on([d_model, d_model], &B::default())?,
240 true,
241 )?;
242 kaiming_uniform(&mut v, d_model)?;
243 Ok(v)
244 };
245 let make_bias = || -> Result<Var<T, B>, B::Error> {
246 Var::new(
247 coeus_tensor::Tensor::zeros_on([d_model], &B::default())?,
248 true,
249 )
250 };
251 Ok(Self {
252 w_q: make_weight()?,
253 b_q: if bias { Some(make_bias()?) } else { None },
254 w_k: make_weight()?,
255 b_k: if bias { Some(make_bias()?) } else { None },
256 w_v: make_weight()?,
257 b_v: if bias { Some(make_bias()?) } else { None },
258 w_o: make_weight()?,
259 b_o: if bias { Some(make_bias()?) } else { None },
260 d_model,
261 _mask: PhantomData,
262 })
263 }
264
265 pub fn forward_cross(
273 &self,
274 query: &Var<T, B>,
275 key: &Var<T, B>,
276 value: &Var<T, B>,
277 key_padding_mask: Option<&Var<T, B>>,
278 ) -> Result<Var<T, B>, ModuleError<B::Error>> {
279 multi_head_attention_cross::<T, B, H, M>(
280 query,
281 key,
282 value,
283 MhaProjectionParams {
284 w_q: &self.w_q,
285 b_q: self.b_q.as_ref(),
286 w_k: &self.w_k,
287 b_k: self.b_k.as_ref(),
288 w_v: &self.w_v,
289 b_v: self.b_v.as_ref(),
290 w_o: &self.w_o,
291 b_o: self.b_o.as_ref(),
292 },
293 key_padding_mask,
294 )
295 }
296}
297
298impl<
300 T: coeus_ops::AttentionScalar + coeus_core::FloatElement + coeus_cpu::RealScalar,
301 B: coeus_ops::BackendOps<T> + coeus_ops::AttentionOps<T> + Default,
302 const H: usize,
303 M: AttentionMask,
304 > Module<T, B> for MultiHeadAttention<T, B, H, M>
305{
306 fn parameters(&self) -> Vec<Var<T, B>> {
307 let mut p = vec![
308 self.w_q.clone(),
309 self.w_k.clone(),
310 self.w_v.clone(),
311 self.w_o.clone(),
312 ];
313 if let Some(ref b) = self.b_q {
314 p.push(b.clone());
315 }
316 if let Some(ref b) = self.b_k {
317 p.push(b.clone());
318 }
319 if let Some(ref b) = self.b_v {
320 p.push(b.clone());
321 }
322 if let Some(ref b) = self.b_o {
323 p.push(b.clone());
324 }
325 p
326 }
327
328 fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
329 let names = [
330 "query.weight",
331 "key.weight",
332 "value.weight",
333 "output.weight",
334 "query.bias",
335 "key.bias",
336 "value.bias",
337 "output.bias",
338 ];
339 self.parameters()
340 .into_iter()
341 .zip(names)
342 .map(|(parameter, name)| coeus_autograd::Parameter::new(parameter, name))
343 .collect()
344 }
345
346 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
347 self.forward_cross(input, input, input, None)
348 }
349}