use crate::tensor::Tensor;
use mathverse_core::error::MathResult;
pub fn qkv_projection(
x: &Tensor,
wq: &Tensor,
wk: &Tensor,
wv: &Tensor,
) -> MathResult<(Tensor, Tensor, Tensor)> {
let batch = x.shape[0];
let seq_len = x.shape[1];
let d_model = x.shape[2];
let d_k = wq.shape[1];
let d_v = wv.shape[1];
let x_flat = Tensor::new(&[batch * seq_len, d_model], &x.data)?;
let q = x_flat.matmul(wq)?.reshape(&[batch, seq_len, d_k])?;
let k = x_flat.matmul(wk)?.reshape(&[batch, seq_len, d_k])?;
let v = x_flat.matmul(wv)?.reshape(&[batch, seq_len, d_v])?;
Ok((q, k, v))
}
pub fn scaled_dot_product_attention(
q: &Tensor,
k: &Tensor,
v: &Tensor,
scale: Option<f64>,
mask: Option<&Tensor>,
) -> MathResult<(Tensor, Tensor)> {
let d_k = q.shape[2] as f64;
let s = scale.unwrap_or(1.0 / d_k.sqrt());
let batch = q.shape[0];
let seq_q = q.shape[1];
let seq_kv = k.shape[1];
let d_v = v.shape[2];
let mut scores_data = vec![0.0; batch * seq_q * seq_kv];
for bi in 0..batch {
for i in 0..seq_q {
for j in 0..seq_kv {
let mut dot = 0.0;
for d in 0..q.shape[2] {
dot += q.data[bi * seq_q * q.shape[2] + i * q.shape[2] + d]
* k.data[bi * seq_kv * k.shape[2] + j * k.shape[2] + d];
}
scores_data[bi * seq_q * seq_kv + i * seq_kv + j] = dot * s;
}
}
}
if let Some(m) = mask {
#[allow(clippy::needless_range_loop)]
for idx in 0..scores_data.len() {
if m.data[idx] < -1e10 {
scores_data[idx] = f64::NEG_INFINITY;
}
}
}
let scores = Tensor::new(&[batch, seq_q, seq_kv], &scores_data)?;
let weights = crate::activations::softmax(&scores, 2)?;
let mut out_data = vec![0.0; batch * seq_q * d_v];
for bi in 0..batch {
for i in 0..seq_q {
for d in 0..d_v {
let mut val = 0.0;
for j in 0..seq_kv {
val += weights.data[bi * seq_q * seq_kv + i * seq_kv + j]
* v.data[bi * seq_kv * d_v + j * d_v + d];
}
out_data[bi * seq_q * d_v + i * d_v + d] = val;
}
}
}
let output = Tensor::new(&[batch, seq_q, d_v], &out_data)?;
Ok((output, weights))
}
pub fn multi_head_attention(
x: &Tensor,
wq: &Tensor,
wk: &Tensor,
wv: &Tensor,
wo: &Tensor,
num_heads: usize,
mask: Option<&Tensor>,
) -> MathResult<Tensor> {
let batch = x.shape[0];
let seq_len = x.shape[1];
let d_model = x.shape[2];
let d_k = wq.shape[1] / num_heads;
let d_v = wv.shape[1] / num_heads;
let (q, k, v) = qkv_projection(x, wq, wk, wv)?;
let q = q.reshape(&[batch, seq_len, num_heads, d_k])?.permute(&[0, 2, 1, 3])?;
let k = k.reshape(&[batch, seq_len, num_heads, d_k])?.permute(&[0, 2, 1, 3])?;
let v = v.reshape(&[batch, seq_len, num_heads, d_v])?.permute(&[0, 2, 1, 3])?;
let mut head_outputs = Vec::new();
let mut all_weights = Vec::new();
#[allow(clippy::needless_range_loop)] for h in 0..num_heads {
let qh = extract_head(&q, h)?;
let kh = extract_head(&k, h)?;
let vh = extract_head(&v, h)?;
let (out, weights) = scaled_dot_product_attention(&qh, &kh, &vh, None, mask)?;
head_outputs.push(out);
all_weights.push(weights);
}
let mut concat_data = vec![0.0; batch * seq_len * num_heads * d_v];
#[allow(clippy::needless_range_loop)] for h in 0..num_heads {
for bi in 0..batch {
for s in 0..seq_len {
for d in 0..d_v {
let src_idx = bi * seq_len * d_v + s * d_v + d;
let dst_idx = bi * seq_len * (num_heads * d_v) + s * (num_heads * d_v) + h * d_v + d;
concat_data[dst_idx] = head_outputs[h].data[src_idx];
}
}
}
}
let concat = Tensor::new(&[batch, seq_len, num_heads * d_v], &concat_data)?;
let flat = concat.reshape(&[batch * seq_len, num_heads * d_v])?;
let out = flat.matmul(wo)?.reshape(&[batch, seq_len, d_model])?;
Ok(out)
}
fn extract_head(t: &Tensor, head: usize) -> MathResult<Tensor> {
let (batch, _nh, seq, d) = (t.shape[0], t.shape[1], t.shape[2], t.shape[3]);
let mut data = vec![0.0; batch * seq * d];
for bi in 0..batch {
for s in 0..seq {
for di in 0..d {
let src = bi * t.shape[1] * seq * d + head * seq * d + s * d + di;
let dst = bi * seq * d + s * d + di;
data[dst] = t.data[src];
}
}
}
Ok(Tensor { shape: vec![batch, seq, d], data })
}
pub fn sinusoidal_encoding(seq_len: usize, d_model: usize, max_len: usize) -> Tensor {
let mut data = vec![0.0; seq_len * d_model];
for pos in 0..seq_len {
for i in 0..d_model / 2 {
let angle = pos as f64 / (max_len as f64).powf(2.0 * i as f64 / d_model as f64);
data[pos * d_model + 2 * i] = angle.sin();
data[pos * d_model + 2 * i + 1] = angle.cos();
}
}
Tensor { shape: vec![1, seq_len, d_model], data }
}
pub fn rope_freqs(d_head: usize, seq_len: usize, base: f64) -> Tensor {
let mut data = vec![0.0; seq_len * d_head];
for pos in 0..seq_len {
for i in 0..d_head / 2 {
let freq = 1.0 / base.powf(2.0 * i as f64 / d_head as f64);
data[pos * d_head + 2 * i] = pos as f64 * freq;
data[pos * d_head + 2 * i + 1] = pos as f64 * freq;
}
}
Tensor { shape: vec![seq_len, d_head], data }
}
pub fn apply_rope(x: &Tensor, seq_len: usize, base: f64) -> Tensor {
let d_head = x.shape[x.shape.len() - 1];
let freqs = rope_freqs(d_head, seq_len, base);
let mut out = x.data.clone();
let nd = x.shape.len();
let outer: usize = x.shape[..nd - 2].iter().product();
let total_pairs = outer * seq_len * (d_head / 2);
for flat in 0..total_pairs {
let pair_idx = flat % (d_head / 2);
let remaining = flat / (d_head / 2);
let pos = remaining % seq_len;
let outer_idx = remaining / seq_len;
let theta = freqs.data[pos * d_head + 2 * pair_idx];
let cos_t = theta.cos();
let sin_t = theta.sin();
let base_offset = outer_idx * seq_len * d_head + pos * d_head;
let i0 = base_offset + 2 * pair_idx;
let i1 = base_offset + 2 * pair_idx + 1;
let x0 = x.data[i0];
let x1 = x.data[i1];
out[i0] = x0 * cos_t - x1 * sin_t;
out[i1] = x0 * sin_t + x1 * cos_t;
}
Tensor { shape: x.shape.clone(), data: out }
}
#[cfg(test)]
mod tests {
use super::*;
const E: f64 = 1e-5;
#[test]
fn scaled_dot_product_attention_test() {
let q = Tensor::new(&[1, 2, 3], &[
1.0, 0.0, 0.0,
0.0, 1.0, 0.0,
]).unwrap();
let k = Tensor::new(&[1, 2, 3], &[
1.0, 0.0, 0.0,
0.0, 1.0, 0.0,
]).unwrap();
let v = Tensor::new(&[1, 2, 3], &[
1.0, 2.0, 3.0,
4.0, 5.0, 6.0,
]).unwrap();
let (out, weights) = scaled_dot_product_attention(&q, &k, &v, None, None).unwrap();
assert_eq!(out.shape, vec![1, 2, 3]);
assert_eq!(weights.shape, vec![1, 2, 2]);
let w_sum: f64 = weights.data[0..2].iter().sum();
assert!((w_sum - 1.0).abs() < E);
}
#[test]
fn attention_mask_test() {
let q = Tensor::new(&[1, 2, 2], &[1.0, 0.0, 0.0, 1.0]).unwrap();
let k = Tensor::new(&[1, 2, 2], &[1.0, 0.0, 0.0, 1.0]).unwrap();
let v = Tensor::new(&[1, 2, 2], &[1.0, 2.0, 3.0, 4.0]).unwrap();
let mask = Tensor::new(&[1, 2, 2], &[
0.0, f64::NEG_INFINITY,
0.0, 0.0,
]).unwrap();
let (out, _) = scaled_dot_product_attention(&q, &k, &v, None, Some(&mask)).unwrap();
assert!((out.data[0] - 1.0).abs() < E);
assert!((out.data[1] - 2.0).abs() < E);
}
#[test]
fn sinusoidal_encoding_test() {
let enc = sinusoidal_encoding(10, 8, 10000);
assert_eq!(enc.shape, vec![1, 10, 8]);
assert!(enc.data[0].abs() < E);
assert!((enc.data[1] - 1.0).abs() < E);
}
#[test]
fn rope_test() {
let freqs = rope_freqs(4, 5, 10000.0);
assert_eq!(freqs.shape, vec![5, 4]);
for i in 0..4 {
assert!(freqs.data[i].abs() < E);
}
}
#[test]
fn apply_rope_preserves_norm() {
let x = Tensor::new(&[1, 4, 4], &(0..16).map(|i| i as f64).collect::<Vec<_>>()).unwrap();
let norm_before: f64 = x.data.iter().map(|v| v * v).sum();
let y = apply_rope(&x, 4, 10000.0);
let norm_after: f64 = y.data.iter().map(|v| v * v).sum();
assert!((norm_before - norm_after).abs() < E);
}
#[test]
fn multi_head_attention_shapes() {
let batch = 2;
let seq = 4;
let d_model = 8;
let num_heads = 2;
let d_k = 4;
let x = Tensor::randn(&[batch, seq, d_model]);
let wq = Tensor::randn(&[d_model, num_heads * d_k]);
let wk = Tensor::randn(&[d_model, num_heads * d_k]);
let wv = Tensor::randn(&[d_model, num_heads * d_k]);
let wo = Tensor::randn(&[num_heads * d_k, d_model]);
let out = multi_head_attention(&x, &wq, &wk, &wv, &wo, num_heads, None).unwrap();
assert_eq!(out.shape, vec![batch, seq, d_model]);
}
}