use crate::error::Result;
use crate::tensor::Tensor;
mod compute;
const SHADER_F64: &str = include_str!("../../shaders/attention/gqa_apply_f64.wgsl");
static SHADER_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_F64.to_string());
const SHADER_SOFTMAX_F64: &str = include_str!("../../shaders/activation/gqa_softmax_f64.wgsl");
static SHADER_SOFTMAX_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_SOFTMAX_F64.to_string());
#[cfg(test)]
mod tests;
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct GQAParams {
pub batch_size: u32,
pub num_q_heads: u32,
pub num_kv_heads: u32,
pub seq_len: u32,
pub head_dim: u32,
pub heads_per_group: u32,
}
pub struct GroupedQueryAttention {
query: Tensor,
key: Tensor,
value: Tensor,
batch_size: usize,
num_q_heads: usize,
num_kv_heads: usize,
seq_len: usize,
head_dim: usize,
heads_per_group: usize,
}
impl GroupedQueryAttention {
pub fn new(query: Tensor, key: Tensor, value: Tensor) -> Result<Self> {
let q_shape = query.shape();
let k_shape = key.shape();
let v_shape = value.shape();
if q_shape.len() != 4 || k_shape.len() != 4 || v_shape.len() != 4 {
return Err(crate::error::BarracudaError::InvalidInput {
message: "All inputs must be 4D tensors [batch, heads, seq_len, head_dim]"
.to_string(),
});
}
let batch_size = q_shape[0];
let num_q_heads = q_shape[1];
let seq_len = q_shape[2];
let head_dim = q_shape[3];
let num_kv_heads = k_shape[1];
if k_shape[0] != batch_size || k_shape[2] != seq_len || k_shape[3] != head_dim {
return Err(crate::error::BarracudaError::shape_mismatch(
vec![batch_size, num_q_heads, seq_len, head_dim],
k_shape.to_vec(),
));
}
if v_shape != k_shape {
return Err(crate::error::BarracudaError::shape_mismatch(
k_shape.to_vec(),
v_shape.to_vec(),
));
}
if !num_q_heads.is_multiple_of(num_kv_heads) {
return Err(crate::error::BarracudaError::InvalidInput {
message: format!(
"num_q_heads ({num_q_heads}) must be divisible by num_kv_heads ({num_kv_heads})"
),
});
}
let heads_per_group = num_q_heads / num_kv_heads;
use std::sync::Arc;
if !Arc::ptr_eq(query.device(), key.device())
|| !Arc::ptr_eq(query.device(), value.device())
{
return Err(crate::error::BarracudaError::device(
"All tensors must be on the same device",
));
}
Ok(Self {
query,
key,
value,
batch_size,
num_q_heads,
num_kv_heads,
seq_len,
head_dim,
heads_per_group,
})
}
pub(super) fn wgsl_shader_matmul() -> &'static str {
static SHADER: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
include_str!("../../shaders/math/gqa_matmul_f64.wgsl").to_string()
});
&SHADER
}
pub(super) fn wgsl_shader_softmax() -> &'static str {
&SHADER_SOFTMAX_F32
}
pub(super) fn wgsl_shader_apply() -> &'static str {
&SHADER_F32
}
pub(super) fn query(&self) -> &Tensor {
&self.query
}
pub(super) fn key(&self) -> &Tensor {
&self.key
}
pub(super) fn value(&self) -> &Tensor {
&self.value
}
pub(super) fn batch_size(&self) -> usize {
self.batch_size
}
pub(super) fn num_q_heads(&self) -> usize {
self.num_q_heads
}
pub(super) fn num_kv_heads(&self) -> usize {
self.num_kv_heads
}
pub(super) fn seq_len(&self) -> usize {
self.seq_len
}
pub(super) fn head_dim(&self) -> usize {
self.head_dim
}
pub(super) fn heads_per_group(&self) -> usize {
self.heads_per_group
}
}
impl Tensor {
pub fn grouped_query_attention(self, key: Self, value: Self) -> Result<Self> {
GroupedQueryAttention::new(self, key, value)?.execute()
}
}