mod compute;
#[cfg(test)]
mod tests;
use crate::error::Result;
use crate::tensor::Tensor;
static WGSL_SDPA_SINGLE_KERNEL_F64: &str =
include_str!("../../shaders/attention/scaled_dot_product_attention_f64.wgsl");
static WGSL_SDPA_SINGLE_KERNEL_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| WGSL_SDPA_SINGLE_KERNEL_F64.to_string());
#[must_use]
pub fn wgsl_sdpa_single_kernel() -> &'static str {
&WGSL_SDPA_SINGLE_KERNEL_F32
}
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub struct AttentionParams {
pub batch_size: u32,
pub num_heads: u32,
pub q_seq_len: u32,
pub kv_seq_len: u32,
pub head_dim: u32,
pub _padding: [u32; 3],
}
pub struct ScaledDotProductAttention {
query: Tensor,
key: Tensor,
value: Tensor,
batch_size: usize,
num_heads: usize,
seq_len: usize,
head_dim: usize,
}
impl ScaledDotProductAttention {
pub fn new(query: Tensor, key: Tensor, value: Tensor) -> Result<Self> {
let q_shape = query.shape();
let k_shape = key.shape();
let v_shape = value.shape();
if q_shape.len() != 4 || k_shape.len() != 4 || v_shape.len() != 4 {
return Err(crate::error::BarracudaError::InvalidInput {
message: "All inputs must be 4D tensors [batch, heads, seq_len, head_dim]"
.to_string(),
});
}
let batch_size = q_shape[0];
let num_heads = q_shape[1];
let seq_len = q_shape[2];
let head_dim = q_shape[3];
if k_shape != q_shape || v_shape != q_shape {
return Err(crate::error::BarracudaError::shape_mismatch(
q_shape.to_vec(),
if k_shape == q_shape {
v_shape.to_vec()
} else {
k_shape.to_vec()
},
));
}
use std::sync::Arc;
if !Arc::ptr_eq(query.device(), key.device())
|| !Arc::ptr_eq(query.device(), value.device())
{
return Err(crate::error::BarracudaError::device(
"All tensors must be on the same device",
));
}
Ok(Self {
query,
key,
value,
batch_size,
num_heads,
seq_len,
head_dim,
})
}
pub(super) fn wgsl_shader_matmul() -> &'static str {
&crate::ops::attention::ATTENTION_MATMUL_F32
}
pub(super) fn wgsl_shader_softmax() -> &'static str {
&crate::ops::attention::ATTENTION_SOFTMAX_F32
}
pub(super) fn wgsl_shader_apply() -> &'static str {
&crate::ops::attention::ATTENTION_APPLY_F32
}
pub(super) fn query(&self) -> &Tensor {
&self.query
}
pub(super) fn key(&self) -> &Tensor {
&self.key
}
pub(super) fn value(&self) -> &Tensor {
&self.value
}
pub(super) fn batch_size(&self) -> usize {
self.batch_size
}
pub(super) fn num_heads(&self) -> usize {
self.num_heads
}
pub(super) fn seq_len(&self) -> usize {
self.seq_len
}
pub(super) fn head_dim(&self) -> usize {
self.head_dim
}
pub fn execute(self) -> Result<Tensor> {
compute::execute_scaled_dot_product_attention(self)
}
}
impl Tensor {
pub fn scaled_dot_product_attention(self, key: Self, value: Self) -> Result<Self> {
ScaledDotProductAttention::new(self, key, value)?.execute()
}
}