use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
mod compute;
#[cfg(test)]
mod tests;
const SHADER_SPARSE_SOFTMAX_F64: &str =
include_str!("../../shaders/activation/sparse_attention_softmax_f64.wgsl");
static SHADER_SPARSE_SOFTMAX_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_SPARSE_SOFTMAX_F64.to_string());
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct SparseAttentionParams {
pub batch_size: u32,
pub num_heads: u32,
pub seq_len: u32,
pub head_dim: u32,
pub stride: u32,
pub _padding: [u32; 3],
}
pub struct SparseAttention {
query: Tensor,
key: Tensor,
value: Tensor,
stride: usize,
}
impl SparseAttention {
pub fn new(query: Tensor, key: Tensor, value: Tensor, stride: usize) -> Result<Self> {
if query.shape().len() != 4 || key.shape().len() != 4 || value.shape().len() != 4 {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
vec![0, 0, 0, 0],
));
}
if query.shape() != key.shape() || query.shape() != value.shape() {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
key.shape().to_vec(),
));
}
if stride == 0 {
return Err(BarracudaError::invalid_op(
"SparseAttention",
"stride must be > 0",
));
}
Ok(Self {
query,
key,
value,
stride,
})
}
pub(super) fn shader_matmul() -> &'static str {
&crate::ops::attention::ATTENTION_MATMUL_F32
}
pub(super) fn shader_sparse_softmax() -> &'static str {
&SHADER_SPARSE_SOFTMAX_F32
}
pub(super) fn shader_apply() -> &'static str {
&crate::ops::attention::ATTENTION_APPLY_F32
}
pub(super) fn query(&self) -> &Tensor {
&self.query
}
pub(super) fn key(&self) -> &Tensor {
&self.key
}
pub(super) fn value(&self) -> &Tensor {
&self.value
}
pub(super) fn stride(&self) -> usize {
self.stride
}
}
impl Tensor {
pub fn sparse_attention(self, key: &Self, value: &Self, stride: usize) -> Result<Self> {
SparseAttention::new(self, key.clone(), value.clone(), stride)?.execute()
}
}