use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
mod compute;
pub(crate) static ATTENTION_MATMUL_F64: &str =
include_str!("../../shaders/math/attention_matmul_f64.wgsl");
pub(crate) static ATTENTION_MATMUL_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| ATTENTION_MATMUL_F64.to_string());
pub(crate) const SDPA_SCORES_F64: &str =
include_str!("../../shaders/attention/sdpa_scores_f64.wgsl");
pub(crate) static SDPA_SCORES_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SDPA_SCORES_F64.to_string());
pub(crate) const ATTENTION_APPLY_F64: &str =
include_str!("../../shaders/attention/attention_apply_f64.wgsl");
pub(crate) static ATTENTION_APPLY_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| ATTENTION_APPLY_F64.to_string());
pub(crate) const ATTENTION_SOFTMAX_F64: &str =
include_str!("../../shaders/activation/attention_softmax_f64.wgsl");
pub(crate) static ATTENTION_SOFTMAX_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| ATTENTION_SOFTMAX_F64.to_string());
#[cfg(test)]
mod tests;
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct AttentionParams {
pub batch_size: u32,
pub num_heads: u32,
pub q_seq_len: u32,
pub kv_seq_len: u32,
pub head_dim: u32,
pub _padding: [u32; 3],
}
pub struct Attention {
query: Tensor,
key: Tensor,
value: Tensor,
}
impl Attention {
pub fn new(query: Tensor, key: Tensor, value: Tensor) -> Result<Self> {
let q_ndim = query.shape().len();
let k_ndim = key.shape().len();
let v_ndim = value.shape().len();
if q_ndim != 4 {
return Err(BarracudaError::invalid_op(
"attention",
format!(
"query requires 4D tensor [batch, heads, seq_len, head_dim], got {q_ndim}D"
),
));
}
if k_ndim != 4 {
return Err(BarracudaError::invalid_op(
"attention",
format!("key requires 4D tensor [batch, heads, seq_len, head_dim], got {k_ndim}D"),
));
}
if v_ndim != 4 {
return Err(BarracudaError::invalid_op(
"attention",
format!(
"value requires 4D tensor [batch, heads, seq_len, head_dim], got {v_ndim}D"
),
));
}
if query.shape()[0] != key.shape()[0]
|| query.shape()[1] != key.shape()[1]
|| query.shape()[3] != key.shape()[3]
{
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
key.shape().to_vec(),
));
}
if key.shape() != value.shape() {
return Err(BarracudaError::shape_mismatch(
key.shape().to_vec(),
value.shape().to_vec(),
));
}
Ok(Self { query, key, value })
}
pub(crate) fn shader_matmul() -> &'static str {
&ATTENTION_MATMUL_F32
}
pub(crate) fn shader_softmax() -> &'static str {
&ATTENTION_SOFTMAX_F32
}
pub(crate) fn shader_apply() -> &'static str {
&ATTENTION_APPLY_F32
}
pub(crate) fn query(&self) -> &Tensor {
&self.query
}
pub(crate) fn key(&self) -> &Tensor {
&self.key
}
pub(crate) fn value(&self) -> &Tensor {
&self.value
}
}
impl Tensor {
pub fn attention(self, key: &Self, value: &Self) -> Result<Self> {
Attention::new(self, key.clone(), value.clone())?.execute()
}
}