use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
mod compute;
#[cfg(test)]
mod tests;
const SHADER_CAUSAL_SOFTMAX_F64: &str =
include_str!("../../shaders/activation/causal_attention_softmax_f64.wgsl");
static SHADER_CAUSAL_SOFTMAX_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_CAUSAL_SOFTMAX_F64.to_string());
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct AttentionParams {
pub batch_size: u32,
pub num_heads: u32,
pub q_seq_len: u32,
pub kv_seq_len: u32,
pub head_dim: u32,
pub _padding: [u32; 3],
}
pub struct CausalAttention {
query: Tensor,
key: Tensor,
value: Tensor,
}
impl CausalAttention {
pub fn new(query: Tensor, key: Tensor, value: Tensor) -> Result<Self> {
if query.shape().len() != 4 || key.shape().len() != 4 || value.shape().len() != 4 {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
vec![0, 0, 0, 0],
));
}
if query.shape() != key.shape() || query.shape() != value.shape() {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
key.shape().to_vec(),
));
}
Ok(Self { query, key, value })
}
pub(super) fn shader_matmul() -> &'static str {
&crate::ops::attention::ATTENTION_MATMUL_F32
}
pub(super) fn shader_causal_softmax() -> &'static str {
&SHADER_CAUSAL_SOFTMAX_F32
}
pub(super) fn shader_apply() -> &'static str {
&crate::ops::attention::ATTENTION_APPLY_F32
}
pub(super) fn query(&self) -> &Tensor {
&self.query
}
pub(super) fn key(&self) -> &Tensor {
&self.key
}
pub(super) fn value(&self) -> &Tensor {
&self.value
}
}
impl Tensor {
pub fn causal_attention(self, key: &Self, value: &Self) -> Result<Self> {
CausalAttention::new(self, key.clone(), value.clone())?.execute()
}
}