use crate::error::{BarracudaError, Result};
use crate::tensor::Tensor;
mod compute;
static CROSS_ATTENTION_MATMUL_F64: &str =
include_str!("../../shaders/math/cross_attention_matmul_f64.wgsl");
static CROSS_ATTENTION_MATMUL_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| CROSS_ATTENTION_MATMUL_F64.to_string());
const SHADER_F64: &str = include_str!("../../shaders/attention/cross_attention_apply_f64.wgsl");
static SHADER_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_F64.to_string());
const SHADER_SOFTMAX_F64: &str =
include_str!("../../shaders/activation/cross_attention_softmax_f64.wgsl");
static SHADER_SOFTMAX_F32: std::sync::LazyLock<String> =
std::sync::LazyLock::new(|| SHADER_SOFTMAX_F64.to_string());
#[cfg(test)]
mod tests;
#[repr(C)]
#[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
pub(crate) struct CrossAttentionParams {
pub batch_size: u32,
pub num_heads: u32,
pub decoder_seq: u32,
pub encoder_seq: u32,
pub head_dim: u32,
pub _padding: [u32; 3],
}
pub struct CrossAttention {
query: Tensor, key: Tensor, value: Tensor, }
impl CrossAttention {
pub fn new(query: Tensor, key: Tensor, value: Tensor) -> Result<Self> {
if query.shape().len() != 4 || key.shape().len() != 4 || value.shape().len() != 4 {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
vec![0, 0, 0, 0],
));
}
if query.shape()[0] != key.shape()[0] || query.shape()[0] != value.shape()[0] {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
key.shape().to_vec(),
));
}
if query.shape()[1] != key.shape()[1] || query.shape()[1] != value.shape()[1] {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
key.shape().to_vec(),
));
}
if query.shape()[3] != key.shape()[3] || query.shape()[3] != value.shape()[3] {
return Err(BarracudaError::shape_mismatch(
query.shape().to_vec(),
key.shape().to_vec(),
));
}
if key.shape()[2] != value.shape()[2] {
return Err(BarracudaError::shape_mismatch(
key.shape().to_vec(),
value.shape().to_vec(),
));
}
Ok(Self { query, key, value })
}
pub(super) fn shader_matmul() -> &'static str {
&CROSS_ATTENTION_MATMUL_F32
}
pub(super) fn shader_softmax() -> &'static str {
&SHADER_SOFTMAX_F32
}
pub(super) fn shader_apply() -> &'static str {
&SHADER_F32
}
pub(super) fn query(&self) -> &Tensor {
&self.query
}
pub(super) fn key(&self) -> &Tensor {
&self.key
}
pub(super) fn value(&self) -> &Tensor {
&self.value
}
}
impl Tensor {
pub fn cross_attention(self, key: &Self, value: &Self) -> Result<Self> {
CrossAttention::new(self, key.clone(), value.clone())?.execute()
}
}