use crate::error::{TptpError, TptpResult};
use crate::kernel::{KernelConfig, KernelResult, PrimitiveKernel};
use crate::memory::{BufferFlags, DType, GpuBuffer, Shape};
use crate::vendor::{VendorBackend, VendorLibrary};
use std::time::Instant;
#[derive(Debug, Clone)]
pub struct AttentionParams {
pub tile_seq: u32,
pub tile_head: u32,
pub vec_width: u32,
}
impl Default for AttentionParams {
fn default() -> Self {
AttentionParams {
tile_seq: 64,
tile_head: 64,
vec_width: 4,
}
}
}
pub struct AttentionKernel {
#[allow(dead_code)]
config: KernelConfig,
#[allow(dead_code)]
vendor: VendorBackend,
pub params: AttentionParams,
}
impl Default for AttentionKernel {
fn default() -> Self {
Self::new()
}
}
impl AttentionKernel {
pub fn new() -> Self {
let vendor = VendorBackend::detect();
let config = KernelConfig::new([32, 1, 1], [256, 1, 1]);
AttentionKernel {
config,
vendor,
params: AttentionParams::default(),
}
}
pub fn with_vendor(vendor: VendorBackend) -> Self {
let config = KernelConfig::new([32, 1, 1], [256, 1, 1]);
AttentionKernel {
config,
vendor,
params: AttentionParams::default(),
}
}
pub fn with_params(mut self, params: AttentionParams) -> Self {
self.params = params;
self
}
pub fn execute(
&self,
q: &GpuBuffer<f32>,
k: &GpuBuffer<f32>,
v: &GpuBuffer<f32>,
scale: Option<f32>,
_mask: Option<&GpuBuffer<f32>>,
) -> TptpResult<GpuBuffer<f32>> {
if q.ndim() != 2 || k.ndim() != 2 || v.ndim() != 2 {
return Err(TptpError::shape_error("Attention requires 2D tensors"));
}
let q_len = q
.dim(0)
.ok_or_else(|| TptpError::shape_error("Q has no dim 0"))?;
let d_k = q
.dim(1)
.ok_or_else(|| TptpError::shape_error("Q has no dim 1"))?;
let kv_len = k
.dim(0)
.ok_or_else(|| TptpError::shape_error("K has no dim 0"))?;
let d_v = v
.dim(1)
.ok_or_else(|| TptpError::shape_error("V has no dim 1"))?;
if k.dim(1) != Some(d_k) {
return Err(TptpError::shape_error("K head_dim must match Q head_dim"));
}
if v.dim(0) != Some(kv_len) {
return Err(TptpError::shape_error("V kv_len must match K kv_len"));
}
let scale_val = scale.unwrap_or_else(|| 1.0 / (d_k as f32).sqrt());
let mut output = GpuBuffer::new(Shape::dim2(q_len, d_v), DType::F32, BufferFlags::STORAGE)?;
let t0 = Instant::now();
if self.vendor.supports_attention() && q_len == kv_len {
self.vendor
.attention(q, k, v, &mut output, scale_val, q_len, d_k)?;
} else {
self.tptir_fallback_attention(q, k, v, &mut output, scale_val, q_len, kv_len, d_k)?;
}
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
log::debug!(
"Attention q_len={} kv_len={} d_k={} via {}: {:.3}ms",
q_len,
kv_len,
d_k,
self.vendor.name(),
elapsed_ms
);
Ok(output)
}
#[allow(clippy::too_many_arguments)]
fn tptir_fallback_attention(
&self,
q: &GpuBuffer<f32>,
k: &GpuBuffer<f32>,
v: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
scale: f32,
q_len: usize,
kv_len: usize,
d_k: usize,
) -> TptpResult<()> {
let d_v = v.dim(1).unwrap_or(d_k);
log::debug!(
"TPTIR Attention fallback: q_len={}, kv_len={}, d_k={}, d_v={}, tile_seq={}",
q_len,
kv_len,
d_k,
d_v,
self.params.tile_seq
);
let mut q_raw = vec![0.0f32; q_len * d_k];
q.copy_to_host(&mut q_raw)?;
let mut k_raw = vec![0.0f32; kv_len * d_k];
k.copy_to_host(&mut k_raw)?;
let mut v_raw = vec![0.0f32; kv_len * d_v];
v.copy_to_host(&mut v_raw)?;
let mut scores = vec![0.0f32; q_len * kv_len];
for i in 0..q_len {
for j in 0..kv_len {
let mut acc = 0.0f32;
for kk in 0..d_k {
acc += q_raw[i * d_k + kk] * k_raw[j * d_k + kk];
}
scores[i * kv_len + j] = acc * scale;
}
}
for i in 0..q_len {
let row = &mut scores[i * kv_len..(i + 1) * kv_len];
let max_val = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let mut sum = 0.0f32;
for s in row.iter_mut() {
*s = (*s - max_val).exp();
sum += *s;
}
if sum > 0.0 {
for s in row.iter_mut() {
*s /= sum;
}
}
}
let mut out_raw = vec![0.0f32; q_len * d_v];
for i in 0..q_len {
for j in 0..d_v {
let mut acc = 0.0f32;
for kk in 0..kv_len {
acc += scores[i * kv_len + kk] * v_raw[kk * d_v + j];
}
out_raw[i * d_v + j] = acc;
}
}
output.copy_from_host(&out_raw)
}
}
impl PrimitiveKernel for AttentionKernel {
fn name(&self) -> &str {
"attention"
}
fn supported_dtypes(&self) -> &[DType] {
&[DType::F32, DType::F16]
}
fn can_execute(&self, inputs: &[&GpuBuffer<f32>]) -> bool {
inputs.len() == 3 && inputs.iter().all(|i| i.ndim() == 2)
}
fn default_config(&self) -> KernelConfig {
KernelConfig::new([32, 1, 1], [256, 1, 1])
}
fn execute(
&self,
inputs: &[&GpuBuffer<f32>],
output: &mut GpuBuffer<f32>,
_config: &KernelConfig,
) -> TptpResult<KernelResult> {
let q = inputs[0];
let k = inputs[1];
let v = inputs[2];
let seq_len = q.dim(0).unwrap_or(0);
let d_k = q.dim(1).unwrap_or(0);
let t0 = Instant::now();
if self.vendor.supports_attention() {
self.vendor
.attention(q, k, v, output, 1.0 / (d_k as f32).sqrt(), seq_len, d_k)?;
}
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
Ok(KernelResult {
outputs: vec![],
execution_time_ms: Some(elapsed_ms),
backend_used: self.vendor.name().to_string(),
})
}
fn execute_with_vendor(
&self,
inputs: &[&GpuBuffer<f32>],
output: &mut GpuBuffer<f32>,
vendor: &VendorBackend,
_config: &KernelConfig,
) -> TptpResult<KernelResult> {
let q = inputs[0];
let k = inputs[1];
let v = inputs[2];
let seq_len = q.dim(0).unwrap_or(0);
let d_k = q.dim(1).unwrap_or(0);
let t0 = Instant::now();
vendor.attention(q, k, v, output, 1.0 / (d_k as f32).sqrt(), seq_len, d_k)?;
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
Ok(KernelResult {
outputs: vec![],
execution_time_ms: Some(elapsed_ms),
backend_used: vendor.name().to_string(),
})
}
}
pub fn attention(
q: &GpuBuffer<f32>,
k: &GpuBuffer<f32>,
v: &GpuBuffer<f32>,
scale: Option<f32>,
) -> TptpResult<GpuBuffer<f32>> {
AttentionKernel::new().execute(q, k, v, scale, None)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_attention_validation() {
let q =
GpuBuffer::<f32>::new(Shape::dim2(8, 64), DType::F32, BufferFlags::STORAGE).unwrap();
let k =
GpuBuffer::<f32>::new(Shape::dim2(8, 64), DType::F32, BufferFlags::STORAGE).unwrap();
let v =
GpuBuffer::<f32>::new(Shape::dim2(8, 64), DType::F32, BufferFlags::STORAGE).unwrap();
let kernel = AttentionKernel::new();
let result = kernel.execute(&q, &k, &v, None, None);
assert!(result.is_ok());
}
#[test]
fn test_attention_shape_mismatch_head_dim() {
let q =
GpuBuffer::<f32>::new(Shape::dim2(8, 64), DType::F32, BufferFlags::STORAGE).unwrap();
let k =
GpuBuffer::<f32>::new(Shape::dim2(8, 32), DType::F32, BufferFlags::STORAGE).unwrap();
let v =
GpuBuffer::<f32>::new(Shape::dim2(8, 64), DType::F32, BufferFlags::STORAGE).unwrap();
let kernel = AttentionKernel::new();
let result = kernel.execute(&q, &k, &v, None, None);
assert!(result.is_err());
}
#[test]
fn test_attention_cross_attention_ok() {
let q = GpuBuffer::<f32>::new(Shape::dim2(2, 8), DType::F32, BufferFlags::STORAGE).unwrap();
let k = GpuBuffer::<f32>::new(Shape::dim2(4, 8), DType::F32, BufferFlags::STORAGE).unwrap();
let v = GpuBuffer::<f32>::new(Shape::dim2(4, 8), DType::F32, BufferFlags::STORAGE).unwrap();
let kernel = AttentionKernel::new();
let result = kernel.execute(&q, &k, &v, None, None);
assert!(result.is_ok());
let out = result.unwrap();
assert_eq!(out.dim(0), Some(2));
assert_eq!(out.dim(1), Some(8));
}
#[test]
fn test_attention_params_default() {
let params = AttentionParams::default();
assert_eq!(params.tile_seq, 64);
assert_eq!(params.tile_head, 64);
}
#[test]
fn test_attention_fallback_computes_real_output() {
let mut q =
GpuBuffer::<f32>::new(Shape::dim2(1, 1), DType::F32, BufferFlags::STORAGE).unwrap();
q.copy_from_host(&[1.0]).unwrap();
let mut k =
GpuBuffer::<f32>::new(Shape::dim2(1, 1), DType::F32, BufferFlags::STORAGE).unwrap();
k.copy_from_host(&[1.0]).unwrap();
let mut v =
GpuBuffer::<f32>::new(Shape::dim2(1, 2), DType::F32, BufferFlags::STORAGE).unwrap();
v.copy_from_host(&[3.0, 4.0]).unwrap();
let kernel = AttentionKernel::new();
let out = kernel.execute(&q, &k, &v, Some(1.0), None).unwrap();
let mut data = [0f32; 2];
out.copy_to_host(&mut data).unwrap();
assert!((data[0] - 3.0).abs() < 1e-5, "expected 3.0 got {}", data[0]);
assert!((data[1] - 4.0).abs() < 1e-5, "expected 4.0 got {}", data[1]);
}
#[test]
fn test_attention_fallback_softmax_distributes_weight() {
let mut q =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
q.copy_from_host(&[1.0, 0.0, 0.0, 1.0]).unwrap();
let mut k =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
k.copy_from_host(&[1.0, 0.0, 0.0, 1.0]).unwrap();
let mut v =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
v.copy_from_host(&[10.0, 0.0, 0.0, 10.0]).unwrap();
let kernel = AttentionKernel::new();
let out = kernel.execute(&q, &k, &v, Some(1.0), None).unwrap();
let mut data = [0f32; 4];
out.copy_to_host(&mut data).unwrap();
assert!(
data[0] > 5.0,
"output[0][0]={} should be > 5 (attending mostly to V[0])",
data[0]
);
assert!(
data[3] > 5.0,
"output[1][1]={} should be > 5 (attending mostly to V[1])",
data[3]
);
}
}