use super::{AttentionParams, ScaledDotProductAttention};
use crate::device::compute_pipeline::ComputeDispatch;
use crate::device::{DeviceCapabilities, WorkloadType};
use crate::error::Result;
use crate::tensor::Tensor;
pub(super) fn execute_scaled_dot_product_attention(
op: ScaledDotProductAttention,
) -> Result<Tensor> {
let device = op.query().device();
let input_size = op.batch_size() * op.num_heads() * op.seq_len() * op.head_dim();
let scores_size = op.batch_size() * op.num_heads() * op.seq_len() * op.seq_len();
let scores_buffer = device.create_buffer_f32(scores_size)?;
let weights_buffer = device.create_buffer_f32(scores_size)?;
let output_buffer = device.create_buffer_f32(input_size)?;
let params = AttentionParams {
batch_size: op.batch_size() as u32,
num_heads: op.num_heads() as u32,
q_seq_len: op.seq_len() as u32,
kv_seq_len: op.seq_len() as u32,
head_dim: op.head_dim() as u32,
_padding: [0; 3],
};
let params_buffer = device
.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Attention Params"),
contents: bytemuck::cast_slice(&[params]),
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
});
let caps = DeviceCapabilities::from_device(device);
const TILE_SIZE: u32 = 16;
let workgroups_x = (op.seq_len() as u32).div_ceil(TILE_SIZE).max(1);
let workgroups_y = (op.seq_len() as u32).div_ceil(TILE_SIZE).max(1);
let workgroups_z = (op.batch_size() * op.num_heads()) as u32;
ComputeDispatch::new(device, "sdp_attn_matmul")
.shader(ScaledDotProductAttention::wgsl_shader_matmul(), "main")
.storage_read(0, op.query().buffer())
.storage_read(1, op.key().buffer())
.storage_rw(2, &scores_buffer)
.uniform(3, ¶ms_buffer)
.dispatch(workgroups_x, workgroups_y, workgroups_z)
.submit()?;
let total_rows = op.batch_size() * op.num_heads() * op.seq_len();
let softmax_workgroups =
(total_rows as u32).div_ceil(caps.optimal_workgroup_size(WorkloadType::MatMul));
ComputeDispatch::new(device, "sdp_attn_softmax")
.shader(ScaledDotProductAttention::wgsl_shader_softmax(), "main")
.storage_read(0, &scores_buffer)
.storage_rw(1, &weights_buffer)
.uniform(2, ¶ms_buffer)
.dispatch(softmax_workgroups, 1, 1)
.submit()?;
let apply_wg_x = (op.head_dim() as u32).div_ceil(TILE_SIZE).max(1);
let apply_wg_y = (op.seq_len() as u32).div_ceil(TILE_SIZE).max(1);
let apply_wg_z = (op.batch_size() * op.num_heads()) as u32;
ComputeDispatch::new(device, "sdp_attn_apply")
.shader(ScaledDotProductAttention::wgsl_shader_apply(), "main")
.storage_read(0, &weights_buffer)
.storage_read(1, op.value().buffer())
.storage_rw(2, &output_buffer)
.uniform(3, ¶ms_buffer)
.dispatch(apply_wg_x, apply_wg_y, apply_wg_z)
.submit()?;
Ok(Tensor::from_buffer(
output_buffer,
vec![op.batch_size(), op.num_heads(), op.seq_len(), op.head_dim()],
device.clone(),
))
}