#![allow(dead_code)]
use crate::attention::MultiHeadAttention;
use crate::cuda_training::cuda_err;
use crate::gpu_model::{Act, FlashShape, Gpu};
use crate::gpu_transformer::{
gemm_lhs_transposed_batched, gemm_plain_batched, gemm_rhs_transposed_batched,
};
use crate::network::NetworkError;
use cudarc::driver::{CudaSlice, CudaView};
#[derive(Clone, Copy, Debug)]
pub(crate) struct CrossShape {
pub(crate) q_len: usize,
pub(crate) kv_len: usize,
pub(crate) sequences: usize,
}
impl CrossShape {
fn q_rows(&self) -> usize {
self.q_len * self.sequences
}
fn kv_rows(&self) -> usize {
self.kv_len * self.sequences
}
}
pub(crate) struct CrossCache {
queries_in: Act,
kv_in: Act,
queries: Act,
kv: Act,
query_weight: std::sync::Arc<Act>,
kv_weights: std::sync::Arc<Act>,
output_weight: std::sync::Arc<Act>,
merged: Act,
log_sum_exp: CudaSlice<f32>,
shape: CrossShape,
}
pub(crate) fn forward(
gpu: &Gpu<'_>,
attention: &MultiHeadAttention,
queries_in: &CudaView<'_, f32>,
kv_in: &CudaView<'_, f32>,
shape: CrossShape,
) -> Result<(CudaSlice<f32>, CrossCache), NetworkError> {
let plan = Plan::new(attention, shape)?;
let (q_rows, kv_rows) = (shape.q_rows(), shape.kv_rows());
let fused = gpu
.context
.flash
.as_ref()
.filter(|_| crate::cuda_flash::eligible(gpu.context.mixed_precision, plan.head_dim));
let narrow = fused.is_some();
let queries_in = gpu.narrowed(queries_in, q_rows * plan.d_model, narrow)?;
let kv_in = gpu.narrowed(kv_in, kv_rows * plan.d_kv, narrow)?;
let query_weight = gpu.pack(&[&attention.query], narrow)?;
let kv_weights = gpu.pack(&[&attention.key, &attention.value], narrow)?;
let output_weight = gpu.pack(&[&attention.output], narrow)?;
let mut queries = gpu.act(q_rows * plan.query_width, narrow)?;
gpu.linear_packed_act(
&query_weight.all(),
plan.query_width,
plan.d_model,
&queries_in.all(),
narrow,
&mut queries,
q_rows,
)?;
let mut kv = gpu.act(kv_rows * plan.kv_stride, narrow)?;
gpu.linear_packed_act(
&kv_weights.all(),
plan.kv_stride,
plan.d_kv,
&kv_in.all(),
narrow,
&mut kv,
kv_rows,
)?;
let mut log_sum_exp = gpu.uninit(plan.heads * q_rows)?;
let mut merged = gpu.act(q_rows * plan.query_width, narrow)?;
if let Some(flash) = fused {
gpu.flash_attention(
flash,
&queries,
&kv,
&mut merged,
&mut log_sum_exp,
plan.flash_shape(shape),
plan.scale,
)?;
} else {
let mut probabilities = gpu.uninit(plan.heads * shape.sequences * plan.block_size)?;
plan.scores(gpu, &queries, &kv, &mut probabilities, shape, plan.scale)?;
gpu.softmax_lse(
&mut probabilities,
&mut log_sum_exp,
plan.heads * shape.sequences * shape.q_len,
shape.kv_len,
false,
)?;
let kv_view = kv.wide();
let mut wide_merged = merged.wide_mut();
for head in 0..plan.heads {
let head_probabilities =
probabilities.slice(head * shape.sequences * plan.block_size..);
let value = kv_view.slice(plan.value_base + plan.kv_base(head)..);
let mut head_merged = wide_merged.slice_mut(head * plan.head_dim..);
gemm_plain_batched(
gpu.context,
&head_probabilities,
shape.kv_len,
&value,
plan.kv_stride,
&mut head_merged,
plan.query_width,
shape.q_len,
plan.head_dim,
shape.kv_len,
1.0,
0.0,
shape.sequences,
plan.block_size,
shape.kv_len * plan.kv_stride,
shape.q_len * plan.query_width,
)?;
}
}
let mut output = gpu.uninit(q_rows * plan.d_model)?;
gpu.linear_packed(
&output_weight.all(),
plan.d_model,
plan.query_width,
&merged.all(),
narrow,
&mut output,
q_rows,
0.0,
)?;
Ok((
output,
CrossCache {
queries_in,
kv_in,
queries,
kv,
query_weight,
kv_weights,
output_weight,
merged,
log_sum_exp,
shape,
},
))
}
pub(crate) fn backward(
gpu: &Gpu<'_>,
attention: &mut MultiHeadAttention,
cache: &CrossCache,
grad_output: &CudaSlice<f32>,
) -> Result<(CudaSlice<f32>, CudaSlice<f32>), NetworkError> {
let shape = cache.shape;
let plan = Plan::new(attention, shape)?;
let (q_rows, kv_rows) = (shape.q_rows(), shape.kv_rows());
let narrow = cache.queries.is_narrow();
let grad_output = gpu.narrowed(&grad_output.slice(..), q_rows * plan.d_model, narrow)?;
let mut grad_merged = gpu.act(q_rows * plan.query_width, narrow)?;
gpu.linear_packed_backward_input(
&cache.output_weight.all(),
plan.d_model,
plan.query_width,
&grad_output.all(),
narrow,
narrow,
grad_merged.destination(),
q_rows,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut attention.output,
&grad_output.all(),
&cache.merged.all(),
narrow,
q_rows,
)?;
let mut grad_queries = gpu.act(q_rows * plan.query_width, narrow)?;
let mut grad_kv = gpu.act(kv_rows * plan.kv_stride, narrow)?;
if narrow {
let flash = gpu
.context
.flash
.as_ref()
.expect("the forward pass ran fused");
let mut delta = gpu.uninit(plan.heads * q_rows)?;
gpu.flash_attention_backward(
flash,
&cache.queries,
&cache.kv,
&cache.merged,
&grad_merged,
&cache.log_sum_exp,
&mut delta,
&grad_queries,
&grad_kv,
plan.flash_shape(shape),
plan.scale,
)?;
} else {
let mut probabilities = gpu.uninit(plan.heads * shape.sequences * plan.block_size)?;
plan.scores(
gpu,
&cache.queries,
&cache.kv,
&mut probabilities,
shape,
plan.scale,
)?;
gpu.probs_from_lse(
&mut probabilities,
&cache.log_sum_exp,
plan.heads * shape.sequences * shape.q_len,
shape.kv_len,
false,
)?;
let mut grad_scores = gpu.uninit(plan.heads * shape.sequences * plan.block_size)?;
{
let mut wide_grad_kv = grad_kv.wide_mut();
gpu.context
.stream
.memset_zeros(&mut wide_grad_kv)
.map_err(cuda_err("gradient clear"))?;
let kv_view = cache.kv.wide();
let queries_view = cache.queries.wide();
let grad_merged_view = grad_merged.wide();
let mut wide_grad_queries = grad_queries.wide_mut();
for head in 0..plan.heads {
let kv_base = plan.kv_base(head);
let upstream = grad_merged_view.slice(head * plan.head_dim..);
let value = kv_view.slice(plan.value_base + kv_base..);
let mut scores = grad_scores.slice_mut(head * shape.sequences * plan.block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&upstream,
plan.query_width,
&value,
plan.kv_stride,
&mut scores,
shape.kv_len,
shape.q_len,
shape.kv_len,
plan.head_dim,
1.0,
0.0,
shape.sequences,
shape.q_len * plan.query_width,
shape.kv_len * plan.kv_stride,
plan.block_size,
)?;
let head_probabilities =
probabilities.slice(head * shape.sequences * plan.block_size..);
let mut grad_value = wide_grad_kv.slice_mut(plan.value_base + kv_base..);
gemm_lhs_transposed_batched(
gpu.context,
&head_probabilities,
shape.kv_len,
&upstream,
plan.query_width,
&mut grad_value,
plan.kv_stride,
shape.q_len,
shape.kv_len,
plan.head_dim,
1.0,
1.0,
shape.sequences,
plan.block_size,
shape.q_len * plan.query_width,
shape.kv_len * plan.kv_stride,
)?;
}
gpu.softmax_backward(
&mut grad_scores,
&probabilities,
plan.heads * shape.sequences * shape.q_len,
shape.kv_len,
false,
)?;
for head in 0..plan.heads {
let kv_base = plan.kv_base(head);
let scores = grad_scores.slice(head * shape.sequences * plan.block_size..);
let key = kv_view.slice(plan.key_base + kv_base..);
let mut grad_query = wide_grad_queries.slice_mut(head * plan.head_dim..);
gemm_plain_batched(
gpu.context,
&scores,
shape.kv_len,
&key,
plan.kv_stride,
&mut grad_query,
plan.query_width,
shape.q_len,
plan.head_dim,
shape.kv_len,
plan.scale,
0.0,
shape.sequences,
plan.block_size,
shape.kv_len * plan.kv_stride,
shape.q_len * plan.query_width,
)?;
let query = queries_view.slice(head * plan.head_dim..);
let mut grad_key = wide_grad_kv.slice_mut(plan.key_base + kv_base..);
gemm_lhs_transposed_batched(
gpu.context,
&scores,
shape.kv_len,
&query,
plan.query_width,
&mut grad_key,
plan.kv_stride,
shape.q_len,
shape.kv_len,
plan.head_dim,
plan.scale,
1.0,
shape.sequences,
plan.block_size,
shape.q_len * plan.query_width,
shape.kv_len * plan.kv_stride,
)?;
}
}
}
let mut grad_queries_in = gpu.uninit(q_rows * plan.d_model)?;
gpu.linear_packed_backward_input(
&cache.query_weight.all(),
plan.query_width,
plan.d_model,
&grad_queries.all(),
narrow,
false,
&mut grad_queries_in,
q_rows,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [&mut attention.query],
&grad_queries.all(),
&cache.queries_in.all(),
narrow,
q_rows,
)?;
let mut grad_kv_in = gpu.uninit(kv_rows * plan.d_kv)?;
gpu.linear_packed_backward_input(
&cache.kv_weights.all(),
plan.kv_stride,
plan.d_kv,
&grad_kv.all(),
narrow,
false,
&mut grad_kv_in,
kv_rows,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [&mut attention.key, &mut attention.value],
&grad_kv.all(),
&cache.kv_in.all(),
narrow,
kv_rows,
)?;
Ok((grad_queries_in, grad_kv_in))
}
struct Plan {
d_model: usize,
d_kv: usize,
heads: usize,
group: usize,
head_dim: usize,
query_width: usize,
kv_stride: usize,
key_base: usize,
value_base: usize,
block_size: usize,
scale: f32,
}
impl Plan {
fn new(attention: &MultiHeadAttention, shape: CrossShape) -> Result<Self, NetworkError> {
if shape.q_len == 0 || shape.kv_len == 0 || shape.sequences == 0 {
return Err(NetworkError::InvalidConfig(format!(
"cross-attention needs a non-empty shape, not {} x {} over {} sequences",
shape.q_len, shape.kv_len, shape.sequences
)));
}
if attention.is_rope_enabled() || attention.is_causal() {
return Err(NetworkError::InvalidConfig(
"device cross-attention wants a layer built with `MultiHeadAttention::cross`: \
rotary positions and the causal mask both have to be off"
.into(),
));
}
let head_dim = attention.head_dim();
let kv_width = attention.num_kv_heads() * head_dim;
Ok(Self {
d_model: attention.d_model(),
d_kv: attention.kv_features(),
heads: attention.num_heads(),
group: attention.num_heads() / attention.num_kv_heads(),
head_dim,
query_width: attention.num_heads() * head_dim,
kv_stride: 2 * kv_width,
key_base: 0,
value_base: kv_width,
block_size: shape.q_len * shape.kv_len,
scale: (head_dim as f32).sqrt().recip(),
})
}
fn kv_base(&self, head: usize) -> usize {
(head / self.group) * self.head_dim
}
fn flash_shape(&self, shape: CrossShape) -> FlashShape {
FlashShape {
rows: shape.q_rows(),
seq_len: shape.q_len,
kv_len: shape.kv_len,
sequences: shape.sequences,
heads: self.heads,
group: self.group,
qkv_width: self.query_width,
kv_width: self.kv_stride,
query_width: self.query_width,
key_base: self.key_base,
value_base: self.value_base,
causal: false,
}
}
fn scores(
&self,
gpu: &Gpu<'_>,
queries: &Act,
kv: &Act,
out: &mut CudaSlice<f32>,
shape: CrossShape,
alpha: f32,
) -> Result<(), NetworkError> {
let queries = queries.wide();
let kv = kv.wide();
for head in 0..self.heads {
let query = queries.slice(head * self.head_dim..);
let key = kv.slice(self.key_base + self.kv_base(head)..);
let mut head_scores = out.slice_mut(head * shape.sequences * self.block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
self.query_width,
&key,
self.kv_stride,
&mut head_scores,
shape.kv_len,
shape.q_len,
shape.kv_len,
self.head_dim,
alpha,
0.0,
shape.sequences,
shape.q_len * self.query_width,
shape.kv_len * self.kv_stride,
self.block_size,
)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cuda_training::cuda_doctor;
use crate::gpu_transformer::GpuContext;
use crate::matrix::Matrix;
use crate::rope::Rope;
use rand::SeedableRng;
use rand::rngs::StdRng;
use std::sync::Arc;
fn cuda_or_skip() -> Option<Arc<GpuContext>> {
match cuda_doctor(0, 8192) {
Ok(_) => Some(GpuContext::new(0).expect("the CUDA doctor passed")),
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
crate::cuda_training::skip_without_cuda();
None
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn ramp(rows: usize, cols: usize, offset: usize) -> Matrix {
Matrix::from_vec(
rows,
cols,
(0..rows * cols)
.map(|i| (((i + offset) * 37) % 23) as f32 / 11.0 - 1.0)
.collect(),
)
}
fn assert_close(label: &str, device: &[f32], host: &[f32], tolerance: f32) {
assert_eq!(device.len(), host.len(), "{label}: length");
let largest = host.iter().fold(1f32, |m, v| m.max(v.abs()));
let tolerance = tolerance * largest;
for (index, (a, b)) in device.iter().zip(host).enumerate() {
assert!(
(a - b).abs() <= tolerance,
"{label}[{index}]: {a} on the device vs {b} on the host"
);
}
}
#[test]
fn cross_attention_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
parity(&context, (8, 6, 4, 2, 4), (3, 5, 2), 1e-4);
}
#[test]
fn fused_cross_attention_matches_the_host_or_skips_without_device() {
if cuda_or_skip().is_none() {
return;
}
let context = GpuContext::with_precision(0, true).unwrap();
for lengths in [(70, 150, 2), (150, 70, 2)] {
parity(&context, (48, 40, 4, 2, 64), lengths, 6e-3);
}
}
fn parity(
context: &Arc<GpuContext>,
(d_model, d_kv, heads, kv_heads, head_dim): (usize, usize, usize, usize, usize),
(q_len, kv_len, sequences): (usize, usize, usize),
tolerance: f32,
) {
let gpu = Gpu { context };
let shape = CrossShape {
q_len,
kv_len,
sequences,
};
let mut rng = StdRng::seed_from_u64(7);
let rope = Rope::new(head_dim, 64, 10_000.0).unwrap();
let mut host =
MultiHeadAttention::cross(d_model, d_kv, heads, kv_heads, head_dim, rope, &mut rng)
.unwrap();
let mut device = host.clone();
let queries_in = ramp(q_len * sequences, d_model, 0);
let kv_in = ramp(kv_len * sequences, d_kv, 5);
let grad_output = ramp(q_len * sequences, d_model, 11);
let (host_output, host_cache) = host
.forward_train_cross(&queries_in, &kv_in, q_len, kv_len)
.unwrap();
let (host_grad_queries, host_grad_kv) =
host.backward_cross(&host_cache, &grad_output).unwrap();
for param in device.params_mut() {
param.move_to_cuda(context).unwrap();
}
let device_queries_in = gpu.upload(&queries_in.data).unwrap();
let device_kv_in = gpu.upload(&kv_in.data).unwrap();
let device_grad_output = gpu.upload(&grad_output.data).unwrap();
let (output, cache) = forward(
&gpu,
&device,
&device_queries_in.slice(..),
&device_kv_in.slice(..),
shape,
)
.expect("the device forward pass");
assert_close(
"cross forward",
&gpu.download(&output).unwrap(),
&host_output.data,
tolerance,
);
let (grad_queries, grad_kv) = backward(&gpu, &mut device, &cache, &device_grad_output)
.expect("the device backward pass");
assert_close(
"cross grad_queries_in",
&gpu.download(&grad_queries).unwrap(),
&host_grad_queries.data,
tolerance,
);
assert_close(
"cross grad_kv_in",
&gpu.download(&grad_kv).unwrap(),
&host_grad_kv.data,
tolerance,
);
for param in device.params_mut() {
param.move_to_cpu().unwrap();
}
for (name, device, host) in [
("query", &device.query, &host.query),
("key", &device.key, &host.key),
("value", &device.value, &host.value),
("output", &device.output, &host.output),
] {
assert_close(
&format!("cross grad_{name}"),
&device.weight.grad.data,
&host.weight.grad.data,
tolerance,
);
}
}
#[test]
fn a_self_attention_layer_is_refused_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let mut rng = StdRng::seed_from_u64(1);
let rope = Rope::new(4, 64, 10_000.0).unwrap();
let layer = MultiHeadAttention::new(8, 2, 2, 4, rope, &mut rng).unwrap();
let queries = gpu.zeros(2 * 8).unwrap();
let keys = gpu.zeros(2 * 8).unwrap();
let shape = CrossShape {
q_len: 2,
kv_len: 2,
sequences: 1,
};
assert!(forward(&gpu, &layer, &queries.slice(..), &keys.slice(..), shape).is_err());
}
}