use crate::batch::TokenBatch;
use crate::cuda_training::{cfg, cuda_alloc_err, cuda_err};
use crate::ffn::SwiGlu;
use crate::gpu_transformer::{
DeviceParam, GpuContext, act_lhs_transposed, act_plain, act_rhs_transposed,
gemm_lhs_transposed, gemm_lhs_transposed_batched, gemm_plain, gemm_plain_batched,
gemm_rhs_transposed, gemm_rhs_transposed_batched,
};
use crate::matrix::Matrix;
use crate::moe::MoeLayer;
use crate::network::NetworkError;
use crate::param::{Linear, Lora, Param};
use crate::rope::Rope;
use crate::transformer::TransformerLm;
use crate::transformer_block::{FeedForward, TransformerBlock};
use cudarc::driver::{
CudaFunction, CudaModule, CudaSlice, CudaView, CudaViewMut, DevicePtr, DevicePtrMut,
PushKernelArg,
};
use half::bf16;
use std::sync::Arc;
pub(crate) struct ModelKernels {
rmsnorm_forward: CudaFunction,
rmsnorm_backward: CudaFunction,
rope: CudaFunction,
softmax_backward: CudaFunction,
probs_from_lse: CudaFunction,
softmax_lse_rows: CudaFunction,
swiglu_forward: CudaFunction,
swiglu_backward: CudaFunction,
adaln_modulate: CudaFunction,
adaln_modulate_backward: CudaFunction,
gate_residual: CudaFunction,
gate_residual_backward: CudaFunction,
softmax_lse: CudaFunction,
topk_gate: CudaFunction,
gather: CudaFunction,
gather_scaled: CudaFunction,
scatter_scaled: CudaFunction,
scatter: CudaFunction,
scatter_negated: CudaFunction,
row_dot: CudaFunction,
stats: CudaFunction,
grad_probabilities: CudaFunction,
aux_grad: CudaFunction,
grad_logits: CudaFunction,
add: CudaFunction,
cross_entropy: CudaFunction,
cross_entropy_bf16: CudaFunction,
to_bf16: CudaFunction,
from_bf16: CudaFunction,
accumulate_bf16: CudaFunction,
cast_act: CudaFunction,
}
impl ModelKernels {
pub(crate) fn load(module: &Arc<CudaModule>) -> Result<Self, NetworkError> {
let get = |name: &str| {
module
.load_function(name)
.map_err(cuda_err("CUDA kernel lookup"))
};
Ok(Self {
rmsnorm_forward: get("rmsnorm_fwd")?,
rmsnorm_backward: get("rmsnorm_bwd")?,
rope: get("rope_rotate")?,
softmax_backward: get("attention_softmax_bwd")?,
probs_from_lse: get("attention_probs_from_lse")?,
softmax_lse_rows: get("attention_softmax_lse")?,
swiglu_forward: get("swiglu_fwd")?,
swiglu_backward: get("swiglu_bwd")?,
adaln_modulate: get("adaln_modulate")?,
adaln_modulate_backward: get("adaln_modulate_bwd")?,
gate_residual: get("gate_residual_fwd")?,
gate_residual_backward: get("gate_residual_bwd")?,
softmax_lse: get("softmax_lse")?,
topk_gate: get("topk_gate")?,
gather: get("gather_rows")?,
gather_scaled: get("gather_scale_rows")?,
scatter_scaled: get("scatter_add_scaled")?,
scatter: get("scatter_add_rows")?,
scatter_negated: get("scatter_rows_neg")?,
row_dot: get("row_dot_scatter")?,
stats: get("moe_stats")?,
grad_probabilities: get("moe_grad_probs")?,
aux_grad: get("moe_aux_grad")?,
grad_logits: get("router_grad_logits")?,
add: get("add_inplace")?,
cross_entropy: get("ce_loss_grad")?,
cross_entropy_bf16: get("ce_loss_grad_bf16")?,
to_bf16: get("to_bf16")?,
from_bf16: get("from_bf16")?,
accumulate_bf16: get("accumulate_bf16")?,
cast_act: get("cast_act")?,
})
}
}
#[derive(Clone, Copy)]
pub(crate) struct FlashShape {
pub(crate) rows: usize,
pub(crate) seq_len: usize,
pub(crate) sequences: usize,
pub(crate) heads: usize,
pub(crate) group: usize,
pub(crate) qkv_width: usize,
pub(crate) query_width: usize,
pub(crate) key_base: usize,
pub(crate) value_base: usize,
pub(crate) causal: bool,
}
pub(crate) struct DeviceRope {
cos: CudaSlice<f32>,
sin: CudaSlice<f32>,
head_dim: usize,
max_seq_len: usize,
}
pub(crate) struct GpuStack {
rows: usize,
seq_len: usize,
sequences: usize,
valid_tokens: usize,
blocks: Vec<BlockCache>,
auxiliary_loss: f32,
}
impl std::fmt::Debug for GpuStack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GpuStack")
.field("rows", &self.rows)
.field("seq_len", &self.seq_len)
.finish()
}
}
pub(crate) struct GpuCache {
stack: GpuStack,
ids: CudaSlice<u32>,
final_input: CudaSlice<f32>,
final_inverse_rms: CudaSlice<f32>,
final_weight: CudaSlice<f32>,
final_output: Act,
}
impl std::fmt::Debug for GpuCache {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.stack.fmt(f)
}
}
impl GpuCache {
pub(crate) fn auxiliary_loss(&self) -> f32 {
self.stack.auxiliary_loss
}
}
pub(crate) struct Act {
bytes: CudaSlice<u8>,
narrow: bool,
len: usize,
}
impl Act {
pub(crate) fn element(narrow: bool) -> usize {
if narrow { 2 } else { 4 }
}
pub(crate) fn is_narrow(&self) -> bool {
self.narrow
}
pub(crate) fn flag(&self) -> i32 {
i32::from(self.narrow)
}
pub(crate) fn at(&self, offset: usize) -> CudaView<'_, u8> {
self.bytes.slice(offset * Act::element(self.narrow)..)
}
pub(crate) fn all(&self) -> CudaView<'_, u8> {
self.bytes.slice(..)
}
pub(crate) fn wide(&self) -> CudaView<'_, f32> {
debug_assert!(!self.narrow, "a narrow activation is not FP32");
unsafe { self.bytes.transmute::<f32>(self.len) }
.expect("a device allocation is aligned for FP32")
}
pub(crate) fn wide_mut(&mut self) -> CudaViewMut<'_, f32> {
debug_assert!(!self.narrow, "a narrow activation is not FP32");
unsafe { self.bytes.transmute_mut::<f32>(self.len) }
.expect("a device allocation is aligned for FP32")
}
pub(crate) fn destination(&mut self) -> &mut CudaSlice<u8> {
&mut self.bytes
}
}
struct BlockCache {
input: CudaSlice<f32>,
attention_weight: CudaSlice<f32>,
attention_inverse_rms: CudaSlice<f32>,
attention_normed: Act,
qkv: Act,
qkv_weights: Act,
log_sum_exp: CudaSlice<f32>,
merged: Act,
output_weight: Act,
residual: CudaSlice<f32>,
feed_forward_weight: CudaSlice<f32>,
feed_forward_inverse_rms: CudaSlice<f32>,
feed_forward_normed: Act,
feed_forward: FfnCache,
}
pub(crate) struct SwiGluCache {
pub(crate) gate_up: CudaSlice<f32>,
pub(crate) hidden: Act,
pub(crate) weights: Act,
pub(crate) down: Option<Act>,
}
type NormOutput = (Act, CudaSlice<f32>, Option<CudaSlice<f32>>);
#[allow(clippy::large_enum_variant)]
enum FfnCache {
Dense(SwiGluCache),
Moe(Box<MoeGpuCache>),
}
struct MoeGpuCache {
probabilities: CudaSlice<f32>,
log_sum_exp: CudaSlice<f32>,
expert_of: CudaSlice<i32>,
token_of: CudaSlice<u32>,
slot_of: CudaSlice<u32>,
gates: CudaSlice<f32>,
offsets: Vec<usize>,
counts: Vec<usize>,
routed: usize,
input: Act,
expert: SwiGluCache,
output: CudaSlice<f32>,
shared: Option<SwiGluCache>,
}
pub(crate) struct Gpu<'a> {
pub(crate) context: &'a Arc<GpuContext>,
}
impl Gpu<'_> {
pub(crate) fn zeros(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
self.context
.stream
.alloc_zeros::<f32>(len.max(1))
.map_err(cuda_alloc_err("device allocation", len.max(1) * 4))
}
pub(crate) fn uninit(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
unsafe { self.context.stream.alloc::<f32>(len.max(1)) }
.map_err(cuda_alloc_err("device allocation", len.max(1) * 4))
}
pub(crate) fn uninit_bf16(&self, len: usize) -> Result<CudaSlice<bf16>, NetworkError> {
unsafe { self.context.stream.alloc::<bf16>(len.max(1)) }
.map_err(cuda_alloc_err("device allocation", len.max(1) * 2))
}
pub(crate) fn act(&self, len: usize, narrow: bool) -> Result<Act, NetworkError> {
let bytes = unsafe {
self.context
.stream
.alloc::<u8>(len.max(1) * Act::element(narrow))
}
.map_err(cuda_alloc_err(
"device allocation",
len.max(1) * Act::element(narrow),
))?;
Ok(Act { bytes, narrow, len })
}
pub(crate) fn cast_into(
&self,
dst: &mut CudaViewMut<'_, u8>,
src: &CudaView<'_, f32>,
n: usize,
narrow: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.cast_act)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.arg(&i32::from(narrow))
.launch(cfg(n))
.map_err(cuda_err("activation cast kernel"))?;
}
Ok(())
}
pub(crate) fn narrowed(
&self,
src: &CudaView<'_, f32>,
len: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
let mut act = self.act(len, narrow)?;
let mut dst = act.bytes.slice_mut(..);
self.cast_into(&mut dst, src, len, narrow)?;
Ok(act)
}
pub(crate) fn cast_to_bf16(
&self,
dst: &mut CudaSlice<bf16>,
src: &CudaView<'_, f32>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.to_bf16)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 cast kernel"))?;
}
Ok(())
}
pub(crate) fn cast_from_bf16(
&self,
dst: &mut CudaViewMut<'_, f32>,
src: &CudaSlice<bf16>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.from_bf16)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 widening kernel"))?;
}
Ok(())
}
pub(crate) fn accumulate_bf16(
&self,
acc: &mut CudaSlice<f32>,
src: &CudaSlice<bf16>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.accumulate_bf16)
.arg(acc)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 accumulate kernel"))?;
}
Ok(())
}
pub(crate) fn upload(&self, data: &[f32]) -> Result<CudaSlice<f32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn upload_indices(&self, data: &[u32]) -> Result<CudaSlice<u32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn upload_flags(&self, data: &[i32]) -> Result<CudaSlice<i32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn download(&self, source: &CudaSlice<f32>) -> Result<Vec<f32>, NetworkError> {
self.context
.stream
.clone_dtoh(source)
.map_err(cuda_err("device to host copy"))
}
#[cfg(test)]
pub(crate) fn download_act(&self, source: &Act) -> Result<Vec<f32>, NetworkError> {
if !source.is_narrow() {
return self
.context
.stream
.clone_dtoh(&source.wide())
.map_err(cuda_err("device to host copy"));
}
let bytes = self
.context
.stream
.clone_dtoh(&source.all())
.map_err(cuda_err("device to host copy"))?;
Ok(bytes
.chunks_exact(2)
.map(|half| f32::from_bits(u32::from(u16::from_le_bytes([half[0], half[1]])) << 16))
.collect())
}
pub(crate) fn upload_signed(&self, data: &[i32]) -> Result<CudaSlice<i32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn download_signed(
&self,
source: &CudaSlice<i32>,
) -> Result<Vec<i32>, NetworkError> {
self.context
.stream
.clone_dtoh(source)
.map_err(cuda_err("device to host copy"))
}
pub(crate) fn matrix(
&self,
source: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<Matrix, NetworkError> {
let mut matrix = Matrix::new(rows, width);
self.context
.stream
.memcpy_dtoh(source, &mut matrix.data)
.map_err(cuda_err("device to host copy"))?;
Ok(matrix)
}
pub(crate) fn accumulate_host_grad(param: &mut Param, values: &[f32]) {
for (slot, value) in param.grad.data.iter_mut().zip(values) {
*slot += value;
}
}
pub(crate) fn rope_tables(&self, rope: &Rope) -> Result<Arc<DeviceRope>, NetworkError> {
let mut slot = self
.context
.rope
.lock()
.map_err(|_| NetworkError::Cuda("the rope table lock was poisoned".into()))?;
if let Some(tables) = slot.as_ref()
&& tables.head_dim == rope.head_dim()
&& tables.max_seq_len == rope.max_seq_len()
{
return Ok(tables.clone());
}
let tables = Arc::new(DeviceRope {
cos: self.upload(rope.cos())?,
sin: self.upload(rope.sin())?,
head_dim: rope.head_dim(),
max_seq_len: rope.max_seq_len(),
});
*slot = Some(tables.clone());
Ok(tables)
}
pub(crate) fn linear<X: DevicePtr<f32>>(
&self,
weight: &DeviceParam,
x: &X,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * weight.rows())?;
gemm_rhs_transposed(
self.context,
x,
weight.cols(),
weight.value(),
weight.cols(),
&mut out,
weight.rows(),
rows,
weight.rows(),
weight.cols(),
1.0,
0.0,
)?;
Ok(out)
}
pub(crate) fn linear_backward_input<G: DevicePtr<f32>, O: DevicePtrMut<f32>>(
&self,
weight: &DeviceParam,
grad_output: &G,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
gemm_plain(
self.context,
grad_output,
weight.rows(),
weight.value(),
weight.cols(),
out,
weight.cols(),
rows,
weight.cols(),
weight.rows(),
1.0,
beta,
)
}
pub(crate) fn accumulate_weight_grad<G: DevicePtr<f32>, X: DevicePtr<f32>>(
&self,
weight: &mut DeviceParam,
grad_output: &G,
input: &X,
rows: usize,
) -> Result<(), NetworkError> {
if weight.is_frozen() {
return Ok(());
}
let (units, cols) = (weight.rows(), weight.cols());
let beta = weight.grad_beta();
gemm_lhs_transposed(
self.context,
grad_output,
units,
input,
cols,
weight.negated_grad_mut(),
cols,
rows,
units,
cols,
-1.0,
beta,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn rmsnorm(
&self,
input: &CudaSlice<f32>,
weight: &CudaSlice<f32>,
rows: usize,
cols: usize,
eps: f32,
narrow: bool,
copy: bool,
) -> Result<NormOutput, NetworkError> {
let mut out = self.act(rows * cols, narrow)?;
let mut inverse = self.uninit(rows)?;
let mut duplicate = self.uninit(if copy { rows * cols } else { 1 })?;
let do_copy = i32::from(copy);
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rmsnorm_forward)
.arg(out.destination())
.arg(&mut inverse)
.arg(input)
.arg(weight)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&eps)
.arg(&i32::from(narrow))
.arg(&mut duplicate)
.arg(&do_copy)
.launch(row_grid(rows))
.map_err(cuda_err("RMSNorm kernel"))?;
}
Ok((out, inverse, copy.then_some(duplicate)))
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments)]
pub(crate) fn rmsnorm_backward(
&self,
input: &CudaSlice<f32>,
grad_output: &CudaSlice<f32>,
weight: &CudaSlice<f32>,
inverse: &CudaSlice<f32>,
grad_weight: &mut CudaViewMut<'_, f32>,
residual: &CudaSlice<f32>,
add_residual: bool,
copy: Option<bool>,
rows: usize,
cols: usize,
) -> Result<(CudaSlice<f32>, Option<Act>), NetworkError> {
let mut grad_input = self.uninit(rows * cols)?;
let add_residual = i32::from(add_residual);
let copy_mode = copy.map_or(0, |narrow| 1 + i32::from(narrow));
let mut copy = self.act(copy.map_or(1, |_| rows * cols), copy.unwrap_or(false))?;
let smem = rmsnorm_smem(cols);
let mut config = row_grid(rows);
config.shared_mem_bytes = smem.unwrap_or(0);
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rmsnorm_backward)
.arg(&mut grad_input)
.arg(grad_weight)
.arg(input)
.arg(grad_output)
.arg(weight)
.arg(inverse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(i32::from(smem.is_some())))
.arg(residual)
.arg(&add_residual)
.arg(&mut copy.bytes)
.arg(©_mode)
.launch(config)
.map_err(cuda_err("RMSNorm backward kernel"))?;
}
Ok((grad_input, (copy_mode != 0).then_some(copy)))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn rope(
&self,
tensor: &mut Act,
tables: &DeviceRope,
rows: usize,
heads: usize,
head_dim: usize,
seq_len: usize,
direction: f32,
width: usize,
offset: usize,
) -> Result<(), NetworkError> {
let elements = rows * heads * head_dim / 2;
let narrow = tensor.flag();
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rope)
.arg(tensor.destination())
.arg(&tables.cos)
.arg(&tables.sin)
.arg(&(rows as i32))
.arg(&(heads as i32))
.arg(&(head_dim as i32))
.arg(&(seq_len as i32))
.arg(&direction)
.arg(&(width as i32))
.arg(&(offset as i32))
.arg(&narrow)
.launch(cfg(elements))
.map_err(cuda_err("RoPE kernel"))?;
}
Ok(())
}
pub(crate) fn softmax_backward(
&self,
grad: &mut CudaSlice<f32>,
probabilities: &CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.softmax_backward)
.arg(grad)
.arg(probabilities)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&i32::from(causal))
.launch(warp_per_row_grid(rows))
.map_err(cuda_err("attention softmax backward kernel"))?;
}
Ok(())
}
pub(crate) fn softmax_lse(
&self,
scores: &mut CudaSlice<f32>,
lse: &mut CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.softmax_lse_rows)
.arg(scores)
.arg(lse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&i32::from(causal))
.launch(warp_per_row_grid(rows))
.map_err(cuda_err("attention softmax kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn flash_attention(
&self,
flash: &crate::cuda_flash::FlashKernels,
qkv: &Act,
merged: &mut Act,
log_sum_exp: &mut CudaSlice<f32>,
shape: FlashShape,
scale: f32,
) -> Result<(), NetworkError> {
debug_assert!(qkv.is_narrow() && merged.is_narrow());
let config = cudarc::driver::LaunchConfig {
grid_dim: (
shape.seq_len.div_ceil(crate::cuda_flash::TILE) as u32,
shape.heads as u32,
shape.sequences as u32,
),
block_dim: (crate::cuda_flash::THREADS, 1, 1),
shared_mem_bytes: crate::cuda_flash::SHARED_BYTES,
};
unsafe {
self.context
.stream
.launch_builder(&flash.forward)
.arg(&qkv.all())
.arg(merged.destination())
.arg(log_sum_exp)
.arg(&(shape.seq_len as i32))
.arg(&(shape.qkv_width as i32))
.arg(&(shape.query_width as i32))
.arg(&(shape.key_base as i32))
.arg(&(shape.value_base as i32))
.arg(&(shape.group as i32))
.arg(&(shape.rows as i32))
.arg(&scale)
.arg(&i32::from(shape.causal))
.launch(config)
.map_err(cuda_err("fused attention kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn flash_attention_backward(
&self,
flash: &crate::cuda_flash::FlashKernels,
qkv: &Act,
merged: &Act,
grad_merged: &Act,
log_sum_exp: &CudaSlice<f32>,
delta: &mut CudaSlice<f32>,
grad_qkv: &mut Act,
shape: FlashShape,
scale: f32,
) -> Result<(), NetworkError> {
let tiles = shape.seq_len.div_ceil(crate::cuda_flash::TILE) as u32;
let warps = shape.heads * shape.rows;
unsafe {
self.context
.stream
.launch_builder(&flash.delta)
.arg(&merged.all())
.arg(&grad_merged.all())
.arg(&mut *delta)
.arg(&(shape.rows as i32))
.arg(&(shape.heads as i32))
.arg(&(shape.query_width as i32))
.arg(&(shape.rows as i32))
.launch(warp_per_row_grid(warps))
.map_err(cuda_err("attention delta kernel"))?;
}
let delta = &*delta;
let grad_merged_view = grad_merged.all();
debug_assert!(qkv.is_narrow() && grad_qkv.is_narrow() && grad_merged.is_narrow());
let qkv = qkv.all();
let grad_qkv = grad_qkv.all();
let launch = |function, grid_y, shared_mem_bytes| {
let config = cudarc::driver::LaunchConfig {
grid_dim: (tiles, grid_y, shape.sequences as u32),
block_dim: (crate::cuda_flash::THREADS, 1, 1),
shared_mem_bytes,
};
unsafe {
self.context
.stream
.launch_builder(function)
.arg(&qkv)
.arg(&grad_merged_view)
.arg(log_sum_exp)
.arg(&*delta)
.arg(&grad_qkv)
.arg(&(shape.seq_len as i32))
.arg(&(shape.qkv_width as i32))
.arg(&(shape.query_width as i32))
.arg(&(shape.key_base as i32))
.arg(&(shape.value_base as i32))
.arg(&(shape.group as i32))
.arg(&(shape.rows as i32))
.arg(&scale)
.arg(&i32::from(shape.causal))
.launch(config)
.map_err(cuda_err("fused attention backward kernel"))
}
};
launch(
&flash.grad_query,
shape.heads as u32,
crate::cuda_flash::DQ_SHARED_BYTES,
)?;
launch(
&flash.grad_key_value,
(shape.heads / shape.group) as u32,
crate::cuda_flash::DKV_SHARED_BYTES,
)?;
Ok(())
}
pub(crate) fn probs_from_lse(
&self,
scores: &mut CudaSlice<f32>,
lse: &CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.probs_from_lse)
.arg(scores)
.arg(lse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&i32::from(causal))
.launch(cfg(rows * cols))
.map_err(cuda_err("attention probability kernel"))?;
}
Ok(())
}
pub(crate) fn add(
&self,
target: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
offset: usize,
len: usize,
keep: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.add)
.arg(target)
.arg(source)
.arg(&(offset as i32))
.arg(&(len as i32))
.arg(&i32::from(keep))
.launch(cfg(if (len | offset) % 4 == 0 {
len / 4
} else {
len
}))
.map_err(cuda_err("addition kernel"))?;
}
Ok(())
}
pub(crate) fn gather(
&self,
out: &mut CudaViewMut<'_, f32>,
source: &CudaView<'_, f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gather)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(&0i32)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("row gather kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gather_scaled(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
scale: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gather_scaled)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(scale)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("scaled row gather kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn scatter_scaled(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
scale: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter_scaled)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(scale)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("scaled row scatter kernel"))?;
}
Ok(())
}
pub(crate) fn scatter(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("row scatter kernel"))?;
}
Ok(())
}
pub(crate) fn scatter_negated(
&self,
grad: &mut CudaSlice<f32>,
upstream: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter_negated)
.arg(grad)
.arg(upstream)
.arg(rows_of)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("embedding scatter kernel"))?;
}
Ok(())
}
pub(crate) fn swiglu(
&self,
gate_up: &CudaSlice<f32>,
rows: usize,
width: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
let mut hidden = self.act(rows * width, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.swiglu_forward)
.arg(hidden.destination())
.arg(gate_up)
.arg(&(rows as i32))
.arg(&(width as i32))
.arg(&i32::from(narrow))
.launch(cfg(if width % 4 == 0 {
rows * width / 4
} else {
rows * width
}))
.map_err(cuda_err("SwiGLU kernel"))?;
}
Ok(hidden)
}
pub(crate) fn swiglu_backward(
&self,
gate_up: &CudaSlice<f32>,
grad_hidden: &CudaSlice<f32>,
rows: usize,
width: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
let mut grad_gate_up = self.act(rows * 2 * width, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.swiglu_backward)
.arg(grad_gate_up.destination())
.arg(gate_up)
.arg(grad_hidden)
.arg(&(rows as i32))
.arg(&(width as i32))
.arg(&i32::from(narrow))
.launch(cfg(if width % 4 == 0 {
rows * width / 4
} else {
rows * width
}))
.map_err(cuda_err("SwiGLU backward kernel"))?;
}
Ok(grad_gate_up)
}
pub(crate) fn adaln_modulate(
&self,
normalized: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.adaln_modulate)
.arg(&mut out)
.arg(normalized)
.arg(triple)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("AdaLN modulation kernel"))?;
}
Ok(out)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn adaln_modulate_backward(
&self,
grad_modulated: &CudaView<'_, f32>,
normalized: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
grad_triple: &mut CudaViewMut<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut grad_normalized = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.adaln_modulate_backward)
.arg(&mut grad_normalized)
.arg(grad_triple)
.arg(grad_modulated)
.arg(normalized)
.arg(triple)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("AdaLN modulation backward kernel"))?;
}
Ok(grad_normalized)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gate_residual(
&self,
residual: &CudaView<'_, f32>,
branch: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gate_residual)
.arg(&mut out)
.arg(residual)
.arg(branch)
.arg(triple)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("gated residual kernel"))?;
}
Ok(out)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gate_residual_backward(
&self,
branch: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
grad_output: &CudaView<'_, f32>,
grad_triple: &mut CudaViewMut<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut grad_branch = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gate_residual_backward)
.arg(&mut grad_branch)
.arg(grad_triple)
.arg(branch)
.arg(triple)
.arg(grad_output)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("gated residual backward kernel"))?;
}
Ok(grad_branch)
}
pub(crate) fn pack(&self, parts: &[&Linear], narrow: bool) -> Result<Act, NetworkError> {
let element = Act::element(narrow);
let mut total = 0;
for part in parts {
let device = device_of(part)?;
total += device.rows() * device.cols();
}
let mut packed = self.act(total, narrow)?;
let mut base = 0;
for part in parts {
let device = device_of(part)?;
let (units, inner) = (device.rows(), device.cols());
let len = units * inner;
let mut slot = packed
.bytes
.slice_mut(base * element..(base + len) * element);
self.cast_into(&mut slot, &device.value().slice(..), len, narrow)?;
if let Some(lora) = &part.lora {
self.fold_lora(lora, &mut slot, units, inner, narrow)?;
}
base += len;
}
Ok(packed)
}
pub(crate) fn fold_lora(
&self,
lora: &Lora,
slot: &mut CudaViewMut<'_, u8>,
units: usize,
inner: usize,
narrow: bool,
) -> Result<(), NetworkError> {
let down = device_param(&lora.down)?;
let up = device_param(&lora.up)?;
let rank = down.rows();
act_plain(
self.context,
&bytes_of(up.value(), units * rank),
rank,
&bytes_of(down.value(), rank * inner),
inner,
false,
narrow,
slot,
inner,
units,
inner,
rank,
lora.scale,
1.0,
)
}
pub(crate) fn pack_gate_up<'a>(
&self,
ffns: impl Iterator<Item = &'a SwiGlu>,
narrow: bool,
) -> Result<Act, NetworkError> {
let mut parts = Vec::new();
for ffn in ffns {
parts.push(&ffn.gate);
parts.push(&ffn.up);
}
self.pack(&parts, narrow)
}
pub(crate) fn pack_down<'a>(
&self,
ffns: impl Iterator<Item = &'a SwiGlu>,
narrow: bool,
) -> Result<Act, NetworkError> {
let parts: Vec<&Linear> = ffns.map(|ffn| &ffn.down).collect();
self.pack(&parts, narrow)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn linear_packed<O: DevicePtrMut<f32>>(
&self,
weights: &CudaView<'_, u8>,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
act_rhs_transposed(
self.context,
x,
inner,
weights,
inner,
narrow,
false,
out,
units,
rows,
units,
inner,
1.0,
beta,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn linear_packed_act(
&self,
weights: &CudaView<'_, u8>,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut Act,
rows: usize,
) -> Result<(), NetworkError> {
let out_narrow = out.is_narrow();
act_rhs_transposed(
self.context,
x,
inner,
weights,
inner,
narrow,
out_narrow,
out.destination(),
units,
rows,
units,
inner,
1.0,
0.0,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn linear_packed_backward_input<T, O: DevicePtrMut<T>>(
&self,
weights: &CudaView<'_, u8>,
units: usize,
inner: usize,
grad_output: &CudaView<'_, u8>,
narrow: bool,
out_narrow: bool,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
act_plain(
self.context,
grad_output,
units,
weights,
inner,
narrow,
out_narrow,
out,
inner,
rows,
inner,
units,
1.0,
beta,
)
}
pub(crate) fn accumulate_weight_grad_act(
&self,
weight: &mut DeviceParam,
grad_output: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
if weight.is_frozen() {
return Ok(());
}
let (units, cols) = (weight.rows(), weight.cols());
let beta = weight.grad_beta();
act_lhs_transposed(
self.context,
grad_output,
units,
input,
cols,
narrow,
weight.negated_grad_mut(),
cols,
rows,
units,
cols,
-1.0,
beta,
)
}
pub(crate) fn accumulate_packed_grad(
&self,
parts: &mut [&mut DeviceParam],
grad: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
if parts.iter().all(|part| part.is_frozen()) {
return Ok(());
}
debug_assert!(
parts.iter().all(|part| !part.is_frozen()),
"a packed projection freezes as a unit"
);
let inner = parts[0].cols();
let units: usize = parts.iter().map(|part| part.rows()).sum();
let mut scratch = self.uninit(units * inner)?;
act_lhs_transposed(
self.context,
grad,
units,
input,
inner,
narrow,
&mut scratch,
inner,
rows,
units,
inner,
-1.0,
0.0,
)?;
let mut base = 0;
for part in parts {
let len = part.rows() * inner;
let keep = part.grad_beta() != 0.0;
self.add(part.negated_grad_mut(), &scratch, base, len, keep)?;
base += len;
}
Ok(())
}
pub(crate) fn accumulate_projection_grad(
&self,
linear: &mut Linear,
grad_output: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
let units = linear.out_features();
self.accumulate_weight_grad_act(device_of_mut(linear)?, grad_output, input, narrow, rows)?;
match &mut linear.lora {
Some(lora) => self.accumulate_lora_grad(lora, grad_output, units, input, narrow, rows),
None => Ok(()),
}
}
pub(crate) fn accumulate_packed_projection_grad(
&self,
parts: &mut [&mut Linear],
grad_output: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
let stride: usize = parts.iter().map(|part| part.out_features()).sum();
{
let mut devices = Vec::with_capacity(parts.len());
for part in parts.iter_mut() {
devices.push(device_of_mut(part)?);
}
self.accumulate_packed_grad(&mut devices, grad_output, input, narrow, rows)?;
}
let element = Act::element(narrow);
let mut base = 0;
for part in parts.iter_mut() {
let units = part.out_features();
if let Some(lora) = &mut part.lora {
self.accumulate_lora_grad(
lora,
&grad_output.slice(base * element..),
stride,
input,
narrow,
rows,
)?;
}
base += units;
}
Ok(())
}
pub(crate) fn accumulate_lora_grad(
&self,
lora: &mut Lora,
grad_output: &CudaView<'_, u8>,
grad_stride: usize,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
let (rank, inner) = {
let down = device_param(&lora.down)?;
(down.rows(), down.cols())
};
let units = device_param(&lora.up)?.rows();
let down_act = self.narrowed(
&device_param(&lora.down)?.value().slice(..),
rank * inner,
narrow,
)?;
let up_act = self.narrowed(
&device_param(&lora.up)?.value().slice(..),
units * rank,
narrow,
)?;
let mut hidden = self.act(rows * rank, narrow)?;
act_rhs_transposed(
self.context,
input,
inner,
&down_act.all(),
inner,
narrow,
narrow,
hidden.destination(),
rank,
rows,
rank,
inner,
lora.scale,
0.0,
)?;
{
let up = device_param_mut(&mut lora.up)?;
let beta = up.grad_beta();
act_lhs_transposed(
self.context,
grad_output,
grad_stride,
&hidden.all(),
rank,
narrow,
up.negated_grad_mut(),
rank,
rows,
units,
rank,
-1.0,
beta,
)?;
}
let mut grad_hidden = self.act(rows * rank, narrow)?;
act_plain(
self.context,
grad_output,
grad_stride,
&up_act.all(),
rank,
narrow,
narrow,
grad_hidden.destination(),
rank,
rows,
rank,
units,
lora.scale,
0.0,
)?;
let down = device_param_mut(&mut lora.down)?;
let beta = down.grad_beta();
act_lhs_transposed(
self.context,
&grad_hidden.all(),
rank,
input,
inner,
narrow,
down.negated_grad_mut(),
inner,
rows,
rank,
inner,
-1.0,
beta,
)
}
}
fn bytes_of(slice: &CudaSlice<f32>, len: usize) -> CudaView<'_, u8> {
unsafe { slice.transmute::<u8>(len * 4) }.expect("a device allocation is byte-aligned")
}
fn device_param(param: &Param) -> Result<&DeviceParam, NetworkError> {
param
.device
.as_ref()
.ok_or_else(|| NetworkError::Cuda("a LoRA adapter is not resident on the device".into()))
}
fn device_param_mut(param: &mut Param) -> Result<&mut DeviceParam, NetworkError> {
param
.device
.as_mut()
.ok_or_else(|| NetworkError::Cuda("a LoRA adapter is not resident on the device".into()))
}
fn head_of(model: &TransformerLm) -> Result<&DeviceParam, NetworkError> {
match &model.lm_head {
Some(head) => device_of(head),
None => model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
}),
}
}
fn head_of_mut(model: &mut TransformerLm) -> Result<&mut DeviceParam, NetworkError> {
match &mut model.lm_head {
Some(head) => device_of_mut(head),
None => model.embedding.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
}),
}
}
pub(crate) fn device_of(linear: &Linear) -> Result<&DeviceParam, NetworkError> {
linear
.weight
.device
.as_ref()
.ok_or_else(|| NetworkError::Cuda("a projection is not resident on the device".into()))
}
pub(crate) fn device_of_mut(linear: &mut Linear) -> Result<&mut DeviceParam, NetworkError> {
linear
.weight
.device
.as_mut()
.ok_or_else(|| NetworkError::Cuda("a projection is not resident on the device".into()))
}
pub(crate) fn unsupported(kind: &str) -> NetworkError {
NetworkError::UnsupportedCuda(format!(
"the device training path implements SwiGLU feed-forwards and MoE layers, not {kind}"
))
}
pub(crate) fn forward(
model: &TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<(Matrix, GpuCache), NetworkError> {
let cache = forward_hidden(model, context, batch)?;
let gpu = Gpu { context };
let rows = cache.stack.rows;
let logits = gpu.linear(head_of(model)?, &cache.final_output.wide(), rows)?;
let logits = gpu.matrix(&logits, rows, model.embedding.vocab_size())?;
Ok((logits, cache))
}
fn forward_hidden(
model: &TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<GpuCache, NetworkError> {
let gpu = Gpu { context };
let rows = batch.rows();
let seq_len = batch.seq_len();
let sequences = batch.batch();
let d_model = model.config.d_model;
let vocab = model.embedding.vocab_size();
for &id in batch.ids() {
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
}
let layout = batch.layout();
let flags: Vec<i32> = (0..rows)
.map(|row| i32::from(layout.is_valid(row)))
.collect();
let valid_tokens = flags.iter().filter(|&&flag| flag != 0).count();
let valid = gpu.upload_flags(&flags)?;
let ids = gpu.upload_indices(batch.ids())?;
let embedding = model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
let mut hidden = gpu.uninit(rows * d_model)?;
gpu.gather(
&mut hidden.slice_mut(..),
&embedding.value().slice(..),
&ids,
rows,
d_model,
)?;
let (hidden, stack) = forward_stack(
&gpu,
&model.blocks,
hidden,
&valid,
valid_tokens,
rows,
seq_len,
sequences,
)?;
let final_weight = gpu.upload(&model.final_norm.weight.value.data)?;
let (final_output, final_inverse_rms, _) = gpu.rmsnorm(
&hidden,
&final_weight,
rows,
d_model,
model.final_norm.eps,
false,
false,
)?;
Ok(GpuCache {
stack,
ids,
final_input: hidden,
final_inverse_rms,
final_weight,
final_output,
})
}
#[allow(clippy::too_many_arguments)]
fn forward_stack(
gpu: &Gpu<'_>,
blocks: &[TransformerBlock],
mut hidden: CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
seq_len: usize,
sequences: usize,
) -> Result<(CudaSlice<f32>, GpuStack), NetworkError> {
let mut caches = Vec::with_capacity(blocks.len());
let mut auxiliary_loss = 0.0;
for block in blocks {
let (output, cache) = forward_block(
gpu,
block,
hidden,
valid,
valid_tokens,
rows,
seq_len,
sequences,
&mut auxiliary_loss,
)?;
caches.push(cache);
hidden = output;
}
Ok((
hidden,
GpuStack {
rows,
seq_len,
sequences,
valid_tokens,
blocks: caches,
auxiliary_loss,
},
))
}
pub(crate) fn stack_forward(
blocks: &[TransformerBlock],
context: &Arc<GpuContext>,
input: &Matrix,
seq_len: usize,
) -> Result<(Matrix, GpuStack), NetworkError> {
let gpu = Gpu { context };
let rows = input.rows;
let valid = gpu.upload_flags(&vec![1; rows])?;
let hidden = gpu.upload(&input.data)?;
let (hidden, stack) = forward_stack(
&gpu,
blocks,
hidden,
&valid,
rows,
rows,
seq_len,
rows / seq_len,
)?;
Ok((gpu.matrix(&hidden, rows, input.cols)?, stack))
}
pub(crate) fn stack_backward(
blocks: &mut [TransformerBlock],
context: &Arc<GpuContext>,
stack: &GpuStack,
grad_output: &Matrix,
) -> Result<Matrix, NetworkError> {
let gpu = Gpu { context };
let rows = stack.rows;
let d_model = grad_output.cols;
if grad_output.rows != rows {
return Err(NetworkError::InvalidInput {
expected: rows,
actual: grad_output.rows,
});
}
let grad = gpu.upload(&grad_output.data)?;
let grad_act = match stack_operand(stack, blocks.len().checked_sub(1)) {
Some(narrow) => Some(gpu.narrowed(&grad.slice(..), rows * d_model, narrow)?),
None => None,
};
let mut grad_norms = gpu.zeros(norm_slots(blocks.len()) * d_model)?;
let grad = backward_stack(&gpu, blocks, stack, grad, grad_act, &mut grad_norms)?;
let values = gpu.download(&grad_norms)?;
accumulate_block_norms(blocks, &values, d_model);
gpu.matrix(&grad, rows, d_model)
}
fn stack_operand(stack: &GpuStack, index: Option<usize>) -> Option<bool> {
stack
.blocks
.get(index?)
.and_then(|block| match &block.feed_forward {
FfnCache::Dense(_) => Some(block.feed_forward_normed.is_narrow()),
FfnCache::Moe(_) => None,
})
}
fn backward_stack(
gpu: &Gpu<'_>,
blocks: &mut [TransformerBlock],
stack: &GpuStack,
mut grad_hidden: CudaSlice<f32>,
mut grad_hidden_act: Option<Act>,
grad_norms: &mut CudaSlice<f32>,
) -> Result<CudaSlice<f32>, NetworkError> {
for (index, (block, block_cache)) in blocks.iter_mut().zip(&stack.blocks).enumerate().rev() {
(grad_hidden, grad_hidden_act) = backward_block(
gpu,
block,
block_cache,
&grad_hidden,
grad_hidden_act.as_ref(),
stack_operand(stack, index.checked_sub(1)),
stack,
grad_norms,
index,
)?;
}
Ok(grad_hidden)
}
fn accumulate_block_norms(blocks: &mut [TransformerBlock], values: &[f32], d_model: usize) {
let at = |slot: usize| &values[slot * d_model..(slot + 1) * d_model];
for (index, block) in blocks.iter_mut().enumerate() {
let (attention, feed_forward) = norm_slot(index);
Gpu::accumulate_host_grad(&mut block.attention_norm.weight, at(attention));
Gpu::accumulate_host_grad(&mut block.feed_forward_norm.weight, at(feed_forward));
}
}
#[allow(clippy::too_many_arguments)]
fn forward_block(
gpu: &Gpu<'_>,
block: &TransformerBlock,
input: CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
seq_len: usize,
sequences: usize,
auxiliary_loss: &mut f32,
) -> Result<(CudaSlice<f32>, BlockCache), NetworkError> {
let attention = &block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let causal = attention.is_causal();
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let narrow = gpu.context.mixed_precision;
let dense_ffn = matches!(block.feed_forward, FeedForward::SwiGlu(_));
let ffn_narrow = narrow && dense_ffn;
let attention_weight = gpu.upload(&block.attention_norm.weight.value.data)?;
let (attention_normed, attention_inverse_rms, residual) = gpu.rmsnorm(
&input,
&attention_weight,
rows,
d_model,
block.attention_norm.eps,
narrow,
true,
)?;
let mut residual = residual.expect("a norm asked for a duplicate returns one");
let kv_width = kv_heads * head_dim;
let qkv_width = query_width + 2 * kv_width;
let fused = gpu
.context
.flash
.as_ref()
.filter(|_| crate::cuda_flash::eligible(gpu.context.mixed_precision, head_dim));
let qkv_weights = gpu.pack(
&[&attention.query, &attention.key, &attention.value],
narrow,
)?;
let mut qkv = gpu.act(rows * qkv_width, narrow && fused.is_some())?;
gpu.linear_packed_act(
&qkv_weights.all(),
qkv_width,
d_model,
&attention_normed.all(),
narrow,
&mut qkv,
rows,
)?;
let key_base = query_width;
let value_base = query_width + kv_width;
let tables = gpu.rope_tables(&attention.rope)?;
if seq_len > tables.max_seq_len {
return Err(NetworkError::SequenceTooLong {
length: seq_len,
max_seq_len: tables.max_seq_len,
});
}
gpu.rope(
&mut qkv, &tables, rows, heads, head_dim, seq_len, 1.0, qkv_width, 0,
)?;
gpu.rope(
&mut qkv, &tables, rows, kv_heads, head_dim, seq_len, 1.0, qkv_width, key_base,
)?;
let group = heads / kv_heads;
let shape = FlashShape {
rows,
seq_len,
sequences,
heads,
group,
qkv_width,
query_width,
key_base,
value_base,
causal,
};
let mut log_sum_exp = gpu.uninit(heads * rows)?;
let mut merged = gpu.act(rows * query_width, narrow && fused.is_some())?;
if let Some(flash) = fused {
gpu.flash_attention(flash, &qkv, &mut merged, &mut log_sum_exp, shape, scale)?;
} else {
let mut wide_merged = merged.wide_mut();
let qkv = qkv.wide();
let block_size = seq_len * seq_len;
let mut probabilities = gpu.uninit(heads * sequences * block_size)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let query = qkv.slice(head * head_dim..);
let key = qkv.slice(key_base + kv_base..);
let mut scores = probabilities.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
qkv_width,
&key,
qkv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
scale,
0.0,
sequences,
seq_len * qkv_width,
seq_len * qkv_width,
block_size,
)?;
}
gpu.softmax_lse(
&mut probabilities,
&mut log_sum_exp,
heads * sequences * seq_len,
seq_len,
causal,
)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let head_probabilities = probabilities.slice(head * sequences * block_size..);
let value = qkv.slice(value_base + kv_base..);
let mut head_merged = wide_merged.slice_mut(head * head_dim..);
gemm_plain_batched(
gpu.context,
&head_probabilities,
seq_len,
&value,
qkv_width,
&mut head_merged,
query_width,
seq_len,
head_dim,
seq_len,
1.0,
0.0,
sequences,
block_size,
seq_len * qkv_width,
seq_len * query_width,
)?;
}
}
let output_weight = gpu.pack(&[&attention.output], merged.is_narrow())?;
gpu.linear_packed(
&output_weight.all(),
d_model,
query_width,
&merged.all(),
merged.is_narrow(),
&mut residual,
rows,
1.0,
)?;
let feed_forward_weight = gpu.upload(&block.feed_forward_norm.weight.value.data)?;
let (feed_forward_normed, feed_forward_inverse_rms, output) = gpu.rmsnorm(
&residual,
&feed_forward_weight,
rows,
d_model,
block.feed_forward_norm.eps,
ffn_narrow,
true,
)?;
let mut output = output.expect("a norm asked for a duplicate returns one");
let feed_forward = match &block.feed_forward {
FeedForward::SwiGlu(ffn) => FfnCache::Dense(forward_swiglu(
gpu,
ffn,
&feed_forward_normed,
&mut output,
rows,
)?),
FeedForward::Gelu(_) => return Err(unsupported("a GELU MLP")),
FeedForward::Moe(moe) => {
let (cache, losses) = forward_moe(
gpu,
moe,
&feed_forward_normed,
&mut output,
valid,
valid_tokens,
rows,
)?;
*auxiliary_loss += losses;
FfnCache::Moe(Box::new(cache))
}
};
Ok((
output,
BlockCache {
input,
attention_weight,
attention_inverse_rms,
attention_normed,
qkv,
qkv_weights,
log_sum_exp,
merged,
output_weight,
residual,
feed_forward_weight,
feed_forward_inverse_rms,
feed_forward_normed,
feed_forward,
},
))
}
pub(crate) fn forward_swiglu(
gpu: &Gpu<'_>,
ffn: &SwiGlu,
input: &Act,
out: &mut CudaSlice<f32>,
rows: usize,
) -> Result<SwiGluCache, NetworkError> {
let width = ffn.d_ff();
let inner = ffn.d_model();
let narrow = input.is_narrow();
let weights = gpu.pack_gate_up(std::iter::once(ffn), narrow)?;
let mut gate_up = gpu.uninit(rows * 2 * width)?;
gpu.linear_packed(
&weights.all(),
2 * width,
inner,
&input.all(),
narrow,
&mut gate_up,
rows,
0.0,
)?;
let hidden = gpu.swiglu(&gate_up, rows, width, narrow)?;
let down = gpu.pack(&[&ffn.down], narrow)?;
gpu.linear_packed(
&down.all(),
inner,
width,
&hidden.all(),
narrow,
out,
rows,
1.0,
)?;
Ok(SwiGluCache {
gate_up,
hidden,
weights,
down: Some(down),
})
}
#[allow(clippy::too_many_arguments)]
fn forward_moe(
gpu: &Gpu<'_>,
moe: &MoeLayer,
input: &Act,
out: &mut CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
) -> Result<(MoeGpuCache, f32), NetworkError> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let logits = gpu.linear(device_of(&moe.router.projection)?, &input.wide(), rows)?;
let mut probabilities = gpu.zeros(rows * experts)?;
let mut log_sum_exp = gpu.zeros(rows)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.softmax_lse)
.arg(&mut probabilities)
.arg(&mut log_sum_exp)
.arg(&logits)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.map_err(cuda_err("router softmax kernel"))?;
}
let mut expert_of = gpu
.context
.stream
.alloc_zeros::<i32>(rows * top_k)
.map_err(cuda_err("device allocation"))?;
let mut gate_of = gpu.zeros(rows * top_k)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.topk_gate)
.arg(&mut expert_of)
.arg(&mut gate_of)
.arg(&probabilities)
.arg(valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.map_err(cuda_err("router top-k kernel"))?;
}
let assigned = gpu.download_signed(&expert_of)?;
let gates_host = gpu.download(&gate_of)?;
let mut counts = vec![0usize; experts];
for &expert in &assigned {
if expert >= 0 {
counts[expert as usize] += 1;
}
}
let mut offsets = Vec::with_capacity(experts);
let mut routed = 0;
for &count in &counts {
offsets.push(routed);
routed += count;
}
let mut cursor = offsets.clone();
let mut token_host = vec![0u32; routed.max(1)];
let mut slot_host = vec![0u32; routed.max(1)];
let mut gate_host = vec![0f32; routed.max(1)];
for (slot, &expert) in assigned.iter().enumerate() {
if expert < 0 {
continue;
}
let position = &mut cursor[expert as usize];
token_host[*position] = (slot / top_k) as u32;
slot_host[*position] = slot as u32;
gate_host[*position] = gates_host[slot];
*position += 1;
}
let token_of = gpu.upload_indices(&token_host)?;
let slot_of = gpu.upload_indices(&slot_host)?;
let gates = gpu.upload(&gate_host)?;
let mut gathered = gpu.act(routed * d_model, false)?;
if routed > 0 {
gpu.gather(
&mut gathered.wide_mut(),
&input.wide(),
&token_of,
routed,
d_model,
)?;
}
let expert_weights = gpu.pack_gate_up(moe.experts.iter(), false)?;
let packed_stride = 2 * width * d_model;
let mut gate_up = gpu.uninit(routed * 2 * width)?;
for (expert, _) in moe.experts.iter().enumerate() {
let count = counts[expert];
if count == 0 {
continue;
}
let rows_in = gathered.at(offsets[expert] * d_model);
let weights = expert_weights.at(expert * packed_stride);
let mut out = gate_up.slice_mut(offsets[expert] * 2 * width..);
gpu.linear_packed(
&weights,
2 * width,
d_model,
&rows_in,
false,
&mut out,
count,
0.0,
)?;
}
let hidden = gpu.swiglu(&gate_up, routed, width, false)?;
let down_weights = gpu.pack_down(moe.experts.iter(), false)?;
let down_stride = d_model * width;
let mut expert_output = gpu.uninit(routed * d_model)?;
for (expert, _) in moe.experts.iter().enumerate() {
let count = counts[expert];
if count == 0 {
continue;
}
let rows_in = hidden.at(offsets[expert] * width);
let mut rows_out = expert_output.slice_mut(offsets[expert] * d_model..);
gpu.linear_packed(
&down_weights.at(expert * down_stride),
d_model,
width,
&rows_in,
false,
&mut rows_out,
count,
0.0,
)?;
}
if routed > 0 {
gpu.scatter_scaled(out, &expert_output, &token_of, &gates, routed, d_model)?;
}
let shared = match &moe.shared {
Some(module) => Some(forward_swiglu(gpu, module, input, out, rows)?),
None => None,
};
let mut sums = gpu.zeros(experts + 1)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.stats)
.arg(&mut sums)
.arg(&probabilities)
.arg(&log_sum_exp)
.arg(valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.map_err(cuda_err("MoE statistics kernel"))?;
}
let sums = gpu.download(&sums)?;
let mut auxiliary = 0.0;
if valid_tokens > 0 {
let slots = (valid_tokens * top_k) as f32;
let balance: f32 = (0..experts)
.map(|expert| (counts[expert] as f32 / slots) * (sums[expert] / valid_tokens as f32))
.sum();
auxiliary += experts as f32 * moe.config.aux_loss_weight * balance;
auxiliary += moe.config.router_z_loss_weight * sums[experts] / valid_tokens as f32;
}
Ok((
MoeGpuCache {
probabilities,
log_sum_exp,
expert_of,
token_of,
slot_of,
gates,
offsets,
counts,
routed,
input: gathered,
expert: SwiGluCache {
gate_up,
hidden,
weights: expert_weights,
down: Some(down_weights),
},
output: expert_output,
shared,
},
auxiliary,
))
}
pub(crate) fn backward(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
cache: &GpuCache,
grad_logits: &Matrix,
) -> Result<(), NetworkError> {
let gpu = Gpu { context };
let rows = cache.stack.rows;
let d_model = model.config.d_model;
if grad_logits.rows != rows {
return Err(NetworkError::InvalidInput {
expected: rows,
actual: grad_logits.rows,
});
}
let upstream = gpu.upload(&grad_logits.data)?;
let mut grad_final = gpu.zeros(rows * d_model)?;
{
let head = head_of_mut(model)?;
gpu.linear_backward_input(head, &upstream, &mut grad_final, rows, 0.0)?;
gpu.accumulate_weight_grad(head, &upstream, &cache.final_output.wide(), rows)?;
}
backward_from_final(model, context, cache, &grad_final)
}
pub(crate) fn train_step(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<(f32, f32), NetworkError> {
let vocab = model.embedding.vocab_size();
let predicted = batch.predicted();
if predicted == 0 {
return Err(NetworkError::InvalidConfig(
"a causal language-modelling loss needs at least two tokens".into(),
));
}
if vocab > i32::MAX as usize {
return Err(NetworkError::InvalidConfig(format!(
"the device loss supports vocabularies up to {}, got {vocab}",
i32::MAX
)));
}
let targets = causal_targets(batch, vocab)?;
let cache = forward_hidden(model, context, batch)?;
let gpu = Gpu { context };
let rows = cache.stack.rows;
let d_model = model.config.d_model;
let targets = gpu.upload_signed(&targets)?;
let mut loss = gpu.zeros(1)?;
let mut grad_final = gpu.zeros(rows * d_model)?;
let inverse_predicted = 1.0 / predicted as f32;
let logit_bytes = if gpu.context.mixed_precision { 2 } else { 4 };
let chunk = head_chunk_rows(rows, vocab, logit_bytes);
{
let head = head_of_mut(model)?;
let mut reduced = gpu
.context
.mixed_precision
.then(|| HeadBf16::new(&gpu, head, chunk))
.transpose()?;
let final_output = cache.final_output.wide();
let mut start = 0;
while start < rows {
let count = chunk.min(rows - start);
let input = final_output.slice(start * d_model..(start + count) * d_model);
let targets = targets.slice(start..start + count);
let mut out = grad_final.slice_mut(start * d_model..(start + count) * d_model);
if let Some(reduced) = reduced.as_mut() {
reduced.chunk(
&gpu,
head,
&input,
&targets,
&mut loss,
&mut out,
count,
inverse_predicted,
)?;
start += count;
continue;
}
let mut logits = gpu.linear(head, &input, count)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.cross_entropy)
.arg(&mut logits)
.arg(&mut loss)
.arg(&targets)
.arg(&(count as i32))
.arg(&(vocab as i32))
.arg(&inverse_predicted)
.launch(row_per_block(count))
.map_err(cuda_err("cross-entropy kernel"))?;
}
gpu.linear_backward_input(head, &logits, &mut out, count, 0.0)?;
gpu.accumulate_weight_grad(head, &logits, &input, count)?;
start += count;
}
}
backward_from_final(model, context, &cache, &grad_final)?;
let lm_loss = gpu.download(&loss)?[0] / predicted as f32;
Ok((lm_loss, cache.stack.auxiliary_loss))
}
struct HeadBf16 {
weight: CudaSlice<bf16>,
weight_grad: CudaSlice<bf16>,
input: CudaSlice<bf16>,
logits: CudaSlice<bf16>,
grad_input: CudaSlice<bf16>,
}
impl HeadBf16 {
fn new(gpu: &Gpu, head: &DeviceParam, chunk: usize) -> Result<Self, NetworkError> {
let (vocab, d_model) = (head.rows(), head.cols());
let mut weight = gpu.uninit_bf16(vocab * d_model)?;
gpu.cast_to_bf16(&mut weight, &head.value().slice(..), vocab * d_model)?;
Ok(Self {
weight,
weight_grad: gpu.uninit_bf16(if head.is_frozen() { 1 } else { vocab * d_model })?,
input: gpu.uninit_bf16(chunk * d_model)?,
logits: gpu.uninit_bf16(chunk * vocab)?,
grad_input: gpu.uninit_bf16(chunk * d_model)?,
})
}
#[allow(clippy::too_many_arguments)]
fn chunk(
&mut self,
gpu: &Gpu,
head: &mut DeviceParam,
input: &CudaView<'_, f32>,
targets: &CudaView<'_, i32>,
loss: &mut CudaSlice<f32>,
grad_input: &mut CudaViewMut<'_, f32>,
count: usize,
inverse_predicted: f32,
) -> Result<(), NetworkError> {
let (vocab, d_model) = (head.rows(), head.cols());
gpu.cast_to_bf16(&mut self.input, input, count * d_model)?;
gemm_rhs_transposed(
gpu.context,
&self.input,
d_model,
&self.weight,
d_model,
&mut self.logits,
vocab,
count,
vocab,
d_model,
bf16::ONE,
bf16::ZERO,
)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.cross_entropy_bf16)
.arg(&mut self.logits)
.arg(loss)
.arg(targets)
.arg(&(count as i32))
.arg(&(vocab as i32))
.arg(&inverse_predicted)
.launch(row_per_block(count))
.map_err(cuda_err("cross-entropy kernel"))?;
}
gemm_plain(
gpu.context,
&self.logits,
vocab,
&self.weight,
d_model,
&mut self.grad_input,
d_model,
count,
d_model,
vocab,
bf16::ONE,
bf16::ZERO,
)?;
gpu.cast_from_bf16(grad_input, &self.grad_input, count * d_model)?;
if head.is_frozen() {
return Ok(());
}
gemm_lhs_transposed(
gpu.context,
&self.logits,
vocab,
&self.input,
d_model,
&mut self.weight_grad,
d_model,
count,
vocab,
d_model,
bf16::NEG_ONE,
bf16::ZERO,
)?;
head.clear_grad()?;
gpu.accumulate_bf16(head.negated_grad_mut(), &self.weight_grad, vocab * d_model)
}
}
fn causal_targets(batch: &TokenBatch, vocab: usize) -> Result<Vec<i32>, NetworkError> {
let ids = batch.ids();
let mut targets = vec![-1i32; batch.rows()];
for row in 0..batch.rows() {
if !batch.predicts(row) {
continue;
}
let id = ids[row + 1];
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
targets[row] = id as i32;
}
Ok(targets)
}
fn head_chunk_rows(rows: usize, vocab: usize, logit_bytes: usize) -> usize {
#[cfg(test)]
{
let forced = CHUNK_ROWS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed);
if forced > 0 {
return forced.min(rows.max(1));
}
}
const BUDGET_BYTES: usize = 256 << 20;
(BUDGET_BYTES / (vocab * logit_bytes)).clamp(1, rows.max(1))
}
#[cfg(test)]
pub(crate) static CHUNK_ROWS_OVERRIDE: std::sync::atomic::AtomicUsize =
std::sync::atomic::AtomicUsize::new(0);
fn row_per_block(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.max(1) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
const ROW_GRID_BLOCKS: usize = 2048;
fn warp_per_row_grid(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.div_ceil(8).clamp(1, ROW_GRID_BLOCKS) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn row_grid(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.clamp(1, ROW_GRID_BLOCKS) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn rmsnorm_smem(cols: usize) -> Option<u32> {
let bytes = cols.checked_mul(4)?;
(bytes <= 48 << 10).then_some(bytes as u32)
}
fn backward_from_final(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
cache: &GpuCache,
grad_final: &CudaSlice<f32>,
) -> Result<(), NetworkError> {
let gpu = Gpu { context };
let rows = cache.stack.rows;
let d_model = model.config.d_model;
let mut grad_norms = gpu.zeros(norm_slots(model.blocks.len()) * d_model)?;
let (grad_hidden, grad_hidden_act) = gpu.rmsnorm_backward(
&cache.final_input,
grad_final,
&cache.final_weight,
&cache.final_inverse_rms,
&mut grad_norms.slice_mut(0..d_model),
grad_final,
false,
stack_operand(&cache.stack, model.blocks.len().checked_sub(1)),
rows,
d_model,
)?;
let grad_hidden = backward_stack(
&gpu,
&mut model.blocks,
&cache.stack,
grad_hidden,
grad_hidden_act,
&mut grad_norms,
)?;
{
let embedding = model.embedding.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
if !embedding.is_frozen() {
embedding.clear_grad()?;
gpu.scatter_negated(
embedding.negated_grad_mut(),
&grad_hidden,
&cache.ids,
rows,
d_model,
)?;
}
}
let values = gpu.download(&grad_norms)?;
Gpu::accumulate_host_grad(&mut model.final_norm.weight, &values[..d_model]);
accumulate_block_norms(&mut model.blocks, &values, d_model);
Ok(())
}
fn norm_slots(blocks: usize) -> usize {
1 + 2 * blocks
}
fn norm_slot(index: usize) -> (usize, usize) {
(1 + 2 * index, 2 + 2 * index)
}
#[allow(clippy::too_many_arguments)]
fn backward_block(
gpu: &Gpu<'_>,
block: &mut TransformerBlock,
cache: &BlockCache,
grad_output: &CudaSlice<f32>,
grad_output_act: Option<&Act>,
output_operand: Option<bool>,
batch: &GpuStack,
grad_norms: &mut CudaSlice<f32>,
index: usize,
) -> Result<(CudaSlice<f32>, Option<Act>), NetworkError> {
let rows = batch.rows;
let d_model = block.attention.d_model();
let (attention_slot, feed_forward_slot) = norm_slot(index);
let mut grad_normed = match block.feed_forward {
FeedForward::SwiGlu(_) => gpu.uninit(rows * d_model)?,
_ => gpu.zeros(rows * d_model)?,
};
match (&mut block.feed_forward, &cache.feed_forward) {
(FeedForward::SwiGlu(ffn), FfnCache::Dense(ffn_cache)) => backward_swiglu(
gpu,
ffn,
ffn_cache,
&cache.feed_forward_normed,
grad_output_act.expect("a dense feed forward was handed its operand"),
&mut grad_normed,
rows,
)?,
(FeedForward::Moe(moe), FfnCache::Moe(moe_cache)) => backward_moe(
gpu,
moe,
moe_cache,
&cache.feed_forward_normed,
grad_output,
&mut grad_normed,
batch.valid_tokens,
rows,
)?,
(FeedForward::Gelu(_), _) => return Err(unsupported("a GELU MLP")),
_ => {
return Err(NetworkError::Cuda(
"the device cache does not match the layer that produced it".into(),
));
}
}
let (grad_residual, grad_residual_act) = gpu.rmsnorm_backward(
&cache.residual,
&grad_normed,
&cache.feed_forward_weight,
&cache.feed_forward_inverse_rms,
&mut grad_norms.slice_mut(feed_forward_slot * d_model..(feed_forward_slot + 1) * d_model),
grad_output,
true,
Some(cache.merged.is_narrow()),
rows,
d_model,
)?;
let grad_residual_act = grad_residual_act.expect("a copy was asked for");
let grad_attention_normed = backward_attention(gpu, block, cache, &grad_residual_act, batch)?;
let (grad_input, grad_input_act) = gpu.rmsnorm_backward(
&cache.input,
&grad_attention_normed,
&cache.attention_weight,
&cache.attention_inverse_rms,
&mut grad_norms.slice_mut(attention_slot * d_model..(attention_slot + 1) * d_model),
&grad_residual,
true,
output_operand,
rows,
d_model,
)?;
Ok((grad_input, grad_input_act))
}
fn backward_attention(
gpu: &Gpu<'_>,
block: &mut TransformerBlock,
cache: &BlockCache,
grad_output: &Act,
batch: &GpuStack,
) -> Result<CudaSlice<f32>, NetworkError> {
let rows = batch.rows;
let seq_len = batch.seq_len;
let sequences = batch.sequences;
let attention = &mut block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let causal = attention.is_causal();
let group = heads / kv_heads;
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let kv_width = kv_heads * head_dim;
let qkv_width = query_width + 2 * kv_width;
let key_base = query_width;
let value_base = query_width + kv_width;
let block_size = seq_len * seq_len;
let merged_narrow = cache.merged.is_narrow();
debug_assert_eq!(grad_output.is_narrow(), merged_narrow);
let grad_output_act = grad_output;
let mut grad_merged = gpu.act(rows * query_width, merged_narrow)?;
gpu.linear_packed_backward_input(
&cache.output_weight.all(),
d_model,
query_width,
&grad_output_act.all(),
merged_narrow,
merged_narrow,
grad_merged.destination(),
rows,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut attention.output,
&grad_output_act.all(),
&cache.merged.all(),
merged_narrow,
rows,
)?;
let fused = gpu.context.flash.as_ref().filter(|_| cache.qkv.is_narrow());
let narrow_qkv = cache.qkv.is_narrow();
let mut grad_qkv = gpu.act(rows * qkv_width, narrow_qkv)?;
if let Some(flash) = fused {
let shape = FlashShape {
rows,
seq_len,
sequences,
heads,
group,
qkv_width,
query_width,
key_base,
value_base,
causal,
};
let mut delta = gpu.uninit(heads * rows)?;
gpu.flash_attention_backward(
flash,
&cache.qkv,
&cache.merged,
&grad_merged,
&cache.log_sum_exp,
&mut delta,
&mut grad_qkv,
shape,
scale,
)?;
} else {
let qkv = cache.qkv.wide();
let wide_grad_merged = grad_merged.wide();
let mut probabilities = gpu.uninit(heads * sequences * block_size)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let query = qkv.slice(head * head_dim..);
let key = qkv.slice(key_base + kv_base..);
let mut scores = probabilities.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
qkv_width,
&key,
qkv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
scale,
0.0,
sequences,
seq_len * qkv_width,
seq_len * qkv_width,
block_size,
)?;
}
gpu.probs_from_lse(
&mut probabilities,
&cache.log_sum_exp,
heads * sequences * seq_len,
seq_len,
causal,
)?;
let mut grad_scores = gpu.uninit(heads * sequences * block_size)?;
let mut grad_qkv = grad_qkv.wide_mut();
gpu.context
.stream
.memset_zeros(&mut grad_qkv)
.map_err(cuda_err("gradient clear"))?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let upstream = wide_grad_merged.slice(head * head_dim..);
let value = qkv.slice(value_base + kv_base..);
let mut scores = grad_scores.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&upstream,
query_width,
&value,
qkv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
1.0,
0.0,
sequences,
seq_len * query_width,
seq_len * qkv_width,
block_size,
)?;
let head_probabilities = probabilities.slice(head * sequences * block_size..);
let mut grad_value = grad_qkv.slice_mut(value_base + kv_base..);
gemm_lhs_transposed_batched(
gpu.context,
&head_probabilities,
seq_len,
&upstream,
query_width,
&mut grad_value,
qkv_width,
seq_len,
seq_len,
head_dim,
1.0,
1.0,
sequences,
block_size,
seq_len * query_width,
seq_len * qkv_width,
)?;
}
gpu.softmax_backward(
&mut grad_scores,
&probabilities,
heads * sequences * seq_len,
seq_len,
causal,
)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let scores = grad_scores.slice(head * sequences * block_size..);
let key = qkv.slice(key_base + kv_base..);
let mut grad_query = grad_qkv.slice_mut(head * head_dim..);
gemm_plain_batched(
gpu.context,
&scores,
seq_len,
&key,
qkv_width,
&mut grad_query,
qkv_width,
seq_len,
head_dim,
seq_len,
scale,
0.0,
sequences,
block_size,
seq_len * qkv_width,
seq_len * qkv_width,
)?;
let query = qkv.slice(head * head_dim..);
let mut grad_key = grad_qkv.slice_mut(key_base + kv_base..);
gemm_lhs_transposed_batched(
gpu.context,
&scores,
seq_len,
&query,
qkv_width,
&mut grad_key,
qkv_width,
seq_len,
seq_len,
head_dim,
scale,
1.0,
sequences,
block_size,
seq_len * qkv_width,
seq_len * qkv_width,
)?;
}
}
let tables = gpu.rope_tables(&attention.rope)?;
gpu.rope(
&mut grad_qkv,
&tables,
rows,
heads,
head_dim,
seq_len,
-1.0,
qkv_width,
0,
)?;
gpu.rope(
&mut grad_qkv,
&tables,
rows,
kv_heads,
head_dim,
seq_len,
-1.0,
qkv_width,
key_base,
)?;
let narrow = cache.qkv_weights.is_narrow();
let grad_qkv = if narrow == narrow_qkv {
grad_qkv
} else {
gpu.narrowed(&grad_qkv.wide(), rows * qkv_width, narrow)?
};
let mut grad_input = gpu.uninit(rows * d_model)?;
gpu.linear_packed_backward_input(
&cache.qkv_weights.all(),
qkv_width,
d_model,
&grad_qkv.all(),
narrow,
false,
&mut grad_input,
rows,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [
&mut attention.query,
&mut attention.key,
&mut attention.value,
],
&grad_qkv.all(),
&cache.attention_normed.all(),
narrow,
rows,
)?;
Ok(grad_input)
}
pub(crate) fn backward_swiglu(
gpu: &Gpu<'_>,
ffn: &mut SwiGlu,
cache: &SwiGluCache,
input: &Act,
grad_output: &Act,
grad_input: &mut CudaSlice<f32>,
rows: usize,
) -> Result<(), NetworkError> {
let width = ffn.d_ff();
let inner = ffn.d_model();
let narrow = input.is_narrow();
debug_assert_eq!(grad_output.is_narrow(), narrow);
let grad_output_act = grad_output;
let down = cache
.down
.as_ref()
.ok_or_else(|| NetworkError::Cuda("the SwiGLU cache has no down projection".into()))?;
let mut grad_hidden = gpu.uninit(rows * width)?;
gpu.linear_packed_backward_input(
&down.all(),
inner,
width,
&grad_output_act.all(),
narrow,
false,
&mut grad_hidden,
rows,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut ffn.down,
&grad_output_act.all(),
&cache.hidden.all(),
narrow,
rows,
)?;
let grad_gate_up = gpu.swiglu_backward(&cache.gate_up, &grad_hidden, rows, width, narrow)?;
gpu.linear_packed_backward_input(
&cache.weights.all(),
2 * width,
inner,
&grad_gate_up.all(),
narrow,
false,
grad_input,
rows,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [&mut ffn.gate, &mut ffn.up],
&grad_gate_up.all(),
&input.all(),
narrow,
rows,
)?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn backward_moe(
gpu: &Gpu<'_>,
moe: &mut MoeLayer,
cache: &MoeGpuCache,
input: &Act,
grad_output: &CudaSlice<f32>,
grad_input: &mut CudaSlice<f32>,
valid_tokens: usize,
rows: usize,
) -> Result<(), NetworkError> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let routed = cache.routed;
if let (Some(shared), Some(shared_cache)) = (moe.shared.as_mut(), &cache.shared) {
let upstream = gpu.narrowed(&grad_output.slice(..), rows * d_model, input.is_narrow())?;
backward_swiglu(
gpu,
shared,
shared_cache,
input,
&upstream,
grad_input,
rows,
)?;
}
let mut grad_gates = gpu.zeros(rows * top_k)?;
if routed > 0 {
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.row_dot)
.arg(&mut grad_gates)
.arg(&cache.slot_of)
.arg(grad_output)
.arg(&cache.token_of)
.arg(&cache.output)
.arg(&(routed as i32))
.arg(&(d_model as i32))
.launch(warp_per_row_grid(routed))
.map_err(cuda_err("gate gradient kernel"))?;
}
let mut grad_expert_output = gpu.uninit(routed * d_model)?;
gpu.gather_scaled(
&mut grad_expert_output,
grad_output,
&cache.token_of,
&cache.gates,
routed,
d_model,
)?;
let down = cache.expert.down.as_ref().ok_or_else(|| {
NetworkError::Cuda("the routed feed-forward cache has no down projection".into())
})?;
let down_stride = d_model * width;
let grad_bytes = bytes_of(&grad_expert_output, routed * d_model);
let mut grad_hidden = gpu.uninit(routed * width)?;
for (expert, module) in moe.experts.iter_mut().enumerate() {
let count = cache.counts[expert];
if count == 0 {
continue;
}
let offset = cache.offsets[expert];
let upstream = grad_bytes.slice(offset * d_model * 4..);
let mut grad = grad_hidden.slice_mut(offset * width..);
gpu.linear_packed_backward_input(
&down.at(expert * down_stride),
d_model,
width,
&upstream,
false,
false,
&mut grad,
count,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut module.down,
&upstream,
&cache.expert.hidden.at(offset * width),
false,
count,
)?;
}
let grad_gate_up =
gpu.swiglu_backward(&cache.expert.gate_up, &grad_hidden, routed, width, false)?;
let packed_stride = 2 * width * d_model;
let mut grad_expert_input = gpu.uninit(routed * d_model)?;
for (expert, module) in moe.experts.iter_mut().enumerate() {
let count = cache.counts[expert];
if count == 0 {
continue;
}
let offset = cache.offsets[expert];
let rows_in = cache.input.at(offset * d_model);
let upstream = grad_gate_up.at(offset * 2 * width);
let weights = cache.expert.weights.at(expert * packed_stride);
let mut grad = grad_expert_input.slice_mut(offset * d_model..);
gpu.linear_packed_backward_input(
&weights,
2 * width,
d_model,
&upstream,
false,
false,
&mut grad,
count,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [&mut module.gate, &mut module.up],
&upstream,
&rows_in,
false,
count,
)?;
}
gpu.scatter(
grad_input,
&grad_expert_input,
&cache.token_of,
routed,
d_model,
)?;
}
let mut grad_probabilities = gpu.zeros(rows * experts)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.grad_probabilities)
.arg(&mut grad_probabilities)
.arg(&grad_gates)
.arg(&cache.probabilities)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.map_err(cuda_err("router gate gradient kernel"))?;
}
if moe.config.aux_loss_weight != 0.0 && valid_tokens > 0 {
let slots = (valid_tokens * top_k) as f32;
let loads: Vec<f32> = cache
.counts
.iter()
.map(|&count| count as f32 / slots)
.collect();
let loads = gpu.upload(&loads)?;
let scale = moe.config.aux_loss_weight * experts as f32 / valid_tokens as f32;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.aux_grad)
.arg(&mut grad_probabilities)
.arg(&loads)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.arg(&scale)
.launch(cfg(rows * experts))
.map_err(cuda_err("load-balancing gradient kernel"))?;
}
}
let z_factor = if moe.config.router_z_loss_weight != 0.0 {
2.0 * moe.config.router_z_loss_weight / valid_tokens.max(1) as f32
} else {
0.0
};
let mut grad_logits = gpu.zeros(rows * experts)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.grad_logits)
.arg(&mut grad_logits)
.arg(&grad_probabilities)
.arg(&cache.probabilities)
.arg(&cache.log_sum_exp)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.arg(&z_factor)
.launch(cfg(rows))
.map_err(cuda_err("router logit gradient kernel"))?;
}
gpu.linear_backward_input(
device_of(&moe.router.projection)?,
&grad_logits,
grad_input,
rows,
1.0,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut moe.router.projection)?,
&grad_logits,
&input.wide(),
rows,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cuda_training::cuda_doctor;
use crate::norm::RmsNorm;
fn cuda_or_skip() -> Option<Arc<GpuContext>> {
match cuda_doctor(0, 8192) {
Ok(_) => Some(GpuContext::new(0).expect("the CUDA doctor passed")),
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
None
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn ramp(len: usize) -> Vec<f32> {
ramp_from(len, 0)
}
fn ramp_from(len: usize, offset: usize) -> Vec<f32> {
(0..len)
.map(|i| (((i + offset) * 37) % 23) as f32 / 11.0 - 1.0)
.collect()
}
fn assert_close(label: &str, device: &[f32], host: &[f32], tolerance: f32) {
assert_eq!(device.len(), host.len(), "{label}: length");
for (index, (a, b)) in device.iter().zip(host).enumerate() {
assert!(
(a - b).abs() <= tolerance,
"{label}[{index}]: {a} on the device vs {b} on the host"
);
}
}
#[test]
fn an_allocation_that_cannot_fit_says_out_of_memory_rather_than_cuda() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
match gpu.zeros(1 << 40) {
Err(NetworkError::CudaOutOfMemory {
requested_mib,
free_mib,
}) => {
assert_eq!(requested_mib, 4 * 1024 * 1024);
assert!(free_mib > 0, "a working device reports some free memory");
}
other => panic!("expected an out-of-memory error, got {other:?}"),
}
}
#[test]
fn the_rmsnorm_kernels_match_the_host_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, cols) = (5, 8);
let mut norm = RmsNorm::new(cols, 1e-6);
norm.weight.value.data = ramp(cols).iter().map(|v| 1.0 + v * 0.25).collect();
let input = Matrix::from_vec(rows, cols, ramp(rows * cols));
let grad_output = Matrix::from_vec(rows, cols, ramp_from(rows * cols, 3));
let weight = gpu.upload(&norm.weight.value.data).unwrap();
let device_input = gpu.upload(&input.data).unwrap();
let (output, inverse, _) = gpu
.rmsnorm(&device_input, &weight, rows, cols, norm.eps, false, false)
.unwrap();
assert_close(
"rmsnorm forward",
&gpu.download_act(&output).unwrap(),
&norm.forward(&input).data,
1e-5,
);
let device_grad = gpu.upload(&grad_output.data).unwrap();
let mut grad_weight = gpu.zeros(cols).unwrap();
let grad_input = gpu
.rmsnorm_backward(
&device_input,
&device_grad,
&weight,
&inverse,
&mut grad_weight.slice_mut(..),
&device_grad,
false,
None,
rows,
cols,
)
.unwrap()
.0;
let host_grad_input = norm.backward(&input, &grad_output);
assert_close(
"rmsnorm grad_input",
&gpu.download(&grad_input).unwrap(),
&host_grad_input.data,
1e-5,
);
assert_close(
"rmsnorm grad_weight",
&gpu.download(&grad_weight).unwrap(),
&norm.weight.grad.data,
1e-5,
);
}
#[test]
fn the_rope_kernel_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (sequences, seq_len, heads, head_dim) = (2, 3, 2, 4);
let rows = sequences * seq_len;
let width = heads * head_dim;
let rope = Rope::new(head_dim, 16, 10_000.0).unwrap();
let tables = gpu.rope_tables(&rope).unwrap();
for (narrow, tolerance) in [(false, 1e-5), (true, 1e-2)] {
let mut host = Matrix::from_vec(rows, width, ramp(rows * width));
let wide = gpu.upload(&host.data).unwrap();
let mut device = gpu.narrowed(&wide.slice(..), rows * width, narrow).unwrap();
gpu.rope(
&mut device,
&tables,
rows,
heads,
head_dim,
seq_len,
1.0,
width,
0,
)
.unwrap();
rope.apply_batched(&mut host, heads, seq_len).unwrap();
assert_close(
"rope",
&gpu.download_act(&device).unwrap(),
&host.data,
tolerance,
);
gpu.rope(
&mut device,
&tables,
rows,
heads,
head_dim,
seq_len,
-1.0,
width,
0,
)
.unwrap();
rope.apply_inverse_batched(&mut host, heads, seq_len)
.unwrap();
assert_close(
"rope inverse",
&gpu.download_act(&device).unwrap(),
&host.data,
tolerance,
);
}
}
#[test]
fn the_causal_probability_kernels_mask_and_normalize_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (sequences, seq_len) = (2, 4);
let rows = sequences * seq_len;
let scores = ramp(rows * seq_len);
let log_sum_exp: Vec<f32> = (0..rows)
.map(|row| {
let source = &scores[row * seq_len..row * seq_len + row % seq_len + 1];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
max + source.iter().map(|v| (v - max).exp()).sum::<f32>().ln()
})
.collect();
let mut device = gpu.upload(&scores).unwrap();
let device_lse = gpu.upload(&log_sum_exp).unwrap();
gpu.probs_from_lse(&mut device, &device_lse, rows, seq_len, true)
.unwrap();
let probabilities = gpu.download(&device).unwrap();
for row in 0..rows {
let visible = row % seq_len + 1;
let slice = &probabilities[row * seq_len..(row + 1) * seq_len];
let mut expected = vec![0.0f32; seq_len];
let source = &scores[row * seq_len..row * seq_len + visible];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let total: f32 = source.iter().map(|v| (v - max).exp()).sum();
for (slot, &value) in expected.iter_mut().zip(source) {
*slot = (value - max).exp() / total;
}
assert_close(
&format!("causal probabilities row {row}"),
slice,
&expected,
1e-6,
);
assert!(slice[visible..].iter().all(|&p| p == 0.0));
}
let mut grad = gpu.upload(&vec![1.0; rows * seq_len]).unwrap();
gpu.softmax_backward(&mut grad, &device, rows, seq_len, true)
.unwrap();
assert_close(
"causal softmax backward",
&gpu.download(&grad).unwrap(),
&vec![0.0; rows * seq_len],
1e-6,
);
}
#[test]
fn the_swiglu_kernel_matches_the_host_formula_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, width) = (4, 4);
let len = rows * width;
let gate = ramp(len);
let up = ramp_from(len, 5);
let mut fused = Vec::with_capacity(2 * len);
for row in 0..rows {
fused.extend_from_slice(&gate[row * width..(row + 1) * width]);
fused.extend_from_slice(&up[row * width..(row + 1) * width]);
}
let gate_up = gpu.upload(&fused).unwrap();
let hidden = gpu.swiglu(&gate_up, rows, width, false).unwrap();
let silu = |x: f32| x / (1.0 + (-x).exp());
let expected: Vec<f32> = gate.iter().zip(&up).map(|(&g, &u)| silu(g) * u).collect();
assert_close(
"swiglu",
&gpu.download_act(&hidden).unwrap(),
&expected,
1e-6,
);
let grad_hidden = gpu.upload(&vec![1.0; len]).unwrap();
let grad_fused = gpu
.swiglu_backward(&gate_up, &grad_hidden, rows, width, false)
.unwrap();
let grad_fused = gpu.download_act(&grad_fused).unwrap();
let mut grad_gate = Vec::with_capacity(len);
let mut grad_up = Vec::with_capacity(len);
for row in 0..rows {
let base = row * 2 * width;
grad_gate.extend_from_slice(&grad_fused[base..base + width]);
grad_up.extend_from_slice(&grad_fused[base + width..base + 2 * width]);
}
let epsilon = 1e-3;
let numeric: Vec<f32> = gate
.iter()
.zip(&up)
.map(|(&g, &u)| (silu(g + epsilon) * u - silu(g - epsilon) * u) / (2.0 * epsilon))
.collect();
assert_close("swiglu grad_gate", &grad_gate, &numeric, 2e-3);
assert_close(
"swiglu grad_up",
&grad_up,
&gate.iter().map(|&g| silu(g)).collect::<Vec<_>>(),
1e-6,
);
}
#[test]
fn the_router_kernels_select_and_renormalize_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, experts, top_k) = (4, 5, 2);
let logits = ramp(rows * experts);
let device_logits = gpu.upload(&logits).unwrap();
let mut probabilities = gpu.zeros(rows * experts).unwrap();
let mut log_sum_exp = gpu.zeros(rows).unwrap();
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.softmax_lse)
.arg(&mut probabilities)
.arg(&mut log_sum_exp)
.arg(&device_logits)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.unwrap();
}
let flags: Vec<i32> = (0..rows).map(|row| i32::from(row + 1 < rows)).collect();
let valid = gpu.upload_flags(&flags).unwrap();
let mut expert_of = gpu.context.stream.alloc_zeros::<i32>(rows * top_k).unwrap();
let mut gate_of = gpu.zeros(rows * top_k).unwrap();
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.topk_gate)
.arg(&mut expert_of)
.arg(&mut gate_of)
.arg(&probabilities)
.arg(&valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.unwrap();
}
let probabilities = gpu.download(&probabilities).unwrap();
let log_sum_exp = gpu.download(&log_sum_exp).unwrap();
let assigned = gpu.download_signed(&expert_of).unwrap();
let gates = gpu.download(&gate_of).unwrap();
for row in 0..rows {
let slice = &probabilities[row * experts..(row + 1) * experts];
let sum: f32 = slice.iter().sum();
assert!((sum - 1.0).abs() < 1e-5, "row {row} softmax sums to {sum}");
let source = &logits[row * experts..(row + 1) * experts];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let expected = max + source.iter().map(|v| (v - max).exp()).sum::<f32>().ln();
assert!((log_sum_exp[row] - expected).abs() < 1e-5);
let selected = &assigned[row * top_k..(row + 1) * top_k];
if row + 1 == rows {
assert!(selected.iter().all(|&expert| expert < 0), "padding routed");
assert!(
gates[row * top_k..(row + 1) * top_k]
.iter()
.all(|&g| g == 0.0)
);
continue;
}
let mut order: Vec<usize> = (0..experts).collect();
order.sort_by(|&a, &b| slice[b].total_cmp(&slice[a]));
assert_eq!(
selected.iter().map(|&e| e as usize).collect::<Vec<_>>(),
order[..top_k],
"row {row} top-{top_k}"
);
let total: f32 = selected.iter().map(|&e| slice[e as usize]).sum();
for (rank, &expert) in selected.iter().enumerate() {
let expected = slice[expert as usize] / total;
assert!((gates[row * top_k + rank] - expected).abs() < 1e-6);
}
}
}
#[derive(Clone, Copy)]
struct AttentionShape {
sequences: usize,
seq_len: usize,
heads: usize,
kv_heads: usize,
head_dim: usize,
scale: f32,
}
fn host_attention(
queries: &[f32],
keys: &[f32],
values: &[f32],
shape: AttentionShape,
) -> (Vec<f32>, Vec<f32>) {
let AttentionShape {
sequences,
seq_len,
heads,
kv_heads,
head_dim,
scale,
} = shape;
let (group, query_width, kv_width) =
(heads / kv_heads, heads * head_dim, kv_heads * head_dim);
let mut out = vec![0.0; queries.len()];
let mut probabilities = vec![0.0; heads * sequences * seq_len * seq_len];
for s in 0..sequences {
for h in 0..heads {
let kv = h / group;
for i in 0..seq_len {
let q = ((s * seq_len + i) * query_width) + h * head_dim;
let mut row = vec![0.0f32; i + 1];
for (j, p) in row.iter_mut().enumerate() {
let k = ((s * seq_len + j) * kv_width) + kv * head_dim;
*p = (0..head_dim)
.map(|d| queries[q + d] * keys[k + d])
.sum::<f32>()
* scale;
}
let top = row.iter().cloned().fold(f32::MIN, f32::max);
let total: f32 = row.iter().map(|p| (p - top).exp()).sum();
let base = ((h * sequences + s) * seq_len + i) * seq_len;
for (j, p) in row.iter().enumerate() {
let weight = (p - top).exp() / total;
probabilities[base + j] = weight;
let v = ((s * seq_len + j) * kv_width) + kv * head_dim;
for d in 0..head_dim {
out[q + d] += weight * values[v + d];
}
}
}
}
}
(out, probabilities)
}
#[test]
fn the_attention_forward_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let shape = AttentionShape {
sequences: 2,
seq_len: 11,
heads: 4,
kv_heads: 2,
head_dim: 8,
scale: 1.0 / 8.0f32.sqrt(),
};
let rows = shape.sequences * shape.seq_len;
let wiggle = |len: usize, offset: usize| -> Vec<f32> {
(0..len)
.map(|i| ((i + offset) as f32 * 0.37).sin() * 0.8)
.collect()
};
let queries = wiggle(rows * shape.heads * shape.head_dim, 0);
let keys = wiggle(rows * shape.kv_heads * shape.head_dim, 5);
let values = wiggle(rows * shape.kv_heads * shape.head_dim, 11);
let (host_out, host_probabilities) = host_attention(&queries, &keys, &values, shape);
let device_queries = gpu.upload(&queries).unwrap();
let device_keys = gpu.upload(&keys).unwrap();
let device_values = gpu.upload(&values).unwrap();
let (group, block_size) = (shape.heads / shape.kv_heads, shape.seq_len * shape.seq_len);
let (query_width, kv_width) = (
shape.heads * shape.head_dim,
shape.kv_heads * shape.head_dim,
);
let scores_of = |probabilities: &mut CudaSlice<f32>| {
for head in 0..shape.heads {
let query = device_queries.slice(head * shape.head_dim..);
let key = device_keys.slice((head / group) * shape.head_dim..);
let mut scores = probabilities.slice_mut(head * shape.sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
query_width,
&key,
kv_width,
&mut scores,
shape.seq_len,
shape.seq_len,
shape.seq_len,
shape.head_dim,
shape.scale,
0.0,
shape.sequences,
shape.seq_len * query_width,
shape.seq_len * kv_width,
block_size,
)
.unwrap();
}
};
let mut probabilities = gpu
.zeros(shape.heads * shape.sequences * block_size)
.unwrap();
scores_of(&mut probabilities);
let mut lse = gpu.zeros(shape.heads * rows).unwrap();
gpu.softmax_lse(
&mut probabilities,
&mut lse,
shape.heads * shape.sequences * shape.seq_len,
shape.seq_len,
true,
)
.unwrap();
assert_close(
"causal softmax probabilities",
&gpu.download(&probabilities).unwrap(),
&host_probabilities,
2e-6,
);
let mut merged = gpu.zeros(queries.len()).unwrap();
for head in 0..shape.heads {
let head_probabilities = probabilities.slice(head * shape.sequences * block_size..);
let value = device_values.slice((head / group) * shape.head_dim..);
let mut head_merged = merged.slice_mut(head * shape.head_dim..);
gemm_plain_batched(
gpu.context,
&head_probabilities,
shape.seq_len,
&value,
kv_width,
&mut head_merged,
query_width,
shape.seq_len,
shape.head_dim,
shape.seq_len,
1.0,
0.0,
shape.sequences,
block_size,
shape.seq_len * kv_width,
shape.seq_len * query_width,
)
.unwrap();
}
assert_close(
"attention forward",
&gpu.download(&merged).unwrap(),
&host_out,
2e-5,
);
scores_of(&mut probabilities);
gpu.probs_from_lse(
&mut probabilities,
&lse,
shape.heads * shape.sequences * shape.seq_len,
shape.seq_len,
true,
)
.unwrap();
assert_close(
"probabilities rebuilt from the log-sum-exp",
&gpu.download(&probabilities).unwrap(),
&host_probabilities,
2e-6,
);
}
}