use crate::batch::TokenBatch;
use crate::cuda_training::{cfg, cuda_alloc_err, cuda_err};
use crate::ffn::SwiGlu;
use crate::gpu_transformer::{
DeviceParam, GpuContext, act_lhs_transposed, act_plain, act_rhs_transposed,
gemm_lhs_transposed, gemm_lhs_transposed_batched, gemm_plain, gemm_plain_batched,
gemm_rhs_transposed, gemm_rhs_transposed_batched,
};
use crate::matrix::Matrix;
use crate::moe::MoeLayer;
use crate::network::NetworkError;
use crate::param::{Linear, Lora, Param};
use crate::rope::Rope;
use crate::transformer::TransformerLm;
use crate::transformer_block::{FeedForward, TransformerBlock};
use cudarc::driver::{
CudaFunction, CudaModule, CudaSlice, CudaView, CudaViewMut, DevicePtr, DevicePtrMut,
PushKernelArg,
};
use half::bf16;
use std::sync::Arc;
use std::sync::atomic::Ordering;
pub(crate) struct ModelKernels {
rmsnorm_forward: CudaFunction,
rmsnorm_backward: CudaFunction,
rope: CudaFunction,
softmax_backward: CudaFunction,
probs_from_lse: CudaFunction,
softmax_lse_rows: CudaFunction,
decode_attention: CudaFunction,
mask_padding: CudaFunction,
decode_attention_merge: CudaFunction,
store_rows: CudaFunction,
quantize_q8: CudaFunction,
quantize_groups_q8: CudaFunction,
dequantize_groups_q8: CudaFunction,
dequantize_q8: CudaFunction,
dequantize_fp8: CudaFunction,
quantize_e4m3: CudaFunction,
dequantize_q4: CudaFunction,
gemv_q8: CudaFunction,
swiglu_forward: CudaFunction,
swiglu_backward: CudaFunction,
adaln_modulate: CudaFunction,
adaln_modulate_backward: CudaFunction,
gate_residual: CudaFunction,
gate_residual_backward: CudaFunction,
softmax_lse: CudaFunction,
topk_gate: CudaFunction,
gemv_routed: CudaFunction,
gemv_routed_q8: CudaFunction,
gather: CudaFunction,
gather_scaled: CudaFunction,
scatter_scaled: CudaFunction,
scatter: CudaFunction,
scatter_negated: CudaFunction,
row_dot: CudaFunction,
stats: CudaFunction,
grad_probabilities: CudaFunction,
aux_grad: CudaFunction,
grad_logits: CudaFunction,
add: CudaFunction,
cross_entropy: CudaFunction,
cross_entropy_bf16: CudaFunction,
pub(crate) to_bf16: CudaFunction,
from_bf16: CudaFunction,
accumulate_bf16: CudaFunction,
cast_act: CudaFunction,
}
impl ModelKernels {
pub(crate) fn load(module: &Arc<CudaModule>) -> Result<Self, NetworkError> {
let get = |name: &str| {
module
.load_function(name)
.map_err(cuda_err("CUDA kernel lookup"))
};
Ok(Self {
rmsnorm_forward: get("rmsnorm_fwd")?,
rmsnorm_backward: get("rmsnorm_bwd")?,
rope: get("rope_rotate")?,
softmax_backward: get("attention_softmax_bwd")?,
probs_from_lse: get("attention_probs_from_lse")?,
softmax_lse_rows: get("attention_softmax_lse")?,
decode_attention: get("decode_attention")?,
mask_padding: get("mask_padding")?,
decode_attention_merge: get("decode_attention_merge")?,
store_rows: get("store_rows")?,
quantize_q8: get("quantize_rows_q8")?,
quantize_groups_q8: get("quantize_groups_q8")?,
dequantize_groups_q8: get("dequantize_groups_q8")?,
dequantize_q8: get("dequantize_q8")?,
dequantize_fp8: get("dequantize_fp8")?,
quantize_e4m3: get("quantize_rows_e4m3")?,
dequantize_q4: get("dequantize_q4")?,
gemv_q8: get("gemv_q8")?,
swiglu_forward: get("swiglu_fwd")?,
swiglu_backward: get("swiglu_bwd")?,
adaln_modulate: get("adaln_modulate")?,
adaln_modulate_backward: get("adaln_modulate_bwd")?,
gate_residual: get("gate_residual_fwd")?,
gate_residual_backward: get("gate_residual_bwd")?,
softmax_lse: get("softmax_lse")?,
topk_gate: get("topk_gate")?,
gemv_routed: get("gemv_routed")?,
gemv_routed_q8: get("gemv_routed_q8")?,
gather: get("gather_rows")?,
gather_scaled: get("gather_scale_rows")?,
scatter_scaled: get("scatter_add_scaled")?,
scatter: get("scatter_add_rows")?,
scatter_negated: get("scatter_rows_neg")?,
row_dot: get("row_dot_scatter")?,
stats: get("moe_stats")?,
grad_probabilities: get("moe_grad_probs")?,
aux_grad: get("moe_aux_grad")?,
grad_logits: get("router_grad_logits")?,
add: get("add_inplace")?,
cross_entropy: get("ce_loss_grad")?,
cross_entropy_bf16: get("ce_loss_grad_bf16")?,
to_bf16: get("to_bf16")?,
from_bf16: get("from_bf16")?,
accumulate_bf16: get("accumulate_bf16")?,
cast_act: get("cast_act")?,
})
}
}
#[derive(Clone, Copy)]
pub(crate) struct FlashShape {
pub(crate) rows: usize,
pub(crate) seq_len: usize,
pub(crate) kv_len: usize,
pub(crate) sequences: usize,
pub(crate) heads: usize,
pub(crate) group: usize,
pub(crate) qkv_width: usize,
pub(crate) kv_width: usize,
pub(crate) query_width: usize,
pub(crate) key_base: usize,
pub(crate) value_base: usize,
pub(crate) causal: bool,
}
impl FlashShape {
fn head_dim(&self) -> usize {
self.query_width / self.heads
}
}
pub(crate) struct DeviceRope {
cos: CudaSlice<f32>,
sin: CudaSlice<f32>,
head_dim: usize,
max_seq_len: usize,
}
pub(crate) struct GpuStack {
rows: usize,
seq_len: usize,
sequences: usize,
valid_tokens: usize,
valid: CudaSlice<i32>,
blocks: Vec<Saved>,
loads: Vec<Vec<f32>>,
auxiliary_loss: f32,
}
#[allow(clippy::large_enum_variant)]
enum Saved {
Cache(BlockCache),
Input(CudaSlice<f32>),
}
impl std::fmt::Debug for GpuStack {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GpuStack")
.field("rows", &self.rows)
.field("seq_len", &self.seq_len)
.finish()
}
}
pub(crate) struct GpuCache {
stack: GpuStack,
ids: CudaSlice<u32>,
final_input: CudaSlice<f32>,
final_inverse_rms: CudaSlice<f32>,
final_weight: CudaSlice<f32>,
final_output: Act,
}
impl std::fmt::Debug for GpuCache {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.stack.fmt(f)
}
}
impl GpuCache {
pub(crate) fn auxiliary_loss(&self) -> f32 {
self.stack.auxiliary_loss
}
pub(crate) fn expert_loads(&self) -> Vec<Vec<f32>> {
self.stack.loads.clone()
}
}
pub(crate) struct Act {
bytes: CudaSlice<u8>,
narrow: bool,
len: usize,
scales: Option<CudaSlice<f32>>,
group: usize,
fp8: bool,
}
impl Act {
pub(crate) fn element(narrow: bool) -> usize {
if narrow { 2 } else { 4 }
}
pub(crate) fn is_narrow(&self) -> bool {
self.narrow
}
pub(crate) fn flag(&self) -> i32 {
i32::from(self.narrow)
}
pub(crate) fn at(&self, offset: usize) -> CudaView<'_, u8> {
self.bytes.slice(offset * Act::element(self.narrow)..)
}
pub(crate) fn all(&self) -> CudaView<'_, u8> {
self.bytes.slice(..)
}
pub(crate) fn int8(&self) -> Option<(&CudaSlice<u8>, &CudaSlice<f32>, usize)> {
match &self.scales {
Some(scales) if !self.fp8 => Some((&self.bytes, scales, self.group)),
_ => None,
}
}
pub(crate) fn wide(&self) -> CudaView<'_, f32> {
debug_assert!(!self.narrow, "a narrow activation is not FP32");
unsafe { self.bytes.transmute::<f32>(self.len) }
.expect("a device allocation is aligned for FP32")
}
pub(crate) fn wide_mut(&mut self) -> CudaViewMut<'_, f32> {
debug_assert!(!self.narrow, "a narrow activation is not FP32");
unsafe { self.bytes.transmute_mut::<f32>(self.len) }
.expect("a device allocation is aligned for FP32")
}
pub(crate) fn destination(&mut self) -> &mut CudaSlice<u8> {
&mut self.bytes
}
}
struct BlockCache {
input: CudaSlice<f32>,
attention_weight: CudaSlice<f32>,
attention_inverse_rms: CudaSlice<f32>,
attention_normed: Act,
qkv: Act,
qkv_weights: Arc<Act>,
log_sum_exp: CudaSlice<f32>,
merged: Act,
output_weight: Arc<Act>,
residual: CudaSlice<f32>,
feed_forward_weight: CudaSlice<f32>,
feed_forward_inverse_rms: CudaSlice<f32>,
feed_forward_normed: Act,
feed_forward: FfnCache,
}
pub(crate) struct SwiGluCache {
pub(crate) gate_up: CudaSlice<f32>,
pub(crate) hidden: Act,
pub(crate) weights: Arc<Act>,
pub(crate) down: Option<Arc<Act>>,
}
type NormOutput = (Act, CudaSlice<f32>, Option<CudaSlice<f32>>);
#[allow(clippy::large_enum_variant)]
enum FfnCache {
Dense(SwiGluCache),
Moe(Box<MoeGpuCache>),
}
struct MoeGpuCache {
probabilities: CudaSlice<f32>,
log_sum_exp: CudaSlice<f32>,
expert_of: CudaSlice<i32>,
token_of: CudaSlice<u32>,
slot_of: CudaSlice<u32>,
gates: CudaSlice<f32>,
offsets: Vec<usize>,
counts: Vec<usize>,
routed: usize,
input: Act,
expert: SwiGluCache,
output: CudaSlice<f32>,
shared: Option<SwiGluCache>,
}
pub(crate) struct Gpu<'a> {
pub(crate) context: &'a Arc<GpuContext>,
}
impl Gpu<'_> {
pub(crate) fn zeros(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
self.context
.stream
.alloc_zeros::<f32>(len.max(1))
.map_err(cuda_alloc_err("device allocation", len.max(1) * 4))
}
pub(crate) fn uninit(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
unsafe { self.context.stream.alloc::<f32>(len.max(1)) }
.map_err(cuda_alloc_err("device allocation", len.max(1) * 4))
}
pub(crate) fn uninit_bf16(&self, len: usize) -> Result<CudaSlice<bf16>, NetworkError> {
unsafe { self.context.stream.alloc::<bf16>(len.max(1)) }
.map_err(cuda_alloc_err("device allocation", len.max(1) * 2))
}
pub(crate) fn act(&self, len: usize, narrow: bool) -> Result<Act, NetworkError> {
let bytes = unsafe {
self.context
.stream
.alloc::<u8>(len.max(1) * Act::element(narrow))
}
.map_err(cuda_alloc_err(
"device allocation",
len.max(1) * Act::element(narrow),
))?;
Ok(Act {
bytes,
narrow,
len,
scales: None,
group: 0,
fp8: false,
})
}
pub(crate) fn cast_into(
&self,
dst: &mut CudaViewMut<'_, u8>,
src: &CudaView<'_, f32>,
n: usize,
narrow: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.cast_act)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.arg(&i32::from(narrow))
.launch(cfg(n))
.map_err(cuda_err("activation cast kernel"))?;
}
Ok(())
}
pub(crate) fn narrowed(
&self,
src: &CudaView<'_, f32>,
len: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
let mut act = self.act(len, narrow)?;
let mut dst = act.bytes.slice_mut(..);
self.cast_into(&mut dst, src, len, narrow)?;
Ok(act)
}
pub(crate) fn cast_to_bf16(
&self,
dst: &mut CudaSlice<bf16>,
src: &CudaView<'_, f32>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.to_bf16)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 cast kernel"))?;
}
Ok(())
}
pub(crate) fn cast_from_bf16(
&self,
dst: &mut CudaViewMut<'_, f32>,
src: &CudaSlice<bf16>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.from_bf16)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 widening kernel"))?;
}
Ok(())
}
pub(crate) fn accumulate_bf16(
&self,
acc: &mut CudaSlice<f32>,
src: &CudaSlice<bf16>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.accumulate_bf16)
.arg(acc)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 accumulate kernel"))?;
}
Ok(())
}
pub(crate) fn upload(&self, data: &[f32]) -> Result<CudaSlice<f32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn upload_indices(&self, data: &[u32]) -> Result<CudaSlice<u32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn upload_flags(&self, data: &[i32]) -> Result<CudaSlice<i32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn download(&self, source: &CudaSlice<f32>) -> Result<Vec<f32>, NetworkError> {
self.context
.stream
.clone_dtoh(source)
.map_err(cuda_err("device to host copy"))
}
#[cfg(test)]
pub(crate) fn download_act(&self, source: &Act) -> Result<Vec<f32>, NetworkError> {
if !source.is_narrow() {
return self
.context
.stream
.clone_dtoh(&source.wide())
.map_err(cuda_err("device to host copy"));
}
let bytes = self
.context
.stream
.clone_dtoh(&source.all())
.map_err(cuda_err("device to host copy"))?;
Ok(bytes
.chunks_exact(2)
.map(|half| f32::from_bits(u32::from(u16::from_le_bytes([half[0], half[1]])) << 16))
.collect())
}
pub(crate) fn upload_signed(&self, data: &[i32]) -> Result<CudaSlice<i32>, NetworkError> {
self.context.stream.clone_htod(data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(data),
))
}
pub(crate) fn download_signed(
&self,
source: &CudaSlice<i32>,
) -> Result<Vec<i32>, NetworkError> {
self.context
.stream
.clone_dtoh(source)
.map_err(cuda_err("device to host copy"))
}
pub(crate) fn matrix(
&self,
source: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<Matrix, NetworkError> {
let mut matrix = Matrix::new(rows, width);
self.context
.stream
.memcpy_dtoh(source, &mut matrix.data)
.map_err(cuda_err("device to host copy"))?;
Ok(matrix)
}
pub(crate) fn accumulate_host_grad(param: &mut Param, values: &[f32]) {
for (slot, value) in param.grad.data.iter_mut().zip(values) {
*slot += value;
}
}
pub(crate) fn rope_tables(&self, rope: &Rope) -> Result<Arc<DeviceRope>, NetworkError> {
let mut slot = self
.context
.rope
.lock()
.map_err(|_| NetworkError::Cuda("the rope table lock was poisoned".into()))?;
if let Some(tables) = slot.as_ref()
&& tables.head_dim == rope.head_dim()
&& tables.max_seq_len == rope.max_seq_len()
{
return Ok(tables.clone());
}
let tables = Arc::new(DeviceRope {
cos: self.upload(rope.cos())?,
sin: self.upload(rope.sin())?,
head_dim: rope.head_dim(),
max_seq_len: rope.max_seq_len(),
});
*slot = Some(tables.clone());
Ok(tables)
}
pub(crate) fn linear<X: DevicePtr<f32>>(
&self,
weight: &DeviceParam,
x: &X,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * weight.rows())?;
gemm_rhs_transposed(
self.context,
x,
weight.cols(),
weight.value(),
weight.cols(),
&mut out,
weight.rows(),
rows,
weight.rows(),
weight.cols(),
1.0,
0.0,
)?;
Ok(out)
}
pub(crate) fn linear_backward_input<G: DevicePtr<f32>, O: DevicePtrMut<f32>>(
&self,
weight: &DeviceParam,
grad_output: &G,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
gemm_plain(
self.context,
grad_output,
weight.rows(),
weight.value(),
weight.cols(),
out,
weight.cols(),
rows,
weight.cols(),
weight.rows(),
1.0,
beta,
)
}
pub(crate) fn accumulate_weight_grad<G: DevicePtr<f32>, X: DevicePtr<f32>>(
&self,
weight: &mut DeviceParam,
grad_output: &G,
input: &X,
rows: usize,
) -> Result<(), NetworkError> {
if weight.is_frozen() {
return Ok(());
}
let (units, cols) = (weight.rows(), weight.cols());
let beta = weight.grad_beta();
gemm_lhs_transposed(
self.context,
grad_output,
units,
input,
cols,
weight.negated_grad_mut(),
cols,
rows,
units,
cols,
-1.0,
beta,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn rmsnorm(
&self,
input: &CudaSlice<f32>,
weight: &CudaView<'_, f32>,
rows: usize,
cols: usize,
eps: f32,
narrow: bool,
copy: bool,
) -> Result<NormOutput, NetworkError> {
let mut out = self.act(rows * cols, narrow)?;
let mut inverse = self.uninit(rows)?;
let mut duplicate = self.uninit(if copy { rows * cols } else { 1 })?;
let do_copy = i32::from(copy);
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rmsnorm_forward)
.arg(out.destination())
.arg(&mut inverse)
.arg(input)
.arg(weight)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&eps)
.arg(&i32::from(narrow))
.arg(&mut duplicate)
.arg(&do_copy)
.launch(row_grid(rows))
.map_err(cuda_err("RMSNorm kernel"))?;
}
Ok((out, inverse, copy.then_some(duplicate)))
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_arguments)]
pub(crate) fn rmsnorm_backward(
&self,
input: &CudaSlice<f32>,
grad_output: &CudaSlice<f32>,
weight: &CudaSlice<f32>,
inverse: &CudaSlice<f32>,
grad_weight: &mut CudaViewMut<'_, f32>,
residual: &CudaSlice<f32>,
add_residual: bool,
copy: Option<bool>,
rows: usize,
cols: usize,
) -> Result<(CudaSlice<f32>, Option<Act>), NetworkError> {
let mut grad_input = self.uninit(rows * cols)?;
let add_residual = i32::from(add_residual);
let copy_mode = copy.map_or(0, |narrow| 1 + i32::from(narrow));
let mut copy = self.act(copy.map_or(1, |_| rows * cols), copy.unwrap_or(false))?;
let smem = rmsnorm_smem(cols);
let mut config = row_grid(rows);
config.shared_mem_bytes = smem.unwrap_or(0);
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rmsnorm_backward)
.arg(&mut grad_input)
.arg(grad_weight)
.arg(input)
.arg(grad_output)
.arg(weight)
.arg(inverse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(i32::from(smem.is_some())))
.arg(residual)
.arg(&add_residual)
.arg(&mut copy.bytes)
.arg(©_mode)
.launch(config)
.map_err(cuda_err("RMSNorm backward kernel"))?;
}
Ok((grad_input, (copy_mode != 0).then_some(copy)))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn rope(
&self,
tensor: &mut Act,
tables: &DeviceRope,
rows: usize,
heads: usize,
head_dim: usize,
seq_len: usize,
direction: f32,
width: usize,
offset: usize,
) -> Result<(), NetworkError> {
let elements = rows * heads * head_dim / 2;
let narrow = tensor.flag();
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rope)
.arg(tensor.destination())
.arg(&tables.cos)
.arg(&tables.sin)
.arg(&(rows as i32))
.arg(&(heads as i32))
.arg(&(head_dim as i32))
.arg(&(seq_len as i32))
.arg(&direction)
.arg(&(width as i32))
.arg(&(offset as i32))
.arg(&narrow)
.arg(&1i32)
.launch(cfg(elements))
.map_err(cuda_err("RoPE kernel"))?;
}
Ok(())
}
pub(crate) fn softmax_backward(
&self,
grad: &mut CudaSlice<f32>,
probabilities: &CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.softmax_backward)
.arg(grad)
.arg(probabilities)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&i32::from(causal))
.launch(warp_per_row_grid(rows))
.map_err(cuda_err("attention softmax backward kernel"))?;
}
Ok(())
}
pub(crate) fn softmax_lse(
&self,
scores: &mut CudaSlice<f32>,
lse: &mut CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
) -> Result<(), NetworkError> {
self.attention_softmax(scores, lse, rows, cols, causal, cols, 0)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn attention_softmax(
&self,
scores: &mut CudaSlice<f32>,
lse: &mut CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
queries: usize,
offset: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.softmax_lse_rows)
.arg(scores)
.arg(lse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&i32::from(causal))
.arg(&(queries as i32))
.arg(&(offset as i32))
.launch(warp_per_row_grid(rows))
.map_err(cuda_err("attention softmax kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn flash_attention(
&self,
flash: &crate::cuda_flash::FlashKernels,
qkv: &Act,
kv: &Act,
merged: &mut Act,
log_sum_exp: &mut CudaSlice<f32>,
shape: FlashShape,
scale: f32,
) -> Result<(), NetworkError> {
debug_assert!(qkv.is_narrow() && kv.is_narrow() && merged.is_narrow());
let config = cudarc::driver::LaunchConfig {
grid_dim: (
shape.seq_len.div_ceil(crate::cuda_flash::TILE) as u32,
shape.heads as u32,
shape.sequences as u32,
),
block_dim: (crate::cuda_flash::THREADS, 1, 1),
shared_mem_bytes: crate::cuda_flash::forward_shared_bytes(shape.head_dim()),
};
let flash = flash.width(shape.head_dim());
unsafe {
self.context
.stream
.launch_builder(&flash.forward)
.arg(&qkv.all())
.arg(&kv.all())
.arg(merged.destination())
.arg(log_sum_exp)
.arg(&(shape.seq_len as i32))
.arg(&(shape.kv_len as i32))
.arg(&(shape.qkv_width as i32))
.arg(&(shape.kv_width as i32))
.arg(&(shape.query_width as i32))
.arg(&(shape.key_base as i32))
.arg(&(shape.value_base as i32))
.arg(&(shape.group as i32))
.arg(&(shape.rows as i32))
.arg(&scale)
.arg(&i32::from(shape.causal))
.launch(config)
.map_err(cuda_err("fused attention kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn flash_attention_backward(
&self,
flash: &crate::cuda_flash::FlashKernels,
qkv: &Act,
kv: &Act,
merged: &Act,
grad_merged: &Act,
log_sum_exp: &CudaSlice<f32>,
delta: &mut CudaSlice<f32>,
grad_qkv: &Act,
grad_kv: &Act,
shape: FlashShape,
scale: f32,
) -> Result<(), NetworkError> {
let head_dim = shape.head_dim();
let flash = flash.width(head_dim);
let warps = shape.heads * shape.rows;
unsafe {
self.context
.stream
.launch_builder(&flash.delta)
.arg(&merged.all())
.arg(&grad_merged.all())
.arg(&mut *delta)
.arg(&(shape.rows as i32))
.arg(&(shape.heads as i32))
.arg(&(shape.query_width as i32))
.arg(&(shape.rows as i32))
.launch(warp_per_row_grid(warps))
.map_err(cuda_err("attention delta kernel"))?;
}
let delta = &*delta;
let grad_merged_view = grad_merged.all();
debug_assert!(qkv.is_narrow() && kv.is_narrow() && grad_merged.is_narrow());
debug_assert!(grad_qkv.is_narrow() && grad_kv.is_narrow());
let qkv = qkv.all();
let kv = kv.all();
let launch = |function, tiles: usize, grid_y, grad: &Act, shared_mem_bytes| {
let config = cudarc::driver::LaunchConfig {
grid_dim: (
tiles.div_ceil(crate::cuda_flash::TILE) as u32,
grid_y,
shape.sequences as u32,
),
block_dim: (crate::cuda_flash::THREADS, 1, 1),
shared_mem_bytes,
};
unsafe {
self.context
.stream
.launch_builder(function)
.arg(&qkv)
.arg(&kv)
.arg(&grad_merged_view)
.arg(log_sum_exp)
.arg(&*delta)
.arg(&grad.all())
.arg(&(shape.seq_len as i32))
.arg(&(shape.kv_len as i32))
.arg(&(shape.qkv_width as i32))
.arg(&(shape.kv_width as i32))
.arg(&(shape.query_width as i32))
.arg(&(shape.key_base as i32))
.arg(&(shape.value_base as i32))
.arg(&(shape.group as i32))
.arg(&(shape.rows as i32))
.arg(&scale)
.arg(&i32::from(shape.causal))
.launch(config)
.map_err(cuda_err("fused attention backward kernel"))
}
};
launch(
&flash.grad_query,
shape.seq_len,
shape.heads as u32,
grad_qkv,
crate::cuda_flash::dq_shared_bytes(head_dim),
)?;
launch(
&flash.grad_key_value,
shape.kv_len,
(shape.heads / shape.group) as u32,
grad_kv,
crate::cuda_flash::dkv_shared_bytes(head_dim),
)?;
Ok(())
}
pub(crate) fn probs_from_lse(
&self,
scores: &mut CudaSlice<f32>,
lse: &CudaSlice<f32>,
rows: usize,
cols: usize,
causal: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.probs_from_lse)
.arg(scores)
.arg(lse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&i32::from(causal))
.launch(cfg(rows * cols))
.map_err(cuda_err("attention probability kernel"))?;
}
Ok(())
}
pub(crate) fn add(
&self,
target: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
offset: usize,
len: usize,
keep: bool,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.add)
.arg(target)
.arg(source)
.arg(&(offset as i32))
.arg(&(len as i32))
.arg(&i32::from(keep))
.launch(cfg(if (len | offset) % 4 == 0 {
len / 4
} else {
len
}))
.map_err(cuda_err("addition kernel"))?;
}
Ok(())
}
pub(crate) fn gather(
&self,
out: &mut CudaViewMut<'_, f32>,
source: &CudaView<'_, f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gather)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(&0i32)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("row gather kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gather_scaled<T>(
&self,
out: &mut Act,
source: &CudaView<'_, T>,
source_narrow: bool,
rows_of: &CudaSlice<u32>,
scale: &CudaSlice<f32>,
scaled: bool,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
let narrow = out.flag();
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gather_scaled)
.arg(out.destination())
.arg(source)
.arg(rows_of)
.arg(scale)
.arg(&(rows as i32))
.arg(&(width as i32))
.arg(&i32::from(scaled))
.arg(&i32::from(source_narrow))
.arg(&narrow)
.launch(cfg(rows * width))
.map_err(cuda_err("scaled row gather kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn scatter_scaled(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
scale: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter_scaled)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(scale)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("scaled row scatter kernel"))?;
}
Ok(())
}
pub(crate) fn scatter(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("row scatter kernel"))?;
}
Ok(())
}
pub(crate) fn scatter_negated(
&self,
grad: &mut CudaSlice<f32>,
upstream: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter_negated)
.arg(grad)
.arg(upstream)
.arg(rows_of)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("embedding scatter kernel"))?;
}
Ok(())
}
pub(crate) fn swiglu(
&self,
gate_up: &CudaSlice<f32>,
rows: usize,
width: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
self.gated(gate_up, rows, width, narrow, false)
}
pub(crate) fn gated(
&self,
gate_up: &CudaSlice<f32>,
rows: usize,
width: usize,
narrow: bool,
gelu: bool,
) -> Result<Act, NetworkError> {
let mut hidden = self.act(rows * width, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.swiglu_forward)
.arg(hidden.destination())
.arg(gate_up)
.arg(&(rows as i32))
.arg(&(width as i32))
.arg(&i32::from(narrow))
.arg(&i32::from(gelu))
.launch(cfg(if width % 4 == 0 {
rows * width / 4
} else {
rows * width
}))
.map_err(cuda_err("SwiGLU kernel"))?;
}
Ok(hidden)
}
pub(crate) fn swiglu_backward(
&self,
gate_up: &CudaSlice<f32>,
grad_hidden: &CudaSlice<f32>,
rows: usize,
width: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
let mut grad_gate_up = self.act(rows * 2 * width, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.swiglu_backward)
.arg(grad_gate_up.destination())
.arg(gate_up)
.arg(grad_hidden)
.arg(&(rows as i32))
.arg(&(width as i32))
.arg(&i32::from(narrow))
.launch(cfg(if width % 4 == 0 {
rows * width / 4
} else {
rows * width
}))
.map_err(cuda_err("SwiGLU backward kernel"))?;
}
Ok(grad_gate_up)
}
pub(crate) fn adaln_modulate(
&self,
normalized: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.adaln_modulate)
.arg(&mut out)
.arg(normalized)
.arg(triple)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("AdaLN modulation kernel"))?;
}
Ok(out)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn adaln_modulate_backward(
&self,
grad_modulated: &CudaView<'_, f32>,
normalized: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
grad_triple: &mut CudaViewMut<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut grad_normalized = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.adaln_modulate_backward)
.arg(&mut grad_normalized)
.arg(grad_triple)
.arg(grad_modulated)
.arg(normalized)
.arg(triple)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("AdaLN modulation backward kernel"))?;
}
Ok(grad_normalized)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gate_residual(
&self,
residual: &CudaView<'_, f32>,
branch: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gate_residual)
.arg(&mut out)
.arg(residual)
.arg(branch)
.arg(triple)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("gated residual kernel"))?;
}
Ok(out)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gate_residual_backward(
&self,
branch: &CudaView<'_, f32>,
triple: &CudaView<'_, f32>,
grad_output: &CudaView<'_, f32>,
grad_triple: &mut CudaViewMut<'_, f32>,
rows: usize,
cols: usize,
seq_len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut grad_branch = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gate_residual_backward)
.arg(&mut grad_branch)
.arg(grad_triple)
.arg(branch)
.arg(triple)
.arg(grad_output)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(seq_len as i32))
.launch(cfg(rows * cols))
.map_err(cuda_err("gated residual backward kernel"))?;
}
Ok(grad_branch)
}
pub(crate) fn pack(&self, parts: &[&Linear], narrow: bool) -> Result<Arc<Act>, NetworkError> {
self.memo_pack(parts, narrow, false)
}
fn memo_pack(
&self,
parts: &[&Linear],
narrow: bool,
int8: bool,
) -> Result<Arc<Act>, NetworkError> {
let first = parts
.first()
.map_or(0, |part| std::ptr::from_ref(*part) as usize);
let key = (first, parts.len(), narrow && !int8);
let fresh = || -> Result<Act, NetworkError> {
if !int8 {
return self.pack_fresh(parts, narrow);
}
let wide = self.pack_fresh(parts, false)?;
let inner = parts
.first()
.map_or(Ok(0), |part| device_of(part).map(|d| d.cols()))?;
self.quantized(&wide.wide(), wide.len / inner.max(1), inner)
};
self.memoize(key, fresh)
}
fn memoize(
&self,
key: (usize, usize, bool),
fresh: impl FnOnce() -> Result<Act, NetworkError>,
) -> Result<Arc<Act>, NetworkError> {
let mut memo = self
.context
.packs
.lock()
.map_err(|_| NetworkError::Cuda("the pack memo lock was poisoned".into()))?;
if let Some(memo) = memo.as_mut() {
if let Some(packed) = memo.get(&key) {
return Ok(packed.clone());
}
let packed = Arc::new(fresh()?);
memo.insert(key, packed.clone());
return Ok(packed);
}
drop(memo);
Ok(Arc::new(fresh()?))
}
pub(crate) fn decode_norm(&self, weight: &[f32]) -> Result<Arc<Act>, NetworkError> {
self.memoize((weight.as_ptr() as usize, weight.len(), false), || {
let mut act = self.act(weight.len(), false)?;
self.context
.stream
.memcpy_htod(weight, &mut act.wide_mut())
.map_err(cuda_alloc_err("host to device copy", size_of_val(weight)))?;
Ok(act)
})
}
pub(crate) fn decode_pack(
&self,
parts: &[&Linear],
narrow: bool,
) -> Result<Arc<Act>, NetworkError> {
let int8 = self.context.int8_decode.load(Ordering::Relaxed);
self.memo_pack(parts, narrow, int8)
}
pub(crate) fn dequantized(
&self,
q: &[i8],
scales: &[f32],
rows: usize,
cols: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let q = self
.context
.stream
.clone_htod(q)
.map_err(cuda_alloc_err("host to device copy", q.len()))?;
let scales = self.upload(scales)?;
let mut wide = self.uninit(rows * cols)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.dequantize_q8)
.arg(&q)
.arg(&scales)
.arg(&mut wide)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&0i32)
.launch(cfg(rows * cols))
.map_err(cuda_err("int8 dequantize kernel"))?;
}
Ok(wide)
}
pub(crate) fn quantized(
&self,
weights: &CudaView<'_, f32>,
rows: usize,
cols: usize,
) -> Result<Act, NetworkError> {
let mut act = self.act(rows * cols / 4 + 1, false)?;
act.len = rows * cols;
let mut scales = self.uninit(rows)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.quantize_q8)
.arg(weights)
.arg(&mut act.bytes)
.arg(&mut scales)
.arg(&(rows as i32))
.arg(&(cols as i32))
.launch(warp_each(rows))
.map_err(cuda_err("int8 quantize kernel"))?;
}
act.scales = Some(scales);
Ok(act)
}
pub(crate) fn quantized_grouped(
&self,
weights: &CudaView<'_, f32>,
rows: usize,
cols: usize,
group: usize,
) -> Result<Act, NetworkError> {
let group = group.clamp(1, cols.max(1));
let groups = cols.div_ceil(group);
let mut act = self.act(rows * cols / 4 + 1, false)?;
act.len = rows * cols;
let mut scales = self.uninit(rows * groups)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.quantize_groups_q8)
.arg(weights)
.arg(&mut act.bytes)
.arg(&mut scales)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(group as i32))
.launch(warp_each(rows * groups))
.map_err(cuda_err("int8 grouped quantize kernel"))?;
}
act.scales = Some(scales);
act.group = group;
Ok(act)
}
pub(crate) fn fp8(
&self,
bytes: &[u8],
scales: &[f32],
rows: usize,
cols: usize,
) -> Result<Act, NetworkError> {
debug_assert_eq!(bytes.len(), rows * cols);
debug_assert_eq!(scales.len(), rows);
Ok(Act {
bytes: self
.context
.stream
.clone_htod(bytes)
.map_err(cuda_alloc_err("host to device copy", bytes.len()))?,
narrow: false,
len: rows * cols,
scales: Some(self.upload(scales)?),
group: 0,
fp8: true,
})
}
pub(crate) fn widened_fp8(
&self,
weights: &Act,
first: usize,
units: usize,
inner: usize,
narrow: bool,
) -> Result<Act, NetworkError> {
debug_assert!(weights.fp8);
let scales = weights.scales.as_ref().expect("an FP8 weight has scales");
let mut wide = self.act(units * inner, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.dequantize_fp8)
.arg(&weights.bytes.slice(first * inner..))
.arg(&scales.slice(first..))
.arg(&mut wide.bytes)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&i32::from(narrow))
.launch(cfg(units * inner))
.map_err(cuda_err("FP8 dequantize kernel"))?;
}
Ok(wide)
}
pub(crate) fn quantized_e4m3_rows(
&self,
x: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
inner: usize,
) -> Result<(CudaSlice<u8>, CudaSlice<f32>), NetworkError> {
let mut q = unsafe { self.context.stream.alloc::<u8>((rows * inner).max(1)) }
.map_err(cuda_alloc_err("device allocation", rows * inner))?;
let mut scales = self.uninit(rows)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.quantize_e4m3)
.arg(x)
.arg(&i32::from(narrow))
.arg(&(inner as i32))
.arg(&mut q)
.arg(&mut scales)
.launch(cudarc::driver::LaunchConfig {
grid_dim: (rows as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
})
.map_err(cuda_err("E4M3 row quantize kernel"))?;
}
Ok((q, scales))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn project_fp8_mma(
&self,
gemm: &CudaFunction,
weights: &Act,
first: usize,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut CudaViewMut<'_, f32>,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
debug_assert!(weights.fp8);
if rows == 0 || units == 0 {
return Ok(());
}
let scales = weights.scales.as_ref().expect("an FP8 weight has scales");
let (xq, xs) = self.quantized_e4m3_rows(x, narrow, rows, inner)?;
let tile = crate::cuda_fp8::TILE;
unsafe {
self.context
.stream
.launch_builder(gemm)
.arg(&xq)
.arg(&xs)
.arg(&weights.bytes.slice(first * inner..))
.arg(&scales.slice(first..))
.arg(out)
.arg(&(rows as i32))
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&beta)
.launch(cudarc::driver::LaunchConfig {
grid_dim: (units.div_ceil(tile) as u32, rows.div_ceil(tile) as u32, 1),
block_dim: (128, 1, 1),
shared_mem_bytes: 0,
})
.map_err(cuda_err("FP8 tensor-core GEMM kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn project(
&self,
weights: &Act,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut CudaViewMut<'_, f32>,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
self.project_at(weights, 0, units, inner, x, narrow, out, rows, beta)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn project_at(
&self,
weights: &Act,
first: usize,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut CudaViewMut<'_, f32>,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
let Some(scales) = &weights.scales else {
let rows_of = weights.at(first * inner);
return self.linear_packed(&rows_of, units, inner, x, narrow, out, rows, beta);
};
if weights.fp8 {
if let Some(gemm) = &self.context.fp8_gemm {
return self.project_fp8_mma(
gemm, weights, first, units, inner, x, narrow, out, rows, beta,
);
}
#[cfg(test)]
if std::env::var_os("RB_TEST_FP8_ACTIVATIONS").is_some() {
let (bytes, scales) = self.quantized_e4m3_rows(x, narrow, rows, inner)?;
let codes = Act {
bytes,
narrow: false,
len: rows * inner,
scales: Some(scales),
group: 0,
fp8: true,
};
let x8 = self.widened_fp8(&codes, 0, rows, inner, narrow)?;
let wide = self.widened_fp8(weights, first, units, inner, narrow)?;
return self.linear_packed(
&wide.all(),
units,
inner,
&x8.all(),
narrow,
out,
rows,
beta,
);
}
let wide = self.widened_fp8(weights, first, units, inner, narrow)?;
return self.linear_packed(&wide.all(), units, inner, x, narrow, out, rows, beta);
}
let bytes = weights.bytes.slice(first * inner..);
if weights.group > 0 {
let groups = inner.div_ceil(weights.group);
let scales = scales.slice(first * groups..);
let mut wide = self.act(units * inner, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.dequantize_groups_q8)
.arg(&bytes)
.arg(&scales)
.arg(&mut wide.bytes)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&i32::from(narrow))
.arg(&(weights.group as i32))
.launch(cfg(units * inner))
.map_err(cuda_err("int8 grouped dequantize kernel"))?;
}
return self.linear_packed(&wide.all(), units, inner, x, narrow, out, rows, beta);
}
let scales = scales.slice(first..);
if rows > Q8_GEMV_ROWS || inner % 16 != 0 {
let mut wide = self.act(units * inner, narrow)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.dequantize_q8)
.arg(&bytes)
.arg(&scales)
.arg(&mut wide.bytes)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&i32::from(narrow))
.launch(cfg(units * inner))
.map_err(cuda_err("int8 dequantize kernel"))?;
}
return self.linear_packed(&wide.all(), units, inner, x, narrow, out, rows, beta);
}
let mma = self
.context
.flash
.as_ref()
.filter(|_| narrow && inner % 32 == 0 && rows >= Q8_MMA_ROWS);
if let Some(flash) = mma {
unsafe {
self.context
.stream
.launch_builder(&flash.gemv_q8)
.arg(&bytes)
.arg(&scales)
.arg(x)
.arg(out)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&(rows as i32))
.arg(&beta)
.launch(cudarc::driver::LaunchConfig {
grid_dim: (units.div_ceil(16) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
})
.map_err(cuda_err("int8 tensor-core GEMV kernel"))?;
}
return Ok(());
}
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gemv_q8)
.arg(&bytes)
.arg(&scales)
.arg(x)
.arg(&i32::from(narrow))
.arg(out)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&(rows as i32))
.arg(&beta)
.launch(warp_each(units))
.map_err(cuda_err("int8 GEMV kernel"))?;
}
Ok(())
}
fn pack_fresh(&self, parts: &[&Linear], narrow: bool) -> Result<Act, NetworkError> {
let element = Act::element(narrow);
let mut total = 0;
for part in parts {
let device = device_of(part)?;
total += device.rows() * device.cols();
}
let mut packed = self.act(total, narrow)?;
let mut base = 0;
for part in parts {
let device = device_of(part)?;
let (units, inner) = (device.rows(), device.cols());
let len = units * inner;
let mut slot = packed
.bytes
.slice_mut(base * element..(base + len) * element);
match device.int8() {
Some((bytes, scales)) => unsafe {
self.context
.stream
.launch_builder(&self.context.model.dequantize_q8)
.arg(bytes)
.arg(scales)
.arg(&mut slot)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&i32::from(narrow))
.launch(cfg(len))
.map_err(cuda_err("int8 dequantize kernel"))?;
},
None => match device.four_bit() {
Some((packed, scales, levels)) => unsafe {
self.context
.stream
.launch_builder(&self.context.model.dequantize_q4)
.arg(packed)
.arg(scales)
.arg(levels)
.arg(&mut slot)
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&i32::from(narrow))
.launch(cfg(len))
.map_err(cuda_err("4-bit dequantize kernel"))?;
},
None => self.cast_into(&mut slot, &device.value().slice(..), len, narrow)?,
},
}
if let Some(lora) = &part.lora {
self.fold_lora(lora, &mut slot, units, inner, narrow)?;
}
base += len;
}
Ok(packed)
}
pub(crate) fn fold_lora(
&self,
lora: &Lora,
slot: &mut CudaViewMut<'_, u8>,
units: usize,
inner: usize,
narrow: bool,
) -> Result<(), NetworkError> {
let down = device_param(&lora.down)?;
let up = device_param(&lora.up)?;
let rank = down.rows();
let up = self.narrowed(&up.value().slice(..), units * rank, narrow)?;
let down = self.narrowed(&down.value().slice(..), rank * inner, narrow)?;
act_plain(
self.context,
&up.all(),
rank,
&down.all(),
inner,
narrow,
narrow,
slot,
inner,
units,
inner,
rank,
lora.scale,
1.0,
)
}
pub(crate) fn pack_gate_up<'a>(
&self,
ffns: impl Iterator<Item = &'a SwiGlu>,
narrow: bool,
) -> Result<Arc<Act>, NetworkError> {
let mut parts = Vec::new();
for ffn in ffns {
parts.push(&ffn.gate);
parts.push(&ffn.up);
}
self.decode_pack(&parts, narrow)
}
pub(crate) fn pack_down<'a>(
&self,
ffns: impl Iterator<Item = &'a SwiGlu>,
narrow: bool,
) -> Result<Arc<Act>, NetworkError> {
let parts: Vec<&Linear> = ffns.map(|ffn| &ffn.down).collect();
self.decode_pack(&parts, narrow)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn linear_packed<O: DevicePtrMut<f32>>(
&self,
weights: &CudaView<'_, u8>,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
act_rhs_transposed(
self.context,
x,
inner,
weights,
inner,
narrow,
false,
out,
units,
rows,
units,
inner,
1.0,
beta,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn linear_packed_act(
&self,
weights: &CudaView<'_, u8>,
units: usize,
inner: usize,
x: &CudaView<'_, u8>,
narrow: bool,
out: &mut Act,
rows: usize,
) -> Result<(), NetworkError> {
let out_narrow = out.is_narrow();
act_rhs_transposed(
self.context,
x,
inner,
weights,
inner,
narrow,
out_narrow,
out.destination(),
units,
rows,
units,
inner,
1.0,
0.0,
)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn linear_packed_backward_input<T, O: DevicePtrMut<T>>(
&self,
weights: &CudaView<'_, u8>,
units: usize,
inner: usize,
grad_output: &CudaView<'_, u8>,
narrow: bool,
out_narrow: bool,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
act_plain(
self.context,
grad_output,
units,
weights,
inner,
narrow,
out_narrow,
out,
inner,
rows,
inner,
units,
1.0,
beta,
)
}
pub(crate) fn accumulate_weight_grad_act(
&self,
weight: &mut DeviceParam,
grad_output: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
if weight.is_frozen() {
return Ok(());
}
let (units, cols) = (weight.rows(), weight.cols());
let beta = weight.grad_beta();
act_lhs_transposed(
self.context,
grad_output,
units,
input,
cols,
narrow,
false,
weight.negated_grad_mut(),
cols,
rows,
units,
cols,
-1.0,
beta,
)
}
pub(crate) fn accumulate_packed_grad(
&self,
parts: &mut [&mut DeviceParam],
grad: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
if parts.iter().all(|part| part.is_frozen()) {
return Ok(());
}
debug_assert!(
parts.iter().all(|part| !part.is_frozen()),
"a packed projection freezes as a unit"
);
let inner = parts[0].cols();
let units: usize = parts.iter().map(|part| part.rows()).sum();
let mut scratch = self.uninit(units * inner)?;
act_lhs_transposed(
self.context,
grad,
units,
input,
inner,
narrow,
false,
&mut scratch,
inner,
rows,
units,
inner,
-1.0,
0.0,
)?;
let mut base = 0;
for part in parts {
let len = part.rows() * inner;
let keep = part.grad_beta() != 0.0;
self.add(part.negated_grad_mut(), &scratch, base, len, keep)?;
base += len;
}
Ok(())
}
pub(crate) fn accumulate_projection_grad(
&self,
linear: &mut Linear,
grad_output: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
let units = linear.out_features();
self.accumulate_weight_grad_act(device_of_mut(linear)?, grad_output, input, narrow, rows)?;
match &mut linear.lora {
Some(lora) => self.accumulate_lora_grad(lora, grad_output, units, input, narrow, rows),
None => Ok(()),
}
}
pub(crate) fn accumulate_packed_projection_grad(
&self,
parts: &mut [&mut Linear],
grad_output: &CudaView<'_, u8>,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
let stride: usize = parts.iter().map(|part| part.out_features()).sum();
{
let mut devices = Vec::with_capacity(parts.len());
for part in parts.iter_mut() {
devices.push(device_of_mut(part)?);
}
self.accumulate_packed_grad(&mut devices, grad_output, input, narrow, rows)?;
}
let element = Act::element(narrow);
let mut base = 0;
for part in parts.iter_mut() {
let units = part.out_features();
if let Some(lora) = &mut part.lora {
self.accumulate_lora_grad(
lora,
&grad_output.slice(base * element..),
stride,
input,
narrow,
rows,
)?;
}
base += units;
}
Ok(())
}
pub(crate) fn accumulate_lora_grad(
&self,
lora: &mut Lora,
grad_output: &CudaView<'_, u8>,
grad_stride: usize,
input: &CudaView<'_, u8>,
narrow: bool,
rows: usize,
) -> Result<(), NetworkError> {
let (rank, inner) = {
let down = device_param(&lora.down)?;
(down.rows(), down.cols())
};
let units = device_param(&lora.up)?.rows();
let down_act = self.narrowed(
&device_param(&lora.down)?.value().slice(..),
rank * inner,
narrow,
)?;
let up_act = self.narrowed(
&device_param(&lora.up)?.value().slice(..),
units * rank,
narrow,
)?;
let mut hidden = self.act(rows * rank, narrow)?;
act_rhs_transposed(
self.context,
input,
inner,
&down_act.all(),
inner,
narrow,
narrow,
hidden.destination(),
rank,
rows,
rank,
inner,
lora.scale,
0.0,
)?;
{
let up = device_param_mut(&mut lora.up)?;
let beta = up.grad_beta();
act_lhs_transposed(
self.context,
grad_output,
grad_stride,
&hidden.all(),
rank,
narrow,
false,
up.negated_grad_mut(),
rank,
rows,
units,
rank,
-1.0,
beta,
)?;
}
let mut grad_hidden = self.act(rows * rank, narrow)?;
act_plain(
self.context,
grad_output,
grad_stride,
&up_act.all(),
rank,
narrow,
narrow,
grad_hidden.destination(),
rank,
rows,
rank,
units,
lora.scale,
0.0,
)?;
let down = device_param_mut(&mut lora.down)?;
let beta = down.grad_beta();
act_lhs_transposed(
self.context,
&grad_hidden.all(),
rank,
input,
inner,
narrow,
false,
down.negated_grad_mut(),
inner,
rows,
rank,
inner,
-1.0,
beta,
)
}
}
fn device_param(param: &Param) -> Result<&DeviceParam, NetworkError> {
param
.device
.as_ref()
.ok_or_else(|| NetworkError::Cuda("a LoRA adapter is not resident on the device".into()))
}
fn device_param_mut(param: &mut Param) -> Result<&mut DeviceParam, NetworkError> {
param
.device
.as_mut()
.ok_or_else(|| NetworkError::Cuda("a LoRA adapter is not resident on the device".into()))
}
fn head_of(model: &TransformerLm) -> Result<&DeviceParam, NetworkError> {
match &model.lm_head {
Some(head) => device_of(head),
None => model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
}),
}
}
fn head_of_mut(model: &mut TransformerLm) -> Result<&mut DeviceParam, NetworkError> {
match &mut model.lm_head {
Some(head) => device_of_mut(head),
None => model.embedding.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
}),
}
}
pub(crate) fn device_of(linear: &Linear) -> Result<&DeviceParam, NetworkError> {
linear
.weight
.device
.as_ref()
.ok_or_else(|| NetworkError::Cuda("a projection is not resident on the device".into()))
}
pub(crate) fn device_of_mut(linear: &mut Linear) -> Result<&mut DeviceParam, NetworkError> {
linear
.weight
.device
.as_mut()
.ok_or_else(|| NetworkError::Cuda("a projection is not resident on the device".into()))
}
pub(crate) fn unsupported(kind: &str) -> NetworkError {
NetworkError::UnsupportedCuda(format!(
"the device training path implements SwiGLU feed-forwards and MoE layers, not {kind}"
))
}
pub(crate) fn forward(
model: &TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<(Matrix, GpuCache), NetworkError> {
let cache = forward_hidden(model, context, batch)?;
let gpu = Gpu { context };
let rows = cache.stack.rows;
let logits = gpu.linear(head_of(model)?, &cache.final_output.wide(), rows)?;
let logits = gpu.matrix(&logits, rows, model.embedding.vocab_size())?;
Ok((logits, cache))
}
fn forward_hidden(
model: &TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<GpuCache, NetworkError> {
let gpu = Gpu { context };
let rows = batch.rows();
let seq_len = batch.seq_len();
let sequences = batch.batch();
let d_model = model.config.d_model;
let vocab = model.embedding.vocab_size();
for &id in batch.ids() {
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
}
let layout = batch.layout();
let flags: Vec<i32> = (0..rows)
.map(|row| i32::from(layout.is_valid(row)))
.collect();
let valid_tokens = flags.iter().filter(|&&flag| flag != 0).count();
let valid = gpu.upload_flags(&flags)?;
let ids = gpu.upload_indices(batch.ids())?;
let embedding = model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
let mut hidden = gpu.uninit(rows * d_model)?;
gpu.gather(
&mut hidden.slice_mut(..),
&embedding.value().slice(..),
&ids,
rows,
d_model,
)?;
let (hidden, stack) = forward_stack(
&gpu,
&model.blocks,
&model.checkpoint,
hidden,
valid,
valid_tokens,
rows,
seq_len,
sequences,
)?;
let final_weight = gpu.upload(&model.final_norm.weight.value.data)?;
let (final_output, final_inverse_rms, _) = gpu.rmsnorm(
&hidden,
&final_weight.as_view(),
rows,
d_model,
model.final_norm.eps,
false,
false,
)?;
Ok(GpuCache {
stack,
ids,
final_input: hidden,
final_inverse_rms,
final_weight,
final_output,
})
}
#[allow(clippy::too_many_arguments)]
fn forward_stack(
gpu: &Gpu<'_>,
blocks: &[TransformerBlock],
checkpoint: &[bool],
mut hidden: CudaSlice<f32>,
valid: CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
seq_len: usize,
sequences: usize,
) -> Result<(CudaSlice<f32>, GpuStack), NetworkError> {
let mut caches = Vec::with_capacity(blocks.len());
let mut loads = Vec::new();
let mut auxiliary_loss = 0.0;
for (index, block) in blocks.iter().enumerate() {
let (output, cache) = forward_block(
gpu,
block,
hidden,
&valid,
valid_tokens,
rows,
seq_len,
sequences,
&mut auxiliary_loss,
)?;
if let FfnCache::Moe(moe) = &cache.feed_forward {
let slots = moe.routed.max(1) as f32;
loads.push(moe.counts.iter().map(|&n| n as f32 / slots).collect());
}
caches.push(if checkpoint.get(index) == Some(&true) {
Saved::Input(cache.input)
} else {
Saved::Cache(cache)
});
hidden = output;
}
Ok((
hidden,
GpuStack {
rows,
seq_len,
sequences,
valid_tokens,
valid,
blocks: caches,
loads,
auxiliary_loss,
},
))
}
pub(crate) fn stack_forward(
blocks: &[TransformerBlock],
context: &Arc<GpuContext>,
input: &Matrix,
seq_len: usize,
) -> Result<(Matrix, GpuStack), NetworkError> {
let gpu = Gpu { context };
let rows = input.rows;
let valid = gpu.upload_flags(&vec![1; rows])?;
let hidden = gpu.upload(&input.data)?;
let (hidden, stack) = forward_stack(
&gpu,
blocks,
&[],
hidden,
valid,
rows,
rows,
seq_len,
rows / seq_len,
)?;
Ok((gpu.matrix(&hidden, rows, input.cols)?, stack))
}
pub(crate) fn stack_backward(
blocks: &mut [TransformerBlock],
context: &Arc<GpuContext>,
stack: &GpuStack,
grad_output: &Matrix,
) -> Result<Matrix, NetworkError> {
let gpu = Gpu { context };
let rows = stack.rows;
let d_model = grad_output.cols;
if grad_output.rows != rows {
return Err(NetworkError::InvalidInput {
expected: rows,
actual: grad_output.rows,
});
}
let grad = gpu.upload(&grad_output.data)?;
let grad_act = match stack_operand(&gpu, blocks, blocks.len().checked_sub(1)) {
Some(narrow) => Some(gpu.narrowed(&grad.slice(..), rows * d_model, narrow)?),
None => None,
};
let mut grad_norms = gpu.zeros(norm_slots(blocks.len()) * d_model)?;
let grad = backward_stack(&gpu, blocks, stack, grad, grad_act, &mut grad_norms)?;
let values = gpu.download(&grad_norms)?;
accumulate_block_norms(blocks, &values, d_model);
gpu.matrix(&grad, rows, d_model)
}
fn stack_operand(gpu: &Gpu<'_>, blocks: &[TransformerBlock], index: Option<usize>) -> Option<bool> {
match blocks.get(index?)?.feed_forward {
FeedForward::SwiGlu(_) => Some(gpu.context.mixed_precision),
_ => None,
}
}
fn backward_stack(
gpu: &Gpu<'_>,
blocks: &mut [TransformerBlock],
stack: &GpuStack,
mut grad_hidden: CudaSlice<f32>,
mut grad_hidden_act: Option<Act>,
grad_norms: &mut CudaSlice<f32>,
) -> Result<CudaSlice<f32>, NetworkError> {
let operands: Vec<_> = (0..blocks.len())
.map(|index| stack_operand(gpu, blocks, index.checked_sub(1)))
.collect();
for (index, (block, saved)) in blocks.iter_mut().zip(&stack.blocks).enumerate().rev() {
let recomputed;
let block_cache = match saved {
Saved::Cache(cache) => cache,
Saved::Input(input) => {
recomputed = forward_block(
gpu,
block,
input.clone(),
&stack.valid,
stack.valid_tokens,
stack.rows,
stack.seq_len,
stack.sequences,
&mut 0.0,
)?
.1;
&recomputed
}
};
(grad_hidden, grad_hidden_act) = backward_block(
gpu,
block,
block_cache,
&grad_hidden,
grad_hidden_act.as_ref(),
operands[index],
stack,
grad_norms,
index,
)?;
}
Ok(grad_hidden)
}
fn accumulate_block_norms(blocks: &mut [TransformerBlock], values: &[f32], d_model: usize) {
let at = |slot: usize| &values[slot * d_model..(slot + 1) * d_model];
for (index, block) in blocks.iter_mut().enumerate() {
let (attention, feed_forward) = norm_slot(index);
Gpu::accumulate_host_grad(&mut block.attention_norm.weight, at(attention));
Gpu::accumulate_host_grad(&mut block.feed_forward_norm.weight, at(feed_forward));
}
}
#[allow(clippy::too_many_arguments)]
fn forward_block(
gpu: &Gpu<'_>,
block: &TransformerBlock,
input: CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
seq_len: usize,
sequences: usize,
auxiliary_loss: &mut f32,
) -> Result<(CudaSlice<f32>, BlockCache), NetworkError> {
let attention = &block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let causal = attention.is_causal();
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let narrow = gpu.context.mixed_precision;
let ffn_narrow = narrow;
let attention_weight = gpu.upload(&block.attention_norm.weight.value.data)?;
let (attention_normed, attention_inverse_rms, residual) = gpu.rmsnorm(
&input,
&attention_weight.as_view(),
rows,
d_model,
block.attention_norm.eps,
narrow,
true,
)?;
let mut residual = residual.expect("a norm asked for a duplicate returns one");
let kv_width = kv_heads * head_dim;
let qkv_width = query_width + 2 * kv_width;
let fused = gpu
.context
.flash
.as_ref()
.filter(|_| crate::cuda_flash::eligible(gpu.context.mixed_precision, head_dim));
let qkv_weights = gpu.pack(
&[&attention.query, &attention.key, &attention.value],
narrow,
)?;
let mut qkv = gpu.act(rows * qkv_width, narrow && fused.is_some())?;
gpu.linear_packed_act(
&qkv_weights.all(),
qkv_width,
d_model,
&attention_normed.all(),
narrow,
&mut qkv,
rows,
)?;
let key_base = query_width;
let value_base = query_width + kv_width;
let tables = gpu.rope_tables(&attention.rope)?;
if seq_len > tables.max_seq_len {
return Err(NetworkError::SequenceTooLong {
length: seq_len,
max_seq_len: tables.max_seq_len,
});
}
gpu.rope(
&mut qkv, &tables, rows, heads, head_dim, seq_len, 1.0, qkv_width, 0,
)?;
gpu.rope(
&mut qkv, &tables, rows, kv_heads, head_dim, seq_len, 1.0, qkv_width, key_base,
)?;
let group = heads / kv_heads;
let shape = FlashShape {
rows,
seq_len,
kv_len: seq_len,
sequences,
heads,
group,
qkv_width,
kv_width: qkv_width,
query_width,
key_base,
value_base,
causal,
};
let mut log_sum_exp = gpu.uninit(heads * rows)?;
let mut merged = gpu.act(rows * query_width, narrow && fused.is_some())?;
if let Some(flash) = fused {
gpu.flash_attention(
flash,
&qkv,
&qkv,
&mut merged,
&mut log_sum_exp,
shape,
scale,
)?;
} else {
let mut wide_merged = merged.wide_mut();
let qkv = qkv.wide();
let block_size = seq_len * seq_len;
let mut probabilities = gpu.uninit(heads * sequences * block_size)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let query = qkv.slice(head * head_dim..);
let key = qkv.slice(key_base + kv_base..);
let mut scores = probabilities.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
qkv_width,
&key,
qkv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
scale,
0.0,
sequences,
seq_len * qkv_width,
seq_len * qkv_width,
block_size,
)?;
}
gpu.softmax_lse(
&mut probabilities,
&mut log_sum_exp,
heads * sequences * seq_len,
seq_len,
causal,
)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let head_probabilities = probabilities.slice(head * sequences * block_size..);
let value = qkv.slice(value_base + kv_base..);
let mut head_merged = wide_merged.slice_mut(head * head_dim..);
gemm_plain_batched(
gpu.context,
&head_probabilities,
seq_len,
&value,
qkv_width,
&mut head_merged,
query_width,
seq_len,
head_dim,
seq_len,
1.0,
0.0,
sequences,
block_size,
seq_len * qkv_width,
seq_len * query_width,
)?;
}
}
let output_weight = gpu.pack(&[&attention.output], merged.is_narrow())?;
gpu.linear_packed(
&output_weight.all(),
d_model,
query_width,
&merged.all(),
merged.is_narrow(),
&mut residual,
rows,
1.0,
)?;
let feed_forward_weight = gpu.upload(&block.feed_forward_norm.weight.value.data)?;
let (feed_forward_normed, feed_forward_inverse_rms, output) = gpu.rmsnorm(
&residual,
&feed_forward_weight.as_view(),
rows,
d_model,
block.feed_forward_norm.eps,
ffn_narrow,
true,
)?;
let mut output = output.expect("a norm asked for a duplicate returns one");
let feed_forward = match &block.feed_forward {
FeedForward::SwiGlu(ffn) => FfnCache::Dense(forward_swiglu(
gpu,
ffn,
&feed_forward_normed,
&mut output,
rows,
)?),
FeedForward::Gelu(_) => return Err(unsupported("a GELU MLP")),
FeedForward::Moe(moe) => {
let (cache, losses) = forward_moe(
gpu,
moe,
&feed_forward_normed,
&mut output,
valid,
valid_tokens,
rows,
)?;
*auxiliary_loss += losses;
FfnCache::Moe(Box::new(cache))
}
};
Ok((
output,
BlockCache {
input,
attention_weight,
attention_inverse_rms,
attention_normed,
qkv,
qkv_weights,
log_sum_exp,
merged,
output_weight,
residual,
feed_forward_weight,
feed_forward_inverse_rms,
feed_forward_normed,
feed_forward,
},
))
}
pub(crate) fn forward_swiglu(
gpu: &Gpu<'_>,
ffn: &SwiGlu,
input: &Act,
out: &mut CudaSlice<f32>,
rows: usize,
) -> Result<SwiGluCache, NetworkError> {
let width = ffn.d_ff();
let inner = ffn.d_model();
let narrow = input.is_narrow();
let weights = gpu.decode_pack(&[&ffn.gate, &ffn.up], narrow)?;
let mut gate_up = gpu.uninit(rows * 2 * width)?;
gpu.project(
&weights,
2 * width,
inner,
&input.all(),
narrow,
&mut gate_up.slice_mut(..),
rows,
0.0,
)?;
let hidden = gpu.swiglu(&gate_up, rows, width, narrow)?;
let down = gpu.decode_pack(&[&ffn.down], narrow)?;
gpu.project(
&down,
inner,
width,
&hidden.all(),
narrow,
&mut out.slice_mut(..),
rows,
1.0,
)?;
Ok(SwiGluCache {
gate_up,
hidden,
weights,
down: Some(down),
})
}
#[allow(clippy::type_complexity)]
fn route(
gpu: &Gpu<'_>,
moe: &MoeLayer,
input: &Act,
valid: &CudaSlice<i32>,
rows: usize,
) -> Result<
(
CudaSlice<f32>,
CudaSlice<f32>,
CudaSlice<i32>,
CudaSlice<f32>,
),
NetworkError,
> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let narrow = input.is_narrow();
let router = gpu.decode_pack(&[&moe.router.projection], narrow)?;
let mut logits = gpu.uninit(rows * experts)?;
gpu.project(
&router,
experts,
d_model,
&input.all(),
narrow,
&mut logits.slice_mut(..),
rows,
0.0,
)?;
let mut probabilities = gpu.zeros(rows * experts)?;
let mut log_sum_exp = gpu.zeros(rows)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.softmax_lse)
.arg(&mut probabilities)
.arg(&mut log_sum_exp)
.arg(&logits)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.map_err(cuda_err("router softmax kernel"))?;
}
let mut expert_of = gpu
.context
.stream
.alloc_zeros::<i32>(rows * top_k)
.map_err(cuda_err("device allocation"))?;
let mut gate_of = gpu.zeros(rows * top_k)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.topk_gate)
.arg(&mut expert_of)
.arg(&mut gate_of)
.arg(&probabilities)
.arg(valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.map_err(cuda_err("router top-k kernel"))?;
}
Ok((probabilities, log_sum_exp, expert_of, gate_of))
}
fn decode_moe(
gpu: &Gpu<'_>,
moe: &MoeLayer,
input: &Act,
out: &mut CudaSlice<f32>,
valid: &CudaSlice<i32>,
rows: usize,
) -> Result<(), NetworkError> {
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let narrow = input.is_narrow();
let (_, _, expert_of, gate_of) = route(gpu, moe, input, valid, rows)?;
let slots = rows * top_k;
let gate_up_weights = gpu.pack_gate_up(moe.experts.iter(), narrow)?;
let mut gate_up = gpu.uninit(slots * 2 * width)?;
let routed_gemv = |weights: &Act,
x: &CudaView<'_, u8>,
y: &mut CudaSlice<f32>,
gated: bool,
outs: usize,
per: usize,
x_div: usize,
units: usize,
inner: usize,
beta: f32| unsafe {
let kernels = &gpu.context.model;
let mut launch = match &weights.scales {
Some(scales) => {
let mut launch = gpu.context.stream.launch_builder(&kernels.gemv_routed_q8);
launch.arg(&weights.bytes).arg(scales);
launch
}
None => {
let mut launch = gpu.context.stream.launch_builder(&kernels.gemv_routed);
launch.arg(&weights.bytes);
launch
}
};
launch
.arg(x)
.arg(&i32::from(narrow))
.arg(y)
.arg(&expert_of)
.arg(&gate_of)
.arg(&i32::from(gated))
.arg(&(outs as i32))
.arg(&(per as i32))
.arg(&(x_div as i32))
.arg(&(units as i32))
.arg(&(inner as i32))
.arg(&beta)
.launch(warp_each(outs * units))
.map(|_| ())
.map_err(cuda_err("routed expert GEMV kernel"))
};
routed_gemv(
&gate_up_weights,
&input.all(),
&mut gate_up,
false,
slots,
1,
top_k,
2 * width,
d_model,
0.0,
)?;
let hidden = gpu.swiglu(&gate_up, slots, width, narrow)?;
let down_weights = gpu.pack_down(moe.experts.iter(), narrow)?;
routed_gemv(
&down_weights,
&hidden.all(),
out,
true,
rows,
top_k,
1,
d_model,
width,
1.0,
)?;
if let Some(module) = &moe.shared {
forward_swiglu(gpu, module, input, out, rows)?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn forward_moe(
gpu: &Gpu<'_>,
moe: &MoeLayer,
input: &Act,
out: &mut CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
) -> Result<(MoeGpuCache, f32), NetworkError> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let narrow = input.is_narrow();
let (probabilities, log_sum_exp, expert_of, gate_of) = route(gpu, moe, input, valid, rows)?;
let assigned = gpu.download_signed(&expert_of)?;
let gates_host = gpu.download(&gate_of)?;
let mut counts = vec![0usize; experts];
for &expert in &assigned {
if expert >= 0 {
counts[expert as usize] += 1;
}
}
let mut offsets = Vec::with_capacity(experts);
let mut routed = 0;
for &count in &counts {
offsets.push(routed);
routed += count;
}
let mut cursor = offsets.clone();
let mut token_host = vec![0u32; routed.max(1)];
let mut slot_host = vec![0u32; routed.max(1)];
let mut gate_host = vec![0f32; routed.max(1)];
for (slot, &expert) in assigned.iter().enumerate() {
if expert < 0 {
continue;
}
let position = &mut cursor[expert as usize];
token_host[*position] = (slot / top_k) as u32;
slot_host[*position] = slot as u32;
gate_host[*position] = gates_host[slot];
*position += 1;
}
let token_of = gpu.upload_indices(&token_host)?;
let slot_of = gpu.upload_indices(&slot_host)?;
let gates = gpu.upload(&gate_host)?;
let mut gathered = gpu.act(routed * d_model, narrow)?;
if routed > 0 {
gpu.gather_scaled(
&mut gathered,
&input.all(),
narrow,
&token_of,
&gates,
false,
routed,
d_model,
)?;
}
let expert_weights = gpu.pack_gate_up(moe.experts.iter(), narrow)?;
let mut gate_up = gpu.uninit(routed * 2 * width)?;
for (expert, _) in moe.experts.iter().enumerate() {
let count = counts[expert];
if count == 0 {
continue;
}
let rows_in = gathered.at(offsets[expert] * d_model);
let mut out = gate_up.slice_mut(offsets[expert] * 2 * width..);
gpu.project_at(
&expert_weights,
expert * 2 * width,
2 * width,
d_model,
&rows_in,
narrow,
&mut out,
count,
0.0,
)?;
}
let hidden = gpu.swiglu(&gate_up, routed, width, narrow)?;
let down_weights = gpu.pack_down(moe.experts.iter(), narrow)?;
let mut expert_output = gpu.uninit(routed * d_model)?;
for (expert, _) in moe.experts.iter().enumerate() {
let count = counts[expert];
if count == 0 {
continue;
}
let rows_in = hidden.at(offsets[expert] * width);
let mut rows_out = expert_output.slice_mut(offsets[expert] * d_model..);
gpu.project_at(
&down_weights,
expert * d_model,
d_model,
width,
&rows_in,
narrow,
&mut rows_out,
count,
0.0,
)?;
}
if routed > 0 {
gpu.scatter_scaled(out, &expert_output, &token_of, &gates, routed, d_model)?;
}
let shared = match &moe.shared {
Some(module) => Some(forward_swiglu(gpu, module, input, out, rows)?),
None => None,
};
let mut sums = gpu.zeros(experts + 1)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.stats)
.arg(&mut sums)
.arg(&probabilities)
.arg(&log_sum_exp)
.arg(valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.map_err(cuda_err("MoE statistics kernel"))?;
}
let sums = gpu.download(&sums)?;
let mut auxiliary = 0.0;
if valid_tokens > 0 {
let slots = (valid_tokens * top_k) as f32;
let balance: f32 = (0..experts)
.map(|expert| (counts[expert] as f32 / slots) * (sums[expert] / valid_tokens as f32))
.sum();
auxiliary += experts as f32 * moe.config.aux_loss_weight * balance;
auxiliary += moe.config.router_z_loss_weight * sums[experts] / valid_tokens as f32;
}
Ok((
MoeGpuCache {
probabilities,
log_sum_exp,
expert_of,
token_of,
slot_of,
gates,
offsets,
counts,
routed,
input: gathered,
expert: SwiGluCache {
gate_up,
hidden,
weights: expert_weights,
down: Some(down_weights),
},
output: expert_output,
shared,
},
auxiliary,
))
}
pub(crate) fn backward(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
cache: &GpuCache,
grad_logits: &Matrix,
) -> Result<(), NetworkError> {
let gpu = Gpu { context };
let rows = cache.stack.rows;
let d_model = model.config.d_model;
if grad_logits.rows != rows {
return Err(NetworkError::InvalidInput {
expected: rows,
actual: grad_logits.rows,
});
}
let upstream = gpu.upload(&grad_logits.data)?;
let mut grad_final = gpu.zeros(rows * d_model)?;
{
let head = head_of_mut(model)?;
gpu.linear_backward_input(head, &upstream, &mut grad_final, rows, 0.0)?;
gpu.accumulate_weight_grad(head, &upstream, &cache.final_output.wide(), rows)?;
}
backward_from_final(model, context, cache, &grad_final)
}
pub(crate) fn train_step(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<(f32, f32), NetworkError> {
let vocab = model.embedding.vocab_size();
let predicted = batch.predicted();
if predicted == 0 {
return Err(NetworkError::InvalidConfig(
"a causal language-modelling loss needs at least two tokens".into(),
));
}
if vocab > i32::MAX as usize {
return Err(NetworkError::InvalidConfig(format!(
"the device loss supports vocabularies up to {}, got {vocab}",
i32::MAX
)));
}
let targets = causal_targets(batch, vocab)?;
let cache = forward_hidden(model, context, batch)?;
let gpu = Gpu { context };
let rows = cache.stack.rows;
let d_model = model.config.d_model;
let targets = gpu.upload_signed(&targets)?;
let mut loss = gpu.zeros(2)?;
let mut grad_final = gpu.zeros(rows * d_model)?;
let inverse_predicted = 1.0 / predicted as f32;
let z_loss = model.z_loss;
let logit_bytes = if gpu.context.mixed_precision { 2 } else { 4 };
let chunk = head_chunk_rows(rows, vocab, logit_bytes);
{
let head = head_of_mut(model)?;
let mut reduced = gpu
.context
.mixed_precision
.then(|| HeadBf16::new(&gpu, head, chunk))
.transpose()?;
let final_output = cache.final_output.wide();
let mut start = 0;
while start < rows {
let count = chunk.min(rows - start);
let input = final_output.slice(start * d_model..(start + count) * d_model);
let targets = targets.slice(start..start + count);
let mut out = grad_final.slice_mut(start * d_model..(start + count) * d_model);
if let Some(reduced) = reduced.as_mut() {
reduced.chunk(
&gpu,
head,
&input,
&targets,
&mut loss,
&mut out,
count,
inverse_predicted,
z_loss,
)?;
start += count;
continue;
}
let mut logits = gpu.linear(head, &input, count)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.cross_entropy)
.arg(&mut logits)
.arg(&mut loss)
.arg(&targets)
.arg(&(count as i32))
.arg(&(vocab as i32))
.arg(&inverse_predicted)
.arg(&z_loss)
.launch(row_per_block(count))
.map_err(cuda_err("cross-entropy kernel"))?;
}
gpu.linear_backward_input(head, &logits, &mut out, count, 0.0)?;
gpu.accumulate_weight_grad(head, &logits, &input, count)?;
start += count;
}
}
backward_from_final(model, context, &cache, &grad_final)?;
let loss = gpu.download(&loss)?;
let lm_loss = loss[0] / predicted as f32;
Ok((
lm_loss,
cache.stack.auxiliary_loss + loss[1] / predicted as f32,
))
}
struct HeadBf16 {
weight: CudaSlice<bf16>,
weight_grad: CudaSlice<bf16>,
input: CudaSlice<bf16>,
logits: CudaSlice<bf16>,
grad_input: CudaSlice<bf16>,
}
impl HeadBf16 {
fn new(gpu: &Gpu, head: &DeviceParam, chunk: usize) -> Result<Self, NetworkError> {
let (vocab, d_model) = (head.rows(), head.cols());
let mut weight = gpu.uninit_bf16(vocab * d_model)?;
gpu.cast_to_bf16(&mut weight, &head.value().slice(..), vocab * d_model)?;
Ok(Self {
weight,
weight_grad: gpu.uninit_bf16(if head.is_frozen() { 1 } else { vocab * d_model })?,
input: gpu.uninit_bf16(chunk * d_model)?,
logits: gpu.uninit_bf16(chunk * vocab)?,
grad_input: gpu.uninit_bf16(chunk * d_model)?,
})
}
#[allow(clippy::too_many_arguments)]
fn chunk(
&mut self,
gpu: &Gpu,
head: &mut DeviceParam,
input: &CudaView<'_, f32>,
targets: &CudaView<'_, i32>,
loss: &mut CudaSlice<f32>,
grad_input: &mut CudaViewMut<'_, f32>,
count: usize,
inverse_predicted: f32,
z_loss: f32,
) -> Result<(), NetworkError> {
let (vocab, d_model) = (head.rows(), head.cols());
gpu.cast_to_bf16(&mut self.input, input, count * d_model)?;
gemm_rhs_transposed(
gpu.context,
&self.input,
d_model,
&self.weight,
d_model,
&mut self.logits,
vocab,
count,
vocab,
d_model,
bf16::ONE,
bf16::ZERO,
)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.cross_entropy_bf16)
.arg(&mut self.logits)
.arg(loss)
.arg(targets)
.arg(&(count as i32))
.arg(&(vocab as i32))
.arg(&inverse_predicted)
.arg(&z_loss)
.launch(row_per_block(count))
.map_err(cuda_err("cross-entropy kernel"))?;
}
gemm_plain(
gpu.context,
&self.logits,
vocab,
&self.weight,
d_model,
&mut self.grad_input,
d_model,
count,
d_model,
vocab,
bf16::ONE,
bf16::ZERO,
)?;
gpu.cast_from_bf16(grad_input, &self.grad_input, count * d_model)?;
if head.is_frozen() {
return Ok(());
}
gemm_lhs_transposed(
gpu.context,
&self.logits,
vocab,
&self.input,
d_model,
&mut self.weight_grad,
d_model,
count,
vocab,
d_model,
bf16::NEG_ONE,
bf16::ZERO,
)?;
head.clear_grad()?;
gpu.accumulate_bf16(head.negated_grad_mut(), &self.weight_grad, vocab * d_model)
}
}
fn causal_targets(batch: &TokenBatch, vocab: usize) -> Result<Vec<i32>, NetworkError> {
let ids = batch.ids();
let mut targets = vec![-1i32; batch.rows()];
for row in 0..batch.rows() {
if !batch.predicts(row) {
continue;
}
let id = ids[row + 1];
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
targets[row] = id as i32;
}
Ok(targets)
}
fn head_chunk_rows(rows: usize, vocab: usize, logit_bytes: usize) -> usize {
#[cfg(test)]
{
let forced = CHUNK_ROWS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed);
if forced > 0 {
return forced.min(rows.max(1));
}
}
const BUDGET_BYTES: usize = 256 << 20;
(BUDGET_BYTES / (vocab * logit_bytes)).clamp(1, rows.max(1))
}
pub(crate) struct DeviceKv {
layers: Vec<CudaSlice<f32>>,
widths: Vec<usize>,
narrow: bool,
length: usize,
capacity: usize,
sequences: usize,
slots: CudaSlice<i32>,
padded: bool,
lead: Vec<usize>,
packs: PackMemo,
head: Option<Arc<Act>>,
int8: bool,
}
pub(crate) type PackMemo = std::collections::HashMap<(usize, usize, bool), Arc<Act>>;
impl DeviceKv {
pub(crate) fn new(
model: &TransformerLm,
context: &Arc<GpuContext>,
sequences: usize,
) -> Result<Self, NetworkError> {
let gpu = Gpu { context };
let narrow = context.mixed_precision
&& context.flash.is_some()
&& model.blocks.iter().all(|block| {
let attention = &block.attention;
crate::cuda_flash::eligible(true, attention.head_dim())
&& attention.num_heads() / attention.num_kv_heads() <= DECODE_GROUP
});
let widths: Vec<usize> = model
.blocks
.iter()
.map(|block| {
let attention = &block.attention;
let kv_width = attention.num_kv_heads() * attention.head_dim();
if narrow {
kv_width
} else {
attention.num_heads() * attention.head_dim() + 2 * kv_width
}
})
.collect();
Ok(Self {
layers: widths
.iter()
.map(|_| gpu.uninit(0))
.collect::<Result<_, _>>()?,
widths,
narrow,
length: 0,
capacity: 0,
sequences,
slots: gpu.upload_flags(&[1])?,
padded: false,
lead: vec![0; sequences],
packs: PackMemo::new(),
head: None,
int8: model.int8_decode,
})
}
pub(crate) fn len(&self) -> usize {
self.length
}
pub(crate) fn truncate(&mut self, positions: usize) {
self.length = self.length.min(positions);
}
fn reserve(&mut self, gpu: &Gpu, positions: usize, limit: usize) -> Result<(), NetworkError> {
if positions <= self.capacity {
return Ok(());
}
let capacity = positions.max(2 * self.capacity).min(limit);
let (kept, rows) = (self.length * self.sequences, capacity * self.sequences);
for (cache, width) in self.layers.iter_mut().zip(&self.widths) {
let mut grown = gpu.uninit(rows * width)?;
if kept > 0 {
gpu.context
.stream
.memcpy_dtod(
&cache.slice(..kept * width),
&mut grown.slice_mut(..kept * width),
)
.map_err(cuda_err("KV cache copy"))?;
}
*cache = grown;
}
let mut slots = gpu.upload_flags(&vec![1; rows])?;
if kept > 0 {
gpu.context
.stream
.memcpy_dtod(&self.slots.slice(..kept), &mut slots.slice_mut(..kept))
.map_err(cuda_err("KV cache copy"))?;
}
self.slots = slots;
self.capacity = capacity;
Ok(())
}
pub(crate) fn fork(&self) -> Result<Self, NetworkError> {
let copy = |slice: &CudaSlice<f32>| slice.try_clone().map_err(cuda_err("KV cache copy"));
Ok(Self {
layers: self.layers.iter().map(copy).collect::<Result<_, _>>()?,
widths: self.widths.clone(),
narrow: self.narrow,
length: self.length,
capacity: self.capacity,
sequences: self.sequences,
slots: self.slots.try_clone().map_err(cuda_err("KV cache copy"))?,
padded: self.padded,
lead: self.lead.clone(),
packs: self.packs.clone(),
head: self.head.clone(),
int8: self.int8,
})
}
}
pub(crate) fn decode(
model: &TransformerLm,
context: &Arc<GpuContext>,
ids: &[u32],
kv: &mut DeviceKv,
positions: usize,
valid: &[i32],
) -> Result<Matrix, NetworkError> {
let gpu = Gpu { context };
let rows = ids.len();
if valid.len() != rows {
return Err(NetworkError::InvalidConfig(format!(
"{rows} ids came with {} padding flags",
valid.len()
)));
}
let sequences = kv.sequences;
let d_model = model.config.d_model;
let vocab = model.embedding.vocab_size();
for &id in ids {
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
}
let embedding = model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
let device_ids = gpu.upload_indices(ids)?;
let mut hidden = gpu.uninit(rows * d_model)?;
gpu.gather(
&mut hidden.slice_mut(..),
&embedding.value().slice(..),
&device_ids,
rows,
d_model,
)?;
kv.reserve(&gpu, kv.length + rows / sequences, model.config.max_seq_len)?;
let first = kv.length * sequences;
context
.stream
.memcpy_htod(valid, &mut kv.slots.slice_mut(first..first + rows))
.map_err(cuda_alloc_err("host to device copy", size_of_val(valid)))?;
kv.padded |= valid.contains(&0);
let padding = kv.padded.then_some(&kv.slots);
if kv.length == 0 {
kv.lead = (0..sequences)
.map(|seq| {
(0..rows / sequences)
.take_while(|&p| valid[p * sequences + seq] == 0)
.count()
})
.collect();
}
let valid = gpu.upload_flags(valid)?;
let lend = |memo: &mut PackMemo, into_context: bool| -> Result<(), NetworkError> {
let mut slot = context
.packs
.lock()
.map_err(|_| NetworkError::Cuda("the pack memo lock was poisoned".into()))?;
if into_context {
*slot = Some(std::mem::take(memo));
} else {
*memo = slot.take().unwrap_or_default();
}
Ok(())
};
lend(&mut kv.packs, true)?;
context.int8_decode.store(kv.int8, Ordering::Relaxed);
let blocks = (|| {
for (block, cache) in model.blocks.iter().zip(&mut kv.layers) {
hidden = decode_block(
&gpu, block, hidden, cache, kv.narrow, kv.length, sequences, rows, &valid,
&kv.slots, padding, &kv.lead,
)?;
}
let final_weight = gpu.decode_norm(&model.final_norm.weight.value.data)?;
Ok::<_, NetworkError>((hidden, final_weight))
})();
context.int8_decode.store(false, Ordering::Relaxed);
lend(&mut kv.packs, false)?;
let (hidden, final_weight) = blocks?;
kv.length += rows / sequences;
let narrow = context.mixed_precision;
let (final_output, _, _) = gpu.rmsnorm(
&hidden,
&final_weight.wide(),
rows,
d_model,
model.final_norm.eps,
narrow,
false,
)?;
let kept = (positions * sequences).min(rows);
if !narrow && !kv.int8 {
let last = final_output.wide().slice((rows - kept) * d_model..);
let logits = gpu.linear(head_of(model)?, &last, kept)?;
return gpu.matrix(&logits, kept, vocab);
}
if kv.head.is_none() {
let head = head_of(model)?.value().slice(..);
kv.head = Some(Arc::new(if kv.int8 {
gpu.quantized(&head, vocab, d_model)?
} else {
gpu.narrowed(&head, vocab * d_model, true)?
}));
}
let head = kv.head.as_ref().expect("made above");
let last = final_output
.all()
.slice((rows - kept) * d_model * Act::element(narrow)..);
let mut logits = gpu.uninit(kept * vocab)?;
gpu.project(
head,
vocab,
d_model,
&last,
narrow,
&mut logits.slice_mut(..),
kept,
0.0,
)?;
gpu.matrix(&logits, kept, vocab)
}
const DECODE_CHUNK: usize = 64;
const DECODE_GROUP: usize = 8;
#[allow(clippy::too_many_arguments)]
fn decode_block(
gpu: &Gpu<'_>,
block: &TransformerBlock,
input: CudaSlice<f32>,
cache: &mut CudaSlice<f32>,
narrow_cache: bool,
offset: usize,
sequences: usize,
rows: usize,
valid: &CudaSlice<i32>,
slots: &CudaSlice<i32>,
padding: Option<&CudaSlice<i32>>,
lead: &[usize],
) -> Result<CudaSlice<f32>, NetworkError> {
let attention = &block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let kv_width = kv_heads * head_dim;
let qkv_width = query_width + 2 * kv_width;
let key_base = query_width;
let value_base = query_width + kv_width;
let queries = rows / sequences;
let history = offset + queries;
let row_stride = sequences * qkv_width;
let narrow = gpu.context.mixed_precision;
let mut scratch = if narrow_cache {
Some(gpu.uninit(rows * qkv_width)?)
} else {
None
};
let start = if narrow_cache {
0
} else {
offset * sequences * qkv_width
};
let attention_weight = gpu.decode_norm(&block.attention_norm.weight.value.data)?;
let (normed, _, residual) = gpu.rmsnorm(
&input,
&attention_weight.wide(),
rows,
d_model,
block.attention_norm.eps,
narrow,
true,
)?;
let mut residual = residual.expect("a norm asked for a duplicate returns one");
let qkv_weights = gpu.decode_pack(
&[&attention.query, &attention.key, &attention.value],
narrow,
)?;
gpu.project(
&qkv_weights,
qkv_width,
d_model,
&normed.all(),
narrow,
&mut scratch.as_mut().unwrap_or(cache).slice_mut(start..),
rows,
0.0,
)?;
let tables = gpu.rope_tables(&attention.rope)?;
let half = head_dim / 2;
for (count, base) in [(heads, 0), (kv_heads, key_base)] {
let elements = rows * count * half;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.rope)
.arg(&mut scratch.as_mut().unwrap_or(cache).slice_mut(start..))
.arg(&tables.cos.slice(offset * half..))
.arg(&tables.sin.slice(offset * half..))
.arg(&(rows as i32))
.arg(&(count as i32))
.arg(&(head_dim as i32))
.arg(&(queries as i32))
.arg(&1.0f32)
.arg(&(qkv_width as i32))
.arg(&(base as i32))
.arg(&0i32)
.arg(&(sequences as i32))
.launch(cfg(elements))
.map_err(cuda_err("RoPE kernel"))?;
}
}
if let Some(scratch) = &scratch {
let cols = 2 * kv_width;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.store_rows)
.arg(scratch)
.arg(&mut cache.slice_mut(offset * sequences * kv_width..))
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(qkv_width as i32))
.arg(&(key_base as i32))
.arg(&(cols as i32))
.arg(&1i32)
.launch(cfg(rows * cols))
.map_err(cuda_err("KV store kernel"))?;
}
}
const THREADS: usize = 256;
const FUSED_ROWS: usize = 16;
let group = heads / kv_heads;
let fused = queries <= FUSED_ROWS && head_dim <= THREADS && group <= DECODE_GROUP;
let flash = gpu.context.flash.as_ref().filter(|_| {
!fused
&& queries > 1
&& (narrow_cache || offset == 0 || queries >= 128)
&& crate::cuda_flash::eligible(narrow, head_dim)
});
let mut merged = gpu.act(rows * query_width, (fused || flash.is_some()) && narrow)?;
if let Some(flash) = flash {
let (qkv, first_query, key_width, key_base, value_base) = if let Some(scratch) = &scratch {
let qkv = gpu.narrowed(&scratch.slice(..), rows * qkv_width, true)?;
(qkv, 0, 2 * kv_width, 0, kv_width)
} else {
let cached = history * sequences * qkv_width;
let qkv = gpu.narrowed(&cache.slice(..cached), cached, true)?;
(qkv, offset * sequences, qkv_width, key_base, value_base)
};
let mut log_sum_exp = gpu.uninit(heads * queries)?;
let forward = &flash.width(head_dim).forward;
if lead.iter().any(|&pad| pad > offset) {
gpu.context
.stream
.memset_zeros(merged.destination())
.map_err(cuda_err("attention output clear"))?;
}
for (seq, &pad) in lead.iter().enumerate() {
let skipped = pad.saturating_sub(offset);
let length = queries - skipped;
let query_row = first_query + skipped * sequences + seq;
let key_row = pad * sequences + seq;
let keys = if narrow_cache {
cache.slice(key_row * kv_width..)
} else {
let bytes = qkv.bytes.slice(key_row * qkv_width * 2..);
unsafe { bytes.transmute(0) }.expect("an empty view fits")
};
let causal = offset + skipped - pad + 1;
unsafe {
gpu.context
.stream
.launch_builder(forward)
.arg(&qkv.bytes.slice(query_row * qkv_width * 2..))
.arg(&keys)
.arg(
&mut merged
.bytes
.slice_mut((skipped * sequences + seq) * query_width * 2..),
)
.arg(&mut log_sum_exp)
.arg(&(length as i32))
.arg(&((history - pad) as i32))
.arg(&(row_stride as i32))
.arg(&((sequences * key_width) as i32))
.arg(&((sequences * query_width) as i32))
.arg(&(key_base as i32))
.arg(&(value_base as i32))
.arg(&(group as i32))
.arg(&(queries as i32))
.arg(&scale)
.arg(&(causal as i32))
.launch(cudarc::driver::LaunchConfig {
grid_dim: (
length.div_ceil(crate::cuda_flash::TILE) as u32,
heads as u32,
1,
),
block_dim: (crate::cuda_flash::THREADS, 1, 1),
shared_mem_bytes: crate::cuda_flash::forward_shared_bytes(head_dim),
})
.map_err(cuda_err("prompt attention kernel"))?;
}
}
} else if fused {
let chunks = history.div_ceil(DECODE_CHUNK);
let mut partial = gpu.uninit(heads * rows * chunks * (head_dim + 2))?;
let (queries_in, kv_row, key_base, value_base) = match &scratch {
Some(scratch) => (scratch.slice(..), 2 * kv_width, 0, kv_width),
None => (cache.slice(start..), qkv_width, key_base, value_base),
};
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.decode_attention)
.arg(&queries_in)
.arg(&*cache)
.arg(&mut partial)
.arg(slots)
.arg(&(history as i32))
.arg(&(queries as i32))
.arg(&(sequences as i32))
.arg(&(qkv_width as i32))
.arg(&(kv_row as i32))
.arg(&(head_dim as i32))
.arg(&(group as i32))
.arg(&(key_base as i32))
.arg(&(value_base as i32))
.arg(&scale)
.arg(&i32::from(narrow_cache))
.launch(cudarc::driver::LaunchConfig {
grid_dim: (kv_heads as u32, rows as u32, chunks as u32),
block_dim: (THREADS as u32, 1, 1),
shared_mem_bytes: 0,
})
.map_err(cuda_err("decode attention kernel"))?;
gpu.context
.stream
.launch_builder(&gpu.context.model.decode_attention_merge)
.arg(&partial)
.arg(merged.destination())
.arg(&(chunks as i32))
.arg(&(head_dim as i32))
.arg(&(query_width as i32))
.arg(&i32::from(fused && narrow))
.launch(cudarc::driver::LaunchConfig {
grid_dim: (heads as u32, rows as u32, 1),
block_dim: (head_dim.next_multiple_of(32) as u32, 1, 1),
shared_mem_bytes: 0,
})
.map_err(cuda_err("decode attention merge kernel"))?;
}
} else {
if narrow_cache {
return Err(unsupported(
"a BF16 key-value cache without flash attention",
));
}
let block_scores = queries * history;
let mut scores = gpu.uninit(heads * sequences * block_scores)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
gemm_rhs_transposed_batched(
gpu.context,
&cache.slice(start + head * head_dim..),
row_stride,
&cache.slice(key_base + kv_base..),
row_stride,
&mut scores.slice_mut(head * sequences * block_scores..),
history,
queries,
history,
head_dim,
scale,
0.0,
sequences,
qkv_width,
qkv_width,
block_scores,
)?;
}
if let Some(slots) = padding {
let rows = heads * rows;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.mask_padding)
.arg(&mut scores)
.arg(slots)
.arg(&(rows as i32))
.arg(&(history as i32))
.arg(&(queries as i32))
.arg(&(sequences as i32))
.arg(&(offset as i32))
.launch(cfg(rows * history))
.map_err(cuda_err("padding mask kernel"))?;
}
}
let mut log_sum_exp = gpu.uninit(heads * rows)?;
gpu.attention_softmax(
&mut scores,
&mut log_sum_exp,
heads * rows,
history,
true,
queries,
offset,
)?;
{
let mut wide = merged.wide_mut();
for head in 0..heads {
let kv_base = (head / group) * head_dim;
gemm_plain_batched(
gpu.context,
&scores.slice(head * sequences * block_scores..),
history,
&cache.slice(value_base + kv_base..),
row_stride,
&mut wide.slice_mut(head * head_dim..),
sequences * query_width,
queries,
head_dim,
history,
1.0,
0.0,
sequences,
block_scores,
qkv_width,
query_width,
)?;
}
}
}
let output_weight = gpu.decode_pack(&[&attention.output], merged.is_narrow())?;
gpu.project(
&output_weight,
d_model,
query_width,
&merged.all(),
merged.is_narrow(),
&mut residual.slice_mut(..),
rows,
1.0,
)?;
let feed_forward_weight = gpu.decode_norm(&block.feed_forward_norm.weight.value.data)?;
let (normed, _, output) = gpu.rmsnorm(
&residual,
&feed_forward_weight.wide(),
rows,
d_model,
block.feed_forward_norm.eps,
narrow,
true,
)?;
let mut output = output.expect("a norm asked for a duplicate returns one");
match &block.feed_forward {
FeedForward::SwiGlu(ffn) => {
forward_swiglu(gpu, ffn, &normed, &mut output, rows)?;
}
FeedForward::Gelu(_) => return Err(unsupported("a GELU MLP")),
FeedForward::Moe(moe)
if rows <= ROUTED_GEMV_ROWS
&& (!gpu.context.int8_decode.load(Ordering::Relaxed)
|| (moe.d_model() % 16 == 0 && moe.config.d_ff % 16 == 0)) =>
{
decode_moe(gpu, moe, &normed, &mut output, valid, rows)?;
}
FeedForward::Moe(moe) => {
forward_moe(gpu, moe, &normed, &mut output, valid, rows, rows)?;
}
}
Ok(output)
}
#[cfg(test)]
pub(crate) static CHUNK_ROWS_OVERRIDE: std::sync::atomic::AtomicUsize =
std::sync::atomic::AtomicUsize::new(0);
fn row_per_block(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.max(1) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
const ROW_GRID_BLOCKS: usize = 2048;
fn warp_each(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.div_ceil(8).max(1) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
const Q8_GEMV_ROWS: usize = 16;
const ROUTED_GEMV_ROWS: usize = 16;
const Q8_MMA_ROWS: usize = 3;
fn warp_per_row_grid(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.div_ceil(8).clamp(1, ROW_GRID_BLOCKS) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn row_grid(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.clamp(1, ROW_GRID_BLOCKS) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn rmsnorm_smem(cols: usize) -> Option<u32> {
let bytes = cols.checked_mul(4)?;
(bytes <= 48 << 10).then_some(bytes as u32)
}
fn backward_from_final(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
cache: &GpuCache,
grad_final: &CudaSlice<f32>,
) -> Result<(), NetworkError> {
let gpu = Gpu { context };
let rows = cache.stack.rows;
let d_model = model.config.d_model;
let mut grad_norms = gpu.zeros(norm_slots(model.blocks.len()) * d_model)?;
let (grad_hidden, grad_hidden_act) = gpu.rmsnorm_backward(
&cache.final_input,
grad_final,
&cache.final_weight,
&cache.final_inverse_rms,
&mut grad_norms.slice_mut(0..d_model),
grad_final,
false,
stack_operand(&gpu, &model.blocks, model.blocks.len().checked_sub(1)),
rows,
d_model,
)?;
let grad_hidden = backward_stack(
&gpu,
&mut model.blocks,
&cache.stack,
grad_hidden,
grad_hidden_act,
&mut grad_norms,
)?;
{
let embedding = model.embedding.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
if !embedding.is_frozen() {
embedding.clear_grad()?;
gpu.scatter_negated(
embedding.negated_grad_mut(),
&grad_hidden,
&cache.ids,
rows,
d_model,
)?;
}
}
let values = gpu.download(&grad_norms)?;
Gpu::accumulate_host_grad(&mut model.final_norm.weight, &values[..d_model]);
accumulate_block_norms(&mut model.blocks, &values, d_model);
Ok(())
}
fn norm_slots(blocks: usize) -> usize {
1 + 2 * blocks
}
fn norm_slot(index: usize) -> (usize, usize) {
(1 + 2 * index, 2 + 2 * index)
}
#[allow(clippy::too_many_arguments)]
fn backward_block(
gpu: &Gpu<'_>,
block: &mut TransformerBlock,
cache: &BlockCache,
grad_output: &CudaSlice<f32>,
grad_output_act: Option<&Act>,
output_operand: Option<bool>,
batch: &GpuStack,
grad_norms: &mut CudaSlice<f32>,
index: usize,
) -> Result<(CudaSlice<f32>, Option<Act>), NetworkError> {
let rows = batch.rows;
let d_model = block.attention.d_model();
let (attention_slot, feed_forward_slot) = norm_slot(index);
let mut grad_normed = match block.feed_forward {
FeedForward::SwiGlu(_) => gpu.uninit(rows * d_model)?,
_ => gpu.zeros(rows * d_model)?,
};
match (&mut block.feed_forward, &cache.feed_forward) {
(FeedForward::SwiGlu(ffn), FfnCache::Dense(ffn_cache)) => backward_swiglu(
gpu,
ffn,
ffn_cache,
&cache.feed_forward_normed,
grad_output_act.expect("a dense feed forward was handed its operand"),
&mut grad_normed,
rows,
)?,
(FeedForward::Moe(moe), FfnCache::Moe(moe_cache)) => backward_moe(
gpu,
moe,
moe_cache,
&cache.feed_forward_normed,
grad_output,
&mut grad_normed,
batch.valid_tokens,
rows,
)?,
(FeedForward::Gelu(_), _) => return Err(unsupported("a GELU MLP")),
_ => {
return Err(NetworkError::Cuda(
"the device cache does not match the layer that produced it".into(),
));
}
}
let (grad_residual, grad_residual_act) = gpu.rmsnorm_backward(
&cache.residual,
&grad_normed,
&cache.feed_forward_weight,
&cache.feed_forward_inverse_rms,
&mut grad_norms.slice_mut(feed_forward_slot * d_model..(feed_forward_slot + 1) * d_model),
grad_output,
true,
Some(cache.merged.is_narrow()),
rows,
d_model,
)?;
let grad_residual_act = grad_residual_act.expect("a copy was asked for");
let grad_attention_normed = backward_attention(gpu, block, cache, &grad_residual_act, batch)?;
let (grad_input, grad_input_act) = gpu.rmsnorm_backward(
&cache.input,
&grad_attention_normed,
&cache.attention_weight,
&cache.attention_inverse_rms,
&mut grad_norms.slice_mut(attention_slot * d_model..(attention_slot + 1) * d_model),
&grad_residual,
true,
output_operand,
rows,
d_model,
)?;
Ok((grad_input, grad_input_act))
}
fn backward_attention(
gpu: &Gpu<'_>,
block: &mut TransformerBlock,
cache: &BlockCache,
grad_output: &Act,
batch: &GpuStack,
) -> Result<CudaSlice<f32>, NetworkError> {
let rows = batch.rows;
let seq_len = batch.seq_len;
let sequences = batch.sequences;
let attention = &mut block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let causal = attention.is_causal();
let group = heads / kv_heads;
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let kv_width = kv_heads * head_dim;
let qkv_width = query_width + 2 * kv_width;
let key_base = query_width;
let value_base = query_width + kv_width;
let block_size = seq_len * seq_len;
let merged_narrow = cache.merged.is_narrow();
debug_assert_eq!(grad_output.is_narrow(), merged_narrow);
let grad_output_act = grad_output;
let mut grad_merged = gpu.act(rows * query_width, merged_narrow)?;
gpu.linear_packed_backward_input(
&cache.output_weight.all(),
d_model,
query_width,
&grad_output_act.all(),
merged_narrow,
merged_narrow,
grad_merged.destination(),
rows,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut attention.output,
&grad_output_act.all(),
&cache.merged.all(),
merged_narrow,
rows,
)?;
let fused = gpu.context.flash.as_ref().filter(|_| cache.qkv.is_narrow());
let narrow_qkv = cache.qkv.is_narrow();
let mut grad_qkv = gpu.act(rows * qkv_width, narrow_qkv)?;
if let Some(flash) = fused {
let shape = FlashShape {
rows,
seq_len,
kv_len: seq_len,
sequences,
heads,
group,
qkv_width,
kv_width: qkv_width,
query_width,
key_base,
value_base,
causal,
};
let mut delta = gpu.uninit(heads * rows)?;
gpu.flash_attention_backward(
flash,
&cache.qkv,
&cache.qkv,
&cache.merged,
&grad_merged,
&cache.log_sum_exp,
&mut delta,
&grad_qkv,
&grad_qkv,
shape,
scale,
)?;
} else {
let qkv = cache.qkv.wide();
let wide_grad_merged = grad_merged.wide();
let mut probabilities = gpu.uninit(heads * sequences * block_size)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let query = qkv.slice(head * head_dim..);
let key = qkv.slice(key_base + kv_base..);
let mut scores = probabilities.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
qkv_width,
&key,
qkv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
scale,
0.0,
sequences,
seq_len * qkv_width,
seq_len * qkv_width,
block_size,
)?;
}
gpu.probs_from_lse(
&mut probabilities,
&cache.log_sum_exp,
heads * sequences * seq_len,
seq_len,
causal,
)?;
let mut grad_scores = gpu.uninit(heads * sequences * block_size)?;
let mut grad_qkv = grad_qkv.wide_mut();
gpu.context
.stream
.memset_zeros(&mut grad_qkv)
.map_err(cuda_err("gradient clear"))?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let upstream = wide_grad_merged.slice(head * head_dim..);
let value = qkv.slice(value_base + kv_base..);
let mut scores = grad_scores.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&upstream,
query_width,
&value,
qkv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
1.0,
0.0,
sequences,
seq_len * query_width,
seq_len * qkv_width,
block_size,
)?;
let head_probabilities = probabilities.slice(head * sequences * block_size..);
let mut grad_value = grad_qkv.slice_mut(value_base + kv_base..);
gemm_lhs_transposed_batched(
gpu.context,
&head_probabilities,
seq_len,
&upstream,
query_width,
&mut grad_value,
qkv_width,
seq_len,
seq_len,
head_dim,
1.0,
1.0,
sequences,
block_size,
seq_len * query_width,
seq_len * qkv_width,
)?;
}
gpu.softmax_backward(
&mut grad_scores,
&probabilities,
heads * sequences * seq_len,
seq_len,
causal,
)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let scores = grad_scores.slice(head * sequences * block_size..);
let key = qkv.slice(key_base + kv_base..);
let mut grad_query = grad_qkv.slice_mut(head * head_dim..);
gemm_plain_batched(
gpu.context,
&scores,
seq_len,
&key,
qkv_width,
&mut grad_query,
qkv_width,
seq_len,
head_dim,
seq_len,
scale,
0.0,
sequences,
block_size,
seq_len * qkv_width,
seq_len * qkv_width,
)?;
let query = qkv.slice(head * head_dim..);
let mut grad_key = grad_qkv.slice_mut(key_base + kv_base..);
gemm_lhs_transposed_batched(
gpu.context,
&scores,
seq_len,
&query,
qkv_width,
&mut grad_key,
qkv_width,
seq_len,
seq_len,
head_dim,
scale,
1.0,
sequences,
block_size,
seq_len * qkv_width,
seq_len * qkv_width,
)?;
}
}
let tables = gpu.rope_tables(&attention.rope)?;
gpu.rope(
&mut grad_qkv,
&tables,
rows,
heads,
head_dim,
seq_len,
-1.0,
qkv_width,
0,
)?;
gpu.rope(
&mut grad_qkv,
&tables,
rows,
kv_heads,
head_dim,
seq_len,
-1.0,
qkv_width,
key_base,
)?;
let narrow = cache.qkv_weights.is_narrow();
let grad_qkv = if narrow == narrow_qkv {
grad_qkv
} else {
gpu.narrowed(&grad_qkv.wide(), rows * qkv_width, narrow)?
};
let mut grad_input = gpu.uninit(rows * d_model)?;
gpu.linear_packed_backward_input(
&cache.qkv_weights.all(),
qkv_width,
d_model,
&grad_qkv.all(),
narrow,
false,
&mut grad_input,
rows,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [
&mut attention.query,
&mut attention.key,
&mut attention.value,
],
&grad_qkv.all(),
&cache.attention_normed.all(),
narrow,
rows,
)?;
Ok(grad_input)
}
pub(crate) fn backward_swiglu(
gpu: &Gpu<'_>,
ffn: &mut SwiGlu,
cache: &SwiGluCache,
input: &Act,
grad_output: &Act,
grad_input: &mut CudaSlice<f32>,
rows: usize,
) -> Result<(), NetworkError> {
let width = ffn.d_ff();
let inner = ffn.d_model();
let narrow = input.is_narrow();
debug_assert_eq!(grad_output.is_narrow(), narrow);
let grad_output_act = grad_output;
let down = cache
.down
.as_ref()
.ok_or_else(|| NetworkError::Cuda("the SwiGLU cache has no down projection".into()))?;
let mut grad_hidden = gpu.uninit(rows * width)?;
gpu.linear_packed_backward_input(
&down.all(),
inner,
width,
&grad_output_act.all(),
narrow,
false,
&mut grad_hidden,
rows,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut ffn.down,
&grad_output_act.all(),
&cache.hidden.all(),
narrow,
rows,
)?;
let grad_gate_up = gpu.swiglu_backward(&cache.gate_up, &grad_hidden, rows, width, narrow)?;
gpu.linear_packed_backward_input(
&cache.weights.all(),
2 * width,
inner,
&grad_gate_up.all(),
narrow,
false,
grad_input,
rows,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [&mut ffn.gate, &mut ffn.up],
&grad_gate_up.all(),
&input.all(),
narrow,
rows,
)?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn backward_moe(
gpu: &Gpu<'_>,
moe: &mut MoeLayer,
cache: &MoeGpuCache,
input: &Act,
grad_output: &CudaSlice<f32>,
grad_input: &mut CudaSlice<f32>,
valid_tokens: usize,
rows: usize,
) -> Result<(), NetworkError> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let routed = cache.routed;
if let (Some(shared), Some(shared_cache)) = (moe.shared.as_mut(), &cache.shared) {
let upstream = gpu.narrowed(&grad_output.slice(..), rows * d_model, input.is_narrow())?;
backward_swiglu(
gpu,
shared,
shared_cache,
input,
&upstream,
grad_input,
rows,
)?;
}
let mut grad_gates = gpu.zeros(rows * top_k)?;
if routed > 0 {
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.row_dot)
.arg(&mut grad_gates)
.arg(&cache.slot_of)
.arg(grad_output)
.arg(&cache.token_of)
.arg(&cache.output)
.arg(&(routed as i32))
.arg(&(d_model as i32))
.launch(warp_per_row_grid(routed))
.map_err(cuda_err("gate gradient kernel"))?;
}
let narrow = cache.input.is_narrow();
let mut grad_expert_output = gpu.act(routed * d_model, narrow)?;
gpu.gather_scaled(
&mut grad_expert_output,
&grad_output.slice(..),
false,
&cache.token_of,
&cache.gates,
true,
routed,
d_model,
)?;
let down = cache.expert.down.as_ref().ok_or_else(|| {
NetworkError::Cuda("the routed feed-forward cache has no down projection".into())
})?;
let down_stride = d_model * width;
let mut grad_hidden = gpu.uninit(routed * width)?;
for (expert, module) in moe.experts.iter_mut().enumerate() {
let count = cache.counts[expert];
if count == 0 {
continue;
}
let offset = cache.offsets[expert];
let upstream = grad_expert_output.at(offset * d_model);
let mut grad = grad_hidden.slice_mut(offset * width..);
gpu.linear_packed_backward_input(
&down.at(expert * down_stride),
d_model,
width,
&upstream,
narrow,
false,
&mut grad,
count,
0.0,
)?;
gpu.accumulate_projection_grad(
&mut module.down,
&upstream,
&cache.expert.hidden.at(offset * width),
narrow,
count,
)?;
}
let grad_gate_up =
gpu.swiglu_backward(&cache.expert.gate_up, &grad_hidden, routed, width, narrow)?;
let packed_stride = 2 * width * d_model;
let mut grad_expert_input = gpu.uninit(routed * d_model)?;
for (expert, module) in moe.experts.iter_mut().enumerate() {
let count = cache.counts[expert];
if count == 0 {
continue;
}
let offset = cache.offsets[expert];
let rows_in = cache.input.at(offset * d_model);
let upstream = grad_gate_up.at(offset * 2 * width);
let weights = cache.expert.weights.at(expert * packed_stride);
let mut grad = grad_expert_input.slice_mut(offset * d_model..);
gpu.linear_packed_backward_input(
&weights,
2 * width,
d_model,
&upstream,
narrow,
false,
&mut grad,
count,
0.0,
)?;
gpu.accumulate_packed_projection_grad(
&mut [&mut module.gate, &mut module.up],
&upstream,
&rows_in,
narrow,
count,
)?;
}
gpu.scatter(
grad_input,
&grad_expert_input,
&cache.token_of,
routed,
d_model,
)?;
}
let mut grad_probabilities = gpu.zeros(rows * experts)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.grad_probabilities)
.arg(&mut grad_probabilities)
.arg(&grad_gates)
.arg(&cache.probabilities)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.map_err(cuda_err("router gate gradient kernel"))?;
}
if moe.config.aux_loss_weight != 0.0 && valid_tokens > 0 {
let slots = (valid_tokens * top_k) as f32;
let loads: Vec<f32> = cache
.counts
.iter()
.map(|&count| count as f32 / slots)
.collect();
let loads = gpu.upload(&loads)?;
let scale = moe.config.aux_loss_weight * experts as f32 / valid_tokens as f32;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.aux_grad)
.arg(&mut grad_probabilities)
.arg(&loads)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.arg(&scale)
.launch(cfg(rows * experts))
.map_err(cuda_err("load-balancing gradient kernel"))?;
}
}
let z_factor = if moe.config.router_z_loss_weight != 0.0 {
2.0 * moe.config.router_z_loss_weight / valid_tokens.max(1) as f32
} else {
0.0
};
let mut grad_logits = gpu.zeros(rows * experts)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.grad_logits)
.arg(&mut grad_logits)
.arg(&grad_probabilities)
.arg(&cache.probabilities)
.arg(&cache.log_sum_exp)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.arg(&z_factor)
.launch(cfg(rows))
.map_err(cuda_err("router logit gradient kernel"))?;
}
gpu.linear_backward_input(
device_of(&moe.router.projection)?,
&grad_logits,
grad_input,
rows,
1.0,
)?;
let grad_logits = gpu.narrowed(&grad_logits.slice(..), rows * experts, input.is_narrow())?;
gpu.accumulate_projection_grad(
&mut moe.router.projection,
&grad_logits.all(),
&input.all(),
input.is_narrow(),
rows,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cuda_training::cuda_doctor;
use crate::norm::RmsNorm;
fn cuda_or_skip() -> Option<Arc<GpuContext>> {
match cuda_doctor(0, 8192) {
Ok(_) => Some(GpuContext::new(0).expect("the CUDA doctor passed")),
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
crate::cuda_training::skip_without_cuda();
None
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn ramp(len: usize) -> Vec<f32> {
ramp_from(len, 0)
}
fn ramp_from(len: usize, offset: usize) -> Vec<f32> {
(0..len)
.map(|i| (((i + offset) * 37) % 23) as f32 / 11.0 - 1.0)
.collect()
}
fn assert_close(label: &str, device: &[f32], host: &[f32], tolerance: f32) {
assert_eq!(device.len(), host.len(), "{label}: length");
for (index, (a, b)) in device.iter().zip(host).enumerate() {
assert!(
(a - b).abs() <= tolerance,
"{label}[{index}]: {a} on the device vs {b} on the host"
);
}
}
#[test]
fn an_allocation_that_cannot_fit_says_out_of_memory_rather_than_cuda() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
match gpu.zeros(1 << 40) {
Err(NetworkError::CudaOutOfMemory {
requested_mib,
free_mib,
}) => {
assert_eq!(requested_mib, 4 * 1024 * 1024);
assert!(free_mib > 0, "a working device reports some free memory");
}
other => panic!("expected an out-of-memory error, got {other:?}"),
}
}
#[test]
fn the_rmsnorm_kernels_match_the_host_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, cols) = (5, 8);
let mut norm = RmsNorm::new(cols, 1e-6);
norm.weight.value.data = ramp(cols).iter().map(|v| 1.0 + v * 0.25).collect();
let input = Matrix::from_vec(rows, cols, ramp(rows * cols));
let grad_output = Matrix::from_vec(rows, cols, ramp_from(rows * cols, 3));
let weight = gpu.upload(&norm.weight.value.data).unwrap();
let device_input = gpu.upload(&input.data).unwrap();
let (output, inverse, _) = gpu
.rmsnorm(
&device_input,
&weight.as_view(),
rows,
cols,
norm.eps,
false,
false,
)
.unwrap();
assert_close(
"rmsnorm forward",
&gpu.download_act(&output).unwrap(),
&norm.forward(&input).data,
1e-5,
);
let device_grad = gpu.upload(&grad_output.data).unwrap();
let mut grad_weight = gpu.zeros(cols).unwrap();
let grad_input = gpu
.rmsnorm_backward(
&device_input,
&device_grad,
&weight,
&inverse,
&mut grad_weight.slice_mut(..),
&device_grad,
false,
None,
rows,
cols,
)
.unwrap()
.0;
let host_grad_input = norm.backward(&input, &grad_output);
assert_close(
"rmsnorm grad_input",
&gpu.download(&grad_input).unwrap(),
&host_grad_input.data,
1e-5,
);
assert_close(
"rmsnorm grad_weight",
&gpu.download(&grad_weight).unwrap(),
&norm.weight.grad.data,
1e-5,
);
}
#[test]
fn the_rope_kernel_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (sequences, seq_len, heads, head_dim) = (2, 3, 2, 4);
let rows = sequences * seq_len;
let width = heads * head_dim;
let rope = Rope::new(head_dim, 16, 10_000.0).unwrap();
let tables = gpu.rope_tables(&rope).unwrap();
for (narrow, tolerance) in [(false, 1e-5), (true, 1e-2)] {
let mut host = Matrix::from_vec(rows, width, ramp(rows * width));
let wide = gpu.upload(&host.data).unwrap();
let mut device = gpu.narrowed(&wide.slice(..), rows * width, narrow).unwrap();
gpu.rope(
&mut device,
&tables,
rows,
heads,
head_dim,
seq_len,
1.0,
width,
0,
)
.unwrap();
rope.apply_batched(&mut host, heads, seq_len).unwrap();
assert_close(
"rope",
&gpu.download_act(&device).unwrap(),
&host.data,
tolerance,
);
gpu.rope(
&mut device,
&tables,
rows,
heads,
head_dim,
seq_len,
-1.0,
width,
0,
)
.unwrap();
rope.apply_inverse_batched(&mut host, heads, seq_len)
.unwrap();
assert_close(
"rope inverse",
&gpu.download_act(&device).unwrap(),
&host.data,
tolerance,
);
}
}
#[test]
fn the_causal_probability_kernels_mask_and_normalize_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (sequences, seq_len) = (2, 4);
let rows = sequences * seq_len;
let scores = ramp(rows * seq_len);
let log_sum_exp: Vec<f32> = (0..rows)
.map(|row| {
let source = &scores[row * seq_len..row * seq_len + row % seq_len + 1];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
max + source.iter().map(|v| (v - max).exp()).sum::<f32>().ln()
})
.collect();
let mut device = gpu.upload(&scores).unwrap();
let device_lse = gpu.upload(&log_sum_exp).unwrap();
gpu.probs_from_lse(&mut device, &device_lse, rows, seq_len, true)
.unwrap();
let probabilities = gpu.download(&device).unwrap();
for row in 0..rows {
let visible = row % seq_len + 1;
let slice = &probabilities[row * seq_len..(row + 1) * seq_len];
let mut expected = vec![0.0f32; seq_len];
let source = &scores[row * seq_len..row * seq_len + visible];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let total: f32 = source.iter().map(|v| (v - max).exp()).sum();
for (slot, &value) in expected.iter_mut().zip(source) {
*slot = (value - max).exp() / total;
}
assert_close(
&format!("causal probabilities row {row}"),
slice,
&expected,
1e-6,
);
assert!(slice[visible..].iter().all(|&p| p == 0.0));
}
let mut grad = gpu.upload(&vec![1.0; rows * seq_len]).unwrap();
gpu.softmax_backward(&mut grad, &device, rows, seq_len, true)
.unwrap();
assert_close(
"causal softmax backward",
&gpu.download(&grad).unwrap(),
&vec![0.0; rows * seq_len],
1e-6,
);
}
#[test]
fn the_swiglu_kernel_matches_the_host_formula_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, width) = (4, 4);
let len = rows * width;
let gate = ramp(len);
let up = ramp_from(len, 5);
let mut fused = Vec::with_capacity(2 * len);
for row in 0..rows {
fused.extend_from_slice(&gate[row * width..(row + 1) * width]);
fused.extend_from_slice(&up[row * width..(row + 1) * width]);
}
let gate_up = gpu.upload(&fused).unwrap();
let hidden = gpu.swiglu(&gate_up, rows, width, false).unwrap();
let silu = |x: f32| x / (1.0 + (-x).exp());
let expected: Vec<f32> = gate.iter().zip(&up).map(|(&g, &u)| silu(g) * u).collect();
assert_close(
"swiglu",
&gpu.download_act(&hidden).unwrap(),
&expected,
1e-6,
);
let grad_hidden = gpu.upload(&vec![1.0; len]).unwrap();
let grad_fused = gpu
.swiglu_backward(&gate_up, &grad_hidden, rows, width, false)
.unwrap();
let grad_fused = gpu.download_act(&grad_fused).unwrap();
let mut grad_gate = Vec::with_capacity(len);
let mut grad_up = Vec::with_capacity(len);
for row in 0..rows {
let base = row * 2 * width;
grad_gate.extend_from_slice(&grad_fused[base..base + width]);
grad_up.extend_from_slice(&grad_fused[base + width..base + 2 * width]);
}
let epsilon = 1e-3;
let numeric: Vec<f32> = gate
.iter()
.zip(&up)
.map(|(&g, &u)| (silu(g + epsilon) * u - silu(g - epsilon) * u) / (2.0 * epsilon))
.collect();
assert_close("swiglu grad_gate", &grad_gate, &numeric, 2e-3);
assert_close(
"swiglu grad_up",
&grad_up,
&gate.iter().map(|&g| silu(g)).collect::<Vec<_>>(),
1e-6,
);
}
#[test]
fn the_router_kernels_select_and_renormalize_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, experts, top_k) = (4, 5, 2);
let logits = ramp(rows * experts);
let device_logits = gpu.upload(&logits).unwrap();
let mut probabilities = gpu.zeros(rows * experts).unwrap();
let mut log_sum_exp = gpu.zeros(rows).unwrap();
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.softmax_lse)
.arg(&mut probabilities)
.arg(&mut log_sum_exp)
.arg(&device_logits)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.unwrap();
}
let flags: Vec<i32> = (0..rows).map(|row| i32::from(row + 1 < rows)).collect();
let valid = gpu.upload_flags(&flags).unwrap();
let mut expert_of = gpu.context.stream.alloc_zeros::<i32>(rows * top_k).unwrap();
let mut gate_of = gpu.zeros(rows * top_k).unwrap();
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.topk_gate)
.arg(&mut expert_of)
.arg(&mut gate_of)
.arg(&probabilities)
.arg(&valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.unwrap();
}
let probabilities = gpu.download(&probabilities).unwrap();
let log_sum_exp = gpu.download(&log_sum_exp).unwrap();
let assigned = gpu.download_signed(&expert_of).unwrap();
let gates = gpu.download(&gate_of).unwrap();
for row in 0..rows {
let slice = &probabilities[row * experts..(row + 1) * experts];
let sum: f32 = slice.iter().sum();
assert!((sum - 1.0).abs() < 1e-5, "row {row} softmax sums to {sum}");
let source = &logits[row * experts..(row + 1) * experts];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let expected = max + source.iter().map(|v| (v - max).exp()).sum::<f32>().ln();
assert!((log_sum_exp[row] - expected).abs() < 1e-5);
let selected = &assigned[row * top_k..(row + 1) * top_k];
if row + 1 == rows {
assert!(selected.iter().all(|&expert| expert < 0), "padding routed");
assert!(
gates[row * top_k..(row + 1) * top_k]
.iter()
.all(|&g| g == 0.0)
);
continue;
}
let mut order: Vec<usize> = (0..experts).collect();
order.sort_by(|&a, &b| slice[b].total_cmp(&slice[a]));
assert_eq!(
selected.iter().map(|&e| e as usize).collect::<Vec<_>>(),
order[..top_k],
"row {row} top-{top_k}"
);
let total: f32 = selected.iter().map(|&e| slice[e as usize]).sum();
for (rank, &expert) in selected.iter().enumerate() {
let expected = slice[expert as usize] / total;
assert!((gates[row * top_k + rank] - expected).abs() < 1e-6);
}
}
}
#[derive(Clone, Copy)]
struct AttentionShape {
sequences: usize,
seq_len: usize,
heads: usize,
kv_heads: usize,
head_dim: usize,
scale: f32,
}
fn host_attention(
queries: &[f32],
keys: &[f32],
values: &[f32],
shape: AttentionShape,
) -> (Vec<f32>, Vec<f32>) {
let AttentionShape {
sequences,
seq_len,
heads,
kv_heads,
head_dim,
scale,
} = shape;
let (group, query_width, kv_width) =
(heads / kv_heads, heads * head_dim, kv_heads * head_dim);
let mut out = vec![0.0; queries.len()];
let mut probabilities = vec![0.0; heads * sequences * seq_len * seq_len];
for s in 0..sequences {
for h in 0..heads {
let kv = h / group;
for i in 0..seq_len {
let q = ((s * seq_len + i) * query_width) + h * head_dim;
let mut row = vec![0.0f32; i + 1];
for (j, p) in row.iter_mut().enumerate() {
let k = ((s * seq_len + j) * kv_width) + kv * head_dim;
*p = (0..head_dim)
.map(|d| queries[q + d] * keys[k + d])
.sum::<f32>()
* scale;
}
let top = row.iter().cloned().fold(f32::MIN, f32::max);
let total: f32 = row.iter().map(|p| (p - top).exp()).sum();
let base = ((h * sequences + s) * seq_len + i) * seq_len;
for (j, p) in row.iter().enumerate() {
let weight = (p - top).exp() / total;
probabilities[base + j] = weight;
let v = ((s * seq_len + j) * kv_width) + kv * head_dim;
for d in 0..head_dim {
out[q + d] += weight * values[v + d];
}
}
}
}
}
(out, probabilities)
}
#[test]
fn the_attention_forward_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let shape = AttentionShape {
sequences: 2,
seq_len: 11,
heads: 4,
kv_heads: 2,
head_dim: 8,
scale: 1.0 / 8.0f32.sqrt(),
};
let rows = shape.sequences * shape.seq_len;
let wiggle = |len: usize, offset: usize| -> Vec<f32> {
(0..len)
.map(|i| ((i + offset) as f32 * 0.37).sin() * 0.8)
.collect()
};
let queries = wiggle(rows * shape.heads * shape.head_dim, 0);
let keys = wiggle(rows * shape.kv_heads * shape.head_dim, 5);
let values = wiggle(rows * shape.kv_heads * shape.head_dim, 11);
let (host_out, host_probabilities) = host_attention(&queries, &keys, &values, shape);
let device_queries = gpu.upload(&queries).unwrap();
let device_keys = gpu.upload(&keys).unwrap();
let device_values = gpu.upload(&values).unwrap();
let (group, block_size) = (shape.heads / shape.kv_heads, shape.seq_len * shape.seq_len);
let (query_width, kv_width) = (
shape.heads * shape.head_dim,
shape.kv_heads * shape.head_dim,
);
let scores_of = |probabilities: &mut CudaSlice<f32>| {
for head in 0..shape.heads {
let query = device_queries.slice(head * shape.head_dim..);
let key = device_keys.slice((head / group) * shape.head_dim..);
let mut scores = probabilities.slice_mut(head * shape.sequences * block_size..);
gemm_rhs_transposed_batched(
gpu.context,
&query,
query_width,
&key,
kv_width,
&mut scores,
shape.seq_len,
shape.seq_len,
shape.seq_len,
shape.head_dim,
shape.scale,
0.0,
shape.sequences,
shape.seq_len * query_width,
shape.seq_len * kv_width,
block_size,
)
.unwrap();
}
};
let mut probabilities = gpu
.zeros(shape.heads * shape.sequences * block_size)
.unwrap();
scores_of(&mut probabilities);
let mut lse = gpu.zeros(shape.heads * rows).unwrap();
gpu.softmax_lse(
&mut probabilities,
&mut lse,
shape.heads * shape.sequences * shape.seq_len,
shape.seq_len,
true,
)
.unwrap();
assert_close(
"causal softmax probabilities",
&gpu.download(&probabilities).unwrap(),
&host_probabilities,
2e-6,
);
let mut merged = gpu.zeros(queries.len()).unwrap();
for head in 0..shape.heads {
let head_probabilities = probabilities.slice(head * shape.sequences * block_size..);
let value = device_values.slice((head / group) * shape.head_dim..);
let mut head_merged = merged.slice_mut(head * shape.head_dim..);
gemm_plain_batched(
gpu.context,
&head_probabilities,
shape.seq_len,
&value,
kv_width,
&mut head_merged,
query_width,
shape.seq_len,
shape.head_dim,
shape.seq_len,
1.0,
0.0,
shape.sequences,
block_size,
shape.seq_len * kv_width,
shape.seq_len * query_width,
)
.unwrap();
}
assert_close(
"attention forward",
&gpu.download(&merged).unwrap(),
&host_out,
2e-5,
);
scores_of(&mut probabilities);
gpu.probs_from_lse(
&mut probabilities,
&lse,
shape.heads * shape.sequences * shape.seq_len,
shape.seq_len,
true,
)
.unwrap();
assert_close(
"probabilities rebuilt from the log-sum-exp",
&gpu.download(&probabilities).unwrap(),
&host_probabilities,
2e-6,
);
}
#[test]
fn an_fp8_weight_widens_to_the_host_bits_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (units, inner) = (41, 300);
let mut state = 0x9e37_79b9_7f4a_7c15u64;
let mut values: Vec<f32> = (0..units * inner)
.map(|i| {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1442695040888963407);
let u = (state >> 40) as f32 / (1u64 << 24) as f32 - 0.5;
u * u * u * 8.0 * 2.0f32.powi((i / inner) as i32 - 20)
})
.collect();
values[7 * inner..8 * inner].fill(0.0);
let host =
crate::quantized::Quantized::from_matrix_fp8(&Matrix::from_vec(units, inner, values));
let (bytes, scales) = host.parts_fp8().unwrap();
assert!(
bytes.iter().any(|b| b & 0x78 == 0 && b & 7 != 0),
"no subnormal codes"
);
let weights = gpu.fp8(bytes, scales, units, inner).unwrap();
let want = host.dequantize();
for first in [0, 5] {
let wide = gpu
.widened_fp8(&weights, first, units - first, inner, false)
.unwrap();
let got = gpu.download_act(&wide).unwrap();
let want = &want.data[first * inner..];
assert_eq!(got.len(), want.len());
for (i, (g, w)) in got.iter().zip(want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"weight {i} from row {first}: {g:e} vs {w:e}"
);
}
}
let rows = 6;
let input = ramp_from(rows * inner, 3);
let mut x = gpu.act(rows * inner, true).unwrap();
gpu.cast_into(
&mut x.bytes.slice_mut(..),
&gpu.upload(&input).unwrap().slice(..),
rows * inner,
true,
)
.unwrap();
let x_host = Matrix::from_vec(rows, inner, gpu.download_act(&x).unwrap());
let mut out = gpu.upload(&vec![0.0; rows * units]).unwrap();
gpu.project(
&weights,
units,
inner,
&x.all(),
true,
&mut out.slice_mut(..),
rows,
0.0,
)
.unwrap();
let got = gpu.download(&out).unwrap();
let expected = host.matmul_rhs_transposed(&x_host);
for (unit, scale) in scales.iter().enumerate() {
for row in 0..rows {
let (g, w) = (got[row * units + unit], expected.data[row * units + unit]);
let bound =
scale * 448.0 * x_host.row(row).iter().map(|v| v.abs()).sum::<f32>() / 256.0;
assert!(
(g - w).abs() <= bound + 1e-30,
"row {row} unit {unit}: {g:e} vs {w:e}"
);
}
}
}
fn fp8_values(rows: usize, inner: usize, seed: u64) -> Vec<f32> {
let mut state = seed;
let mut values: Vec<f32> = (0..rows * inner)
.map(|i| {
state = state
.wrapping_mul(6_364_136_223_846_793_005)
.wrapping_add(1442695040888963407);
let u = (state >> 40) as f32 / (1u64 << 24) as f32 - 0.5;
u * u * u * 8.0 * 2.0f32.powi(((i / inner) % 41) as i32 - 20)
})
.collect();
if rows > 2 {
values[2 * inner..3 * inner].fill(0.0);
}
values
}
fn fp8_operand(gpu: &Gpu<'_>, values: &[f32], rows: usize, narrow: bool) -> (Act, Matrix) {
let mut x = gpu.act(values.len(), narrow).unwrap();
gpu.cast_into(
&mut x.bytes.slice_mut(..),
&gpu.upload(values).unwrap().slice(..),
values.len(),
narrow,
)
.unwrap();
let host = Matrix::from_vec(rows, values.len() / rows, gpu.download_act(&x).unwrap());
(x, host)
}
#[test]
fn e4m3_row_quantization_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
for (rows, inner) in [(43, 300), (1, 1), (5, 4097)] {
for narrow in [false, true] {
let values = fp8_values(rows, inner, 0x9e37_79b9_7f4a_7c15);
let (x, host) = fp8_operand(&gpu, &values, rows, narrow);
let (q, scales) = gpu
.quantized_e4m3_rows(&x.all(), narrow, rows, inner)
.unwrap();
let want = crate::quantized::Quantized::from_matrix_fp8(&host);
let (want_q, want_scales) = want.parts_fp8().unwrap();
let got_q = context.stream.clone_dtoh(&q).unwrap();
let got_scales = gpu.download(&scales).unwrap();
assert_eq!(
&got_q[..rows * inner],
want_q,
"{rows}x{inner} narrow {narrow}"
);
let bits = |v: &[f32]| v.iter().map(|s| s.to_bits()).collect::<Vec<_>>();
assert_eq!(bits(&got_scales), bits(want_scales));
}
}
}
#[test]
#[ignore = "needs sm_89+"]
fn the_fp8_gemm_matches_the_host_reference_on_sm_89_or_skips() {
let Some(context) = cuda_or_skip() else {
return;
};
let device = context.stream.context();
let (major, minor) = device.compute_capability().unwrap();
let Some(gemm) = crate::cuda_fp8::load(device) else {
eprintln!("skipped: needs sm_89+, device 0 is {major}.{minor}");
return;
};
let gpu = Gpu { context: &context };
let rel_l2 = |got: &[f32], want: &[f32]| {
let (mut diff, mut norm) = (0.0f64, 0.0f64);
for (g, w) in got.iter().zip(want) {
diff += f64::from(g - w).powi(2);
norm += f64::from(*w).powi(2);
}
(diff / norm.max(1e-300)).sqrt()
};
for (rows, units, inner, first) in [
(64, 64, 64, 0),
(130, 77, 300, 0),
(1, 200, 3072, 0),
(257, 129, 33, 3),
(7, 3072, 96, 0),
(100, 70, 4097, 0),
] {
let w = fp8_values(first + units, inner, 0x2545_f491_4f6c_dd1d);
let w = Matrix::from_vec(first + units, inner, w);
let host_w = crate::quantized::Quantized::from_matrix_fp8(&w);
let (bytes, scales) = host_w.parts_fp8().unwrap();
let weights = gpu.fp8(bytes, scales, first + units, inner).unwrap();
let unit_rows = Matrix::from_vec(units, inner, w.data[first * inner..].to_vec());
let host_w = crate::quantized::Quantized::from_matrix_fp8(&unit_rows);
for (narrow, beta) in [(true, 0.0), (false, 1.0)] {
let values: Vec<f32> = ramp_from(rows * inner, 7)
.iter()
.enumerate()
.map(|(i, v)| v * (1.0 + (i / inner) as f32))
.collect();
let (x, x_host) = fp8_operand(&gpu, &values, rows, narrow);
let start = ramp_from(rows * units, 2);
let mut out = gpu.upload(&start).unwrap();
gpu.project_fp8_mma(
&gemm,
&weights,
first,
units,
inner,
&x.all(),
narrow,
&mut out.slice_mut(..),
rows,
beta,
)
.unwrap();
let got = gpu.download(&out).unwrap();
let mut reference = host_w.matmul_fp8_activations(&x_host);
let mut exact = Matrix::new(rows, units);
x_host.dot_rhs_transposed(&unit_rows, &mut exact);
for (i, s) in start.iter().enumerate() {
reference.data[i] += beta * s;
exact.data[i] += beta * s;
}
let (to_host, to_f32) = (rel_l2(&got, &reference.data), rel_l2(&got, &exact.data));
eprintln!(
"{rows}x{units}x{inner} first {first} narrow {narrow} beta {beta}: \
rel_l2 {to_host:.3e} to the host FP8 math, {to_f32:.3e} to FP32"
);
assert!(to_host < 1e-3, "against the host FP8 math: {to_host:e}");
assert!(to_f32 < 0.1, "against FP32: {to_f32:e}");
}
}
}
#[test]
fn an_int8_projection_matches_the_host_for_every_batch_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
for (units, inner) in [(37, 288), (37, 48)] {
let weights = gpu.upload(&ramp(units * inner)).unwrap();
let packed = gpu.quantized(&weights.slice(..), units, inner).unwrap();
let bytes = context.stream.clone_dtoh(&packed.all()).unwrap();
let scales = gpu.download(packed.scales.as_ref().unwrap()).unwrap();
for rows in [1, 4, 6, 8, 9, 16] {
for beta in [0.0, 0.5] {
let input = gpu.upload(&ramp_from(rows * inner, 5)).unwrap();
let mut x = gpu.act(rows * inner, true).unwrap();
gpu.cast_into(
&mut x.bytes.slice_mut(..),
&input.slice(..),
rows * inner,
true,
)
.unwrap();
let x_host = gpu.download_act(&x).unwrap();
let start = ramp_from(rows * units, 9);
let mut out = gpu.upload(&start).unwrap();
gpu.project(
&packed,
units,
inner,
&x.all(),
true,
&mut out.slice_mut(..),
rows,
beta,
)
.unwrap();
let expected: Vec<f32> = (0..rows * units)
.map(|i| {
let (row, unit) = (i / units, i % units);
let dot: f32 = (0..inner)
.map(|k| {
x_host[row * inner + k]
* f32::from(bytes[unit * inner + k] as i8)
})
.sum();
dot * scales[unit] + beta * start[i]
})
.collect();
assert_close(
&format!("{inner} wide, {rows} rows, beta {beta}"),
&gpu.download(&out).unwrap(),
&expected,
1e-4,
);
}
}
}
}
}