use crate::batch::TokenBatch;
use crate::cuda_training::{cfg, cuda_err};
use crate::ffn::SwiGlu;
use crate::gpu_transformer::{
DeviceParam, GpuContext, gemm_lhs_transposed, gemm_lhs_transposed_batched, gemm_plain,
gemm_plain_batched, gemm_rhs_transposed, gemm_rhs_transposed_batched,
};
use crate::matrix::Matrix;
use crate::moe::MoeLayer;
use crate::network::NetworkError;
use crate::param::{Linear, Param};
use crate::rope::Rope;
use crate::transformer::TransformerLm;
use crate::transformer_block::{FeedForward, TransformerBlock};
use cudarc::driver::{
CudaFunction, CudaModule, CudaSlice, CudaView, CudaViewMut, DevicePtr, DevicePtrMut,
PushKernelArg,
};
use half::bf16;
use std::sync::Arc;
pub(crate) struct ModelKernels {
rmsnorm_forward: CudaFunction,
rmsnorm_backward: CudaFunction,
rope: CudaFunction,
causal_softmax: CudaFunction,
causal_softmax_backward: CudaFunction,
swiglu_forward: CudaFunction,
swiglu_backward: CudaFunction,
softmax_lse: CudaFunction,
topk_gate: CudaFunction,
gather: CudaFunction,
gather_scaled: CudaFunction,
scatter_scaled: CudaFunction,
scatter: CudaFunction,
scatter_negated: CudaFunction,
row_dot: CudaFunction,
stats: CudaFunction,
grad_probabilities: CudaFunction,
aux_grad: CudaFunction,
grad_logits: CudaFunction,
add: CudaFunction,
cross_entropy: CudaFunction,
cross_entropy_bf16: CudaFunction,
to_bf16: CudaFunction,
from_bf16: CudaFunction,
accumulate_bf16: CudaFunction,
}
impl ModelKernels {
pub(crate) fn load(module: &Arc<CudaModule>) -> Result<Self, NetworkError> {
let get = |name: &str| {
module
.load_function(name)
.map_err(cuda_err("CUDA kernel lookup"))
};
Ok(Self {
rmsnorm_forward: get("rmsnorm_fwd")?,
rmsnorm_backward: get("rmsnorm_bwd")?,
rope: get("rope_rotate")?,
causal_softmax: get("causal_softmax")?,
causal_softmax_backward: get("causal_softmax_bwd")?,
swiglu_forward: get("swiglu_fwd")?,
swiglu_backward: get("swiglu_bwd")?,
softmax_lse: get("softmax_lse")?,
topk_gate: get("topk_gate")?,
gather: get("gather_rows")?,
gather_scaled: get("gather_scale_rows")?,
scatter_scaled: get("scatter_add_scaled")?,
scatter: get("scatter_add_rows")?,
scatter_negated: get("scatter_rows_neg")?,
row_dot: get("row_dot_scatter")?,
stats: get("moe_stats")?,
grad_probabilities: get("moe_grad_probs")?,
aux_grad: get("moe_aux_grad")?,
grad_logits: get("router_grad_logits")?,
add: get("add_inplace")?,
cross_entropy: get("ce_loss_grad")?,
cross_entropy_bf16: get("ce_loss_grad_bf16")?,
to_bf16: get("to_bf16")?,
from_bf16: get("from_bf16")?,
accumulate_bf16: get("accumulate_bf16")?,
})
}
}
pub(crate) struct DeviceRope {
cos: CudaSlice<f32>,
sin: CudaSlice<f32>,
head_dim: usize,
max_seq_len: usize,
}
pub(crate) struct GpuCache {
rows: usize,
seq_len: usize,
sequences: usize,
ids: CudaSlice<u32>,
valid_tokens: usize,
blocks: Vec<BlockCache>,
final_input: CudaSlice<f32>,
final_inverse_rms: CudaSlice<f32>,
final_weight: CudaSlice<f32>,
final_output: CudaSlice<f32>,
auxiliary_loss: f32,
}
impl std::fmt::Debug for GpuCache {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GpuCache")
.field("rows", &self.rows)
.field("seq_len", &self.seq_len)
.finish()
}
}
impl GpuCache {
pub(crate) fn auxiliary_loss(&self) -> f32 {
self.auxiliary_loss
}
}
struct BlockCache {
input: CudaSlice<f32>,
attention_weight: CudaSlice<f32>,
attention_inverse_rms: CudaSlice<f32>,
attention_normed: CudaSlice<f32>,
queries: CudaSlice<f32>,
keys: CudaSlice<f32>,
values: CudaSlice<f32>,
probabilities: CudaSlice<f32>,
merged: CudaSlice<f32>,
residual: CudaSlice<f32>,
feed_forward_weight: CudaSlice<f32>,
feed_forward_inverse_rms: CudaSlice<f32>,
feed_forward_normed: CudaSlice<f32>,
feed_forward: FfnCache,
}
struct SwiGluCache {
gate: CudaSlice<f32>,
up: CudaSlice<f32>,
hidden: CudaSlice<f32>,
}
enum FfnCache {
Dense(SwiGluCache),
Moe(Box<MoeGpuCache>),
}
struct MoeGpuCache {
probabilities: CudaSlice<f32>,
log_sum_exp: CudaSlice<f32>,
expert_of: CudaSlice<i32>,
token_of: CudaSlice<u32>,
slot_of: CudaSlice<u32>,
gates: CudaSlice<f32>,
offsets: Vec<usize>,
counts: Vec<usize>,
routed: usize,
input: CudaSlice<f32>,
expert: SwiGluCache,
output: CudaSlice<f32>,
shared: Option<SwiGluCache>,
}
struct Gpu<'a> {
context: &'a Arc<GpuContext>,
}
impl Gpu<'_> {
fn zeros(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
self.context
.stream
.alloc_zeros::<f32>(len.max(1))
.map_err(cuda_err("device allocation"))
}
fn uninit(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
unsafe { self.context.stream.alloc::<f32>(len.max(1)) }
.map_err(cuda_err("device allocation"))
}
fn uninit_bf16(&self, len: usize) -> Result<CudaSlice<bf16>, NetworkError> {
unsafe { self.context.stream.alloc::<bf16>(len.max(1)) }
.map_err(cuda_err("device allocation"))
}
fn cast_to_bf16(
&self,
dst: &mut CudaSlice<bf16>,
src: &CudaView<'_, f32>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.to_bf16)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 cast kernel"))?;
}
Ok(())
}
fn cast_from_bf16(
&self,
dst: &mut CudaViewMut<'_, f32>,
src: &CudaSlice<bf16>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.from_bf16)
.arg(dst)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 widening kernel"))?;
}
Ok(())
}
fn accumulate_bf16(
&self,
acc: &mut CudaSlice<f32>,
src: &CudaSlice<bf16>,
n: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.accumulate_bf16)
.arg(acc)
.arg(src)
.arg(&(n as i32))
.launch(cfg(n))
.map_err(cuda_err("BF16 accumulate kernel"))?;
}
Ok(())
}
fn upload(&self, data: &[f32]) -> Result<CudaSlice<f32>, NetworkError> {
self.context
.stream
.clone_htod(data)
.map_err(cuda_err("host to device copy"))
}
fn upload_indices(&self, data: &[u32]) -> Result<CudaSlice<u32>, NetworkError> {
self.context
.stream
.clone_htod(data)
.map_err(cuda_err("host to device copy"))
}
fn upload_flags(&self, data: &[i32]) -> Result<CudaSlice<i32>, NetworkError> {
self.context
.stream
.clone_htod(data)
.map_err(cuda_err("host to device copy"))
}
fn download(&self, source: &CudaSlice<f32>) -> Result<Vec<f32>, NetworkError> {
self.context
.stream
.clone_dtoh(source)
.map_err(cuda_err("device to host copy"))
}
fn upload_signed(&self, data: &[i32]) -> Result<CudaSlice<i32>, NetworkError> {
self.context
.stream
.clone_htod(data)
.map_err(cuda_err("host to device copy"))
}
fn download_signed(&self, source: &CudaSlice<i32>) -> Result<Vec<i32>, NetworkError> {
self.context
.stream
.clone_dtoh(source)
.map_err(cuda_err("device to host copy"))
}
fn duplicate(&self, source: &CudaSlice<f32>) -> Result<CudaSlice<f32>, NetworkError> {
self.context
.stream
.clone_dtod(source)
.map_err(cuda_err("device to device copy"))
}
fn matrix(
&self,
source: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<Matrix, NetworkError> {
let mut matrix = Matrix::new(rows, width);
self.context
.stream
.memcpy_dtoh(source, &mut matrix.data)
.map_err(cuda_err("device to host copy"))?;
Ok(matrix)
}
fn accumulate_host_grad(param: &mut Param, values: &[f32]) {
for (slot, value) in param.grad.data.iter_mut().zip(values) {
*slot += value;
}
}
fn rope_tables(&self, rope: &Rope) -> Result<Arc<DeviceRope>, NetworkError> {
let mut slot = self
.context
.rope
.lock()
.map_err(|_| NetworkError::Cuda("the rope table lock was poisoned".into()))?;
if let Some(tables) = slot.as_ref()
&& tables.head_dim == rope.head_dim()
&& tables.max_seq_len == rope.max_seq_len()
{
return Ok(tables.clone());
}
let tables = Arc::new(DeviceRope {
cos: self.upload(rope.cos())?,
sin: self.upload(rope.sin())?,
head_dim: rope.head_dim(),
max_seq_len: rope.max_seq_len(),
});
*slot = Some(tables.clone());
Ok(tables)
}
fn linear<X: DevicePtr<f32>>(
&self,
weight: &DeviceParam,
x: &X,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut out = self.uninit(rows * weight.rows())?;
gemm_rhs_transposed(
&self.context.blas,
x,
weight.cols(),
weight.value(),
weight.cols(),
&mut out,
weight.rows(),
rows,
weight.rows(),
weight.cols(),
1.0,
0.0,
)?;
Ok(out)
}
#[allow(clippy::too_many_arguments)]
fn linear_into<X: DevicePtr<f32>, O: DevicePtrMut<f32>>(
&self,
weight: &DeviceParam,
x: &X,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
gemm_rhs_transposed(
&self.context.blas,
x,
weight.cols(),
weight.value(),
weight.cols(),
out,
weight.rows(),
rows,
weight.rows(),
weight.cols(),
1.0,
beta,
)
}
fn linear_backward_input<G: DevicePtr<f32>, O: DevicePtrMut<f32>>(
&self,
weight: &DeviceParam,
grad_output: &G,
out: &mut O,
rows: usize,
beta: f32,
) -> Result<(), NetworkError> {
gemm_plain(
&self.context.blas,
grad_output,
weight.rows(),
weight.value(),
weight.cols(),
out,
weight.cols(),
rows,
weight.cols(),
weight.rows(),
1.0,
beta,
)
}
fn accumulate_weight_grad<G: DevicePtr<f32>, X: DevicePtr<f32>>(
&self,
weight: &mut DeviceParam,
grad_output: &G,
input: &X,
rows: usize,
) -> Result<(), NetworkError> {
let (units, cols) = (weight.rows(), weight.cols());
gemm_lhs_transposed(
&self.context.blas,
grad_output,
units,
input,
cols,
weight.negated_grad_mut(),
cols,
rows,
units,
cols,
-1.0,
1.0,
)
}
fn rmsnorm(
&self,
input: &CudaSlice<f32>,
weight: &CudaSlice<f32>,
rows: usize,
cols: usize,
eps: f32,
) -> Result<(CudaSlice<f32>, CudaSlice<f32>), NetworkError> {
let mut out = self.uninit(rows * cols)?;
let mut inverse = self.uninit(rows)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rmsnorm_forward)
.arg(&mut out)
.arg(&mut inverse)
.arg(input)
.arg(weight)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&eps)
.launch(row_grid(rows))
.map_err(cuda_err("RMSNorm kernel"))?;
}
Ok((out, inverse))
}
#[allow(clippy::too_many_arguments)]
fn rmsnorm_backward(
&self,
input: &CudaSlice<f32>,
grad_output: &CudaSlice<f32>,
weight: &CudaSlice<f32>,
inverse: &CudaSlice<f32>,
grad_weight: &mut CudaViewMut<'_, f32>,
rows: usize,
cols: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut grad_input = self.uninit(rows * cols)?;
let smem = rmsnorm_smem(cols);
let mut config = row_grid(rows);
config.shared_mem_bytes = smem.unwrap_or(0);
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rmsnorm_backward)
.arg(&mut grad_input)
.arg(grad_weight)
.arg(input)
.arg(grad_output)
.arg(weight)
.arg(inverse)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(i32::from(smem.is_some())))
.launch(config)
.map_err(cuda_err("RMSNorm backward kernel"))?;
}
Ok(grad_input)
}
#[allow(clippy::too_many_arguments)]
fn rope(
&self,
tensor: &mut CudaSlice<f32>,
tables: &DeviceRope,
rows: usize,
heads: usize,
head_dim: usize,
seq_len: usize,
direction: f32,
) -> Result<(), NetworkError> {
let elements = rows * heads * head_dim / 2;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.rope)
.arg(tensor)
.arg(&tables.cos)
.arg(&tables.sin)
.arg(&(rows as i32))
.arg(&(heads as i32))
.arg(&(head_dim as i32))
.arg(&(seq_len as i32))
.arg(&direction)
.launch(cfg(elements))
.map_err(cuda_err("RoPE kernel"))?;
}
Ok(())
}
fn causal_softmax(
&self,
scores: &mut CudaSlice<f32>,
rows: usize,
seq_len: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.causal_softmax)
.arg(scores)
.arg(&(rows as i32))
.arg(&(seq_len as i32))
.launch(warp_per_row_grid(rows))
.map_err(cuda_err("causal softmax kernel"))?;
}
Ok(())
}
fn causal_softmax_backward(
&self,
grad: &mut CudaSlice<f32>,
probabilities: &CudaSlice<f32>,
rows: usize,
seq_len: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.causal_softmax_backward)
.arg(grad)
.arg(probabilities)
.arg(&(rows as i32))
.arg(&(seq_len as i32))
.launch(warp_per_row_grid(rows))
.map_err(cuda_err("causal softmax backward kernel"))?;
}
Ok(())
}
fn add(
&self,
target: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
len: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.add)
.arg(target)
.arg(source)
.arg(&(len as i32))
.launch(cfg(len))
.map_err(cuda_err("addition kernel"))?;
}
Ok(())
}
fn gather(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gather)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(&0i32)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("row gather kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn gather_scaled(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
scale: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.gather_scaled)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(scale)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("scaled row gather kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn scatter_scaled(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
scale: &CudaSlice<f32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter_scaled)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(scale)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("scaled row scatter kernel"))?;
}
Ok(())
}
fn scatter(
&self,
out: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter)
.arg(out)
.arg(source)
.arg(rows_of)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("row scatter kernel"))?;
}
Ok(())
}
fn scatter_negated(
&self,
grad: &mut CudaSlice<f32>,
upstream: &CudaSlice<f32>,
rows_of: &CudaSlice<u32>,
rows: usize,
width: usize,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.context.model.scatter_negated)
.arg(grad)
.arg(upstream)
.arg(rows_of)
.arg(&(rows as i32))
.arg(&(width as i32))
.launch(cfg(rows * width))
.map_err(cuda_err("embedding scatter kernel"))?;
}
Ok(())
}
fn swiglu(
&self,
gate: &CudaSlice<f32>,
up: &CudaSlice<f32>,
len: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut hidden = self.uninit(len)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.swiglu_forward)
.arg(&mut hidden)
.arg(gate)
.arg(up)
.arg(&(len as i32))
.launch(cfg(len))
.map_err(cuda_err("SwiGLU kernel"))?;
}
Ok(hidden)
}
fn swiglu_backward(
&self,
cache: &SwiGluCache,
grad_hidden: &CudaSlice<f32>,
len: usize,
) -> Result<(CudaSlice<f32>, CudaSlice<f32>), NetworkError> {
let mut grad_gate = self.uninit(len)?;
let mut grad_up = self.uninit(len)?;
unsafe {
self.context
.stream
.launch_builder(&self.context.model.swiglu_backward)
.arg(&mut grad_gate)
.arg(&mut grad_up)
.arg(&cache.gate)
.arg(&cache.up)
.arg(grad_hidden)
.arg(&(len as i32))
.launch(cfg(len))
.map_err(cuda_err("SwiGLU backward kernel"))?;
}
Ok((grad_gate, grad_up))
}
}
fn head_of(model: &TransformerLm) -> Result<&DeviceParam, NetworkError> {
match &model.lm_head {
Some(head) => device_of(head),
None => model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
}),
}
}
fn head_of_mut(model: &mut TransformerLm) -> Result<&mut DeviceParam, NetworkError> {
match &mut model.lm_head {
Some(head) => device_of_mut(head),
None => model.embedding.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
}),
}
}
fn device_of(linear: &Linear) -> Result<&DeviceParam, NetworkError> {
linear.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("a projection is not resident on the device".into())
})
}
fn device_of_mut(linear: &mut Linear) -> Result<&mut DeviceParam, NetworkError> {
linear.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("a projection is not resident on the device".into())
})
}
fn unsupported(kind: &str) -> NetworkError {
NetworkError::UnsupportedCuda(format!(
"the device training path implements SwiGLU feed-forwards and MoE layers, not {kind}"
))
}
pub(crate) fn forward(
model: &TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<(Matrix, GpuCache), NetworkError> {
let cache = forward_hidden(model, context, batch)?;
let gpu = Gpu { context };
let logits = gpu.linear(head_of(model)?, &cache.final_output, cache.rows)?;
let logits = gpu.matrix(&logits, cache.rows, model.embedding.vocab_size())?;
Ok((logits, cache))
}
fn forward_hidden(
model: &TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<GpuCache, NetworkError> {
let gpu = Gpu { context };
let rows = batch.rows();
let seq_len = batch.seq_len();
let sequences = batch.batch();
let d_model = model.config.d_model;
let vocab = model.embedding.vocab_size();
for &id in batch.ids() {
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
}
let layout = batch.layout();
let flags: Vec<i32> = (0..rows).map(|row| i32::from(layout.is_valid(row))).collect();
let valid_tokens = flags.iter().filter(|&&flag| flag != 0).count();
let valid = gpu.upload_flags(&flags)?;
let ids = gpu.upload_indices(batch.ids())?;
let embedding = model.embedding.weight.device.as_ref().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
let mut hidden = gpu.uninit(rows * d_model)?;
gpu.gather(&mut hidden, embedding.value(), &ids, rows, d_model)?;
let mut blocks = Vec::with_capacity(model.blocks.len());
let mut auxiliary_loss = 0.0;
for block in &model.blocks {
let (output, cache) = forward_block(
&gpu,
block,
hidden,
&valid,
valid_tokens,
rows,
seq_len,
sequences,
&mut auxiliary_loss,
)?;
blocks.push(cache);
hidden = output;
}
let final_weight = gpu.upload(&model.final_norm.weight.value.data)?;
let (final_output, final_inverse_rms) = gpu.rmsnorm(
&hidden,
&final_weight,
rows,
d_model,
model.final_norm.eps,
)?;
Ok(GpuCache {
rows,
seq_len,
sequences,
ids,
valid_tokens,
blocks,
final_input: hidden,
final_inverse_rms,
final_weight,
final_output,
auxiliary_loss,
})
}
#[allow(clippy::too_many_arguments)]
fn forward_block(
gpu: &Gpu<'_>,
block: &TransformerBlock,
input: CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
seq_len: usize,
sequences: usize,
auxiliary_loss: &mut f32,
) -> Result<(CudaSlice<f32>, BlockCache), NetworkError> {
let attention = &block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let group = heads / kv_heads;
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let kv_width = kv_heads * head_dim;
let attention_weight = gpu.upload(&block.attention_norm.weight.value.data)?;
let (attention_normed, attention_inverse_rms) = gpu.rmsnorm(
&input,
&attention_weight,
rows,
d_model,
block.attention_norm.eps,
)?;
let mut queries = gpu.linear(device_of(&attention.query)?, &attention_normed, rows)?;
let mut keys = gpu.linear(device_of(&attention.key)?, &attention_normed, rows)?;
let values = gpu.linear(device_of(&attention.value)?, &attention_normed, rows)?;
let tables = gpu.rope_tables(&attention.rope)?;
if seq_len > tables.max_seq_len {
return Err(NetworkError::SequenceTooLong {
length: seq_len,
max_seq_len: tables.max_seq_len,
});
}
gpu.rope(&mut queries, &tables, rows, heads, head_dim, seq_len, 1.0)?;
gpu.rope(&mut keys, &tables, rows, kv_heads, head_dim, seq_len, 1.0)?;
let mut probabilities = gpu.uninit(heads * sequences * seq_len * seq_len)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let query = queries.slice(head * head_dim..);
let key = keys.slice(kv_base..);
let mut scores = probabilities.slice_mut(head * sequences * seq_len * seq_len..);
gemm_rhs_transposed_batched(
&gpu.context.blas,
&query,
query_width,
&key,
kv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
scale,
0.0,
sequences,
seq_len * query_width,
seq_len * kv_width,
seq_len * seq_len,
)?;
}
gpu.causal_softmax(&mut probabilities, heads * sequences * seq_len, seq_len)?;
let mut merged = gpu.uninit(rows * query_width)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let scores = probabilities.slice(head * sequences * seq_len * seq_len..);
let value = values.slice(kv_base..);
let mut out = merged.slice_mut(head * head_dim..);
gemm_plain_batched(
&gpu.context.blas,
&scores,
seq_len,
&value,
kv_width,
&mut out,
query_width,
seq_len,
head_dim,
seq_len,
1.0,
0.0,
sequences,
seq_len * seq_len,
seq_len * kv_width,
seq_len * query_width,
)?;
}
let mut residual = gpu.duplicate(&input)?;
gpu.linear_into(
device_of(&attention.output)?,
&merged,
&mut residual,
rows,
1.0,
)?;
let feed_forward_weight = gpu.upload(&block.feed_forward_norm.weight.value.data)?;
let (feed_forward_normed, feed_forward_inverse_rms) = gpu.rmsnorm(
&residual,
&feed_forward_weight,
rows,
d_model,
block.feed_forward_norm.eps,
)?;
let mut output = gpu.duplicate(&residual)?;
let feed_forward = match &block.feed_forward {
FeedForward::SwiGlu(ffn) => FfnCache::Dense(forward_swiglu(
gpu,
ffn,
&feed_forward_normed,
&mut output,
rows,
)?),
FeedForward::Gelu(_) => return Err(unsupported("a GELU MLP")),
FeedForward::Moe(moe) => {
let (cache, losses) = forward_moe(
gpu,
moe,
&feed_forward_normed,
&mut output,
valid,
valid_tokens,
rows,
)?;
*auxiliary_loss += losses;
FfnCache::Moe(Box::new(cache))
}
};
Ok((
output,
BlockCache {
input,
attention_weight,
attention_inverse_rms,
attention_normed,
queries,
keys,
values,
probabilities,
merged,
residual,
feed_forward_weight,
feed_forward_inverse_rms,
feed_forward_normed,
feed_forward,
},
))
}
fn forward_swiglu(
gpu: &Gpu<'_>,
ffn: &SwiGlu,
input: &CudaSlice<f32>,
out: &mut CudaSlice<f32>,
rows: usize,
) -> Result<SwiGluCache, NetworkError> {
let width = ffn.d_ff();
let gate = gpu.linear(device_of(&ffn.gate)?, input, rows)?;
let up = gpu.linear(device_of(&ffn.up)?, input, rows)?;
let hidden = gpu.swiglu(&gate, &up, rows * width)?;
gpu.linear_into(device_of(&ffn.down)?, &hidden, out, rows, 1.0)?;
Ok(SwiGluCache { gate, up, hidden })
}
#[allow(clippy::too_many_arguments)]
fn forward_moe(
gpu: &Gpu<'_>,
moe: &MoeLayer,
input: &CudaSlice<f32>,
out: &mut CudaSlice<f32>,
valid: &CudaSlice<i32>,
valid_tokens: usize,
rows: usize,
) -> Result<(MoeGpuCache, f32), NetworkError> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let logits = gpu.linear(device_of(&moe.router.projection)?, input, rows)?;
let mut probabilities = gpu.zeros(rows * experts)?;
let mut log_sum_exp = gpu.zeros(rows)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.softmax_lse)
.arg(&mut probabilities)
.arg(&mut log_sum_exp)
.arg(&logits)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.map_err(cuda_err("router softmax kernel"))?;
}
let mut expert_of = gpu
.context
.stream
.alloc_zeros::<i32>(rows * top_k)
.map_err(cuda_err("device allocation"))?;
let mut gate_of = gpu.zeros(rows * top_k)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.topk_gate)
.arg(&mut expert_of)
.arg(&mut gate_of)
.arg(&probabilities)
.arg(valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.map_err(cuda_err("router top-k kernel"))?;
}
let assigned = gpu.download_signed(&expert_of)?;
let gates_host = gpu.download(&gate_of)?;
let mut counts = vec![0usize; experts];
for &expert in &assigned {
if expert >= 0 {
counts[expert as usize] += 1;
}
}
let mut offsets = Vec::with_capacity(experts);
let mut routed = 0;
for &count in &counts {
offsets.push(routed);
routed += count;
}
let mut cursor = offsets.clone();
let mut token_host = vec![0u32; routed.max(1)];
let mut slot_host = vec![0u32; routed.max(1)];
let mut gate_host = vec![0f32; routed.max(1)];
for (slot, &expert) in assigned.iter().enumerate() {
if expert < 0 {
continue;
}
let position = &mut cursor[expert as usize];
token_host[*position] = (slot / top_k) as u32;
slot_host[*position] = slot as u32;
gate_host[*position] = gates_host[slot];
*position += 1;
}
let token_of = gpu.upload_indices(&token_host)?;
let slot_of = gpu.upload_indices(&slot_host)?;
let gates = gpu.upload(&gate_host)?;
let mut gathered = gpu.uninit(routed * d_model)?;
if routed > 0 {
gpu.gather(&mut gathered, input, &token_of, routed, d_model)?;
}
let mut gate_activations = gpu.uninit(routed * width)?;
let mut up_activations = gpu.uninit(routed * width)?;
for (expert, module) in moe.experts.iter().enumerate() {
let count = counts[expert];
if count == 0 {
continue;
}
let rows_in = gathered.slice(offsets[expert] * d_model..);
let mut gate = gate_activations.slice_mut(offsets[expert] * width..);
gpu.linear_into(device_of(&module.gate)?, &rows_in, &mut gate, count, 0.0)?;
let mut up = up_activations.slice_mut(offsets[expert] * width..);
gpu.linear_into(device_of(&module.up)?, &rows_in, &mut up, count, 0.0)?;
}
let hidden = gpu.swiglu(&gate_activations, &up_activations, routed * width)?;
let mut expert_output = gpu.uninit(routed * d_model)?;
for (expert, module) in moe.experts.iter().enumerate() {
let count = counts[expert];
if count == 0 {
continue;
}
let rows_in = hidden.slice(offsets[expert] * width..);
let mut rows_out = expert_output.slice_mut(offsets[expert] * d_model..);
gpu.linear_into(device_of(&module.down)?, &rows_in, &mut rows_out, count, 0.0)?;
}
if routed > 0 {
gpu.scatter_scaled(out, &expert_output, &token_of, &gates, routed, d_model)?;
}
let shared = match &moe.shared {
Some(module) => Some(forward_swiglu(gpu, module, input, out, rows)?),
None => None,
};
let mut sums = gpu.zeros(experts + 1)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.stats)
.arg(&mut sums)
.arg(&probabilities)
.arg(&log_sum_exp)
.arg(valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.map_err(cuda_err("MoE statistics kernel"))?;
}
let sums = gpu.download(&sums)?;
let mut auxiliary = 0.0;
if valid_tokens > 0 {
let slots = (valid_tokens * top_k) as f32;
let balance: f32 = (0..experts)
.map(|expert| {
(counts[expert] as f32 / slots) * (sums[expert] / valid_tokens as f32)
})
.sum();
auxiliary += experts as f32 * moe.config.aux_loss_weight * balance;
auxiliary += moe.config.router_z_loss_weight * sums[experts] / valid_tokens as f32;
}
Ok((
MoeGpuCache {
probabilities,
log_sum_exp,
expert_of,
token_of,
slot_of,
gates,
offsets,
counts,
routed,
input: gathered,
expert: SwiGluCache {
gate: gate_activations,
up: up_activations,
hidden,
},
output: expert_output,
shared,
},
auxiliary,
))
}
pub(crate) fn backward(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
cache: &GpuCache,
grad_logits: &Matrix,
) -> Result<(), NetworkError> {
let gpu = Gpu { context };
let rows = cache.rows;
let d_model = model.config.d_model;
if grad_logits.rows != rows {
return Err(NetworkError::InvalidInput {
expected: rows,
actual: grad_logits.rows,
});
}
let upstream = gpu.upload(&grad_logits.data)?;
let mut grad_final = gpu.zeros(rows * d_model)?;
{
let head = head_of_mut(model)?;
gpu.linear_backward_input(head, &upstream, &mut grad_final, rows, 0.0)?;
gpu.accumulate_weight_grad(head, &upstream, &cache.final_output, rows)?;
}
backward_from_final(model, context, cache, &grad_final)
}
pub(crate) fn train_step(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
batch: &TokenBatch,
) -> Result<(f32, f32), NetworkError> {
let vocab = model.embedding.vocab_size();
let predicted = batch.predicted();
if predicted == 0 {
return Err(NetworkError::InvalidConfig(
"a causal language-modelling loss needs at least two tokens".into(),
));
}
if vocab > i32::MAX as usize {
return Err(NetworkError::InvalidConfig(format!(
"the device loss supports vocabularies up to {}, got {vocab}",
i32::MAX
)));
}
let targets = causal_targets(batch, vocab)?;
let cache = forward_hidden(model, context, batch)?;
let gpu = Gpu { context };
let rows = cache.rows;
let d_model = model.config.d_model;
let targets = gpu.upload_signed(&targets)?;
let mut loss = gpu.zeros(1)?;
let mut grad_final = gpu.zeros(rows * d_model)?;
let inverse_predicted = 1.0 / predicted as f32;
let logit_bytes = if gpu.context.mixed_precision { 2 } else { 4 };
let chunk = head_chunk_rows(rows, vocab, logit_bytes);
{
let head = head_of_mut(model)?;
let mut reduced = gpu
.context
.mixed_precision
.then(|| HeadBf16::new(&gpu, head, chunk))
.transpose()?;
let mut start = 0;
while start < rows {
let count = chunk.min(rows - start);
let input = cache.final_output.slice(start * d_model..(start + count) * d_model);
let targets = targets.slice(start..start + count);
let mut out = grad_final.slice_mut(start * d_model..(start + count) * d_model);
if let Some(reduced) = reduced.as_mut() {
reduced.chunk(
&gpu,
head,
&input,
&targets,
&mut loss,
&mut out,
count,
inverse_predicted,
)?;
start += count;
continue;
}
let mut logits = gpu.linear(head, &input, count)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.cross_entropy)
.arg(&mut logits)
.arg(&mut loss)
.arg(&targets)
.arg(&(count as i32))
.arg(&(vocab as i32))
.arg(&inverse_predicted)
.launch(row_per_block(count))
.map_err(cuda_err("cross-entropy kernel"))?;
}
gpu.linear_backward_input(head, &logits, &mut out, count, 0.0)?;
gpu.accumulate_weight_grad(head, &logits, &input, count)?;
start += count;
}
}
backward_from_final(model, context, &cache, &grad_final)?;
let lm_loss = gpu.download(&loss)?[0] / predicted as f32;
Ok((lm_loss, cache.auxiliary_loss))
}
struct HeadBf16 {
weight: CudaSlice<bf16>,
weight_grad: CudaSlice<bf16>,
input: CudaSlice<bf16>,
logits: CudaSlice<bf16>,
grad_input: CudaSlice<bf16>,
}
impl HeadBf16 {
fn new(gpu: &Gpu, head: &DeviceParam, chunk: usize) -> Result<Self, NetworkError> {
let (vocab, d_model) = (head.rows(), head.cols());
let mut weight = gpu.uninit_bf16(vocab * d_model)?;
gpu.cast_to_bf16(&mut weight, &head.value().slice(..), vocab * d_model)?;
Ok(Self {
weight,
weight_grad: gpu.uninit_bf16(vocab * d_model)?,
input: gpu.uninit_bf16(chunk * d_model)?,
logits: gpu.uninit_bf16(chunk * vocab)?,
grad_input: gpu.uninit_bf16(chunk * d_model)?,
})
}
#[allow(clippy::too_many_arguments)]
fn chunk(
&mut self,
gpu: &Gpu,
head: &mut DeviceParam,
input: &CudaView<'_, f32>,
targets: &CudaView<'_, i32>,
loss: &mut CudaSlice<f32>,
grad_input: &mut CudaViewMut<'_, f32>,
count: usize,
inverse_predicted: f32,
) -> Result<(), NetworkError> {
let (vocab, d_model) = (head.rows(), head.cols());
let blas = &gpu.context.blas;
gpu.cast_to_bf16(&mut self.input, input, count * d_model)?;
gemm_rhs_transposed(
blas,
&self.input,
d_model,
&self.weight,
d_model,
&mut self.logits,
vocab,
count,
vocab,
d_model,
bf16::ONE,
bf16::ZERO,
)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.cross_entropy_bf16)
.arg(&mut self.logits)
.arg(loss)
.arg(targets)
.arg(&(count as i32))
.arg(&(vocab as i32))
.arg(&inverse_predicted)
.launch(row_per_block(count))
.map_err(cuda_err("cross-entropy kernel"))?;
}
gemm_plain(
blas,
&self.logits,
vocab,
&self.weight,
d_model,
&mut self.grad_input,
d_model,
count,
d_model,
vocab,
bf16::ONE,
bf16::ZERO,
)?;
gpu.cast_from_bf16(grad_input, &self.grad_input, count * d_model)?;
gemm_lhs_transposed(
blas,
&self.logits,
vocab,
&self.input,
d_model,
&mut self.weight_grad,
d_model,
count,
vocab,
d_model,
bf16::NEG_ONE,
bf16::ZERO,
)?;
gpu.accumulate_bf16(head.negated_grad_mut(), &self.weight_grad, vocab * d_model)
}
}
fn causal_targets(batch: &TokenBatch, vocab: usize) -> Result<Vec<i32>, NetworkError> {
let seq_len = batch.seq_len();
let ids = batch.ids();
let mut targets = vec![-1i32; batch.rows()];
for (sequence, &length) in batch.lengths().iter().enumerate() {
for position in 0..length.saturating_sub(1) {
let row = sequence * seq_len + position;
let id = ids[row + 1];
if id as usize >= vocab {
return Err(NetworkError::TokenOutOfRange {
id,
vocab_size: vocab,
});
}
targets[row] = id as i32;
}
}
Ok(targets)
}
fn head_chunk_rows(rows: usize, vocab: usize, logit_bytes: usize) -> usize {
#[cfg(test)]
{
let forced = CHUNK_ROWS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed);
if forced > 0 {
return forced.min(rows.max(1));
}
}
const BUDGET_BYTES: usize = 256 << 20;
(BUDGET_BYTES / (vocab * logit_bytes)).clamp(1, rows.max(1))
}
#[cfg(test)]
pub(crate) static CHUNK_ROWS_OVERRIDE: std::sync::atomic::AtomicUsize =
std::sync::atomic::AtomicUsize::new(0);
fn row_per_block(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.max(1) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
const ROW_GRID_BLOCKS: usize = 2048;
fn warp_per_row_grid(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.div_ceil(8).clamp(1, ROW_GRID_BLOCKS) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn row_grid(rows: usize) -> cudarc::driver::LaunchConfig {
cudarc::driver::LaunchConfig {
grid_dim: (rows.clamp(1, ROW_GRID_BLOCKS) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn rmsnorm_smem(cols: usize) -> Option<u32> {
let bytes = cols.checked_mul(4)?;
(bytes <= 48 << 10).then_some(bytes as u32)
}
fn backward_from_final(
model: &mut TransformerLm,
context: &Arc<GpuContext>,
cache: &GpuCache,
grad_final: &CudaSlice<f32>,
) -> Result<(), NetworkError> {
let gpu = Gpu { context };
let rows = cache.rows;
let d_model = model.config.d_model;
let mut grad_norms = gpu.zeros(norm_slots(model.blocks.len()) * d_model)?;
let mut grad_hidden = gpu.rmsnorm_backward(
&cache.final_input,
grad_final,
&cache.final_weight,
&cache.final_inverse_rms,
&mut grad_norms.slice_mut(0..d_model),
rows,
d_model,
)?;
for (index, (block, block_cache)) in
model.blocks.iter_mut().zip(&cache.blocks).enumerate().rev()
{
grad_hidden = backward_block(
&gpu,
block,
block_cache,
&grad_hidden,
cache,
&mut grad_norms,
index,
)?;
}
{
let embedding = model.embedding.weight.device.as_mut().ok_or_else(|| {
NetworkError::Cuda("the embedding table is not resident on the device".into())
})?;
gpu.scatter_negated(
embedding.negated_grad_mut(),
&grad_hidden,
&cache.ids,
rows,
d_model,
)?;
}
let values = gpu.download(&grad_norms)?;
let at = |slot: usize| &values[slot * d_model..(slot + 1) * d_model];
Gpu::accumulate_host_grad(&mut model.final_norm.weight, at(0));
for (index, block) in model.blocks.iter_mut().enumerate() {
let (attention, feed_forward) = norm_slot(index);
Gpu::accumulate_host_grad(&mut block.attention_norm.weight, at(attention));
Gpu::accumulate_host_grad(&mut block.feed_forward_norm.weight, at(feed_forward));
}
Ok(())
}
fn norm_slots(blocks: usize) -> usize {
1 + 2 * blocks
}
fn norm_slot(index: usize) -> (usize, usize) {
(1 + 2 * index, 2 + 2 * index)
}
#[allow(clippy::too_many_arguments)]
fn backward_block(
gpu: &Gpu<'_>,
block: &mut TransformerBlock,
cache: &BlockCache,
grad_output: &CudaSlice<f32>,
batch: &GpuCache,
grad_norms: &mut CudaSlice<f32>,
index: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let rows = batch.rows;
let d_model = block.attention.d_model();
let (attention_slot, feed_forward_slot) = norm_slot(index);
let mut grad_normed = match block.feed_forward {
FeedForward::SwiGlu(_) => gpu.uninit(rows * d_model)?,
_ => gpu.zeros(rows * d_model)?,
};
match (&mut block.feed_forward, &cache.feed_forward) {
(FeedForward::SwiGlu(ffn), FfnCache::Dense(ffn_cache)) => backward_swiglu(
gpu,
ffn,
ffn_cache,
&cache.feed_forward_normed,
grad_output,
&mut grad_normed,
rows,
)?,
(FeedForward::Moe(moe), FfnCache::Moe(moe_cache)) => backward_moe(
gpu,
moe,
moe_cache,
&cache.feed_forward_normed,
grad_output,
&mut grad_normed,
batch.valid_tokens,
rows,
)?,
(FeedForward::Gelu(_), _) => return Err(unsupported("a GELU MLP")),
_ => {
return Err(NetworkError::Cuda(
"the device cache does not match the layer that produced it".into(),
));
}
}
let mut grad_residual = gpu.rmsnorm_backward(
&cache.residual,
&grad_normed,
&cache.feed_forward_weight,
&cache.feed_forward_inverse_rms,
&mut grad_norms.slice_mut(feed_forward_slot * d_model..(feed_forward_slot + 1) * d_model),
rows,
d_model,
)?;
gpu.add(&mut grad_residual, grad_output, rows * d_model)?;
let grad_attention_normed = backward_attention(gpu, block, cache, &grad_residual, batch)?;
let mut grad_input = gpu.rmsnorm_backward(
&cache.input,
&grad_attention_normed,
&cache.attention_weight,
&cache.attention_inverse_rms,
&mut grad_norms.slice_mut(attention_slot * d_model..(attention_slot + 1) * d_model),
rows,
d_model,
)?;
gpu.add(&mut grad_input, &grad_residual, rows * d_model)?;
Ok(grad_input)
}
fn backward_attention(
gpu: &Gpu<'_>,
block: &mut TransformerBlock,
cache: &BlockCache,
grad_output: &CudaSlice<f32>,
batch: &GpuCache,
) -> Result<CudaSlice<f32>, NetworkError> {
let rows = batch.rows;
let seq_len = batch.seq_len;
let sequences = batch.sequences;
let attention = &mut block.attention;
let d_model = attention.d_model();
let heads = attention.num_heads();
let kv_heads = attention.num_kv_heads();
let head_dim = attention.head_dim();
let group = heads / kv_heads;
let scale = (head_dim as f32).sqrt().recip();
let query_width = heads * head_dim;
let kv_width = kv_heads * head_dim;
let block_size = seq_len * seq_len;
let mut grad_merged = gpu.uninit(rows * query_width)?;
gpu.linear_backward_input(
device_of(&attention.output)?,
grad_output,
&mut grad_merged,
rows,
0.0,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut attention.output)?,
grad_output,
&cache.merged,
rows,
)?;
let mut grad_scores = gpu.uninit(heads * sequences * block_size)?;
let mut grad_values = gpu.zeros(rows * kv_width)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let upstream = grad_merged.slice(head * head_dim..);
let value = cache.values.slice(kv_base..);
let mut scores = grad_scores.slice_mut(head * sequences * block_size..);
gemm_rhs_transposed_batched(
&gpu.context.blas,
&upstream,
query_width,
&value,
kv_width,
&mut scores,
seq_len,
seq_len,
seq_len,
head_dim,
1.0,
0.0,
sequences,
seq_len * query_width,
seq_len * kv_width,
block_size,
)?;
let probabilities = cache.probabilities.slice(head * sequences * block_size..);
let mut grad_value = grad_values.slice_mut(kv_base..);
gemm_lhs_transposed_batched(
&gpu.context.blas,
&probabilities,
seq_len,
&upstream,
query_width,
&mut grad_value,
kv_width,
seq_len,
seq_len,
head_dim,
1.0,
1.0,
sequences,
block_size,
seq_len * query_width,
seq_len * kv_width,
)?;
}
gpu.causal_softmax_backward(
&mut grad_scores,
&cache.probabilities,
heads * sequences * seq_len,
seq_len,
)?;
let mut grad_queries = gpu.uninit(rows * query_width)?;
let mut grad_keys = gpu.zeros(rows * kv_width)?;
for head in 0..heads {
let kv_base = (head / group) * head_dim;
let scores = grad_scores.slice(head * sequences * block_size..);
let key = cache.keys.slice(kv_base..);
let mut grad_query = grad_queries.slice_mut(head * head_dim..);
gemm_plain_batched(
&gpu.context.blas,
&scores,
seq_len,
&key,
kv_width,
&mut grad_query,
query_width,
seq_len,
head_dim,
seq_len,
scale,
0.0,
sequences,
block_size,
seq_len * kv_width,
seq_len * query_width,
)?;
let query = cache.queries.slice(head * head_dim..);
let mut grad_key = grad_keys.slice_mut(kv_base..);
gemm_lhs_transposed_batched(
&gpu.context.blas,
&scores,
seq_len,
&query,
query_width,
&mut grad_key,
kv_width,
seq_len,
seq_len,
head_dim,
scale,
1.0,
sequences,
block_size,
seq_len * query_width,
seq_len * kv_width,
)?;
}
let tables = gpu.rope_tables(&attention.rope)?;
gpu.rope(
&mut grad_queries,
&tables,
rows,
heads,
head_dim,
seq_len,
-1.0,
)?;
gpu.rope(
&mut grad_keys,
&tables,
rows,
kv_heads,
head_dim,
seq_len,
-1.0,
)?;
let mut grad_input = gpu.uninit(rows * d_model)?;
gpu.linear_backward_input(
device_of(&attention.query)?,
&grad_queries,
&mut grad_input,
rows,
0.0,
)?;
gpu.linear_backward_input(
device_of(&attention.key)?,
&grad_keys,
&mut grad_input,
rows,
1.0,
)?;
gpu.linear_backward_input(
device_of(&attention.value)?,
&grad_values,
&mut grad_input,
rows,
1.0,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut attention.query)?,
&grad_queries,
&cache.attention_normed,
rows,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut attention.key)?,
&grad_keys,
&cache.attention_normed,
rows,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut attention.value)?,
&grad_values,
&cache.attention_normed,
rows,
)?;
Ok(grad_input)
}
fn backward_swiglu(
gpu: &Gpu<'_>,
ffn: &mut SwiGlu,
cache: &SwiGluCache,
input: &CudaSlice<f32>,
grad_output: &CudaSlice<f32>,
grad_input: &mut CudaSlice<f32>,
rows: usize,
) -> Result<(), NetworkError> {
let width = ffn.d_ff();
let mut grad_hidden = gpu.uninit(rows * width)?;
gpu.linear_backward_input(
device_of(&ffn.down)?,
grad_output,
&mut grad_hidden,
rows,
0.0,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut ffn.down)?,
grad_output,
&cache.hidden,
rows,
)?;
let (grad_gate, grad_up) = gpu.swiglu_backward(cache, &grad_hidden, rows * width)?;
gpu.linear_backward_input(device_of(&ffn.gate)?, &grad_gate, grad_input, rows, 0.0)?;
gpu.accumulate_weight_grad(device_of_mut(&mut ffn.gate)?, &grad_gate, input, rows)?;
gpu.linear_backward_input(device_of(&ffn.up)?, &grad_up, grad_input, rows, 1.0)?;
gpu.accumulate_weight_grad(device_of_mut(&mut ffn.up)?, &grad_up, input, rows)?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn backward_moe(
gpu: &Gpu<'_>,
moe: &mut MoeLayer,
cache: &MoeGpuCache,
input: &CudaSlice<f32>,
grad_output: &CudaSlice<f32>,
grad_input: &mut CudaSlice<f32>,
valid_tokens: usize,
rows: usize,
) -> Result<(), NetworkError> {
let experts = moe.config.num_experts;
let top_k = moe.config.experts_per_token;
let d_model = moe.d_model();
let width = moe.config.d_ff;
let routed = cache.routed;
if let (Some(shared), Some(shared_cache)) = (moe.shared.as_mut(), &cache.shared) {
backward_swiglu(
gpu,
shared,
shared_cache,
input,
grad_output,
grad_input,
rows,
)?;
}
let mut grad_gates = gpu.zeros(rows * top_k)?;
if routed > 0 {
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.row_dot)
.arg(&mut grad_gates)
.arg(&cache.slot_of)
.arg(grad_output)
.arg(&cache.token_of)
.arg(&cache.output)
.arg(&(routed as i32))
.arg(&(d_model as i32))
.launch(warp_per_row_grid(routed))
.map_err(cuda_err("gate gradient kernel"))?;
}
let mut grad_expert_output = gpu.uninit(routed * d_model)?;
gpu.gather_scaled(
&mut grad_expert_output,
grad_output,
&cache.token_of,
&cache.gates,
routed,
d_model,
)?;
let mut grad_hidden = gpu.uninit(routed * width)?;
for (expert, module) in moe.experts.iter_mut().enumerate() {
let count = cache.counts[expert];
if count == 0 {
continue;
}
let offset = cache.offsets[expert];
let upstream = grad_expert_output.slice(offset * d_model..);
let mut grad = grad_hidden.slice_mut(offset * width..);
gpu.linear_backward_input(device_of(&module.down)?, &upstream, &mut grad, count, 0.0)?;
let hidden = cache.expert.hidden.slice(offset * width..);
gpu.accumulate_weight_grad(
device_of_mut(&mut module.down)?,
&upstream,
&hidden,
count,
)?;
}
let (grad_gate, grad_up) = gpu.swiglu_backward(&cache.expert, &grad_hidden, routed * width)?;
let mut grad_expert_input = gpu.uninit(routed * d_model)?;
for (expert, module) in moe.experts.iter_mut().enumerate() {
let count = cache.counts[expert];
if count == 0 {
continue;
}
let offset = cache.offsets[expert];
let rows_in = cache.input.slice(offset * d_model..);
let upstream = grad_gate.slice(offset * width..);
let mut grad = grad_expert_input.slice_mut(offset * d_model..);
gpu.linear_backward_input(device_of(&module.gate)?, &upstream, &mut grad, count, 0.0)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut module.gate)?,
&upstream,
&rows_in,
count,
)?;
let upstream = grad_up.slice(offset * width..);
let mut grad = grad_expert_input.slice_mut(offset * d_model..);
gpu.linear_backward_input(device_of(&module.up)?, &upstream, &mut grad, count, 1.0)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut module.up)?,
&upstream,
&rows_in,
count,
)?;
}
gpu.scatter(
grad_input,
&grad_expert_input,
&cache.token_of,
routed,
d_model,
)?;
}
let mut grad_probabilities = gpu.zeros(rows * experts)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.grad_probabilities)
.arg(&mut grad_probabilities)
.arg(&grad_gates)
.arg(&cache.probabilities)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.map_err(cuda_err("router gate gradient kernel"))?;
}
if moe.config.aux_loss_weight != 0.0 && valid_tokens > 0 {
let slots = (valid_tokens * top_k) as f32;
let loads: Vec<f32> = cache
.counts
.iter()
.map(|&count| count as f32 / slots)
.collect();
let loads = gpu.upload(&loads)?;
let scale = moe.config.aux_loss_weight * experts as f32 / valid_tokens as f32;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.aux_grad)
.arg(&mut grad_probabilities)
.arg(&loads)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.arg(&scale)
.launch(cfg(rows * experts))
.map_err(cuda_err("load-balancing gradient kernel"))?;
}
}
let z_factor = if moe.config.router_z_loss_weight != 0.0 {
2.0 * moe.config.router_z_loss_weight / valid_tokens.max(1) as f32
} else {
0.0
};
let mut grad_logits = gpu.zeros(rows * experts)?;
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.grad_logits)
.arg(&mut grad_logits)
.arg(&grad_probabilities)
.arg(&cache.probabilities)
.arg(&cache.log_sum_exp)
.arg(&cache.expert_of)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.arg(&z_factor)
.launch(cfg(rows))
.map_err(cuda_err("router logit gradient kernel"))?;
}
gpu.linear_backward_input(
device_of(&moe.router.projection)?,
&grad_logits,
grad_input,
rows,
1.0,
)?;
gpu.accumulate_weight_grad(
device_of_mut(&mut moe.router.projection)?,
&grad_logits,
input,
rows,
)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cuda_training::cuda_doctor;
use crate::norm::RmsNorm;
fn cuda_or_skip() -> Option<Arc<GpuContext>> {
match cuda_doctor(0, 8192) {
Ok(_) => Some(GpuContext::new(0).expect("the CUDA doctor passed")),
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
None
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn ramp(len: usize) -> Vec<f32> {
ramp_from(len, 0)
}
fn ramp_from(len: usize, offset: usize) -> Vec<f32> {
(0..len)
.map(|i| (((i + offset) * 37) % 23) as f32 / 11.0 - 1.0)
.collect()
}
fn assert_close(label: &str, device: &[f32], host: &[f32], tolerance: f32) {
assert_eq!(device.len(), host.len(), "{label}: length");
for (index, (a, b)) in device.iter().zip(host).enumerate() {
assert!(
(a - b).abs() <= tolerance,
"{label}[{index}]: {a} on the device vs {b} on the host"
);
}
}
#[test]
fn the_rmsnorm_kernels_match_the_host_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, cols) = (5, 8);
let mut norm = RmsNorm::new(cols, 1e-6);
norm.weight.value.data = ramp(cols).iter().map(|v| 1.0 + v * 0.25).collect();
let input = Matrix::from_vec(rows, cols, ramp(rows * cols));
let grad_output = Matrix::from_vec(rows, cols, ramp_from(rows * cols, 3));
let weight = gpu.upload(&norm.weight.value.data).unwrap();
let device_input = gpu.upload(&input.data).unwrap();
let (output, inverse) = gpu
.rmsnorm(&device_input, &weight, rows, cols, norm.eps)
.unwrap();
assert_close(
"rmsnorm forward",
&gpu.download(&output).unwrap(),
&norm.forward(&input).data,
1e-5,
);
let device_grad = gpu.upload(&grad_output.data).unwrap();
let mut grad_weight = gpu.zeros(cols).unwrap();
let grad_input = gpu
.rmsnorm_backward(
&device_input,
&device_grad,
&weight,
&inverse,
&mut grad_weight.slice_mut(..),
rows,
cols,
)
.unwrap();
let host_grad_input = norm.backward(&input, &grad_output);
assert_close(
"rmsnorm grad_input",
&gpu.download(&grad_input).unwrap(),
&host_grad_input.data,
1e-5,
);
assert_close(
"rmsnorm grad_weight",
&gpu.download(&grad_weight).unwrap(),
&norm.weight.grad.data,
1e-5,
);
}
#[test]
fn the_rope_kernel_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (sequences, seq_len, heads, head_dim) = (2, 3, 2, 4);
let rows = sequences * seq_len;
let rope = Rope::new(head_dim, 16, 10_000.0).unwrap();
let mut host = Matrix::from_vec(rows, heads * head_dim, ramp(rows * heads * head_dim));
let mut device = gpu.upload(&host.data).unwrap();
let tables = gpu.rope_tables(&rope).unwrap();
gpu.rope(&mut device, &tables, rows, heads, head_dim, seq_len, 1.0)
.unwrap();
rope.apply_batched(&mut host, heads, seq_len).unwrap();
assert_close("rope", &gpu.download(&device).unwrap(), &host.data, 1e-5);
gpu.rope(&mut device, &tables, rows, heads, head_dim, seq_len, -1.0)
.unwrap();
rope.apply_inverse_batched(&mut host, heads, seq_len).unwrap();
assert_close(
"rope inverse",
&gpu.download(&device).unwrap(),
&host.data,
1e-5,
);
}
#[test]
fn the_causal_softmax_kernel_masks_and_normalizes_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (sequences, seq_len) = (2, 4);
let rows = sequences * seq_len;
let scores = ramp(rows * seq_len);
let mut device = gpu.upload(&scores).unwrap();
gpu.causal_softmax(&mut device, rows, seq_len).unwrap();
let probabilities = gpu.download(&device).unwrap();
for row in 0..rows {
let visible = row % seq_len + 1;
let slice = &probabilities[row * seq_len..(row + 1) * seq_len];
let mut expected = vec![0.0f32; seq_len];
let source = &scores[row * seq_len..row * seq_len + visible];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let total: f32 = source.iter().map(|v| (v - max).exp()).sum();
for (slot, &value) in expected.iter_mut().zip(source) {
*slot = (value - max).exp() / total;
}
assert_close(&format!("causal softmax row {row}"), slice, &expected, 1e-6);
assert!(slice[visible..].iter().all(|&p| p == 0.0));
}
let mut grad = gpu.upload(&vec![1.0; rows * seq_len]).unwrap();
gpu.causal_softmax_backward(&mut grad, &device, rows, seq_len)
.unwrap();
assert_close(
"causal softmax backward",
&gpu.download(&grad).unwrap(),
&vec![0.0; rows * seq_len],
1e-6,
);
}
#[test]
fn the_swiglu_kernel_matches_the_host_formula_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let len = 16;
let gate = ramp(len);
let up = ramp_from(len, 5);
let cache = SwiGluCache {
gate: gpu.upload(&gate).unwrap(),
up: gpu.upload(&up).unwrap(),
hidden: gpu.zeros(len).unwrap(),
};
let hidden = gpu.swiglu(&cache.gate, &cache.up, len).unwrap();
let silu = |x: f32| x / (1.0 + (-x).exp());
let expected: Vec<f32> = gate.iter().zip(&up).map(|(&g, &u)| silu(g) * u).collect();
assert_close("swiglu", &gpu.download(&hidden).unwrap(), &expected, 1e-6);
let grad_hidden = gpu.upload(&vec![1.0; len]).unwrap();
let (grad_gate, grad_up) = gpu.swiglu_backward(&cache, &grad_hidden, len).unwrap();
let epsilon = 1e-3;
let numeric: Vec<f32> = gate
.iter()
.zip(&up)
.map(|(&g, &u)| (silu(g + epsilon) * u - silu(g - epsilon) * u) / (2.0 * epsilon))
.collect();
assert_close(
"swiglu grad_gate",
&gpu.download(&grad_gate).unwrap(),
&numeric,
2e-3,
);
assert_close(
"swiglu grad_up",
&gpu.download(&grad_up).unwrap(),
&gate.iter().map(|&g| silu(g)).collect::<Vec<_>>(),
1e-6,
);
}
#[test]
fn the_router_kernels_select_and_renormalize_or_skip_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let gpu = Gpu { context: &context };
let (rows, experts, top_k) = (4, 5, 2);
let logits = ramp(rows * experts);
let device_logits = gpu.upload(&logits).unwrap();
let mut probabilities = gpu.zeros(rows * experts).unwrap();
let mut log_sum_exp = gpu.zeros(rows).unwrap();
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.softmax_lse)
.arg(&mut probabilities)
.arg(&mut log_sum_exp)
.arg(&device_logits)
.arg(&(rows as i32))
.arg(&(experts as i32))
.launch(cfg(rows))
.unwrap();
}
let flags: Vec<i32> = (0..rows).map(|row| i32::from(row + 1 < rows)).collect();
let valid = gpu.upload_flags(&flags).unwrap();
let mut expert_of = gpu.context.stream.alloc_zeros::<i32>(rows * top_k).unwrap();
let mut gate_of = gpu.zeros(rows * top_k).unwrap();
unsafe {
gpu.context
.stream
.launch_builder(&gpu.context.model.topk_gate)
.arg(&mut expert_of)
.arg(&mut gate_of)
.arg(&probabilities)
.arg(&valid)
.arg(&(rows as i32))
.arg(&(experts as i32))
.arg(&(top_k as i32))
.launch(cfg(rows))
.unwrap();
}
let probabilities = gpu.download(&probabilities).unwrap();
let log_sum_exp = gpu.download(&log_sum_exp).unwrap();
let assigned = gpu.download_signed(&expert_of).unwrap();
let gates = gpu.download(&gate_of).unwrap();
for row in 0..rows {
let slice = &probabilities[row * experts..(row + 1) * experts];
let sum: f32 = slice.iter().sum();
assert!((sum - 1.0).abs() < 1e-5, "row {row} softmax sums to {sum}");
let source = &logits[row * experts..(row + 1) * experts];
let max = source.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let expected = max + source.iter().map(|v| (v - max).exp()).sum::<f32>().ln();
assert!((log_sum_exp[row] - expected).abs() < 1e-5);
let selected = &assigned[row * top_k..(row + 1) * top_k];
if row + 1 == rows {
assert!(selected.iter().all(|&expert| expert < 0), "padding routed");
assert!(gates[row * top_k..(row + 1) * top_k].iter().all(|&g| g == 0.0));
continue;
}
let mut order: Vec<usize> = (0..experts).collect();
order.sort_by(|&a, &b| slice[b].total_cmp(&slice[a]));
assert_eq!(
selected.iter().map(|&e| e as usize).collect::<Vec<_>>(),
order[..top_k],
"row {row} top-{top_k}"
);
let total: f32 = selected.iter().map(|&e| slice[e as usize]).sum();
for (rank, &expert) in selected.iter().enumerate() {
let expected = slice[expert as usize] / total;
assert!((gates[row * top_k + rank] - expected).abs() < 1e-6);
}
}
}
}