use crate::activations::softmax;
use crate::cuda_training::{cfg, cuda_err, device_context};
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::optimizers::Optimizer;
use cudarc::cublas::{
CudaBlas, Gemm, GemmConfig, StridedBatchedConfig, result as cublas, sys as cublas_sys,
sys::cublasOperation_t,
};
use cudarc::driver::{
CudaFunction, CudaSlice, CudaStream, CudaView, DevicePtr, DevicePtrMut, PushKernelArg,
};
use std::any::TypeId;
use std::sync::{Arc, Mutex};
pub struct GpuContext {
pub device: usize,
pub(crate) stream: Arc<CudaStream>,
pub(crate) blas: CudaBlas,
pub(crate) mixed_precision: bool,
adam: CudaFunction,
sgd: CudaFunction,
scale: CudaFunction,
gather: CudaFunction,
scatter: CudaFunction,
pub(crate) model: crate::gpu_model::ModelKernels,
pub(crate) flash: Option<crate::cuda_flash::FlashKernels>,
pub(crate) rope: Mutex<Option<Arc<crate::gpu_model::DeviceRope>>>,
poison: Mutex<Option<String>>,
}
impl std::fmt::Debug for GpuContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GpuContext")
.field("device", &self.device)
.finish()
}
}
impl GpuContext {
pub fn new(device: usize) -> Result<Arc<Self>, NetworkError> {
Self::with_precision(device, false)
}
pub fn with_precision(device: usize, mixed_precision: bool) -> Result<Arc<Self>, NetworkError> {
let (context, module) = device_context(device)?;
let stream = context.new_stream().map_err(cuda_err("stream creation"))?;
let blas = CudaBlas::new(stream.clone()).map_err(cuda_err("cuBLAS initialization"))?;
if mixed_precision {
let status = unsafe {
cudarc::cublas::sys::cublasSetMathMode(
*blas.handle(),
cudarc::cublas::sys::cublasMath_t::CUBLAS_TF32_TENSOR_OP_MATH,
)
};
if status != cudarc::cublas::sys::cublasStatus_t::CUBLAS_STATUS_SUCCESS {
return Err(NetworkError::Cuda(format!(
"cuBLAS rejected the TF32 math mode: {status:?}"
)));
}
}
let get = |name: &str| {
module
.load_function(name)
.map_err(cuda_err("CUDA kernel lookup"))
};
Ok(Arc::new(Self {
device,
stream,
blas,
mixed_precision,
adam: get("adam")?,
sgd: get("sgd")?,
scale: get("scale_inplace")?,
gather: get("gather_rows")?,
scatter: get("scatter_rows_neg")?,
model: crate::gpu_model::ModelKernels::load(&module)?,
flash: crate::cuda_flash::flash_kernels(device, &context),
rope: Mutex::new(None),
poison: Mutex::new(None),
}))
}
pub(crate) fn poison(&self, error: NetworkError) {
if let Ok(mut slot) = self.poison.lock() {
slot.get_or_insert_with(|| error.to_string());
}
}
pub(crate) fn guard<T>(&self, result: Result<T, NetworkError>, fallback: T) -> T {
match result {
Ok(value) => value,
Err(error) => {
self.poison(error);
fallback
}
}
}
pub fn synchronize(&self) -> Result<(), NetworkError> {
self.stream
.synchronize()
.map_err(cuda_err("stream synchronize"))
}
pub fn check(&self) -> Result<(), NetworkError> {
let taken = self.poison.lock().ok().and_then(|mut slot| slot.take());
match taken {
None => Ok(()),
Some(message) => Err(NetworkError::Cuda(message)),
}
}
pub(crate) fn upload(&self, matrix: &Matrix) -> Result<CudaSlice<f32>, NetworkError> {
self.stream
.clone_htod(&matrix.data)
.map_err(cuda_err("host to device copy"))
}
pub(crate) fn download(
&self,
source: &CudaSlice<f32>,
target: &mut Matrix,
) -> Result<(), NetworkError> {
self.stream
.memcpy_dtoh(source, &mut target.data)
.map_err(cuda_err("device to host copy"))
}
pub(crate) fn zeros(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
self.stream
.alloc_zeros::<f32>(len)
.map_err(cuda_err("device allocation"))
}
#[allow(clippy::too_many_arguments)]
fn attention(
&self,
queries: &Matrix,
keys: &Matrix,
values: &Matrix,
num_heads: usize,
num_kv_heads: usize,
head_dim: usize,
position_offset: usize,
keep_probabilities: bool,
) -> Result<(Vec<Matrix>, Matrix), NetworkError> {
let tokens = queries.rows;
let history = keys.rows;
let group_size = num_heads / num_kv_heads;
let scale = (head_dim as f32).sqrt().recip();
let device_queries = self.upload(queries)?;
let device_keys = self.upload(keys)?;
let device_values = self.upload(values)?;
let mut device_scores = self.zeros(tokens * history)?;
let mut device_merged = self.zeros(tokens * num_heads * head_dim)?;
let mut probabilities = Vec::with_capacity(if keep_probabilities { num_heads } else { 0 });
let mut scores = Matrix::new(tokens, history);
for head in 0..num_heads {
let query_base = head * head_dim;
let kv_base = (head / group_size) * head_dim;
gemm_rhs_transposed(
self,
&device_queries.slice(query_base..),
queries.cols,
&device_keys.slice(kv_base..),
keys.cols,
&mut device_scores.slice_mut(0..),
history,
tokens,
history,
head_dim,
scale,
0.0,
)?;
self.download(&device_scores, &mut scores)?;
for query in 0..tokens {
let visible = position_offset + query + 1;
let row = scores.row_mut(query);
softmax(&mut row[..visible]);
row[visible..].fill(0.0);
}
self.stream
.memcpy_htod(&scores.data, &mut device_scores)
.map_err(cuda_err("host to device copy"))?;
gemm_plain(
self,
&device_scores.slice(0..),
history,
&device_values.slice(kv_base..),
values.cols,
&mut device_merged.slice_mut(query_base..),
num_heads * head_dim,
tokens,
head_dim,
history,
1.0,
0.0,
)?;
if keep_probabilities {
probabilities.push(scores.clone());
}
}
let mut merged = Matrix::new(tokens, num_heads * head_dim);
self.download(&device_merged, &mut merged)?;
Ok((probabilities, merged))
}
}
pub struct DeviceParam {
context: Arc<GpuContext>,
rows: usize,
cols: usize,
value: CudaSlice<f32>,
negated_grad: CudaSlice<f32>,
grad_dirty: bool,
moment1: CudaSlice<f32>,
moment2: CudaSlice<f32>,
}
impl std::fmt::Debug for DeviceParam {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeviceParam")
.field("device", &self.context.device)
.field("rows", &self.rows)
.field("cols", &self.cols)
.finish()
}
}
impl Clone for DeviceParam {
fn clone(&self) -> Self {
let copy = |source: &CudaSlice<f32>| {
self.context
.stream
.clone_dtod(source)
.map_err(cuda_err("device to device copy"))
};
let fallback = |len: usize| self.context.stream.alloc_zeros::<f32>(len).ok();
let mut slices = Vec::with_capacity(4);
for source in [
&self.value,
&self.negated_grad,
&self.moment1,
&self.moment2,
] {
match copy(source) {
Ok(slice) => slices.push(Some(slice)),
Err(error) => {
self.context.poison(error);
slices.push(fallback(self.rows * self.cols));
}
}
}
let mut take = |index: usize| {
slices[index]
.take()
.unwrap_or_else(|| panic!("CUDA allocation failed while cloning a parameter"))
};
Self {
context: self.context.clone(),
rows: self.rows,
cols: self.cols,
value: take(0),
negated_grad: take(1),
grad_dirty: self.grad_dirty,
moment1: take(2),
moment2: take(3),
}
}
}
impl DeviceParam {
pub(crate) fn new(context: Arc<GpuContext>, value: &Matrix) -> Result<Self, NetworkError> {
let len = value.data.len();
Ok(Self {
rows: value.rows,
cols: value.cols,
value: context.upload(value)?,
negated_grad: context.zeros(len)?,
grad_dirty: false,
moment1: context.zeros(len)?,
moment2: context.zeros(len)?,
context,
})
}
pub(crate) fn context(&self) -> &Arc<GpuContext> {
&self.context
}
pub(crate) fn rows(&self) -> usize {
self.rows
}
pub(crate) fn cols(&self) -> usize {
self.cols
}
pub(crate) fn value(&self) -> &CudaSlice<f32> {
&self.value
}
pub(crate) fn negated_grad_mut(&mut self) -> &mut CudaSlice<f32> {
&mut self.negated_grad
}
pub(crate) fn grad_beta(&mut self) -> f32 {
f32::from(u8::from(std::mem::replace(&mut self.grad_dirty, true)))
}
pub(crate) fn clear_grad(&mut self) -> Result<(), NetworkError> {
if std::mem::replace(&mut self.grad_dirty, true) {
return Ok(());
}
self.context
.stream
.memset_zeros(&mut self.negated_grad)
.map_err(cuda_err("gradient reset"))
}
pub(crate) fn download_value(&self, target: &mut Matrix) -> Result<(), NetworkError> {
self.context.download(&self.value, target)
}
pub(crate) fn download_moments(
&self,
first: &mut Matrix,
second: &mut Matrix,
) -> Result<(), NetworkError> {
self.context.download(&self.moment1, first)?;
self.context.download(&self.moment2, second)
}
pub(crate) fn upload_moments(
&mut self,
first: &Matrix,
second: &Matrix,
) -> Result<(), NetworkError> {
self.moment1 = self.context.upload(first)?;
self.moment2 = self.context.upload(second)?;
Ok(())
}
pub(crate) fn download_grad(&self, target: &mut Matrix) -> Result<(), NetworkError> {
if !self.grad_dirty {
target.zeros();
return Ok(());
}
self.context.download(&self.negated_grad, target)?;
for slot in &mut target.data {
*slot = -*slot;
}
Ok(())
}
pub(crate) fn zero_grad(&mut self) {
self.grad_dirty = false;
}
pub(crate) fn matmul_rhs_transposed(&self, input: &Matrix) -> Matrix {
let mut output = Matrix::new(input.rows, self.rows);
let result = (|| {
let device_input = self.context.upload(input)?;
let mut device_output = self.context.zeros(output.data.len())?;
gemm_rhs_transposed(
&self.context,
&device_input,
self.cols,
&self.value,
self.cols,
&mut device_output,
self.rows,
input.rows,
self.rows,
self.cols,
1.0,
0.0,
)?;
self.context.download(&device_output, &mut output)
})();
self.context.guard(result, ());
output
}
pub(crate) fn matmul(&self, input: &Matrix) -> Matrix {
let mut output = Matrix::new(input.rows, self.cols);
let result = (|| {
let device_input = self.context.upload(input)?;
let mut device_output = self.context.zeros(output.data.len())?;
gemm_plain(
&self.context,
&device_input,
self.rows,
&self.value,
self.cols,
&mut device_output,
self.cols,
input.rows,
self.cols,
self.rows,
1.0,
0.0,
)?;
self.context.download(&device_output, &mut output)
})();
self.context.guard(result, ());
output
}
pub(crate) fn accumulate_grad(&mut self, grad_output: &Matrix, input: &Matrix) {
let result = (|| {
let device_grad_output = self.context.upload(grad_output)?;
let device_input = self.context.upload(input)?;
let beta = self.grad_beta();
gemm_lhs_transposed(
&self.context,
&device_grad_output,
self.rows,
&device_input,
self.cols,
&mut self.negated_grad,
self.cols,
grad_output.rows,
self.rows,
self.cols,
-1.0,
beta,
)
})();
self.context.guard(result, ());
}
pub(crate) fn gather(&self, ids: &[u32]) -> Matrix {
let mut output = Matrix::new(ids.len(), self.cols);
let result = (|| {
let device_ids = self
.context
.stream
.clone_htod(&ids.to_vec())
.map_err(cuda_err("host to device copy"))?;
let mut device_output = self.context.zeros(output.data.len())?;
let elements = ids.len() * self.cols;
unsafe {
self.context
.stream
.launch_builder(&self.context.gather)
.arg(&mut device_output)
.arg(&self.value)
.arg(&device_ids)
.arg(&0i32)
.arg(&(ids.len() as i32))
.arg(&(self.cols as i32))
.launch(cfg(elements))
.map_err(cuda_err("embedding gather kernel"))?;
}
self.context.download(&device_output, &mut output)
})();
self.context.guard(result, ());
output
}
pub(crate) fn scatter_grad(&mut self, ids: &[u32], grad_output: &Matrix) {
let result = (|| -> Result<(), NetworkError> {
let device_ids = self
.context
.stream
.clone_htod(&ids.to_vec())
.map_err(cuda_err("host to device copy"))?;
let device_grad = self.context.upload(grad_output)?;
self.clear_grad()?;
let elements = ids.len() * self.cols;
unsafe {
self.context
.stream
.launch_builder(&self.context.scatter)
.arg(&mut self.negated_grad)
.arg(&device_grad)
.arg(&device_ids)
.arg(&(ids.len() as i32))
.arg(&(self.cols as i32))
.launch(cfg(elements))
.map_err(cuda_err("embedding scatter kernel"))?;
}
Ok(())
})();
self.context.guard(result, ());
}
pub(crate) fn step(&mut self, optimizer: &Optimizer, step: usize, scale: f32) {
let elements = self.rows * self.cols;
let result = (|| -> Result<(), NetworkError> {
self.clear_grad()?;
if scale != 1.0 {
unsafe {
self.context
.stream
.launch_builder(&self.context.scale)
.arg(&mut self.negated_grad)
.arg(&(elements as i32))
.arg(&scale)
.launch(cfg(elements))
.map_err(cuda_err("gradient scaling kernel"))?;
}
}
match *optimizer {
Optimizer::Sgd { learning_rate } => unsafe {
self.context
.stream
.launch_builder(&self.context.sgd)
.arg(&mut self.value)
.arg(&self.negated_grad)
.arg(&(elements as i32))
.arg(&learning_rate)
.launch(cfg(elements))
.map_err(cuda_err("SGD update kernel"))?;
},
Optimizer::Adam {
learning_rate,
beta1,
beta2,
epsilon,
weight_decay,
} => {
let correction1 = 1.0 - beta1.powi(step as i32);
let correction2 = 1.0 - beta2.powi(step as i32);
unsafe {
self.context
.stream
.launch_builder(&self.context.adam)
.arg(&mut self.value)
.arg(&self.negated_grad)
.arg(&mut self.moment1)
.arg(&mut self.moment2)
.arg(&(elements as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&epsilon)
.arg(&correction1)
.arg(&correction2)
.arg(&weight_decay)
.launch(cfg(elements))
.map_err(cuda_err("Adam update kernel"))?;
}
}
}
self.grad_dirty = false;
Ok(())
})();
self.context.guard(result, ());
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn attention_heads(
context: &Arc<GpuContext>,
queries: &Matrix,
keys: &Matrix,
values: &Matrix,
num_heads: usize,
num_kv_heads: usize,
head_dim: usize,
position_offset: usize,
keep_probabilities: bool,
) -> (Vec<Matrix>, Matrix) {
let fallback = (Vec::new(), Matrix::new(queries.rows, num_heads * head_dim));
let result = context.attention(
queries,
keys,
values,
num_heads,
num_kv_heads,
head_dim,
position_offset,
keep_probabilities,
);
context.guard(result, fallback)
}
unsafe fn gemm_dispatch<T: 'static, A: DevicePtr<T>, B: DevicePtr<T>, C: DevicePtrMut<T>>(
context: &GpuContext,
config: GemmConfig<T>,
a: &A,
b: &B,
c: &mut C,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
if !context.mixed_precision || TypeId::of::<T>() != TypeId::of::<f32>() {
return unsafe { context.blas.gemm(config, a, b, c) }.map_err(cuda_err("cuBLAS GEMM"));
}
let (a_pointer, _a_guard) = a.device_ptr(&context.stream);
let (b_pointer, _b_guard) = b.device_ptr(&context.stream);
let (c_pointer, _c_guard) = c.device_ptr_mut(&context.stream);
unsafe {
cublas::gemm_ex(
*context.blas.handle(),
config.transa,
config.transb,
config.m,
config.n,
config.k,
&config.alpha as *const T as *const std::ffi::c_void,
a_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
config.lda,
b_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
config.ldb,
&config.beta as *const T as *const std::ffi::c_void,
c_pointer as *mut std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
config.ldc,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP,
)
}
.map_err(cuda_err("cuBLAS BF16 GEMM"))
}
unsafe fn act_dispatch<T, C: DevicePtrMut<T>>(
context: &GpuContext,
config: GemmConfig<f32>,
a: &CudaView<'_, u8>,
b: &CudaView<'_, u8>,
narrow: bool,
out_narrow: bool,
c: &mut C,
) -> Result<(), NetworkError> {
let (operand, compute) = if narrow {
(
cublas_sys::cudaDataType_t::CUDA_R_16BF,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F,
)
} else if context.mixed_precision {
(
cublas_sys::cudaDataType_t::CUDA_R_32F,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
)
} else {
(
cublas_sys::cudaDataType_t::CUDA_R_32F,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F,
)
};
let (a_pointer, _a_guard) = a.device_ptr(&context.stream);
let (b_pointer, _b_guard) = b.device_ptr(&context.stream);
let (c_pointer, _c_guard) = c.device_ptr_mut(&context.stream);
unsafe {
cublas::gemm_ex(
*context.blas.handle(),
config.transa,
config.transb,
config.m,
config.n,
config.k,
&config.alpha as *const f32 as *const std::ffi::c_void,
a_pointer as *const std::ffi::c_void,
operand,
config.lda,
b_pointer as *const std::ffi::c_void,
operand,
config.ldb,
&config.beta as *const f32 as *const std::ffi::c_void,
c_pointer as *mut std::ffi::c_void,
if out_narrow {
cublas_sys::cudaDataType_t::CUDA_R_16BF
} else {
cublas_sys::cudaDataType_t::CUDA_R_32F
},
config.ldc,
compute,
cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP,
)
}
.map_err(cuda_err("cuBLAS narrow GEMM"))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn act_rhs_transposed<T, O: DevicePtrMut<T>>(
context: &GpuContext,
x: &CudaView<'_, u8>,
x_stride: usize,
w: &CudaView<'_, u8>,
w_stride: usize,
narrow: bool,
out_narrow: bool,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
inner: usize,
alpha: f32,
beta: f32,
) -> Result<(), NetworkError> {
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { act_dispatch(context, config, w, x, narrow, out_narrow, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn act_plain<T, O: DevicePtrMut<T>>(
context: &GpuContext,
x: &CudaView<'_, u8>,
x_stride: usize,
w: &CudaView<'_, u8>,
w_stride: usize,
narrow: bool,
out_narrow: bool,
out: &mut O,
out_stride: usize,
rows: usize,
cols: usize,
inner: usize,
alpha: f32,
beta: f32,
) -> Result<(), NetworkError> {
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: cols as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { act_dispatch(context, config, w, x, narrow, out_narrow, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn act_lhs_transposed<O: DevicePtrMut<f32>>(
context: &GpuContext,
d: &CudaView<'_, u8>,
d_stride: usize,
x: &CudaView<'_, u8>,
x_stride: usize,
narrow: bool,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
cols: usize,
alpha: f32,
beta: f32,
) -> Result<(), NetworkError> {
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: cols as i32,
n: units as i32,
k: rows as i32,
alpha,
lda: x_stride as i32,
ldb: d_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { act_dispatch(context, config, x, d, narrow, false, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_rhs_transposed<
T: 'static,
X: DevicePtr<T>,
W: DevicePtr<T>,
O: DevicePtrMut<T>,
>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
inner: usize,
alpha: T,
beta: T,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { gemm_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
unsafe fn gemm_strided_batched_dispatch<
A: DevicePtr<f32>,
B: DevicePtr<f32>,
C: DevicePtrMut<f32>,
>(
context: &GpuContext,
config: StridedBatchedConfig<f32>,
a: &A,
b: &B,
c: &mut C,
) -> Result<(), NetworkError> {
if !context.mixed_precision {
return unsafe { context.blas.gemm_strided_batched(config, a, b, c) }
.map_err(cuda_err("cuBLAS batched GEMM"));
}
let (a_pointer, _a_guard) = a.device_ptr(&context.stream);
let (b_pointer, _b_guard) = b.device_ptr(&context.stream);
let (c_pointer, _c_guard) = c.device_ptr_mut(&context.stream);
let gemm = config.gemm;
unsafe {
cublas::gemm_strided_batched_ex(
*context.blas.handle(),
gemm.transa,
gemm.transb,
gemm.m,
gemm.n,
gemm.k,
&gemm.alpha as *const f32 as *const std::ffi::c_void,
a_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
gemm.lda,
config.stride_a,
b_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
gemm.ldb,
config.stride_b,
&gemm.beta as *const f32 as *const std::ffi::c_void,
c_pointer as *mut std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
gemm.ldc,
config.stride_c,
config.batch_size,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP,
)
}
.map_err(cuda_err("cuBLAS batched BF16 GEMM"))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_rhs_transposed_batched<
X: DevicePtr<f32>,
W: DevicePtr<f32>,
O: DevicePtrMut<f32>,
>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
inner: usize,
alpha: f32,
beta: f32,
count: usize,
x_batch_stride: usize,
w_batch_stride: usize,
out_batch_stride: usize,
) -> Result<(), NetworkError> {
let config = StridedBatchedConfig {
gemm: GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
},
batch_size: count as i32,
stride_a: w_batch_stride as i64,
stride_b: x_batch_stride as i64,
stride_c: out_batch_stride as i64,
};
unsafe { gemm_strided_batched_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_plain_batched<X: DevicePtr<f32>, W: DevicePtr<f32>, O: DevicePtrMut<f32>>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
cols: usize,
inner: usize,
alpha: f32,
beta: f32,
count: usize,
x_batch_stride: usize,
w_batch_stride: usize,
out_batch_stride: usize,
) -> Result<(), NetworkError> {
let config = StridedBatchedConfig {
gemm: GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: cols as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
},
batch_size: count as i32,
stride_a: w_batch_stride as i64,
stride_b: x_batch_stride as i64,
stride_c: out_batch_stride as i64,
};
unsafe { gemm_strided_batched_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_lhs_transposed_batched<
D: DevicePtr<f32>,
X: DevicePtr<f32>,
O: DevicePtrMut<f32>,
>(
context: &GpuContext,
d: &D,
d_stride: usize,
x: &X,
x_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
cols: usize,
alpha: f32,
beta: f32,
count: usize,
d_batch_stride: usize,
x_batch_stride: usize,
out_batch_stride: usize,
) -> Result<(), NetworkError> {
let config = StridedBatchedConfig {
gemm: GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: cols as i32,
n: units as i32,
k: rows as i32,
alpha,
lda: x_stride as i32,
ldb: d_stride as i32,
beta,
ldc: out_stride as i32,
},
batch_size: count as i32,
stride_a: x_batch_stride as i64,
stride_b: d_batch_stride as i64,
stride_c: out_batch_stride as i64,
};
unsafe { gemm_strided_batched_dispatch(context, config, x, d, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_plain<T: 'static, X: DevicePtr<T>, W: DevicePtr<T>, O: DevicePtrMut<T>>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
cols: usize,
inner: usize,
alpha: T,
beta: T,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: cols as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { gemm_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_lhs_transposed<
T: 'static,
D: DevicePtr<T>,
X: DevicePtr<T>,
O: DevicePtrMut<T>,
>(
context: &GpuContext,
d: &D,
d_stride: usize,
x: &X,
x_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
cols: usize,
alpha: T,
beta: T,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: cols as i32,
n: units as i32,
k: rows as i32,
alpha,
lda: x_stride as i32,
ldb: d_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { gemm_dispatch(context, config, x, d, out) }
}
#[cfg(test)]
mod tests {
use crate::cuda_training::cuda_doctor;
use crate::network::NetworkError;
use crate::optimizers::Optimizer;
use crate::transformer::{TransformerBuilder, TransformerLm};
fn cuda_or_skip() -> bool {
match cuda_doctor(0, 8192) {
Ok(_) => true,
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
false
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn tiny() -> TransformerBuilder {
TransformerLm::builder()
.vocab_size(24)
.d_model(16)
.n_layers(2)
.heads(4, 2, 4)
.d_ff(32)
.moe_d_ff(12)
.experts(4, 2)
.moe_layers([1])
.shared_expert(true)
.max_seq_len(32)
.optimizer(Optimizer::adam(1e-2))
.seed(1234)
.mixed_precision(false)
}
fn assert_close(label: &str, gpu: &[f32], cpu: &[f32], tolerance: f32) {
assert_eq!(gpu.len(), cpu.len(), "{label}: length");
for (index, (a, b)) in gpu.iter().zip(cpu).enumerate() {
assert!(
(a - b).abs() <= tolerance,
"{label}[{index}]: {a} on the device vs {b} on the host"
);
}
}
#[test]
fn gpu_forward_matches_cpu_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5, 2, 7];
let (host_logits, _) = cpu.forward_train(&[&ids[..]]).unwrap();
let (device_logits, _) = gpu.forward_train(&[&ids[..]]).unwrap();
assert_close("logits", &device_logits.data, &host_logits.data, 1e-3);
}
#[test]
fn gpu_backward_matches_cpu_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5, 2, 7];
let host = cpu.train_step(&[&ids[..]]).unwrap();
let device = gpu.train_step(&[&ids[..]]).unwrap();
gpu.sync_from_device().unwrap();
assert!((host.total() - device.total()).abs() < 1e-3);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn a_batched_gpu_forward_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5, 2, 7], vec![9, 4, 4, 0, 11, 6]];
let (host_logits, _) = cpu.forward_train(&batch).unwrap();
let (device_logits, _) = gpu.forward_train(&batch).unwrap();
assert_eq!(device_logits.rows, 12);
assert_close(
"batched logits",
&device_logits.data,
&host_logits.data,
1e-3,
);
}
#[test]
fn a_padded_gpu_batch_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5, 2, 7], vec![9, 4, 4], vec![2, 6]];
let (host_logits, host_cache) = cpu.forward_train(&batch).unwrap();
let (device_logits, device_cache) = gpu.forward_train(&batch).unwrap();
assert_close(
"padded logits",
&device_logits.data,
&host_logits.data,
1e-3,
);
assert!(
(device_cache.auxiliary_loss() - host_cache.auxiliary_loss()).abs() < 1e-4,
"auxiliary loss {} on the device vs {} on the host",
device_cache.auxiliary_loss(),
host_cache.auxiliary_loss()
);
}
#[test]
fn a_batched_gpu_train_step_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [
vec![3u32, 8, 1, 5, 2, 7],
vec![9, 4, 4, 0, 11, 6],
vec![2, 6, 1],
];
let host = cpu.train_step(&batch).unwrap();
let device = gpu.train_step(&batch).unwrap();
gpu.sync_from_device().unwrap();
assert!((host.lm_loss - device.lm_loss).abs() < 1e-3);
assert!((host.auxiliary_loss - device.auxiliary_loss).abs() < 1e-4);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn a_chunked_gpu_head_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [
vec![3u32, 8, 1, 5, 2, 7],
vec![9, 4, 4, 0, 11, 6],
vec![2, 6, 1],
];
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(5, std::sync::atomic::Ordering::Relaxed);
let host = cpu.train_step(&batch).unwrap();
let device = gpu.train_step(&batch).unwrap();
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(0, std::sync::atomic::Ordering::Relaxed);
gpu.sync_from_device().unwrap();
assert!(
(host.lm_loss - device.lm_loss).abs() < 1e-3,
"chunked loss {} on the device vs {} on the host",
device.lm_loss,
host.lm_loss
);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn mixed_precision_tracks_the_fp32_device_path_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut baseline = tiny().build().unwrap();
let mut reduced = tiny().mixed_precision(true).build().unwrap();
assert_eq!(
baseline, reduced,
"the two models start from the same weights"
);
baseline.to_cuda(0, 8192).unwrap();
reduced.to_cuda(0, 8192).unwrap();
let batch = [
vec![3u32, 8, 1, 5, 2, 7],
vec![9, 4, 4, 0, 11, 6],
vec![2, 6, 1],
];
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(5, std::sync::atomic::Ordering::Relaxed);
for step in 0..2 {
let exact = baseline.train_step(&batch).unwrap();
let approximate = reduced.train_step(&batch).unwrap();
assert!(
(exact.lm_loss - approximate.lm_loss).abs() < 1e-2,
"step {step}: reduced-precision loss {} vs FP32 {}",
approximate.lm_loss,
exact.lm_loss
);
}
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(0, std::sync::atomic::Ordering::Relaxed);
}
#[test]
fn repeated_gpu_batched_steps_stay_in_step_with_the_host_or_skip_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5], vec![9, 4, 4, 0]];
for step in 0..4 {
let host = cpu.train_step(&batch).unwrap();
let device = gpu.train_step(&batch).unwrap();
assert!(
(host.total() - device.total()).abs() < 2e-3,
"step {step}: {} on the device vs {} on the host",
device.total(),
host.total()
);
}
gpu.sync_from_device().unwrap();
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
2e-3,
);
}
}
#[test]
fn a_cached_decode_on_the_device_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5];
let mut host_caches = cpu.new_kv_caches();
let mut device_caches = gpu.new_kv_caches();
let host = cpu.forward_cached(&ids, &mut host_caches).unwrap();
let device = gpu.forward_cached(&ids, &mut device_caches).unwrap();
assert_close("prefill logits", &device.data, &host.data, 1e-3);
}
#[test]
fn to_cuda_fails_closed_without_a_device() {
if cuda_or_skip() {
return;
}
let mut model = tiny().build().unwrap();
assert!(matches!(model.to_cuda(0, 8192), Err(NetworkError::Cuda(_))));
assert!(!model.on_device());
}
#[test]
fn an_impossible_memory_budget_is_refused_before_any_upload() {
let mut model = tiny()
.d_model(64)
.heads(4, 2, 16)
.d_ff(256)
.moe_d_ff(64)
.build()
.unwrap();
assert!(matches!(
model.to_cuda(0, 1),
Err(NetworkError::CudaMemoryBudget { .. })
));
assert!(!model.on_device());
}
#[test]
fn fused_attention_matches_the_three_kernel_path_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let model = || {
TransformerLm::builder()
.vocab_size(32)
.d_model(128)
.n_layers(2)
.heads(2, 1, 64)
.d_ff(64)
.moe_layers([0usize; 0])
.max_seq_len(128)
.optimizer(Optimizer::adam(1e-2))
.seed(99)
.mixed_precision(true)
.build()
.unwrap()
};
let ids: Vec<u32> = (0..100).map(|index| (index * 7 % 32) as u32).collect();
let (host_logits, _) = model().forward_train(&[&ids[..]]).unwrap();
let mut fused = model();
fused.to_cuda(0, 8192).unwrap();
let (fused_logits, _) = fused.forward_train(&[&ids[..]]).unwrap();
crate::cuda_flash::DISABLED.store(true, std::sync::atomic::Ordering::Relaxed);
let mut split = model();
split.to_cuda(0, 8192).unwrap();
let split_forward = split.forward_train(&[&ids[..]]);
crate::cuda_flash::DISABLED.store(false, std::sync::atomic::Ordering::Relaxed);
let (split_logits, _) = split_forward.unwrap();
let error = |logits: &crate::matrix::Matrix| {
logits
.data
.iter()
.zip(&host_logits.data)
.map(|(device, host)| (device - host).abs())
.fold(0.0f32, f32::max)
};
let (fused_error, split_error) = (error(&fused_logits), error(&split_logits));
assert!(
fused_error <= 2.0 * split_error,
"the fused path is {fused_error} from the host where the three-kernel path is {split_error}"
);
assert!(
fused_error < 1e-2,
"both device paths drifted: {fused_error}"
);
}
#[test]
fn fused_attention_backward_matches_the_three_kernel_path_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let model = || {
TransformerLm::builder()
.vocab_size(32)
.d_model(256)
.n_layers(2)
.heads(4, 2, 64)
.d_ff(64)
.moe_layers([0usize; 0])
.max_seq_len(128)
.optimizer(Optimizer::sgd(1.0))
.seed(99)
.mixed_precision(true)
.build()
.unwrap()
};
let ids: Vec<u32> = (0..100).map(|index| (index * 7 % 32) as u32).collect();
let mut host = model();
host.train_step(&[&ids[..]]).unwrap();
let (host_logits, _) = host.forward_train(&[&ids[..]]).unwrap();
let mut fused = model();
fused.to_cuda(0, 8192).unwrap();
fused.train_step(&[&ids[..]]).unwrap();
let (fused_logits, _) = fused.forward_train(&[&ids[..]]).unwrap();
crate::cuda_flash::DISABLED.store(true, std::sync::atomic::Ordering::Relaxed);
let mut split = model();
split.to_cuda(0, 8192).unwrap();
let stepped = split
.train_step(&[&ids[..]])
.and_then(|_| split.forward_train(&[&ids[..]]));
crate::cuda_flash::DISABLED.store(false, std::sync::atomic::Ordering::Relaxed);
let (split_logits, _) = stepped.unwrap();
let error = |logits: &crate::matrix::Matrix| {
logits
.data
.iter()
.zip(&host_logits.data)
.map(|(device, host)| (device - host).abs())
.fold(0.0f32, f32::max)
};
let (fused_error, split_error) = (error(&fused_logits), error(&split_logits));
assert!(
fused_error <= 2.0 * split_error,
"the fused path is {fused_error} from the host where the three-kernel path is {split_error}"
);
assert!(
fused_error < 0.1,
"both device paths drifted: {fused_error}"
);
}
}