use crate::activations::softmax;
use crate::cuda_training::{cfg, cuda_alloc_err, cuda_err, device_context};
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::optimizers::Optimizer;
use cudarc::cublas::{
CudaBlas, Gemm, GemmConfig, StridedBatchedConfig, result as cublas, sys as cublas_sys,
sys::cublasOperation_t,
};
use cudarc::driver::{
CudaFunction, CudaSlice, CudaStream, CudaView, DevicePtr, DevicePtrMut, PushKernelArg,
};
use std::any::TypeId;
use std::sync::{Arc, Mutex};
pub struct GpuContext {
pub device: usize,
pub(crate) stream: Arc<CudaStream>,
pub(crate) blas: CudaBlas,
pub(crate) mixed_precision: bool,
pub(crate) half_accumulate: bool,
adam: CudaFunction,
lion: CudaFunction,
sgd: CudaFunction,
scale: CudaFunction,
gather: CudaFunction,
scatter: CudaFunction,
im2col_train: CudaFunction,
col2im_train: CudaFunction,
pub(crate) model: crate::gpu_model::ModelKernels,
pub(crate) flash: Option<crate::cuda_flash::FlashKernels>,
pub(crate) rope: Mutex<Option<Arc<crate::gpu_model::DeviceRope>>>,
poison: Mutex<Option<String>>,
}
impl std::fmt::Debug for GpuContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GpuContext")
.field("device", &self.device)
.finish()
}
}
impl GpuContext {
pub fn new(device: usize) -> Result<Arc<Self>, NetworkError> {
Self::with_precision(device, false)
}
pub(crate) fn half_accumulate(device: usize) -> Result<Arc<Self>, NetworkError> {
let mut context = Self::with_precision(device, false)?;
Arc::get_mut(&mut context)
.expect("a context this function just built is unshared")
.half_accumulate = true;
Ok(context)
}
pub fn with_precision(device: usize, mixed_precision: bool) -> Result<Arc<Self>, NetworkError> {
let (context, module) = device_context(device)?;
let stream = context.new_stream().map_err(cuda_err("stream creation"))?;
let blas = CudaBlas::new(stream.clone()).map_err(cuda_err("cuBLAS initialization"))?;
if mixed_precision {
let status = unsafe {
cudarc::cublas::sys::cublasSetMathMode(
*blas.handle(),
cudarc::cublas::sys::cublasMath_t::CUBLAS_TF32_TENSOR_OP_MATH,
)
};
if status != cudarc::cublas::sys::cublasStatus_t::CUBLAS_STATUS_SUCCESS {
return Err(NetworkError::Cuda(format!(
"cuBLAS rejected the TF32 math mode: {status:?}"
)));
}
}
let get = |name: &str| {
module
.load_function(name)
.map_err(cuda_err("CUDA kernel lookup"))
};
Ok(Arc::new(Self {
device,
stream,
blas,
mixed_precision,
half_accumulate: false,
adam: get("adam")?,
lion: get("lion")?,
sgd: get("sgd")?,
scale: get("scale_inplace")?,
gather: get("gather_rows")?,
scatter: get("scatter_rows_neg")?,
im2col_train: get("im2col_train")?,
col2im_train: get("col2im_train")?,
model: crate::gpu_model::ModelKernels::load(&module)?,
flash: crate::cuda_flash::flash_kernels(device, &context),
rope: Mutex::new(None),
poison: Mutex::new(None),
}))
}
pub(crate) fn poison(&self, error: NetworkError) {
if let Ok(mut slot) = self.poison.lock() {
slot.get_or_insert_with(|| error.to_string());
}
}
pub(crate) fn guard<T>(&self, result: Result<T, NetworkError>, fallback: T) -> T {
match result {
Ok(value) => value,
Err(error) => {
self.poison(error);
fallback
}
}
}
pub fn synchronize(&self) -> Result<(), NetworkError> {
self.stream
.synchronize()
.map_err(cuda_err("stream synchronize"))
}
pub fn check(&self) -> Result<(), NetworkError> {
let taken = self.poison.lock().ok().and_then(|mut slot| slot.take());
match taken {
None => Ok(()),
Some(message) => Err(NetworkError::Cuda(message)),
}
}
pub(crate) fn upload(&self, matrix: &Matrix) -> Result<CudaSlice<f32>, NetworkError> {
self.stream.clone_htod(&matrix.data).map_err(cuda_alloc_err(
"host to device copy",
std::mem::size_of_val(&matrix.data[..]),
))
}
pub(crate) fn download(
&self,
source: &CudaSlice<f32>,
target: &mut Matrix,
) -> Result<(), NetworkError> {
self.stream
.memcpy_dtoh(source, &mut target.data)
.map_err(cuda_err("device to host copy"))
}
pub(crate) fn zeros(&self, len: usize) -> Result<CudaSlice<f32>, NetworkError> {
self.stream
.alloc_zeros::<f32>(len)
.map_err(cuda_alloc_err("device allocation", len * 4))
}
pub(crate) fn im2col(
&self,
input: &CudaSlice<f32>,
shape: &crate::conv::ConvGeometry,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut columns = self.zeros(shape.column_elements())?;
self.launch_conv(
&self.im2col_train,
&mut columns,
input,
shape,
"im2col kernel",
)?;
Ok(columns)
}
pub(crate) fn col2im(
&self,
columns: &CudaSlice<f32>,
shape: &crate::conv::ConvGeometry,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut grad = self.zeros(shape.input_elements())?;
self.launch_conv(
&self.col2im_train,
&mut grad,
columns,
shape,
"col2im kernel",
)?;
Ok(grad)
}
fn launch_conv(
&self,
kernel: &CudaFunction,
destination: &mut CudaSlice<f32>,
source: &CudaSlice<f32>,
shape: &crate::conv::ConvGeometry,
stage: &'static str,
) -> Result<(), NetworkError> {
let elements = destination.len();
let dimensions = [
shape.batch,
shape.channels,
shape.height,
shape.width,
shape.kernel,
shape.stride,
shape.padding,
shape.out_height,
shape.out_width,
]
.map(|value| value as i32);
unsafe {
let mut launch = self.stream.launch_builder(kernel);
launch.arg(destination).arg(source);
for value in &dimensions {
launch.arg(value);
}
launch.launch(cfg(elements)).map_err(cuda_err(stage))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn attention(
&self,
queries: &Matrix,
keys: &Matrix,
values: &Matrix,
num_heads: usize,
num_kv_heads: usize,
head_dim: usize,
position_offset: usize,
keep_probabilities: bool,
) -> Result<(Vec<Matrix>, Matrix), NetworkError> {
let tokens = queries.rows;
let history = keys.rows;
let group_size = num_heads / num_kv_heads;
let scale = (head_dim as f32).sqrt().recip();
let device_queries = self.upload(queries)?;
let device_keys = self.upload(keys)?;
let device_values = self.upload(values)?;
let mut device_scores = self.zeros(tokens * history)?;
let mut device_merged = self.zeros(tokens * num_heads * head_dim)?;
let mut probabilities = Vec::with_capacity(if keep_probabilities { num_heads } else { 0 });
let mut scores = Matrix::new(tokens, history);
for head in 0..num_heads {
let query_base = head * head_dim;
let kv_base = (head / group_size) * head_dim;
gemm_rhs_transposed(
self,
&device_queries.slice(query_base..),
queries.cols,
&device_keys.slice(kv_base..),
keys.cols,
&mut device_scores.slice_mut(0..),
history,
tokens,
history,
head_dim,
scale,
0.0,
)?;
self.download(&device_scores, &mut scores)?;
for query in 0..tokens {
let visible = position_offset + query + 1;
let row = scores.row_mut(query);
softmax(&mut row[..visible]);
row[visible..].fill(0.0);
}
self.stream
.memcpy_htod(&scores.data, &mut device_scores)
.map_err(cuda_err("host to device copy"))?;
gemm_plain(
self,
&device_scores.slice(0..),
history,
&device_values.slice(kv_base..),
values.cols,
&mut device_merged.slice_mut(query_base..),
num_heads * head_dim,
tokens,
head_dim,
history,
1.0,
0.0,
)?;
if keep_probabilities {
probabilities.push(scores.clone());
}
}
let mut merged = Matrix::new(tokens, num_heads * head_dim);
self.download(&device_merged, &mut merged)?;
Ok((probabilities, merged))
}
}
pub struct DeviceParam {
context: Arc<GpuContext>,
rows: usize,
cols: usize,
value: CudaSlice<f32>,
negated_grad: CudaSlice<f32>,
grad_dirty: bool,
moment1: CudaSlice<f32>,
moment2: CudaSlice<f32>,
frozen: bool,
}
impl std::fmt::Debug for DeviceParam {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeviceParam")
.field("device", &self.context.device)
.field("rows", &self.rows)
.field("cols", &self.cols)
.finish()
}
}
impl Clone for DeviceParam {
fn clone(&self) -> Self {
let copy = |source: &CudaSlice<f32>| {
self.context
.stream
.clone_dtod(source)
.map_err(cuda_err("device to device copy"))
};
let fallback = |len: usize| self.context.stream.alloc_zeros::<f32>(len).ok();
let mut slices = Vec::with_capacity(4);
for source in [
&self.value,
&self.negated_grad,
&self.moment1,
&self.moment2,
] {
match copy(source) {
Ok(slice) => slices.push(Some(slice)),
Err(error) => {
self.context.poison(error);
slices.push(fallback(self.rows * self.cols));
}
}
}
let mut take = |index: usize| {
slices[index]
.take()
.unwrap_or_else(|| panic!("CUDA allocation failed while cloning a parameter"))
};
Self {
context: self.context.clone(),
rows: self.rows,
cols: self.cols,
value: take(0),
negated_grad: take(1),
grad_dirty: self.grad_dirty,
moment1: take(2),
moment2: take(3),
frozen: self.frozen,
}
}
}
impl DeviceParam {
pub(crate) fn new(
context: Arc<GpuContext>,
value: &Matrix,
frozen: bool,
) -> Result<Self, NetworkError> {
let len = if frozen { 1 } else { value.data.len() };
Ok(Self {
rows: value.rows,
cols: value.cols,
value: context.upload(value)?,
negated_grad: context.zeros(len)?,
grad_dirty: false,
moment1: context.zeros(len)?,
moment2: context.zeros(len)?,
frozen,
context,
})
}
pub(crate) fn is_frozen(&self) -> bool {
self.frozen
}
pub(crate) fn context(&self) -> &Arc<GpuContext> {
&self.context
}
pub(crate) fn rows(&self) -> usize {
self.rows
}
pub(crate) fn cols(&self) -> usize {
self.cols
}
pub(crate) fn value(&self) -> &CudaSlice<f32> {
&self.value
}
pub(crate) fn negated_grad_mut(&mut self) -> &mut CudaSlice<f32> {
&mut self.negated_grad
}
pub(crate) fn grad_beta(&mut self) -> f32 {
f32::from(u8::from(std::mem::replace(&mut self.grad_dirty, true)))
}
pub(crate) fn clear_grad(&mut self) -> Result<(), NetworkError> {
if std::mem::replace(&mut self.grad_dirty, true) {
return Ok(());
}
self.context
.stream
.memset_zeros(&mut self.negated_grad)
.map_err(cuda_err("gradient reset"))
}
pub(crate) fn download_value(&self, target: &mut Matrix) -> Result<(), NetworkError> {
self.context.download(&self.value, target)
}
pub(crate) fn upload_value(&mut self, value: &Matrix) -> Result<(), NetworkError> {
self.value = self.context.upload(value)?;
Ok(())
}
pub(crate) fn download_moments(
&self,
first: &mut Matrix,
second: &mut Matrix,
) -> Result<(), NetworkError> {
if self.frozen {
return Ok(());
}
self.context.download(&self.moment1, first)?;
first.data.iter_mut().for_each(|value| *value = -*value);
self.context.download(&self.moment2, second)
}
pub(crate) fn upload_moments(
&mut self,
first: &Matrix,
second: &Matrix,
) -> Result<(), NetworkError> {
if self.frozen {
return Ok(());
}
let negated: Vec<f32> = first.data.iter().map(|value| -value).collect();
self.context
.stream
.memcpy_htod(&negated, &mut self.moment1)
.map_err(cuda_err("host to device copy"))?;
self.context
.stream
.memcpy_htod(&second.data, &mut self.moment2)
.map_err(cuda_err("host to device copy"))
}
pub(crate) fn grad_norm(&self) -> Result<f32, NetworkError> {
if self.frozen || !self.grad_dirty {
return Ok(0.0);
}
let (pointer, _guard) = self.negated_grad.device_ptr(&self.context.stream);
let mut norm = 0.0f32;
let status = unsafe {
cublas_sys::cublasSnrm2_v2(
*self.context.blas.handle(),
(self.rows * self.cols) as i32,
pointer as *const f32,
1,
&mut norm,
)
};
if status != cublas_sys::cublasStatus_t::CUBLAS_STATUS_SUCCESS {
return Err(NetworkError::Cuda(format!(
"cuBLAS gradient norm failed: {status:?}"
)));
}
Ok(norm)
}
pub(crate) fn download_grad(&self, target: &mut Matrix) -> Result<(), NetworkError> {
if self.frozen || !self.grad_dirty {
target.zeros();
return Ok(());
}
self.context.download(&self.negated_grad, target)?;
for slot in &mut target.data {
*slot = -*slot;
}
Ok(())
}
pub(crate) fn zero_grad(&mut self) {
self.grad_dirty = false;
}
pub(crate) fn matmul_rhs_transposed(&self, input: &Matrix) -> Matrix {
let mut output = Matrix::new(input.rows, self.rows);
let result = (|| {
let device_input = self.context.upload(input)?;
let mut device_output = self.context.zeros(output.data.len())?;
gemm_rhs_transposed(
&self.context,
&device_input,
self.cols,
&self.value,
self.cols,
&mut device_output,
self.rows,
input.rows,
self.rows,
self.cols,
1.0,
0.0,
)?;
self.context.download(&device_output, &mut output)
})();
self.context.guard(result, ());
output
}
pub(crate) fn matmul_rhs_transposed_device(
&self,
input: &CudaSlice<f32>,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut output = self.context.zeros(rows * self.rows)?;
gemm_rhs_transposed(
&self.context,
input,
self.cols,
&self.value,
self.cols,
&mut output,
self.rows,
rows,
self.rows,
self.cols,
1.0,
0.0,
)?;
Ok(output)
}
pub(crate) fn matmul_device(
&self,
input: &CudaSlice<f32>,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut output = self.context.zeros(rows * self.cols)?;
gemm_plain(
&self.context,
input,
self.rows,
&self.value,
self.cols,
&mut output,
self.cols,
rows,
self.cols,
self.rows,
1.0,
0.0,
)?;
Ok(output)
}
pub(crate) fn accumulate_grad_device(
&mut self,
grad_output: &CudaSlice<f32>,
input: &CudaSlice<f32>,
rows: usize,
) -> Result<(), NetworkError> {
if self.frozen {
return Ok(());
}
let beta = self.grad_beta();
gemm_lhs_transposed(
&self.context,
grad_output,
self.rows,
input,
self.cols,
&mut self.negated_grad,
self.cols,
rows,
self.rows,
self.cols,
-1.0,
beta,
)
}
pub(crate) fn matmul(&self, input: &Matrix) -> Matrix {
let mut output = Matrix::new(input.rows, self.cols);
let result = (|| {
let device_input = self.context.upload(input)?;
let mut device_output = self.context.zeros(output.data.len())?;
gemm_plain(
&self.context,
&device_input,
self.rows,
&self.value,
self.cols,
&mut device_output,
self.cols,
input.rows,
self.cols,
self.rows,
1.0,
0.0,
)?;
self.context.download(&device_output, &mut output)
})();
self.context.guard(result, ());
output
}
pub(crate) fn accumulate_grad(&mut self, grad_output: &Matrix, input: &Matrix) {
if self.frozen {
return;
}
let result = (|| {
let device_grad_output = self.context.upload(grad_output)?;
let device_input = self.context.upload(input)?;
let beta = self.grad_beta();
gemm_lhs_transposed(
&self.context,
&device_grad_output,
self.rows,
&device_input,
self.cols,
&mut self.negated_grad,
self.cols,
grad_output.rows,
self.rows,
self.cols,
-1.0,
beta,
)
})();
self.context.guard(result, ());
}
pub(crate) fn gather(&self, ids: &[u32]) -> Matrix {
let mut output = Matrix::new(ids.len(), self.cols);
let result = (|| {
let device_ids = self
.context
.stream
.clone_htod(&ids.to_vec())
.map_err(cuda_err("host to device copy"))?;
let mut device_output = self.context.zeros(output.data.len())?;
let elements = ids.len() * self.cols;
unsafe {
self.context
.stream
.launch_builder(&self.context.gather)
.arg(&mut device_output)
.arg(&self.value)
.arg(&device_ids)
.arg(&0i32)
.arg(&(ids.len() as i32))
.arg(&(self.cols as i32))
.launch(cfg(elements))
.map_err(cuda_err("embedding gather kernel"))?;
}
self.context.download(&device_output, &mut output)
})();
self.context.guard(result, ());
output
}
pub(crate) fn scatter_grad(&mut self, ids: &[u32], grad_output: &Matrix) {
if self.frozen {
return;
}
let result = (|| -> Result<(), NetworkError> {
let device_ids = self
.context
.stream
.clone_htod(&ids.to_vec())
.map_err(cuda_err("host to device copy"))?;
let device_grad = self.context.upload(grad_output)?;
self.clear_grad()?;
let elements = ids.len() * self.cols;
unsafe {
self.context
.stream
.launch_builder(&self.context.scatter)
.arg(&mut self.negated_grad)
.arg(&device_grad)
.arg(&device_ids)
.arg(&(ids.len() as i32))
.arg(&(self.cols as i32))
.launch(cfg(elements))
.map_err(cuda_err("embedding scatter kernel"))?;
}
Ok(())
})();
self.context.guard(result, ());
}
pub(crate) fn step(&mut self, optimizer: &Optimizer, step: usize, scale: f32) {
if self.frozen {
return;
}
let elements = self.rows * self.cols;
let result = (|| -> Result<(), NetworkError> {
self.clear_grad()?;
if scale != 1.0 {
unsafe {
self.context
.stream
.launch_builder(&self.context.scale)
.arg(&mut self.negated_grad)
.arg(&(elements as i32))
.arg(&scale)
.launch(cfg(elements))
.map_err(cuda_err("gradient scaling kernel"))?;
}
}
match *optimizer {
Optimizer::Sgd { learning_rate } => unsafe {
self.context
.stream
.launch_builder(&self.context.sgd)
.arg(&mut self.value)
.arg(&self.negated_grad)
.arg(&(elements as i32))
.arg(&learning_rate)
.launch(cfg(elements))
.map_err(cuda_err("SGD update kernel"))?;
},
Optimizer::Adam {
learning_rate,
beta1,
beta2,
epsilon,
weight_decay,
} => {
let correction1 = 1.0 - beta1.powi(step as i32);
let correction2 = 1.0 - beta2.powi(step as i32);
unsafe {
self.context
.stream
.launch_builder(&self.context.adam)
.arg(&mut self.value)
.arg(&self.negated_grad)
.arg(&mut self.moment1)
.arg(&mut self.moment2)
.arg(&(elements as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&epsilon)
.arg(&correction1)
.arg(&correction2)
.arg(&weight_decay)
.launch(cfg(elements))
.map_err(cuda_err("Adam update kernel"))?;
}
}
Optimizer::Lion {
learning_rate,
beta1,
beta2,
weight_decay,
} => unsafe {
self.context
.stream
.launch_builder(&self.context.lion)
.arg(&mut self.value)
.arg(&self.negated_grad)
.arg(&mut self.moment1)
.arg(&(elements as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&weight_decay)
.launch(cfg(elements))
.map_err(cuda_err("Lion update kernel"))?;
},
}
self.grad_dirty = false;
Ok(())
})();
self.context.guard(result, ());
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn attention_heads(
context: &Arc<GpuContext>,
queries: &Matrix,
keys: &Matrix,
values: &Matrix,
num_heads: usize,
num_kv_heads: usize,
head_dim: usize,
position_offset: usize,
keep_probabilities: bool,
) -> (Vec<Matrix>, Matrix) {
let fallback = (Vec::new(), Matrix::new(queries.rows, num_heads * head_dim));
let result = context.attention(
queries,
keys,
values,
num_heads,
num_kv_heads,
head_dim,
position_offset,
keep_probabilities,
);
context.guard(result, fallback)
}
unsafe fn gemm_dispatch<T: 'static, A: DevicePtr<T>, B: DevicePtr<T>, C: DevicePtrMut<T>>(
context: &GpuContext,
config: GemmConfig<T>,
a: &A,
b: &B,
c: &mut C,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
if !context.mixed_precision || TypeId::of::<T>() != TypeId::of::<f32>() {
return unsafe { context.blas.gemm(config, a, b, c) }.map_err(cuda_err("cuBLAS GEMM"));
}
let (a_pointer, _a_guard) = a.device_ptr(&context.stream);
let (b_pointer, _b_guard) = b.device_ptr(&context.stream);
let (c_pointer, _c_guard) = c.device_ptr_mut(&context.stream);
unsafe {
cublas::gemm_ex(
*context.blas.handle(),
config.transa,
config.transb,
config.m,
config.n,
config.k,
&config.alpha as *const T as *const std::ffi::c_void,
a_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
config.lda,
b_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
config.ldb,
&config.beta as *const T as *const std::ffi::c_void,
c_pointer as *mut std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
config.ldc,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP,
)
}
.map_err(cuda_err("cuBLAS BF16 GEMM"))
}
unsafe fn act_dispatch<T, C: DevicePtrMut<T>>(
context: &GpuContext,
config: GemmConfig<f32>,
a: &CudaView<'_, u8>,
b: &CudaView<'_, u8>,
narrow: bool,
out_narrow: bool,
c: &mut C,
) -> Result<(), NetworkError> {
let (operand, compute) = if narrow && context.half_accumulate {
(
cublas_sys::cudaDataType_t::CUDA_R_16F,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_16F,
)
} else if narrow {
(
cublas_sys::cudaDataType_t::CUDA_R_16BF,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F,
)
} else if context.mixed_precision {
(
cublas_sys::cudaDataType_t::CUDA_R_32F,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
)
} else {
(
cublas_sys::cudaDataType_t::CUDA_R_32F,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F,
)
};
let half = compute == cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_16F;
let scalars = (
half::f16::from_f32(config.alpha),
half::f16::from_f32(config.beta),
);
let (alpha, beta): (*const std::ffi::c_void, *const std::ffi::c_void) = match half {
true => (
&scalars.0 as *const half::f16 as *const _,
&scalars.1 as *const half::f16 as *const _,
),
false => (
&config.alpha as *const f32 as *const _,
&config.beta as *const f32 as *const _,
),
};
let (a_pointer, _a_guard) = a.device_ptr(&context.stream);
let (b_pointer, _b_guard) = b.device_ptr(&context.stream);
let (c_pointer, _c_guard) = c.device_ptr_mut(&context.stream);
unsafe {
cublas::gemm_ex(
*context.blas.handle(),
config.transa,
config.transb,
config.m,
config.n,
config.k,
alpha,
a_pointer as *const std::ffi::c_void,
operand,
config.lda,
b_pointer as *const std::ffi::c_void,
operand,
config.ldb,
beta,
c_pointer as *mut std::ffi::c_void,
match (out_narrow, half) {
(true, true) => cublas_sys::cudaDataType_t::CUDA_R_16F,
(true, false) => cublas_sys::cudaDataType_t::CUDA_R_16BF,
(false, _) => cublas_sys::cudaDataType_t::CUDA_R_32F,
},
config.ldc,
compute,
match half {
true => cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT,
false => cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP,
},
)
}
.map_err(cuda_err("cuBLAS narrow GEMM"))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn act_rhs_transposed<T, O: DevicePtrMut<T>>(
context: &GpuContext,
x: &CudaView<'_, u8>,
x_stride: usize,
w: &CudaView<'_, u8>,
w_stride: usize,
narrow: bool,
out_narrow: bool,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
inner: usize,
alpha: f32,
beta: f32,
) -> Result<(), NetworkError> {
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { act_dispatch(context, config, w, x, narrow, out_narrow, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn act_plain<T, O: DevicePtrMut<T>>(
context: &GpuContext,
x: &CudaView<'_, u8>,
x_stride: usize,
w: &CudaView<'_, u8>,
w_stride: usize,
narrow: bool,
out_narrow: bool,
out: &mut O,
out_stride: usize,
rows: usize,
cols: usize,
inner: usize,
alpha: f32,
beta: f32,
) -> Result<(), NetworkError> {
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: cols as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { act_dispatch(context, config, w, x, narrow, out_narrow, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn act_lhs_transposed<O: DevicePtrMut<f32>>(
context: &GpuContext,
d: &CudaView<'_, u8>,
d_stride: usize,
x: &CudaView<'_, u8>,
x_stride: usize,
narrow: bool,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
cols: usize,
alpha: f32,
beta: f32,
) -> Result<(), NetworkError> {
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: cols as i32,
n: units as i32,
k: rows as i32,
alpha,
lda: x_stride as i32,
ldb: d_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { act_dispatch(context, config, x, d, narrow, false, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_rhs_transposed<
T: 'static,
X: DevicePtr<T>,
W: DevicePtr<T>,
O: DevicePtrMut<T>,
>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
inner: usize,
alpha: T,
beta: T,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { gemm_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
unsafe fn gemm_strided_batched_dispatch<
A: DevicePtr<f32>,
B: DevicePtr<f32>,
C: DevicePtrMut<f32>,
>(
context: &GpuContext,
config: StridedBatchedConfig<f32>,
a: &A,
b: &B,
c: &mut C,
) -> Result<(), NetworkError> {
if !context.mixed_precision {
return unsafe { context.blas.gemm_strided_batched(config, a, b, c) }
.map_err(cuda_err("cuBLAS batched GEMM"));
}
let (a_pointer, _a_guard) = a.device_ptr(&context.stream);
let (b_pointer, _b_guard) = b.device_ptr(&context.stream);
let (c_pointer, _c_guard) = c.device_ptr_mut(&context.stream);
let gemm = config.gemm;
unsafe {
cublas::gemm_strided_batched_ex(
*context.blas.handle(),
gemm.transa,
gemm.transb,
gemm.m,
gemm.n,
gemm.k,
&gemm.alpha as *const f32 as *const std::ffi::c_void,
a_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
gemm.lda,
config.stride_a,
b_pointer as *const std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
gemm.ldb,
config.stride_b,
&gemm.beta as *const f32 as *const std::ffi::c_void,
c_pointer as *mut std::ffi::c_void,
cublas_sys::cudaDataType_t::CUDA_R_32F,
gemm.ldc,
config.stride_c,
config.batch_size,
cublas_sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
cublas_sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP,
)
}
.map_err(cuda_err("cuBLAS batched BF16 GEMM"))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_rhs_transposed_batched<
X: DevicePtr<f32>,
W: DevicePtr<f32>,
O: DevicePtrMut<f32>,
>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
inner: usize,
alpha: f32,
beta: f32,
count: usize,
x_batch_stride: usize,
w_batch_stride: usize,
out_batch_stride: usize,
) -> Result<(), NetworkError> {
let config = StridedBatchedConfig {
gemm: GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
},
batch_size: count as i32,
stride_a: w_batch_stride as i64,
stride_b: x_batch_stride as i64,
stride_c: out_batch_stride as i64,
};
unsafe { gemm_strided_batched_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_plain_batched<X: DevicePtr<f32>, W: DevicePtr<f32>, O: DevicePtrMut<f32>>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
cols: usize,
inner: usize,
alpha: f32,
beta: f32,
count: usize,
x_batch_stride: usize,
w_batch_stride: usize,
out_batch_stride: usize,
) -> Result<(), NetworkError> {
let config = StridedBatchedConfig {
gemm: GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: cols as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
},
batch_size: count as i32,
stride_a: w_batch_stride as i64,
stride_b: x_batch_stride as i64,
stride_c: out_batch_stride as i64,
};
unsafe { gemm_strided_batched_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_lhs_transposed_batched<
D: DevicePtr<f32>,
X: DevicePtr<f32>,
O: DevicePtrMut<f32>,
>(
context: &GpuContext,
d: &D,
d_stride: usize,
x: &X,
x_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
cols: usize,
alpha: f32,
beta: f32,
count: usize,
d_batch_stride: usize,
x_batch_stride: usize,
out_batch_stride: usize,
) -> Result<(), NetworkError> {
let config = StridedBatchedConfig {
gemm: GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: cols as i32,
n: units as i32,
k: rows as i32,
alpha,
lda: x_stride as i32,
ldb: d_stride as i32,
beta,
ldc: out_stride as i32,
},
batch_size: count as i32,
stride_a: x_batch_stride as i64,
stride_b: d_batch_stride as i64,
stride_c: out_batch_stride as i64,
};
unsafe { gemm_strided_batched_dispatch(context, config, x, d, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_plain<T: 'static, X: DevicePtr<T>, W: DevicePtr<T>, O: DevicePtrMut<T>>(
context: &GpuContext,
x: &X,
x_stride: usize,
w: &W,
w_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
cols: usize,
inner: usize,
alpha: T,
beta: T,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: cols as i32,
n: rows as i32,
k: inner as i32,
alpha,
lda: w_stride as i32,
ldb: x_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { gemm_dispatch(context, config, w, x, out) }
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gemm_lhs_transposed<
T: 'static,
D: DevicePtr<T>,
X: DevicePtr<T>,
O: DevicePtrMut<T>,
>(
context: &GpuContext,
d: &D,
d_stride: usize,
x: &X,
x_stride: usize,
out: &mut O,
out_stride: usize,
rows: usize,
units: usize,
cols: usize,
alpha: T,
beta: T,
) -> Result<(), NetworkError>
where
CudaBlas: Gemm<T>,
{
let config = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: cols as i32,
n: units as i32,
k: rows as i32,
alpha,
lda: x_stride as i32,
ldb: d_stride as i32,
beta,
ldc: out_stride as i32,
};
unsafe { gemm_dispatch(context, config, x, d, out) }
}
#[cfg(test)]
mod tests {
use crate::cuda_training::cuda_doctor;
use crate::network::NetworkError;
use crate::optimizers::Optimizer;
use crate::transformer::{TransformerBuilder, TransformerLm};
fn cuda_or_skip() -> bool {
match cuda_doctor(0, 8192) {
Ok(_) => true,
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
false
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn tiny() -> TransformerBuilder {
TransformerLm::builder()
.vocab_size(24)
.d_model(16)
.n_layers(2)
.heads(4, 2, 4)
.d_ff(32)
.moe_d_ff(12)
.experts(4, 2)
.moe_layers([1])
.shared_expert(true)
.max_seq_len(32)
.optimizer(Optimizer::adam(1e-2))
.seed(1234)
.mixed_precision(false)
}
fn assert_close(label: &str, gpu: &[f32], cpu: &[f32], tolerance: f32) {
assert_eq!(gpu.len(), cpu.len(), "{label}: length");
for (index, (a, b)) in gpu.iter().zip(cpu).enumerate() {
assert!(
(a - b).abs() <= tolerance,
"{label}[{index}]: {a} on the device vs {b} on the host"
);
}
}
#[test]
fn gpu_grad_norm_matches_cpu_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = crate::batch::TokenBatch::new(&[[3u32, 8, 1, 5, 2, 7]]).unwrap();
cpu.zero_grad();
gpu.zero_grad();
assert_eq!(gpu.grad_norm().unwrap(), 0.0);
cpu.accumulate_step(&batch).unwrap();
gpu.accumulate_step(&batch).unwrap();
let (host, device) = (cpu.grad_norm().unwrap(), gpu.grad_norm().unwrap());
assert!(host > 0.0);
assert!(
(device - host).abs() <= 1e-3 * host,
"{device} on the device vs {host} on the host"
);
}
#[test]
fn gpu_forward_matches_cpu_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5, 2, 7];
let (host_logits, _) = cpu.forward_train(&[&ids[..]]).unwrap();
let (device_logits, _) = gpu.forward_train(&[&ids[..]]).unwrap();
assert_close("logits", &device_logits.data, &host_logits.data, 1e-3);
}
#[test]
fn gpu_backward_matches_cpu_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5, 2, 7];
let host = cpu.train_step(&[&ids[..]]).unwrap();
let device = gpu.train_step(&[&ids[..]]).unwrap();
gpu.sync_from_device().unwrap();
assert!((host.total() - device.total()).abs() < 1e-3);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn a_gpu_lora_step_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
cpu.add_lora(crate::transformer::LoraConfig::new(4).alpha(8.0))
.unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5, 2, 7];
let before = cpu.blocks[0].attention.query.weight.value.clone();
for step in 0..2 {
let host = cpu.train_step(&[&ids[..]]).unwrap();
let device = gpu.train_step(&[&ids[..]]).unwrap();
assert!(
(host.total() - device.total()).abs() < 1e-3,
"step {step}: {} on the device vs {} on the host",
device.total(),
host.total()
);
}
gpu.sync_from_device().unwrap();
assert_close(
"frozen base weight",
&gpu.blocks[0].attention.query.weight.value.data,
&before.data,
0.0,
);
let adapter = gpu.blocks[0]
.attention
.query
.lora
.as_ref()
.expect("the query projection carries an adapter");
assert!(
adapter.up.value.data.iter().any(|&value| value != 0.0),
"the device step left the adapter at its zero initialization"
);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn a_batched_gpu_forward_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5, 2, 7], vec![9, 4, 4, 0, 11, 6]];
let (host_logits, _) = cpu.forward_train(&batch).unwrap();
let (device_logits, _) = gpu.forward_train(&batch).unwrap();
assert_eq!(device_logits.rows, 12);
assert_close(
"batched logits",
&device_logits.data,
&host_logits.data,
1e-3,
);
}
#[test]
fn a_padded_gpu_batch_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5, 2, 7], vec![9, 4, 4], vec![2, 6]];
let (host_logits, host_cache) = cpu.forward_train(&batch).unwrap();
let (device_logits, device_cache) = gpu.forward_train(&batch).unwrap();
assert_close(
"padded logits",
&device_logits.data,
&host_logits.data,
1e-3,
);
assert!(
(device_cache.auxiliary_loss() - host_cache.auxiliary_loss()).abs() < 1e-4,
"auxiliary loss {} on the device vs {} on the host",
device_cache.auxiliary_loss(),
host_cache.auxiliary_loss()
);
}
#[test]
fn a_batched_gpu_train_step_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [
vec![3u32, 8, 1, 5, 2, 7],
vec![9, 4, 4, 0, 11, 6],
vec![2, 6, 1],
];
let host = cpu.train_step(&batch).unwrap();
let device = gpu.train_step(&batch).unwrap();
gpu.sync_from_device().unwrap();
assert!((host.lm_loss - device.lm_loss).abs() < 1e-3);
assert!((host.auxiliary_loss - device.auxiliary_loss).abs() < 1e-4);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn adam_moments_survive_a_move_each_way_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut host = tiny().build().unwrap();
let mut moved = host.clone();
let batch = [vec![3u32, 8, 1, 5, 2, 7], vec![9, 4, 4, 0, 11, 6]];
host.train_step(&batch).unwrap();
moved.train_step(&batch).unwrap();
moved.to_cuda(0, 8192).unwrap();
host.train_step(&batch).unwrap();
moved.train_step(&batch).unwrap();
moved.to_cpu().unwrap();
host.train_step(&batch).unwrap();
moved.train_step(&batch).unwrap();
for (index, (moved_param, host_param)) in
moved.params_mut().iter().zip(host.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&moved_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn a_lion_step_on_the_device_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny()
.optimizer(Optimizer::lion_with_weight_decay(1e-2, 0.1))
.build()
.unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5, 2, 7], vec![9, 4, 4, 0, 11, 6]];
for _ in 0..2 {
cpu.train_step(&batch).unwrap();
gpu.train_step(&batch).unwrap();
}
gpu.sync_from_device().unwrap();
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn a_chunked_gpu_head_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [
vec![3u32, 8, 1, 5, 2, 7],
vec![9, 4, 4, 0, 11, 6],
vec![2, 6, 1],
];
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(5, std::sync::atomic::Ordering::Relaxed);
let host = cpu.train_step(&batch).unwrap();
let device = gpu.train_step(&batch).unwrap();
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(0, std::sync::atomic::Ordering::Relaxed);
gpu.sync_from_device().unwrap();
assert!(
(host.lm_loss - device.lm_loss).abs() < 1e-3,
"chunked loss {} on the device vs {} on the host",
device.lm_loss,
host.lm_loss
);
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
1e-3,
);
}
}
#[test]
fn mixed_precision_tracks_the_fp32_device_path_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let mut baseline = tiny().build().unwrap();
let mut reduced = tiny().mixed_precision(true).build().unwrap();
assert_eq!(
baseline, reduced,
"the two models start from the same weights"
);
baseline.to_cuda(0, 8192).unwrap();
reduced.to_cuda(0, 8192).unwrap();
let batch = [
vec![3u32, 8, 1, 5, 2, 7],
vec![9, 4, 4, 0, 11, 6],
vec![2, 6, 1],
];
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(5, std::sync::atomic::Ordering::Relaxed);
for step in 0..2 {
let exact = baseline.train_step(&batch).unwrap();
let approximate = reduced.train_step(&batch).unwrap();
assert!(
(exact.lm_loss - approximate.lm_loss).abs() < 1e-2,
"step {step}: reduced-precision loss {} vs FP32 {}",
approximate.lm_loss,
exact.lm_loss
);
}
crate::gpu_model::CHUNK_ROWS_OVERRIDE.store(0, std::sync::atomic::Ordering::Relaxed);
}
#[test]
fn repeated_gpu_batched_steps_stay_in_step_with_the_host_or_skip_without_device() {
if !cuda_or_skip() {
return;
}
let mut cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let batch = [vec![3u32, 8, 1, 5], vec![9, 4, 4, 0]];
for step in 0..4 {
let host = cpu.train_step(&batch).unwrap();
let device = gpu.train_step(&batch).unwrap();
assert!(
(host.total() - device.total()).abs() < 2e-3,
"step {step}: {} on the device vs {} on the host",
device.total(),
host.total()
);
}
gpu.sync_from_device().unwrap();
for (index, (device_param, host_param)) in
gpu.params_mut().iter().zip(cpu.params_mut()).enumerate()
{
assert_close(
&format!("parameter {index}"),
&device_param.value.data,
&host_param.value.data,
2e-3,
);
}
}
#[test]
fn a_cached_decode_on_the_device_matches_the_host_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let cpu = tiny().build().unwrap();
let mut gpu = cpu.clone();
gpu.to_cuda(0, 8192).unwrap();
let ids = [3u32, 8, 1, 5];
let mut host_caches = cpu.new_kv_caches();
let mut device_caches = gpu.new_kv_caches();
let host = cpu.forward_cached(&ids, &mut host_caches).unwrap();
let device = gpu.forward_cached(&ids, &mut device_caches).unwrap();
assert_close("prefill logits", &device.data, &host.data, 1e-3);
}
#[test]
fn to_cuda_fails_closed_without_a_device() {
if cuda_or_skip() {
return;
}
let mut model = tiny().build().unwrap();
assert!(matches!(model.to_cuda(0, 8192), Err(NetworkError::Cuda(_))));
assert!(!model.on_device());
}
#[test]
fn an_impossible_memory_budget_is_refused_before_any_upload() {
let mut model = tiny()
.d_model(64)
.heads(4, 2, 16)
.d_ff(256)
.moe_d_ff(64)
.build()
.unwrap();
assert!(matches!(
model.to_cuda(0, 1),
Err(NetworkError::CudaMemoryBudget { .. })
));
assert!(!model.on_device());
}
#[test]
fn fused_attention_matches_the_three_kernel_path_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
for bidirectional in [false, true] {
fused_attention_forward_parity(bidirectional);
}
}
fn fused_attention_forward_parity(bidirectional: bool) {
let model = || {
TransformerLm::builder()
.vocab_size(32)
.d_model(128)
.n_layers(2)
.heads(2, 1, 64)
.d_ff(64)
.moe_layers([0usize; 0])
.max_seq_len(128)
.optimizer(Optimizer::adam(1e-2))
.seed(99)
.mixed_precision(true)
.bidirectional(bidirectional)
.build()
.unwrap()
};
let ids: Vec<u32> = (0..100).map(|index| (index * 7 % 32) as u32).collect();
let (host_logits, _) = model().forward_train(&[&ids[..]]).unwrap();
let mut fused = model();
fused.to_cuda(0, 8192).unwrap();
let (fused_logits, _) = fused.forward_train(&[&ids[..]]).unwrap();
crate::cuda_flash::DISABLED.store(true, std::sync::atomic::Ordering::Relaxed);
let mut split = model();
split.to_cuda(0, 8192).unwrap();
let split_forward = split.forward_train(&[&ids[..]]);
crate::cuda_flash::DISABLED.store(false, std::sync::atomic::Ordering::Relaxed);
let (split_logits, _) = split_forward.unwrap();
let error = |logits: &crate::matrix::Matrix| {
logits
.data
.iter()
.zip(&host_logits.data)
.map(|(device, host)| (device - host).abs())
.fold(0.0f32, f32::max)
};
let (fused_error, split_error) = (error(&fused_logits), error(&split_logits));
assert!(
fused_error <= 2.0 * split_error,
"bidirectional {bidirectional}: the fused path is {fused_error} from the host \
where the three-kernel path is {split_error}"
);
assert!(
fused_error < 1e-2,
"bidirectional {bidirectional}: both device paths drifted: {fused_error}"
);
}
#[test]
fn fused_attention_backward_matches_the_three_kernel_path_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
for bidirectional in [false, true] {
fused_attention_backward_parity(bidirectional);
}
}
fn fused_attention_backward_parity(bidirectional: bool) {
let model = || {
TransformerLm::builder()
.vocab_size(32)
.d_model(256)
.n_layers(2)
.heads(4, 2, 64)
.d_ff(64)
.moe_layers([0usize; 0])
.max_seq_len(128)
.optimizer(Optimizer::sgd(1.0))
.seed(99)
.mixed_precision(true)
.bidirectional(bidirectional)
.build()
.unwrap()
};
let ids: Vec<u32> = (0..100).map(|index| (index * 7 % 32) as u32).collect();
let mut host = model();
host.train_step(&[&ids[..]]).unwrap();
let (host_logits, _) = host.forward_train(&[&ids[..]]).unwrap();
let mut fused = model();
fused.to_cuda(0, 8192).unwrap();
fused.train_step(&[&ids[..]]).unwrap();
let (fused_logits, _) = fused.forward_train(&[&ids[..]]).unwrap();
crate::cuda_flash::DISABLED.store(true, std::sync::atomic::Ordering::Relaxed);
let mut split = model();
split.to_cuda(0, 8192).unwrap();
let stepped = split
.train_step(&[&ids[..]])
.and_then(|_| split.forward_train(&[&ids[..]]));
crate::cuda_flash::DISABLED.store(false, std::sync::atomic::Ordering::Relaxed);
let (split_logits, _) = stepped.unwrap();
let error = |logits: &crate::matrix::Matrix| {
logits
.data
.iter()
.zip(&host_logits.data)
.map(|(device, host)| (device - host).abs())
.fold(0.0f32, f32::max)
};
let (fused_error, split_error) = (error(&fused_logits), error(&split_logits));
assert!(
fused_error <= 2.0 * split_error,
"bidirectional {bidirectional}: the fused path is {fused_error} from the host \
where the three-kernel path is {split_error}"
);
assert!(
fused_error < 0.1,
"bidirectional {bidirectional}: both device paths drifted: {fused_error}"
);
}
}