//! Fail-closed FP32 CUDA fitting. All state below is owned by `CudaSlice`s;
//! host pointers are only used by cudarc's checked copy operations.
use crate::cuda_flash::compile_ptx;
use crate::{
Activation, CudaTrainingCheckpoint, Dataset, Loss, Network, NetworkError, Optimizer,
TrainConfig, TrainingHistory,
};
use cudarc::cublas::{CudaBlas, Gemm, GemmConfig, sys::cublasOperation_t};
use cudarc::driver::{
CudaContext, CudaFunction, CudaModule, CudaSlice, CudaStream, LaunchConfig, PushKernelArg,
};
use cudarc::nvrtc::Ptx;
use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
use std::{
collections::HashMap,
sync::{Arc, Mutex, OnceLock},
time::Instant,
};
/// Backwards-compatible name for the shared session measurements.
pub use crate::accelerator::AcceleratorStats as CudaTrainingStats;
pub(crate) use crate::accelerator::MIB;
/// Re-exported so `cuda_training::estimate_tensor_memory_mib` keeps resolving;
/// the estimate itself is backend-independent and lives in `accelerator`.
pub use crate::accelerator::estimate_tensor_memory_mib;
const KERNELS: &str = r#"
// bfloat16 helpers, written against the raw bit pattern rather than
// `cuda_bf16.h`: NVRTC compiles this string without a header search path, and
// bf16 is just FP32 with the low 16 mantissa bits dropped. Rounding is
// round-to-nearest-even, the same rule the tensor cores use, so a value that
// makes a round trip through bf16 and back matches what cuBLAS saw.
typedef unsigned short bf16_t;
__device__ __forceinline__ float bf16_to_f32(bf16_t b){return __uint_as_float((unsigned int)b<<16);}
__device__ __forceinline__ bf16_t f32_to_bf16(float v){
unsigned int u=__float_as_uint(v);
if((u&0x7f800000u)==0x7f800000u&&(u&0x007fffffu))return (bf16_t)0x7fc0u;
unsigned int r=((u>>16)&1u)+0x7fffu;
return (bf16_t)((u+r)>>16);
}
// A GEMM operand is stored narrow whenever the context is in reduced
// precision, so the kernels that produce one take their destination as raw
// bytes plus a `narrow` flag rather than a typed pointer. The branch is
// uniform across the whole grid, and writing two bytes per element instead of
// four is what the caller is paying for.
__device__ __forceinline__ void store_act(void*p,size_t i,float v,int narrow){
if(narrow)((bf16_t*)p)[i]=f32_to_bf16(v); else ((float*)p)[i]=v;
}
__device__ __forceinline__ void store_act4(void*p,size_t i,float4 v,int narrow){
if(narrow){ushort4 o;o.x=f32_to_bf16(v.x);o.y=f32_to_bf16(v.y);o.z=f32_to_bf16(v.z);o.w=f32_to_bf16(v.w);((ushort4*)p)[i]=o;}
else ((float4*)p)[i]=v;
}
__device__ __forceinline__ float load_act(const void*p,size_t i,int narrow){
return narrow ? __uint_as_float((unsigned)((const unsigned short*)p)[i]<<16) : ((const float*)p)[i];
}
// `dst = src`, narrowed or not, for an operand whose producer is a cuBLAS call
// rather than one of the kernels above.
extern "C" __global__ void cast_act(void*dst,const float*src,int n,int narrow){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)store_act(dst,i,src[i],narrow);
}
extern "C" __global__ void bias_act(float *x,const float*b,int rows,int cols,int act){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<rows*cols){float v=x[i]+b[i%cols];if(act==1)v=v>0?v:0;else if(act==2)v=1.f/(1.f+expf(-v));else if(act==3)v=tanhf(v);else if(act==4)v=v/(1.f+expf(-v));x[i]=v;}}
// bias_act for SiLU or none, a float4 a thread: rows of `cols4` float4s.
extern "C" __global__ void bias_act4(float4*x,const float4*b,int n4,int cols4,int act){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n4){float4 v=x[i],c=b[i%cols4];v.x+=c.x;v.y+=c.y;v.z+=c.z;v.w+=c.w;if(act==4){v.x/=1.f+expf(-v.x);v.y/=1.f+expf(-v.y);v.z/=1.f+expf(-v.z);v.w/=1.f+expf(-v.w);}x[i]=v;}}
extern "C" __global__ void output_delta(float*d,const float*y,const float*t,int n,int act){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float v=t[i]-y[i];float a=y[i];if(act==1)v*=a>0;else if(act==2)v*=a*(1-a);else if(act==3)v*=1-a*a;d[i]=v;}}
extern "C" __global__ void act_derivative(float*d,const float*a,int n,int act){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float v=a[i];if(act==1)d[i]*=v>0;else if(act==2)d[i]*=v*(1-v);else if(act==3)d[i]*=1-v*v;}}
extern "C" __global__ void grad_b(float*g,const float*d,int batch,int out){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<out){float s=0;for(int b=0;b<batch;b++)s+=d[b*out+i];g[i]=s/(float)batch;}}
extern "C" __global__ void sgd(float*x,const float*g,int n,float lr){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)x[i]+=lr*g[i];}
extern "C" __global__ void adam(float*x,const float*g,float*m,float*v,int n,float lr,float b1,float b2,float eps,float c1,float c2,float wd){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float q=g[i];float mi=b1*m[i]+(1-b1)*q;float vi=b2*v[i]+(1-b2)*q*q;m[i]=mi;v[i]=vi;x[i]-=lr*wd*x[i];x[i]+=lr*(mi/c1)/(sqrtf(vi/c2)+eps);}}
__device__ unsigned char nearest8(const float*q,float x){int k=0;for(int s=128;s>0;s>>=1)if(k+s<256&&q[k+s]<=x)k+=s;if(k<255&&q[k+1]-x<x-q[k])k++;return(unsigned char)k;}
extern "C" __global__ void adam8(float*x,const float*g,unsigned char*m,unsigned char*v,float*mx,float*vx,const float*qm,const float*qv,int n,float lr,float b1,float b2,float eps,float c1,float c2,float wd){__shared__ float sm[256],sv[256],rm[8],rv[8];int t=threadIdx.x,b=blockIdx.x,i=b*256+t;sm[t]=qm[t];sv[t]=qv[t];__syncthreads();float mi=0,vi=0;if(i<n){float q=g[i];mi=b1*sm[m[i]]*mx[b]+(1-b1)*q;vi=b2*sv[v[i]]*vx[b]+(1-b2)*q*q;x[i]-=lr*wd*x[i];x[i]+=lr*(mi/c1)/(sqrtf(vi/c2)+eps);}float am=fabsf(mi),av=vi;for(int o=16;o>0;o>>=1){am=fmaxf(am,__shfl_xor_sync(0xffffffff,am,o));av=fmaxf(av,__shfl_xor_sync(0xffffffff,av,o));}if((t&31)==0){rm[t>>5]=am;rv[t>>5]=av;}__syncthreads();am=rm[0];av=rv[0];for(int w=1;w<8;w++){am=fmaxf(am,rm[w]);av=fmaxf(av,rv[w]);}if(i<n){m[i]=nearest8(sm,am>0?mi/am:0);v[i]=nearest8(sv,av>0?vi/av:0);}if(t==0){mx[b]=am;vx[b]=av;}}
extern "C" __global__ void lion(float*x,const float*g,float*m,int n,float lr,float b1,float b2,float wd){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float q=g[i];float u=b1*m[i]+(1-b1)*q;m[i]=b2*m[i]+(1-b2)*q;x[i]-=lr*wd*x[i];x[i]+=lr*(float)((u>0)-(u<0));}}
extern "C" __global__ void mse_sum(float *out,const float*y,const float*t,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float z=y[i]-t[i];atomicAdd(out,z*z/(float)n);}}
extern "C" __global__ void mse_epoch_sum(float *out,const float*y,const float*t,int n,float scale){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float z=y[i]-t[i];atomicAdd(out,z*z*scale);}}
extern "C" __global__ void gather_rows(float*out,const float*all,const unsigned int*order,int start,int rows,int width){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<rows*width){int r=i/width,c=i%width;out[i]=all[order[start+r]*width+c];}}
extern "C" __global__ void scale_inplace(float*g,int n,float s){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)g[i]*=s;}
extern "C" __global__ void fill_f32(float*x,int n,float v){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)x[i]=v;}
// im2col and its transpose for a trainable convolution, in FP32.
//
// `crate::cuda_image` has an FP16 im2col for inference; this pair is the
// training one, so it keeps the precision the optimizer needs and it has the
// scatter half that a backward pass needs. Layout: the input is
// `[batch * height * width, channels]` with the channel fastest, and a column
// row holds `(ky * kernel + kx) * channels + channel`, which is the ordering
// the weight `[out_channels, in_channels * kernel * kernel]` was trained in.
// 64-bit division costs several times the 32-bit one, so the index type
// follows the column matrix's size; and with channels in fours each thread
// moves a float4 (the launch is a quarter the size), a quarter of the index
// arithmetic per value.
__device__ __forceinline__ void put_col(float*p,long long i,float v){p[i]=v;}
__device__ __forceinline__ void put_col(float4*p,long long i,float4 v){p[i]=v;}
__device__ __forceinline__ void put_col(bf16_t*p,long long i,float v){p[i]=f32_to_bf16(v);}
__device__ __forceinline__ void put_col(ushort4*p,long long i,float4 v){
ushort4 o;o.x=f32_to_bf16(v.x);o.y=f32_to_bf16(v.y);o.z=f32_to_bf16(v.z);o.w=f32_to_bf16(v.w);p[i]=o;
}
template<typename I,typename V,typename W> __device__ void im2col_at(W*columns,const V*input,I idx,
int channels,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
I patch=(I)channels*kernel*kernel;
I row=idx/patch,off=idx%patch;
int c=(int)(off%channels);I t=off/channels;
int kx=(int)(t%kernel),ky=(int)(t/kernel);
I plane=(I)out_h*out_w;
I b=row/plane;int rem=(int)(row%plane);
int oy=rem/out_w,ox=rem%out_w;
int y=oy*stride+ky-pad,x=ox*stride+kx-pad;
V v={};
if(y>=0&&y<height&&x>=0&&x<width)v=input[(((long long)b*height+y)*width+x)*channels+c];
put_col(columns,idx,v);
}
extern "C" __global__ void im2col_train(float*columns,const float*input,
int batch,int channels,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
long long idx=(long long)blockIdx.x*blockDim.x+threadIdx.x;
bool four=channels%4==0;
int lanes=four?channels/4:channels;
long long total=(long long)batch*out_h*out_w*lanes*kernel*kernel;
if(idx>=total)return;
if(four){
if(total<2147483647LL)im2col_at<int>((float4*)columns,(const float4*)input,(int)idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
else im2col_at<long long>((float4*)columns,(const float4*)input,idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
}else{
if(total<2147483647LL)im2col_at<int>(columns,input,(int)idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
else im2col_at<long long>(columns,input,idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
}
}
// The same columns rounded to bf16, for a GEMM that would round them anyway.
extern "C" __global__ void im2col_bf16(bf16_t*columns,const float*input,
int batch,int channels,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
long long idx=(long long)blockIdx.x*blockDim.x+threadIdx.x;
bool four=channels%4==0;
int lanes=four?channels/4:channels;
long long total=(long long)batch*out_h*out_w*lanes*kernel*kernel;
if(idx>=total)return;
if(four){
if(total<2147483647LL)im2col_at<int>((ushort4*)columns,(const float4*)input,(int)idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
else im2col_at<long long>((ushort4*)columns,(const float4*)input,idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
}else{
if(total<2147483647LL)im2col_at<int>(columns,input,(int)idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
else im2col_at<long long>(columns,input,idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
}
}
// One thread per input element, gathering every column entry that read it.
// A scatter would need atomics and would not be reproducible run to run; this
// costs kernel*kernel reads per element and is deterministic.
__device__ __forceinline__ float get_col(const float*p,long long i){return p[i];}
__device__ __forceinline__ float get_col(const bf16_t*p,long long i){return bf16_to_f32(p[i]);}
template<typename T> __device__ void col2im_at(float*grad,const T*columns,
int batch,int channels,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
long long idx=(long long)blockIdx.x*blockDim.x+threadIdx.x;
long long total=(long long)batch*height*width*channels;
if(idx>=total)return;
int c=(int)(idx%channels);long long p=idx/channels;
int x=(int)(p%width);long long q=p/width;
int y=(int)(q%height);int b=(int)(q/height);
long long patch=(long long)channels*kernel*kernel;
float s=0.f;
for(int ky=0;ky<kernel;ky++){
int sy=y+pad-ky;if(sy<0||sy%stride)continue;int oy=sy/stride;if(oy>=out_h)continue;
for(int kx=0;kx<kernel;kx++){
int sx=x+pad-kx;if(sx<0||sx%stride)continue;int ox=sx/stride;if(ox>=out_w)continue;
long long row=((long long)b*out_h+oy)*out_w+ox;
s+=get_col(columns,row*patch+(long long)(ky*kernel+kx)*channels+c);
}
}
grad[idx]=s;
}
extern "C" __global__ void col2im_train(float*grad,const float*columns,
int batch,int channels,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
col2im_at(grad,columns,batch,channels,height,width,kernel,stride,pad,out_h,out_w);
}
// The same sum over bf16 column gradients, accumulated in FP32. With channels
// in fours a thread sums four neighbouring channels, so a warp reads 256
// contiguous bytes a tap rather than 64; the launch is a quarter the size.
template<typename I> __device__ void col2im_bf16_four(float4*grad,const ushort4*columns,I idx,
int lanes,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
int c=(int)(idx%lanes);I p=idx/lanes;
int x=(int)(p%width);I q=p/width;
int y=(int)(q%height);I b=q/height;
I patch=(I)lanes*kernel*kernel;
float4 s=make_float4(0.f,0.f,0.f,0.f);
for(int ky=0;ky<kernel;ky++){
int sy=y+pad-ky;if(sy<0||sy%stride)continue;int oy=sy/stride;if(oy>=out_h)continue;
for(int kx=0;kx<kernel;kx++){
int sx=x+pad-kx;if(sx<0||sx%stride)continue;int ox=sx/stride;if(ox>=out_w)continue;
I row=(b*out_h+oy)*out_w+ox;
ushort4 v=columns[row*patch+(I)(ky*kernel+kx)*lanes+c];
s.x+=bf16_to_f32(v.x);s.y+=bf16_to_f32(v.y);s.z+=bf16_to_f32(v.z);s.w+=bf16_to_f32(v.w);
}
}
grad[idx]=s;
}
extern "C" __global__ void col2im_bf16(float*grad,const bf16_t*columns,
int batch,int channels,int height,int width,int kernel,int stride,int pad,int out_h,int out_w){
if(channels%4){col2im_at(grad,columns,batch,channels,height,width,kernel,stride,pad,out_h,out_w);return;}
long long idx=(long long)blockIdx.x*blockDim.x+threadIdx.x;
int lanes=channels/4;
long long total=(long long)batch*height*width*lanes;
if(idx>=total)return;
// The column index reaches out_h * out_w * batch * patch.
if((long long)batch*out_h*out_w*lanes*kernel*kernel<2147483647LL)
col2im_bf16_four<int>((float4*)grad,(const ushort4*)columns,(int)idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
else col2im_bf16_four<long long>((float4*)grad,(const ushort4*)columns,idx,lanes,height,width,kernel,stride,pad,out_h,out_w);
}
// Group norm and SiLU over a device-resident `[batch * pixels, channels]`
// image batch, for `conv::TrainableGroupNorm` and `conv::silu_device`. One
// block per (sample, group) for the reductions, which sum in two passes like
// the host layer does.
__device__ float img_block_sum(float v){
__shared__ float s[32];int l=threadIdx.x&31,w=threadIdx.x>>5;
for(int o=16;o>0;o>>=1)v+=__shfl_xor_sync(0xffffffff,v,o);
__syncthreads();if(l==0)s[w]=v;__syncthreads();
v=threadIdx.x<(blockDim.x>>5)?s[threadIdx.x]:0.f;
if(w==0)for(int o=16;o>0;o>>=1)v+=__shfl_xor_sync(0xffffffff,v,o);
if(threadIdx.x==0)s[0]=v;__syncthreads();float r=s[0];__syncthreads();return r;
}
extern "C" __global__ void gn_stats(float*mean,float*inv,const float*x,int pixels,int C,int cg,float eps){
int bg=blockIdx.x,G=C/cg,k=bg%G;long long b=bg/G;int n=pixels*cg;
const float*xs=x+b*pixels*C+k*cg;
float s=0.f;for(int i=threadIdx.x;i<n;i+=blockDim.x)s+=xs[(long long)(i/cg)*C+i%cg];
float m=img_block_sum(s)/(float)n;
float v=0.f;for(int i=threadIdx.x;i<n;i+=blockDim.x){float d=xs[(long long)(i/cg)*C+i%cg]-m;v+=d*d;}
v=img_block_sum(v)/(float)n;
if(threadIdx.x==0){mean[bg]=m;inv[bg]=1.f/sqrtf(v+eps);}
}
extern "C" __global__ void gn_apply(float*y,float*xhat,const float*x,const float*mean,const float*inv,
const float*gamma,const float*beta,int total,int pixels,int C,int cg){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,bg=(i/C/pixels)*(C/cg)+c/cg;
float h=(x[i]-mean[bg])*inv[bg];xhat[i]=h;y[i]=h*gamma[c]+beta[c];
}
extern "C" __global__ void gn_bwd_stats(float*m1,float*m2,const float*g,const float*xhat,const float*gamma,
int pixels,int C,int cg){
int bg=blockIdx.x,G=C/cg,k=bg%G;long long b=bg/G;int n=pixels*cg;
long long base=b*pixels*C+k*cg;
float a=0.f,z=0.f;
for(int i=threadIdx.x;i<n;i+=blockDim.x){long long at=base+(long long)(i/cg)*C+i%cg;
float d=g[at]*gamma[k*cg+i%cg];a+=d;z+=d*xhat[at];}
a=img_block_sum(a);z=img_block_sum(z);
if(threadIdx.x==0){m1[bg]=a/(float)n;m2[bg]=z/(float)n;}
}
extern "C" __global__ void gn_bwd_apply(float*dx,const float*g,const float*xhat,const float*inv,
const float*m1,const float*m2,const float*gamma,int total,int pixels,int C,int cg){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,bg=(i/C/pixels)*(C/cg)+c/cg;
dx[i]=inv[bg]*(g[i]*gamma[c]-m1[bg]-xhat[i]*m2[bg]);
}
__device__ float img_block_max(float v){
__shared__ float s[32];int l=threadIdx.x&31,w=threadIdx.x>>5;
for(int o=16;o>0;o>>=1)v=fmaxf(v,__shfl_xor_sync(0xffffffff,v,o));
__syncthreads();if(l==0)s[w]=v;__syncthreads();
v=threadIdx.x<(blockDim.x>>5)?s[threadIdx.x]:-3.402823466e38f;
if(w==0)for(int o=16;o>0;o>>=1)v=fmaxf(v,__shfl_xor_sync(0xffffffff,v,o));
if(threadIdx.x==0)s[0]=v;__syncthreads();float r=s[0];__syncthreads();return r;
}
__device__ float unit_coord(int i,int n){return n==1?0.f:-1.f+2.f*(float)i/(float)(n-1);}
// One block per (sample, channel): the softmax over that channel's pixels and
// its expected x and y. `out` is `[batch, 2 * C]`, every x before every y.
extern "C" __global__ void ssm_fwd(float*prob,float*out,const float*x,int h,int w,int C){
int b=blockIdx.x/C,c=blockIdx.x%C,n=h*w;long long base=(long long)b*n*C+c;
float m=-3.402823466e38f;for(int p=threadIdx.x;p<n;p+=blockDim.x)m=fmaxf(m,x[base+(long long)p*C]);
m=img_block_max(m);
float s=0.f;for(int p=threadIdx.x;p<n;p+=blockDim.x){float e=expf(x[base+(long long)p*C]-m);prob[base+(long long)p*C]=e;s+=e;}
s=img_block_sum(s);
float ex=0.f,ey=0.f;for(int p=threadIdx.x;p<n;p+=blockDim.x){long long at=base+(long long)p*C;float q=prob[at]/s;prob[at]=q;ex+=q*unit_coord(p%w,w);ey+=q*unit_coord(p/w,h);}
ex=img_block_sum(ex);ey=img_block_sum(ey);
if(threadIdx.x==0){out[b*2*C+c]=ex;out[b*2*C+C+c]=ey;}
}
extern "C" __global__ void ssm_bwd(float*dx,const float*prob,const float*g,const float*out,int total,int h,int w,int C){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,r=i/C,p=r%(h*w),b=r/(h*w);const float*gs=g+b*2*C;const float*es=out+b*2*C;
dx[i]=prob[i]*(gs[c]*(unit_coord(p%w,w)-es[c])+gs[C+c]*(unit_coord(p/w,h)-es[C+c]));
}
// One block per (sample, channel), the mean over its pixels.
extern "C" __global__ void gap_fwd(float*out,const float*x,int n,int C){
int b=blockIdx.x/C,c=blockIdx.x%C;long long base=(long long)b*n*C+c;
float s=0.f;for(int p=threadIdx.x;p<n;p+=blockDim.x)s+=x[base+(long long)p*C];
s=img_block_sum(s);if(threadIdx.x==0)out[blockIdx.x]=s/(float)n;
}
extern "C" __global__ void gap_bwd(float*dx,const float*g,int total,int n,int C){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
dx[i]=g[(i/C/n)*C+i%C]/(float)n;
}
// Batch norm over `[rows, channels]`: the column sums go through cuBLAS, the
// rest is one thread per element or channel.
extern "C" __global__ void bn_finish(float*mean,float*var,float*inv,float*run,const float*shift,int C,float eps,float momentum,float unbias,int training){
int c=blockIdx.x*blockDim.x+threadIdx.x;if(c>=C)return;
// Training sums came shifted by `shift` (col_sums' mode 1).
if(training){float s=mean[c],m=shift[c]+s;mean[c]=m;var[c]=fmaxf(var[c]-s*s,0.f);
run[c]=(1.f-momentum)*run[c]+momentum*m;run[C+c]=(1.f-momentum)*run[C+c]+momentum*var[c]*unbias;}
else{mean[c]=run[c];var[c]=run[C+c];}
inv[c]=1.f/sqrtf(var[c]+eps);
}
// `act` runs SiLU over the normalized output. The backward pass recomputes
// the normalized values from x, so they are not written out.
extern "C" __global__ void bn_apply(float*y,const float*x,const float*mean,const float*inv,
const float*gamma,const float*beta,int total,int C,int act){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C;float h=(x[i]-mean[c])*inv[c];float v=h*gamma[c]+beta[c];
y[i]=act?v/(1.f+expf(-v)):v;
}
// The gradient at the batch norm's output from the one after its SiLU,
// when `act` says a SiLU follows.
__device__ __forceinline__ float bn_gy(float d,float xhat,float gamma,float beta,int act){
if(act){float v=xhat*gamma+beta,s=1.f/(1.f+expf(-v));d*=s*(1.f+v*(1.f-s));}
return d;
}
// `scale` times each channel's sum over the rows of `[rows, C]`: of x under
// mode 0, of x - other[c] into `out` and its square into `out2` under mode 1
// (one pass for a mean and variance, shifted by a guess at the mean so the
// variance does not cancel away), and under mode 2 of x into `out`
// and of x * xhat into `out2` from the same read, xhat being
// (other - mu[c]) * inv[c], x first taken through `bn_gy` when `act` is set. One thread per (chunk of
// COL_ROWS rows, channel), channel fastest so a warp reads neighbouring
// channels; partial sums are added into a zeroed `out`.
#define COL_ROWS 64
extern "C" __global__ void col_sums(float*out,float*out2,const float*x,const float*other,
int rows,int C,float scale,int mode,const float*gamma,const float*beta,int act,
const float*mu,const float*inv){
long long i=(long long)blockIdx.x*blockDim.x+threadIdx.x;
if(i>=(long long)(rows+COL_ROWS-1)/COL_ROWS*C)return;
int c=i%C,chunk=i/C;float a=0.f,b=0.f,m=mode==1?other[c]:0.f;
float ga=act?gamma[c]:0.f,be=act?beta[c]:0.f,mc=mode==2?mu[c]:0.f,ic=mode==2?inv[c]:0.f;
for(int r=chunk*COL_ROWS,end=min(rows,r+COL_ROWS);r<end;r++){
long long at=(long long)r*C+c;float v=x[at];
if(mode==1){v-=m;b+=v*v;}else if(mode==2){float o=(other[at]-mc)*ic;v=bn_gy(v,o,ga,be,act);b+=v*o;}
a+=v;}
atomicAdd(&out[c],a*scale);
if(mode)atomicAdd(&out2[c],b*scale);
}
extern "C" __global__ void bn_bwd_apply(float*dx,const float*g,const float*input,const float*mu,
const float*inv,const float*m1,const float*m2,const float*gamma,const float*beta,int total,int C,int act){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C;float x=(input[i]-mu[c])*inv[c],d=bn_gy(g[i],x,gamma[c],beta[c],act);
dx[i]=inv[c]*gamma[c]*(d-m1[c]-x*m2[c]);
}
extern "C" __global__ void upsample_nearest(float*y,const float*x,int total,int H,int W,int C,int f){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,r=i/C,ox=r%(W*f),q=r/(W*f),oy=q%(H*f),b=q/(H*f);
y[i]=x[((long long)(b*H+oy/f)*W+ox/f)*C+c];
}
extern "C" __global__ void upsample_nearest_bwd(float*dx,const float*g,int total,int H,int W,int C,int f){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,r=i/C,x=r%W,q=r/W,y=q%H,b=q/H;float s=0.f;
for(int dy=0;dy<f;dy++)for(int dx2=0;dx2<f;dx2++)
s+=g[((long long)(b*H*f+y*f+dy)*W*f+x*f+dx2)*C+c];
dx[i]=s;
}
// `add` adds into dst instead of overwriting it.
extern "C" __global__ void copy_channels(float*dst,const float*src,int rows,int width,int dC,int doff,int sC,int soff,int add){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*width)return;
long long r=i/width;int c=i%width;float v=src[r*sC+soff+c];float*d=dst+r*dC+doff+c;*d=add?*d+v:v;
}
// Stride-one max pool padded to keep the size; `arg` is the winner's place in
// the window, the first maximum on a tie as on the host.
extern "C" __global__ void maxpool_same(float*y,int*arg,const float*x,int total,int H,int W,int C,int k){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,r=i/C,px=r%W,q=r/W,py=q%H,b=q/H,pad=k/2;float m=-3.402823466e38f;int a=0;
for(int ky=0;ky<k;ky++){int sy=py+ky-pad;if(sy<0||sy>=H)continue;
for(int kx=0;kx<k;kx++){int sx=px+kx-pad;if(sx<0||sx>=W)continue;
float v=x[((long long)(b*H+sy)*W+sx)*C+c];if(v>m){m=v;a=ky*k+kx;}}}
y[i]=m;arg[i]=a;
}
extern "C" __global__ void maxpool_same_bwd(float*dx,const float*g,const int*arg,int total,int H,int W,int C,int k){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;
int c=i%C,r=i/C,px=r%W,q=r/W,py=q%H,b=q/H,pad=k/2;float s=0.f;
for(int ky=0;ky<k;ky++){int oy=py-ky+pad;if(oy<0||oy>=H)continue;
for(int kx=0;kx<k;kx++){int ox=px-kx+pad;if(ox<0||ox>=W)continue;
long long o=((long long)(b*H+oy)*W+ox)*C+c;if(arg[o]==ky*k+kx)s+=g[o];}}
dx[i]=s;
}
// Depthwise convolution, padded by k/2. The input gradient gathers like
// col2im; the weight gradient is one block per (channel, tap).
extern "C" __global__ void dw_fwd(float*y,const float*x,const float*w,int B,int H,int W,int C,int k,int s,int OH,int OW){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=B*OH*OW*C)return;
int c=i%C,r=i/C,ox=r%OW,q=r/OW,oy=q%OH,b=q/OH,pad=k/2;float a=0.f;
for(int t=0;t<k*k;t++){int sy=oy*s+t/k-pad,sx=ox*s+t%k-pad;
if(sy>=0&&sy<H&&sx>=0&&sx<W)a+=x[((long long)(b*H+sy)*W+sx)*C+c]*w[c*k*k+t];}
y[i]=a;
}
extern "C" __global__ void dw_bwd_input(float*dx,const float*g,const float*w,int B,int H,int W,int C,int k,int s,int OH,int OW){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=B*H*W*C)return;
int c=i%C,r=i/C,px=r%W,q=r/W,py=q%H,b=q/H,pad=k/2;float a=0.f;
for(int t=0;t<k*k;t++){int sy=py+pad-t/k,sx=px+pad-t%k;
if(sy<0||sx<0||sy%s||sx%s)continue;int oy=sy/s,ox=sx/s;if(oy>=OH||ox>=OW)continue;
a+=g[((long long)(b*OH+oy)*OW+ox)*C+c]*w[c*k*k+t];}
dx[i]=a;
}
// One thread per (chunk of DW_ROWS output pixels, tap, channel), channel
// fastest so a warp reads neighbouring channels; partial sums are added into
// a zeroed `dw`.
#define DW_ROWS 64
extern "C" __global__ void dw_bwd_weight(float*dw,const float*g,const float*x,int B,int H,int W,int C,int k,int s,int OH,int OW){
long long i=(long long)blockIdx.x*blockDim.x+threadIdx.x;int kk=k*k,n=B*OH*OW;
if(i>=(long long)(n+DW_ROWS-1)/DW_ROWS*kk*C)return;
int c=i%C,t=(i/C)%kk,chunk=i/((long long)C*kk),pad=k/2,dy=t/k-pad,dx=t%k-pad;float a=0.f;
for(int r=chunk*DW_ROWS,end=min(n,r+DW_ROWS);r<end;r++){int ox=r%OW,q=r/OW,oy=q%OH,b=q/OH;
int sy=oy*s+dy,sx=ox*s+dx;
if(sy>=0&&sy<H&&sx>=0&&sx<W)a+=g[(long long)r*C+c]*x[((long long)(b*H+sy)*W+sx)*C+c];}
if(a!=0.f)atomicAdd(&dw[c*kk+t],a);
}
extern "C" __global__ void add_f32(float*out,const float*a,const float*b,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=a[i]+b[i];}
extern "C" __global__ void mul_f32(float*out,const float*a,const float*b,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=a[i]*b[i];}
extern "C" __global__ void silu_f32(float*y,const float*x,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float v=x[i];y[i]=v/(1.f+expf(-v));}}
extern "C" __global__ void silu_bwd_f32(float*dx,const float*g,const float*x,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float v=x[i],s=1.f/(1.f+expf(-v));dx[i]=g[i]*s*(1.f+v*(1.f-s));}}
extern "C" __global__ void scatter_rows_neg(float*grad,const float*upstream,const unsigned int*rows_of,int rows,int width){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<rows*width){int r=i/width,c=i%width;atomicAdd(&grad[rows_of[r]*width+c],-upstream[i]);}}
// The transformer path's fused kernels. Everything a decoder layer does
// between two matmuls lives here, so a layer runs device-side end to end: see
// `gpu_model` for how they are sequenced.
// One block of ROW_THREADS threads cooperates on one row, striding over the
// columns so neighbouring threads touch neighbouring addresses. The previous
// one-thread-per-row shape read a row serially, which made every access
// uncoalesced: adjacent threads were `cols` floats apart. Rows outnumber
// blocks, so each block walks a grid-stride slice of them.
#define ROW_THREADS 256
__device__ __forceinline__ float row_sum(float v,float*red){
int tid=threadIdx.x;red[tid]=v;__syncthreads();
for(int s=ROW_THREADS/2;s>0;s>>=1){if(tid<s)red[tid]+=red[tid+s];__syncthreads();}
float r=red[0];__syncthreads();return r;
}
// `do_copy` asks for a plain duplicate of `x` in `copy`. A residual branch
// needs one: the projection below it accumulates with `beta = 1` over the
// block input. Writing it here is a store on a row this kernel has already
// read, which is cheaper than the separate device-to-device copy it replaces.
extern "C" __global__ void rmsnorm_fwd(void*out,float*inv,const float*x,const float*w,int rows,int cols,float eps,int narrow,float*copy,int do_copy){
__shared__ float red[ROW_THREADS];int tid=threadIdx.x;
for(int r=blockIdx.x;r<rows;r+=gridDim.x){
const float*src=x+(size_t)r*cols;
float*dup=copy+(size_t)r*cols;
float s=0.f;for(int c=tid;c<cols;c+=ROW_THREADS){float v=src[c];if(do_copy)dup[c]=v;s+=v*v;}
s=row_sum(s,red);
float t=1.f/sqrtf(s/(float)cols+eps);
if(tid==0)inv[r]=t;
size_t base=(size_t)r*cols;
for(int c=tid;c<cols;c+=ROW_THREADS)store_act(out,base+c,src[c]*t*w[c],narrow);
}
}
// `use_smem` says the host sized the dynamic shared memory to hold `cols`
// floats, so the scale gradient accumulates per block and lands in `gw` with
// one atomic per column instead of one per element. A model wide enough to
// overflow shared memory falls back to the direct atomic.
extern "C" __global__ void rmsnorm_bwd(float*gx,float*gw,const float*x,const float*gy,const float*w,const float*inv,int rows,int cols,int use_smem,const float*res,int add_res,void*copy,int copy_mode){
extern __shared__ float acc[];
__shared__ float red[ROW_THREADS];int tid=threadIdx.x;
if(use_smem)for(int c=tid;c<cols;c+=ROW_THREADS)acc[c]=0.f;
__syncthreads();
for(int r=blockIdx.x;r<rows;r+=gridDim.x){
size_t base=(size_t)r*cols;
const float*src=x+base;const float*up=gy+base;float t=inv[r];
float proj=0.f;for(int c=tid;c<cols;c+=ROW_THREADS)proj+=up[c]*w[c]*src[c];
proj=row_sum(proj,red);
float shared=proj*t*t*t/(float)cols;
float*dst=gx+base;
for(int c=tid;c<cols;c+=ROW_THREADS){
float g_in=up[c]*w[c]*t-src[c]*shared;
// The residual carries the upstream gradient past the branch, so it is
// one more addend here rather than a separate pass over the whole tensor.
if(add_res)g_in+=res[base+c];
dst[c]=g_in;
// The GEMMs downstream want this gradient as an operand, so the copy
// they read is written here instead of by a separate pass.
if(copy_mode)store_act(copy,base+c,g_in,copy_mode-1);
float g=up[c]*src[c]*t;
if(use_smem)acc[c]+=g;else atomicAdd(&gw[c],g);
}
}
if(use_smem){__syncthreads();for(int c=tid;c<cols;c+=ROW_THREADS)atomicAdd(&gw[c],acc[c]);}
}
// `width` and `off` locate the rotated block inside a wider row: queries and
// keys are two slices of one fused projection output, so they share a row
// stride and differ only in where they start.
extern "C" __global__ void rope_rotate(void*x,const float*cs,const float*sn,int rows,int heads,int head_dim,int seq_len,float dir,int width,int off,int narrow,int per){int half=head_dim/2;int total=rows*heads*half;int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=total)return;int ch=i%half;int h=(i/half)%heads;int r=i/(half*heads);int t=((r/per)%seq_len)*half+ch;float c=cs[t];float s=dir*sn[t];size_t base=(size_t)r*width+off+(size_t)h*head_dim+ch;float lo=load_act(x,base,narrow);float hi=load_act(x,base+half,narrow);store_act(x,base,lo*c-hi*s,narrow);store_act(x,base+half,hi*c+lo*s,narrow);}
// Attention rows are short (one per query position, and only the positions up
// to it are visible), so a whole block per row would leave most of its threads
// idle. One warp per row instead, reducing through shuffles with no shared
// memory and no block-wide barrier.
#define WARP 32
__device__ __forceinline__ float warp_max(float v){
for(int o=16;o>0;o>>=1)v=fmaxf(v,__shfl_down_sync(0xffffffff,v,o));
return __shfl_sync(0xffffffff,v,0);
}
__device__ __forceinline__ float warp_sum(float v){
for(int o=16;o>0;o>>=1)v+=__shfl_down_sync(0xffffffff,v,o);
return __shfl_sync(0xffffffff,v,0);
}
// A row short enough to sit in registers is read once and written once instead
// of streamed three times for the maximum, the exponential and the scale. Each
// lane holds `SOFTMAX_ROW` of the row's values, so the bound is a register
// budget rather than a shape: 512 columns at 32 lanes. A longer row falls back
// to the streaming loop below, which is the same arithmetic over more passes.
//
// The shape decoder's cross-attention is exactly at the bound --- 512 latents
// --- and its score matrix is the largest buffer in the step, so this is where
// the three-pass version cost the most.
#define SOFTMAX_ROW 16
// Row softmax in place, plus the one log-sum-exp per query the backward pass
// needs to rebuild these same probabilities. Writing that number here is what
// lets the forward cache hold a few kilobytes per layer instead of the whole
// probability matrix.
//
// `causal` masks each query to the keys at or before it, which is what a
// decoder wants. Cross-attention clears the flag: it has two independent
// lengths and every query sees every key, so `cols` is the key count rather
// than the query count and the row index carries no position at all.
// Scores `[heads, sequences, queries, cols]` after `offset` cached positions:
// a key whose slot is padding is pushed out of the softmax, except a padding
// query's own key, so its row still has something to normalise.
extern "C" __global__ void mask_padding(float*s,const int*slots,int rows,int cols,int queries,int sequences,int offset){
long i=(long)blockIdx.x*blockDim.x+threadIdx.x;if(i>=(long)rows*cols)return;
int r=i/cols,j=i-(long)r*cols,q=r%queries,seq=(r/queries)%sequences;
if(j!=offset+q&&slots[(long)j*sequences+seq]==0)s[i]=-3.0e38f;
}
extern "C" __global__ void attention_softmax_lse(float*s,float*lse,int rows,int cols,int causal,int queries,int offset){
int lane=threadIdx.x%WARP,warps=blockDim.x/WARP;
int stride=gridDim.x*warps;
for(int i=blockIdx.x*warps+threadIdx.x/WARP;i<rows;i+=stride){
int seq_len=cols;
int vis=causal?offset+i%queries+1:cols;float*row=s+(size_t)i*seq_len;
if(vis<=WARP*SOFTMAX_ROW){
float v[SOFTMAX_ROW];float m=-3.0e38f;
#pragma unroll
for(int k=0;k<SOFTMAX_ROW;k++){int j=lane+k*WARP;v[k]=j<vis?row[j]:-3.0e38f;m=fmaxf(m,v[k]);}
m=warp_max(m);
float part=0.f;
#pragma unroll
for(int k=0;k<SOFTMAX_ROW;k++){int j=lane+k*WARP;v[k]=j<vis?__expf(v[k]-m):0.f;part+=v[k];}
float sum=warp_sum(part);
if(lane==0)lse[i]=m+__logf(sum);
float inv=sum>0.f?1.f/sum:1.f;
#pragma unroll
for(int k=0;k<SOFTMAX_ROW;k++){int j=lane+k*WARP;if(j<vis)row[j]=v[k]*inv;}
for(int j=vis+lane;j<seq_len;j+=WARP)row[j]=0.f;
continue;
}
float m=-3.0e38f;for(int j=lane;j<vis;j+=WARP)m=fmaxf(m,row[j]);
m=warp_max(m);
float part=0.f;for(int j=lane;j<vis;j+=WARP){float e=__expf(row[j]-m);row[j]=e;part+=e;}
float sum=warp_sum(part);
if(lane==0)lse[i]=m+__logf(sum);
if(sum>0.f){float inv=1.f/sum;for(int j=lane;j<vis;j+=WARP)row[j]*=inv;}
for(int j=vis+lane;j<seq_len;j+=WARP)row[j]=0.f;
}
}
// Decode attention for a few new rows per sequence, split the
// flash-decoding way so a batch-1 step still fills the card. Block
// (kv head, row, chunk) scores DECODE_CHUNK positions of the history for
// every query head sharing that kv head, so each key and value is loaded once
// for the whole group. Row r of the `queries * sequences` new rows is
// sequence r % sequences at the r / sequences-th new position, and sees the
// history up to and including itself. Queries are FP32 rows `q_width` apart;
// keys and values are read in place from the device cache, rows `kv_width`
// apart and BF16 when `narrow`, position p of sequence `seq` at row
// p*sequences+seq. `slots` at the same index is zero where that position is
// padding, which no query attends to. It writes, per query head, its running
// max, its exp-sum and its unnormalised weighted sum of values;
// `decode_attention_merge` rescales and adds the chunks. Launch with
// DECODE_THREADS threads and `group` at most DECODE_GROUP.
#define DECODE_CHUNK 64
#define DECODE_GROUP 8
#define DECODE_THREADS 256
extern "C" __global__ void decode_attention(const float*q,const void*kv,float*partial,const int*slots,int history,int queries,int sequences,int q_width,int kv_width,int head_dim,int group,int key_base,int value_base,float scale,int narrow){
__shared__ float s[DECODE_GROUP][DECODE_CHUNK];
__shared__ float red[DECODE_GROUP][2];
__shared__ float part[DECODE_GROUP*DECODE_THREADS];
int kvh=blockIdx.x,row=blockIdx.y,chunk=blockIdx.z,chunks=gridDim.z,heads=gridDim.x*group;
int seq=row%sequences,seen=history-queries+row/sequences+1;
int first=chunk*DECODE_CHUNK,count=min(DECODE_CHUNK,seen-first);
int lane=threadIdx.x%WARP,warp=threadIdx.x/WARP;
long row_stride=(long)sequences*kv_width;
q+=(long)row*q_width+kvh*group*head_dim;
long k=(long)seq*kv_width+key_base+kvh*head_dim+first*row_stride;
long v=(long)seq*kv_width+value_base+kvh*head_dim+first*row_stride;
// A few threads per position, so every key load is in flight at once.
{int per=DECODE_THREADS/DECODE_CHUNK,p=threadIdx.x/per,sub=threadIdx.x%per;float d[DECODE_GROUP];
#pragma unroll
for(int g=0;g<DECODE_GROUP;g++)d[g]=0.f;
if(p<count){long kp=k+p*row_stride;
#pragma unroll 4
for(int i=sub;i<head_dim;i+=per){float key=load_act(kv,kp+i,narrow);
#pragma unroll
for(int g=0;g<DECODE_GROUP;g++)if(g<group)d[g]+=q[g*head_dim+i]*key;}}
bool live=p<count&&slots[(long)(first+p)*sequences+seq];
#pragma unroll
for(int g=0;g<DECODE_GROUP;g++){float x=d[g];
for(int o=per/2;o>0;o>>=1)x+=__shfl_xor_sync(0xffffffff,x,o);
if(g<group&&sub==0&&p<count)s[g][p]=live?x*scale:-3.0e38f;}}
__syncthreads();
if(warp<group){
float m=-3.0e38f;for(int p=lane;p<count;p+=WARP)m=fmaxf(m,s[warp][p]);
m=warp_max(m);
float sum=0.f;for(int p=lane;p<count;p+=WARP){float e=__expf(s[warp][p]-m);s[warp][p]=e;sum+=e;}
sum=warp_sum(sum);
if(lane==0){red[warp][0]=m;red[warp][1]=sum;}
}
__syncthreads();
int slices=DECODE_THREADS/head_dim,t=threadIdx.x%head_dim,slice=threadIdx.x/head_dim;
float acc[DECODE_GROUP];
#pragma unroll
for(int g=0;g<DECODE_GROUP;g++)acc[g]=0.f;
if(slice<slices){
#pragma unroll 8
for(int p=slice;p<count;p+=slices){float value=load_act(kv,v+p*row_stride+t,narrow);
#pragma unroll
for(int g=0;g<DECODE_GROUP;g++)if(g<group)acc[g]+=s[g][p]*value;}}
#pragma unroll
for(int g=0;g<DECODE_GROUP;g++)if(g<group)part[g*DECODE_THREADS+threadIdx.x]=acc[g];
__syncthreads();
for(int g=0;g<group;g++){
float*out=partial+((long)(row*heads+kvh*group+g)*chunks+chunk)*(head_dim+2);
if(threadIdx.x<head_dim){
float total=0.f;for(int i=0;i<slices;i++)total+=part[g*DECODE_THREADS+i*head_dim+threadIdx.x];
out[2+threadIdx.x]=total;
}
if(threadIdx.x==0){out[0]=red[g][0];out[1]=red[g][1];}
}
}
// `dst` row r, `cols` wide and `dst_width` apart, takes `src` row r from
// column `src_off`, rows `src_width` apart, narrowed when `narrow`.
extern "C" __global__ void store_rows(const float*src,void*dst,int rows,int cols,int src_width,int src_off,int dst_width,int narrow){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*cols)return;
int r=i/cols,c=i%cols;
store_act(dst,(size_t)r*dst_width+c,src[(size_t)r*src_width+src_off+c],narrow);
}
// Block (head, row), one thread per output element.
extern "C" __global__ void decode_attention_merge(const float*partial,void*out,int chunks,int head_dim,int query_width,int narrow){
int head=blockIdx.x,seq=blockIdx.y,t=threadIdx.x;
if(t>=head_dim)return;
const float*in=partial+(long)(seq*gridDim.x+head)*chunks*(head_dim+2);
float m=-3.0e38f;for(int c=0;c<chunks;c++)m=fmaxf(m,in[c*(head_dim+2)]);
float sum=0.f,acc=0.f;
for(int c=0;c<chunks;c++){const float*p=in+c*(head_dim+2);float w=__expf(p[0]-m);sum+=w*p[1];acc+=w*p[2+t];}
store_act(out,(size_t)seq*query_width+head*head_dim+t,acc/sum,narrow);
}
// Weight-only int8 for device decode, the scheme `src/quantized.rs` uses on
// the host: per row, `scale = absmax / 127` and `round(w / scale)` as int8.
// One warp per row.
extern "C" __global__ void quantize_rows_q8(const float*w,signed char*q,float*scales,int rows,int cols){
int row=(blockIdx.x*blockDim.x+threadIdx.x)/WARP,lane=threadIdx.x%WARP;
if(row>=rows)return;
const float*in=w+(size_t)row*cols;
float m=0.f;for(int k=lane;k<cols;k+=WARP)m=fmaxf(m,fabsf(in[k]));
for(int o=WARP/2;o>0;o>>=1)m=fmaxf(m,__shfl_xor_sync(0xffffffff,m,o));
float scale=m/127.f,inv=scale>0.f?1.f/scale:0.f;
for(int k=lane;k<cols;k+=WARP)q[(size_t)row*cols+k]=(signed char)fminf(fmaxf(rintf(in[k]*inv),-127.f),127.f);
if(lane==0)scales[row]=scale;
}
// The same with one scale per `group` columns of a row, `groups = ceil(cols /
// group)` scales a row. One warp per group.
extern "C" __global__ void quantize_groups_q8(const float*w,signed char*q,float*scales,int rows,int cols,int group){
int groups=(cols+group-1)/group;
long g=((long)blockIdx.x*blockDim.x+threadIdx.x)/WARP;int lane=threadIdx.x%WARP;
if(g>=(long)rows*groups)return;
int row=g/groups,start=(g%groups)*group,end=min(start+group,cols);
const float*in=w+(size_t)row*cols;
float m=0.f;for(int k=start+lane;k<end;k+=WARP)m=fmaxf(m,fabsf(in[k]));
for(int o=WARP/2;o>0;o>>=1)m=fmaxf(m,__shfl_xor_sync(0xffffffff,m,o));
float scale=m/127.f,inv=scale>0.f?1.f/scale:0.f;
for(int k=start+lane;k<end;k+=WARP)q[(size_t)row*cols+k]=(signed char)fminf(fmaxf(rintf(in[k]*inv),-127.f),127.f);
if(lane==0)scales[g]=scale;
}
extern "C" __global__ void dequantize_groups_q8(const signed char*q,const float*scales,void*out,int units,int inner,int narrow,int group){
size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x;if(i>=(size_t)units*inner)return;
int groups=(inner+group-1)/group;
store_act(out,i,(float)q[i]*scales[i/inner*groups+(i%inner)/group],narrow);
}
// `out = w * scale` per element, for a prompt, which takes the cuBLAS path.
extern "C" __global__ void dequantize_q8(const signed char*q,const float*scales,void*out,int units,int inner,int narrow){
size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x;if(i>=(size_t)units*inner)return;
store_act(out,i,(float)q[i]*scales[i/inner],narrow);
}
// FP8 E4M3 (`e4m3fn`: no infinities, 0x7f and 0xff the only NaN) by its bits,
// the value `safetensors::f8_to_f32` decodes on the host, times a scale per
// row. sm_86 has no FP8 arithmetic, so a prompt widens the weight first.
__device__ __forceinline__ float e4m3_to_f32(unsigned char b){
unsigned e=(b>>3)&15u,m=b&7u;float v;
if(e==0u)v=(float)m*0.001953125f;
else if(e==15u&&m==7u)return __int_as_float(0x7fc00000);
else v=__uint_as_float(((e+120u)<<23)|(m<<20));
return (b&0x80)?-v:v;
}
extern "C" __global__ void dequantize_fp8(const unsigned char*q,const float*scales,void*out,int units,int inner,int narrow){
size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x;if(i>=(size_t)units*inner)return;
store_act(out,i,e4m3_to_f32(q[i])*scales[i/inner],narrow);
}
// `quantized::f32_to_e4m3`, op for op: round to nearest even, saturate at 448,
// NaN to 0x7f.
__device__ __forceinline__ unsigned char f32_to_e4m3(float x){
if(isnan(x))return 0x7f;
unsigned sign=signbit(x)?0x80u:0u;float a=fabsf(x);
if(a>=448.f)return sign|0x7eu;
int octave=max((int)(__float_as_uint(a)>>23)-127,-6);
int steps=(int)rintf(a*__uint_as_float((unsigned)(130-octave)<<23));
return sign|min(((octave+7)<<3)+steps-8,0x7e);
}
// One block a row: the activations of the sm_89+ FP8 GEMM (`cuda_fp8.rs`) as
// `Quantized::from_matrix_fp8` stores a weight, scale = absmax / 448 and a
// zero row a zero scale. No FP8 instruction, so it runs (and is tested) on
// sm_86 too.
extern "C" __global__ void quantize_rows_e4m3(const void*x,int narrow,int inner,unsigned char*q,float*scales){
size_t base=(size_t)blockIdx.x*inner;float m=0.f;
for(int c=threadIdx.x;c<inner;c+=blockDim.x)m=fmaxf(m,fabsf(load_act(x,base+c,narrow)));
float scale=img_block_max(m)/448.f;
if(threadIdx.x==0)scales[blockIdx.x]=scale;
for(int c=threadIdx.x;c<inner;c+=blockDim.x)
q[base+c]=scale>0.f?f32_to_e4m3(load_act(x,base+c,narrow)/scale):0;
}
// The 4-bit layout of `quantized.rs`: rows padded to blocks of 64, byte j of a
// block holding weight j low and weight j+32 high, one scale a block, and the
// codebook's 16 levels in `levels`.
extern "C" __global__ void dequantize_q4(const unsigned char*packed,const float*scales,const float*levels,void*out,int units,int inner,int narrow){
size_t i=(size_t)blockIdx.x*blockDim.x+threadIdx.x;if(i>=(size_t)units*inner)return;
size_t unit=i/inner;int k=i%inner,blocks=(inner+63)/64,block=k/64,offset=k%64;
unsigned char byte=packed[(unit*blocks+block)*32+offset%32];
store_act(out,i,levels[(byte>>(offset<32?0:4))&15]*scales[unit*blocks+block],narrow);
}
// `y[r, n] = scales[n] * (x[r, ..] . q[n, ..]) + beta * y[r, n]`, one warp per
// output unit. A lane reads 16 weights at a time and keeps them for up to
// four rows of `x`, so a decode step reads each weight byte once. `inner` is a
// multiple of 16; the caller sends other widths through `dequantize_q8`.
#define Q8_ROWS 4
extern "C" __global__ void gemv_q8(const signed char*q,const float*scales,const void*x,int narrow,float*y,int units,int inner,int rows,float beta){
int unit=(blockIdx.x*blockDim.x+threadIdx.x)/WARP,lane=threadIdx.x%WARP;
if(unit>=units)return;
const signed char*w=q+(size_t)unit*inner;
float scale=scales[unit];
for(int r0=0;r0<rows;r0+=Q8_ROWS){
int nr=min(Q8_ROWS,rows-r0);
float acc[Q8_ROWS]={0.f,0.f,0.f,0.f};
{
for(int k=lane*16;k<inner;k+=WARP*16){
int4 packed=*(const int4*)(w+k);const signed char*b=(const signed char*)&packed;
#pragma unroll
for(int j=0;j<Q8_ROWS;j++){if(j>=nr)break;size_t base=(size_t)(r0+j)*inner+k;float v[16];
if(narrow){const uint4*p=(const uint4*)((const unsigned short*)x+base);uint4 h[2]={p[0],p[1]};const unsigned short*u=(const unsigned short*)h;
#pragma unroll
for(int e=0;e<16;e++)v[e]=bf16_to_f32(u[e]);}
else{const float4*p=(const float4*)((const float*)x+base);
#pragma unroll
for(int e=0;e<4;e++){float4 f=p[e];v[4*e]=f.x;v[4*e+1]=f.y;v[4*e+2]=f.z;v[4*e+3]=f.w;}}
float a=0.f;
#pragma unroll
for(int e=0;e<16;e++)a+=v[e]*(float)b[e];
acc[j]+=a;}
}
}
#pragma unroll
for(int j=0;j<Q8_ROWS;j++){float a=acc[j];for(int o=WARP/2;o>0;o>>=1)a+=__shfl_xor_sync(0xffffffff,a,o);
if(lane==0&&j<nr){size_t i=(size_t)(r0+j)*units+unit;y[i]=beta==0.f?a*scale:a*scale+beta*y[i];}}
}
}
extern "C" __global__ void attention_softmax_bwd(float*g,const float*p,int rows,int cols,int causal){
int lane=threadIdx.x%WARP,warps=blockDim.x/WARP;
int stride=gridDim.x*warps;
for(int i=blockIdx.x*warps+threadIdx.x/WARP;i<rows;i+=stride){
int seq_len=cols;
int vis=causal?i%cols+1:cols;float*gr=g+(size_t)i*seq_len;const float*pr=p+(size_t)i*seq_len;
if(vis<=WARP*SOFTMAX_ROW){
float pv[SOFTMAX_ROW],gv[SOFTMAX_ROW];float part=0.f;
#pragma unroll
for(int k=0;k<SOFTMAX_ROW;k++){int j=lane+k*WARP;int in=j<vis;pv[k]=in?pr[j]:0.f;gv[k]=in?gr[j]:0.f;part+=pv[k]*gv[k];}
float dot=warp_sum(part);
#pragma unroll
for(int k=0;k<SOFTMAX_ROW;k++){int j=lane+k*WARP;if(j<vis)gr[j]=pv[k]*(gv[k]-dot);}
for(int j=vis+lane;j<seq_len;j+=WARP)gr[j]=0.f;
continue;
}
float part=0.f;for(int j=lane;j<vis;j+=WARP)part+=pr[j]*gr[j];
float dot=warp_sum(part);
for(int j=lane;j<vis;j+=WARP)gr[j]=pr[j]*(gr[j]-dot);
for(int j=vis+lane;j<seq_len;j+=WARP)gr[j]=0.f;
}
}
// Rebuild the attention probabilities from the scaled scores and the stored
// log-sum-exp. The forward pass kept only that one number per query, so the
// backward pass pays an exponential per element instead of a second reduction,
// and the probability matrix lives only for the length of one block's backward.
extern "C" __global__ void attention_probs_from_lse(
float*s,const float*lse,int rows,int cols,int causal){
int i=blockIdx.x*blockDim.x+threadIdx.x;
if(i>=rows*cols)return;
int r=i/cols,c=i-r*cols;
s[i]=(!causal||c<=r%cols)?__expf(s[i]-lse[r]):0.f;
}
// Gate and up share an input and a shape, so they are one GEMM at twice the
// width: `gu` holds both halves of every row, gate first. cuBLAS is close to
// twice as fast on the wider shape, which is why the two projections are not
// kept apart here.
__device__ inline float silu_mul(float x,float u){return x*u/(1.f+expf(-x));}
// `gelu` picks the tanh GELU on the first half instead, the gated feed-forward
// T5 1.1 trains.
__device__ inline float gate_mul(float x,float u,int gelu){if(!gelu)return silu_mul(x,u);return 0.5f*x*(1.f+tanhf(0.7978846f*(x+0.044715f*x*x*x)))*u;}
__device__ inline void silu_mul_grad(float x,float u,float p,float&dg,float&du){float s=1.f/(1.f+expf(-x));dg=p*u*s*(1.f+x*(1.f-s));du=p*x*s;}
// These two are memory bound, so an even width is read and written four floats
// at a time. The host picks the thread count to match; the scalar tail below
// serves an odd width, which no preset uses but a caller may configure.
extern "C" __global__ void swiglu_fwd(void*h,const float*gu,int rows,int width,int narrow,int gelu){int i=blockIdx.x*blockDim.x+threadIdx.x;
if((width&3)==0){int q=width>>2;if(i>=rows*q)return;int r=i/q;size_t b=(size_t)r*2*q+(i-r*q);const float4*s=(const float4*)gu;float4 g=s[b],u=s[b+q];float4 o;o.x=gate_mul(g.x,u.x,gelu);o.y=gate_mul(g.y,u.y,gelu);o.z=gate_mul(g.z,u.z,gelu);o.w=gate_mul(g.w,u.w,gelu);store_act4(h,i,o,narrow);return;}
if(i>=rows*width)return;int r=i/width;size_t b=(size_t)r*2*width+(i-r*width);store_act(h,i,gate_mul(gu[b],gu[b+width],gelu),narrow);}
extern "C" __global__ void swiglu_bwd(void*ggu,const float*gu,const float*gh,int rows,int width,int narrow){int i=blockIdx.x*blockDim.x+threadIdx.x;
if((width&3)==0){int q=width>>2;if(i>=rows*q)return;int r=i/q;size_t b=(size_t)r*2*q+(i-r*q);const float4*s=(const float4*)gu;float4 g=s[b],u=s[b+q],p=((const float4*)gh)[i];float4 dg,du;silu_mul_grad(g.x,u.x,p.x,dg.x,du.x);silu_mul_grad(g.y,u.y,p.y,dg.y,du.y);silu_mul_grad(g.z,u.z,p.z,dg.z,du.z);silu_mul_grad(g.w,u.w,p.w,dg.w,du.w);store_act4(ggu,b,dg,narrow);store_act4(ggu,b+q,du,narrow);return;}
if(i>=rows*width)return;int r=i/width;size_t b=(size_t)r*2*width+(i-r*width);float dg,du;silu_mul_grad(gu[b],gu[b+width],gh[i],dg,du);store_act(ggu,b,dg,narrow);store_act(ggu,b+width,du,narrow);}
// AdaLN, in the shape `src/adaln.rs` describes: the conditioning is one row
// per sequence and every token of that sequence reads it. A row of `triple` is
// `[shift, scale, gate]`, each `cols` wide, so the gate the residual below
// needs is the same buffer at offset `2 * cols` rather than a tensor of its
// own.
extern "C" __global__ void adaln_modulate(float*out,const float*x,const float*triple,int rows,int cols,int seq_len){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*cols)return;int r=i/cols,c=i-r*cols;
const float*t=triple+(size_t)(r/seq_len)*3*cols;out[i]=x[i]*(1.f+t[cols+c])+t[c];}
// The shift and scale gradients are sums over a sequence's tokens, so they land
// with atomics: the whole conditioning gradient is one zeroed buffer that every
// sub-layer of every block accumulates into, and it is downloaded once.
extern "C" __global__ void adaln_modulate_bwd(float*gx,float*gt,const float*gy,const float*x,const float*triple,int rows,int cols,int seq_len){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*cols)return;int r=i/cols,c=i-r*cols;
size_t base=(size_t)(r/seq_len)*3*cols;float u=gy[i];
gx[i]=u*(1.f+triple[base+cols+c]);
atomicAdd(>[base+c],u);atomicAdd(>[base+cols+c],u*x[i]);}
// `out = residual + gate * branch`, the other half of an AdaLN sub-layer.
extern "C" __global__ void gate_residual_fwd(float*out,const float*residual,const float*branch,const float*triple,int rows,int cols,int seq_len){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*cols)return;int r=i/cols,c=i-r*cols;
out[i]=residual[i]+triple[(size_t)(r/seq_len)*3*cols+2*cols+c]*branch[i];}
// `dL/dresidual` is the upstream gradient unchanged, so it is not written: the
// caller already holds that buffer.
extern "C" __global__ void gate_residual_bwd(float*gb,float*gt,const float*branch,const float*triple,const float*gout,int rows,int cols,int seq_len){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*cols)return;int r=i/cols,c=i-r*cols;
size_t gate=(size_t)(r/seq_len)*3*cols+2*cols+c;float u=gout[i];
gb[i]=u*triple[gate];atomicAdd(>[gate],u*branch[i]);}
extern "C" __global__ void softmax_lse(float*p,float*lse,const float*x,int rows,int cols){int r=blockIdx.x*blockDim.x+threadIdx.x;if(r>=rows)return;const float*src=x+(size_t)r*cols;float*dst=p+(size_t)r*cols;float m=-3.0e38f;for(int c=0;c<cols;c++)m=fmaxf(m,src[c]);float sum=0;for(int c=0;c<cols;c++){float e=expf(src[c]-m);dst[c]=e;sum+=e;}lse[r]=m+logf(sum);if(sum>0)for(int c=0;c<cols;c++)dst[c]/=sum;}
// A decode step's routed experts without the host: one warp per output unit
// `n` of output row `o`, summing over that row's `per` slots `s = o*per + j`
// the product of expert `expert_of[s]`'s weight row `n` with input row
// `s / x_div`, times `gate_of[s]` when `gated`. A slot routed nowhere (`-1`)
// adds nothing. The gate/up product is `per = 1, x_div = top_k`, one output
// row per slot; the down product is `per = top_k, x_div = 1` with the gates,
// one output row per token.
extern "C" __global__ void gemv_routed(const void*w,const void*x,int narrow,float*y,const int*expert_of,const float*gate_of,int gated,int outs,int per,int x_div,int units,int inner,float beta){
int warp=(blockIdx.x*blockDim.x+threadIdx.x)/WARP,lane=threadIdx.x%WARP;
if(warp>=outs*units)return;
int o=warp/units,n=warp%units;float acc=0.f;
for(int j=0;j<per;j++){int s=o*per+j,e=expert_of[s];if(e<0)continue;
size_t wrow=((size_t)e*units+n)*inner,xrow=(size_t)(s/x_div)*inner;float a=0.f;
if(narrow&&(inner&7)==0){const unsigned short*wp=(const unsigned short*)w+wrow,*xp=(const unsigned short*)x+xrow;
for(int k=lane*8;k<inner;k+=WARP*8){uint4 wv=*(const uint4*)(wp+k),xv=*(const uint4*)(xp+k);const unsigned short*wu=(const unsigned short*)&wv,*xu=(const unsigned short*)&xv;
#pragma unroll
for(int t=0;t<8;t++)a+=bf16_to_f32(wu[t])*bf16_to_f32(xu[t]);}}
else for(int k=lane;k<inner;k+=WARP)a+=load_act(w,wrow+k,narrow)*load_act(x,xrow+k,narrow);
acc+=gated?gate_of[s]*a:a;}
acc=warp_sum(acc);
if(lane==0){size_t i=(size_t)o*units+n;y[i]=acc+(beta!=0.f?beta*y[i]:0.f);}
}
// The same over int8 experts with a scale per row, the `gemv_q8` layout:
// `w` is `[experts * units, inner]` bytes and `inner` a multiple of 16.
extern "C" __global__ void gemv_routed_q8(const signed char*w,const float*scales,const void*x,int narrow,float*y,const int*expert_of,const float*gate_of,int gated,int outs,int per,int x_div,int units,int inner,float beta){
int warp=(blockIdx.x*blockDim.x+threadIdx.x)/WARP,lane=threadIdx.x%WARP;
if(warp>=outs*units)return;
int o=warp/units,n=warp%units;float acc=0.f;
for(int j=0;j<per;j++){int s=o*per+j,e=expert_of[s];if(e<0)continue;
size_t row=(size_t)e*units+n,xrow=(size_t)(s/x_div)*inner;float a=0.f;
for(int k=lane*16;k<inner;k+=WARP*16){int4 packed=*(const int4*)(w+row*inner+k);const signed char*b=(const signed char*)&packed;
#pragma unroll
for(int t=0;t<16;t++)a+=load_act(x,xrow+k+t,narrow)*(float)b[t];}
a*=scales[row];acc+=gated?gate_of[s]*a:a;}
acc=warp_sum(acc);
if(lane==0){size_t i=(size_t)o*units+n;y[i]=acc+(beta!=0.f?beta*y[i]:0.f);}
}
extern "C" __global__ void topk_gate(int*expert_of,float*gate_of,const float*p,const int*valid,int rows,int experts,int top_k){int r=blockIdx.x*blockDim.x+threadIdx.x;if(r>=rows)return;int base=r*top_k;if(valid!=0&&valid[r]==0){for(int k=0;k<top_k;k++){expert_of[base+k]=-1;gate_of[base+k]=0;}return;}const float*row=p+(size_t)r*experts;float total=0;for(int k=0;k<top_k;k++){int best=-1;for(int e=0;e<experts;e++){int taken=0;for(int j=0;j<k;j++)if(expert_of[base+j]==e)taken=1;if(taken)continue;if(best<0||row[e]>row[best])best=e;}expert_of[base+k]=best;gate_of[base+k]=row[best];total+=row[best];}if(total>0)for(int k=0;k<top_k;k++)gate_of[base+k]/=total;}
// The routed layer's gathers: rows of `src` by `rows_of`, times `scale` if
// `scaled`, read and written at either precision so the expert GEMMs read BF16.
extern "C" __global__ void gather_scale_rows(void*out,const void*src,const unsigned int*rows_of,const float*scale,int rows,int width,int scaled,int src_narrow,int narrow){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*width)return;int r=i/width;int c=i%width;float v=load_act(src,(size_t)rows_of[r]*width+c,src_narrow);store_act(out,i,scaled?scale[r]*v:v,narrow);}
extern "C" __global__ void scatter_add_scaled(float*out,const float*src,const unsigned int*rows_of,const float*scale,int rows,int width){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*width)return;int r=i/width;int c=i%width;atomicAdd(&out[(size_t)rows_of[r]*width+c],scale[r]*src[i]);}
extern "C" __global__ void scatter_add_rows(float*out,const float*src,const unsigned int*rows_of,int rows,int width){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*width)return;int r=i/width;int c=i%width;atomicAdd(&out[(size_t)rows_of[r]*width+c],src[i]);}
// One warp per routed row. A thread per row reads `width` consecutive floats
// alone, which is the worst access pattern the memory system has; a warp
// striding the row reads it in coalesced 128-byte lines instead and was about
// nine times faster on the default MoE configuration.
extern "C" __global__ void row_dot_scatter(float*out,const unsigned int*out_of,const float*a,const unsigned int*rows_of,const float*b,int rows,int width){
int lane=threadIdx.x&(WARP-1);
int warp=(blockIdx.x*blockDim.x+threadIdx.x)/WARP;
int warps=(gridDim.x*blockDim.x)/WARP;
for(int r=warp;r<rows;r+=warps){
const float*x=a+(size_t)rows_of[r]*width;
const float*y=b+(size_t)r*width;
float s=0;
for(int c=lane;c<width;c+=WARP)s+=x[c]*y[c];
s=warp_sum(s);
if(lane==0)out[out_of[r]]=s;
}
}
extern "C" __global__ void moe_stats(float*sums,const float*p,const float*lse,const int*valid,int rows,int experts){int r=blockIdx.x*blockDim.x+threadIdx.x;if(r>=rows)return;if(valid!=0&&valid[r]==0)return;const float*row=p+(size_t)r*experts;for(int e=0;e<experts;e++)atomicAdd(&sums[e],row[e]);atomicAdd(&sums[experts],lse[r]*lse[r]);}
extern "C" __global__ void moe_grad_probs(float*gp,const float*gg,const float*p,const int*expert_of,int rows,int experts,int top_k){int r=blockIdx.x*blockDim.x+threadIdx.x;if(r>=rows)return;int base=r*top_k;if(expert_of[base]<0)return;const float*row=p+(size_t)r*experts;float total=0;float weighted=0;for(int k=0;k<top_k;k++){int e=expert_of[base+k];total+=row[e];weighted+=gg[base+k]*row[e];}if(total<=0)return;float*dst=gp+(size_t)r*experts;for(int k=0;k<top_k;k++){int e=expert_of[base+k];dst[e]+=gg[base+k]/total-weighted/(total*total);}}
extern "C" __global__ void moe_aux_grad(float*gp,const float*loads,const int*expert_of,int rows,int experts,int top_k,float scale){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=rows*experts)return;int r=i/experts;int e=i%experts;if(expert_of[r*top_k]<0)return;gp[i]+=scale*loads[e];}
extern "C" __global__ void router_grad_logits(float*gl,const float*gp,const float*p,const float*lse,const int*expert_of,int rows,int experts,int top_k,float zfactor){int r=blockIdx.x*blockDim.x+threadIdx.x;if(r>=rows)return;float*dst=gl+(size_t)r*experts;if(expert_of[r*top_k]<0){for(int e=0;e<experts;e++)dst[e]=0;return;}const float*prob=p+(size_t)r*experts;const float*up=gp+(size_t)r*experts;float dot=0;for(int e=0;e<experts;e++)dot+=prob[e]*up[e];float f=zfactor*lse[r];for(int e=0;e<experts;e++)dst[e]=prob[e]*(up[e]-dot)+f*prob[e];}
// `keep` says `a` already holds a value worth adding to. A gradient buffer the
// optimizer has not refilled since the last step holds a stale one instead, and
// the first writer of the step overwrites it rather than paying for a zeroing
// pass over every parameter.
extern "C" __global__ void add_inplace(float*a,const float*b,int off,int n,int keep){int i=blockIdx.x*blockDim.x+threadIdx.x;
if(((n|off)&3)==0){int q=n>>2;if(i>=q)return;float4*d=(float4*)a;float4 y=((const float4*)(b+off))[i];if(keep){float4 x=d[i];y.x+=x.x;y.y+=x.y;y.z+=x.z;y.w+=x.w;}d[i]=y;return;}
if(i<n)a[i]=keep?a[i]+b[off+i]:b[off+i];}
// Cross-entropy over one chunk of logit rows, in place: the row comes in as
// logits and leaves as dL/dlogits, so the largest tensor in a training step is
// never held twice. `target` is the next token, or -1 for a row that predicts
// nothing (padding, and the last position of every sequence), whose gradient
// is zero. One block of CE_THREADS threads per row; `loss` accumulates the
// unscaled negative log-likelihood, which the host divides by the predicted
// count. `z_weight` adds the z-loss of `TransformerBuilder::z_loss`, whose
// unscaled sum lands in `loss[1]`.
extern "C" __global__ void to_bf16(bf16_t*out,const float*in,int n){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=f32_to_bf16(in[i]);
}
extern "C" __global__ void from_bf16(float*out,const bf16_t*in,int n){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=bf16_to_f32(in[i]);
}
extern "C" __global__ void accumulate_bf16(float*acc,const bf16_t*in,int n){
int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)acc[i]+=bf16_to_f32(in[i]);
}
#define CE_THREADS 256
extern "C" __global__ void ce_loss_grad(float*x,float*loss,const int*target,int rows,int vocab,float inv_predicted,float z_weight){
int r=blockIdx.x;if(r>=rows)return;
float*row=x+(size_t)r*vocab;
int tid=threadIdx.x;
int t=target[r];
if(t<0){for(int c=tid;c<vocab;c+=CE_THREADS)row[c]=0.f;return;}
// The running (max, sum) pair of the online softmax: one read pass over the
// row instead of a max pass followed by a sum pass. At this vocabulary the
// row is far too big for any cache, so the second pass cost a full trip to
// memory. Rescaling the accumulator only when a new maximum arrives keeps the
// common element down to a single exponential, and the fast intrinsic is
// ample for a result that is stored as bfloat16.
__shared__ float redm[CE_THREADS];
__shared__ float reds[CE_THREADS];
__shared__ float tgt;
float m=-3.0e38f,acc=0.f;
for(int c=tid;c<vocab;c+=CE_THREADS){float v=row[c];if(v>m){acc=acc*__expf(m-v)+1.f;m=v;}else acc+=__expf(v-m);}
redm[tid]=m;reds[tid]=acc;__syncthreads();
for(int s=CE_THREADS/2;s>0;s>>=1){
if(tid<s){float a=redm[tid],b=redm[tid+s];float nm=fmaxf(a,b);
reds[tid]=reds[tid]*expf(a-nm)+reds[tid+s]*expf(b-nm);redm[tid]=nm;}
__syncthreads();
}
m=redm[0];
float sum=reds[0];
if(tid==0)tgt=row[t];
__syncthreads();
// The z-loss z*lse^2 adds 2*z*lse*softmax to the gradient, which folds into
// the softmax scale, and accumulates separately in loss[1].
float lse=m+logf(sum);
float scale=inv_predicted*(1.f+2.f*z_weight*lse)/sum;
if(tid==0){atomicAdd(loss,lse-tgt);if(z_weight!=0.f)atomicAdd(loss+1,z_weight*lse*lse);}
for(int c=tid;c<vocab;c+=CE_THREADS)row[c]=__expf(row[c]-m)*scale;
__syncthreads();
if(tid==0)row[t]-=inv_predicted;
}
// The bf16 mirror of `ce_loss_grad`. Identical arithmetic in FP32 registers;
// only the row's storage is half as wide, which halves the traffic over the
// largest tensor a training step touches.
extern "C" __global__ void ce_loss_grad_bf16(bf16_t*x,float*loss,const int*target,int rows,int vocab,float inv_predicted,float z_weight){
int r=blockIdx.x;if(r>=rows)return;
bf16_t*row=x+(size_t)r*vocab;
int tid=threadIdx.x;
int t=target[r];
if(t<0){for(int c=tid;c<vocab;c+=CE_THREADS)row[c]=(bf16_t)0;return;}
// The running (max, sum) pair of the online softmax: one read pass over the
// row instead of a max pass followed by a sum pass. At this vocabulary the
// row is far too big for any cache, so the second pass cost a full trip to
// memory. Rescaling the accumulator only when a new maximum arrives keeps the
// common element down to a single exponential, and the fast intrinsic is
// ample for a result that is stored as bfloat16.
__shared__ float redm[CE_THREADS];
__shared__ float reds[CE_THREADS];
__shared__ float tgt;
// Both passes over the row move two values per access, because a warp of
// single bfloat16 loads only asks for 64 bytes at a time and this row is the
// largest tensor in the step. An odd vocabulary has no pairs and falls to the
// scalar tail, which is also where an even vocabulary's last value never
// lands.
int half=(vocab&1)==0?vocab>>1:0;
float m=-3.0e38f,acc=0.f;
{
const unsigned int*pairs=(const unsigned int*)row;
for(int c=tid;c<half;c+=CE_THREADS){
unsigned int p=pairs[c];
float v=bf16_to_f32((bf16_t)(p&0xffffu));
if(v>m){acc=acc*__expf(m-v)+1.f;m=v;}else acc+=__expf(v-m);
v=bf16_to_f32((bf16_t)(p>>16));
if(v>m){acc=acc*__expf(m-v)+1.f;m=v;}else acc+=__expf(v-m);
}
}
for(int c=half*2+tid;c<vocab;c+=CE_THREADS){float v=bf16_to_f32(row[c]);if(v>m){acc=acc*__expf(m-v)+1.f;m=v;}else acc+=__expf(v-m);}
redm[tid]=m;reds[tid]=acc;__syncthreads();
for(int s=CE_THREADS/2;s>0;s>>=1){
if(tid<s){float a=redm[tid],b=redm[tid+s];float nm=fmaxf(a,b);
reds[tid]=reds[tid]*expf(a-nm)+reds[tid+s]*expf(b-nm);redm[tid]=nm;}
__syncthreads();
}
m=redm[0];
float sum=reds[0];
if(tid==0)tgt=bf16_to_f32(row[t]);
__syncthreads();
// The z-loss z*lse^2 adds 2*z*lse*softmax to the gradient, which folds into
// the softmax scale, and accumulates separately in loss[1].
float lse=m+logf(sum);
float scale=inv_predicted*(1.f+2.f*z_weight*lse)/sum;
if(tid==0){atomicAdd(loss,lse-tgt);if(z_weight!=0.f)atomicAdd(loss+1,z_weight*lse*lse);}
{
unsigned int*pairs=(unsigned int*)row;
for(int c=tid;c<half;c+=CE_THREADS){
unsigned int p=pairs[c];
unsigned int lo=f32_to_bf16(__expf(bf16_to_f32((bf16_t)(p&0xffffu))-m)*scale);
unsigned int hi=f32_to_bf16(__expf(bf16_to_f32((bf16_t)(p>>16))-m)*scale);
pairs[c]=(hi<<16)|lo;
}
}
for(int c=half*2+tid;c<vocab;c+=CE_THREADS)row[c]=f32_to_bf16(__expf(bf16_to_f32(row[c])-m)*scale);
__syncthreads();
if(tid==0)row[t]=f32_to_bf16(bf16_to_f32(row[t])-inv_predicted);
}
"#;
#[derive(Clone, Debug)]
pub struct CudaDoctorReport {
pub device: usize,
pub name: String,
pub compute_capability: String,
pub total_memory_mib: usize,
pub free_memory_mib: Option<usize>,
pub requested_budget_mib: usize,
pub allocation_test: bool,
pub cublas_available: bool,
pub kernel_available: bool,
}
/// Called by a CUDA test that found no device and is about to return early.
/// A machine with a GPU sets `RUSTING_BRAIN_REQUIRE_CUDA=1`, so a test that
/// should have run fails instead of passing without doing anything.
#[cfg(test)]
pub(crate) fn skip_without_cuda() {
assert!(
std::env::var_os("RUSTING_BRAIN_REQUIRE_CUDA").is_none(),
"RUSTING_BRAIN_REQUIRE_CUDA is set, but this test found no CUDA device"
);
eprintln!("skipped: no CUDA device");
}
pub fn cuda_doctor(
device: usize,
requested_budget_mib: usize,
) -> Result<CudaDoctorReport, NetworkError> {
let ctx = CudaContext::new(device).map_err(|e| {
NetworkError::Cuda(format!(
"CUDA driver/device {device} initialization failed: {e}"
))
})?;
let stream = ctx.new_stream().map_err(cuda_err("stream creation"))?;
let _blas = CudaBlas::new(stream.clone()).map_err(cuda_err("cuBLAS initialization"))?;
let ptx = compile_ptx(KERNELS)
.map_err(|e| NetworkError::Cuda(format!("CUDA kernel compilation failed: {e}")))?;
let module = ctx
.load_module(ptx)
.map_err(cuda_err("CUDA module loading"))?;
let _ = module
.load_function("sgd")
.map_err(cuda_err("CUDA kernel lookup"))?;
let allocation_test = stream.alloc_zeros::<f32>(256).is_ok();
let total = unsafe { cudarc::driver::result::device::total_mem(ctx.cu_device()) }
.map_err(cuda_err("CUDA memory query"))?
/ MIB;
let cc = ctx
.compute_capability()
.map(|(a, b)| format!("{a}.{b}"))
.unwrap_or_else(|_| "unknown".into());
let free_memory_mib = cudarc::runtime::result::get_mem_info()
.ok()
.map(|(free, _)| free / MIB);
Ok(CudaDoctorReport {
device,
name: ctx.name().unwrap_or_else(|_| "NVIDIA CUDA device".into()),
compute_capability: cc,
total_memory_mib: total,
free_memory_mib,
requested_budget_mib,
allocation_test,
cublas_available: true,
kernel_available: true,
})
}
pub(crate) fn cuda_err<E: std::fmt::Display>(
stage: &'static str,
) -> impl FnOnce(E) -> NetworkError {
move |e| NetworkError::Cuda(format!("{stage} failed: {e}"))
}
/// As [`cuda_err`], for a call that asks the driver for `bytes` of device
/// memory.
///
/// A driver out-of-memory becomes [`NetworkError::CudaOutOfMemory`], which a
/// caller can match on to retry with a smaller batch. Every other failure keeps
/// its [`NetworkError::Cuda`], because nothing about it says a smaller
/// allocation would have worked.
pub(crate) fn cuda_alloc_err<E: std::fmt::Display>(
stage: &'static str,
bytes: usize,
) -> impl FnOnce(E) -> NetworkError {
move |e| {
let message = e.to_string();
if !is_out_of_memory(&message) {
return NetworkError::Cuda(format!("{stage} failed: {e}"));
}
NetworkError::CudaOutOfMemory {
requested_mib: bytes.div_ceil(MIB),
free_mib: cudarc::runtime::result::get_mem_info()
.map(|(free, _)| free / MIB)
.unwrap_or(0),
}
}
}
/// Whether a driver error is the one that says the allocation did not fit.
///
/// Matched on the text because cudarc's error types differ between the driver
/// and the runtime API and both reach this crate; the spelling is the driver's
/// own `CUDA_ERROR_OUT_OF_MEMORY` and the runtime's `cudaErrorMemoryAllocation`.
fn is_out_of_memory(message: &str) -> bool {
let lowered = message.to_ascii_lowercase();
lowered.contains("out_of_memory")
|| lowered.contains("out of memory")
|| lowered.contains("memoryallocation")
}
/// nvrtc compilation of [`KERNELS`] costs a few hundred milliseconds and the
/// source never varies, so a campaign running dozens of sessions compiles once.
fn kernel_ptx() -> Result<&'static Ptx, NetworkError> {
static PTX: OnceLock<Result<Ptx, String>> = OnceLock::new();
PTX.get_or_init(|| compile_ptx(KERNELS).map_err(|e| e.to_string()))
.as_ref()
.map_err(|e| NetworkError::Cuda(format!("CUDA kernel compilation failed: {e}")))
}
/// Creating a `CudaContext` and loading a module are per-process costs, not
/// per-session ones. Device buffers still belong to their session and are freed
/// when it drops; only the context and the compiled module are shared.
type LoadedDevice = (Arc<CudaContext>, Arc<CudaModule>);
pub(crate) fn device_context(
device: usize,
) -> Result<(Arc<CudaContext>, Arc<CudaModule>), NetworkError> {
static CONTEXTS: OnceLock<Mutex<HashMap<usize, LoadedDevice>>> = OnceLock::new();
let mut contexts = CONTEXTS
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.map_err(|_| NetworkError::Cuda("CUDA context cache lock was poisoned".into()))?;
if let Some(entry) = contexts.get(&device) {
return Ok(entry.clone());
}
let ctx = CudaContext::new(device).map_err(cuda_err("CUDA driver/device initialization"))?;
keep_freed_memory(&ctx)?;
let module = ctx
.load_module(kernel_ptx()?.clone())
.map_err(cuda_err("CUDA module loading"))?;
contexts.insert(device, (ctx.clone(), module.clone()));
Ok((ctx, module))
}
/// Stops the device's memory pool from handing freed memory back to the
/// driver at every synchronization.
///
/// The pool's default release threshold is zero, so every sync returns the
/// step's freed buffers to the driver and the next step's allocations map
/// them back in. On the shape VAE that sync took 44 ms of a 129 ms step with
/// the GPU idle. Memory freed by this process stays reserved
/// for it until the context goes away, which is what a training loop wants.
fn keep_freed_memory(ctx: &CudaContext) -> Result<(), NetworkError> {
use cudarc::driver::sys;
unsafe {
let mut pool = std::mem::MaybeUninit::uninit();
sys::cuDeviceGetDefaultMemPool(pool.as_mut_ptr(), ctx.cu_device())
.result()
.map_err(cuda_err("CUDA memory pool lookup"))?;
let mut keep = u64::MAX;
sys::cuMemPoolSetAttribute(
pool.assume_init(),
sys::CUmemPool_attribute::CU_MEMPOOL_ATTR_RELEASE_THRESHOLD,
(&mut keep as *mut u64).cast(),
)
.result()
.map_err(cuda_err("CUDA memory pool setup"))
}
}
/// Waits for the device, then hands the memory [`keep_freed_memory`] kept
/// back to the driver, so that `memory_info` reports it free again. A model
/// that drops one network to make room for another calls it in between.
pub(crate) fn release_freed_memory(device: usize) -> Result<(), NetworkError> {
use cudarc::driver::sys;
let (ctx, _) = device_context(device)?;
ctx.synchronize()
.map_err(cuda_err("CUDA synchronize before a pool trim"))?;
unsafe {
let mut pool = std::mem::MaybeUninit::uninit();
sys::cuDeviceGetDefaultMemPool(pool.as_mut_ptr(), ctx.cu_device())
.result()
.map_err(cuda_err("CUDA memory pool lookup"))?;
sys::cuMemPoolTrimTo(pool.assume_init(), 0)
.result()
.map_err(cuda_err("CUDA memory pool trim"))
}
}
/// The most device memory the default pool has held since the last call, in
/// bytes; the call starts the next window. Measurement only.
#[cfg(test)]
pub(crate) fn reserved_high_water(device: usize) -> Result<u64, NetworkError> {
use cudarc::driver::sys;
let (ctx, _) = device_context(device)?;
ctx.synchronize()
.map_err(cuda_err("CUDA synchronize before a pool read"))?;
unsafe {
let mut pool = std::mem::MaybeUninit::uninit();
sys::cuDeviceGetDefaultMemPool(pool.as_mut_ptr(), ctx.cu_device())
.result()
.map_err(cuda_err("CUDA memory pool lookup"))?;
let pool = pool.assume_init();
let mut high = 0u64;
sys::cuMemPoolGetAttribute(
pool,
sys::CUmemPool_attribute::CU_MEMPOOL_ATTR_RESERVED_MEM_HIGH,
(&mut high as *mut u64).cast(),
)
.result()
.map_err(cuda_err("CUDA memory pool read"))?;
let mut zero = 0u64;
sys::cuMemPoolSetAttribute(
pool,
sys::CUmemPool_attribute::CU_MEMPOOL_ATTR_RESERVED_MEM_HIGH,
(&mut zero as *mut u64).cast(),
)
.result()
.map_err(cuda_err("CUDA memory pool reset"))?;
Ok(high)
}
}
/// Every kernel handle resolved once. `CudaModule::load_function` is a driver
/// lookup; calling it inside the batch loop cost more than the kernels it found.
struct Kernels {
bias_act: CudaFunction,
output_delta: CudaFunction,
act_derivative: CudaFunction,
grad_b: CudaFunction,
sgd: CudaFunction,
adam: CudaFunction,
lion: CudaFunction,
mse_epoch_sum: CudaFunction,
gather_rows: CudaFunction,
}
impl Kernels {
fn load(module: &Arc<CudaModule>) -> Result<Self, NetworkError> {
let get = |name: &str| {
module
.load_function(name)
.map_err(cuda_err("CUDA kernel lookup"))
};
Ok(Self {
bias_act: get("bias_act")?,
output_delta: get("output_delta")?,
act_derivative: get("act_derivative")?,
grad_b: get("grad_b")?,
sgd: get("sgd")?,
adam: get("adam")?,
lion: get("lion")?,
mse_epoch_sum: get("mse_epoch_sum")?,
gather_rows: get("gather_rows")?,
})
}
}
pub(crate) fn cfg(n: usize) -> LaunchConfig {
LaunchConfig {
grid_dim: ((n as u32).div_ceil(256), 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
fn act(a: Activation) -> Result<i32, NetworkError> {
match a {
Activation::Linear => Ok(0),
Activation::Relu => Ok(1),
Activation::Sigmoid => Ok(2),
Activation::Tanh => Ok(3),
Activation::Softmax => Err(NetworkError::UnsupportedCuda("Softmax activation".into())),
}
}
struct DevLayer {
w: CudaSlice<f32>,
b: CudaSlice<f32>,
mw: CudaSlice<f32>,
vw: CudaSlice<f32>,
mb: CudaSlice<f32>,
vb: CudaSlice<f32>,
gw: CudaSlice<f32>,
gb: CudaSlice<f32>,
a: CudaSlice<f32>,
d: CudaSlice<f32>,
}
struct State {
_ctx: Arc<CudaContext>,
stream: Arc<CudaStream>,
blas: CudaBlas,
kernels: Kernels,
layers: Vec<DevLayer>,
input: CudaSlice<f32>,
target: CudaSlice<f32>,
loss: CudaSlice<f32>,
all_input: Option<CudaSlice<f32>>,
all_target: Option<CudaSlice<f32>>,
order: Option<CudaSlice<u32>>,
/// Validation set, uploaded once by `prepare_validation` (P09: device-side
/// validation avoids a full model host round-trip every epoch). Identity
/// order, since validation never shuffles.
eval_input: Option<CudaSlice<f32>>,
eval_target: Option<CudaSlice<f32>>,
eval_order: Option<CudaSlice<u32>>,
eval_loss: Option<CudaSlice<f32>>,
}
pub(crate) fn fit_cuda(
net: &mut Network,
dataset: &Dataset,
config: TrainConfig,
device: usize,
budget: usize,
) -> Result<TrainingHistory, NetworkError> {
let mut session = CudaTrainingSession::new(net, dataset, config, device, budget)?;
let mut history = Vec::with_capacity(config.epochs);
for _ in 0..config.epochs {
history.push(session.train_epoch()?);
}
session.synchronize_network(net)?;
Ok(TrainingHistory { losses: history })
}
/// Persistent CUDA state for epoch-at-a-time training and explicit snapshots.
pub struct CudaTrainingSession {
state: State,
model: Network,
/// Flat row-major copies, kept only when the dataset did not fit on the
/// device. When it did, the device owns the only copy and holding a second
/// one on the host wasted as much memory as the dataset itself.
host: Option<HostDataset>,
rows: usize,
input_width: usize,
target_width: usize,
order: Vec<usize>,
config: TrainConfig,
batch_size: usize,
epoch: usize,
stats: CudaTrainingStats,
staged_input: Vec<f32>,
staged_target: Vec<f32>,
/// Row count of the device-resident validation set, or 0 when
/// `prepare_validation` has not been called (or declined for budget).
eval_rows: usize,
}
struct HostDataset {
inputs: Vec<f32>,
targets: Vec<f32>,
}
impl CudaTrainingSession {
pub fn new(
net: &Network,
dataset: &Dataset,
config: TrainConfig,
device: usize,
budget_mib: usize,
) -> Result<Self, NetworkError> {
let started = Instant::now();
if dataset.is_empty() {
return Err(NetworkError::EmptyDataset);
}
if net.loss != Loss::Mse {
return Err(NetworkError::UnsupportedCuda(
"only MSE loss is implemented".into(),
));
}
for layer in &net.layers {
act(layer.activation)?;
}
for (x, y) in dataset.inputs.iter().zip(&dataset.targets) {
net.validate_input(x)?;
net.validate_target(y)?;
}
let mut batch = config.batch_size.max(1).min(dataset.len());
while estimate_tensor_memory_mib(net, batch)? > budget_mib {
if batch == 1 {
return Err(NetworkError::CudaMemoryBudget {
estimated_mib: estimate_tensor_memory_mib(net, 1)?,
budget_mib,
});
}
batch = batch.div_ceil(2);
}
let xs: Vec<f32> = dataset.inputs.iter().flatten().copied().collect();
let ys: Vec<f32> = dataset.targets.iter().flatten().copied().collect();
let dataset_bytes =
(xs.len() + ys.len()) * size_of::<f32>() + dataset.len() * size_of::<u32>();
let resident = estimate_tensor_memory_mib(net, batch)?
.saturating_mul(MIB)
.saturating_add(dataset_bytes)
<= budget_mib.saturating_mul(MIB);
let state = State::new(
net,
batch,
device,
resident.then_some((&xs, &ys, dataset.len())),
)?;
let parameter_bytes = net
.layers
.iter()
.map(|l| (l.weights.data.len() + l.biases.data.len()) * 4 * size_of::<f32>())
.sum::<usize>();
let peak = state.allocated_bytes();
if peak > budget_mib.saturating_mul(MIB) {
return Err(NetworkError::CudaMemoryBudget {
estimated_mib: peak.div_ceil(MIB),
budget_mib,
});
}
Ok(Self {
state,
model: net.clone(),
host: (!resident).then_some(HostDataset {
inputs: xs,
targets: ys,
}),
rows: dataset.len(),
input_width: net.input_size,
target_width: net.output_size(),
order: (0..dataset.len()).collect(),
config,
batch_size: batch,
epoch: 0,
staged_input: Vec::with_capacity(batch * net.input_size),
staged_target: Vec::with_capacity(batch * net.output_size()),
eval_rows: 0,
stats: CudaTrainingStats {
peak_allocated_bytes: peak,
setup_time: started.elapsed(),
host_to_device_bytes: parameter_bytes
+ if resident {
dataset_bytes - dataset.len() * size_of::<u32>()
} else {
0
},
dataset_resident: resident,
..Default::default()
},
})
}
/// Restores host checkpoint state and reconstructs the deterministic
/// shuffle position before creating fresh device resources.
pub fn from_checkpoint(
checkpoint: CudaTrainingCheckpoint,
dataset: &Dataset,
mut config: TrainConfig,
device: usize,
budget_mib: usize,
) -> Result<Self, NetworkError> {
let completed_epochs = checkpoint.epoch;
config.seed = checkpoint.shuffle_seed;
let mut model = Network::builder()
.input_size(checkpoint.input_size)
.dense(1, Activation::Linear)
.build();
model.restore_cuda_checkpoint(checkpoint)?;
let mut session = Self::new(&model, dataset, config, device, budget_mib)?;
if config.shuffle {
for epoch in 0..completed_epochs {
if let Some(seed) = config.seed {
session
.order
.shuffle(&mut StdRng::seed_from_u64(seed + epoch as u64));
} else {
return Err(NetworkError::InvalidCudaCheckpoint(
"cannot deterministically resume an unseeded shuffle".into(),
));
}
}
}
session.epoch = completed_epochs;
Ok(session)
}
pub fn train_epoch(&mut self) -> Result<f32, NetworkError> {
let started = Instant::now();
if self.config.shuffle {
if let Some(seed) = self.config.seed {
self.order
.shuffle(&mut StdRng::seed_from_u64(seed + self.epoch as u64));
} else {
self.order.shuffle(&mut rand::thread_rng());
}
}
if let Some(device_order) = &mut self.state.order {
let order: Vec<u32> = self.order.iter().map(|&i| i as u32).collect();
self.state
.stream
.memcpy_htod(&order, device_order)
.map_err(cuda_err("shuffle order upload"))?;
self.stats.host_to_device_bytes += order.len() * size_of::<u32>();
}
self.state
.stream
.memset_zeros(&mut self.state.loss)
.map_err(cuda_err("epoch loss reset"))?;
let batches = self.rows.div_ceil(self.batch_size);
for start in (0..self.rows).step_by(self.batch_size) {
let rows = self.batch_size.min(self.rows - start);
if let Some(host) = &self.host {
let ids = &self.order[start..start + rows];
self.staged_input.clear();
self.staged_target.clear();
for &i in ids {
let x = i * self.input_width;
let y = i * self.target_width;
self.staged_input
.extend_from_slice(&host.inputs[x..x + self.input_width]);
self.staged_target
.extend_from_slice(&host.targets[y..y + self.target_width]);
}
self.state
.stream
.memcpy_htod(&self.staged_input, &mut self.state.input)
.map_err(cuda_err("input upload"))?;
self.state
.stream
.memcpy_htod(&self.staged_target, &mut self.state.target)
.map_err(cuda_err("target upload"))?;
self.stats.host_to_device_bytes +=
(self.staged_input.len() + self.staged_target.len()) * size_of::<f32>();
} else {
self.state
.gather_batch(start, rows, self.input_width, self.target_width)?;
}
self.state.train_batch(&mut self.model, rows, batches)?;
self.stats.batches += 1;
}
self.state
.stream
.synchronize()
.map_err(cuda_err("epoch synchronization"))?;
let loss = self
.state
.stream
.clone_dtoh(&self.state.loss)
.map_err(cuda_err("epoch loss download"))?[0];
self.stats.device_to_host_bytes += size_of::<f32>();
if !loss.is_finite() {
return Err(NetworkError::Cuda(
"numerical preflight failed: non-finite loss".into(),
));
}
self.epoch += 1;
self.stats.epochs += 1;
self.stats.training_time += started.elapsed();
Ok(loss)
}
pub fn checkpoint(&mut self) -> Result<CudaTrainingCheckpoint, NetworkError> {
let started = Instant::now();
self.state.copy_back(&mut self.model)?;
self.stats.device_to_host_bytes += self.state.checkpoint_bytes();
self.stats.checkpoint_time += started.elapsed();
Ok(self.model.cuda_checkpoint(self.epoch, self.config.seed))
}
pub fn synchronize_network(&mut self, net: &mut Network) -> Result<(), NetworkError> {
net.restore_cuda_checkpoint(self.checkpoint()?)
}
pub fn stats(&self) -> &CudaTrainingStats {
&self.stats
}
pub fn epoch(&self) -> usize {
self.epoch
}
/// Uploads `dataset` to the device once so `validate_epoch` can score it
/// without a host round-trip. Returns `Ok(false)` (not an error) when it
/// would not fit under `budget_mib` alongside what is already resident;
/// the caller should fall back to host-side validation in that case.
pub fn prepare_validation(
&mut self,
dataset: &Dataset,
budget_mib: usize,
) -> Result<bool, NetworkError> {
if dataset.is_empty() {
return Err(NetworkError::EmptyDataset);
}
for (x, y) in dataset.inputs.iter().zip(&dataset.targets) {
self.model.validate_input(x)?;
self.model.validate_target(y)?;
}
let xs: Vec<f32> = dataset.inputs.iter().flatten().copied().collect();
let ys: Vec<f32> = dataset.targets.iter().flatten().copied().collect();
let rows = dataset.len();
let bytes = (xs.len() + ys.len()) * size_of::<f32>() + rows * size_of::<u32>();
if self.state.allocated_bytes() + bytes > budget_mib.saturating_mul(MIB) {
return Ok(false);
}
let stream = self.state.stream.clone();
let eval_input = stream
.clone_htod(&xs)
.map_err(cuda_err("validation dataset upload"))?;
let eval_target = stream
.clone_htod(&ys)
.map_err(cuda_err("validation dataset upload"))?;
// Identity order: validation is never shuffled, but `gather_rows`
// already does exactly the contiguous-batch copy this needs.
let identity: Vec<u32> = (0..rows as u32).collect();
let eval_order = stream
.clone_htod(&identity)
.map_err(cuda_err("validation order upload"))?;
let eval_loss = stream
.alloc_zeros::<f32>(1)
.map_err(cuda_err("device allocation"))?;
self.stats.host_to_device_bytes += bytes;
self.state.eval_input = Some(eval_input);
self.state.eval_target = Some(eval_target);
self.state.eval_order = Some(eval_order);
self.state.eval_loss = Some(eval_loss);
self.eval_rows = rows;
Ok(true)
}
/// Device-side validation loss over the set uploaded by
/// `prepare_validation`, using the weights currently resident on the
/// device (no `copy_back`/host round-trip of the full model). Forward
/// pass only: no gradients, no optimizer step, no mutation of training
/// state. Numerically the same computation as `Network::evaluate_loss`
/// (MSE averaged over every output element), just done on-device.
pub fn validate_epoch(&mut self) -> Result<f32, NetworkError> {
if self.eval_rows == 0 {
return Err(NetworkError::Accelerator(
"validate_epoch called before prepare_validation succeeded".into(),
));
}
let rows = self.eval_rows;
let output_size = self.target_width;
{
let loss = self.state.eval_loss.as_mut().expect("prepared");
self.state
.stream
.memset_zeros(loss)
.map_err(cuda_err("validation loss reset"))?;
}
let scale = 1.0 / (rows * output_size) as f32;
for start in (0..rows).step_by(self.batch_size) {
let b = self.batch_size.min(rows - start);
let f = &self.state.kernels.gather_rows;
let all_input = self.state.eval_input.as_ref().expect("prepared");
let all_target = self.state.eval_target.as_ref().expect("prepared");
let order = self.state.eval_order.as_ref().expect("prepared");
unsafe {
self.state
.stream
.launch_builder(f)
.arg(&mut self.state.input)
.arg(all_input)
.arg(order)
.arg(&(start as i32))
.arg(&(b as i32))
.arg(&(self.input_width as i32))
.launch(cfg(b * self.input_width))
.map_err(cuda_err("validation input gather kernel"))?;
self.state
.stream
.launch_builder(f)
.arg(&mut self.state.target)
.arg(all_target)
.arg(order)
.arg(&(start as i32))
.arg(&(b as i32))
.arg(&(self.target_width as i32))
.launch(cfg(b * self.target_width))
.map_err(cuda_err("validation target gather kernel"))?;
}
forward_pass(
&self.state.stream,
&self.state.blas,
&self.state.kernels,
&mut self.state.layers,
&self.model,
&self.state.input,
b,
)?;
let last = self.state.layers.len() - 1;
let la = &self.state.layers[last].a;
let n = b * output_size;
let fl = &self.state.kernels.mse_epoch_sum;
let loss = self.state.eval_loss.as_mut().expect("prepared");
unsafe {
self.state
.stream
.launch_builder(fl)
.arg(loss)
.arg(la)
.arg(&self.state.target)
.arg(&(n as i32))
.arg(&scale)
.launch(cfg(n))
.map_err(cuda_err("validation loss kernel"))?;
}
}
self.state
.stream
.synchronize()
.map_err(cuda_err("validation synchronization"))?;
let loss = self
.state
.stream
.clone_dtoh(self.state.eval_loss.as_ref().expect("prepared"))
.map_err(cuda_err("validation loss download"))?[0];
self.stats.device_to_host_bytes += size_of::<f32>();
if !loss.is_finite() {
return Err(NetworkError::Cuda(
"numerical preflight failed: non-finite validation loss".into(),
));
}
Ok(loss)
}
}
impl State {
fn new(
net: &Network,
batch: usize,
device: usize,
resident: Option<(&[f32], &[f32], usize)>,
) -> Result<Self, NetworkError> {
let (ctx, module) = device_context(device)?;
let stream = ctx.new_stream().map_err(cuda_err("stream creation"))?;
let blas = CudaBlas::new(stream.clone()).map_err(cuda_err("cuBLAS initialization"))?;
let kernels = Kernels::load(&module)?;
let mut layers = Vec::new();
for (i, l) in net.layers.iter().enumerate() {
let a = batch * l.weights.rows;
let z = |v: &[f32]| {
stream
.clone_htod(v)
.map_err(cuda_err("device allocation/upload"))
};
let zeros = |n| {
stream
.alloc_zeros::<f32>(n)
.map_err(cuda_err("device allocation"))
};
layers.push(DevLayer {
w: z(&l.weights.data)?,
b: z(&l.biases.data)?,
mw: z(&net.adam_m_weights[i].data)?,
vw: z(&net.adam_v_weights[i].data)?,
mb: z(&net.adam_m_biases[i].data)?,
vb: z(&net.adam_v_biases[i].data)?,
gw: zeros(l.weights.data.len())?,
gb: zeros(l.biases.data.len())?,
a: zeros(a)?,
d: zeros(a)?,
});
}
let (all_input, all_target, order) = if let Some((xs, ys, rows)) = resident {
(
Some(stream.clone_htod(xs).map_err(cuda_err("dataset upload"))?),
Some(stream.clone_htod(ys).map_err(cuda_err("dataset upload"))?),
Some(
stream
.alloc_zeros::<u32>(rows)
.map_err(cuda_err("order allocation"))?,
),
)
} else {
(None, None, None)
};
Ok(Self {
_ctx: ctx,
stream: stream.clone(),
blas,
kernels,
layers,
input: stream
.alloc_zeros(batch * net.input_size)
.map_err(cuda_err("device allocation"))?,
target: stream
.alloc_zeros(batch * net.output_size())
.map_err(cuda_err("device allocation"))?,
loss: stream
.alloc_zeros(1)
.map_err(cuda_err("device allocation"))?,
all_input,
all_target,
order,
eval_input: None,
eval_target: None,
eval_order: None,
eval_loss: None,
})
}
fn gather_batch(
&mut self,
start: usize,
rows: usize,
input_width: usize,
target_width: usize,
) -> Result<(), NetworkError> {
let f = &self.kernels.gather_rows;
let all_input = self.all_input.as_ref().expect("resident input invariant");
let all_target = self.all_target.as_ref().expect("resident target invariant");
let order = self.order.as_ref().expect("resident order invariant");
unsafe {
self.stream
.launch_builder(f)
.arg(&mut self.input)
.arg(all_input)
.arg(order)
.arg(&(start as i32))
.arg(&(rows as i32))
.arg(&(input_width as i32))
.launch(cfg(rows * input_width))
.map_err(cuda_err("input gather kernel"))?;
self.stream
.launch_builder(f)
.arg(&mut self.target)
.arg(all_target)
.arg(order)
.arg(&(start as i32))
.arg(&(rows as i32))
.arg(&(target_width as i32))
.launch(cfg(rows * target_width))
.map_err(cuda_err("target gather kernel"))?;
}
Ok(())
}
fn checkpoint_bytes(&self) -> usize {
self.layers
.iter()
.map(|l| l.w.len() + l.b.len() + l.mw.len() + l.vw.len() + l.mb.len() + l.vb.len())
.sum::<usize>()
* size_of::<f32>()
}
fn allocated_bytes(&self) -> usize {
let layers = self
.layers
.iter()
.map(|l| {
l.w.len()
+ l.b.len()
+ l.mw.len()
+ l.vw.len()
+ l.mb.len()
+ l.vb.len()
+ l.gw.len()
+ l.gb.len()
+ l.a.len()
+ l.d.len()
})
.sum::<usize>()
* size_of::<f32>();
layers
+ (self.input.len()
+ self.target.len()
+ self.loss.len()
+ self.all_input.as_ref().map_or(0, CudaSlice::len)
+ self.all_target.as_ref().map_or(0, CudaSlice::len)
+ self.eval_input.as_ref().map_or(0, CudaSlice::len)
+ self.eval_target.as_ref().map_or(0, CudaSlice::len)
+ self.eval_loss.as_ref().map_or(0, CudaSlice::len))
* size_of::<f32>()
+ (self.order.as_ref().map_or(0, CudaSlice::len)
+ self.eval_order.as_ref().map_or(0, CudaSlice::len))
* size_of::<u32>()
}
fn train_batch(
&mut self,
net: &mut Network,
b: usize,
epoch_batches: usize,
) -> Result<(), NetworkError> {
for i in 0..self.layers.len() {
let units = if i == 0 {
net.input_size
} else {
net.layers[i - 1].weights.rows
};
let out_units = net.layers[i].weights.rows;
if i == 0 {
let l = &mut self.layers[0];
gemm(&self.blas, &self.input, &l.w, &mut l.a, b, units, out_units)?;
} else {
let (left, right) = self.layers.split_at_mut(i);
let previous = &left[i - 1].a;
let l = &mut right[0];
gemm(&self.blas, previous, &l.w, &mut l.a, b, units, out_units)?;
}
let f = &self.kernels.bias_act;
let l = &mut self.layers[i];
unsafe {
self.stream
.launch_builder(f)
.arg(&mut l.a)
.arg(&l.b)
.arg(&(b as i32))
.arg(&(net.layers[i].weights.rows as i32))
.arg(&act(net.layers[i].activation)?)
.launch(cfg(b * net.layers[i].weights.rows))
.map_err(cuda_err("bias/activation kernel"))?;
}
}
let last = self.layers.len() - 1;
let f = &self.kernels.output_delta;
{
let l = &mut self.layers[last];
unsafe {
self.stream
.launch_builder(f)
.arg(&mut l.d)
.arg(&l.a)
.arg(&self.target)
.arg(&((b * net.output_size()) as i32))
.arg(&act(net.layers[last].activation)?)
.launch(cfg(b * net.output_size()))
.map_err(cuda_err("output delta kernel"))?;
}
}
for i in (0..self.layers.len()).rev() {
let out = net.layers[i].weights.rows;
let inp = net.layers[i].weights.cols;
let fb = &self.kernels.grad_b;
// GW[out, inp] = (1/b) * D^T . X, through cuBLAS rather than a
// hand-written kernel: this is a real GEMM, and a kernel giving
// each of the out*inp threads a serial strided loop over the batch
// reaches a fraction of the same bandwidth.
if i == 0 {
let l = &mut self.layers[0];
grad_weights(&self.blas, &l.d, &self.input, &mut l.gw, b, out, inp)?;
unsafe {
self.stream
.launch_builder(fb)
.arg(&mut l.gb)
.arg(&l.d)
.arg(&(b as i32))
.arg(&(out as i32))
.launch(cfg(out))
.map_err(cuda_err("bias gradient kernel"))?;
}
} else {
let (left, right) = self.layers.split_at_mut(i);
let source = &left[i - 1].a;
let l = &mut right[0];
grad_weights(&self.blas, &l.d, source, &mut l.gw, b, out, inp)?;
unsafe {
self.stream
.launch_builder(fb)
.arg(&mut l.gb)
.arg(&l.d)
.arg(&(b as i32))
.arg(&(out as i32))
.launch(cfg(out))
.map_err(cuda_err("bias gradient kernel"))?;
}
}
if i > 0 {
let fd = &self.kernels.act_derivative;
let (left, right) = self.layers.split_at_mut(i);
let prev = &mut left[i - 1];
let cur = &right[0];
// PD[b, inp] = D_next[b, out] . W[out, inp], then scaled in
// place by the previous layer's activation derivative.
propagate_delta(&self.blas, &cur.d, &cur.w, &mut prev.d, b, out, inp)?;
unsafe {
self.stream
.launch_builder(fd)
.arg(&mut prev.d)
.arg(&prev.a)
.arg(&((b * inp) as i32))
.arg(&act(net.layers[i - 1].activation)?)
.launch(cfg(b * inp))
.map_err(cuda_err("activation derivative kernel"))?;
}
}
}
self.update(net)?;
let fl = &self.kernels.mse_epoch_sum;
let la = &self.layers[last].a;
let n = b * net.output_size();
let scale = 1.0 / (n * epoch_batches) as f32;
unsafe {
self.stream
.launch_builder(fl)
.arg(&mut self.loss)
.arg(la)
.arg(&self.target)
.arg(&(n as i32))
.arg(&scale)
.launch(cfg(b * net.output_size()))
.map_err(cuda_err("loss kernel"))?;
}
Ok(())
}
fn update(&mut self, net: &mut Network) -> Result<(), NetworkError> {
let (f, adam) = match net.optimizer.clone() {
Optimizer::Sgd { .. } => (&self.kernels.sgd, false),
Optimizer::Adam { .. } => (&self.kernels.adam, true),
Optimizer::Lion { .. } => (&self.kernels.lion, false),
};
if adam {
net.adam_step += 1;
}
for l in &mut self.layers {
let nw = l.w.len();
let nb = l.b.len();
unsafe {
if let Optimizer::Sgd { learning_rate } = net.optimizer {
self.stream
.launch_builder(f)
.arg(&mut l.w)
.arg(&l.gw)
.arg(&(nw as i32))
.arg(&learning_rate)
.launch(cfg(nw))
.map_err(cuda_err("SGD update kernel"))?;
self.stream
.launch_builder(f)
.arg(&mut l.b)
.arg(&l.gb)
.arg(&(nb as i32))
.arg(&learning_rate)
.launch(cfg(nb))
.map_err(cuda_err("SGD update kernel"))?;
} else if let Optimizer::Adam {
learning_rate,
beta1,
beta2,
epsilon,
weight_decay,
} = net.optimizer
{
let c1 = 1. - beta1.powi(net.adam_step as i32);
let c2 = 1. - beta2.powi(net.adam_step as i32);
let no_decay = 0.0f32;
self.stream
.launch_builder(f)
.arg(&mut l.w)
.arg(&l.gw)
.arg(&mut l.mw)
.arg(&mut l.vw)
.arg(&(nw as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&epsilon)
.arg(&c1)
.arg(&c2)
.arg(&weight_decay)
.launch(cfg(nw))
.map_err(cuda_err("Adam update kernel"))?;
self.stream
.launch_builder(f)
.arg(&mut l.b)
.arg(&l.gb)
.arg(&mut l.mb)
.arg(&mut l.vb)
.arg(&(nb as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&epsilon)
.arg(&c1)
.arg(&c2)
.arg(&no_decay)
.launch(cfg(nb))
.map_err(cuda_err("Adam update kernel"))?;
} else if let Optimizer::Lion {
learning_rate,
beta1,
beta2,
weight_decay,
} = net.optimizer
{
// Lion keeps one moment, the one Adam calls `m`.
let no_decay = 0.0f32;
self.stream
.launch_builder(f)
.arg(&mut l.w)
.arg(&l.gw)
.arg(&mut l.mw)
.arg(&(nw as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&weight_decay)
.launch(cfg(nw))
.map_err(cuda_err("Lion update kernel"))?;
self.stream
.launch_builder(f)
.arg(&mut l.b)
.arg(&l.gb)
.arg(&mut l.mb)
.arg(&(nb as i32))
.arg(&learning_rate)
.arg(&beta1)
.arg(&beta2)
.arg(&no_decay)
.launch(cfg(nb))
.map_err(cuda_err("Lion update kernel"))?;
}
}
}
Ok(())
}
fn copy_back(&self, net: &mut Network) -> Result<(), NetworkError> {
for (i, l) in self.layers.iter().enumerate() {
net.layers[i].weights.data = self
.stream
.clone_dtoh(&l.w)
.map_err(cuda_err("weight download"))?;
net.layers[i].biases.data = self
.stream
.clone_dtoh(&l.b)
.map_err(cuda_err("bias download"))?;
net.adam_m_weights[i].data = self
.stream
.clone_dtoh(&l.mw)
.map_err(cuda_err("moment download"))?;
net.adam_v_weights[i].data = self
.stream
.clone_dtoh(&l.vw)
.map_err(cuda_err("moment download"))?;
net.adam_m_biases[i].data = self
.stream
.clone_dtoh(&l.mb)
.map_err(cuda_err("moment download"))?;
net.adam_v_biases[i].data = self
.stream
.clone_dtoh(&l.vb)
.map_err(cuda_err("moment download"))?;
}
Ok(())
}
}
/// Forward pass only: fills every `layers[i].a` from `input`, leaving
/// `layers.last().a` holding the network's output. No gradients, no
/// parameter update. Used by `CudaTrainingSession::validate_epoch`; kept
/// separate from `State::train_batch`'s own (duplicated) forward section so
/// this addition cannot change what the tested training path does.
fn forward_pass(
stream: &Arc<CudaStream>,
blas: &CudaBlas,
kernels: &Kernels,
layers: &mut [DevLayer],
net: &Network,
input: &CudaSlice<f32>,
b: usize,
) -> Result<(), NetworkError> {
for i in 0..layers.len() {
let units = if i == 0 {
net.input_size
} else {
net.layers[i - 1].weights.rows
};
let out_units = net.layers[i].weights.rows;
if i == 0 {
let l = &mut layers[0];
gemm(blas, input, &l.w, &mut l.a, b, units, out_units)?;
} else {
let (left, right) = layers.split_at_mut(i);
let previous = &left[i - 1].a;
let l = &mut right[0];
gemm(blas, previous, &l.w, &mut l.a, b, units, out_units)?;
}
let f = &kernels.bias_act;
let l = &mut layers[i];
unsafe {
stream
.launch_builder(f)
.arg(&mut l.a)
.arg(&l.b)
.arg(&(b as i32))
.arg(&(net.layers[i].weights.rows as i32))
.arg(&act(net.layers[i].activation)?)
.launch(cfg(b * net.layers[i].weights.rows))
.map_err(cuda_err("bias/activation kernel"))?;
}
}
Ok(())
}
fn gemm(
blas: &CudaBlas,
a: &CudaSlice<f32>,
w: &CudaSlice<f32>,
out: &mut CudaSlice<f32>,
batch: usize,
input: usize,
units: usize,
) -> Result<(), NetworkError> {
let c = GemmConfig {
// `w` is stored row-major as [units, input]. cuBLAS reads the same
// bytes as column-major [input, units] (W^T), so OP_T is required to
// compute Z^T = W * X^T. OP_N would also typecheck here, and is
// wrong: it multiplies that column-major reinterpretation of W, which
// only coincides with W itself when the layer is square.
transa: cublasOperation_t::CUBLAS_OP_T,
transb: cublasOperation_t::CUBLAS_OP_N,
m: units as i32,
n: batch as i32,
k: input as i32,
alpha: 1.,
lda: input as i32,
ldb: input as i32,
beta: 0.,
ldc: units as i32,
};
unsafe { blas.gemm(c, w, a, out).map_err(cuda_err("cuBLAS GEMM")) }
}
/// `gw[out, in] = (1/batch) * d[batch, out]^T . x[batch, in]`, all row-major.
///
/// cuBLAS is column-major, so a row-major `[r, c]` buffer is read as `[c, r]`.
/// The result is written as its own transpose: column-major `[in, out]` with
/// leading dimension `in` is exactly row-major `[out, in]`.
fn grad_weights(
blas: &CudaBlas,
d: &CudaSlice<f32>,
x: &CudaSlice<f32>,
gw: &mut CudaSlice<f32>,
batch: usize,
out: usize,
input: usize,
) -> Result<(), NetworkError> {
let c = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_T,
m: input as i32,
n: out as i32,
k: batch as i32,
alpha: 1.0 / batch as f32,
lda: input as i32,
ldb: out as i32,
beta: 0.,
ldc: input as i32,
};
unsafe {
blas.gemm(c, x, d, gw)
.map_err(cuda_err("cuBLAS weight-gradient GEMM"))
}
}
/// `prev[batch, in] = next[batch, out] . w[out, in]`, all row-major. The
/// activation derivative is applied afterwards by the `act_derivative` kernel.
fn propagate_delta(
blas: &CudaBlas,
next: &CudaSlice<f32>,
w: &CudaSlice<f32>,
prev: &mut CudaSlice<f32>,
batch: usize,
out: usize,
input: usize,
) -> Result<(), NetworkError> {
let c = GemmConfig {
transa: cublasOperation_t::CUBLAS_OP_N,
transb: cublasOperation_t::CUBLAS_OP_N,
m: input as i32,
n: batch as i32,
k: out as i32,
alpha: 1.,
lda: input as i32,
ldb: out as i32,
beta: 0.,
ldc: input as i32,
};
unsafe {
blas.gemm(c, w, next, prev)
.map_err(cuda_err("cuBLAS hidden-delta GEMM"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Network, Optimizer};
#[test]
fn memory_estimate_is_nonzero_and_budget_is_checked_before_cuda() {
let model = Network::builder()
.input_size(2)
.dense(3, Activation::Relu)
.dense(1, Activation::Linear)
.optimizer(Optimizer::sgd(0.1))
.build();
assert!(estimate_tensor_memory_mib(&model, 4).unwrap() >= 1);
let data = Dataset::new(vec![vec![0.0, 0.0]], vec![vec![0.0]]);
let error = fit_cuda(
&mut model.clone(),
&data,
TrainConfig {
epochs: 1,
batch_size: 1,
shuffle: false,
seed: Some(1),
},
usize::MAX,
0,
)
.unwrap_err();
assert!(matches!(error, NetworkError::CudaMemoryBudget { .. }));
}
fn cuda_or_skip() -> bool {
match cuda_doctor(0, 8192) {
Ok(_) => true,
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
skip_without_cuda();
false
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
fn assert_update_parity(optimizer: Optimizer, tolerance: f32) {
if !cuda_or_skip() {
return;
}
let data = Dataset::new(
vec![vec![0.2, -0.3], vec![0.4, 0.1]],
vec![vec![0.7], vec![-0.2]],
);
let base = Network::builder()
.input_size(2)
.dense(3, Activation::Tanh)
.dense(1, Activation::Linear)
.loss(Loss::Mse)
.optimizer(optimizer)
.seed(19)
.build();
let mut cpu = base.clone();
let mut gpu = base;
cpu.train_batch(&data.inputs, &data.targets).unwrap();
gpu.fit_with_backend(
&data,
TrainConfig {
epochs: 1,
batch_size: 2,
shuffle: false,
seed: Some(3),
},
crate::TrainingBackend::Cuda {
device: 0,
memory_budget_mib: 8192,
},
)
.unwrap();
for (a, b) in cpu.layers.iter().zip(&gpu.layers) {
for (x, y) in a
.weights
.data
.iter()
.chain(&a.biases.data)
.zip(b.weights.data.iter().chain(&b.biases.data))
{
assert!(
(x - y).abs() <= tolerance,
"CPU/GPU update differs: {x} vs {y}"
);
}
}
}
#[test]
fn cuda_sgd_one_batch_update_matches_cpu_or_skips_without_device() {
assert_update_parity(Optimizer::sgd(0.01), 1e-5);
}
#[test]
fn cuda_adam_one_batch_update_matches_cpu_or_skips_without_device() {
assert_update_parity(Optimizer::adam(0.01), 2e-5);
}
#[test]
fn cuda_adam_weight_decay_one_batch_update_matches_cpu_or_skips_without_device() {
assert_update_parity(Optimizer::adam_with_weight_decay(0.01, 0.01), 2e-5);
}
#[test]
fn cuda_lion_one_batch_update_matches_cpu_or_skips_without_device() {
assert_update_parity(Optimizer::lion_with_weight_decay(0.01, 0.1), 2e-5);
}
/// This deliberately exercises the failure mode that a one-layer/one-batch
/// test misses: rectangular dense layers, a short final batch, shuffling,
/// and a complete optimizer epoch. Keep this fixture aligned with the
/// production universe feature width when RustingTrade changes it.
fn assert_full_epoch_parity(optimizer: Optimizer, tolerance: f32) {
if !cuda_or_skip() {
return;
}
let inputs: Vec<Vec<f32>> = (0..19)
.map(|row| {
(0..8)
.map(|col| ((row * 13 + col * 7) as f32 - 80.0) / 37.0)
.collect()
})
.collect();
let targets: Vec<Vec<f32>> = inputs
.iter()
.map(|x| vec![0.3 * x[0] - 0.2 * x[3] + 0.1 * x[6] + 0.05])
.collect();
let data = Dataset::new(inputs, targets);
let base = Network::builder()
.input_size(8)
.dense(13, Activation::Relu)
.dense(7, Activation::Tanh)
.dense(1, Activation::Linear)
.loss(Loss::Mse)
.optimizer(optimizer)
.seed(0x5eed)
.build();
let config = TrainConfig {
epochs: 1,
batch_size: 6,
shuffle: true,
seed: Some(0x1234),
};
let mut cpu = base.clone();
let mut gpu = base;
let cpu_history = cpu.fit(&data, config).unwrap();
let gpu_history = gpu
.fit_with_backend(
&data,
config,
crate::TrainingBackend::Cuda {
device: 0,
memory_budget_mib: 8192,
},
)
.unwrap();
assert!(
(cpu_history.losses[0] - gpu_history.losses[0]).abs() <= tolerance,
"full-epoch loss differs: CPU={} GPU={}",
cpu_history.losses[0],
gpu_history.losses[0]
);
for (cpu_layer, gpu_layer) in cpu.layers.iter().zip(&gpu.layers) {
for (left, right) in cpu_layer
.weights
.data
.iter()
.chain(&cpu_layer.biases.data)
.zip(gpu_layer.weights.data.iter().chain(&gpu_layer.biases.data))
{
assert!(
(left - right).abs() <= tolerance,
"full-epoch parameter mismatch: CPU={left} GPU={right}"
);
}
}
}
#[test]
fn cuda_full_network_sgd_epoch_matches_cpu_or_skips_without_device() {
assert_full_epoch_parity(Optimizer::sgd(0.01), 1e-5);
}
#[test]
fn cuda_full_network_adam_epoch_matches_cpu_or_skips_without_device() {
assert_full_epoch_parity(Optimizer::adam(0.005), 2e-5);
}
#[test]
fn persistent_session_checkpoint_resume_matches_uninterrupted_or_skips() {
if !cuda_or_skip() {
return;
}
let data = Dataset::new(
(0..17)
.map(|i| vec![i as f32 / 17.0, (i % 3) as f32])
.collect(),
(0..17).map(|i| vec![(i % 5) as f32 / 5.0]).collect(),
);
let base = Network::builder()
.input_size(2)
.dense(5, Activation::Tanh)
.dense(1, Activation::Linear)
.optimizer(Optimizer::adam(0.003))
.seed(7)
.build();
let config = TrainConfig {
epochs: 2,
batch_size: 6,
shuffle: true,
seed: Some(91),
};
let mut uninterrupted = CudaTrainingSession::new(&base, &data, config, 0, 8192).unwrap();
uninterrupted.train_epoch().unwrap();
uninterrupted.train_epoch().unwrap();
let expected = uninterrupted.checkpoint().unwrap();
let mut first = CudaTrainingSession::new(&base, &data, config, 0, 8192).unwrap();
first.train_epoch().unwrap();
assert_eq!(first.stats().device_to_host_bytes, size_of::<f32>());
let checkpoint = first.checkpoint().unwrap();
let mut resumed =
CudaTrainingSession::from_checkpoint(checkpoint, &data, config, 0, 8192).unwrap();
resumed.train_epoch().unwrap();
let actual = resumed.checkpoint().unwrap();
assert_eq!(actual.optimizer_step, expected.optimizer_step);
for (a, b) in actual.layers.iter().zip(&expected.layers) {
for (x, y) in a
.weights
.data
.iter()
.chain(&a.biases.data)
.zip(b.weights.data.iter().chain(&b.biases.data))
{
assert!(
(x - y).abs() <= 2e-5,
"resumed parameter differs: {x} vs {y}"
);
}
}
assert!(resumed.stats().peak_allocated_bytes > 0);
assert_eq!(resumed.stats().epochs, 1);
}
/// P09: `validate_epoch`'s device-side forward pass must agree with the
/// CPU `Network::evaluate_loss` path it replaces, or early stopping
/// would silently start deciding on wrong numbers. Trains a couple of
/// epochs first so weights are not at their (numerically forgiving)
/// random init, then checks both paths against the same, unsynced,
/// device-resident weights.
#[test]
fn cuda_validate_epoch_matches_cpu_evaluate_loss_or_skips_without_device() {
if !cuda_or_skip() {
return;
}
let train = Dataset::new(
(0..23)
.map(|i| vec![(i as f32 - 11.0) / 9.0, ((i * 3) % 7) as f32 / 7.0])
.collect(),
(0..23)
.map(|i| vec![((i * 5) % 11) as f32 / 11.0 - 0.5])
.collect(),
);
let validation = Dataset::new(
(0..13)
.map(|i| vec![(i as f32 - 6.0) / 5.0, ((i * 2) % 5) as f32 / 5.0])
.collect(),
(0..13)
.map(|i| vec![((i * 3) % 7) as f32 / 7.0 - 0.3])
.collect(),
);
let base = Network::builder()
.input_size(2)
.dense(6, Activation::Tanh)
.dense(1, Activation::Linear)
.loss(Loss::Mse)
.optimizer(Optimizer::adam(0.01))
.seed(0xC0FFEE)
.build();
let config = TrainConfig {
epochs: 1,
batch_size: 5,
shuffle: true,
seed: Some(42),
};
let mut session = CudaTrainingSession::new(&base, &train, config, 0, 8192).unwrap();
assert!(session.prepare_validation(&validation, 8192).unwrap());
session.train_epoch().unwrap();
session.train_epoch().unwrap();
let device_loss = session.validate_epoch().unwrap();
let mut host_network = base;
session.synchronize_network(&mut host_network).unwrap();
let cpu_loss = host_network.evaluate_loss(&validation).unwrap();
assert!(
(device_loss - cpu_loss).abs() <= 1e-5,
"device validation loss {device_loss} vs CPU {cpu_loss}"
);
// validate_epoch must not have perturbed training: another epoch and
// checkpoint should still look like ordinary continued training.
let loss_after = session.train_epoch().unwrap();
assert!(loss_after.is_finite());
}
}