use cudarc::driver::{CudaContext, CudaFunction};
use cudarc::nvrtc::{CompileOptions, compile_ptx_with_opts};
use std::sync::Arc;
const FP8_MMA_ARCH: i32 = 89;
pub(crate) const TILE: usize = 64;
pub(crate) const KERNELS: &str = r#"
#define FP8_LD 48
__device__ __forceinline__ void fp8_stage(unsigned char*s,const unsigned char*g,
int count,int inner,int first,int k0){
int r=threadIdx.x>>1,c=(threadIdx.x&1)*16,row=first+r,k=k0+c;
uint4 v=make_uint4(0u,0u,0u,0u);
if(row<count){
const unsigned char*p=g+(size_t)row*inner+k;
if((inner&15)==0&&k+16<=inner)v=*(const uint4*)p;
else{
unsigned w[4]={0u,0u,0u,0u};
for(int i=0;i<16&&k+i<inner;i++)w[i>>2]|=(unsigned)p[i]<<(8*(i&3));
v=make_uint4(w[0],w[1],w[2],w[3]);
}
}
*(uint4*)(s+r*FP8_LD+c)=v;
}
extern "C" __global__ __launch_bounds__(128) void gemm_fp8_mma(
const unsigned char* __restrict__ xq,const float* __restrict__ xs,
const unsigned char* __restrict__ w,const float* __restrict__ ws,
float* __restrict__ y,int rows,int units,int inner,float beta)
{
__shared__ __align__(16) unsigned char sa[64*FP8_LD],sb[64*FP8_LD];
int warp=threadIdx.x>>5,lane=threadIdx.x&31,g=lane>>2,t=lane&3;
int m0=blockIdx.y*64,n0=blockIdx.x*64,wm=(warp>>1)*32,wn=(warp&1)*32;
float d[2][4][4];
#pragma unroll
for(int i=0;i<2;i++)
#pragma unroll
for(int j=0;j<4;j++)
#pragma unroll
for(int e=0;e<4;e++)d[i][j][e]=0.f;
for(int k0=0;k0<inner;k0+=32){
fp8_stage(sa,xq,rows,inner,m0,k0);
fp8_stage(sb,w,units,inner,n0,k0);
__syncthreads();
unsigned a[2][4],b[4][2];
#pragma unroll
for(int i=0;i<2;i++){
const unsigned char*p=sa+(wm+i*16+g)*FP8_LD+t*4;
a[i][0]=*(const unsigned*)p;
a[i][1]=*(const unsigned*)(p+8*FP8_LD);
a[i][2]=*(const unsigned*)(p+16);
a[i][3]=*(const unsigned*)(p+8*FP8_LD+16);
}
#pragma unroll
for(int j=0;j<4;j++){
const unsigned char*p=sb+(wn+j*8+g)*FP8_LD+t*4;
b[j][0]=*(const unsigned*)p;
b[j][1]=*(const unsigned*)(p+16);
}
#pragma unroll
for(int i=0;i<2;i++)
#pragma unroll
for(int j=0;j<4;j++)
asm volatile("mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
"{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};"
:"+f"(d[i][j][0]),"+f"(d[i][j][1]),"+f"(d[i][j][2]),"+f"(d[i][j][3])
:"r"(a[i][0]),"r"(a[i][1]),"r"(a[i][2]),"r"(a[i][3]),"r"(b[j][0]),"r"(b[j][1]));
__syncthreads();
}
#pragma unroll
for(int i=0;i<2;i++)
#pragma unroll
for(int j=0;j<4;j++)
#pragma unroll
for(int e=0;e<4;e++){
int row=m0+wm+i*16+g+(e>=2?8:0),col=n0+wn+j*8+2*t+(e&1);
if(row>=rows||col>=units)continue;
size_t o=(size_t)row*units+col;
float v=d[i][j][e]*xs[row]*ws[col];
y[o]=beta==0.f?v:v+beta*y[o];
}
}
"#;
pub(crate) fn enabled() -> bool {
std::env::var_os("RUSTING_BRAIN_FP8_MMA").is_some()
}
pub(crate) fn load(context: &Arc<CudaContext>) -> Option<CudaFunction> {
let (major, minor) = context.compute_capability().ok()?;
if major * 10 + minor < FP8_MMA_ARCH {
return None;
}
let arch = match crate::cuda_flash::nvrtc_arch() {
Some((number, name)) if number >= FP8_MMA_ARCH => name,
_ => "compute_89",
};
let options = CompileOptions {
arch: Some(arch),
..Default::default()
};
let ptx = compile_ptx_with_opts(KERNELS, options).ok()?;
context
.load_module(ptx)
.ok()?
.load_function("gemm_fp8_mma")
.ok()
}
#[cfg(test)]
mod tests {
use super::KERNELS;
use cudarc::nvrtc::result as nvrtc;
use std::ffi::CString;
fn nvrtc_compile(arch: &str) -> Result<(usize, String), String> {
let src = CString::new(KERNELS).unwrap();
let program = nvrtc::create_program(&src, None).map_err(|e| e.to_string())?;
let options = [
format!("--gpu-architecture={arch}"),
"--ptxas-options=-v".to_string(),
];
unsafe {
let compiled = nvrtc::compile_program(program, &options);
let log = nvrtc::get_program_log(program)
.map(|log| {
let bytes: Vec<u8> = log.iter().map(|&c| c as u8).collect();
String::from_utf8_lossy(&bytes)
.trim_end_matches('\0')
.to_string()
})
.unwrap_or_default();
let result = match compiled {
Err(error) => Err(format!("{error}: {log}")),
Ok(()) if arch.starts_with("sm_") => {
use cudarc::nvrtc::sys::{nvrtcGetCUBINSize, nvrtcResult};
let mut size = 0usize;
match nvrtcGetCUBINSize(program, &mut size) {
nvrtcResult::NVRTC_SUCCESS => Ok((size, log)),
other => Err(format!("{other:?}: {log}")),
}
}
Ok(()) => nvrtc::get_ptx(program)
.map(|ptx| (ptx.len(), log))
.map_err(|e| e.to_string()),
};
let _ = nvrtc::destroy_program(program);
result
}
}
#[test]
fn the_fp8_gemm_compiles_for_sm_89_and_sm_120_but_not_sm_86() {
if nvrtc::create_program(c"", None).is_err() {
eprintln!("skipped: NVRTC not available");
return;
}
for arch in ["compute_89", "compute_120", "sm_89", "sm_120"] {
match nvrtc_compile(arch) {
Ok((size, log)) => {
eprintln!("{arch}: {size} bytes\n{log}");
assert!(size > 0, "{arch}: empty output");
}
Err(log) => panic!("{arch} failed:\n{log}"),
}
}
let rejected = nvrtc_compile("sm_86");
eprintln!("sm_86: {:?}", rejected.as_ref().map(|(size, _)| size));
assert!(rejected.is_err(), "sm_86 accepted FP8 mma");
}
#[test]
fn the_fp8_gemm_is_not_loaded_below_sm_89_or_skips_without_device() {
let Ok(context) = cudarc::driver::CudaContext::new(0) else {
crate::cuda_training::skip_without_cuda();
return;
};
let (major, minor) = context.compute_capability().unwrap();
let loaded = super::load(&context);
eprintln!(
"device 0 is {major}.{minor}, FP8 GEMM loaded: {}",
loaded.is_some()
);
if major * 10 + minor < super::FP8_MMA_ARCH {
assert!(loaded.is_none());
} else {
assert!(
loaded.is_some(),
"an sm_89+ device failed to load the FP8 GEMM"
);
}
}
}