use crate::error::{TptpError, TptpResult};
use crate::kernel::{KernelConfig, KernelResult, PrimitiveKernel};
use crate::memory::{BufferFlags, DType, GpuBuffer, Shape};
use crate::vendor::VendorBackend;
use std::time::Instant;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FusedActivation {
None,
Relu,
Gelu,
Silu, Tanh,
}
impl std::fmt::Display for FusedActivation {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FusedActivation::None => write!(f, "none"),
FusedActivation::Relu => write!(f, "relu"),
FusedActivation::Gelu => write!(f, "gelu"),
FusedActivation::Silu => write!(f, "silu"),
FusedActivation::Tanh => write!(f, "tanh"),
}
}
}
impl FusedActivation {
fn apply(self, x: f32) -> f32 {
match self {
FusedActivation::None => x,
FusedActivation::Relu => x.max(0.0),
FusedActivation::Gelu => {
const SQRT_2_OVER_PI: f32 = 0.797_884_6;
0.5 * x * (1.0 + (SQRT_2_OVER_PI * (x + 0.044715 * x.powi(3))).tanh())
}
FusedActivation::Silu => x / (1.0 + (-x).exp()),
FusedActivation::Tanh => x.tanh(),
}
}
}
#[derive(Debug, Clone)]
pub struct FusedGemmParams {
pub tile_m: u32,
pub tile_n: u32,
pub tile_k: u32,
pub vec_width: u32,
pub unroll: u32,
pub num_warps: u32,
}
impl Default for FusedGemmParams {
fn default() -> Self {
FusedGemmParams {
tile_m: 128,
tile_n: 128,
tile_k: 32,
vec_width: 8,
unroll: 4,
num_warps: 8,
}
}
}
impl FusedGemmParams {
pub fn for_problem_size(m: usize, n: usize, k: usize) -> Self {
if k <= 1024 && (m >= 2048 || n >= 2048) {
FusedGemmParams {
tile_m: 128,
tile_n: 128,
tile_k: 32,
vec_width: 8,
unroll: 4,
num_warps: 8,
}
} else if m <= 512 && n <= 512 {
FusedGemmParams {
tile_m: 64,
tile_n: 64,
tile_k: 16,
vec_width: 4,
unroll: 2,
num_warps: 4,
}
} else if m >= 4096 && n >= 4096 && k >= 4096 {
FusedGemmParams {
tile_m: 256,
tile_n: 128,
tile_k: 64,
vec_width: 8,
unroll: 8,
num_warps: 16,
}
} else {
Self::default()
}
}
}
pub struct FusedGemmKernel {
#[allow(dead_code)]
config: KernelConfig,
#[allow(dead_code)]
vendor: VendorBackend,
pub params: FusedGemmParams,
pub activation: FusedActivation,
}
impl FusedGemmKernel {
pub fn new(activation: FusedActivation) -> Self {
let vendor = VendorBackend::detect();
let config = KernelConfig::new([128, 1, 1], [256, 1, 1]);
FusedGemmKernel {
config,
vendor,
params: FusedGemmParams::default(),
activation,
}
}
pub fn with_vendor(vendor: VendorBackend, activation: FusedActivation) -> Self {
let config = KernelConfig::new([128, 1, 1], [256, 1, 1]);
FusedGemmKernel {
config,
vendor,
params: FusedGemmParams::default(),
activation,
}
}
pub fn with_params(mut self, params: FusedGemmParams) -> Self {
self.params = params;
self
}
pub fn with_config(mut self, config: KernelConfig) -> Self {
self.config = config;
self
}
pub fn execute_with_bias(
&self,
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
bias: &GpuBuffer<f32>,
mut c: Option<&mut GpuBuffer<f32>>,
alpha: f32,
) -> TptpResult<GpuBuffer<f32>> {
if a.ndim() != 2 || b.ndim() != 2 {
return Err(TptpError::shape_error("Fused GEMM requires 2D matrices"));
}
let m = a
.dim(0)
.ok_or_else(|| TptpError::shape_error("A has no dim 0"))?;
let k_a = a
.dim(1)
.ok_or_else(|| TptpError::shape_error("A has no dim 1"))?;
let k_b = b
.dim(0)
.ok_or_else(|| TptpError::shape_error("B has no dim 0"))?;
let n = b
.dim(1)
.ok_or_else(|| TptpError::shape_error("B has no dim 1"))?;
if k_a != k_b {
return Err(TptpError::ShapeError {
message: format!(
"inner dimensions must match: A is {}x{}, B is {}x{}",
m, k_a, k_b, n
),
expected: Some(k_a.to_string()),
got: Some(k_b.to_string()),
});
}
let k = k_a;
if bias.dim(0) != Some(n) {
return Err(TptpError::shape_error(format!(
"bias dimension {} does not match N={}",
bias.dim(0).unwrap_or(0),
n
)));
}
let mut output_owned: Option<GpuBuffer<f32>> = None;
let output: &mut GpuBuffer<f32> = if let Some(ref mut c) = c {
if c.dim(0) != Some(m) || c.dim(1) != Some(n) {
return Err(TptpError::shape_error(format!(
"C shape [{},{}] does not match output [{},{}]",
c.dim(0).unwrap_or(0),
c.dim(1).unwrap_or(0),
m,
n
)));
}
c
} else {
output_owned = Some(GpuBuffer::new(
Shape::dim2(m, n),
DType::F32,
BufferFlags::STORAGE,
)?);
output_owned.as_mut().expect("just initialized")
};
let t0 = Instant::now();
let params = FusedGemmParams::for_problem_size(m, n, k);
self.tptir_fused_gemm_with_bias(a, b, bias, output, alpha, m, n, k, ¶ms)?;
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
log::debug!(
"Fused GEMM {}x{}x{} + bias + {} via TPTIR: {:.3}ms (tile={}x{}x{}, vec={}, unroll={})",
m,
n,
k,
self.activation,
elapsed_ms,
params.tile_m,
params.tile_n,
params.tile_k,
params.vec_width,
params.unroll
);
match c {
Some(c) => Ok(c.clone()),
None => Ok(output_owned.expect("output_owned initialized when c is None")),
}
}
pub fn execute(
&self,
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
mut c: Option<&mut GpuBuffer<f32>>,
alpha: f32,
) -> TptpResult<GpuBuffer<f32>> {
if a.ndim() != 2 || b.ndim() != 2 {
return Err(TptpError::shape_error("Fused GEMM requires 2D matrices"));
}
let m = a
.dim(0)
.ok_or_else(|| TptpError::shape_error("A has no dim 0"))?;
let k_a = a
.dim(1)
.ok_or_else(|| TptpError::shape_error("A has no dim 1"))?;
let k_b = b
.dim(0)
.ok_or_else(|| TptpError::shape_error("B has no dim 0"))?;
let n = b
.dim(1)
.ok_or_else(|| TptpError::shape_error("B has no dim 1"))?;
if k_a != k_b {
return Err(TptpError::ShapeError {
message: format!(
"inner dimensions must match: A is {}x{}, B is {}x{}",
m, k_a, k_b, n
),
expected: Some(k_a.to_string()),
got: Some(k_b.to_string()),
});
}
let k = k_a;
let mut output_owned: Option<GpuBuffer<f32>> = None;
let output: &mut GpuBuffer<f32> = if let Some(ref mut c) = c {
if c.dim(0) != Some(m) || c.dim(1) != Some(n) {
return Err(TptpError::shape_error(format!(
"C shape [{},{}] does not match output [{},{}]",
c.dim(0).unwrap_or(0),
c.dim(1).unwrap_or(0),
m,
n
)));
}
c
} else {
output_owned = Some(GpuBuffer::new(
Shape::dim2(m, n),
DType::F32,
BufferFlags::STORAGE,
)?);
output_owned.as_mut().expect("just initialized")
};
let t0 = Instant::now();
let params = FusedGemmParams::for_problem_size(m, n, k);
self.tptir_fused_gemm(a, b, output, alpha, m, n, k, ¶ms)?;
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
log::debug!(
"Fused GEMM {}x{}x{} + {} via TPTIR: {:.3}ms (tile={}x{}x{}, vec={}, unroll={})",
m,
n,
k,
self.activation,
elapsed_ms,
params.tile_m,
params.tile_n,
params.tile_k,
params.vec_width,
params.unroll
);
match c {
Some(c) => Ok(c.clone()),
None => Ok(output_owned.expect("output_owned initialized when c is None")),
}
}
#[allow(clippy::too_many_arguments)]
fn tptir_fused_gemm_with_bias(
&self,
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
bias: &GpuBuffer<f32>,
c: &mut GpuBuffer<f32>,
alpha: f32,
m: usize,
n: usize,
k: usize,
params: &FusedGemmParams,
) -> TptpResult<()> {
log::debug!(
"TPTIR Fused GEMM with bias: M={}, N={}, K={}, activation={}, tile={}x{}x{}, vec_width={}, unroll={}",
m,
n,
k,
self.activation,
params.tile_m,
params.tile_n,
params.tile_k,
params.vec_width,
params.unroll
);
let mut a_raw = vec![0.0f32; m * k];
a.copy_to_host(&mut a_raw)?;
let mut b_raw = vec![0.0f32; k * n];
b.copy_to_host(&mut b_raw)?;
let mut bias_raw = vec![0.0f32; n];
bias.copy_to_host(&mut bias_raw)?;
let mut c_raw = vec![0.0f32; m * n];
for i in 0..m {
for j in 0..n {
let mut acc = 0.0f32;
for kk in 0..k {
acc += a_raw[i * k + kk] * b_raw[kk * n + j];
}
c_raw[i * n + j] = self.activation.apply(alpha * acc + bias_raw[j]);
}
}
c.copy_from_host(&c_raw)
}
#[allow(clippy::too_many_arguments)]
fn tptir_fused_gemm(
&self,
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
c: &mut GpuBuffer<f32>,
alpha: f32,
m: usize,
n: usize,
k: usize,
params: &FusedGemmParams,
) -> TptpResult<()> {
log::debug!(
"TPTIR Fused GEMM: M={}, N={}, K={}, activation={}, tile={}x{}x{}, vec_width={}, unroll={}",
m,
n,
k,
self.activation,
params.tile_m,
params.tile_n,
params.tile_k,
params.vec_width,
params.unroll
);
let mut a_raw = vec![0.0f32; m * k];
a.copy_to_host(&mut a_raw)?;
let mut b_raw = vec![0.0f32; k * n];
b.copy_to_host(&mut b_raw)?;
let mut c_raw = vec![0.0f32; m * n];
for i in 0..m {
for j in 0..n {
let mut acc = 0.0f32;
for kk in 0..k {
acc += a_raw[i * k + kk] * b_raw[kk * n + j];
}
c_raw[i * n + j] = self.activation.apply(alpha * acc);
}
}
c.copy_from_host(&c_raw)?;
Ok(())
}
}
impl PrimitiveKernel for FusedGemmKernel {
fn name(&self) -> &str {
"fused_gemm"
}
fn supported_dtypes(&self) -> &[DType] {
&[DType::F32, DType::F16, DType::BF16]
}
fn can_execute(&self, inputs: &[&GpuBuffer<f32>]) -> bool {
inputs.len() >= 2 && inputs[0].ndim() == 2 && inputs[1].ndim() == 2
}
fn default_config(&self) -> KernelConfig {
KernelConfig::new([128, 1, 1], [256, 1, 1])
}
fn execute(
&self,
inputs: &[&GpuBuffer<f32>],
output: &mut GpuBuffer<f32>,
_config: &KernelConfig,
) -> TptpResult<KernelResult> {
let a = inputs[0];
let b = inputs[1];
let m = a.dim(0).unwrap_or(0);
let n = b.dim(1).unwrap_or(0);
let t0 = Instant::now();
let params = FusedGemmParams::for_problem_size(m, n, a.dim(1).unwrap_or(0));
self.tptir_fused_gemm(a, b, output, 1.0, m, n, a.dim(1).unwrap_or(0), ¶ms)?;
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
Ok(KernelResult {
outputs: vec![],
execution_time_ms: Some(elapsed_ms),
backend_used: "tptir-fused".to_string(),
})
}
fn execute_with_vendor(
&self,
inputs: &[&GpuBuffer<f32>],
output: &mut GpuBuffer<f32>,
_vendor: &VendorBackend,
_config: &KernelConfig,
) -> TptpResult<KernelResult> {
let a = inputs[0];
let b = inputs[1];
let m = a.dim(0).unwrap_or(0);
let n = b.dim(1).unwrap_or(0);
let t0 = Instant::now();
let params = FusedGemmParams::for_problem_size(m, n, a.dim(1).unwrap_or(0));
self.tptir_fused_gemm(a, b, output, 1.0, m, n, a.dim(1).unwrap_or(0), ¶ms)?;
let elapsed_ms = t0.elapsed().as_secs_f64() * 1000.0;
Ok(KernelResult {
outputs: vec![],
execution_time_ms: Some(elapsed_ms),
backend_used: "tptir-fused".to_string(),
})
}
}
pub fn fused_gemm_relu(
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
alpha: f32,
) -> TptpResult<GpuBuffer<f32>> {
FusedGemmKernel::new(FusedActivation::Relu).execute(a, b, None, alpha)
}
pub fn fused_gemm_bias_relu(
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
bias: &GpuBuffer<f32>,
alpha: f32,
) -> TptpResult<GpuBuffer<f32>> {
FusedGemmKernel::new(FusedActivation::Relu).execute_with_bias(a, b, bias, None, alpha)
}
pub fn fused_gemm_gelu(
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
alpha: f32,
) -> TptpResult<GpuBuffer<f32>> {
FusedGemmKernel::new(FusedActivation::Gelu).execute(a, b, None, alpha)
}
pub fn fused_gemm_silu(
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
alpha: f32,
) -> TptpResult<GpuBuffer<f32>> {
FusedGemmKernel::new(FusedActivation::Silu).execute(a, b, None, alpha)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fused_gemm_returns_computed_buffer() {
let mut a =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
a.copy_from_host(&[1.0, 0.0, 0.0, 1.0]).unwrap();
let mut b =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
b.copy_from_host(&[3.0, 4.0, 5.0, 6.0]).unwrap();
let mut c =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
let kernel = FusedGemmKernel::new(FusedActivation::None);
let out = kernel.execute(&a, &b, Some(&mut c), 1.0).unwrap();
let mut data = [0f32; 4];
out.copy_to_host(&mut data).unwrap();
assert_eq!(data, [3.0, 4.0, 5.0, 6.0]);
}
#[test]
fn test_fused_gemm_with_bias_returns_computed_buffer() {
let mut a =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
a.copy_from_host(&[1.0, 0.0, 0.0, 1.0]).unwrap();
let mut b =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
b.copy_from_host(&[2.0, 3.0, 4.0, 5.0]).unwrap();
let mut bias =
GpuBuffer::<f32>::new(Shape::dim2(2, 1), DType::F32, BufferFlags::STORAGE).unwrap();
bias.copy_from_host(&[1.0, 2.0]).unwrap();
let mut c =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
let kernel = FusedGemmKernel::new(FusedActivation::None);
let out = kernel
.execute_with_bias(&a, &b, &bias, Some(&mut c), 1.0)
.unwrap();
let mut data = [0f32; 4];
out.copy_to_host(&mut data).unwrap();
assert_eq!(data, [3.0, 5.0, 5.0, 7.0]);
}
#[test]
fn test_fused_gemm_returns_local_buffer_when_no_c() {
let a = GpuBuffer::<f32>::new(Shape::dim2(2, 3), DType::F32, BufferFlags::STORAGE).unwrap();
let b = GpuBuffer::<f32>::new(Shape::dim2(3, 2), DType::F32, BufferFlags::STORAGE).unwrap();
let kernel = FusedGemmKernel::new(FusedActivation::None);
let out = kernel.execute(&a, &b, None, 1.0).unwrap();
assert_eq!(out.shape(), &Shape::dim2(2, 2));
}
#[test]
fn test_fused_gemm_computes_real_product_no_activation() {
let mut a =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
a.copy_from_host(&[1.0, 2.0, 3.0, 4.0]).unwrap();
let mut b =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
b.copy_from_host(&[5.0, 6.0, 7.0, 8.0]).unwrap();
let kernel = FusedGemmKernel::new(FusedActivation::None);
let out = kernel.execute(&a, &b, None, 1.0).unwrap();
let mut data = [0f32; 4];
out.copy_to_host(&mut data).unwrap();
assert_eq!(data, [19.0, 22.0, 43.0, 50.0]);
}
#[test]
fn test_fused_gemm_relu_zeros_negatives() {
let mut a =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
a.copy_from_host(&[1.0, -1.0, -1.0, 1.0]).unwrap();
let mut b =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
b.copy_from_host(&[1.0, 0.0, 0.0, 1.0]).unwrap();
let kernel = FusedGemmKernel::new(FusedActivation::Relu);
let out = kernel.execute(&a, &b, None, 1.0).unwrap();
let mut data = [0f32; 4];
out.copy_to_host(&mut data).unwrap();
assert_eq!(data, [1.0, 0.0, 0.0, 1.0]);
}
#[test]
fn test_fused_gemm_with_bias_computes_real_product() {
let mut a =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
a.copy_from_host(&[1.0, 0.0, 0.0, 1.0]).unwrap();
let mut b =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
b.copy_from_host(&[2.0, 3.0, 4.0, 5.0]).unwrap();
let mut bias =
GpuBuffer::<f32>::new(Shape::dim2(2, 1), DType::F32, BufferFlags::STORAGE).unwrap();
bias.copy_from_host(&[1.0, 2.0]).unwrap();
let kernel = FusedGemmKernel::new(FusedActivation::None);
let out = kernel.execute_with_bias(&a, &b, &bias, None, 1.0).unwrap();
let mut data = [0f32; 4];
out.copy_to_host(&mut data).unwrap();
assert_eq!(data, [3.0, 5.0, 5.0, 7.0]);
}
}