use crate::error::{TptpError, TptpResult};
use crate::kernel::KernelConfig;
use crate::memory::{BufferFlags, DType, GpuBuffer, Shape};
use crate::vendor::VendorBackend;
pub const DEFAULT_GROUP_SIZE: u32 = 128;
#[derive(Debug, Clone)]
pub struct QuantGemmParams {
pub bits: u8,
pub tile_m: u32,
pub tile_n: u32,
pub tile_k: u32,
pub group_size: u32,
}
impl Default for QuantGemmParams {
fn default() -> Self {
QuantGemmParams {
bits: 4,
tile_m: 64,
tile_n: 64,
tile_k: 32,
group_size: DEFAULT_GROUP_SIZE,
}
}
}
pub struct QuantGemmKernel {
#[allow(dead_code)]
config: KernelConfig,
vendor: VendorBackend,
pub params: QuantGemmParams,
}
impl QuantGemmKernel {
pub fn new() -> Self {
let vendor = VendorBackend::detect();
let config = KernelConfig::new([128, 1, 1], [256, 1, 1]);
QuantGemmKernel {
config,
vendor,
params: QuantGemmParams::default(),
}
}
pub fn with_vendor(vendor: VendorBackend) -> Self {
let config = KernelConfig::new([128, 1, 1], [256, 1, 1]);
QuantGemmKernel {
config,
vendor,
params: QuantGemmParams::default(),
}
}
pub fn with_params(mut self, params: QuantGemmParams) -> Self {
self.params = params;
self
}
pub fn execute(
&self,
a_packed: &GpuBuffer<i8>,
b_f32: &GpuBuffer<f32>,
scales: &GpuBuffer<f32>,
zpoints: &GpuBuffer<i8>,
) -> TptpResult<GpuBuffer<f32>> {
if b_f32.ndim() != 2 {
return Err(TptpError::shape_error(
"QuantGemm: activation B must be 2D [k, n]",
));
}
let k = b_f32
.dim(0)
.ok_or_else(|| TptpError::shape_error("B has no dim 0"))?;
let n = b_f32
.dim(1)
.ok_or_else(|| TptpError::shape_error("B has no dim 1"))?;
let pack_factor = (8usize / self.params.bits.max(1) as usize).max(1);
let packed_cols = k.div_ceil(pack_factor);
let m = a_packed.num_elements() / packed_cols;
if m == 0 {
return Err(TptpError::shape_error(
"QuantGemm: cannot infer M from packed weight buffer",
));
}
let mut output = GpuBuffer::<f32>::new(Shape::dim2(m, n), DType::F32, BufferFlags::STORAGE)
.map_err(|e| TptpError::device_error(e.to_string()))?;
if self.vendor.supports_int8_gemm() {
self.vendor.quant_gemm(
a_packed,
b_f32,
&mut output,
scales,
zpoints,
m,
n,
k,
self.params.bits,
)?;
} else {
self.tptir_fallback(a_packed, b_f32, &mut output, scales, zpoints, m, n, k)?;
}
Ok(output)
}
#[allow(clippy::too_many_arguments)]
fn tptir_fallback(
&self,
a_packed: &GpuBuffer<i8>,
b_f32: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
scales: &GpuBuffer<f32>,
zpoints: &GpuBuffer<i8>,
m: usize,
n: usize,
k: usize,
) -> TptpResult<()> {
let bits = self.params.bits as usize;
let group_size = self.params.group_size as usize;
let pack_factor = (8 / bits.max(1)).max(1);
let mask = ((1u16 << bits) - 1) as u8;
let mut a_raw = vec![0i8; a_packed.num_elements()];
let _ = a_packed.copy_to_host(&mut a_raw);
let mut sc_raw = vec![0.0f32; scales.num_elements()];
let _ = scales.copy_to_host(&mut sc_raw);
let mut zp_raw = vec![0i8; zpoints.num_elements()];
let _ = zpoints.copy_to_host(&mut zp_raw);
let packed_cols = k.div_ceil(pack_factor);
let groups_per_row = k.div_ceil(group_size);
let mut a_f32 = vec![0.0f32; m * k];
for row in 0..m {
for col in 0..k {
let packed_col = col / pack_factor;
let shift = (col % pack_factor) * bits;
let byte_idx = row * packed_cols + packed_col;
let raw_byte = if byte_idx < a_raw.len() {
a_raw[byte_idx] as u8
} else {
0
};
let raw = (raw_byte >> shift) & mask;
let group_idx = col / group_size;
let sc_idx = row * groups_per_row + group_idx;
let scale = if sc_idx < sc_raw.len() {
sc_raw[sc_idx]
} else {
1.0
};
let zp = if sc_idx < zp_raw.len() {
zp_raw[sc_idx] as f32
} else {
0.0
};
a_f32[row * k + col] = (raw as f32 - zp) * scale;
}
}
let mut b_raw = vec![0.0f32; k * n];
let _ = b_f32.copy_to_host(&mut b_raw);
let mut c_raw = vec![0.0f32; m * n];
for i in 0..m {
for j in 0..n {
let mut acc = 0.0f32;
for kk in 0..k {
acc += a_f32[i * k + kk] * b_raw[kk * n + j];
}
c_raw[i * n + j] = acc;
}
}
output
.copy_from_host(&c_raw)
.map_err(|e| TptpError::device_error(e.to_string()))
}
}
impl Default for QuantGemmKernel {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn quant_gemm_params_default() {
let p = QuantGemmParams::default();
assert_eq!(p.bits, 4);
assert_eq!(p.group_size, DEFAULT_GROUP_SIZE);
}
#[test]
fn fallback_2x2_identity_8bit() {
let kernel = QuantGemmKernel::new().with_params(QuantGemmParams {
bits: 8,
tile_m: 2,
tile_n: 2,
tile_k: 2,
group_size: 2,
});
let mut a_buf =
GpuBuffer::<i8>::new(Shape::dim2(2, 2), DType::I8, BufferFlags::STORAGE).unwrap();
a_buf.copy_from_host(&[1i8, 2, 3, 4]).unwrap();
let mut b_buf =
GpuBuffer::<f32>::new(Shape::dim2(2, 2), DType::F32, BufferFlags::STORAGE).unwrap();
b_buf.copy_from_host(&[1.0f32, 0.0, 0.0, 1.0]).unwrap();
let mut s_buf =
GpuBuffer::<f32>::new(Shape::dim2(2, 1), DType::F32, BufferFlags::STORAGE).unwrap();
s_buf.copy_from_host(&[1.0f32, 1.0]).unwrap();
let mut z_buf =
GpuBuffer::<i8>::new(Shape::dim2(2, 1), DType::I8, BufferFlags::STORAGE).unwrap();
z_buf.copy_from_host(&[0i8, 0]).unwrap();
let result = kernel.execute(&a_buf, &b_buf, &s_buf, &z_buf).unwrap();
let mut out = vec![0.0f32; 4];
result.copy_to_host(&mut out).unwrap();
assert!((out[0] - 1.0).abs() < 0.01, "C[0,0]={}", out[0]);
assert!((out[1] - 2.0).abs() < 0.01, "C[0,1]={}", out[1]);
assert!((out[2] - 3.0).abs() < 0.01, "C[1,0]={}", out[2]);
assert!((out[3] - 4.0).abs() < 0.01, "C[1,1]={}", out[3]);
}
}