use candle_core::quantized::k_quants::{self, GgmlType};
use candle_core::{bail, Result};
use half::f16;
pub fn decode_raw_to_f32(dtype: u32, bytes: &[u8], elems: usize) -> Result<Option<Vec<f32>>> {
use candle_core::quantized::k_quants::{
BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ4_1, BlockQ5K, BlockQ5_0, BlockQ5_1, BlockQ6K,
BlockQ8_0, BlockQ8_1, BlockQ8K,
};
macro_rules! kquant {
($ty:ty) => {{
let block_bytes = std::mem::size_of::<$ty>();
let blck = <$ty as GgmlType>::BLCK_SIZE;
if elems % blck != 0 || bytes.len() != elems / blck * block_bytes {
bail!(
"dtype {dtype}: {} bytes for {elems} elements ({} per {blck}-block)",
bytes.len(),
block_bytes
);
}
let blocks = unsafe {
std::slice::from_raw_parts(bytes.as_ptr() as *const $ty, bytes.len() / block_bytes)
};
let mut out = vec![0f32; elems];
<$ty as GgmlType>::to_float(blocks, &mut out);
out
}};
}
let out = match dtype {
0 => return Ok(None), 1 => {
if bytes.len() != elems * 2 {
bail!("dtype F16: {} bytes for {elems} elements", bytes.len());
}
bytes
.chunks_exact(2)
.map(|c| f16::from_le_bytes([c[0], c[1]]).to_f32())
.collect()
}
crate::iq2xxs::GGML_TYPE_IQ2_XXS => {
let blocks = crate::iq2xxs::blocks_from_bytes(bytes)?;
let mut out = vec![0f32; elems];
crate::iq2xxs::dequantize(blocks, &mut out)?;
out
}
2 => kquant!(BlockQ4_0),
3 => kquant!(BlockQ4_1),
6 => kquant!(BlockQ5_0),
7 => kquant!(BlockQ5_1),
8 => kquant!(BlockQ8_0),
9 => kquant!(BlockQ8_1),
10 => kquant!(BlockQ2K),
11 => kquant!(BlockQ3K),
12 => kquant!(BlockQ4K),
13 => kquant!(BlockQ5K),
14 => kquant!(BlockQ6K),
15 => kquant!(BlockQ8K),
_ => return Ok(None),
};
Ok(Some(out))
}
pub fn matmul_kquant<T: GgmlType>(
(m, k, n): (usize, usize, usize),
lhs: &[f32],
blocks: &[T],
dst: &mut [f32],
) -> Result<()> {
let blocks_per_row = validate(m, k, n, lhs, blocks, dst)?;
matmul_kquant_impl((m, k, n), blocks_per_row, lhs, blocks, dst, true)
}
#[cfg(test)]
pub fn matmul_kquant_serial<T: GgmlType>(
(m, k, n): (usize, usize, usize),
lhs: &[f32],
blocks: &[T],
dst: &mut [f32],
) -> Result<()> {
let blocks_per_row = validate(m, k, n, lhs, blocks, dst)?;
matmul_kquant_impl((m, k, n), blocks_per_row, lhs, blocks, dst, false)
}
fn validate<T: GgmlType>(
m: usize,
k: usize,
n: usize,
lhs: &[f32],
blocks: &[T],
dst: &mut [f32],
) -> Result<usize> {
if !k.is_multiple_of(T::BLCK_SIZE) {
bail!(
"quantized matmul: k={k} is not a multiple of the block size {}",
T::BLCK_SIZE
);
}
let blocks_per_row = k / T::BLCK_SIZE;
if blocks.len() != n * blocks_per_row {
bail!(
"quantized matmul: rhs holds {} blocks, expected {}",
blocks.len(),
n * blocks_per_row
);
}
if lhs.len() != m * k || dst.len() != m * n {
bail!(
"quantized matmul: lhs/dst sized {}/{} expected {}/{}",
lhs.len(),
dst.len(),
m * k,
m * n
);
}
Ok(blocks_per_row)
}
fn matmul_kquant_impl<T: GgmlType>(
mkn: (usize, usize, usize),
blocks_per_row: usize,
lhs: &[f32],
blocks: &[T],
dst: &mut [f32],
parallel: bool,
) -> Result<()> {
let (m, k, n) = mkn;
if m == 0 || n == 0 {
return Ok(());
}
if try_avx2_kquant(mkn, blocks_per_row, lhs, blocks, dst, parallel) {
return Ok(());
}
k_quants::matmul((m, k, n), lhs, blocks, dst)
}
#[cfg(target_arch = "x86_64")]
fn try_avx2_kquant<T: GgmlType>(
mkn: (usize, usize, usize),
blocks_per_row: usize,
lhs: &[f32],
blocks: &[T],
dst: &mut [f32],
parallel: bool,
) -> bool {
let (m, k, n) = mkn;
if !(crate::simd::avx2_fma_available() && k.is_multiple_of(8)) {
return false;
}
let block_bytes = unsafe {
std::slice::from_raw_parts(
blocks.as_ptr() as *const u8,
std::mem::size_of_val(blocks),
)
};
if crate::kquant_dot::try_matmul_fused_avx2(T::DTYPE, mkn, lhs, block_bytes, dst, parallel) {
return true;
}
let dst_ptr = crate::simd::DstPtr::new(dst);
let worker = move |rows: &[usize]| {
let mut wrow = vec![0f32; k];
for &row in rows {
T::to_float(&blocks[row * blocks_per_row..(row + 1) * blocks_per_row], &mut wrow);
unsafe { dot_row_avx2(m, k, n, lhs, &wrow, row, &dst_ptr) }
}
};
if parallel {
crate::simd::for_each_row_chunks(n, worker);
} else {
let rows: Vec<usize> = (0..n).collect();
worker(&rows);
}
true
}
#[cfg(not(target_arch = "x86_64"))]
fn try_avx2_kquant<T: GgmlType>(
_mkn: (usize, usize, usize),
_blocks_per_row: usize,
_lhs: &[f32],
_blocks: &[T],
_dst: &mut [f32],
_parallel: bool,
) -> bool {
false
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2,fma")]
#[allow(clippy::too_many_arguments)] unsafe fn dot_row_avx2(
m: usize,
k: usize,
n: usize,
lhs: &[f32],
wrow: &[f32],
row: usize,
dst: &crate::simd::DstPtr,
) {
use std::arch::x86_64::*;
const MTILE: usize = 4;
let mut m0 = 0;
while m0 < m {
let mcnt = (m - m0).min(MTILE);
let mut acc = [_mm256_setzero_ps(); MTILE];
let mut c = 0;
while c + 8 <= k {
let wv = _mm256_loadu_ps(wrow[c..c + 8].as_ptr());
for i in 0..mcnt {
let a = _mm256_loadu_ps(lhs[(m0 + i) * k + c..].as_ptr());
acc[i] = _mm256_fmadd_ps(a, wv, acc[i]);
}
c += 8;
}
for (i, acc_i) in acc.iter().enumerate().take(mcnt) {
dst.write((m0 + i) * n + row, crate::simd::hsum256(*acc_i));
}
m0 += MTILE;
}
}
#[cfg(test)]
mod tests {
use super::*;
use candle_core::quantized::k_quants::{BlockQ2K, BlockQ4K, BlockQ8_0};
use candle_core::quantized::{GgmlDType, QTensor};
use candle_core::{Device, Tensor};
fn quantized_blocks<T: GgmlType>(n: usize, k: usize, dtype: GgmlDType) -> Vec<T> {
let data: Vec<f32> = (0..n * k)
.map(|i| ((((i as u64) * 2654435761) % 100000) as f32 / 1000.0) - 50.0)
.collect();
let t = Tensor::from_vec(data, (n, k), &Device::Cpu).unwrap();
let qt = QTensor::quantize(&t, dtype).unwrap();
let bytes = qt.data().unwrap();
assert!(bytes.len().is_multiple_of(std::mem::size_of::<T>()));
let ptr = bytes.as_ptr() as *const T;
let len = bytes.len() / std::mem::size_of::<T>();
let blocks = unsafe { std::slice::from_raw_parts(ptr, len) };
blocks.to_vec()
}
fn run_case<T: GgmlType>(dtype: GgmlDType, m: usize, k: usize, n: usize) {
let blocks = quantized_blocks::<T>(n, k, dtype);
let lhs: Vec<f32> = (0..m * k)
.map(|i| (((i * 40503) % 1000) as f32 / 100.0) - 5.0)
.collect();
let mut fast = vec![0f32; m * n];
matmul_kquant::<T>((m, k, n), &lhs, &blocks, &mut fast).unwrap();
let mut w = vec![0f32; n * k];
T::to_float(&blocks, &mut w);
for (i, fastv) in fast.iter().enumerate() {
let mut acc = 0f32;
let (ii, jj) = (i / n, i % n);
for (kk, wv) in w[jj * k..(jj + 1) * k].iter().enumerate() {
acc += lhs[ii * k + kk] * wv;
}
let tol = 1e-4 * acc.abs().max(1.0);
assert!(
(fastv - acc).abs() <= tol,
"{dtype:?}: dst[{i}] fast={} f32-gemm={} (tol {tol})",
fast[i],
acc
);
}
}
#[test]
fn still_close_to_candle_quantised_activation_path() {
let (m, k, n) = (2, 256, 9);
let lhs: Vec<f32> = (0..m * k).map(|i| (i as f32) * 0.01 - 1.0).collect();
macro_rules! check {
($ty:ty, $dt:expr) => {{
let blocks = quantized_blocks::<$ty>(n, k, $dt);
let mut fast = vec![0f32; m * n];
matmul_kquant::<$ty>((m, k, n), &lhs, &blocks, &mut fast).unwrap();
let mut candle = vec![0f32; m * n];
k_quants::matmul((m, k, n), &lhs, &blocks, &mut candle).unwrap();
for (i, v) in fast.iter().enumerate() {
let tol = 1e-1 * candle[i].abs().max(1.0);
assert!(
(v - candle[i]).abs() <= tol,
"{:?}: dst[{i}] fast={} candle={} (tol {tol})",
$dt,
fast[i],
candle[i]
);
}
}};
}
check!(BlockQ8_0, GgmlDType::Q8_0);
check!(BlockQ2K, GgmlDType::Q2K);
check!(BlockQ4K, GgmlDType::Q4K);
}
#[test]
fn q8_0_matches_candle_scalar() {
run_case::<BlockQ8_0>(GgmlDType::Q8_0, 3, 512, 17);
run_case::<BlockQ8_0>(GgmlDType::Q8_0, 1, 256, 64);
run_case::<BlockQ8_0>(GgmlDType::Q8_0, 5, 128, 9);
}
#[test]
fn q2k_matches_candle_scalar() {
run_case::<BlockQ2K>(GgmlDType::Q2K, 3, 512, 11);
run_case::<BlockQ2K>(GgmlDType::Q2K, 1, 256, 40);
}
#[test]
fn q4k_matches_candle_scalar() {
run_case::<BlockQ4K>(GgmlDType::Q4K, 3, 512, 13);
run_case::<BlockQ4K>(GgmlDType::Q4K, 4, 768, 25);
}
#[test]
fn decode_raw_to_f32_handles_all_k_quants() {
use candle_core::quantized::k_quants::{
BlockQ2K, BlockQ3K, BlockQ4K, BlockQ5K, BlockQ6K, BlockQ8K,
};
macro_rules! check {
($id:expr, $ty:ty) => {{
let n = 3usize;
let k = 256usize; let blocks = quantized_blocks::<$ty>(n, k, <$ty as GgmlType>::DTYPE);
let elems = n * k;
let block_bytes = std::mem::size_of::<$ty>();
let mut bytes = Vec::with_capacity(blocks.len() * block_bytes);
for b in &blocks {
let ptr = b as *const $ty as *const u8;
bytes.extend_from_slice(unsafe {
std::slice::from_raw_parts(ptr, block_bytes)
});
}
let decoded = decode_raw_to_f32($id, &bytes, elems)
.unwrap()
.unwrap_or_else(|| panic!("dtype {} must decode", $id));
let mut w = vec![0f32; elems];
<$ty as GgmlType>::to_float(&blocks, &mut w);
for (i, (x, y)) in decoded.iter().zip(w.iter()).enumerate() {
let tol = 1e-6 * y.abs().max(1.0);
assert!(
(x - y).abs() <= tol,
"dtype {} elem {i}: decoded={x} to_float={y} (tol {tol})",
$id
);
}
}};
}
check!(10, BlockQ2K);
check!(11, BlockQ3K);
check!(12, BlockQ4K);
check!(13, BlockQ5K);
check!(14, BlockQ6K);
check!(15, BlockQ8K);
}
#[test]
fn threaded_matches_serial_bit_exact() {
let blocks = quantized_blocks::<BlockQ8_0>(24, 256, GgmlDType::Q8_0);
let lhs: Vec<f32> = (0..3 * 256).map(|i| (i as f32) * 0.01 - 1.0).collect();
let mut par = vec![0f32; 3 * 24];
matmul_kquant::<BlockQ8_0>((3, 256, 24), &lhs, &blocks, &mut par).unwrap();
let mut ser = vec![0f32; 3 * 24];
matmul_kquant_serial::<BlockQ8_0>((3, 256, 24), &lhs, &blocks, &mut ser).unwrap();
assert_eq!(par, ser, "parallel and serial matmul must be bit-identical");
}
#[test]
fn rejects_bad_shapes() {
let blocks = quantized_blocks::<BlockQ8_0>(2, 256, GgmlDType::Q8_0);
let lhs = vec![0f32; 256];
let mut dst = vec![0f32; 2];
assert!(matmul_kquant::<BlockQ8_0>((1, 100, 2), &lhs, &blocks, &mut dst).is_err());
assert!(matmul_kquant::<BlockQ8_0>((1, 256, 4), &lhs, &blocks, &mut dst).is_err());
assert!(matmul_kquant::<BlockQ8_0>((1, 256, 2), &[0.0], &blocks, &mut dst).is_err());
}
}