use super::blocks::{
BlockQ8_0, BlockQ8K, QK_K, f16_at, get_scale_min_k4, q4k, q5k, q6k, q8_0, scales_at,
};
use super::{KernelError, QuantFormat};
fn weight_blocks<'a>(
bytes: &'a [u8],
format: QuantFormat,
activation_blocks: usize,
) -> Result<std::slice::ChunksExact<'a, u8>, KernelError> {
let block_bytes = format.block_bytes();
if !bytes.len().is_multiple_of(block_bytes) {
return Err(KernelError::RowBytesNotBlockMultiple {
format,
len: bytes.len(),
block_bytes,
});
}
let weight_blocks = bytes.len() / block_bytes;
if weight_blocks != activation_blocks {
return Err(KernelError::BlockCountMismatch {
weight_blocks,
activation_blocks,
});
}
Ok(bytes.chunks_exact(block_bytes))
}
#[inline]
fn k_quant_affine_block(
aux: &[i8; QK_K],
scales: &[u8; 12],
y: &BlockQ8K,
d: f32,
dmin: f32,
) -> f32 {
let mut sc = [0u8; 8];
let mut mins = [0u8; 8];
for j in 0..8 {
let (s, m) = get_scale_min_k4(j, scales);
sc[j] = s;
mins[j] = m;
}
let mut sumi_mins = 0i32;
for (g, &bs) in y.bsums.iter().enumerate() {
sumi_mins += i32::from(bs) * i32::from(mins[g / 2]);
}
let mut sumi = 0i32;
for j in 0..8 {
let mut s = 0i32;
for l in 0..32 {
s += i32::from(aux[32 * j + l]) * i32::from(y.qs[32 * j + l]);
}
sumi += i32::from(sc[j]) * s;
}
d * sumi as f32 - dmin * sumi_mins as f32
}
pub fn vec_dot_q4_k_q8_k(weight_row: &[u8], acts: &[BlockQ8K]) -> Result<f32, KernelError> {
let blocks = weight_blocks(weight_row, QuantFormat::Q4_K, acts.len())?;
let mut sumf = 0f32;
for (block, y) in blocks.zip(acts) {
let d = f16_at(block, q4k::D) * y.d;
let dmin = f16_at(block, q4k::DMIN) * y.d;
let qs = &block[q4k::QS..q4k::BYTES];
let mut aux = [0i8; QK_K];
for span in 0..4 {
let q = &qs[32 * span..32 * span + 32];
for l in 0..32 {
aux[64 * span + l] = (q[l] & 0xF) as i8;
aux[64 * span + 32 + l] = (q[l] >> 4) as i8;
}
}
sumf += k_quant_affine_block(&aux, scales_at(block, q4k::SCALES), y, d, dmin);
}
Ok(sumf)
}
pub fn vec_dot_q5_k_q8_k(weight_row: &[u8], acts: &[BlockQ8K]) -> Result<f32, KernelError> {
let blocks = weight_blocks(weight_row, QuantFormat::Q5_K, acts.len())?;
let mut sumf = 0f32;
for (block, y) in blocks.zip(acts) {
let d = f16_at(block, q5k::D) * y.d;
let dmin = f16_at(block, q5k::DMIN) * y.d;
let qh = &block[q5k::QH..q5k::QS];
let qs = &block[q5k::QS..q5k::BYTES];
let mut aux = [0i8; QK_K];
for j in 0..8 {
let q = &qs[32 * (j / 2)..32 * (j / 2) + 32];
let mask = 1u8 << j;
for l in 0..32 {
let low = if j % 2 == 0 { q[l] & 0xF } else { q[l] >> 4 };
let hi = if qh[l] & mask != 0 { 16 } else { 0 };
aux[32 * j + l] = (low + hi) as i8;
}
}
sumf += k_quant_affine_block(&aux, scales_at(block, q5k::SCALES), y, d, dmin);
}
Ok(sumf)
}
pub fn vec_dot_q6_k_q8_k(weight_row: &[u8], acts: &[BlockQ8K]) -> Result<f32, KernelError> {
let blocks = weight_blocks(weight_row, QuantFormat::Q6_K, acts.len())?;
let mut sumf = 0f32;
for (block, y) in blocks.zip(acts) {
let mut aux = [0i8; QK_K];
for half in 0..2 {
let ql = &block[q6k::QL + 64 * half..];
let qh = &block[q6k::QH + 32 * half..];
let a = &mut aux[128 * half..];
for l in 0..32 {
let h = i32::from(qh[l]);
a[l] = ((i32::from(ql[l] & 0xF) | ((h & 3) << 4)) - 32) as i8;
a[l + 32] = ((i32::from(ql[l + 32] & 0xF) | (((h >> 2) & 3) << 4)) - 32) as i8;
a[l + 64] = ((i32::from(ql[l] >> 4) | (((h >> 4) & 3) << 4)) - 32) as i8;
a[l + 96] = ((i32::from(ql[l + 32] >> 4) | (((h >> 6) & 3) << 4)) - 32) as i8;
}
}
let mut sumi = 0i32;
for g in 0..16 {
let scale = i32::from(block[q6k::SCALES + g] as i8);
let mut s = 0i32;
for l in 0..16 {
s += i32::from(aux[16 * g + l]) * i32::from(y.qs[16 * g + l]);
}
sumi += scale * s;
}
sumf += f16_at(block, q6k::D) * y.d * sumi as f32;
}
Ok(sumf)
}
pub fn vec_dot_q8_0_q8_0(weight_row: &[u8], acts: &[BlockQ8_0]) -> Result<f32, KernelError> {
let blocks = weight_blocks(weight_row, QuantFormat::Q8_0, acts.len())?;
let mut sumf = 0f32;
for (block, y) in blocks.zip(acts) {
let mut sumi = 0i32;
for (l, &b) in block[q8_0::QS..q8_0::BYTES].iter().enumerate() {
sumi += i32::from(b as i8) * i32::from(y.qs[l]);
}
sumf += sumi as f32 * (f16_at(block, q8_0::D) * y.d);
}
Ok(sumf)
}