use super::{BlockQ8_0, BlockQ8K, KernelError, QuantFormat, dot, quantize};
#[inline]
pub fn avx2_fma_available() -> bool {
#[cfg(target_arch = "x86_64")]
{
std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma")
}
#[cfg(not(target_arch = "x86_64"))]
{
false
}
}
#[cfg(target_arch = "x86_64")]
fn validate_row(
bytes: &[u8],
format: QuantFormat,
activation_blocks: usize,
) -> Result<(), KernelError> {
let block_bytes = format.block_bytes();
if !bytes.len().is_multiple_of(block_bytes) {
return Err(KernelError::RowBytesNotBlockMultiple {
format,
len: bytes.len(),
block_bytes,
});
}
let weight_blocks = bytes.len() / block_bytes;
if weight_blocks != activation_blocks {
return Err(KernelError::BlockCountMismatch {
weight_blocks,
activation_blocks,
});
}
Ok(())
}
#[cfg(target_arch = "x86_64")]
fn validate_quantize(
format: QuantFormat,
floats: usize,
out_blocks: usize,
) -> Result<(), KernelError> {
if floats != out_blocks * format.block_weights() {
return Err(KernelError::QuantizeLenMismatch {
format,
floats,
out_blocks,
});
}
Ok(())
}
pub fn vec_dot_q4_k_q8_k(
weight_row: &[u8],
acts: &[BlockQ8K],
force_scalar: bool,
) -> Result<f32, KernelError> {
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
validate_row(weight_row, QuantFormat::Q4_K, acts.len())?;
return Ok(unsafe { x86::vec_dot_q4_k_q8_k(weight_row, acts) });
}
#[cfg(not(target_arch = "x86_64"))]
let _ = force_scalar;
dot::vec_dot_q4_k_q8_k(weight_row, acts)
}
pub fn vec_dot_q4_k_q8_k_rows(
weight_row: &[u8],
acts: &[BlockQ8K],
n_acts: usize,
out: &mut [f32],
force_scalar: bool,
) -> Result<(), KernelError> {
dot_rows(Tiled::Q4K, weight_row, acts, n_acts, out, force_scalar)
}
pub fn vec_dot_q6_k_q8_k_rows(
weight_row: &[u8],
acts: &[BlockQ8K],
n_acts: usize,
out: &mut [f32],
force_scalar: bool,
) -> Result<(), KernelError> {
dot_rows(Tiled::Q6K, weight_row, acts, n_acts, out, force_scalar)
}
#[derive(Clone, Copy)]
enum Tiled {
Q4K,
Q6K,
}
impl Tiled {
fn format(self) -> QuantFormat {
match self {
Self::Q4K => QuantFormat::Q4_K,
Self::Q6K => QuantFormat::Q6_K,
}
}
fn single(
self,
weight_row: &[u8],
acts: &[BlockQ8K],
force_scalar: bool,
) -> Result<f32, KernelError> {
match self {
Self::Q4K => vec_dot_q4_k_q8_k(weight_row, acts, force_scalar),
Self::Q6K => vec_dot_q6_k_q8_k(weight_row, acts, force_scalar),
}
}
fn scalar(self, weight_row: &[u8], acts: &[BlockQ8K]) -> Result<f32, KernelError> {
match self {
Self::Q4K => dot::vec_dot_q4_k_q8_k(weight_row, acts),
Self::Q6K => dot::vec_dot_q6_k_q8_k(weight_row, acts),
}
}
#[cfg(target_arch = "x86_64")]
unsafe fn one(self, weight_row: &[u8], acts: &[BlockQ8K]) -> f32 {
unsafe {
match self {
Self::Q4K => x86::vec_dot_q4_k_q8_k(weight_row, acts),
Self::Q6K => x86::vec_dot_q6_k_q8_k(weight_row, acts),
}
}
}
#[cfg(target_arch = "x86_64")]
unsafe fn four(self, weight_row: &[u8], acts: [&[BlockQ8K]; 4]) -> [f32; 4] {
unsafe {
match self {
Self::Q4K => x86::vec_dot_q4_k_q8_k_x4(weight_row, acts),
Self::Q6K => x86::vec_dot_q6_k_q8_k_x4(weight_row, acts),
}
}
}
}
fn dot_rows(
tiled: Tiled,
weight_row: &[u8],
acts: &[BlockQ8K],
n_acts: usize,
out: &mut [f32],
force_scalar: bool,
) -> Result<(), KernelError> {
let format = tiled.format();
let block_bytes = format.block_bytes();
if !weight_row.len().is_multiple_of(block_bytes) {
return Err(KernelError::RowBytesNotBlockMultiple {
format,
len: weight_row.len(),
block_bytes,
});
}
let n_blocks = weight_row.len() / block_bytes;
if n_blocks.checked_mul(n_acts) != Some(acts.len()) {
return Err(KernelError::BlockCountMismatch {
weight_blocks: n_blocks.saturating_mul(n_acts),
activation_blocks: acts.len(),
});
}
if out.len() != n_acts {
return Err(KernelError::LengthMismatch {
what: "vec_dot_k_q8_k_rows: out vs n_acts",
left: out.len(),
right: n_acts,
});
}
if n_blocks == 0 {
for o in out.iter_mut() {
*o = tiled.single(weight_row, &[], force_scalar)?;
}
return Ok(());
}
let row = |a: usize| &acts[a * n_blocks..(a + 1) * n_blocks];
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
let mut a = 0;
while a + 4 <= n_acts {
let four =
unsafe { tiled.four(weight_row, [row(a), row(a + 1), row(a + 2), row(a + 3)]) };
out[a..a + 4].copy_from_slice(&four);
a += 4;
}
for (k, o) in out[a..].iter_mut().enumerate() {
*o = unsafe { tiled.one(weight_row, row(a + k)) };
}
return Ok(());
}
for (a, o) in out.iter_mut().enumerate() {
*o = tiled.scalar(weight_row, row(a))?;
}
Ok(())
}
pub fn vec_dot_q5_k_q8_k(
weight_row: &[u8],
acts: &[BlockQ8K],
force_scalar: bool,
) -> Result<f32, KernelError> {
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
validate_row(weight_row, QuantFormat::Q5_K, acts.len())?;
return Ok(unsafe { x86::vec_dot_q5_k_q8_k(weight_row, acts) });
}
#[cfg(not(target_arch = "x86_64"))]
let _ = force_scalar;
dot::vec_dot_q5_k_q8_k(weight_row, acts)
}
pub fn vec_dot_q6_k_q8_k(
weight_row: &[u8],
acts: &[BlockQ8K],
force_scalar: bool,
) -> Result<f32, KernelError> {
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
validate_row(weight_row, QuantFormat::Q6_K, acts.len())?;
return Ok(unsafe { x86::vec_dot_q6_k_q8_k(weight_row, acts) });
}
#[cfg(not(target_arch = "x86_64"))]
let _ = force_scalar;
dot::vec_dot_q6_k_q8_k(weight_row, acts)
}
pub fn vec_dot_q8_0_q8_0(
weight_row: &[u8],
acts: &[BlockQ8_0],
force_scalar: bool,
) -> Result<f32, KernelError> {
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
validate_row(weight_row, QuantFormat::Q8_0, acts.len())?;
return Ok(unsafe { x86::vec_dot_q8_0_q8_0(weight_row, acts) });
}
#[cfg(not(target_arch = "x86_64"))]
let _ = force_scalar;
dot::vec_dot_q8_0_q8_0(weight_row, acts)
}
pub fn quantize_row_q8_0(
x: &[f32],
out: &mut [BlockQ8_0],
force_scalar: bool,
) -> Result<(), KernelError> {
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
validate_quantize(QuantFormat::Q8_0, x.len(), out.len())?;
unsafe { x86::quantize_row_q8_0(x, out) };
return Ok(());
}
#[cfg(not(target_arch = "x86_64"))]
let _ = force_scalar;
quantize::quantize_row_q8_0(x, out)
}
pub fn quantize_row_q8_k(
x: &[f32],
out: &mut [BlockQ8K],
force_scalar: bool,
) -> Result<(), KernelError> {
#[cfg(target_arch = "x86_64")]
if !force_scalar && avx2_fma_available() {
validate_quantize(QuantFormat::Q8_K, x.len(), out.len())?;
unsafe { x86::quantize_row_q8_k(x, out) };
return Ok(());
}
#[cfg(not(target_arch = "x86_64"))]
let _ = force_scalar;
quantize::quantize_row_q8_k(x, out)
}
#[cfg(target_arch = "x86_64")]
mod x86 {
use std::arch::x86_64::*;
use super::super::blocks::{QK_K, QK8_0, f16_at, q4k, q5k, q6k, q8_0, scales_at};
use super::super::f16::{f16_to_f32, f32_to_f16};
use super::super::{BlockQ8_0, BlockQ8K};
#[inline]
pub(super) fn unpack_scales_mins(scales: &[u8; 12]) -> ([u8; 8], [u8; 8]) {
const KMASK1: u32 = 0x3f3f3f3f;
const KMASK2: u32 = 0x0f0f0f0f;
const KMASK3: u32 = 0x03030303;
let mut utmp = [
u32::from_le_bytes([scales[0], scales[1], scales[2], scales[3]]),
u32::from_le_bytes([scales[4], scales[5], scales[6], scales[7]]),
u32::from_le_bytes([scales[8], scales[9], scales[10], scales[11]]),
0u32,
];
utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4);
let uaux = utmp[1] & KMASK1;
utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4);
utmp[2] = uaux;
utmp[0] &= KMASK1;
let mut sc = [0u8; 8];
let mut mins = [0u8; 8];
sc[..4].copy_from_slice(&utmp[0].to_le_bytes());
sc[4..].copy_from_slice(&utmp[1].to_le_bytes());
mins[..4].copy_from_slice(&utmp[2].to_le_bytes());
mins[4..].copy_from_slice(&utmp[3].to_le_bytes());
(sc, mins)
}
#[target_feature(enable = "avx2,fma")]
unsafe fn hsum_ps_8(v: __m256) -> f32 {
let s = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps::<1>(v));
let s = _mm_add_ps(s, _mm_movehl_ps(s, s));
let s = _mm_add_ss(s, _mm_movehdup_ps(s));
_mm_cvtss_f32(s)
}
#[target_feature(enable = "avx2,fma")]
unsafe fn hsum_ps_4(v: __m128) -> f32 {
let s = _mm_add_ps(v, _mm_movehl_ps(v, v));
let s = _mm_add_ss(s, _mm_movehdup_ps(s));
_mm_cvtss_f32(s)
}
#[target_feature(enable = "avx2,fma")]
unsafe fn hmax_ps_8(v: __m256) -> f32 {
let m = _mm_max_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps::<1>(v));
let m = _mm_max_ps(m, _mm_movehl_ps(m, m));
let m = _mm_max_ss(m, _mm_movehdup_ps(m));
_mm_cvtss_f32(m)
}
#[target_feature(enable = "avx2,fma")]
unsafe fn hsum_epi32_8(v: __m256i) -> i32 {
let s = _mm_add_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v));
let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b0100_1110>(s));
let s = _mm_add_epi32(s, _mm_shuffle_epi32::<0b1011_0001>(s));
_mm_cvtsi128_si32(s)
}
#[target_feature(enable = "avx2,fma")]
unsafe fn mins_dot(y: &BlockQ8K, mins: &[u8; 8]) -> __m128i {
unsafe {
let q8sums = _mm256_loadu_si256(y.bsums.as_ptr().cast());
let q8s = _mm_hadd_epi16(
_mm256_castsi256_si128(q8sums),
_mm256_extracti128_si256::<1>(q8sums),
);
let minsv = _mm_cvtepu8_epi16(_mm_loadl_epi64(mins.as_ptr().cast()));
_mm_madd_epi16(minsv, q8s)
}
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn vec_dot_q4_k_q8_k(weight_row: &[u8], acts: &[BlockQ8K]) -> f32 {
unsafe {
let m4 = _mm256_set1_epi8(0xF);
let mut acc = _mm256_setzero_ps();
let mut acc_m = _mm_setzero_ps();
for (block, y) in weight_row.chunks_exact(q4k::BYTES).zip(acts) {
let d = f16_at(block, q4k::D) * y.d;
let dmin = f16_at(block, q4k::DMIN) * y.d;
let (sc, mins) = unpack_scales_mins(scales_at(block, q4k::SCALES));
let prod = mins_dot(y, &mins);
acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
let qs = &block[q4k::QS..q4k::BYTES];
let mut sumi = _mm256_setzero_si256();
for j in 0..4 {
let q4bits = _mm256_loadu_si256(qs.as_ptr().add(32 * j).cast());
let q4l = _mm256_and_si256(q4bits, m4);
let q4h = _mm256_and_si256(_mm256_srli_epi16::<4>(q4bits), m4);
let q8l = _mm256_loadu_si256(y.qs.as_ptr().add(64 * j).cast());
let q8h = _mm256_loadu_si256(y.qs.as_ptr().add(64 * j + 32).cast());
let p16l = _mm256_madd_epi16(
_mm256_set1_epi16(i16::from(sc[2 * j])),
_mm256_maddubs_epi16(q4l, q8l),
);
let p16h = _mm256_madd_epi16(
_mm256_set1_epi16(i16::from(sc[2 * j + 1])),
_mm256_maddubs_epi16(q4h, q8h),
);
sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16l, p16h));
}
acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), acc);
}
hsum_ps_8(acc) - hsum_ps_4(acc_m)
}
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn vec_dot_q4_k_q8_k_x4(
weight_row: &[u8],
acts: [&[BlockQ8K]; 4],
) -> [f32; 4] {
unsafe {
let m4 = _mm256_set1_epi8(0xF);
let mut acc = [_mm256_setzero_ps(); 4];
let mut acc_m = [_mm_setzero_ps(); 4];
for (b, block) in weight_row.chunks_exact(q4k::BYTES).enumerate() {
let ys = [&acts[0][b], &acts[1][b], &acts[2][b], &acts[3][b]];
let dw = f16_at(block, q4k::D);
let dminw = f16_at(block, q4k::DMIN);
let (sc, mins) = unpack_scales_mins(scales_at(block, q4k::SCALES));
for (y, acc_m) in ys.iter().zip(acc_m.iter_mut()) {
let dmin = dminw * y.d;
let prod = mins_dot(y, &mins);
*acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), *acc_m);
}
let qs = &block[q4k::QS..q4k::BYTES];
let mut sumi = [_mm256_setzero_si256(); 4];
for j in 0..4 {
let q4bits = _mm256_loadu_si256(qs.as_ptr().add(32 * j).cast());
let q4l = _mm256_and_si256(q4bits, m4);
let q4h = _mm256_and_si256(_mm256_srli_epi16::<4>(q4bits), m4);
let scl = _mm256_set1_epi16(i16::from(sc[2 * j]));
let sch = _mm256_set1_epi16(i16::from(sc[2 * j + 1]));
for (y, sumi) in ys.iter().zip(sumi.iter_mut()) {
let q8l = _mm256_loadu_si256(y.qs.as_ptr().add(64 * j).cast());
let q8h = _mm256_loadu_si256(y.qs.as_ptr().add(64 * j + 32).cast());
let p16l = _mm256_madd_epi16(scl, _mm256_maddubs_epi16(q4l, q8l));
let p16h = _mm256_madd_epi16(sch, _mm256_maddubs_epi16(q4h, q8h));
*sumi = _mm256_add_epi32(*sumi, _mm256_add_epi32(p16l, p16h));
}
}
for ((y, sumi), acc) in ys.iter().zip(sumi).zip(acc.iter_mut()) {
let d = dw * y.d;
*acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), *acc);
}
}
[0, 1, 2, 3].map(|a| hsum_ps_8(acc[a]) - hsum_ps_4(acc_m[a]))
}
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn vec_dot_q5_k_q8_k(weight_row: &[u8], acts: &[BlockQ8K]) -> f32 {
unsafe {
let m4 = _mm256_set1_epi8(0xF);
let mut acc = _mm256_setzero_ps();
let mut acc_m = _mm_setzero_ps();
for (block, y) in weight_row.chunks_exact(q5k::BYTES).zip(acts) {
let d = f16_at(block, q5k::D) * y.d;
let dmin = f16_at(block, q5k::DMIN) * y.d;
let (sc, mins) = unpack_scales_mins(scales_at(block, q5k::SCALES));
let prod = mins_dot(y, &mins);
acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m);
let qh = &block[q5k::QH..q5k::QS];
let qs = &block[q5k::QS..q5k::BYTES];
let hbits = _mm256_loadu_si256(qh.as_ptr().cast());
let mut hmask = _mm256_set1_epi8(1);
let mut sumi = _mm256_setzero_si256();
for j in 0..4 {
let q5bits = _mm256_loadu_si256(qs.as_ptr().add(32 * j).cast());
let h0 = _mm256_slli_epi16::<4>(_mm256_srl_epi16(
_mm256_and_si256(hbits, hmask),
_mm_cvtsi32_si128(2 * j as i32),
));
let q5_0 = _mm256_add_epi8(_mm256_and_si256(q5bits, m4), h0);
hmask = _mm256_slli_epi16::<1>(hmask);
let h1 = _mm256_slli_epi16::<4>(_mm256_srl_epi16(
_mm256_and_si256(hbits, hmask),
_mm_cvtsi32_si128(2 * j as i32 + 1),
));
let q5_1 =
_mm256_add_epi8(_mm256_and_si256(_mm256_srli_epi16::<4>(q5bits), m4), h1);
hmask = _mm256_slli_epi16::<1>(hmask);
let q8l = _mm256_loadu_si256(y.qs.as_ptr().add(64 * j).cast());
let q8h = _mm256_loadu_si256(y.qs.as_ptr().add(64 * j + 32).cast());
let p16l = _mm256_madd_epi16(
_mm256_set1_epi16(i16::from(sc[2 * j])),
_mm256_maddubs_epi16(q5_0, q8l),
);
let p16h = _mm256_madd_epi16(
_mm256_set1_epi16(i16::from(sc[2 * j + 1])),
_mm256_maddubs_epi16(q5_1, q8h),
);
sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16l, p16h));
}
acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), acc);
}
hsum_ps_8(acc) - hsum_ps_4(acc_m)
}
}
#[target_feature(enable = "avx2,fma")]
unsafe fn scale_pair(lo: i8, hi: i8) -> __m256i {
_mm256_set_m128i(_mm_set1_epi16(i16::from(hi)), _mm_set1_epi16(i16::from(lo)))
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn vec_dot_q6_k_q8_k(weight_row: &[u8], acts: &[BlockQ8K]) -> f32 {
unsafe {
let m4 = _mm256_set1_epi8(0xF);
let m2 = _mm256_set1_epi8(3);
let m32s = _mm256_set1_epi8(32);
let mut acc = _mm256_setzero_ps();
for (block, y) in weight_row.chunks_exact(q6k::BYTES).zip(acts) {
let d = f16_at(block, q6k::D) * y.d;
let scs = &block[q6k::SCALES..q6k::SCALES + 16];
let mut sumi = _mm256_setzero_si256();
for half in 0..2 {
let base = 128 * half; let q4bits1 =
_mm256_loadu_si256(block.as_ptr().add(q6k::QL + 64 * half).cast());
let q4bits2 =
_mm256_loadu_si256(block.as_ptr().add(q6k::QL + 64 * half + 32).cast());
let bits_h = _mm256_loadu_si256(block.as_ptr().add(q6k::QH + 32 * half).cast());
let q4h_0 = _mm256_slli_epi16::<4>(_mm256_and_si256(bits_h, m2));
let q4h_1 = _mm256_slli_epi16::<4>(_mm256_and_si256(
_mm256_srli_epi16::<2>(bits_h),
m2,
));
let q4h_2 = _mm256_slli_epi16::<4>(_mm256_and_si256(
_mm256_srli_epi16::<4>(bits_h),
m2,
));
let q4h_3 = _mm256_slli_epi16::<4>(_mm256_and_si256(
_mm256_srli_epi16::<6>(bits_h),
m2,
));
let q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0);
let q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1);
let q4_2 = _mm256_or_si256(
_mm256_and_si256(_mm256_srli_epi16::<4>(q4bits1), m4),
q4h_2,
);
let q4_3 = _mm256_or_si256(
_mm256_and_si256(_mm256_srli_epi16::<4>(q4bits2), m4),
q4h_3,
);
let q8_0 = _mm256_loadu_si256(y.qs.as_ptr().add(base).cast());
let q8_1 = _mm256_loadu_si256(y.qs.as_ptr().add(base + 32).cast());
let q8_2 = _mm256_loadu_si256(y.qs.as_ptr().add(base + 64).cast());
let q8_3 = _mm256_loadu_si256(y.qs.as_ptr().add(base + 96).cast());
let p16_0 = _mm256_sub_epi16(
_mm256_maddubs_epi16(q4_0, q8_0),
_mm256_maddubs_epi16(m32s, q8_0),
);
let p16_1 = _mm256_sub_epi16(
_mm256_maddubs_epi16(q4_1, q8_1),
_mm256_maddubs_epi16(m32s, q8_1),
);
let p16_2 = _mm256_sub_epi16(
_mm256_maddubs_epi16(q4_2, q8_2),
_mm256_maddubs_epi16(m32s, q8_2),
);
let p16_3 = _mm256_sub_epi16(
_mm256_maddubs_epi16(q4_3, q8_3),
_mm256_maddubs_epi16(m32s, q8_3),
);
let s = 8 * half;
let p32_0 =
_mm256_madd_epi16(scale_pair(scs[s] as i8, scs[s + 1] as i8), p16_0);
let p32_1 =
_mm256_madd_epi16(scale_pair(scs[s + 2] as i8, scs[s + 3] as i8), p16_1);
let p32_2 =
_mm256_madd_epi16(scale_pair(scs[s + 4] as i8, scs[s + 5] as i8), p16_2);
let p32_3 =
_mm256_madd_epi16(scale_pair(scs[s + 6] as i8, scs[s + 7] as i8), p16_3);
sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p32_0, p32_1));
sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p32_2, p32_3));
}
acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), acc);
}
hsum_ps_8(acc)
}
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn vec_dot_q6_k_q8_k_x4(
weight_row: &[u8],
acts: [&[BlockQ8K]; 4],
) -> [f32; 4] {
unsafe {
let m4 = _mm256_set1_epi8(0xF);
let m2 = _mm256_set1_epi8(3);
let m32s = _mm256_set1_epi8(32);
let mut acc = [_mm256_setzero_ps(); 4];
for (b, block) in weight_row.chunks_exact(q6k::BYTES).enumerate() {
let ys = [&acts[0][b], &acts[1][b], &acts[2][b], &acts[3][b]];
let dw = f16_at(block, q6k::D);
let scs = &block[q6k::SCALES..q6k::SCALES + 16];
let mut sumi = [_mm256_setzero_si256(); 4];
for half in 0..2 {
let base = 128 * half;
let q4bits1 =
_mm256_loadu_si256(block.as_ptr().add(q6k::QL + 64 * half).cast());
let q4bits2 =
_mm256_loadu_si256(block.as_ptr().add(q6k::QL + 64 * half + 32).cast());
let bits_h = _mm256_loadu_si256(block.as_ptr().add(q6k::QH + 32 * half).cast());
let q4h_0 = _mm256_slli_epi16::<4>(_mm256_and_si256(bits_h, m2));
let q4h_1 = _mm256_slli_epi16::<4>(_mm256_and_si256(
_mm256_srli_epi16::<2>(bits_h),
m2,
));
let q4h_2 = _mm256_slli_epi16::<4>(_mm256_and_si256(
_mm256_srli_epi16::<4>(bits_h),
m2,
));
let q4h_3 = _mm256_slli_epi16::<4>(_mm256_and_si256(
_mm256_srli_epi16::<6>(bits_h),
m2,
));
let q4 = [
_mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0),
_mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1),
_mm256_or_si256(
_mm256_and_si256(_mm256_srli_epi16::<4>(q4bits1), m4),
q4h_2,
),
_mm256_or_si256(
_mm256_and_si256(_mm256_srli_epi16::<4>(q4bits2), m4),
q4h_3,
),
];
let s = 8 * half;
let sc = [
scale_pair(scs[s] as i8, scs[s + 1] as i8),
scale_pair(scs[s + 2] as i8, scs[s + 3] as i8),
scale_pair(scs[s + 4] as i8, scs[s + 5] as i8),
scale_pair(scs[s + 6] as i8, scs[s + 7] as i8),
];
for (y, sumi) in ys.iter().zip(sumi.iter_mut()) {
let mut p32 = [_mm256_setzero_si256(); 4];
for (k, p) in p32.iter_mut().enumerate() {
let q8 = _mm256_loadu_si256(y.qs.as_ptr().add(base + 32 * k).cast());
let p16 = _mm256_sub_epi16(
_mm256_maddubs_epi16(q4[k], q8),
_mm256_maddubs_epi16(m32s, q8),
);
*p = _mm256_madd_epi16(sc[k], p16);
}
*sumi = _mm256_add_epi32(*sumi, _mm256_add_epi32(p32[0], p32[1]));
*sumi = _mm256_add_epi32(*sumi, _mm256_add_epi32(p32[2], p32[3]));
}
}
for ((y, sumi), acc) in ys.iter().zip(sumi).zip(acc.iter_mut()) {
let d = dw * y.d;
*acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sumi), *acc);
}
}
[0, 1, 2, 3].map(|a| hsum_ps_8(acc[a]))
}
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn vec_dot_q8_0_q8_0(weight_row: &[u8], acts: &[BlockQ8_0]) -> f32 {
unsafe {
let ones = _mm256_set1_epi16(1);
let mut acc = _mm256_setzero_ps();
for (block, y) in weight_row.chunks_exact(q8_0::BYTES).zip(acts) {
let d = f16_at(block, q8_0::D) * y.d;
let qx = _mm256_loadu_si256(block.as_ptr().add(q8_0::QS).cast());
let qy = _mm256_loadu_si256(y.qs.as_ptr().cast());
let ax = _mm256_sign_epi8(qx, qx);
let sy = _mm256_sign_epi8(qy, qx);
let dot16 = _mm256_maddubs_epi16(ax, sy);
let sum32 = _mm256_madd_epi16(ones, dot16);
acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sum32), acc);
}
hsum_ps_8(acc)
}
}
#[target_feature(enable = "avx2,fma")]
unsafe fn pack_store_i8(dst: *mut i8, i0: __m256i, i1: __m256i, i2: __m256i, i3: __m256i) {
unsafe {
let p01 = _mm256_packs_epi32(i0, i1);
let p23 = _mm256_packs_epi32(i2, i3);
let p = _mm256_packs_epi16(p01, p23);
let perm = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
let p = _mm256_permutevar8x32_epi32(p, perm);
_mm256_storeu_si256(dst.cast(), p);
}
}
#[target_feature(enable = "avx2,fma")]
unsafe fn round_away_ps(x: __m256) -> __m256 {
let t = _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(x);
let diff = _mm256_sub_ps(x, t);
let one = _mm256_set1_ps(1.0);
let zero = _mm256_setzero_ps();
let tie_up = _mm256_and_ps(
_mm256_cmp_ps::<_CMP_EQ_OQ>(diff, _mm256_set1_ps(0.5)),
_mm256_cmp_ps::<_CMP_GT_OQ>(x, zero),
);
let tie_dn = _mm256_and_ps(
_mm256_cmp_ps::<_CMP_EQ_OQ>(diff, _mm256_set1_ps(-0.5)),
_mm256_cmp_ps::<_CMP_LT_OQ>(x, zero),
);
let t = _mm256_add_ps(t, _mm256_and_ps(tie_up, one));
_mm256_sub_ps(t, _mm256_and_ps(tie_dn, one))
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn quantize_row_q8_0(x: &[f32], out: &mut [BlockQ8_0]) {
unsafe {
let sign_mask = _mm256_set1_ps(-0.0);
for (xs, block) in x.chunks_exact(QK8_0).zip(out.iter_mut()) {
let v0 = _mm256_loadu_ps(xs.as_ptr());
let v1 = _mm256_loadu_ps(xs.as_ptr().add(8));
let v2 = _mm256_loadu_ps(xs.as_ptr().add(16));
let v3 = _mm256_loadu_ps(xs.as_ptr().add(24));
let a01 = _mm256_max_ps(
_mm256_andnot_ps(sign_mask, v0),
_mm256_andnot_ps(sign_mask, v1),
);
let a23 = _mm256_max_ps(
_mm256_andnot_ps(sign_mask, v2),
_mm256_andnot_ps(sign_mask, v3),
);
let amax = hmax_ps_8(_mm256_max_ps(a01, a23));
let d = amax / 127.0;
let id = if d != 0.0 { 1.0 / d } else { 0.0 };
block.d = f16_to_f32(f32_to_f16(d));
let idv = _mm256_set1_ps(id);
let i0 = _mm256_cvtps_epi32(round_away_ps(_mm256_mul_ps(v0, idv)));
let i1 = _mm256_cvtps_epi32(round_away_ps(_mm256_mul_ps(v1, idv)));
let i2 = _mm256_cvtps_epi32(round_away_ps(_mm256_mul_ps(v2, idv)));
let i3 = _mm256_cvtps_epi32(round_away_ps(_mm256_mul_ps(v3, idv)));
pack_store_i8(block.qs.as_mut_ptr(), i0, i1, i2, i3);
}
}
}
#[target_feature(enable = "avx2,fma")]
pub(super) unsafe fn quantize_row_q8_k(x: &[f32], out: &mut [BlockQ8K]) {
unsafe {
let sign_mask = _mm256_set1_ps(-0.0);
for (xs, block) in x.chunks_exact(QK_K).zip(out.iter_mut()) {
let mut amax_v = _mm256_setzero_ps();
for k in 0..QK_K / 8 {
let v = _mm256_loadu_ps(xs.as_ptr().add(8 * k));
amax_v = _mm256_max_ps(amax_v, _mm256_andnot_ps(sign_mask, v));
}
let amax = hmax_ps_8(amax_v);
if amax == 0.0 {
*block = BlockQ8K::default();
continue;
}
let target = _mm256_set1_ps(amax);
let mut max = 0.0f32;
for k in 0..QK_K / 8 {
let v = _mm256_loadu_ps(xs.as_ptr().add(8 * k));
let eq = _mm256_cmp_ps::<_CMP_EQ_OQ>(_mm256_andnot_ps(sign_mask, v), target);
let m = _mm256_movemask_ps(eq);
if m != 0 {
max = xs[8 * k + m.trailing_zeros() as usize];
break;
}
}
let iscale = -127.0 / max;
let iv = _mm256_set1_ps(iscale);
for c in 0..QK_K / 32 {
let base = 32 * c;
let i0 = _mm256_cvtps_epi32(_mm256_mul_ps(
_mm256_loadu_ps(xs.as_ptr().add(base)),
iv,
));
let i1 = _mm256_cvtps_epi32(_mm256_mul_ps(
_mm256_loadu_ps(xs.as_ptr().add(base + 8)),
iv,
));
let i2 = _mm256_cvtps_epi32(_mm256_mul_ps(
_mm256_loadu_ps(xs.as_ptr().add(base + 16)),
iv,
));
let i3 = _mm256_cvtps_epi32(_mm256_mul_ps(
_mm256_loadu_ps(xs.as_ptr().add(base + 24)),
iv,
));
block.bsums[2 * c] = hsum_epi32_8(_mm256_add_epi32(i0, i1)) as i16;
block.bsums[2 * c + 1] = hsum_epi32_8(_mm256_add_epi32(i2, i3)) as i16;
pack_store_i8(block.qs.as_mut_ptr().add(base), i0, i1, i2, i3);
}
block.d = 1.0 / iscale;
}
}
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::{
Lcg, quantize_row_q4_k_test, quantize_row_q5_k_test, quantize_row_q6_k_test,
quantize_row_q8_0_test,
};
use super::super::{
dequantize_row_q4_k, dequantize_row_q5_k, dequantize_row_q6_k, dequantize_row_q8_0,
f32_to_f16, quantize,
};
use super::*;
const DIMS: [usize; 3] = [2048, 768, 4096];
type DispatchDot = fn(&[u8], &[BlockQ8K], bool) -> Result<f32, KernelError>;
fn assert_close(avx2: f32, scalar: f32, w: &[f32], a: &[f32]) {
assert_eq!(w.len(), a.len());
let mag: f64 = w
.iter()
.zip(a)
.map(|(&wi, &ai)| (f64::from(wi) * f64::from(ai)).abs())
.sum();
let tol = 1e-6 * mag.max(1e-20);
let err = (f64::from(avx2) - f64::from(scalar)).abs();
assert!(
err <= tol,
"avx2 {avx2} vs scalar {scalar}: err {err} > tol {tol} (mag {mag})"
);
}
fn q8_k_acts(x: &[f32]) -> Vec<BlockQ8K> {
let mut acts = vec![BlockQ8K::default(); x.len() / 256];
quantize::quantize_row_q8_k(x, &mut acts).unwrap();
acts
}
fn dequant_q8_k(blocks: &[BlockQ8K]) -> Vec<f32> {
blocks
.iter()
.flat_map(|b| b.qs.iter().map(move |&q| b.d * f32::from(q)))
.collect()
}
fn dequant_q8_0_act(blocks: &[BlockQ8_0]) -> Vec<f32> {
blocks
.iter()
.flat_map(|b| b.qs.iter().map(move |&q| b.d * f32::from(q)))
.collect()
}
fn k_quant_avx2_vs_scalar(
quant: fn(&[f32]) -> Vec<u8>,
dequant: fn(&[u8]) -> Result<Vec<f32>, KernelError>,
dot: DispatchDot,
seed: u64,
) {
if !avx2_fma_available() {
return; }
for n in DIMS {
let mut rng = Lcg(seed ^ n as u64);
let xw: Vec<f32> = (0..n).map(|_| rng.next_f32() * 4.0).collect();
let xa: Vec<f32> = (0..n).map(|_| rng.next_f32() * 3.0).collect();
let wb = quant(&xw);
let acts = q8_k_acts(&xa);
let scalar = dot(&wb, &acts, true).unwrap();
let fast = dot(&wb, &acts, false).unwrap();
assert_close(fast, scalar, &dequant(&wb).unwrap(), &dequant_q8_k(&acts));
}
}
#[test]
fn avx2_vs_scalar_q4_k() {
k_quant_avx2_vs_scalar(
quantize_row_q4_k_test,
dequantize_row_q4_k,
vec_dot_q4_k_q8_k,
0xB4,
);
}
#[test]
fn avx2_vs_scalar_q5_k() {
k_quant_avx2_vs_scalar(
quantize_row_q5_k_test,
dequantize_row_q5_k,
vec_dot_q5_k_q8_k,
0xB5,
);
}
#[test]
fn avx2_vs_scalar_q6_k() {
k_quant_avx2_vs_scalar(
quantize_row_q6_k_test,
dequantize_row_q6_k,
vec_dot_q6_k_q8_k,
0xB6,
);
}
#[test]
fn avx2_vs_scalar_q8_0() {
if !avx2_fma_available() {
return;
}
for n in DIMS {
let mut rng = Lcg(0xB8 ^ n as u64);
let xw: Vec<f32> = (0..n).map(|_| rng.next_f32() * 4.0).collect();
let xa: Vec<f32> = (0..n).map(|_| rng.next_f32() * 3.0).collect();
let wb = quantize_row_q8_0_test(&xw);
let mut acts = vec![BlockQ8_0::default(); n / 32];
quantize::quantize_row_q8_0(&xa, &mut acts).unwrap();
let scalar = vec_dot_q8_0_q8_0(&wb, &acts, true).unwrap();
let fast = vec_dot_q8_0_q8_0(&wb, &acts, false).unwrap();
assert_close(
fast,
scalar,
&dequantize_row_q8_0(&wb).unwrap(),
&dequant_q8_0_act(&acts),
);
}
}
#[test]
fn the_four_row_q4_k_tile_is_bit_identical_to_one_row_at_a_time() {
let mut rng = Lcg(0x4_7113);
for in_dim in [256usize, 768, 2048, 4096] {
let xw: Vec<f32> = (0..in_dim).map(|_| rng.next_f32() * 4.0).collect();
let wb = quantize_row_q4_k_test(&xw);
for n_acts in 0..=13usize {
let xa: Vec<f32> = (0..in_dim * n_acts).map(|_| rng.next_f32() * 3.0).collect();
let acts = q8_k_acts(&xa);
let blocks = in_dim / 256;
for force in [false, true] {
let mut got = vec![f32::NAN; n_acts];
vec_dot_q4_k_q8_k_rows(&wb, &acts, n_acts, &mut got, force).unwrap();
for (a, &g) in got.iter().enumerate() {
let want =
vec_dot_q4_k_q8_k(&wb, &acts[a * blocks..(a + 1) * blocks], force)
.unwrap();
assert_eq!(
g.to_bits(),
want.to_bits(),
"in_dim {in_dim}, n_acts {n_acts}, row {a}, force_scalar {force}"
);
}
}
}
}
}
#[test]
fn the_four_row_q4_k_tile_is_bit_identical_on_adversarial_blocks() {
let rows = [
build_q4k(1.0, 1.0, [0xFF; 12], 0xFF),
build_q4k(0.0, 0.0, [0; 12], 0x00),
build_q4k(-2.5, 3.0, ALL_MIN_SCALES, 0xA5),
build_q4k(65504.0, -65504.0, [0x3F; 12], 0x0F),
];
let extreme = |q: i8, d: f32| BlockQ8K {
d,
qs: [q; 256],
bsums: [i16::from(q) * 16; 16],
};
let mut acts = vec![
extreme(127, 1.0),
extreme(-128, 1.0),
extreme(-128, 1e-3),
extreme(0, 0.0),
];
for seed in 0..5u64 {
acts.extend(random_q8_k_block(0xAD_0000 + seed));
}
for row in &rows {
for n_acts in [1usize, 4, 5, 9] {
let acts = &acts[..n_acts];
for force in [false, true] {
let mut got = vec![f32::NAN; n_acts];
vec_dot_q4_k_q8_k_rows(row, acts, n_acts, &mut got, force).unwrap();
for (a, &g) in got.iter().enumerate() {
let want = vec_dot_q4_k_q8_k(row, &acts[a..=a], force).unwrap();
assert_eq!(g.to_bits(), want.to_bits(), "n_acts {n_acts}, row {a}");
}
}
}
}
}
#[test]
fn the_four_row_q6_k_tile_is_bit_identical_to_one_row_at_a_time() {
let mut rng = Lcg(0x6_7113);
for in_dim in [256usize, 768, 2048, 4096] {
let xw: Vec<f32> = (0..in_dim).map(|_| rng.next_f32() * 4.0).collect();
let wb = quantize_row_q6_k_test(&xw);
for n_acts in 0..=13usize {
let xa: Vec<f32> = (0..in_dim * n_acts).map(|_| rng.next_f32() * 3.0).collect();
let acts = q8_k_acts(&xa);
let blocks = in_dim / 256;
for force in [false, true] {
let mut got = vec![f32::NAN; n_acts];
vec_dot_q6_k_q8_k_rows(&wb, &acts, n_acts, &mut got, force).unwrap();
for (a, &g) in got.iter().enumerate() {
let want =
vec_dot_q6_k_q8_k(&wb, &acts[a * blocks..(a + 1) * blocks], force)
.unwrap();
assert_eq!(
g.to_bits(),
want.to_bits(),
"in_dim {in_dim}, n_acts {n_acts}, row {a}, force_scalar {force}"
);
}
}
}
}
}
#[test]
fn the_four_row_q6_k_tile_is_bit_identical_on_adversarial_blocks() {
let rows = [
build_q6k(0xFF, 0xFF, 127, 1.0),
build_q6k(0x00, 0x00, -128, 1.0),
build_q6k(0xA5, 0x5A, -1, -2.5),
build_q6k(0x0F, 0xF0, 64, 65504.0),
];
let extreme = |q: i8, d: f32| BlockQ8K {
d,
qs: [q; 256],
bsums: [i16::from(q) * 16; 16],
};
let mut acts = vec![
extreme(127, 1.0),
extreme(-128, 1.0),
extreme(-128, 1e-3),
extreme(0, 0.0),
];
for seed in 0..5u64 {
acts.extend(random_q8_k_block(0x6AD_0000 + seed));
}
for row in &rows {
for n_acts in [1usize, 4, 5, 9] {
let acts = &acts[..n_acts];
for force in [false, true] {
let mut got = vec![f32::NAN; n_acts];
vec_dot_q6_k_q8_k_rows(row, acts, n_acts, &mut got, force).unwrap();
for (a, &g) in got.iter().enumerate() {
let want = vec_dot_q6_k_q8_k(row, &acts[a..=a], force).unwrap();
assert_eq!(g.to_bits(), want.to_bits(), "n_acts {n_acts}, row {a}");
}
}
}
}
}
#[test]
fn the_four_row_q4_k_tile_refuses_mis_sized_inputs() {
let wb = vec![0u8; 2 * 144];
let acts = vec![BlockQ8K::default(); 6];
let mut out = vec![0f32; 3];
assert!(vec_dot_q4_k_q8_k_rows(&wb[..143], &acts, 3, &mut out, false).is_err());
assert!(vec_dot_q4_k_q8_k_rows(&wb, &acts[..5], 3, &mut out, false).is_err());
assert!(vec_dot_q4_k_q8_k_rows(&wb, &acts, 3, &mut out[..2], false).is_err());
assert!(vec_dot_q4_k_q8_k_rows(&wb, &acts, usize::MAX, &mut out, false).is_err());
vec_dot_q4_k_q8_k_rows(&wb, &acts, 3, &mut out, false).unwrap();
}
#[test]
fn force_scalar_is_bitwise_scalar() {
let mut rng = Lcg(0xF5);
let xw: Vec<f32> = (0..768).map(|_| rng.next_f32() * 4.0).collect();
let xa: Vec<f32> = (0..768).map(|_| rng.next_f32() * 3.0).collect();
let wb = quantize_row_q4_k_test(&xw);
let acts = q8_k_acts(&xa);
let forced = vec_dot_q4_k_q8_k(&wb, &acts, true).unwrap();
let reference = dot::vec_dot_q4_k_q8_k(&wb, &acts).unwrap();
assert_eq!(forced.to_bits(), reference.to_bits());
}
fn build_q4k(d: f32, dmin: f32, scales: [u8; 12], qs: u8) -> Vec<u8> {
let mut b = Vec::with_capacity(144);
b.extend_from_slice(&f32_to_f16(d).to_le_bytes());
b.extend_from_slice(&f32_to_f16(dmin).to_le_bytes());
b.extend_from_slice(&scales);
b.extend_from_slice(&[qs; 128]);
b
}
fn build_q5k(d: f32, dmin: f32, scales: [u8; 12], qh: u8, qs: u8) -> Vec<u8> {
let mut b = Vec::with_capacity(176);
b.extend_from_slice(&f32_to_f16(d).to_le_bytes());
b.extend_from_slice(&f32_to_f16(dmin).to_le_bytes());
b.extend_from_slice(&scales);
b.extend_from_slice(&[qh; 32]);
b.extend_from_slice(&[qs; 128]);
b
}
fn build_q6k(ql: u8, qh: u8, scale: i8, d: f32) -> Vec<u8> {
let mut b = Vec::with_capacity(210);
b.extend_from_slice(&[ql; 128]);
b.extend_from_slice(&[qh; 64]);
b.extend_from_slice(&[scale as u8; 16]);
b.extend_from_slice(&f32_to_f16(d).to_le_bytes());
b
}
const ALL_MIN_SCALES: [u8; 12] = [0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xF0, 0xF0, 0xF0];
fn random_q8_k_block(seed: u64) -> Vec<BlockQ8K> {
let mut rng = Lcg(seed);
let xa: Vec<f32> = (0..256).map(|_| rng.next_f32() * 2.0).collect();
q8_k_acts(&xa)
}
#[test]
fn adversarial_q4_k_matches_scalar() {
if !avx2_fma_available() {
return;
}
let acts = random_q8_k_block(0xE4);
let a = dequant_q8_k(&acts);
let cases = [
build_q4k(1.0, 0.5, [0xFF; 12], 0xFF),
build_q4k(4.0, 2.0, [0; 12], 0x5A),
build_q4k(0.0, 0.0, [0x3F; 12], 0xFF),
build_q4k(1.0, 0.5, ALL_MIN_SCALES, 0x00),
];
for bytes in &cases {
let scalar = vec_dot_q4_k_q8_k(bytes, &acts, true).unwrap();
let fast = vec_dot_q4_k_q8_k(bytes, &acts, false).unwrap();
assert_close(fast, scalar, &dequantize_row_q4_k(bytes).unwrap(), &a);
}
let zero_acts = vec![BlockQ8K::default()];
assert_eq!(
vec_dot_q4_k_q8_k(&cases[1], &zero_acts, false).unwrap(),
0.0
);
}
#[test]
fn adversarial_q5_k_matches_scalar() {
if !avx2_fma_available() {
return;
}
let acts = random_q8_k_block(0xE5);
let a = dequant_q8_k(&acts);
let cases = [
build_q5k(1.0, 0.5, [0xFF; 12], 0xFF, 0xFF),
build_q5k(1.0, 0.5, [0xFF; 12], 0x00, 0xFF),
build_q5k(4.0, 2.0, [0; 12], 0xAA, 0x5A),
build_q5k(1.0, 0.5, ALL_MIN_SCALES, 0xAA, 0x00),
];
for bytes in &cases {
let scalar = vec_dot_q5_k_q8_k(bytes, &acts, true).unwrap();
let fast = vec_dot_q5_k_q8_k(bytes, &acts, false).unwrap();
assert_close(fast, scalar, &dequantize_row_q5_k(bytes).unwrap(), &a);
}
}
#[test]
fn adversarial_q6_k_matches_scalar() {
if !avx2_fma_available() {
return;
}
let acts = random_q8_k_block(0xE6);
let a = dequant_q8_k(&acts);
let cases = [
build_q6k(0xFF, 0xFF, -128, 1.0),
build_q6k(0xFF, 0xFF, 127, 0.25),
build_q6k(0x00, 0x00, 127, 1.0),
build_q6k(0x5A, 0xA5, 0, 4.0),
];
for bytes in &cases {
let scalar = vec_dot_q6_k_q8_k(bytes, &acts, true).unwrap();
let fast = vec_dot_q6_k_q8_k(bytes, &acts, false).unwrap();
assert_close(fast, scalar, &dequantize_row_q6_k(bytes).unwrap(), &a);
}
}
#[test]
fn adversarial_q8_0_matches_scalar() {
if !avx2_fma_available() {
return;
}
let mut wb = Vec::new();
wb.extend_from_slice(&0x7BFFu16.to_le_bytes()); wb.extend_from_slice(&[0x80; 32]); let acts = [BlockQ8_0 {
d: 65504.0,
qs: [127; 32],
}];
let scalar = vec_dot_q8_0_q8_0(&wb, &acts, true).unwrap();
let fast = vec_dot_q8_0_q8_0(&wb, &acts, false).unwrap();
assert_close(
fast,
scalar,
&dequantize_row_q8_0(&wb).unwrap(),
&dequant_q8_0_act(&acts),
);
assert_eq!(
vec_dot_q8_0_q8_0(&wb, &[BlockQ8_0::default()], false).unwrap(),
0.0
);
}
fn quantizer_inputs(n: usize, seed: u64) -> Vec<Vec<f32>> {
let mut rng = Lcg(seed);
let mut cases = vec![
(0..n).map(|_| rng.next_f32() * 8.0).collect::<Vec<f32>>(),
vec![0.0; n],
vec![-3.75; n],
vec![12.5; n],
];
let mut tie = vec![0.0f32; n];
for (i, v) in tie.iter_mut().enumerate() {
*v = match i % 4 {
0 => 2.0,
1 => -1.0,
2 => 0.5,
_ => -0.5,
};
}
cases.push(tie);
let mut neg = vec![0.5f32; n];
neg[0] = -4.0;
cases.push(neg);
let mut pos = vec![-0.5f32; n];
pos[1] = 4.0;
cases.push(pos);
let mut dup = vec![0.25f32; n];
dup[3] = -6.0;
dup[7] = 6.0;
cases.push(dup);
cases
}
#[test]
fn quantize_q8_0_bytes_identical_to_scalar() {
if !avx2_fma_available() {
return;
}
for n in DIMS {
for x in quantizer_inputs(n, 0xC0 ^ n as u64) {
let mut scalar = vec![BlockQ8_0::default(); n / 32];
quantize::quantize_row_q8_0(&x, &mut scalar).unwrap();
let mut fast = vec![BlockQ8_0::default(); n / 32];
quantize_row_q8_0(&x, &mut fast, false).unwrap();
for (s, f) in scalar.iter().zip(&fast) {
assert_eq!(s.d.to_bits(), f.d.to_bits());
assert_eq!(s.qs, f.qs);
}
}
}
}
#[test]
fn quantize_q8_k_bytes_identical_to_scalar() {
if !avx2_fma_available() {
return;
}
for n in DIMS {
for x in quantizer_inputs(n, 0xC8 ^ n as u64) {
let mut scalar = vec![BlockQ8K::default(); n / 256];
quantize::quantize_row_q8_k(&x, &mut scalar).unwrap();
let mut fast = vec![BlockQ8K::default(); n / 256];
quantize_row_q8_k(&x, &mut fast, false).unwrap();
for (s, f) in scalar.iter().zip(&fast) {
assert_eq!(s.d.to_bits(), f.d.to_bits());
assert_eq!(s.qs, f.qs);
assert_eq!(s.bsums, f.bsums);
}
}
}
}
#[test]
fn misaligned_weight_row_matches() {
if !avx2_fma_available() {
return;
}
let mut rng = Lcg(0xA11);
let xw: Vec<f32> = (0..2048).map(|_| rng.next_f32() * 4.0).collect();
let xa: Vec<f32> = (0..2048).map(|_| rng.next_f32() * 3.0).collect();
let acts = q8_k_acts(&xa);
for (wb, dot) in [
(
quantize_row_q4_k_test(&xw),
vec_dot_q4_k_q8_k as DispatchDot,
),
(quantize_row_q5_k_test(&xw), vec_dot_q5_k_q8_k),
(quantize_row_q6_k_test(&xw), vec_dot_q6_k_q8_k),
] {
let aligned = dot(&wb, &acts, false).unwrap();
let mut shifted = vec![0u8; wb.len() + 1];
shifted[1..].copy_from_slice(&wb);
let misaligned = dot(&shifted[1..], &acts, false).unwrap();
assert_eq!(aligned.to_bits(), misaligned.to_bits());
}
let wb = quantize_row_q8_0_test(&xw);
let mut acts0 = vec![BlockQ8_0::default(); 2048 / 32];
quantize::quantize_row_q8_0(&xa, &mut acts0).unwrap();
let aligned = vec_dot_q8_0_q8_0(&wb, &acts0, false).unwrap();
let mut shifted = vec![0u8; wb.len() + 1];
shifted[1..].copy_from_slice(&wb);
let misaligned = vec_dot_q8_0_q8_0(&shifted[1..], &acts0, false).unwrap();
assert_eq!(aligned.to_bits(), misaligned.to_bits());
}
#[test]
fn dispatch_validates_like_scalar() {
let acts2 = vec![BlockQ8K::default(); 2];
assert_eq!(
vec_dot_q4_k_q8_k(&[0; 144], &acts2, false).unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 1,
activation_blocks: 2,
},
);
assert_eq!(
vec_dot_q5_k_q8_k(&[0; 175], &acts2, false).unwrap_err(),
KernelError::RowBytesNotBlockMultiple {
format: QuantFormat::Q5_K,
len: 175,
block_bytes: 176,
},
);
assert_eq!(
vec_dot_q6_k_q8_k(&[0; 630], &acts2, false).unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 3,
activation_blocks: 2,
},
);
assert_eq!(
vec_dot_q8_0_q8_0(&[0; 35], &[BlockQ8_0::default(); 8], false).unwrap_err(),
KernelError::RowBytesNotBlockMultiple {
format: QuantFormat::Q8_0,
len: 35,
block_bytes: 34,
},
);
let x = [0f32; 100];
assert_eq!(
quantize_row_q8_0(&x, &mut [BlockQ8_0::default(); 3], false).unwrap_err(),
KernelError::QuantizeLenMismatch {
format: QuantFormat::Q8_0,
floats: 100,
out_blocks: 3,
}
);
assert_eq!(
quantize_row_q8_k(&x, &mut [BlockQ8K::default()], false).unwrap_err(),
KernelError::QuantizeLenMismatch {
format: QuantFormat::Q8_K,
floats: 100,
out_blocks: 1,
}
);
}
#[cfg(target_arch = "x86_64")]
#[test]
fn scale_unpack_matches_get_scale_min_k4() {
use super::super::blocks::get_scale_min_k4;
let mut rng = Lcg(0x5CA1E5);
for _ in 0..64 {
let mut scales = [0u8; 12];
for b in &mut scales {
*b = (rng.next_f32() * 128.0 + 128.0) as u8;
}
let (sc, mins) = super::x86::unpack_scales_mins(&scales);
for j in 0..8 {
let (s, m) = get_scale_min_k4(j, &scales);
assert_eq!(sc[j], s, "scale {j} of {scales:?}");
assert_eq!(mins[j], m, "min {j} of {scales:?}");
}
}
}
}