use super::KernelError;
use super::quants::{BlockQ8_0, BlockQ8K, QuantFormat, avx2};
use std::ops::Range;
use std::sync::OnceLock;
fn force_scalar() -> bool {
static FORCE: OnceLock<bool> = OnceLock::new();
*FORCE.get_or_init(|| std::env::var_os("RAMVAMP_FORCE_SCALAR").is_some())
}
const ROW_RANGE_ORDER: &str = "gemv rows: row range start vs end";
const ROW_RANGE_BOUND: &str = "gemv rows: row range end vs out_dim";
const FUSED_OUT: &str = "gemv_q8_k_fused_rows: out vs range";
const FUSED_PART: &str = "gemv_q8_k_fused_rows: part weight bytes vs rows / out vs range";
type DotQ8K = fn(&[u8], &[BlockQ8K], bool) -> Result<f32, KernelError>;
#[derive(Debug, Clone, Copy)]
struct Matrix<'a> {
format: QuantFormat,
bytes: &'a [u8],
in_dim: usize,
out_dim: usize,
}
struct Batch<'a, T> {
blocks: &'a [T],
n_acts: usize,
}
impl<'a, T> Batch<'a, T> {
fn single(blocks: &'a [T]) -> Self {
Self { blocks, n_acts: 1 }
}
}
fn validate(
what: &'static str,
m: Matrix<'_>,
activation_blocks: usize,
n_acts: usize,
rows: &Range<usize>,
out_len: usize,
) -> Result<usize, KernelError> {
let row_bytes = m.format.row_bytes(m.in_dim)?;
match row_bytes.checked_mul(m.out_dim) {
Some(expected) if expected == m.bytes.len() => {}
_ => {
return Err(KernelError::LengthMismatch {
what,
left: m.bytes.len(),
right: row_bytes.saturating_mul(m.out_dim),
});
}
}
if rows.start > rows.end {
return Err(KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: rows.start,
right: rows.end,
});
}
if rows.end > m.out_dim {
return Err(KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: rows.end,
right: m.out_dim,
});
}
let wanted = (rows.end - rows.start).saturating_mul(n_acts);
if out_len != wanted {
return Err(KernelError::LengthMismatch {
what,
left: out_len,
right: wanted,
});
}
let weight_blocks = (m.in_dim / m.format.block_weights()).saturating_mul(n_acts);
if weight_blocks != activation_blocks {
return Err(KernelError::BlockCountMismatch {
weight_blocks,
activation_blocks,
});
}
Ok(row_bytes)
}
fn k_quant_dot(what: &'static str, format: QuantFormat) -> Result<DotQ8K, KernelError> {
match format {
QuantFormat::Q4_K => Ok(avx2::vec_dot_q4_k_q8_k),
QuantFormat::Q5_K => Ok(avx2::vec_dot_q5_k_q8_k),
QuantFormat::Q6_K => Ok(avx2::vec_dot_q6_k_q8_k),
QuantFormat::Q8_0 | QuantFormat::Q8_K => {
Err(KernelError::UnsupportedFormat { what, format })
}
}
}
fn gemv_impl<T, F>(
dot: F,
what: &'static str,
m: Matrix<'_>,
acts: Batch<'_, T>,
rows: Range<usize>,
out: &mut [f32],
scalar: bool,
) -> Result<(), KernelError>
where
F: Fn(&[u8], &[T], bool) -> Result<f32, KernelError>,
{
let row_bytes = validate(what, m, acts.blocks.len(), acts.n_acts, &rows, out.len())?;
if acts.n_acts == 0 {
return Ok(());
}
let act_blocks = m.in_dim / m.format.block_weights();
for (i, dst) in out.chunks_mut(acts.n_acts).enumerate() {
let r = rows.start + i;
let weight_row = &m.bytes[r * row_bytes..(r + 1) * row_bytes];
for (a, o) in dst.iter_mut().enumerate() {
*o = dot(
weight_row,
&acts.blocks[a * act_blocks..(a + 1) * act_blocks],
scalar,
)?;
}
}
Ok(())
}
pub fn gemv_q8_k(
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
out: &mut [f32],
) -> Result<(), KernelError> {
let dot = k_quant_dot("gemv_q8_k", format)?;
gemv_impl(
dot,
"gemv_q8_k: weight bytes vs rows / out vs out_dim",
Matrix {
format,
bytes: weight,
in_dim,
out_dim,
},
Batch::single(acts),
0..out_dim,
out,
force_scalar(),
)
}
pub fn gemv_q8_k_rows(
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
rows: Range<usize>,
out: &mut [f32],
) -> Result<(), KernelError> {
let dot = k_quant_dot("gemv_q8_k_rows", format)?;
gemv_impl(
dot,
"gemv_q8_k_rows: weight bytes vs rows / out vs range",
Matrix {
format,
bytes: weight,
in_dim,
out_dim,
},
Batch::single(acts),
rows,
out,
force_scalar(),
)
}
#[allow(clippy::too_many_arguments)]
pub fn gemv_q8_k_batched(
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
n_acts: usize,
rows: Range<usize>,
out: &mut [f32],
) -> Result<(), KernelError> {
if matches!(format, QuantFormat::Q4_K | QuantFormat::Q6_K) && n_acts >= ROW_TILE {
return gemv_tiled_batched(format, weight, in_dim, out_dim, acts, n_acts, rows, out);
}
let dot = k_quant_dot("gemv_q8_k_batched", format)?;
gemv_impl(
dot,
"gemv_q8_k_batched: weight bytes vs rows / out vs range x n_acts",
Matrix {
format,
bytes: weight,
in_dim,
out_dim,
},
Batch {
blocks: acts,
n_acts,
},
rows,
out,
force_scalar(),
)
}
const ROW_TILE: usize = 4;
#[allow(clippy::too_many_arguments)]
fn gemv_tiled_batched(
format: QuantFormat,
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8K],
n_acts: usize,
rows: Range<usize>,
out: &mut [f32],
) -> Result<(), KernelError> {
let rows_dot = match format {
QuantFormat::Q6_K => avx2::vec_dot_q6_k_q8_k_rows,
_ => avx2::vec_dot_q4_k_q8_k_rows,
};
let m = Matrix {
format,
bytes: weight,
in_dim,
out_dim,
};
let row_bytes = validate(
"gemv_q8_k_batched: weight bytes vs rows / out vs range x n_acts",
m,
acts.len(),
n_acts,
&rows,
out.len(),
)?;
if n_acts == 0 {
return Ok(());
}
let scalar = force_scalar();
for (i, dst) in out.chunks_mut(n_acts).enumerate() {
let r = rows.start + i;
let weight_row = &weight[r * row_bytes..(r + 1) * row_bytes];
rows_dot(weight_row, acts, n_acts, dst, scalar)?;
}
Ok(())
}
pub fn gemv_q8_0(
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8_0],
out: &mut [f32],
) -> Result<(), KernelError> {
gemv_impl(
avx2::vec_dot_q8_0_q8_0,
"gemv_q8_0: weight bytes vs rows / out vs out_dim",
Matrix {
format: QuantFormat::Q8_0,
bytes: weight,
in_dim,
out_dim,
},
Batch::single(acts),
0..out_dim,
out,
force_scalar(),
)
}
pub fn gemv_q8_0_rows(
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8_0],
rows: Range<usize>,
out: &mut [f32],
) -> Result<(), KernelError> {
gemv_impl(
avx2::vec_dot_q8_0_q8_0,
"gemv_q8_0_rows: weight bytes vs rows / out vs range",
Matrix {
format: QuantFormat::Q8_0,
bytes: weight,
in_dim,
out_dim,
},
Batch::single(acts),
rows,
out,
force_scalar(),
)
}
pub fn gemv_q8_0_batched(
weight: &[u8],
in_dim: usize,
out_dim: usize,
acts: &[BlockQ8_0],
n_acts: usize,
rows: Range<usize>,
out: &mut [f32],
) -> Result<(), KernelError> {
gemv_impl(
avx2::vec_dot_q8_0_q8_0,
"gemv_q8_0_batched: weight bytes vs rows / out vs range x n_acts",
Matrix {
format: QuantFormat::Q8_0,
bytes: weight,
in_dim,
out_dim,
},
Batch {
blocks: acts,
n_acts,
},
rows,
out,
force_scalar(),
)
}
#[derive(Debug, Clone, Copy)]
pub struct FusedPart<'a> {
pub format: QuantFormat,
pub weight: &'a [u8],
pub in_dim: usize,
pub out_dim: usize,
pub acts: &'a [BlockQ8K],
}
impl FusedPart<'_> {
#[must_use]
pub fn empty() -> Self {
Self {
format: QuantFormat::Q4_K,
weight: &[],
in_dim: 0,
out_dim: 0,
acts: &[],
}
}
}
#[must_use]
pub fn fused_out_dim(parts: &[FusedPart<'_>]) -> usize {
parts
.iter()
.fold(0usize, |total, part| total.saturating_add(part.out_dim))
}
pub fn fused_row_parts<'p>(
parts: &'p [FusedPart<'_>],
rows: Range<usize>,
) -> impl Iterator<Item = (usize, Range<usize>)> + 'p {
let mut start = 0usize;
parts.iter().enumerate().filter_map(move |(p, part)| {
let base = start;
let end = base.saturating_add(part.out_dim);
start = end;
let lo = rows.start.clamp(base, end);
let hi = rows.end.clamp(base, end);
(lo < hi).then(|| (p, lo - base..hi - base))
})
}
pub fn gemv_q8_k_fused_rows(
parts: &[FusedPart<'_>],
rows: Range<usize>,
out: &mut [f32],
) -> Result<(), KernelError> {
gemv_q8_k_fused_rows_impl(parts, rows, out, force_scalar())
}
fn gemv_q8_k_fused_rows_impl(
parts: &[FusedPart<'_>],
rows: Range<usize>,
out: &mut [f32],
scalar: bool,
) -> Result<(), KernelError> {
let total = fused_out_dim(parts);
if rows.start > rows.end {
return Err(KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: rows.start,
right: rows.end,
});
}
if rows.end > total {
return Err(KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: rows.end,
right: total,
});
}
let wanted = rows.end - rows.start;
if out.len() != wanted {
return Err(KernelError::LengthMismatch {
what: FUSED_OUT,
left: out.len(),
right: wanted,
});
}
let mut rest = out;
for (p, local) in fused_row_parts(parts, rows) {
let part = parts[p];
if local.len() > rest.len() {
return Err(KernelError::LengthMismatch {
what: FUSED_OUT,
left: rest.len(),
right: local.len(),
});
}
let (head, tail) = rest.split_at_mut(local.len());
let dot = k_quant_dot("gemv_q8_k_fused_rows", part.format)?;
gemv_impl(
dot,
FUSED_PART,
Matrix {
format: part.format,
bytes: part.weight,
in_dim: part.in_dim,
out_dim: part.out_dim,
},
Batch::single(part.acts),
local,
head,
scalar,
)?;
rest = tail;
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::io::Write as _;
use super::super::quants::{quantize_row_q8_0, quantize_row_q8_k};
use super::*;
struct Lcg(u64);
impl Lcg {
fn next_u64(&mut self) -> u64 {
self.0 = self
.0
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
self.0
}
fn next_f32(&mut self) -> f32 {
(self.next_u64() >> 40) as f32 / (1u64 << 24) as f32 * 2.0 - 1.0
}
fn block_bytes(&mut self, format: QuantFormat, scale_offs: &[usize]) -> Vec<u8> {
let mut b: Vec<u8> = (0..format.block_bytes())
.map(|_| (self.next_u64() >> 32) as u8)
.collect();
for &off in scale_offs {
let d = super::super::quants::f32_to_f16(0.01 + 0.02 * (self.next_f32() + 1.0));
b[off..off + 2].copy_from_slice(&d.to_le_bytes());
}
b
}
}
fn synth_matrix(format: QuantFormat, in_dim: usize, out_dim: usize, seed: u64) -> Vec<u8> {
let scale_offs: &[usize] = match format {
QuantFormat::Q4_K | QuantFormat::Q5_K => &[0, 2], QuantFormat::Q6_K => &[208],
_ => &[0],
};
let mut rng = Lcg(seed);
let blocks_per_row = in_dim / format.block_weights();
let mut w = Vec::with_capacity(out_dim * blocks_per_row * format.block_bytes());
for _ in 0..out_dim * blocks_per_row {
w.extend_from_slice(&rng.block_bytes(format, scale_offs));
}
w
}
fn q8_k_acts(n: usize, seed: u64) -> Vec<BlockQ8K> {
let mut rng = Lcg(seed);
let x: Vec<f32> = (0..n).map(|_| rng.next_f32() * 3.0).collect();
let mut acts = vec![BlockQ8K::default(); n / 256];
quantize_row_q8_k(&x, &mut acts).unwrap();
acts
}
fn q8_0_acts(n: usize, seed: u64) -> Vec<BlockQ8_0> {
let mut rng = Lcg(seed);
let x: Vec<f32> = (0..n).map(|_| rng.next_f32() * 3.0).collect();
let mut acts = vec![BlockQ8_0::default(); n / 32];
quantize_row_q8_0(&x, &mut acts).unwrap();
acts
}
fn act_seed(seed: u64, a: usize) -> u64 {
seed ^ (a as u64)
.wrapping_add(1)
.wrapping_mul(0x9E37_79B9_7F4A_7C15)
}
fn q8_k_batch(in_dim: usize, n_acts: usize, seed: u64) -> Vec<BlockQ8K> {
let mut v = Vec::with_capacity(n_acts * (in_dim / 256));
for a in 0..n_acts {
v.extend_from_slice(&q8_k_acts(in_dim, act_seed(seed, a)));
}
v
}
fn q8_0_batch(in_dim: usize, n_acts: usize, seed: u64) -> Vec<BlockQ8_0> {
let mut v = Vec::with_capacity(n_acts * (in_dim / 32));
for a in 0..n_acts {
v.extend_from_slice(&q8_0_acts(in_dim, act_seed(seed, a)));
}
v
}
struct Fixture {
name: &'static str,
format: QuantFormat,
in_dim: usize,
out_dim: usize,
weight: Vec<u8>,
}
impl Fixture {
fn new(name: &'static str, format: QuantFormat, in_dim: usize, out_dim: usize) -> Self {
let seed = 0x6E0 ^ (in_dim as u64) << 20 ^ (out_dim as u64) << 4 ^ format as u64;
Self {
name,
format,
in_dim,
out_dim,
weight: synth_matrix(format, in_dim, out_dim, seed),
}
}
fn matrix(&self) -> Matrix<'_> {
Matrix {
format: self.format,
bytes: &self.weight,
in_dim: self.in_dim,
out_dim: self.out_dim,
}
}
}
fn k_quant_fixtures() -> Vec<Fixture> {
vec![
Fixture::new("expert gate (Q4_K)", QuantFormat::Q4_K, 2048, 768),
Fixture::new("expert up (Q4_K)", QuantFormat::Q4_K, 2048, 768),
Fixture::new("expert down (Q6_K layers)", QuantFormat::Q6_K, 768, 2048),
Fixture::new("expert down (Q4_K layers)", QuantFormat::Q4_K, 768, 2048),
Fixture::new("attn_q (Q4_K)", QuantFormat::Q4_K, 2048, 4096),
Fixture::new("attn_v (Q6_K layers)", QuantFormat::Q6_K, 2048, 512),
Fixture::new("attn_v (Q4_K layers)", QuantFormat::Q4_K, 2048, 512),
Fixture::new("attn_output (Q5_K)", QuantFormat::Q5_K, 4096, 2048),
Fixture::new(
"lm_head (Q6_K, out_dim trimmed)",
QuantFormat::Q6_K,
2048,
1024,
),
]
}
const BATCH_OUT_DIM: usize = 66;
const BATCH_WIDTHS: [usize; 5] = [1, 2, 3, 8, 33];
fn k_quant_batch_fixtures() -> Vec<Fixture> {
vec![
Fixture::new(
"expert gate/up (Q4_K, 1152 B rows)",
QuantFormat::Q4_K,
2048,
BATCH_OUT_DIM,
),
Fixture::new(
"expert down (Q4_K, 432 B rows)",
QuantFormat::Q4_K,
768,
BATCH_OUT_DIM,
),
Fixture::new(
"expert down (Q6_K, 630 B rows)",
QuantFormat::Q6_K,
768,
BATCH_OUT_DIM,
),
Fixture::new(
"attn_v / lm_head (Q6_K, 1680 B rows)",
QuantFormat::Q6_K,
2048,
BATCH_OUT_DIM,
),
Fixture::new(
"attn_output (Q5_K, 2816 B rows)",
QuantFormat::Q5_K,
4096,
BATCH_OUT_DIM,
),
]
}
fn chunked(out_dim: usize, chunk: usize) -> Vec<Range<usize>> {
let chunk = chunk.max(1);
let mut v = Vec::new();
let mut s = 0;
while s < out_dim {
let e = (s + chunk).min(out_dim);
v.push(s..e);
s = e;
}
if v.is_empty() {
v.push(0..0);
}
v
}
fn partitions(out_dim: usize) -> Vec<(&'static str, Vec<Range<usize>>)> {
assert!(out_dim >= 6, "partition schemes assume a real out_dim");
let half = out_dim / 2;
let third = out_dim / 3;
let pool_ceil = chunked(out_dim, out_dim.div_ceil(6));
let floor = (out_dim / 6).max(1);
let mut pool_floor: Vec<Range<usize>> =
(0..5).map(|i| i * floor..(i + 1) * floor).collect();
pool_floor.push(5 * floor..out_dim);
vec![
("whole", chunked(out_dim, out_dim)),
("halves", vec![0..half, half..out_dim]),
("pool6-ceil", pool_ceil),
("pool6-floor-remainder", pool_floor),
("stride7-remainder", chunked(out_dim, 7)),
("single-rows", (0..out_dim).map(|r| r..r + 1).collect()),
(
"uneven-with-empties",
vec![
0..0,
0..1,
1..1,
1..third,
third..third,
third..out_dim - 1,
out_dim - 1..out_dim,
out_dim..out_dim,
],
),
(
"empty-middle",
vec![0..half, half..half, half..out_dim, out_dim..out_dim],
),
]
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum KernelPaths {
Both,
ScalarOnly,
}
impl KernelPaths {
fn flags(self) -> &'static [bool] {
match self {
KernelPaths::Both => &[false, true],
KernelPaths::ScalarOnly => &[true],
}
}
}
fn kernel_paths(claim: &str) -> KernelPaths {
let dispatch = avx2::avx2_fma_available();
#[cfg(target_arch = "x86_64")]
{
let cpu = std::arch::is_x86_feature_detected!("avx2")
&& std::arch::is_x86_feature_detected!("fma");
assert_eq!(
dispatch, cpu,
"kernel dispatch disagrees with this x86_64 CPU: CPUID reports \
avx2+fma = {cpu}, `avx2::avx2_fma_available()` reports \
{dispatch}. Every GEMV dispatches on the latter, so this is a \
live mis-dispatch, not an unsupported host."
);
}
if dispatch {
return KernelPaths::Both;
}
let mut err = std::io::stderr();
let _ = writeln!(
err,
"\n\
##########################################################\n\
# NOT PROVEN ON THIS HOST: {claim}\n\
#\n\
# This host has no AVX2+FMA, so `scalar = false` dispatches to\n\
# the same scalar kernel as `scalar = true`. The row-partition\n\
# logic was exercised on the scalar path only; the\n\
# AVX2-vs-scalar half of phase 5's bit-identity gate was NOT\n\
# tested. A green run here is not evidence for it. Re-run the\n\
# kernel tests on an AVX2+FMA machine before relying on it.\n\
##########################################################\n"
);
let _ = err.flush();
KernelPaths::ScalarOnly
}
fn assert_partitions_bit_identical<F>(f: &Fixture, scalar: bool, width: usize, mut run: F)
where
F: FnMut(&Fixture, Range<usize>, &mut [f32], bool),
{
let path = if scalar { "scalar" } else { "avx2" };
let len = f.out_dim * width;
let mut full = vec![f32::NAN; len];
run(f, 0..f.out_dim, &mut full, scalar);
assert!(
full.iter().all(|v| v.is_finite()),
"{} [{path}]: fixture produced a non-finite reference",
f.name,
);
for (scheme, ranges) in partitions(f.out_dim) {
let mut got = vec![f32::NAN; len];
let mut rest: &mut [f32] = &mut got;
let mut cursor = 0usize;
for r in &ranges {
assert_eq!(
r.start, cursor,
"{} [{path}] {scheme}: ranges must tile 0..out_dim in order",
f.name,
);
let (head, tail) = rest.split_at_mut(r.len() * width);
run(f, r.clone(), head, scalar);
rest = tail;
cursor = r.end;
}
assert_eq!(
cursor, f.out_dim,
"{} [{path}] {scheme}: short tiling",
f.name
);
assert!(rest.is_empty());
for (i, (&g, &w)) in got.iter().zip(&full).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"{} [{path}] {scheme} (width {width}): row {} act {} differs ({g} vs {w})",
f.name,
i / width,
i % width,
);
}
}
}
fn run_k(
f: &Fixture,
acts: &[BlockQ8K],
n_acts: usize,
rows: Range<usize>,
out: &mut [f32],
scalar: bool,
) {
let dot = k_quant_dot("test", f.format).unwrap();
let acts = Batch {
blocks: acts,
n_acts,
};
gemv_impl(dot, "test", f.matrix(), acts, rows, out, scalar).unwrap();
}
fn run_0(
f: &Fixture,
acts: &[BlockQ8_0],
n_acts: usize,
rows: Range<usize>,
out: &mut [f32],
scalar: bool,
) {
let acts = Batch {
blocks: acts,
n_acts,
};
gemv_impl(
avx2::vec_dot_q8_0_q8_0,
"test",
f.matrix(),
acts,
rows,
out,
scalar,
)
.unwrap();
}
#[test]
fn gemv_q8_k_matches_per_row_dots() {
for (format, in_dim, out_dim) in [
(QuantFormat::Q4_K, 2048, 64),
(QuantFormat::Q5_K, 4096, 32),
(QuantFormat::Q6_K, 768, 96),
] {
let w = synth_matrix(format, in_dim, out_dim, 0x6E0 ^ in_dim as u64);
let acts = q8_k_acts(in_dim, 0xAC ^ in_dim as u64);
let mut out = vec![0f32; out_dim];
gemv_q8_k(format, &w, in_dim, out_dim, &acts, &mut out).unwrap();
let dot = match format {
QuantFormat::Q4_K => avx2::vec_dot_q4_k_q8_k,
QuantFormat::Q5_K => avx2::vec_dot_q5_k_q8_k,
QuantFormat::Q6_K => avx2::vec_dot_q6_k_q8_k,
_ => unreachable!(),
};
let row_bytes = format.row_bytes(in_dim).unwrap();
for (r, &o) in out.iter().enumerate() {
let want = dot(
&w[r * row_bytes..(r + 1) * row_bytes],
&acts,
force_scalar(),
)
.unwrap();
assert_eq!(o.to_bits(), want.to_bits(), "{format:?} row {r}");
}
}
}
#[test]
fn gemv_q8_0_matches_per_row_dots() {
let (in_dim, out_dim) = (2048, 64);
let w = synth_matrix(QuantFormat::Q8_0, in_dim, out_dim, 0x8_0);
let mut rng = Lcg(0xAC7);
let x: Vec<f32> = (0..in_dim).map(|_| rng.next_f32() * 3.0).collect();
let mut acts = vec![BlockQ8_0::default(); in_dim / 32];
quantize_row_q8_0(&x, &mut acts).unwrap();
let mut out = vec![0f32; out_dim];
gemv_q8_0(&w, in_dim, out_dim, &acts, &mut out).unwrap();
let row_bytes = QuantFormat::Q8_0.row_bytes(in_dim).unwrap();
for (r, &o) in out.iter().enumerate() {
let want = avx2::vec_dot_q8_0_q8_0(
&w[r * row_bytes..(r + 1) * row_bytes],
&acts,
force_scalar(),
)
.unwrap();
assert_eq!(o.to_bits(), want.to_bits(), "row {r}");
}
}
#[test]
fn k_quant_row_partitions_are_bit_identical() {
let paths = kernel_paths(
"k-quant row partitions are bit-identical on both the AVX2 and the \
scalar kernel path",
);
for f in k_quant_fixtures() {
let acts = q8_k_acts(f.in_dim, 0xAC ^ f.in_dim as u64);
for &scalar in paths.flags() {
assert_partitions_bit_identical(&f, scalar, 1, |f, rows, out, scalar| {
run_k(f, &acts, 1, rows, out, scalar)
});
}
}
}
#[test]
fn q8_0_row_partitions_are_bit_identical() {
let paths = kernel_paths(
"Q8_0 row partitions are bit-identical on both the AVX2 and the \
scalar kernel path",
);
let f = Fixture::new("attn_k (Q8_0)", QuantFormat::Q8_0, 2048, 512);
let acts = q8_0_acts(f.in_dim, 0xAC7);
for &scalar in paths.flags() {
assert_partitions_bit_identical(&f, scalar, 1, |f, rows, out, scalar| {
run_0(f, &acts, 1, rows, out, scalar)
});
}
}
fn assert_batch_matches_single<F>(
f: &Fixture,
n_acts: usize,
scalar: bool,
got: &[f32],
mut single: F,
) where
F: FnMut(usize, &mut [f32]),
{
let path = if scalar { "scalar" } else { "avx2" };
assert_eq!(got.len(), f.out_dim * n_acts, "{}: bad test buffer", f.name);
let mut first: Option<Vec<f32>> = None;
let mut columns_differ = false;
for a in 0..n_acts {
let mut want = vec![f32::NAN; f.out_dim];
single(a, &mut want);
assert!(
want.iter().all(|v| v.is_finite()),
"{} [{path}] n_acts {n_acts}: activation {a} produced a \
non-finite reference",
f.name,
);
for (r, w) in want.iter().enumerate() {
let g = got[r * n_acts + a];
assert_eq!(
g.to_bits(),
w.to_bits(),
"{} [{path}] n_acts {n_acts}: row {r} activation {a} \
differs ({g} vs {w})",
f.name,
);
}
match &first {
None => first = Some(want),
Some(f0) => {
columns_differ |= f0
.iter()
.zip(&want)
.any(|(x, y)| x.to_bits() != y.to_bits());
}
}
}
assert!(
n_acts < 2 || columns_differ,
"{}: every activation row produced the same column, so this \
fixture cannot distinguish a batched kernel that ignores the \
activation index",
f.name,
);
}
#[test]
fn k_quant_batched_matches_single_vector_bitwise() {
let paths = kernel_paths(
"batched k-quant GEMV is bit-identical to the single-vector GEMV \
on both the AVX2 and the scalar kernel path",
);
for f in k_quant_batch_fixtures() {
let blocks = f.in_dim / f.format.block_weights();
for n_acts in BATCH_WIDTHS {
let acts = q8_k_batch(f.in_dim, n_acts, 0xBA7C ^ f.in_dim as u64);
for &scalar in paths.flags() {
let mut got = vec![f32::NAN; f.out_dim * n_acts];
run_k(&f, &acts, n_acts, 0..f.out_dim, &mut got, scalar);
assert_batch_matches_single(&f, n_acts, scalar, &got, |a, want| {
let row = &acts[a * blocks..(a + 1) * blocks];
run_k(&f, row, 1, 0..f.out_dim, want, scalar);
});
}
}
}
}
#[test]
fn q8_0_batched_matches_single_vector_bitwise() {
let paths = kernel_paths(
"batched Q8_0 GEMV is bit-identical to the single-vector GEMV on \
both the AVX2 and the scalar kernel path",
);
let f = Fixture::new(
"attn_k (Q8_0, 2176 B rows)",
QuantFormat::Q8_0,
2048,
BATCH_OUT_DIM,
);
let blocks = f.in_dim / f.format.block_weights();
for n_acts in BATCH_WIDTHS {
let acts = q8_0_batch(f.in_dim, n_acts, 0xBA70);
for &scalar in paths.flags() {
let mut got = vec![f32::NAN; f.out_dim * n_acts];
run_0(&f, &acts, n_acts, 0..f.out_dim, &mut got, scalar);
assert_batch_matches_single(&f, n_acts, scalar, &got, |a, want| {
let row = &acts[a * blocks..(a + 1) * blocks];
run_0(&f, row, 1, 0..f.out_dim, want, scalar);
});
}
}
}
#[test]
fn k_quant_batched_row_partitions_are_bit_identical() {
let paths = kernel_paths(
"batched k-quant row partitions are bit-identical on both the \
AVX2 and the scalar kernel path",
);
for f in k_quant_batch_fixtures() {
for n_acts in BATCH_WIDTHS {
let acts = q8_k_batch(f.in_dim, n_acts, 0x9A27 ^ f.in_dim as u64);
for &scalar in paths.flags() {
assert_partitions_bit_identical(&f, scalar, n_acts, |f, rows, out, scalar| {
run_k(f, &acts, n_acts, rows, out, scalar)
});
}
}
}
}
#[test]
fn q8_0_batched_row_partitions_are_bit_identical() {
let paths = kernel_paths(
"batched Q8_0 row partitions are bit-identical on both the AVX2 \
and the scalar kernel path",
);
let f = Fixture::new(
"attn_k (Q8_0, 2176 B rows)",
QuantFormat::Q8_0,
2048,
BATCH_OUT_DIM,
);
for n_acts in BATCH_WIDTHS {
let acts = q8_0_batch(f.in_dim, n_acts, 0x9A20);
for &scalar in paths.flags() {
assert_partitions_bit_identical(&f, scalar, n_acts, |f, rows, out, scalar| {
run_0(f, &acts, n_acts, rows, out, scalar)
});
}
}
}
#[test]
fn batched_misaligned_weight_slab_is_bit_identical() {
let paths = kernel_paths(
"batched GEMV is bit-identical at 1- and 2-byte weight-slab \
alignment on both the AVX2 and the scalar kernel path",
);
let n_acts = 5;
for f in [
Fixture::new("expert down (Q6_K, 630 B rows)", QuantFormat::Q6_K, 768, 34),
Fixture::new("expert down (Q4_K, 432 B rows)", QuantFormat::Q4_K, 768, 34),
] {
let acts = q8_k_batch(f.in_dim, n_acts, 0xA11 ^ f.in_dim as u64);
let dot = k_quant_dot("test", f.format).unwrap();
for &scalar in paths.flags() {
let mut want = vec![f32::NAN; f.out_dim * n_acts];
run_k(&f, &acts, n_acts, 0..f.out_dim, &mut want, scalar);
assert!(
want.iter().all(|v| v.is_finite()),
"{}: fixture produced a non-finite reference",
f.name,
);
for shift in [1usize, 2] {
let mut shifted = vec![0u8; f.weight.len() + shift];
shifted[shift..].copy_from_slice(&f.weight);
let m = Matrix {
format: f.format,
bytes: &shifted[shift..],
in_dim: f.in_dim,
out_dim: f.out_dim,
};
let acts = Batch {
blocks: &acts,
n_acts,
};
let mut got = vec![f32::NAN; f.out_dim * n_acts];
gemv_impl(dot, "test", m, acts, 0..f.out_dim, &mut got, scalar).unwrap();
for (i, (&g, &w)) in got.iter().zip(&want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"{} [shift {shift}]: row {} activation {} differs \
({g} vs {w})",
f.name,
i / n_acts,
i % n_acts,
);
}
}
}
}
}
#[test]
fn public_batched_entry_points_match_single_vector() {
let f = Fixture::new("expert gate (Q4_K)", QuantFormat::Q4_K, 2048, 768);
let n_acts = 5;
let blocks = f.in_dim / 256;
let acts = q8_k_batch(f.in_dim, n_acts, 0x51CE);
let mut whole = vec![f32::NAN; f.out_dim * n_acts];
gemv_q8_k_batched(
f.format,
&f.weight,
f.in_dim,
f.out_dim,
&acts,
n_acts,
0..f.out_dim,
&mut whole,
)
.unwrap();
assert_batch_matches_single(&f, n_acts, force_scalar(), &whole, |a, want| {
let row = &acts[a * blocks..(a + 1) * blocks];
gemv_q8_k(f.format, &f.weight, f.in_dim, f.out_dim, row, want).unwrap();
});
let chunk = f.out_dim.div_ceil(6);
let mut sharded = vec![f32::NAN; f.out_dim * n_acts];
std::thread::scope(|s| {
for (i, part) in sharded.chunks_mut(chunk * n_acts).enumerate() {
let (f, acts) = (&f, &acts);
let rows = i * chunk..i * chunk + part.len() / n_acts;
s.spawn(move || {
gemv_q8_k_batched(
f.format, &f.weight, f.in_dim, f.out_dim, acts, n_acts, rows, part,
)
.unwrap();
});
}
});
for (i, (&g, &w)) in sharded.iter().zip(&whole).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"gemv_q8_k_batched shard: row {} activation {}",
i / n_acts,
i % n_acts,
);
}
let f0 = Fixture::new("attn_k (Q8_0)", QuantFormat::Q8_0, 2048, 512);
let n0 = 3;
let blocks0 = f0.in_dim / 32;
let acts0 = q8_0_batch(f0.in_dim, n0, 0x51CF);
let mut got0 = vec![f32::NAN; f0.out_dim * n0];
let mut rest: &mut [f32] = &mut got0;
let mut cursor = 0usize;
for end in [1usize, 333, 512] {
let (head, tail) = rest.split_at_mut((end - cursor) * n0);
gemv_q8_0_batched(
&f0.weight,
f0.in_dim,
f0.out_dim,
&acts0,
n0,
cursor..end,
head,
)
.unwrap();
rest = tail;
cursor = end;
}
assert_batch_matches_single(&f0, n0, force_scalar(), &got0, |a, want| {
let row = &acts0[a * blocks0..(a + 1) * blocks0];
gemv_q8_0(&f0.weight, f0.in_dim, f0.out_dim, row, want).unwrap();
});
}
const BATCHED_K_WHAT: &str = "gemv_q8_k_batched: weight bytes vs rows / out vs range x n_acts";
const BATCHED_0_WHAT: &str = "gemv_q8_0_batched: weight bytes vs rows / out vs range x n_acts";
#[test]
fn batched_rejects_wrong_formats() {
let acts = vec![BlockQ8K::default(); 2];
let mut out = [0f32; 2];
for format in [QuantFormat::Q8_0, QuantFormat::Q8_K] {
assert_eq!(
gemv_q8_k_batched(format, &[], 256, 1, &acts, 2, 0..1, &mut out).unwrap_err(),
KernelError::UnsupportedFormat {
what: "gemv_q8_k_batched",
format,
},
);
}
}
#[test]
fn batched_rejects_bad_sizes() {
let row = QuantFormat::Q4_K.row_bytes(2048).unwrap(); let w = vec![0u8; row * 4];
let acts = vec![BlockQ8K::default(); 24]; let mut out = vec![0f32; 12];
assert_eq!(
gemv_q8_k_batched(
QuantFormat::Q4_K,
&w,
2048,
4,
&acts,
3,
0..4,
&mut out[..4]
)
.unwrap_err(),
KernelError::LengthMismatch {
what: BATCHED_K_WHAT,
left: 4,
right: 12,
},
);
assert_eq!(
gemv_q8_k_batched(
QuantFormat::Q4_K,
&w,
2048,
4,
&acts[..20],
3,
0..4,
&mut out
)
.unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 24,
activation_blocks: 20,
},
);
assert_eq!(
gemv_q8_k_batched(
QuantFormat::Q4_K,
&w,
2048,
4,
&acts,
2,
0..4,
&mut out[..8]
)
.unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 16,
activation_blocks: 24,
},
);
assert_eq!(
gemv_q8_k_batched(
QuantFormat::Q4_K,
&w[..row * 3],
2048,
4,
&acts,
3,
0..4,
&mut out
)
.unwrap_err(),
KernelError::LengthMismatch {
what: BATCHED_K_WHAT,
left: row * 3,
right: row * 4,
},
);
assert_eq!(
gemv_q8_k_batched(QuantFormat::Q4_K, &w, 2000, 4, &acts, 3, 0..4, &mut out)
.unwrap_err(),
KernelError::IndivisibleRow {
format: QuantFormat::Q4_K,
in_dim: 2000,
block_weights: 256,
},
);
assert_eq!(
gemv_q8_k_batched(
QuantFormat::Q4_K,
&w,
2048,
4,
&acts,
usize::MAX,
0..4,
&mut out
)
.unwrap_err(),
KernelError::LengthMismatch {
what: BATCHED_K_WHAT,
left: 12,
right: usize::MAX,
},
);
assert_eq!(
gemv_q8_k_batched(
QuantFormat::Q4_K,
&w,
2048,
4,
&acts,
usize::MAX,
0..0,
&mut []
)
.unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: usize::MAX,
activation_blocks: 24,
},
);
let row0 = QuantFormat::Q8_0.row_bytes(2048).unwrap();
let w0 = vec![0u8; row0 * 2];
let acts0 = vec![BlockQ8_0::default(); 128]; assert_eq!(
gemv_q8_0_batched(&w0, 2048, 2, &acts0, 2, 0..2, &mut out[..3]).unwrap_err(),
KernelError::LengthMismatch {
what: BATCHED_0_WHAT,
left: 3,
right: 4,
},
);
assert_eq!(
gemv_q8_0_batched(&w0, 2048, 2, &acts0[..100], 2, 0..2, &mut out[..4]).unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 128,
activation_blocks: 100,
},
);
}
#[test]
fn batched_rejects_bad_row_ranges() {
let row = QuantFormat::Q4_K.row_bytes(2048).unwrap();
let w = vec![0u8; row * 4]; let acts = vec![BlockQ8K::default(); 16]; let mut out = vec![0f32; 8];
let rows_err = |rows: Range<usize>, out: &mut [f32]| {
gemv_q8_k_batched(QuantFormat::Q4_K, &w, 2048, 4, &acts, 2, rows, out).unwrap_err()
};
assert_eq!(
rows_err(raw_range(3, 1), &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: 3,
right: 1,
},
);
assert_eq!(
rows_err(2..5, &mut out[..6]),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: 5,
right: 4,
},
);
assert_eq!(
rows_err(9..9, &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: 9,
right: 4,
},
);
assert_eq!(
rows_err(usize::MAX - 1..usize::MAX, &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: usize::MAX,
right: 4,
},
);
assert_eq!(
rows_err(1..3, &mut out),
KernelError::LengthMismatch {
what: BATCHED_K_WHAT,
left: 8,
right: 4,
},
);
let row0 = QuantFormat::Q8_0.row_bytes(2048).unwrap();
let w0 = vec![0u8; row0 * 2];
let acts0 = vec![BlockQ8_0::default(); 128];
assert_eq!(
gemv_q8_0_batched(&w0, 2048, 2, &acts0, 2, raw_range(2, 1), &mut []).unwrap_err(),
KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: 2,
right: 1,
},
);
assert_eq!(
gemv_q8_0_batched(&w0, 2048, 2, &acts0, 2, 0..3, &mut out[..6]).unwrap_err(),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: 3,
right: 2,
},
);
}
#[test]
fn batched_zero_width_and_empty_range_are_no_ops() {
let f = Fixture::new("expert gate (Q4_K)", QuantFormat::Q4_K, 2048, 768);
let mut none: [f32; 0] = [];
gemv_q8_k_batched(
f.format,
&f.weight,
f.in_dim,
f.out_dim,
&[],
0,
0..f.out_dim,
&mut none,
)
.unwrap();
let empty_k = vec![BlockQ8K::default(); 4];
gemv_q8_k_batched(QuantFormat::Q4_K, &[], 256, 0, &empty_k, 4, 0..0, &mut none).unwrap();
let n_acts = 4;
let acts = q8_k_batch(f.in_dim, n_acts, 0xE377);
let mut out = vec![7.5f32; f.out_dim * n_acts];
{
let (_, mid) = out.split_at_mut(300 * n_acts);
let (empty, _) = mid.split_at_mut(0);
gemv_q8_k_batched(
f.format,
&f.weight,
f.in_dim,
f.out_dim,
&acts,
n_acts,
300..300,
empty,
)
.unwrap();
}
assert!(out.iter().all(|&v| v == 7.5));
let f0 = Fixture::new("attn_k (Q8_0)", QuantFormat::Q8_0, 2048, 512);
gemv_q8_0_batched(
&f0.weight,
f0.in_dim,
f0.out_dim,
&[],
0,
0..f0.out_dim,
&mut none,
)
.unwrap();
let empty_0 = vec![BlockQ8_0::default(); 4 * (256 / 32)];
gemv_q8_0_batched(&[], 256, 0, &empty_0, 4, 0..0, &mut none).unwrap();
}
#[test]
fn public_row_range_entry_points_match_whole_matrix() {
let f = Fixture::new("expert gate (Q4_K)", QuantFormat::Q4_K, 2048, 768);
let acts = q8_k_acts(f.in_dim, 0x51CE);
let mut want = vec![0f32; f.out_dim];
gemv_q8_k(f.format, &f.weight, f.in_dim, f.out_dim, &acts, &mut want).unwrap();
let chunk = f.out_dim.div_ceil(6);
let mut got = vec![f32::NAN; f.out_dim];
std::thread::scope(|s| {
for (i, part) in got.chunks_mut(chunk).enumerate() {
let (f, acts) = (&f, &acts);
let rows = i * chunk..i * chunk + part.len();
s.spawn(move || {
gemv_q8_k_rows(f.format, &f.weight, f.in_dim, f.out_dim, acts, rows, part)
.unwrap();
});
}
});
for (r, (&g, &w)) in got.iter().zip(&want).enumerate() {
assert_eq!(g.to_bits(), w.to_bits(), "gemv_q8_k_rows row {r}");
}
let f0 = Fixture::new("attn_k (Q8_0)", QuantFormat::Q8_0, 2048, 512);
let acts0 = q8_0_acts(f0.in_dim, 0x51CF);
let mut want0 = vec![0f32; f0.out_dim];
gemv_q8_0(&f0.weight, f0.in_dim, f0.out_dim, &acts0, &mut want0).unwrap();
let mut got0 = vec![f32::NAN; f0.out_dim];
let mut rest: &mut [f32] = &mut got0;
let mut cursor = 0usize;
for end in [1usize, 333, 512] {
let (head, tail) = rest.split_at_mut(end - cursor);
gemv_q8_0_rows(&f0.weight, f0.in_dim, f0.out_dim, &acts0, cursor..end, head).unwrap();
rest = tail;
cursor = end;
}
for (r, (&g, &w)) in got0.iter().zip(&want0).enumerate() {
assert_eq!(g.to_bits(), w.to_bits(), "gemv_q8_0_rows row {r}");
}
}
#[test]
fn rejects_wrong_formats() {
let acts = vec![BlockQ8K::default()];
let mut out = [0f32; 1];
for format in [QuantFormat::Q8_0, QuantFormat::Q8_K] {
assert_eq!(
gemv_q8_k(format, &[], 256, 1, &acts, &mut out).unwrap_err(),
KernelError::UnsupportedFormat {
what: "gemv_q8_k",
format,
},
);
assert_eq!(
gemv_q8_k_rows(format, &[], 256, 1, &acts, 0..1, &mut out).unwrap_err(),
KernelError::UnsupportedFormat {
what: "gemv_q8_k_rows",
format,
},
);
}
}
#[test]
fn rejects_bad_sizes() {
let acts = vec![BlockQ8K::default(); 8]; let mut out = vec![0f32; 4];
let row = QuantFormat::Q4_K.row_bytes(2048).unwrap();
assert_eq!(
gemv_q8_k(QuantFormat::Q4_K, &[0; 1152], 2000, 1, &acts, &mut out[..1]).unwrap_err(),
KernelError::IndivisibleRow {
format: QuantFormat::Q4_K,
in_dim: 2000,
block_weights: 256,
},
);
assert_eq!(
gemv_q8_k(
QuantFormat::Q4_K,
&vec![0; row * 3],
2048,
4,
&acts,
&mut out
)
.unwrap_err(),
KernelError::LengthMismatch {
what: "gemv_q8_k: weight bytes vs rows / out vs out_dim",
left: row * 3,
right: row * 4,
},
);
assert_eq!(
gemv_q8_k(
QuantFormat::Q4_K,
&vec![0; row * 4],
2048,
4,
&acts,
&mut out[..2]
)
.unwrap_err(),
KernelError::LengthMismatch {
what: "gemv_q8_k: weight bytes vs rows / out vs out_dim",
left: 2,
right: 4,
},
);
assert_eq!(
gemv_q8_k(
QuantFormat::Q4_K,
&vec![0; row * 4],
2048,
4,
&acts[..4],
&mut out
)
.unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 8,
activation_blocks: 4,
},
);
let acts0 = vec![BlockQ8_0::default(); 64];
assert_eq!(
gemv_q8_0(&[0; 2176], 2048, 2, &acts0, &mut out[..2]).unwrap_err(),
KernelError::LengthMismatch {
what: "gemv_q8_0: weight bytes vs rows / out vs out_dim",
left: 2176,
right: 4352,
},
);
}
fn raw_range(start: usize, end: usize) -> Range<usize> {
Range { start, end }
}
#[test]
fn rejects_bad_row_ranges() {
let acts = vec![BlockQ8K::default(); 8]; let row = QuantFormat::Q4_K.row_bytes(2048).unwrap();
let w = vec![0u8; row * 4]; let mut out = vec![0f32; 4];
let rows_err = |rows: Range<usize>, out: &mut [f32]| {
gemv_q8_k_rows(QuantFormat::Q4_K, &w, 2048, 4, &acts, rows, out).unwrap_err()
};
assert_eq!(
rows_err(raw_range(3, 1), &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: 3,
right: 1,
},
);
assert_eq!(
rows_err(2..5, &mut out[..3]),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: 5,
right: 4,
},
);
assert_eq!(
rows_err(9..9, &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: 9,
right: 4,
},
);
assert_eq!(
rows_err(usize::MAX - 1..usize::MAX, &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: usize::MAX,
right: 4,
},
);
assert_eq!(
rows_err(raw_range(usize::MAX, 0), &mut []),
KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: usize::MAX,
right: 0,
},
);
assert_eq!(
rows_err(1..3, &mut out),
KernelError::LengthMismatch {
what: "gemv_q8_k_rows: weight bytes vs rows / out vs range",
left: 4,
right: 2,
},
);
assert_eq!(
gemv_q8_k_rows(
QuantFormat::Q4_K,
&w[..row * 3],
2048,
4,
&acts,
0..1,
&mut out[..1]
)
.unwrap_err(),
KernelError::LengthMismatch {
what: "gemv_q8_k_rows: weight bytes vs rows / out vs range",
left: row * 3,
right: row * 4,
},
);
let acts0 = vec![BlockQ8_0::default(); 64];
let row0 = QuantFormat::Q8_0.row_bytes(2048).unwrap();
let w0 = vec![0u8; row0 * 2];
assert_eq!(
gemv_q8_0_rows(&w0, 2048, 2, &acts0, raw_range(2, 1), &mut []).unwrap_err(),
KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: 2,
right: 1,
},
);
assert_eq!(
gemv_q8_0_rows(&w0, 2048, 2, &acts0, 0..3, &mut out[..3]).unwrap_err(),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: 3,
right: 2,
},
);
assert_eq!(
gemv_q8_0_rows(&w0, 2048, 2, &acts0, 0..2, &mut out).unwrap_err(),
KernelError::LengthMismatch {
what: "gemv_q8_0_rows: weight bytes vs rows / out vs range",
left: 4,
right: 2,
},
);
}
#[test]
fn zero_out_dim_is_a_no_op() {
let acts = q8_k_acts(256, 1);
let mut out: [f32; 0] = [];
gemv_q8_k(QuantFormat::Q4_K, &[], 256, 0, &acts, &mut out).unwrap();
gemv_q8_k_rows(QuantFormat::Q4_K, &[], 256, 0, &acts, 0..0, &mut out).unwrap();
}
fn part<'a>(f: &'a Fixture, acts: &'a [BlockQ8K]) -> FusedPart<'a> {
FusedPart {
format: f.format,
weight: &f.weight,
in_dim: f.in_dim,
out_dim: f.out_dim,
acts,
}
}
fn fused_fixtures() -> (Vec<Fixture>, Vec<Vec<BlockQ8K>>) {
let fx = vec![
Fixture::new("e0 gate (Q4_K)", QuantFormat::Q4_K, 2048, 96),
Fixture::new("e0 up (Q4_K)", QuantFormat::Q4_K, 2048, 96),
Fixture::new("e1 gate (Q4_K)", QuantFormat::Q4_K, 2048, 96),
Fixture::new("e1 up (Q4_K)", QuantFormat::Q4_K, 2048, 96),
Fixture::new("attn_v (Q6_K)", QuantFormat::Q6_K, 2048, 64),
Fixture::new("e0 down (Q4_K)", QuantFormat::Q4_K, 768, 128),
Fixture::new("e1 down (Q6_K)", QuantFormat::Q6_K, 768, 128),
];
let shared = q8_k_acts(2048, 0xF05E_D000);
let acts = vec![
shared.clone(),
shared.clone(),
shared.clone(),
shared.clone(),
shared,
q8_k_acts(768, 0xF05E_D001),
q8_k_acts(768, 0xF05E_D002),
];
(fx, acts)
}
const FUSED_TOTAL: usize = 4 * 96 + 64 + 2 * 128;
#[test]
fn a_fused_row_space_is_bit_identical_to_the_single_matrix_sequence() {
let paths = kernel_paths(
"a fused multi-matrix fan-out is bit-identical to the sequence of \
single-matrix fan-outs on both the AVX2 and the scalar kernel path",
);
let (fx, acts) = fused_fixtures();
let parts: Vec<FusedPart<'_>> =
fx.iter().zip(&acts).map(|(f, a)| part(f, &a[..])).collect();
let total = fused_out_dim(&parts);
assert_eq!(total, FUSED_TOTAL);
for &scalar in paths.flags() {
let path = if scalar { "scalar" } else { "avx2" };
let mut want = vec![f32::NAN; total];
let mut rest: &mut [f32] = &mut want;
for (f, a) in fx.iter().zip(&acts) {
let (head, tail) = rest.split_at_mut(f.out_dim);
run_k(f, &a[..], 1, 0..f.out_dim, head, scalar);
rest = tail;
}
assert!(
want.iter().all(|v| v.is_finite()),
"[{path}]: the fused fixture produced a non-finite reference",
);
for (scheme, ranges) in partitions(total) {
let mut got = vec![f32::NAN; total];
let mut rest: &mut [f32] = &mut got;
let mut cursor = 0usize;
for r in &ranges {
assert_eq!(r.start, cursor, "[{path}] {scheme}: bad tiling");
let (head, tail) = rest.split_at_mut(r.len());
gemv_q8_k_fused_rows_impl(&parts, r.clone(), head, scalar).unwrap();
rest = tail;
cursor = r.end;
}
assert_eq!(cursor, total, "[{path}] {scheme}: short tiling");
for (i, (&g, &w)) in got.iter().zip(&want).enumerate() {
assert_eq!(
g.to_bits(),
w.to_bits(),
"[{path}] {scheme}: fused row {i} differs ({g} vs {w})",
);
}
}
}
}
#[test]
fn the_fused_row_map_covers_every_part_row_exactly_once() {
let (fx, acts) = fused_fixtures();
let parts: Vec<FusedPart<'_>> =
fx.iter().zip(&acts).map(|(f, a)| part(f, &a[..])).collect();
let total = fused_out_dim(&parts);
for (scheme, ranges) in partitions(total) {
let mut seen: Vec<Vec<u32>> = parts.iter().map(|p| vec![0u32; p.out_dim]).collect();
let mut rows_yielded = 0usize;
for r in &ranges {
let mut taken = 0usize;
for (p, local) in fused_row_parts(&parts, r.clone()) {
assert!(
local.end <= parts[p].out_dim,
"{scheme}: part {p} local range {local:?} runs past its \
own out_dim {}",
parts[p].out_dim,
);
assert!(!local.is_empty(), "{scheme}: an empty part was yielded");
for row in local.clone() {
seen[p][row] += 1;
}
taken += local.len();
}
assert_eq!(
taken,
r.len(),
"{scheme}: range {r:?} mapped to {taken} part rows",
);
rows_yielded += taken;
}
assert_eq!(rows_yielded, total, "{scheme}: the tiling lost rows");
for (p, counts) in seen.iter().enumerate() {
for (row, &n) in counts.iter().enumerate() {
assert_eq!(n, 1, "{scheme}: part {p} row {row} computed {n} times");
}
}
}
let padded = vec![
FusedPart::empty(),
part(&fx[0], &acts[0]),
FusedPart::empty(),
part(&fx[4], &acts[4]),
FusedPart::empty(),
];
assert_eq!(fused_out_dim(&padded), fx[0].out_dim + fx[4].out_dim);
let walked: Vec<(usize, Range<usize>)> =
fused_row_parts(&padded, 0..fused_out_dim(&padded)).collect();
assert_eq!(walked, vec![(1, 0..fx[0].out_dim), (3, 0..fx[4].out_dim)]);
let tail: Vec<(usize, Range<usize>)> =
fused_row_parts(&padded, fx[0].out_dim + 5..fx[0].out_dim + 9).collect();
assert_eq!(tail, vec![(3, 5..9)]);
}
#[test]
fn fused_rejects_bad_geometry() {
let (fx, acts) = fused_fixtures();
let parts: Vec<FusedPart<'_>> =
fx.iter().zip(&acts).map(|(f, a)| part(f, &a[..])).collect();
let total = fused_out_dim(&parts);
let mut out = vec![0f32; total];
assert_eq!(
gemv_q8_k_fused_rows(&parts, raw_range(9, 2), &mut []).unwrap_err(),
KernelError::LengthMismatch {
what: ROW_RANGE_ORDER,
left: 9,
right: 2,
},
);
assert_eq!(
gemv_q8_k_fused_rows(&parts, 0..total + 1, &mut out).unwrap_err(),
KernelError::LengthMismatch {
what: ROW_RANGE_BOUND,
left: total + 1,
right: total,
},
);
assert_eq!(
gemv_q8_k_fused_rows(&parts, 0..total, &mut out[..total - 1]).unwrap_err(),
KernelError::LengthMismatch {
what: FUSED_OUT,
left: total - 1,
right: total,
},
);
let mut bad = parts.clone();
bad[1].acts = &acts[5];
assert_eq!(
gemv_q8_k_fused_rows(&bad, 0..total, &mut out).unwrap_err(),
KernelError::BlockCountMismatch {
weight_blocks: 2048 / 256,
activation_blocks: 768 / 256,
},
);
let mut wrong = parts.clone();
wrong[0].format = QuantFormat::Q8_0;
assert_eq!(
gemv_q8_k_fused_rows(&wrong, 0..total, &mut out).unwrap_err(),
KernelError::UnsupportedFormat {
what: "gemv_q8_k_fused_rows",
format: QuantFormat::Q8_0,
},
);
assert_eq!(fused_out_dim(&[]), 0);
gemv_q8_k_fused_rows(&[], 0..0, &mut []).unwrap();
let huge = [
FusedPart {
out_dim: usize::MAX,
..parts[0]
},
parts[1],
];
assert_eq!(fused_out_dim(&huge), usize::MAX);
let walked: Vec<(usize, Range<usize>)> = fused_row_parts(&huge, 0..4).collect();
assert_eq!(walked, vec![(0, 0..4)], "part 1 cannot alias part 0's rows");
}
#[test]
fn empty_row_range_writes_nothing() {
let f = Fixture::new("expert gate (Q4_K)", QuantFormat::Q4_K, 2048, 768);
let acts = q8_k_acts(f.in_dim, 0xE377);
let mut out = vec![7.5f32; f.out_dim];
{
let (_, mid) = out.split_at_mut(300);
let (empty, _) = mid.split_at_mut(0);
gemv_q8_k_rows(
f.format,
&f.weight,
f.in_dim,
f.out_dim,
&acts,
300..300,
empty,
)
.unwrap();
}
assert!(out.iter().all(|&v| v == 7.5));
}
}