Skip to main content

trueno/backends/q6k/
colmajor.rs

1//! Column-major Q6_K matrix-vector multiplication.
2//!
3//! This module implements column-major GEMV for GGML/GGUF format weights,
4//! where weights are stored column-first for cache-efficient streaming.
5
6use super::{f16_to_f32, SUPER_BLOCK_BYTES, SUPER_BLOCK_SIZE};
7
8/// Extract the 6-bit value of element `idx` of one Q6_K super-block in GGML's layout; the same
9/// mapping as the row-major kernel (`gemv.rs::extract_q6k_scalar`), which documents it.
10#[inline(always)]
11fn extract_q6k_value(ql: &[u8], qh: &[u8], idx: usize) -> i8 {
12    let (n, r) = (idx / 128, idx % 128);
13    let (k, l) = (r / 32, r % 32);
14    let ql_byte = ql[64 * n + l + 32 * (k & 1)];
15    let low4 = if k < 2 { ql_byte & 0x0F } else { ql_byte >> 4 };
16    let high2 = (qh[32 * n + l] >> (2 * k)) & 0x03;
17    (low4 | (high2 << 4)) as i8 - 32
18}
19
20/// Accumulate one Q6_K superblock into output (column-major layout).
21#[inline]
22fn accumulate_q6k_superblock_colmajor(
23    sb_data: &[u8],
24    x_j: f32,
25    output: &mut [f32],
26    output_offset: usize,
27    ne0: usize,
28) {
29    let ql = sb_data.get(0..128).expect("Q6_K: need ≥128 bytes for ql");
30    let qh = sb_data.get(128..192).expect("Q6_K: need ≥192 bytes for qh");
31    let scales = sb_data.get(192..208).expect("Q6_K: need ≥208 bytes for scales");
32    let d = f16_to_f32(u16::from_le_bytes([sb_data[208], sb_data[209]]));
33
34    for group in 0..16 {
35        let scale = (scales[group] as i8) as f32;
36        let group_offset = group * 16;
37
38        for j in 0..16 {
39            let idx = group_offset + j;
40            let output_idx = output_offset + idx;
41            if output_idx >= ne0 {
42                continue;
43            }
44            let q6 = extract_q6k_value(ql, qh, idx);
45            output[output_idx] += x_j * d * scale * q6 as f32;
46        }
47    }
48}
49
50/// Fused Q6_K matrix-vector multiply for GGML column-major layout
51///
52/// Computes: output = input @ Q6K_weight (GGML convention: y = x @ W)
53/// where weight is stored in Q6_K format with GGML column-major super-block organization.
54///
55/// # Arguments
56/// * `q6k_data` - Raw Q6K bytes in GGML column-major layout [ne0, ne1]
57/// * `input` - F32 input vector [ne1] (input/reduction dimension)
58/// * `ne0` - Size of output dimension (rows in GGML, output size)
59/// * `ne1` - Size of input/reduction dimension (columns in GGML, input size)
60///
61/// # Returns
62/// F32 output vector [ne0]
63#[deprecated(
64    since = "0.15.0",
65    note = "LAYOUT-001: Use row-major kernels. APR/GGUF data is transposed at import boundary."
66)]
67pub fn matmul_q6k_f32_colmajor(
68    q6k_data: &[u8],
69    input: &[f32],
70    ne0: usize, // output dimension (rows)
71    ne1: usize, // input/reduction dimension (columns)
72) -> Vec<f32> {
73    assert_eq!(input.len(), ne1, "Input length must match ne1 (input dimension)");
74
75    let blocks_per_col = (ne0 + SUPER_BLOCK_SIZE - 1) / SUPER_BLOCK_SIZE;
76    let col_bytes = blocks_per_col * SUPER_BLOCK_BYTES;
77
78    let mut output = vec![0.0f32; ne0];
79
80    for col_idx in 0..ne1 {
81        let col_start = col_idx * col_bytes;
82        let x_j = input[col_idx];
83
84        if x_j == 0.0 {
85            continue;
86        }
87
88        for sb_idx in 0..blocks_per_col {
89            let sb_start = col_start + sb_idx * SUPER_BLOCK_BYTES;
90            if sb_start + SUPER_BLOCK_BYTES > q6k_data.len() {
91                break;
92            }
93            let sb_data = &q6k_data[sb_start..sb_start + SUPER_BLOCK_BYTES];
94            let output_offset = sb_idx * SUPER_BLOCK_SIZE;
95            accumulate_q6k_superblock_colmajor(sb_data, x_j, &mut output, output_offset, ne0);
96        }
97    }
98
99    output
100}
101
102/// Runtime dispatch for column-major Q6K matmul
103///
104/// Uses scalar implementation for correctness.
105/// Critical for lm_head which is typically 151936 x 1536 (233M elements).
106#[deprecated(
107    since = "0.15.0",
108    note = "LAYOUT-001: Use row-major kernels. APR/GGUF data is transposed at import boundary."
109)]
110#[inline]
111pub fn matmul_q6k_f32_colmajor_dispatch(
112    q6k_data: &[u8],
113    input: &[f32],
114    ne0: usize,
115    ne1: usize,
116) -> Vec<f32> {
117    #[allow(deprecated)]
118    matmul_q6k_f32_colmajor(q6k_data, input, ne0, ne1)
119}