ferrox_quant/encode/

q4_k.rs

//! The Q4_K weight encoder: a transcription of llama.cpp b7650's
//! `quantize_row_q4_K_ref` (`ggml/src/ggml-quants.c:1280`), not a
//! reimplementation of it.
//!
//! A K-quant is NOT min/max over a block. Q4_K's 256-element
//! super-block is fitted in three stages, all of which live in
//! [`super::fit`] because Q5_K's fit is the same three stages with four
//! numbers changed:
//!
//! 1. Each of the 8 sub-blocks of 32 gets an **iterative** affine fit
//!    (`make_qkx2_quants`): 21 candidate inverse scales are tried, each
//!    one re-solves a weighted least-squares for (scale, min) from the
//!    integer codes it produced, and the lowest weighted squared error
//!    wins. The weights are `sqrt(mean(x^2)) + |x|`, so a sub-block's
//!    large values pull the fit toward themselves.
//! 2. The 8 scales and 8 mins are themselves quantized to 6 bits
//!    against the super-block's `d`/`dmin` and packed into 12 bytes.
//! 3. The 4-bit codes are then recomputed **against the 6-bit-rounded**
//!    scale and min, not against the fit from stage 1 -- so stage 3
//!    sees a slightly different affine map than stage 1 did.
//!
//! A naive min/max encoder skips all three and produces a file that
//! loads and generates measurably worse text. That is the failure this
//! module exists to not ship, so the arithmetic in [`super::fit`] is
//! deliberately the same shape as the C, down to the operation order in
//! the least-squares accumulation.
//!
//! With an importance matrix the same three stages run with the
//! weights, grid and stage 2 of `quantize_row_q4_K_impl`
//! (`ggml-quants.c:1376`); that switch lives in [`super::fit`] too, and
//! this module only passes the slice through.
//!
//! What is left HERE is only what is Q4_K's own: the candidate grid it
//! passes to the shared fit, and the nibble packing.

use super::fit::{fit_qk_super_block, make_qkx2_quants, QkFit, QK_SUB_ELEMS};
use crate::{Q4_K_BLOCK_BYTES, Q4_K_BLOCK_ELEMS};

/// Q4_K's half of the shared super-block fit: 4-bit codes, the
/// `(-1.0, 0.1, 20)` candidate grid from `ggml-quants.c:1301`, and NO
/// clamp on the imatrix scale codes (`:1411-1424` packs `Ls[j]` as is).
const Q4_K_FIT: QkFit = QkFit {
    nmax: 15,
    rmin: -1.0,
    rdelta: 0.1,
    nstep: 20,
    imatrix_clamps_scale_codes: false,
};

/// Runs one 32-element sub-block through exactly the path
/// [`encode_block_q4_k`] uses and returns its `(scale, min)`.
///
/// Tooling, not a code path: it exists so a single sub-block can be
/// compared against llama.cpp's own `make_qkx2_quants` on the same
/// input. Chasing a floating-point difference that affects 0.55% of
/// super-blocks by quantizing whole checkpoints is far too coarse a
/// loop, and `examples/q4k_probe.rs` is the other half of it.
#[doc(hidden)]
pub fn probe_sub_block(xs: &[f32]) -> (f32, f32) {
    assert_eq!(xs.len(), QK_SUB_ELEMS);
    let mut l = [0u8; QK_SUB_ELEMS];
    let mut laux = [0u8; QK_SUB_ELEMS];
    let mut weights = [0f32; QK_SUB_ELEMS];
    super::fit::qk_sub_block_weights(xs, &mut weights);
    make_qkx2_quants(
        xs,
        &weights,
        &mut l,
        &mut laux,
        Q4_K_FIT.nmax,
        Q4_K_FIT.rmin,
        Q4_K_FIT.rdelta,
        Q4_K_FIT.nstep,
        false,
    )
}

/// Encodes one Q4_K super-block (exactly [`Q4_K_BLOCK_ELEMS`] values)
/// and appends its [`Q4_K_BLOCK_BYTES`] bytes to `out`.
///
/// `qw` is this super-block's slice of the importance matrix
/// (llama.cpp's `quant_weights + QK_K*i`), or `None` for the plain fit.
pub fn encode_block_q4_k(
    block: &[f32; Q4_K_BLOCK_ELEMS],
    qw: Option<&[f32; Q4_K_BLOCK_ELEMS]>,
    out: &mut Vec<u8>,
) {
    let fitted = fit_qk_super_block(block, Q4_K_FIT, qw);

    out.reserve(Q4_K_BLOCK_BYTES);
    out.extend_from_slice(&fitted.d.to_le_bytes());
    out.extend_from_slice(&fitted.dmin.to_le_bytes());
    out.extend_from_slice(&fitted.packed);
    // Two 32-element halves of every 64 elements share a byte: the low
    // nibble is the first half, the high nibble the second. The reader
    // in `dequant_q4_k` walks the same pairing.
    for j in (0..Q4_K_BLOCK_ELEMS).step_by(64) {
        for i in 0..32 {
            out.push(fitted.l[j + i] | (fitted.l[j + i + 32] << 4));
        }
    }
}

/// Encodes a whole row (or any slice whose length is a multiple of
/// [`Q4_K_BLOCK_ELEMS`]) into Q4_K super-blocks, appending to `out`.
///
/// `qw` is the row's importance-matrix weights, one per column and the
/// same for every row of the tensor, or `None` for the plain fit. Its
/// length must equal `src.len()`; a mismatch is refused like a ragged
/// row, because a shifted slice would weight every element by its
/// neighbour's importance and still produce a plausible file.
///
/// Returns `None` when `src.len()` is not a multiple of the super-block
/// size. llama.cpp handles that case by silently *changing type* --
/// `convert_incompatible_tensor` rewrites a Q4_K tensor with an awkward
/// row length to Q5_0, and to F16 if that does not fit either -- and
/// ferrox has neither encoder, so this refuses instead of padding.
/// Padding would write more elements than the tensor's shape declares
/// and every following row would decode shifted.
pub fn encode_row_q4_k(src: &[f32], qw: Option<&[f32]>, out: &mut Vec<u8>) -> Option<()> {
    let (blocks, rest) = src.as_chunks::<Q4_K_BLOCK_ELEMS>();
    if !rest.is_empty() {
        return None;
    }
    let qw_blocks = super::imatrix_blocks::<Q4_K_BLOCK_ELEMS>(qw, blocks.len())?;
    out.reserve(blocks.len() * Q4_K_BLOCK_BYTES);
    for (i, block) in blocks.iter().enumerate() {
        encode_block_q4_k(block, qw_blocks.map(|q| &q[i]), out);
    }
    Some(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::encode::fit::QK_SUBS;
    use crate::encode::testdata::k_quant_fixture;
    use crate::{dequant_q4_k, q4_k_scale_min, Q4_K_SCALE_BYTES};
    use half::f16;

    /// llama.cpp's own bytes for [`k_quant_fixture`].
    ///
    /// Produced by the C harness described in the PR body: it links
    /// llama.cpp b7650's `libggml-base` and calls the exported
    /// `quantize_row_q4_K_ref` on the f32s
    /// `encode::testdata::dump_the_fixture_the_c_harness_reads` writes.
    /// The same harness also calls `ggml_quantize_chunk(GGML_TYPE_Q4_K,
    /// ...)` --
    /// the entry point `llama-quantize` itself goes through -- and
    /// asserts the two agree, so this is what the real tool writes and
    /// not merely what a reference function does.
    const LLAMA_CPP_Q4_K_GOLDEN: [u8; 12 * Q4_K_BLOCK_BYTES] = [
        0x32, 0x10, 0x14, 0x1c, 0x05, 0x0c, 0x59, 0xff, 0x04, 0x0b, 0x58, 0xff, 0x55, 0xcc, 0x8a,
        0xb3, 0xed, 0xc8, 0xba, 0x4e, 0xeb, 0x91, 0x85, 0xa6, 0x9c, 0x87, 0xd8, 0xab, 0x42, 0xe9,
        0x87, 0x0b, 0xb3, 0x82, 0x59, 0xb2, 0xc0, 0x80, 0x87, 0xa7, 0x98, 0x62, 0x75, 0x94, 0x31,
        0x0a, 0x89, 0xda, 0xc5, 0x32, 0xd4, 0xfa, 0xf6, 0xd6, 0xc1, 0xbd, 0xf1, 0xc8, 0x6c, 0xbf,
        0xc4, 0xa0, 0xeb, 0x46, 0x7d, 0xb0, 0xf4, 0xb7, 0x95, 0xbc, 0xd1, 0xe6, 0x84, 0x8d, 0x77,
        0x1d, 0x01, 0xd7, 0x1f, 0xda, 0x1b, 0xf6, 0x4f, 0x62, 0x3f, 0xce, 0x28, 0x47, 0x5b, 0xba,
        0xeb, 0xfc, 0x04, 0xb3, 0xba, 0x44, 0x94, 0xe0, 0xd6, 0xbf, 0x7e, 0x02, 0xf0, 0xac, 0x4c,
        0xda, 0xbf, 0x21, 0x4d, 0xc7, 0xd1, 0xb0, 0x6b, 0xf0, 0xb2, 0x0a, 0x8d, 0x25, 0xbc, 0x2c,
        0xda, 0xd7, 0xa9, 0x51, 0x32, 0xa2, 0xc0, 0x5e, 0x1c, 0x86, 0x95, 0x53, 0x40, 0x7d, 0xf1,
        0xf5, 0x34, 0xf8, 0x9c, 0xf0, 0x9f, 0xa8, 0x4c, 0x48, 0x2d, 0x10, 0xeb, 0x1b, 0x00, 0x10,
        0x48, 0xc6, 0x00, 0x00, 0x40, 0xcf, 0x45, 0xcc, 0xaa, 0xff, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0,
        0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0,
        0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xe6, 0xba, 0x13,
        0x8b, 0x45, 0x3b, 0x24, 0x4e, 0x69, 0xbb, 0x96, 0xd7, 0xc9, 0xe9, 0x13, 0xad, 0x75, 0xeb,
        0xeb, 0x8a, 0x0e, 0x9e, 0x76, 0xfb, 0x0d, 0x17, 0xfe, 0x5a, 0x07, 0x7e, 0x6d, 0x95, 0xa5,
        0x30, 0x33, 0xe7, 0xf1, 0x3d, 0x29, 0xfa, 0x07, 0x84, 0x99, 0xea, 0xca, 0x06, 0xe3, 0x27,
        0xa5, 0x40, 0x07, 0x12, 0xf3, 0x55, 0x72, 0xe5, 0xa1, 0x12, 0x35, 0xee, 0x06, 0xf2, 0x51,
        0x0d, 0x11, 0x70, 0x92, 0xc1, 0xd3, 0x7c, 0x99, 0x33, 0x74, 0x49, 0x6e, 0x6d, 0x2a, 0xdc,
        0xa2, 0x57, 0x35, 0xbe, 0xd3, 0x65, 0xbb, 0xf1, 0x14, 0x05, 0x09, 0xd4, 0x87, 0x3e, 0xbf,
        0x4c, 0xc8, 0xd1, 0x30, 0x10, 0xdc, 0x1b, 0x05, 0x00, 0x58, 0xff, 0x05, 0x00, 0x58, 0xff,
        0x55, 0xdd, 0x89, 0xce, 0x44, 0xa8, 0xc2, 0x9c, 0xec, 0x84, 0x2c, 0x8f, 0x6f, 0x30, 0x74,
        0xae, 0x66, 0x2b, 0x16, 0x5b, 0xe6, 0xe0, 0x96, 0x69, 0x66, 0x8f, 0xe4, 0x5c, 0x57, 0x24,
        0x06, 0x52, 0x67, 0xa1, 0xa0, 0x43, 0xaa, 0x5d, 0x43, 0x4d, 0x7c, 0xbf, 0x78, 0x16, 0x59,
        0xf9, 0x30, 0x58, 0x03, 0x12, 0x73, 0xed, 0x8d, 0x00, 0xdd, 0x49, 0xe2, 0xf9, 0xa1, 0x88,
        0x2c, 0x80, 0x90, 0x0f, 0xb3, 0x2b, 0xf5, 0xc9, 0x72, 0x61, 0x6a, 0x85, 0x99, 0xc3, 0x02,
        0xd4, 0xd8, 0x2a, 0xee, 0x20, 0xa9, 0xcd, 0x9a, 0xa8, 0xed, 0x6b, 0x95, 0x98, 0x8c, 0x96,
        0x6f, 0x1f, 0xda, 0x13, 0xf9, 0xc7, 0x75, 0xdd, 0x55, 0x17, 0x71, 0xd4, 0xbd, 0xc5, 0x79,
        0xa0, 0x2d, 0xcb, 0x7b, 0x40, 0x76, 0x1b, 0xf4, 0x04, 0x56, 0xd2, 0x1b, 0x24, 0x44, 0x25,
        0x68, 0x01, 0x33, 0xa1, 0x92, 0xf5, 0x1f, 0x69, 0xed, 0xd1, 0xa8, 0x28, 0x3c, 0x10, 0x2d,
        0x1c, 0x05, 0x0c, 0x58, 0xff, 0x05, 0x0c, 0x53, 0xff, 0x55, 0xcc, 0x88, 0x9f, 0xba, 0xf2,
        0xc5, 0x51, 0xfe, 0x43, 0xec, 0x47, 0x96, 0x64, 0x14, 0x78, 0xf3, 0x6b, 0x46, 0x52, 0x79,
        0x15, 0x26, 0x05, 0x50, 0x9f, 0xdd, 0xec, 0x0b, 0x0d, 0x5a, 0x8f, 0xe1, 0x15, 0x76, 0x87,
        0x1c, 0x6a, 0xf7, 0xe1, 0xe2, 0x46, 0xc4, 0xcc, 0x90, 0x95, 0x40, 0x67, 0xdb, 0x70, 0x53,
        0xd4, 0x70, 0xb4, 0xcd, 0x80, 0x52, 0xb4, 0x0b, 0xc1, 0xd5, 0xda, 0x17, 0x15, 0x1e, 0x99,
        0x57, 0x22, 0x9c, 0x58, 0xc3, 0xc4, 0x5e, 0xd2, 0x78, 0x37, 0x69, 0xe2, 0x21, 0xf7, 0x83,
        0x5c, 0xa1, 0x6a, 0xbd, 0xbe, 0x72, 0xa4, 0x3d, 0x61, 0x76, 0xcb, 0x55, 0x2a, 0x01, 0x8d,
        0x14, 0xcb, 0xdc, 0x4f, 0x6f, 0x15, 0x46, 0x6e, 0xe8, 0x5d, 0x6d, 0xf0, 0xea, 0x0d, 0xaa,
        0x8f, 0xdd, 0xd7, 0x3e, 0x52, 0x20, 0x24, 0x1b, 0x15, 0x62, 0x98, 0x0a, 0xf4, 0x42, 0x9b,
        0xdc, 0x8a, 0xb7, 0xab, 0xae, 0x57, 0xb2, 0x04, 0x0f, 0x11, 0xe9, 0xe8, 0xf8, 0xf4, 0xac,
        0xa9, 0xb1, 0xd7, 0xc6, 0xc5, 0x5c, 0xff, 0x7c, 0xa8, 0xf7, 0x8b, 0x09, 0x68, 0x8b, 0xa0,
        0x69, 0x6f, 0x9d, 0xb0, 0xbb, 0xe7, 0xc9, 0x87, 0x6e, 0xfe, 0x58, 0xca, 0x56, 0xad, 0xb9,
        0xb4, 0xd4, 0x7b, 0x96, 0x0b, 0x19, 0x71, 0xea, 0xd2, 0x87, 0x37, 0x18, 0x9e, 0x47, 0x09,
        0x0b, 0x37, 0x05, 0x22, 0x43, 0x2a, 0x6a, 0x7a, 0x07, 0x3d, 0x53, 0x0f, 0x28, 0x37, 0x57,
        0x3b, 0x7b, 0x68, 0x7c, 0x55, 0x99, 0x0a, 0x6c, 0x26, 0x20, 0xf7, 0x81, 0x7c, 0x53, 0x43,
        0xf5, 0xd4, 0x77, 0x37, 0x66, 0x89, 0xc7, 0x17, 0xbf, 0xad, 0x73, 0x48, 0xc2, 0x86, 0x39,
        0x7e, 0x86, 0x63, 0x5d, 0x5c, 0xb0, 0x73, 0x04, 0x77, 0x98, 0x72, 0xd3, 0xcd, 0x97, 0x44,
        0x35, 0xb6, 0xb5, 0x06, 0x75, 0xcc, 0xb2, 0x81, 0xa5, 0xe7, 0x67, 0x76, 0xf1, 0xbc, 0x4a,
        0xa6, 0x77, 0x9b, 0x78, 0x95, 0xb5, 0x96, 0x53, 0x95, 0x9e, 0xd2, 0x61, 0x86, 0xa9, 0x90,
        0xfe, 0x09, 0x83, 0x16, 0xb6, 0xb2, 0xa9, 0xff, 0xad, 0xa8, 0xaa, 0xbf, 0x3b, 0x7a, 0xcf,
        0x04, 0xc8, 0xba, 0x14, 0x42, 0x87, 0x7a, 0xb5, 0x8a, 0x65, 0x84, 0x84, 0x04, 0x56, 0x96,
        0xab, 0x24, 0xb2, 0x61, 0xc8, 0x85, 0x53, 0x84, 0x65, 0x80, 0x6e, 0x57, 0x73, 0x7a, 0x35,
        0x5a, 0x0b, 0xf7, 0xbc, 0x88, 0xa8, 0xca, 0x82, 0x85, 0x30, 0xba, 0xaa, 0x98, 0x97, 0x88,
        0x78, 0xfc, 0x7a, 0x96, 0xd8, 0x07, 0x9d, 0x98, 0x6c, 0x88, 0x78, 0x7a, 0xa2, 0x4b, 0x8b,
        0xa8, 0xcf, 0xdd, 0x85, 0xb4, 0xc7, 0x93, 0xa8, 0x6e, 0x47, 0x26, 0x74, 0x96, 0x53, 0xba,
        0x53, 0xba, 0x67, 0x79, 0xc5, 0x06, 0xf6, 0x9a, 0xe0, 0x4e, 0x74, 0x03, 0x74, 0x83, 0x82,
        0xa8, 0x9a, 0x8a, 0xb6, 0xad, 0xbb, 0x74, 0x7f, 0x37, 0xc0, 0x49, 0x9b, 0x16, 0x68, 0x84,
        0x89, 0x39, 0x98, 0x49, 0x7c, 0x25, 0x5a, 0x23, 0x99, 0x37, 0x45, 0x05, 0x03, 0x27, 0x35,
        0x49, 0x32, 0xed, 0x65, 0x1b, 0x34, 0xcd, 0xa6, 0x89, 0x58, 0x09, 0x9c, 0x14, 0xb0, 0xe2,
        0xbf, 0xee, 0xe8, 0xe5, 0xbf, 0xfa, 0xaf, 0x12, 0xff, 0xf2, 0x76, 0x65, 0x95, 0x54, 0x70,
        0x99, 0xa5, 0x14, 0x00, 0xfc, 0x93, 0xa3, 0x24, 0xc7, 0x78, 0x27, 0x58, 0x73, 0x54, 0x53,
        0x5c, 0xf6, 0x3a, 0xa4, 0x56, 0xaa, 0x64, 0x36, 0x83, 0x6d, 0x72, 0x7f, 0xe6, 0x57, 0x57,
        0x67, 0xc2, 0x98, 0x54, 0x06, 0x86, 0x58, 0xbf, 0x7a, 0x99, 0xf4, 0xb7, 0xa8, 0xf6, 0x7b,
        0xc9, 0xb8, 0x58, 0x97, 0x96, 0xc5, 0x30, 0xb8, 0xa6, 0x77, 0x89, 0x99, 0xd4, 0xa7, 0x46,
        0x79, 0x8a, 0x89, 0x9c, 0x78, 0x09, 0x83, 0x99, 0x2c, 0x74, 0x75, 0x78, 0xb9, 0x89, 0x40,
        0x88, 0xfa, 0x84, 0x6e, 0x7c, 0x93, 0xab, 0x26, 0x87, 0x4e, 0xa8, 0x5c, 0x6a, 0x97, 0xaa,
        0x57, 0x92, 0x84, 0x96, 0xd0, 0x93, 0x6c, 0x07, 0x86, 0x87, 0xa5, 0x8b, 0xc9, 0xd9, 0xa9,
        0xa4, 0x98, 0x81, 0xc6, 0x94, 0xeb, 0xd4, 0x9a, 0x74, 0xaa, 0x4a, 0x85, 0x5f, 0x33, 0x7c,
        0x86, 0x5a, 0xd6, 0xb7, 0x08, 0x9e, 0x15, 0xf5, 0xfa, 0xf6, 0xff, 0xa1, 0x9c, 0xa1, 0xbf,
        0xc8, 0x69, 0x65, 0xcf, 0x13, 0x08, 0x63, 0x85, 0x10, 0x75, 0x37, 0x21, 0x5b, 0xb1, 0x27,
        0x07, 0xe7, 0x18, 0x17, 0x12, 0x13, 0x8b, 0x51, 0x64, 0x2f, 0x35, 0x67, 0x73, 0x37, 0x62,
        0x25, 0x96, 0xbe, 0x4b, 0x93, 0x44, 0xb5, 0x81, 0x98, 0x95, 0x96, 0xa5, 0x43, 0xba, 0x6a,
        0x06, 0x9d, 0xb9, 0x9a, 0xc8, 0xc4, 0xf3, 0x82, 0xa4, 0xdc, 0xc0, 0xb6, 0xd7, 0xb4, 0xa5,
        0x90, 0xaf, 0xbb, 0x7a, 0xad, 0x75, 0xc2, 0x97, 0x69, 0x23, 0x72, 0xaa, 0x73, 0x25, 0x9e,
        0x4c, 0xa8, 0xe8, 0x74, 0x65, 0x6d, 0xf2, 0x84, 0xa9, 0xa0, 0x4c, 0xc8, 0x87, 0xb5, 0x4a,
        0xa4, 0x5c, 0x8d, 0x85, 0xc3, 0xd2, 0x33, 0xa3, 0x87, 0x0b, 0x0b, 0x75, 0x4f, 0x97, 0x77,
        0x84, 0xa8, 0xb9, 0xc8, 0x8a, 0x97, 0x66, 0x65, 0x9b, 0x68, 0xf4, 0x6c, 0x86, 0x49, 0x80,
        0x66, 0x35, 0x75, 0x75, 0xb2, 0x74, 0x26, 0x88, 0x5b, 0xa3, 0x65, 0xa9, 0x80, 0x09, 0x7e,
        0x16, 0xfb, 0xf5, 0xf9, 0xb5, 0xa6, 0xb2, 0xf4, 0x6c, 0x60, 0x2f, 0xfb, 0xed, 0x56, 0x2a,
        0xf7, 0x93, 0xc5, 0xb5, 0x85, 0xc5, 0xb5, 0x83, 0xa4, 0x79, 0xb2, 0x74, 0x96, 0x98, 0x79,
        0x96, 0x38, 0xb2, 0xc8, 0xb7, 0x66, 0x85, 0x46, 0x8f, 0x44, 0x95, 0x6d, 0xb8, 0xa7, 0x00,
        0x7b, 0x40, 0x7b, 0x58, 0x8f, 0xa9, 0xc6, 0x9b, 0x6d, 0x6e, 0x55, 0x8c, 0x49, 0xd9, 0xe8,
        0x61, 0x29, 0x8b, 0x88, 0x76, 0x95, 0x1a, 0xb9, 0x45, 0x6f, 0x0f, 0x89, 0xf6, 0x86, 0x5d,
        0x7a, 0xaa, 0x8d, 0x48, 0xa1, 0xa7, 0x29, 0x47, 0x97, 0x74, 0x57, 0x69, 0x3b, 0x98, 0x51,
        0x78, 0x79, 0x40, 0x68, 0x42, 0x5d, 0x09, 0x8c, 0x5f, 0x78, 0x94, 0xf6, 0x89, 0x98, 0x62,
        0x46, 0x5a, 0x68, 0x34, 0x98, 0x0b, 0x4a, 0x4b, 0x9c, 0x28, 0x7e, 0x00, 0x66, 0xad, 0xca,
        0x69, 0x4d, 0x7c, 0x8a, 0xc6, 0x7e, 0x8e, 0x27, 0x3d, 0x2b, 0x6f, 0x0a, 0x7c, 0x79, 0xa6,
        0xaa, 0xf7, 0x6e, 0x06, 0xbb, 0x49, 0xaf, 0x08, 0x4f, 0x14, 0xfe, 0xb6, 0xef, 0xfb, 0xff,
        0xee, 0xed, 0xed, 0xef, 0x9e, 0xb9, 0xee, 0xb8, 0x17, 0x4b, 0xbf, 0x84, 0x35, 0x98, 0x98,
        0x59, 0x18, 0xe0, 0x64, 0x45, 0xc6, 0x04, 0x58, 0xa6, 0xac, 0x44, 0x67, 0x9a, 0x59, 0x98,
        0xc5, 0x12, 0x44, 0xf8, 0x20, 0x63, 0x66, 0x18, 0x59, 0xe2, 0x23, 0x32, 0xb9, 0x49, 0x28,
        0x6d, 0x86, 0x4a, 0xdb, 0xbb, 0xa7, 0x75, 0x32, 0x05, 0x7a, 0x18, 0x3c, 0x4f, 0x16, 0x05,
        0xd0, 0x85, 0x84, 0x58, 0x21, 0x96, 0xa7, 0x62, 0x8b, 0xa9, 0x73, 0x1a, 0xf8, 0xd7, 0x83,
        0x57, 0x57, 0x53, 0x6f, 0x80, 0x56, 0xcb, 0xaa, 0x96, 0x73, 0x79, 0xf6, 0xa5, 0xb8, 0x47,
        0x78, 0x14, 0xad, 0xa7, 0xb3, 0x08, 0xd5, 0xb3, 0xd6, 0xf4, 0x48, 0xf3, 0x7a, 0x49, 0xa5,
        0x94, 0x35, 0xeb, 0x98, 0x5d, 0x0f, 0x82, 0x66, 0x88, 0x77, 0x46, 0xac, 0x2c, 0x67, 0xc7,
        0x72, 0x98, 0xf7, 0x78, 0x50, 0x75, 0x7d, 0x32, 0x92, 0xa5, 0xcd, 0x9a, 0x59, 0x60, 0xf9,
        0xd9, 0x0a, 0x01, 0x18, 0xb0, 0xf4, 0xfd, 0xf5, 0x6b, 0x68, 0xb3, 0xdf, 0x9d, 0x7c, 0x7f,
        0xf9, 0x94, 0xd8, 0xa7, 0xf0, 0x7e, 0x9f, 0x9a, 0x99, 0x7e, 0x35, 0x85, 0x6c, 0x69, 0x5a,
        0x74, 0x0d, 0x98, 0x76, 0x64, 0x99, 0x38, 0x8c, 0x5d, 0x74, 0x6a, 0xb4, 0x8e, 0x77, 0x8c,
        0x63, 0x7a, 0x8d, 0x07, 0xb7, 0x77, 0x88, 0x4c, 0x6a, 0xf8, 0x35, 0x9b, 0x53, 0x83, 0x23,
        0x28, 0xc4, 0x48, 0x5a, 0x84, 0x45, 0x4c, 0x17, 0xc2, 0x38, 0xa7, 0xc5, 0x90, 0x4f, 0x7c,
        0x47, 0x86, 0x89, 0x79, 0x58, 0x3a, 0x41, 0x92, 0x7b, 0x41, 0x30, 0x25, 0x69, 0x89, 0x22,
        0x37, 0x05, 0x25, 0x64, 0xf5, 0x38, 0x42, 0x32, 0x22, 0x54, 0x5a, 0x0f, 0x52, 0x52, 0x94,
        0xa4, 0x21, 0x20, 0x79, 0x2c, 0x17, 0x46, 0xa2, 0x96, 0xa5, 0xc5, 0xb6, 0x97, 0xa6, 0xc5,
        0xb5, 0xa0, 0x87, 0x93, 0xb3, 0xb2, 0xb9, 0x87, 0xd6, 0xa4, 0x0b, 0xb7, 0x6a, 0xc6, 0x3b,
        0xf7, 0xc2, 0xc2, 0x85, 0xc5, 0xb7, 0xa6, 0x7b, 0xef, 0x45, 0x0b, 0xc5, 0x16, 0xf5, 0xf0,
        0xaf, 0xef, 0xef, 0xfd, 0xa4, 0xf2, 0x87, 0x43, 0x8b, 0xff, 0xc7, 0x1b, 0x83, 0xc9, 0x19,
        0x36, 0x72, 0x75, 0xcf, 0x90, 0xcc, 0x88, 0xf7, 0x54, 0x52, 0x2a, 0x0e, 0x85, 0x8a, 0x17,
        0xaa, 0xe1, 0xa6, 0xd6, 0xe4, 0xe4, 0x73, 0xd3, 0xf7, 0xfd, 0xa6, 0x7d, 0xb7, 0x34, 0x87,
        0x68, 0xd7, 0x52, 0xc2, 0xb3, 0xb3, 0x39, 0xa4, 0xc5, 0xcc, 0x80, 0x21, 0xd5, 0x5c, 0x64,
        0x83, 0x99, 0x66, 0x76, 0x85, 0x6a, 0xea, 0x8e, 0xb3, 0xb2, 0x9f, 0x29, 0x52, 0x07, 0x47,
        0x46, 0x69, 0x77, 0x89, 0x8a, 0x48, 0xa4, 0x8c, 0x82, 0xa7, 0x6a, 0x78, 0x66, 0xaf, 0x99,
        0x61, 0x77, 0xb4, 0xb8, 0x26, 0x65, 0xba, 0x7a, 0xed, 0x59, 0x09, 0xab, 0x80, 0xf6, 0x16,
        0x2b, 0xa6, 0x62, 0x9b, 0x53, 0x17, 0x5c, 0xb4, 0x8a, 0xd7, 0x87, 0xd0, 0x4d, 0x69, 0xac,
        0x8b, 0x39, 0x0d, 0x97, 0x54, 0x84, 0x8b, 0xb9, 0xa7, 0xff, 0x75, 0x60, 0x8a, 0x7d, 0xb7,
        0xa8, 0x98, 0x57,
    ];

    /// The property that makes `ferrox quantize --type q4_k_s --pure`'s
    /// output a file llama.cpp would have written, rather than one that
    /// merely decodes to similar numbers.
    ///
    /// An encoder that is within Q4_K's error bound passes any
    /// tolerance test and still writes a different file. Only this
    /// catches that.
    #[test]
    fn q4_k_matches_llama_cpp_quantize_row_q4_k_ref() {
        let x = k_quant_fixture();
        let mut got = Vec::new();
        encode_row_q4_k(&x, None, &mut got).unwrap();
        assert_eq!(got.len(), LLAMA_CPP_Q4_K_GOLDEN.len());
        for (b, (g, w)) in got
            .as_chunks::<Q4_K_BLOCK_BYTES>()
            .0
            .iter()
            .zip(LLAMA_CPP_Q4_K_GOLDEN.as_chunks::<Q4_K_BLOCK_BYTES>().0)
            .enumerate()
        {
            assert_eq!(g, w, "super-block {b} disagrees with llama.cpp");
        }
    }

    /// A row that is not a whole number of super-blocks is refused, not
    /// padded. llama.cpp answers this case by changing the tensor's
    /// TYPE (Q4_K -> Q5_0 -> F16); ferrox has neither encoder, and
    /// padding would shift every following row on decode.
    #[test]
    fn a_row_that_is_not_a_whole_number_of_super_blocks_is_refused() {
        let mut out = Vec::new();
        assert!(encode_row_q4_k(&[0.5; Q4_K_BLOCK_ELEMS + 1], None, &mut out).is_none());
        // 32 is a Q8_0 block and a Q4_K sub-block, and still not a
        // Q4_K row: the block size that matters here is 256.
        assert!(encode_row_q4_k(&[0.5; 32], None, &mut out).is_none());
        assert!(encode_row_q4_k(&[], None, &mut out).is_some());
        // An importance slice that does not cover the row is refused
        // the same way: a shifted weight is a wrong file, not an error
        // the reader can see.
        assert!(encode_row_q4_k(&[0.5; Q4_K_BLOCK_ELEMS], Some(&[1.0; 32]), &mut out).is_none());
    }

    /// Round trip through this crate's own reader, against an exact
    /// property rather than a tolerance: for every element, **no
    /// representable level is strictly closer** than the one the
    /// encoder chose.
    ///
    /// A tolerance would have to be invented, and an invented tolerance
    /// is what this whole issue exists to avoid. This is a fact instead:
    /// stage 3 rounds to the nearest of the 16 levels `d*sc*k -
    /// dmin*m`, so a nibble packed into the wrong half-byte, a scale
    /// unpacked from the wrong bits, or an off-by-one in the sub-block
    /// stride all move some element off its nearest level and turn this
    /// red. It says nothing about whether the *fit* is good -- that is
    /// what the golden above is for, and this is the weak half.
    ///
    /// (A sub-block whose 6-bit scale rounded to zero has all 16 levels
    /// equal, so it passes trivially. Sub-block 17 is that case, on
    /// purpose.)
    #[test]
    fn every_element_lands_on_its_nearest_representable_level() {
        let x = k_quant_fixture();
        let mut bytes = Vec::new();
        encode_row_q4_k(&x, None, &mut bytes).unwrap();
        let back = dequant_q4_k(&bytes).unwrap();
        assert_eq!(back.len(), x.len());

        for (b, block) in bytes.as_chunks::<Q4_K_BLOCK_BYTES>().0.iter().enumerate() {
            let d = f16::from_le_bytes([block[0], block[1]]).to_f32();
            let dmin = f16::from_le_bytes([block[2], block[3]]).to_f32();
            let packed: [u8; Q4_K_SCALE_BYTES] = block[4..16].try_into().unwrap();
            for j in 0..QK_SUBS {
                let (sc, m) = q4_k_scale_min(j, &packed);
                let (dj, dm) = (d * sc as f32, dmin * m as f32);
                for ii in 0..QK_SUB_ELEMS {
                    let idx = b * Q4_K_BLOCK_ELEMS + QK_SUB_ELEMS * j + ii;
                    let chosen = (x[idx] - back[idx]).abs();
                    for k in 0..=15u8 {
                        let level = dj * k as f32 - dm;
                        assert!(
                            (x[idx] - level).abs() >= chosen,
                            "block {b} sub-block {j} element {ii}: {} is closer to {} than to the \
                             chosen {}",
                            x[idx],
                            level,
                            back[idx]
                        );
                    }
                }
            }
        }
    }
}