Struct QuantizedMatrixQ4 

pub struct QuantizedMatrixQ4 {
    pub data: Vec<u8>,
    pub scales: Vec<f32>,
    pub n: usize,
    pub k: usize,
}

A weight matrix quantized to symmetric int4, packed two values per byte.

Layout mirrors crate::int8::QuantizedMatrix: [n, k] row-major in the checkpoint’s own nn.Linear orientation, one f32 scale per output channel, so no transpose is ever materialized.

Fields

data: Vec<u8>

n * k.div_ceil(2) bytes: row-major, two biased nibbles per byte, low nibble first.

scales: Vec<f32>

One scale per output channel.

n: usize

k: usize

Implementations

impl QuantizedMatrixQ4

pub fn quantize(weight: &[f32], n: usize, k: usize) -> Self

Quantizes an [n, k] f32 weight matrix.

Panics

If weight.len() != n * k, or a weight is non-finite — a NaN reaching the quantizer means the graph upstream is already corrupt, and refusing loudly beats baking it into an artifact.

Examples found in repository

examples/int4_speed_gate.rs (line 84)

fn main() {
    // (label, n, k) for one microdecoder layer, at decode geometry m = 1.
    const HIDDEN: usize = 1024;
    const INTERMEDIATE: usize = 3072;
    const Q_WIDTH: usize = 16 * 128;
    const KV_WIDTH: usize = 8 * 128;
    let projections: &[(&str, usize, usize)] = &[
        ("q_proj", Q_WIDTH, HIDDEN),
        ("k_proj", KV_WIDTH, HIDDEN),
        ("v_proj", KV_WIDTH, HIDDEN),
        ("o_proj", HIDDEN, Q_WIDTH),
        ("gate_up", INTERMEDIATE * 2, HIDDEN),
        ("down_proj", HIDDEN, INTERMEDIATE),
    ];

    // Fifteen depths x five layers is how often this body is walked per frame.
    const ROUNDS: usize = 15 * 5;

    let dispatched = Int8Tier::dispatch();
    println!("dispatched int8 route: {}", dispatched.as_str());
    println!("shapes: microdecoder layer at m=1, {ROUNDS} rounds (15 depths x 5 layers)\n");
    // The ratio columns are SPEED ratios (q4's throughput relative to the q8 variant):
    // t_q8 / t_q4, so 1.0 means parity and 0.05 means q4 runs at 5% of q8's speed.
    println!(
        "{:<10} {:>6} {:>6} {:>10} {:>10} {:>10} {:>9} {:>9}",
        "proj", "n", "k", "q8-scalar", "q4-scalar", "q8-route", "spd/q8scl", "spd/route"
    );

    let mut total_q8_scalar = 0.0_f64;
    let mut total_q4 = 0.0_f64;
    let mut total_q8_route = 0.0_f64;

    for &(label, n, k) in projections {
        let weight = deterministic(n * k, 0x51ED_0000 + n as u64);
        let activation = deterministic(k, 0xA0C7_0000 + k as u64);
        let mut x_q = vec![0_i8; k];
        let scale = quantize_row_q8(&activation, &mut x_q);

        let q8 = QuantizedMatrix::quantize(&weight, n, k);
        let q4 = QuantizedMatrixQ4::quantize(&weight, n, k);
        let mut out = vec![0.0_f32; n];

        // Warm the caches for whichever side runs first, so the ordering does not decide it.
        linear_q8(&x_q, &[scale], &q8, None, 1, &mut out, Int8Tier::Scalar);
        linear_q4(&x_q, &[scale], &q4, None, 1, &mut out);

        // INTERLEAVED repeats, reporting the minimum of each variant.
        //
        // A first attempt timed the three variants in sequential blocks and produced 98.60 ms for
        // k_proj against 5.14 ms for v_proj — identical 1024x1024 shapes, 19x apart. That is
        // scheduler and thermal noise, and it is precisely what doctrine #8's same-thermal-window
        // rule exists to prevent. Interleaving puts all three variants in the same window on every
        // repeat, and the minimum is the least noise-contaminated estimator of a deterministic
        // kernel: noise only ever adds time.
        const REPEATS: usize = 7;
        let mut q8_scalar = f64::MAX;
        let mut q4_ms = f64::MAX;
        let mut q8_route = f64::MAX;
        for _ in 0..REPEATS {
            let started = Instant::now();
            for _ in 0..ROUNDS {
                linear_q8(&x_q, &[scale], &q8, None, 1, &mut out, Int8Tier::Scalar);
            }
            q8_scalar = q8_scalar.min(started.elapsed().as_secs_f64() * 1000.0);

            let started = Instant::now();
            for _ in 0..ROUNDS {
                linear_q4(&x_q, &[scale], &q4, None, 1, &mut out);
            }
            q4_ms = q4_ms.min(started.elapsed().as_secs_f64() * 1000.0);

            let started = Instant::now();
            for _ in 0..ROUNDS {
                linear_q8(&x_q, &[scale], &q8, None, 1, &mut out, dispatched);
            }
            q8_route = q8_route.min(started.elapsed().as_secs_f64() * 1000.0);
        }

        total_q8_scalar += q8_scalar;
        total_q4 += q4_ms;
        total_q8_route += q8_route;

        println!(
            "{label:<10} {n:>6} {k:>6} {q8_scalar:>9.2}m {q4_ms:>9.2}m {q8_route:>9.2}m {:>8.2}x {:>8.2}x",
            q8_scalar / q4_ms,
            q8_route / q4_ms
        );
    }

    println!(
        "\ntotals (one layer, {ROUNDS} rounds): q8-scalar {total_q8_scalar:.1} ms, \
         q4-scalar {total_q4:.1} ms, q8-route {total_q8_route:.1} ms"
    );
    println!(
        "int4 vs scalar int8 : {:.2}x  ({})",
        total_q8_scalar / total_q4,
        if total_q4 < total_q8_scalar {
            "FASTER - the halved-bytes thesis holds at equal implementation quality"
        } else {
            "SLOWER - unpack cost exceeds the traffic saving even against scalar"
        }
    );
    println!(
        "int4 vs shipping int8: {:.2}x  ({})",
        total_q8_route / total_q4,
        if total_q4 < total_q8_route {
            "FASTER - gate (a) PASSES; the listening gate is now worth running"
        } else {
            "SLOWER - gate (a) FAILS as built; int4 needs an in-register SIMD unpack to compete"
        }
    );

    // Bytes are the thesis; report them so the ratio can be read against the traffic it saves.
    let q8_bytes: usize = projections.iter().map(|(_, n, k)| n * k).sum();
    println!(
        "\nweight bytes per layer: q8 {:.1} MB, q4 {:.1} MB",
        q8_bytes as f64 / 1e6,
        q8_bytes as f64 / 2e6
    );
}

pub fn dequantize_row(&self, row: usize) -> Vec<f32>

Dequantizes one output channel back to f32, for parity comparison against the f32 weights.

pub fn packed_bytes(&self) -> usize

Bytes of weight storage, the number this lever exists to shrink.

Trait Implementations

impl Clone for QuantizedMatrixQ4

fn clone(&self) -> QuantizedMatrixQ4

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for QuantizedMatrixQ4

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq for QuantizedMatrixQ4

fn eq(&self, other: &QuantizedMatrixQ4) -> bool

Equality operator ==.

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=.

impl StructuralPartialEq for QuantizedMatrixQ4