Struct QuantizedMatrix 

pub struct QuantizedMatrix {
    pub data: Vec<i8>,
    pub scales: Vec<f32>,
    pub n: usize,
    pub k: usize,
}

A weight matrix quantized with per-output-channel symmetric Q8 scales.

Layout is the nn.Linear layout the checkpoint stores: data is [n, k] row-major with one f32 scale per output row. Quantized once at hydration; the borrowed f32 tensor is untouched.

Fields

data: Vec<i8>

Q8 bytes, [n, k] row-major, each value in [-127, 127].

scales: Vec<f32>

One symmetric scale per output row, [n].

n: usize

Output rows.

k: usize

Reduction length of one output element.

Implementations

impl QuantizedMatrix

pub fn concat_rows(parts: &[&Self]) -> Self

Stacks matrices with a shared reduction length into one taller matrix.

Row bytes and scales are byte-identical to quantizing each part separately — this exists so fused projections (QKV, gate‖up) can run as ONE kernel dispatch while every output row keeps exactly the per-channel quantization it would have had alone.

Panics

Panics if the parts disagree on k or the list is empty.

pub fn quantize(weight: &[f32], n: usize, k: usize) -> Self

Quantizes an [n, k] f32 weight matrix one output channel at a time.

Panics

Panics if weight.len() != n * k or any value is non-finite.

Examples found in repository

examples/int8_shape_bench.rs (line 84)

fn main() {
    let tiers: Vec<Int8Tier> = Int8Tier::available();
    println!("int8 shape bench — tiers available: {:?}", {
        tiers.iter().map(|t| t.as_str()).collect::<Vec<_>>()
    });
    println!(
        "interleaved rounds={ROUNDS} (+{WARMUP_ROUNDS} warmup); per-sample = mean over calls in one round; cv% over rounds"
    );

    for &m in &[1_usize, 16] {
        println!(
            "\n== m = {m} {} ==",
            if m == 1 {
                "(decode GEMV)"
            } else {
                "(seq-16 verify GEMM)"
            }
        );
        for &(label, n, k) in SHAPES {
            let calls: usize = (32 / m).max(2);
            let weight = pseudo_random_f32(n * k, 0xbe0_0001 ^ (n as u64) << 20 ^ k as u64);
            let x = pseudo_random_f32(m * k, 0xbe0_0002 ^ (m as u64) << 32 ^ k as u64);
            let quantized = QuantizedMatrix::quantize(&weight, n, k);
            let mut out = vec![0.0_f32; m * n];
            let mut x_q = vec![0_i8; m * k];
            let mut x_scales = vec![0.0_f32; m];

            // One arm per route, all interleaved inside every round.
            let mut f32_samples = Vec::with_capacity(ROUNDS);
            let mut tier_samples: Vec<Vec<f64>> =
                tiers.iter().map(|_| Vec::with_capacity(ROUNDS)).collect();

            for round in 0..ROUNDS + WARMUP_ROUNDS {
                // f32 arm
                let start = Instant::now();
                for _ in 0..calls {
                    f32ref::linear(
                        black_box(&x),
                        black_box(&weight),
                        None,
                        m,
                        k,
                        n,
                        black_box(&mut out),
                    );
                }
                let f32_us = start.elapsed().as_secs_f64() * 1e6 / calls as f64;

                // W8A8 arms, including dynamic activation quantization each call.
                let mut this_round = Vec::with_capacity(tiers.len());
                for &tier in &tiers {
                    let start = Instant::now();
                    for _ in 0..calls {
                        for ((x_row, q_row), scale) in x
                            .chunks_exact(k)
                            .zip(x_q.chunks_exact_mut(k))
                            .zip(x_scales.iter_mut())
                        {
                            *scale = quantize_row_q8(black_box(x_row), q_row);
                        }
                        linear_q8(
                            black_box(&x_q),
                            black_box(&x_scales),
                            black_box(&quantized),
                            None,
                            m,
                            black_box(&mut out),
                            tier,
                        );
                    }
                    this_round.push(start.elapsed().as_secs_f64() * 1e6 / calls as f64);
                }

                if round >= WARMUP_ROUNDS {
                    f32_samples.push(f32_us);
                    for (samples, sample) in tier_samples.iter_mut().zip(&this_round) {
                        samples.push(*sample);
                    }
                }
            }

            let f32_stats = stats(&f32_samples);
            // The f32 reference loops rows outermost, so it streams the weight matrix once per
            // activation row (m times per call); the q8 kernel is weight-stationary and streams
            // it exactly once per call. The column reports actual weight bytes moved per second.
            let f32_bytes = (n * k * 4 * m) as f64;
            println!(
                "{label}  f32     {:9.1} us  cv {:4.1}%  ({:5.1} GB/s weight-stream)",
                f32_stats.mean_us,
                f32_stats.cv_percent,
                f32_bytes / (f32_stats.mean_us * 1e-6) / 1e9,
            );
            for (tier, samples) in tiers.iter().zip(&tier_samples) {
                let tier_stats = stats(samples);
                let q8_bytes = (n * k) as f64;
                let verdict = if tier_stats.cv_percent > 5.0 || f32_stats.cv_percent > 5.0 {
                    "REFUSED (cv>5%)"
                } else {
                    ""
                };
                println!(
                    "{label}  q8 {:9} {:9.1} us  cv {:4.1}%  ({:5.1} GB/s weight-stream)  x{:.2} vs f32 {verdict}",
                    tier.as_str(),
                    tier_stats.mean_us,
                    tier_stats.cv_percent,
                    q8_bytes / (tier_stats.mean_us * 1e-6) / 1e9,
                    f32_stats.mean_us / tier_stats.mean_us,
                );
            }
        }
    }
    println!(
        "\nNOTE: ratios above compare routes inside this tree (self-comparison = maintenance),\nnever a pinned incumbent. cv%>5 rows are refused, not averaged."
    );
}

examples/int4_speed_gate.rs (line 83)

fn main() {
    // (label, n, k) for one microdecoder layer, at decode geometry m = 1.
    const HIDDEN: usize = 1024;
    const INTERMEDIATE: usize = 3072;
    const Q_WIDTH: usize = 16 * 128;
    const KV_WIDTH: usize = 8 * 128;
    let projections: &[(&str, usize, usize)] = &[
        ("q_proj", Q_WIDTH, HIDDEN),
        ("k_proj", KV_WIDTH, HIDDEN),
        ("v_proj", KV_WIDTH, HIDDEN),
        ("o_proj", HIDDEN, Q_WIDTH),
        ("gate_up", INTERMEDIATE * 2, HIDDEN),
        ("down_proj", HIDDEN, INTERMEDIATE),
    ];

    // Fifteen depths x five layers is how often this body is walked per frame.
    const ROUNDS: usize = 15 * 5;

    let dispatched = Int8Tier::dispatch();
    println!("dispatched int8 route: {}", dispatched.as_str());
    println!("shapes: microdecoder layer at m=1, {ROUNDS} rounds (15 depths x 5 layers)\n");
    // The ratio columns are SPEED ratios (q4's throughput relative to the q8 variant):
    // t_q8 / t_q4, so 1.0 means parity and 0.05 means q4 runs at 5% of q8's speed.
    println!(
        "{:<10} {:>6} {:>6} {:>10} {:>10} {:>10} {:>9} {:>9}",
        "proj", "n", "k", "q8-scalar", "q4-scalar", "q8-route", "spd/q8scl", "spd/route"
    );

    let mut total_q8_scalar = 0.0_f64;
    let mut total_q4 = 0.0_f64;
    let mut total_q8_route = 0.0_f64;

    for &(label, n, k) in projections {
        let weight = deterministic(n * k, 0x51ED_0000 + n as u64);
        let activation = deterministic(k, 0xA0C7_0000 + k as u64);
        let mut x_q = vec![0_i8; k];
        let scale = quantize_row_q8(&activation, &mut x_q);

        let q8 = QuantizedMatrix::quantize(&weight, n, k);
        let q4 = QuantizedMatrixQ4::quantize(&weight, n, k);
        let mut out = vec![0.0_f32; n];

        // Warm the caches for whichever side runs first, so the ordering does not decide it.
        linear_q8(&x_q, &[scale], &q8, None, 1, &mut out, Int8Tier::Scalar);
        linear_q4(&x_q, &[scale], &q4, None, 1, &mut out);

        // INTERLEAVED repeats, reporting the minimum of each variant.
        //
        // A first attempt timed the three variants in sequential blocks and produced 98.60 ms for
        // k_proj against 5.14 ms for v_proj — identical 1024x1024 shapes, 19x apart. That is
        // scheduler and thermal noise, and it is precisely what doctrine #8's same-thermal-window
        // rule exists to prevent. Interleaving puts all three variants in the same window on every
        // repeat, and the minimum is the least noise-contaminated estimator of a deterministic
        // kernel: noise only ever adds time.
        const REPEATS: usize = 7;
        let mut q8_scalar = f64::MAX;
        let mut q4_ms = f64::MAX;
        let mut q8_route = f64::MAX;
        for _ in 0..REPEATS {
            let started = Instant::now();
            for _ in 0..ROUNDS {
                linear_q8(&x_q, &[scale], &q8, None, 1, &mut out, Int8Tier::Scalar);
            }
            q8_scalar = q8_scalar.min(started.elapsed().as_secs_f64() * 1000.0);

            let started = Instant::now();
            for _ in 0..ROUNDS {
                linear_q4(&x_q, &[scale], &q4, None, 1, &mut out);
            }
            q4_ms = q4_ms.min(started.elapsed().as_secs_f64() * 1000.0);

            let started = Instant::now();
            for _ in 0..ROUNDS {
                linear_q8(&x_q, &[scale], &q8, None, 1, &mut out, dispatched);
            }
            q8_route = q8_route.min(started.elapsed().as_secs_f64() * 1000.0);
        }

        total_q8_scalar += q8_scalar;
        total_q4 += q4_ms;
        total_q8_route += q8_route;

        println!(
            "{label:<10} {n:>6} {k:>6} {q8_scalar:>9.2}m {q4_ms:>9.2}m {q8_route:>9.2}m {:>8.2}x {:>8.2}x",
            q8_scalar / q4_ms,
            q8_route / q4_ms
        );
    }

    println!(
        "\ntotals (one layer, {ROUNDS} rounds): q8-scalar {total_q8_scalar:.1} ms, \
         q4-scalar {total_q4:.1} ms, q8-route {total_q8_route:.1} ms"
    );
    println!(
        "int4 vs scalar int8 : {:.2}x  ({})",
        total_q8_scalar / total_q4,
        if total_q4 < total_q8_scalar {
            "FASTER - the halved-bytes thesis holds at equal implementation quality"
        } else {
            "SLOWER - unpack cost exceeds the traffic saving even against scalar"
        }
    );
    println!(
        "int4 vs shipping int8: {:.2}x  ({})",
        total_q8_route / total_q4,
        if total_q4 < total_q8_route {
            "FASTER - gate (a) PASSES; the listening gate is now worth running"
        } else {
            "SLOWER - gate (a) FAILS as built; int4 needs an in-register SIMD unpack to compete"
        }
    );

    // Bytes are the thesis; report them so the ratio can be read against the traffic it saves.
    let q8_bytes: usize = projections.iter().map(|(_, n, k)| n * k).sum();
    println!(
        "\nweight bytes per layer: q8 {:.1} MB, q4 {:.1} MB",
        q8_bytes as f64 / 1e6,
        q8_bytes as f64 / 2e6
    );
}

Trait Implementations

impl Clone for QuantizedMatrix

fn clone(&self) -> QuantizedMatrix

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for QuantizedMatrix

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.