oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! KV-cache quantization (int8 / FP8 / NF4) with per-head or per-layer scales.

use crate::weight_quant::{
    absmax_scale, decode_scalar, encode_scalar, read_packed_public, write_packed_public,
    WeightQuantization,
};
use serde::{Deserialize, Serialize};

/// KV element format.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum KvQuantization {
    Int8,
    Nf4,
    #[serde(rename = "fp8_e4m3")]
    Fp8E4M3,
    #[serde(rename = "fp8_e5m2")]
    Fp8E5M2,
}

impl KvQuantization {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Int8 => "int8",
            Self::Nf4 => "nf4",
            Self::Fp8E4M3 => "fp8_e4m3",
            Self::Fp8E5M2 => "fp8_e5m2",
        }
    }

    pub fn bits(self) -> u32 {
        match self {
            Self::Int8 | Self::Fp8E4M3 | Self::Fp8E5M2 => 8,
            Self::Nf4 => 4,
        }
    }

    pub fn parse(value: &str) -> Result<Self, String> {
        match value.trim().to_ascii_lowercase().as_str() {
            "int8" | "i8" | "8" => Ok(Self::Int8),
            "nf4" => Ok(Self::Nf4),
            "fp8" | "fp8_e4m3" | "e4m3" => Ok(Self::Fp8E4M3),
            "fp8_e5m2" | "e5m2" => Ok(Self::Fp8E5M2),
            other => Err(format!(
                "unknown KV quantization '{other}'; expected int8|nf4|fp8_e4m3|fp8_e5m2"
            )),
        }
    }

    pub fn weight_format(self) -> WeightQuantization {
        match self {
            Self::Int8 => WeightQuantization::Int8,
            Self::Nf4 => WeightQuantization::Nf4,
            Self::Fp8E4M3 => WeightQuantization::Fp8E4M3,
            Self::Fp8E5M2 => WeightQuantization::Fp8E5M2,
        }
    }
}

/// Scale granularity over `[B, H, S, D]` KV tensors.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum KvGranularity {
    /// One scale for the whole K (or V) tensor of a layer.
    PerLayer,
    /// One scale per KV head (shared across batch and sequence).
    PerHead,
}

impl KvGranularity {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::PerLayer => "layer",
            Self::PerHead => "head",
        }
    }

    pub fn parse(value: &str) -> Result<Self, String> {
        match value.trim().to_ascii_lowercase().as_str() {
            "layer" | "per-layer" | "per_layer" | "tensor" => Ok(Self::PerLayer),
            "head" | "per-head" | "per_head" => Ok(Self::PerHead),
            other => Err(format!(
                "unknown KV granularity '{other}'; expected layer|head"
            )),
        }
    }
}

/// Complete KV-cache quantization scheme (symmetric).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct KvQuantScheme {
    pub format: KvQuantization,
    pub granularity: KvGranularity,
}

impl KvQuantScheme {
    pub fn int8_per_head() -> Self {
        Self {
            format: KvQuantization::Int8,
            granularity: KvGranularity::PerHead,
        }
    }

    pub fn scale_count(self, heads: usize) -> usize {
        match self.granularity {
            KvGranularity::PerLayer => 1,
            KvGranularity::PerHead => heads.max(1),
        }
    }

    pub fn bytes_per_element(self) -> f32 {
        self.format.bits() as f32 / 8.0
    }
}

/// Packed K or V tensor for one attention layer.
#[derive(Clone, Debug)]
pub struct PackedKvTensor {
    pub scheme: KvQuantScheme,
    pub packed: Vec<u32>,
    pub scales: Vec<f32>,
    pub batch: usize,
    pub heads: usize,
    pub seq: usize,
    pub head_dim: usize,
}

impl PackedKvTensor {
    pub fn numel(&self) -> usize {
        self.batch
            .saturating_mul(self.heads)
            .saturating_mul(self.seq)
            .saturating_mul(self.head_dim)
    }

    pub fn storage_bytes(&self) -> usize {
        self.packed.len() * std::mem::size_of::<u32>()
            + self.scales.len() * std::mem::size_of::<f32>()
    }

    pub fn dequantize(&self) -> Vec<f32> {
        let format = self.scheme.format.weight_format();
        let mut values = vec![0.0; self.numel()];
        let head_stride = self.seq * self.head_dim;
        for batch in 0..self.batch {
            for head in 0..self.heads {
                let scale = match self.scheme.granularity {
                    KvGranularity::PerLayer => self.scales[0],
                    KvGranularity::PerHead => self.scales[head],
                };
                let base = (batch * self.heads + head) * head_stride;
                for index in 0..head_stride {
                    let flat = base + index;
                    let raw = read_packed_public(&self.packed, flat, format);
                    values[flat] = decode_scalar(raw, scale, 0.0, format, true);
                }
            }
        }
        values
    }
}

/// Pack a `[B, H, S, D]` FP32 KV tensor.
pub fn pack_kv(
    values: &[f32],
    batch: usize,
    heads: usize,
    seq: usize,
    head_dim: usize,
    scheme: KvQuantScheme,
) -> Result<PackedKvTensor, String> {
    let expected = batch * heads * seq * head_dim;
    if values.len() != expected {
        return Err(format!(
            "KV length {} != B·H·S·D ({expected})",
            values.len()
        ));
    }
    let format = scheme.format.weight_format();
    let scale_count = scheme.scale_count(heads);
    let mut scales = vec![1.0; scale_count];
    let head_stride = seq * head_dim;
    match scheme.granularity {
        KvGranularity::PerLayer => {
            scales[0] = absmax_scale(values, format).max(1e-12);
        }
        KvGranularity::PerHead => {
            for head in 0..heads {
                let mut absmax = 0.0f32;
                for batch_index in 0..batch {
                    let start = (batch_index * heads + head) * head_stride;
                    let end = start + head_stride;
                    for &value in &values[start..end] {
                        absmax = absmax.max(value.abs());
                    }
                }
                scales[head] = absmax_scale(&[absmax], format).max(1e-12);
            }
        }
    }

    let words = expected.div_ceil(format.values_per_word());
    let mut packed = vec![0u32; words];
    for batch_index in 0..batch {
        for head in 0..heads {
            let scale = match scheme.granularity {
                KvGranularity::PerLayer => scales[0],
                KvGranularity::PerHead => scales[head],
            };
            let base = (batch_index * heads + head) * head_stride;
            for index in 0..head_stride {
                let flat = base + index;
                let encoded = encode_scalar(values[flat], scale, 0.0, format, true);
                write_packed_public(&mut packed, flat, encoded, format);
            }
        }
    }

    Ok(PackedKvTensor {
        scheme,
        packed,
        scales,
        batch,
        heads,
        seq,
        head_dim,
    })
}

/// Estimate quantized KV-cache bytes for a full model cache.
pub fn estimate_kv_cache_bytes_quantized(
    num_layers: usize,
    batch: usize,
    seq: usize,
    kv_heads: usize,
    head_dim: usize,
    scheme: KvQuantScheme,
) -> usize {
    let elements_per_layer = batch
        .saturating_mul(kv_heads)
        .saturating_mul(seq)
        .saturating_mul(head_dim)
        .saturating_mul(2); // K + V
    let packed_bytes =
        ((elements_per_layer as f32) * scheme.bytes_per_element()).ceil() as usize;
    let scale_bytes = scheme.scale_count(kv_heads) * 2 * std::mem::size_of::<f32>();
    num_layers.saturating_mul(packed_bytes.saturating_add(scale_bytes))
}

/// Running absmax collector for offline KV range calibration.
#[derive(Clone, Debug, Default)]
pub struct KvRangeCollector {
    pub key_absmax: f32,
    pub value_absmax: f32,
    pub per_head_key: Vec<f32>,
    pub per_head_value: Vec<f32>,
}

impl KvRangeCollector {
    pub fn new(heads: usize) -> Self {
        Self {
            key_absmax: 0.0,
            value_absmax: 0.0,
            per_head_key: vec![0.0; heads],
            per_head_value: vec![0.0; heads],
        }
    }

    pub fn observe(
        &mut self,
        key: &[f32],
        value: &[f32],
        batch: usize,
        heads: usize,
        seq: usize,
        head_dim: usize,
    ) {
        if self.per_head_key.len() != heads {
            self.per_head_key = vec![0.0; heads];
            self.per_head_value = vec![0.0; heads];
        }
        let head_stride = seq * head_dim;
        for batch_index in 0..batch {
            for head in 0..heads {
                let start = (batch_index * heads + head) * head_stride;
                let end = start + head_stride;
                for &v in &key[start..end] {
                    let abs = v.abs();
                    self.key_absmax = self.key_absmax.max(abs);
                    self.per_head_key[head] = self.per_head_key[head].max(abs);
                }
                for &v in &value[start..end] {
                    let abs = v.abs();
                    self.value_absmax = self.value_absmax.max(abs);
                    self.per_head_value[head] = self.per_head_value[head].max(abs);
                }
            }
        }
    }

    pub fn scales(&self, scheme: KvQuantScheme) -> (Vec<f32>, Vec<f32>) {
        let to_scale = |absmax: f32| -> f32 {
            if absmax <= 0.0 {
                return 1.0;
            }
            match scheme.format {
                KvQuantization::Int8 => absmax / 127.0,
                _ => absmax,
            }
        };
        match scheme.granularity {
            KvGranularity::PerLayer => (vec![to_scale(self.key_absmax)], vec![to_scale(self.value_absmax)]),
            KvGranularity::PerHead => (
                self.per_head_key.iter().copied().map(to_scale).collect(),
                self.per_head_value.iter().copied().map(to_scale).collect(),
            ),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn int8_per_head_roundtrip() {
        let batch = 1;
        let heads = 2;
        let seq = 3;
        let head_dim = 4;
        let mut values = Vec::new();
        for i in 0..(batch * heads * seq * head_dim) {
            values.push((i as f32) * 0.01 - 0.2);
        }
        let packed = pack_kv(
            &values,
            batch,
            heads,
            seq,
            head_dim,
            KvQuantScheme::int8_per_head(),
        )
        .unwrap();
        assert_eq!(packed.scales.len(), heads);
        let recon = packed.dequantize();
        for (original, restored) in values.iter().zip(recon.iter()) {
            assert!(
                (original - restored).abs() < 0.02,
                "{original} vs {restored}"
            );
        }
    }

    #[test]
    fn nf4_per_layer_packs() {
        let values: Vec<f32> = (0..16).map(|i| (i as f32) * 0.05 - 0.4).collect();
        let packed = pack_kv(
            &values,
            1,
            1,
            4,
            4,
            KvQuantScheme {
                format: KvQuantization::Nf4,
                granularity: KvGranularity::PerLayer,
            },
        )
        .unwrap();
        assert_eq!(packed.scales.len(), 1);
        assert!(!packed.packed.is_empty());
    }
}