oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Activation quantization for mixed-precision inference (W8A8 / W4A8 / W4A16).
//!
//! Supports static and dynamic int8 activations at per-tensor or per-token
//! granularity. W4A16 is weight-only (activations stay FP32/FP16).

use crate::weight_quant::WeightQuantization;
use serde::{Deserialize, Serialize};

/// Activation element format (int8 for A8 modes).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ActivationQuantization {
    Int8,
}

impl ActivationQuantization {
    pub fn as_str(self) -> &'static str {
        "int8"
    }

    pub fn bits(self) -> u32 {
        8
    }

    pub fn parse(value: &str) -> Result<Self, String> {
        match value.trim().to_ascii_lowercase().as_str() {
            "int8" | "i8" | "8" | "a8" => Ok(Self::Int8),
            other => Err(format!(
                "unknown activation quantization '{other}'; expected int8"
            )),
        }
    }

    pub fn weight_format(self) -> WeightQuantization {
        WeightQuantization::Int8
    }

    pub fn max_q(self) -> f32 {
        127.0
    }
}

/// Scale granularity over the activation matrix `[rows, features]`.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ActGranularity {
    /// One scale for the entire activation tensor.
    PerTensor,
    /// One scale per row (token / batch×seq position).
    PerToken,
}

impl ActGranularity {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::PerTensor => "tensor",
            Self::PerToken => "token",
        }
    }

    pub fn parse(value: &str) -> Result<Self, String> {
        match value.trim().to_ascii_lowercase().as_str() {
            "tensor" | "per-tensor" | "per_tensor" => Ok(Self::PerTensor),
            "token" | "per-token" | "per_token" | "row" => Ok(Self::PerToken),
            other => Err(format!(
                "unknown activation granularity '{other}'; expected tensor|token"
            )),
        }
    }
}

/// How activation scales are obtained.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ActQuantMode {
    /// Scales fixed from offline calibration (per-tensor only).
    Static,
    /// Scales computed from the live activation tensor each forward.
    Dynamic,
}

impl ActQuantMode {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Static => "static",
            Self::Dynamic => "dynamic",
        }
    }

    pub fn parse(value: &str) -> Result<Self, String> {
        match value.trim().to_ascii_lowercase().as_str() {
            "static" | "offline" => Ok(Self::Static),
            "dynamic" | "online" => Ok(Self::Dynamic),
            other => Err(format!(
                "unknown activation mode '{other}'; expected static|dynamic"
            )),
        }
    }
}

/// Complete activation quantization scheme.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActQuantScheme {
    pub format: ActivationQuantization,
    pub granularity: ActGranularity,
    pub mode: ActQuantMode,
}

impl ActQuantScheme {
    pub fn int8_dynamic_per_token() -> Self {
        Self {
            format: ActivationQuantization::Int8,
            granularity: ActGranularity::PerToken,
            mode: ActQuantMode::Dynamic,
        }
    }

    pub fn int8_dynamic_per_tensor() -> Self {
        Self {
            format: ActivationQuantization::Int8,
            granularity: ActGranularity::PerTensor,
            mode: ActQuantMode::Dynamic,
        }
    }

    pub fn int8_static_per_tensor() -> Self {
        Self {
            format: ActivationQuantization::Int8,
            granularity: ActGranularity::PerTensor,
            mode: ActQuantMode::Static,
        }
    }

    pub fn validate(self) -> Result<(), String> {
        if self.mode == ActQuantMode::Static && self.granularity != ActGranularity::PerTensor {
            return Err(
                "static activation quantization only supports per-tensor granularity".into(),
            );
        }
        Ok(())
    }
}

/// Named mixed-precision inference modes.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub enum InferenceQuantMode {
    /// int8 weights × int8 activations.
    W8A8,
    /// int4 weights × int8 activations.
    W4A8,
    /// int4 weights × FP16/FP32 activations (weight-only).
    W4A16,
}

impl InferenceQuantMode {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::W8A8 => "W8A8",
            Self::W4A8 => "W4A8",
            Self::W4A16 => "W4A16",
        }
    }

    pub fn parse(value: &str) -> Result<Self, String> {
        match value.trim().to_ascii_uppercase().as_str() {
            "W8A8" => Ok(Self::W8A8),
            "W4A8" => Ok(Self::W4A8),
            "W4A16" | "W4A32" => Ok(Self::W4A16),
            other => Err(format!(
                "unknown inference mode '{other}'; expected W8A8|W4A8|W4A16"
            )),
        }
    }

    pub fn weight_format(self) -> WeightQuantization {
        match self {
            Self::W8A8 => WeightQuantization::Int8,
            Self::W4A8 | Self::W4A16 => WeightQuantization::Int4,
        }
    }

    pub fn activation_scheme(self) -> Option<ActQuantScheme> {
        match self {
            Self::W8A8 | Self::W4A8 => Some(ActQuantScheme::int8_dynamic_per_token()),
            Self::W4A16 => None,
        }
    }
}

/// Live / calibrated scales attached to a Linear.
#[derive(Clone, Debug, PartialEq)]
pub struct ActQuantConfig {
    pub scheme: ActQuantScheme,
    /// Required when `scheme.mode == Static` (per-tensor absmax scale).
    pub static_scale: Option<f32>,
}

impl ActQuantConfig {
    pub fn dynamic(scheme: ActQuantScheme) -> Result<Self, String> {
        let mut scheme = scheme;
        scheme.mode = ActQuantMode::Dynamic;
        scheme.validate()?;
        Ok(Self {
            scheme,
            static_scale: None,
        })
    }

    pub fn static_per_tensor(scale: f32) -> Self {
        Self {
            scheme: ActQuantScheme::int8_static_per_tensor(),
            static_scale: Some(if scale > 0.0 && scale.is_finite() {
                scale
            } else {
                1.0
            }),
        }
    }

    pub fn validate(&self) -> Result<(), String> {
        self.scheme.validate()?;
        if self.scheme.mode == ActQuantMode::Static && self.static_scale.is_none() {
            return Err("static activation quant requires a calibrated scale".into());
        }
        Ok(())
    }
}

/// Quantized activation matrix ready for int8×int matmul.
#[derive(Clone, Debug)]
pub struct QuantizedActivations {
    /// Symmetric int8 codes, row-major `[rows * features]`.
    pub codes: Vec<i8>,
    /// Per-row scales (length `rows`) or a single tensor scale (length 1).
    pub scales: Vec<f32>,
    pub rows: usize,
    pub features: usize,
    pub granularity: ActGranularity,
}

impl QuantizedActivations {
    pub fn scale_for_row(&self, row: usize) -> f32 {
        match self.granularity {
            ActGranularity::PerTensor => self.scales[0],
            ActGranularity::PerToken => self.scales[row],
        }
    }

    /// Fake-quantized FP32 reconstruction (for GPU / W4A8 dequant paths).
    pub fn dequantized(&self) -> Vec<f32> {
        let mut out = vec![0.0; self.rows * self.features];
        for row in 0..self.rows {
            let scale = self.scale_for_row(row);
            let offset = row * self.features;
            for feature in 0..self.features {
                out[offset + feature] = self.codes[offset + feature] as f32 * scale;
            }
        }
        out
    }
}

/// Abs-max → int8 scale for a value slice.
pub fn int8_scale(values: &[f32]) -> f32 {
    let absmax = values.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
    if absmax > 0.0 && absmax.is_finite() {
        absmax / ActivationQuantization::Int8.max_q()
    } else {
        1.0
    }
}

/// Quantize a row-major activation matrix.
pub fn quantize_activations(
    values: &[f32],
    rows: usize,
    features: usize,
    config: &ActQuantConfig,
) -> Result<QuantizedActivations, String> {
    config.validate()?;
    if values.len() != rows * features {
        return Err(format!(
            "activation length {} != rows ({rows}) * features ({features})",
            values.len()
        ));
    }
    let max_q = config.scheme.format.max_q();
    let mut codes = vec![0i8; values.len()];
    let scales = match (config.scheme.mode, config.scheme.granularity) {
        (ActQuantMode::Static, ActGranularity::PerTensor) => {
            let scale = config.static_scale.unwrap_or(1.0).max(1e-12);
            for (index, &value) in values.iter().enumerate() {
                codes[index] = (value / scale).round().clamp(-max_q, max_q) as i8;
            }
            vec![scale]
        }
        (ActQuantMode::Dynamic, ActGranularity::PerTensor) => {
            let scale = int8_scale(values).max(1e-12);
            for (index, &value) in values.iter().enumerate() {
                codes[index] = (value / scale).round().clamp(-max_q, max_q) as i8;
            }
            vec![scale]
        }
        (ActQuantMode::Dynamic, ActGranularity::PerToken) => {
            let mut row_scales = vec![1.0; rows];
            for row in 0..rows {
                let start = row * features;
                let end = start + features;
                let scale = int8_scale(&values[start..end]).max(1e-12);
                row_scales[row] = scale;
                for feature in 0..features {
                    let value = values[start + feature];
                    codes[start + feature] = (value / scale).round().clamp(-max_q, max_q) as i8;
                }
            }
            row_scales
        }
        (ActQuantMode::Static, ActGranularity::PerToken) => {
            return Err("static per-token activation quantization is unsupported".into());
        }
    };
    Ok(QuantizedActivations {
        codes,
        scales,
        rows,
        features,
        granularity: config.scheme.granularity,
    })
}

/// Collect a running per-tensor absmax for static calibration.
pub fn update_absmax(current: &mut f32, values: &[f32]) {
    for &value in values {
        let abs = value.abs();
        if abs > *current {
            *current = abs;
        }
    }
}

/// Convert a collected absmax into a static int8 scale.
pub fn scale_from_absmax(absmax: f32) -> f32 {
    if absmax > 0.0 && absmax.is_finite() {
        absmax / ActivationQuantization::Int8.max_q()
    } else {
        1.0
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn dynamic_per_token_roundtrip() {
        let rows = 2;
        let features = 4;
        let values = vec![0.5, -1.0, 0.25, 0.0, 2.0, -2.0, 1.0, -0.5];
        let config = ActQuantConfig::dynamic(ActQuantScheme::int8_dynamic_per_token()).unwrap();
        let quantized = quantize_activations(&values, rows, features, &config).unwrap();
        let recon = quantized.dequantized();
        for (original, restored) in values.iter().zip(recon.iter()) {
            assert!(
                (original - restored).abs() < 0.02,
                "{original} vs {restored}"
            );
        }
    }

    #[test]
    fn static_per_tensor_uses_calibrated_scale() {
        let values = vec![1.0f32, -1.0, 0.5, -0.5];
        let scale = int8_scale(&values);
        let config = ActQuantConfig::static_per_tensor(scale);
        let quantized = quantize_activations(&values, 1, 4, &config).unwrap();
        assert_eq!(quantized.scales.len(), 1);
        assert!((quantized.scales[0] - scale).abs() < 1e-6);
    }

    #[test]
    fn inference_mode_parse() {
        assert_eq!(
            InferenceQuantMode::parse("w8a8").unwrap(),
            InferenceQuantMode::W8A8
        );
        assert!(InferenceQuantMode::W4A16.activation_scheme().is_none());
        assert!(InferenceQuantMode::W8A8.activation_scheme().is_some());
    }
}