oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Quantization-aware training (QAT) configuration and fake-quant helpers.
//!
//! Supports fixed absmax fake-quant (STE) and learnable per-channel/per-tensor
//! scales (LSQ-style). Group granularity is available for fixed-scale QAT;
//! learnable scales require per-tensor or per-channel granularity.

use crate::weight_quant::{QuantGranularity, QuantScheme, ScaleMethod, WeightQuantization};
use crate::Tensor;
use serde::{Deserialize, Serialize};
use std::fmt;

fn default_qat_format() -> WeightQuantization {
    WeightQuantization::Int8
}

/// Trainer / CLI configuration for quantization-aware training.
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct QatConfig {
    pub format: WeightQuantization,
    /// When set, use per-group scales along `in_features` (32 / 64 / 128).
    /// `None` means per-output-channel (default RTN).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub group_size: Option<usize>,
    /// Learned step-size quantization: scales are trainable parameters.
    #[serde(default)]
    pub learnable_scales: bool,
}

impl Default for QatConfig {
    fn default() -> Self {
        Self {
            format: default_qat_format(),
            group_size: None,
            learnable_scales: false,
        }
    }
}

impl QatConfig {
    pub fn from_format(format: WeightQuantization) -> Self {
        Self {
            format,
            ..Self::default()
        }
    }

    pub fn scheme(&self) -> QuantScheme {
        QuantScheme {
            format: self.format,
            granularity: match self.group_size {
                Some(size) => QuantGranularity::PerGroup { size },
                None => QuantGranularity::PerChannel,
            },
            symmetric: true,
            scale_method: ScaleMethod::AbsMax,
            percentile: 99.99,
        }
    }

    pub fn validate(&self) -> Result<(), QatError> {
        if !self.format.is_integer() {
            return Err(QatError::Invalid(
                "QAT currently supports integer formats only (int8/int4/int3/int2)".into(),
            ));
        }
        self.scheme()
            .validate()
            .map_err(|message| QatError::Invalid(message))?;
        if self.learnable_scales && self.group_size.is_some() {
            return Err(QatError::Invalid(
                "learnable QAT scales require per-channel (omit group_size); use fixed-scale QAT for groups".into(),
            ));
        }
        Ok(())
    }

    pub fn as_str(&self) -> String {
        let mut parts = vec![self.format.as_str().to_string()];
        if let Some(size) = self.group_size {
            parts.push(format!("group:{size}"));
        }
        if self.learnable_scales {
            parts.push("lsq".into());
        }
        parts.join("+")
    }
}

impl<'de> Deserialize<'de> for QatConfig {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: serde::Deserializer<'de>,
    {
        #[derive(Deserialize)]
        #[serde(untagged)]
        enum QatConfigDe {
            Format(WeightQuantization),
            Full {
                format: WeightQuantization,
                #[serde(default)]
                group_size: Option<usize>,
                #[serde(default)]
                learnable_scales: bool,
            },
        }
        match QatConfigDe::deserialize(deserializer)? {
            QatConfigDe::Format(format) => Ok(Self::from_format(format)),
            QatConfigDe::Full {
                format,
                group_size,
                learnable_scales,
            } => Ok(Self {
                format,
                group_size,
                learnable_scales,
            }),
        }
    }
}

#[derive(Debug)]
pub enum QatError {
    Invalid(String),
}

impl fmt::Display for QatError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Invalid(message) => write!(formatter, "invalid QAT configuration: {message}"),
        }
    }
}

impl std::error::Error for QatError {}

/// Runtime QAT state attached to a Linear.
#[derive(Clone, Debug)]
pub struct QatRuntime {
    pub config: QatConfig,
    /// Present when `config.learnable_scales`; shape `[scale_count]`.
    pub scales: Option<Tensor>,
}

impl QatRuntime {
    pub fn new(
        config: QatConfig,
        in_features: usize,
        out_features: usize,
        init_scales: Option<Vec<f32>>,
    ) -> Result<Self, QatError> {
        config.validate()?;
        let scales = if config.learnable_scales {
            let scheme = config.scheme();
            let count = scheme.scale_count(in_features, out_features);
            let values = match init_scales {
                Some(values) if values.len() == count => values,
                Some(values) => {
                    return Err(QatError::Invalid(format!(
                        "expected {count} QAT scales, got {}",
                        values.len()
                    )));
                }
                None => vec![1.0; count],
            };
            Some(Tensor::new(values, vec![count], true))
        } else {
            None
        };
        Ok(Self { config, scales })
    }
}

/// Initialize per-channel/per-tensor absmax scales from logical weight values.
pub fn init_absmax_scales(
    in_features: usize,
    out_features: usize,
    scheme: QuantScheme,
    mut logical_value: impl FnMut(usize, usize) -> f32,
) -> Vec<f32> {
    let max_q = scheme
        .format
        .int_maximum()
        .expect("QAT scales require an integer format");
    let count = scheme.scale_count(in_features, out_features);
    let mut scales = vec![1.0f32; count];
    for scale_index in 0..count {
        let mut absmax = 0.0f32;
        match scheme.granularity {
            QuantGranularity::PerTensor => {
                for input in 0..in_features {
                    for output in 0..out_features {
                        absmax = absmax.max(logical_value(input, output).abs());
                    }
                }
            }
            QuantGranularity::PerChannel => {
                let output = scale_index;
                for input in 0..in_features {
                    absmax = absmax.max(logical_value(input, output).abs());
                }
            }
            QuantGranularity::PerGroup { size } => {
                let groups = in_features.div_ceil(size.max(1));
                let output = scale_index / groups;
                let group = scale_index % groups;
                let start = group * size;
                let end = (start + size).min(in_features);
                for input in start..end {
                    absmax = absmax.max(logical_value(input, output).abs());
                }
            }
        }
        scales[scale_index] = if absmax > 0.0 { absmax / max_q } else { 1.0 };
    }
    scales
}

/// Symmetric fake-quant into storage order using provided scales.
pub fn fake_quantize_storage(
    in_features: usize,
    out_features: usize,
    transpose_weight: bool,
    scheme: QuantScheme,
    scales: &[f32],
    mut logical_value: impl FnMut(usize, usize) -> f32,
) -> Vec<f32> {
    let max_q = scheme
        .format
        .int_maximum()
        .expect("fake-quant requires an integer format");
    let mut storage = vec![0.0f32; in_features * out_features];
    for input in 0..in_features {
        for output in 0..out_features {
            let scale_index = match scheme.granularity {
                QuantGranularity::PerTensor => 0,
                QuantGranularity::PerChannel => output,
                QuantGranularity::PerGroup { size } => {
                    let groups = in_features.div_ceil(size.max(1));
                    output * groups + input / size.max(1)
                }
            };
            let scale = scales[scale_index].max(1e-12);
            let value = logical_value(input, output);
            let q = (value / scale).round().clamp(-max_q, max_q);
            let dequant = q * scale;
            if transpose_weight {
                storage[output * in_features + input] = dequant;
            } else {
                storage[input * out_features + output] = dequant;
            }
        }
    }
    storage
}

/// Integer codes (as f32) in storage order for LSQ `q * scale` reconstruction.
pub fn quantized_codes_storage(
    in_features: usize,
    out_features: usize,
    transpose_weight: bool,
    scheme: QuantScheme,
    scales: &[f32],
    mut logical_value: impl FnMut(usize, usize) -> f32,
) -> Vec<f32> {
    let max_q = scheme
        .format
        .int_maximum()
        .expect("fake-quant requires an integer format");
    let mut storage = vec![0.0f32; in_features * out_features];
    for input in 0..in_features {
        for output in 0..out_features {
            let scale_index = match scheme.granularity {
                QuantGranularity::PerTensor => 0,
                QuantGranularity::PerChannel => output,
                QuantGranularity::PerGroup { size } => {
                    let groups = in_features.div_ceil(size.max(1));
                    output * groups + input / size.max(1)
                }
            };
            let scale = scales[scale_index].max(1e-12);
            let value = logical_value(input, output);
            let q = (value / scale).round().clamp(-max_q, max_q);
            if transpose_weight {
                storage[output * in_features + input] = q;
            } else {
                storage[input * out_features + output] = q;
            }
        }
    }
    storage
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn deserialize_format_string_and_object() {
        let from_str: QatConfig = serde_json::from_str("\"int4\"").unwrap();
        assert_eq!(from_str.format, WeightQuantization::Int4);
        assert!(!from_str.learnable_scales);

        let from_obj: QatConfig = serde_json::from_str(
            r#"{ "format": "int8", "group_size": 64, "learnable_scales": false }"#,
        )
        .unwrap();
        assert_eq!(from_obj.group_size, Some(64));
    }

    #[test]
    fn learnable_rejects_group_size() {
        let config = QatConfig {
            format: WeightQuantization::Int8,
            group_size: Some(64),
            learnable_scales: true,
        };
        assert!(config.validate().is_err());
    }
}