use crate::weight_quant::{QuantGranularity, QuantScheme, ScaleMethod, WeightQuantization};
use crate::Tensor;
use serde::{Deserialize, Serialize};
use std::fmt;
fn default_qat_format() -> WeightQuantization {
WeightQuantization::Int8
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct QatConfig {
pub format: WeightQuantization,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub group_size: Option<usize>,
#[serde(default)]
pub learnable_scales: bool,
}
impl Default for QatConfig {
fn default() -> Self {
Self {
format: default_qat_format(),
group_size: None,
learnable_scales: false,
}
}
}
impl QatConfig {
pub fn from_format(format: WeightQuantization) -> Self {
Self {
format,
..Self::default()
}
}
pub fn scheme(&self) -> QuantScheme {
QuantScheme {
format: self.format,
granularity: match self.group_size {
Some(size) => QuantGranularity::PerGroup { size },
None => QuantGranularity::PerChannel,
},
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
}
}
pub fn validate(&self) -> Result<(), QatError> {
if !self.format.is_integer() {
return Err(QatError::Invalid(
"QAT currently supports integer formats only (int8/int4/int3/int2)".into(),
));
}
self.scheme()
.validate()
.map_err(|message| QatError::Invalid(message))?;
if self.learnable_scales && self.group_size.is_some() {
return Err(QatError::Invalid(
"learnable QAT scales require per-channel (omit group_size); use fixed-scale QAT for groups".into(),
));
}
Ok(())
}
pub fn as_str(&self) -> String {
let mut parts = vec![self.format.as_str().to_string()];
if let Some(size) = self.group_size {
parts.push(format!("group:{size}"));
}
if self.learnable_scales {
parts.push("lsq".into());
}
parts.join("+")
}
}
impl<'de> Deserialize<'de> for QatConfig {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum QatConfigDe {
Format(WeightQuantization),
Full {
format: WeightQuantization,
#[serde(default)]
group_size: Option<usize>,
#[serde(default)]
learnable_scales: bool,
},
}
match QatConfigDe::deserialize(deserializer)? {
QatConfigDe::Format(format) => Ok(Self::from_format(format)),
QatConfigDe::Full {
format,
group_size,
learnable_scales,
} => Ok(Self {
format,
group_size,
learnable_scales,
}),
}
}
}
#[derive(Debug)]
pub enum QatError {
Invalid(String),
}
impl fmt::Display for QatError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => write!(formatter, "invalid QAT configuration: {message}"),
}
}
}
impl std::error::Error for QatError {}
#[derive(Clone, Debug)]
pub struct QatRuntime {
pub config: QatConfig,
pub scales: Option<Tensor>,
}
impl QatRuntime {
pub fn new(
config: QatConfig,
in_features: usize,
out_features: usize,
init_scales: Option<Vec<f32>>,
) -> Result<Self, QatError> {
config.validate()?;
let scales = if config.learnable_scales {
let scheme = config.scheme();
let count = scheme.scale_count(in_features, out_features);
let values = match init_scales {
Some(values) if values.len() == count => values,
Some(values) => {
return Err(QatError::Invalid(format!(
"expected {count} QAT scales, got {}",
values.len()
)));
}
None => vec![1.0; count],
};
Some(Tensor::new(values, vec![count], true))
} else {
None
};
Ok(Self { config, scales })
}
}
pub fn init_absmax_scales(
in_features: usize,
out_features: usize,
scheme: QuantScheme,
mut logical_value: impl FnMut(usize, usize) -> f32,
) -> Vec<f32> {
let max_q = scheme
.format
.int_maximum()
.expect("QAT scales require an integer format");
let count = scheme.scale_count(in_features, out_features);
let mut scales = vec![1.0f32; count];
for scale_index in 0..count {
let mut absmax = 0.0f32;
match scheme.granularity {
QuantGranularity::PerTensor => {
for input in 0..in_features {
for output in 0..out_features {
absmax = absmax.max(logical_value(input, output).abs());
}
}
}
QuantGranularity::PerChannel => {
let output = scale_index;
for input in 0..in_features {
absmax = absmax.max(logical_value(input, output).abs());
}
}
QuantGranularity::PerGroup { size } => {
let groups = in_features.div_ceil(size.max(1));
let output = scale_index / groups;
let group = scale_index % groups;
let start = group * size;
let end = (start + size).min(in_features);
for input in start..end {
absmax = absmax.max(logical_value(input, output).abs());
}
}
}
scales[scale_index] = if absmax > 0.0 { absmax / max_q } else { 1.0 };
}
scales
}
pub fn fake_quantize_storage(
in_features: usize,
out_features: usize,
transpose_weight: bool,
scheme: QuantScheme,
scales: &[f32],
mut logical_value: impl FnMut(usize, usize) -> f32,
) -> Vec<f32> {
let max_q = scheme
.format
.int_maximum()
.expect("fake-quant requires an integer format");
let mut storage = vec![0.0f32; in_features * out_features];
for input in 0..in_features {
for output in 0..out_features {
let scale_index = match scheme.granularity {
QuantGranularity::PerTensor => 0,
QuantGranularity::PerChannel => output,
QuantGranularity::PerGroup { size } => {
let groups = in_features.div_ceil(size.max(1));
output * groups + input / size.max(1)
}
};
let scale = scales[scale_index].max(1e-12);
let value = logical_value(input, output);
let q = (value / scale).round().clamp(-max_q, max_q);
let dequant = q * scale;
if transpose_weight {
storage[output * in_features + input] = dequant;
} else {
storage[input * out_features + output] = dequant;
}
}
}
storage
}
pub fn quantized_codes_storage(
in_features: usize,
out_features: usize,
transpose_weight: bool,
scheme: QuantScheme,
scales: &[f32],
mut logical_value: impl FnMut(usize, usize) -> f32,
) -> Vec<f32> {
let max_q = scheme
.format
.int_maximum()
.expect("fake-quant requires an integer format");
let mut storage = vec![0.0f32; in_features * out_features];
for input in 0..in_features {
for output in 0..out_features {
let scale_index = match scheme.granularity {
QuantGranularity::PerTensor => 0,
QuantGranularity::PerChannel => output,
QuantGranularity::PerGroup { size } => {
let groups = in_features.div_ceil(size.max(1));
output * groups + input / size.max(1)
}
};
let scale = scales[scale_index].max(1e-12);
let value = logical_value(input, output);
let q = (value / scale).round().clamp(-max_q, max_q);
if transpose_weight {
storage[output * in_features + input] = q;
} else {
storage[input * out_features + output] = q;
}
}
}
storage
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn deserialize_format_string_and_object() {
let from_str: QatConfig = serde_json::from_str("\"int4\"").unwrap();
assert_eq!(from_str.format, WeightQuantization::Int4);
assert!(!from_str.learnable_scales);
let from_obj: QatConfig = serde_json::from_str(
r#"{ "format": "int8", "group_size": 64, "learnable_scales": false }"#,
)
.unwrap();
assert_eq!(from_obj.group_size, Some(64));
}
#[test]
fn learnable_rejects_group_size() {
let config = QatConfig {
format: WeightQuantization::Int8,
group_size: Some(64),
learnable_scales: true,
};
assert!(config.validate().is_err());
}
}