use crate::weight_quant::WeightQuantization;
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ActivationQuantization {
Int8,
}
impl ActivationQuantization {
pub fn as_str(self) -> &'static str {
"int8"
}
pub fn bits(self) -> u32 {
8
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_lowercase().as_str() {
"int8" | "i8" | "8" | "a8" => Ok(Self::Int8),
other => Err(format!(
"unknown activation quantization '{other}'; expected int8"
)),
}
}
pub fn weight_format(self) -> WeightQuantization {
WeightQuantization::Int8
}
pub fn max_q(self) -> f32 {
127.0
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ActGranularity {
PerTensor,
PerToken,
}
impl ActGranularity {
pub fn as_str(self) -> &'static str {
match self {
Self::PerTensor => "tensor",
Self::PerToken => "token",
}
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_lowercase().as_str() {
"tensor" | "per-tensor" | "per_tensor" => Ok(Self::PerTensor),
"token" | "per-token" | "per_token" | "row" => Ok(Self::PerToken),
other => Err(format!(
"unknown activation granularity '{other}'; expected tensor|token"
)),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ActQuantMode {
Static,
Dynamic,
}
impl ActQuantMode {
pub fn as_str(self) -> &'static str {
match self {
Self::Static => "static",
Self::Dynamic => "dynamic",
}
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_lowercase().as_str() {
"static" | "offline" => Ok(Self::Static),
"dynamic" | "online" => Ok(Self::Dynamic),
other => Err(format!(
"unknown activation mode '{other}'; expected static|dynamic"
)),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ActQuantScheme {
pub format: ActivationQuantization,
pub granularity: ActGranularity,
pub mode: ActQuantMode,
}
impl ActQuantScheme {
pub fn int8_dynamic_per_token() -> Self {
Self {
format: ActivationQuantization::Int8,
granularity: ActGranularity::PerToken,
mode: ActQuantMode::Dynamic,
}
}
pub fn int8_dynamic_per_tensor() -> Self {
Self {
format: ActivationQuantization::Int8,
granularity: ActGranularity::PerTensor,
mode: ActQuantMode::Dynamic,
}
}
pub fn int8_static_per_tensor() -> Self {
Self {
format: ActivationQuantization::Int8,
granularity: ActGranularity::PerTensor,
mode: ActQuantMode::Static,
}
}
pub fn validate(self) -> Result<(), String> {
if self.mode == ActQuantMode::Static && self.granularity != ActGranularity::PerTensor {
return Err(
"static activation quantization only supports per-tensor granularity".into(),
);
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "UPPERCASE")]
pub enum InferenceQuantMode {
W8A8,
W4A8,
W4A16,
}
impl InferenceQuantMode {
pub fn as_str(self) -> &'static str {
match self {
Self::W8A8 => "W8A8",
Self::W4A8 => "W4A8",
Self::W4A16 => "W4A16",
}
}
pub fn parse(value: &str) -> Result<Self, String> {
match value.trim().to_ascii_uppercase().as_str() {
"W8A8" => Ok(Self::W8A8),
"W4A8" => Ok(Self::W4A8),
"W4A16" | "W4A32" => Ok(Self::W4A16),
other => Err(format!(
"unknown inference mode '{other}'; expected W8A8|W4A8|W4A16"
)),
}
}
pub fn weight_format(self) -> WeightQuantization {
match self {
Self::W8A8 => WeightQuantization::Int8,
Self::W4A8 | Self::W4A16 => WeightQuantization::Int4,
}
}
pub fn activation_scheme(self) -> Option<ActQuantScheme> {
match self {
Self::W8A8 | Self::W4A8 => Some(ActQuantScheme::int8_dynamic_per_token()),
Self::W4A16 => None,
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct ActQuantConfig {
pub scheme: ActQuantScheme,
pub static_scale: Option<f32>,
}
impl ActQuantConfig {
pub fn dynamic(scheme: ActQuantScheme) -> Result<Self, String> {
let mut scheme = scheme;
scheme.mode = ActQuantMode::Dynamic;
scheme.validate()?;
Ok(Self {
scheme,
static_scale: None,
})
}
pub fn static_per_tensor(scale: f32) -> Self {
Self {
scheme: ActQuantScheme::int8_static_per_tensor(),
static_scale: Some(if scale > 0.0 && scale.is_finite() {
scale
} else {
1.0
}),
}
}
pub fn validate(&self) -> Result<(), String> {
self.scheme.validate()?;
if self.scheme.mode == ActQuantMode::Static && self.static_scale.is_none() {
return Err("static activation quant requires a calibrated scale".into());
}
Ok(())
}
}
#[derive(Clone, Debug)]
pub struct QuantizedActivations {
pub codes: Vec<i8>,
pub scales: Vec<f32>,
pub rows: usize,
pub features: usize,
pub granularity: ActGranularity,
}
impl QuantizedActivations {
pub fn scale_for_row(&self, row: usize) -> f32 {
match self.granularity {
ActGranularity::PerTensor => self.scales[0],
ActGranularity::PerToken => self.scales[row],
}
}
pub fn dequantized(&self) -> Vec<f32> {
let mut out = vec![0.0; self.rows * self.features];
for row in 0..self.rows {
let scale = self.scale_for_row(row);
let offset = row * self.features;
for feature in 0..self.features {
out[offset + feature] = self.codes[offset + feature] as f32 * scale;
}
}
out
}
}
pub fn int8_scale(values: &[f32]) -> f32 {
let absmax = values.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
if absmax > 0.0 && absmax.is_finite() {
absmax / ActivationQuantization::Int8.max_q()
} else {
1.0
}
}
pub fn quantize_activations(
values: &[f32],
rows: usize,
features: usize,
config: &ActQuantConfig,
) -> Result<QuantizedActivations, String> {
config.validate()?;
if values.len() != rows * features {
return Err(format!(
"activation length {} != rows ({rows}) * features ({features})",
values.len()
));
}
let max_q = config.scheme.format.max_q();
let mut codes = vec![0i8; values.len()];
let scales = match (config.scheme.mode, config.scheme.granularity) {
(ActQuantMode::Static, ActGranularity::PerTensor) => {
let scale = config.static_scale.unwrap_or(1.0).max(1e-12);
for (index, &value) in values.iter().enumerate() {
codes[index] = (value / scale).round().clamp(-max_q, max_q) as i8;
}
vec![scale]
}
(ActQuantMode::Dynamic, ActGranularity::PerTensor) => {
let scale = int8_scale(values).max(1e-12);
for (index, &value) in values.iter().enumerate() {
codes[index] = (value / scale).round().clamp(-max_q, max_q) as i8;
}
vec![scale]
}
(ActQuantMode::Dynamic, ActGranularity::PerToken) => {
let mut row_scales = vec![1.0; rows];
for row in 0..rows {
let start = row * features;
let end = start + features;
let scale = int8_scale(&values[start..end]).max(1e-12);
row_scales[row] = scale;
for feature in 0..features {
let value = values[start + feature];
codes[start + feature] = (value / scale).round().clamp(-max_q, max_q) as i8;
}
}
row_scales
}
(ActQuantMode::Static, ActGranularity::PerToken) => {
return Err("static per-token activation quantization is unsupported".into());
}
};
Ok(QuantizedActivations {
codes,
scales,
rows,
features,
granularity: config.scheme.granularity,
})
}
pub fn update_absmax(current: &mut f32, values: &[f32]) {
for &value in values {
let abs = value.abs();
if abs > *current {
*current = abs;
}
}
}
pub fn scale_from_absmax(absmax: f32) -> f32 {
if absmax > 0.0 && absmax.is_finite() {
absmax / ActivationQuantization::Int8.max_q()
} else {
1.0
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dynamic_per_token_roundtrip() {
let rows = 2;
let features = 4;
let values = vec![0.5, -1.0, 0.25, 0.0, 2.0, -2.0, 1.0, -0.5];
let config = ActQuantConfig::dynamic(ActQuantScheme::int8_dynamic_per_token()).unwrap();
let quantized = quantize_activations(&values, rows, features, &config).unwrap();
let recon = quantized.dequantized();
for (original, restored) in values.iter().zip(recon.iter()) {
assert!(
(original - restored).abs() < 0.02,
"{original} vs {restored}"
);
}
}
#[test]
fn static_per_tensor_uses_calibrated_scale() {
let values = vec![1.0f32, -1.0, 0.5, -0.5];
let scale = int8_scale(&values);
let config = ActQuantConfig::static_per_tensor(scale);
let quantized = quantize_activations(&values, 1, 4, &config).unwrap();
assert_eq!(quantized.scales.len(), 1);
assert!((quantized.scales[0] - scale).abs() < 1e-6);
}
#[test]
fn inference_mode_parse() {
assert_eq!(
InferenceQuantMode::parse("w8a8").unwrap(),
InferenceQuantMode::W8A8
);
assert!(InferenceQuantMode::W4A16.activation_scheme().is_none());
assert!(InferenceQuantMode::W8A8.activation_scheme().is_some());
}
}