use crate::checkpoint::{
save_inference_checkpoint, CheckpointQuantization, QuantizedLayerEntry, QUANTIZATION_SCHEMA_VERSION,
};
use crate::layers::{QuantGranularity, QuantScheme, ScaleMethod, WeightQuantization};
use crate::model::CausalLM;
use crate::quality::{evaluate_held_out_text, evaluate_kl_held_out_text};
use crate::tokenizer::BpeTokenizer;
use crate::trainer::{load_checkpoint, TrainerError};
use serde::{Deserialize, Serialize};
use std::error::Error;
use std::fmt;
use std::fs;
use std::path::{Path, PathBuf};
pub const DEFAULT_EXCLUDE: &[&str] = &["embed", "lm_head"];
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum QuantizeMethod {
Rtn,
Gptq,
Awq,
#[serde(rename = "smoothquant")]
SmoothQuant,
Hqq,
Spqr,
#[serde(rename = "quip")]
Quip,
Aqlm,
}
impl QuantizeMethod {
pub fn as_str(self) -> &'static str {
match self {
Self::Rtn => "rtn",
Self::Gptq => "gptq",
Self::Awq => "awq",
Self::SmoothQuant => "smoothquant",
Self::Hqq => "hqq",
Self::Spqr => "spqr",
Self::Quip => "quip",
Self::Aqlm => "aqlm",
}
}
pub fn requires_calibration(self) -> bool {
matches!(
self,
Self::Gptq | Self::Awq | Self::SmoothQuant
)
}
pub fn parse(value: &str) -> Result<Self, QuantizeError> {
match value.trim().to_ascii_lowercase().as_str() {
"rtn" | "round-to-nearest" | "round_to_nearest" => Ok(Self::Rtn),
"gptq" => Ok(Self::Gptq),
"awq" => Ok(Self::Awq),
"smoothquant" | "smooth_quant" | "smooth-quant" => Ok(Self::SmoothQuant),
"hqq" => Ok(Self::Hqq),
"spqr" => Ok(Self::Spqr),
"quip" | "quip#" | "quip_sharp" | "quip-sharp" => Ok(Self::Quip),
"aqlm" => Ok(Self::Aqlm),
other => Err(QuantizeError::Invalid(format!(
"unknown quantize method '{other}'; expected rtn|gptq|awq|smoothquant|hqq|spqr|quip|aqlm"
))),
}
}
}
#[derive(Clone, Debug)]
pub struct QuantizeOptions {
pub method: QuantizeMethod,
pub format: Option<WeightQuantization>,
pub bits: u32,
pub group_size: Option<usize>,
pub symmetric: bool,
pub scale_method: ScaleMethod,
pub percentile: f32,
pub calibration: Option<PathBuf>,
pub calibration_sequences: usize,
pub calibration_batch_size: usize,
pub calibration_seed: u64,
pub smoothquant_alpha: Option<f32>,
pub exclude: Vec<String>,
pub include: Option<Vec<String>>,
pub eval_corpus: Option<PathBuf>,
pub eval_sequence_length: Option<usize>,
pub eval_batch_size: usize,
pub max_ppl_delta: Option<f32>,
pub max_kl: Option<f32>,
pub json_out: Option<PathBuf>,
pub inference_mode: Option<crate::act_quant::InferenceQuantMode>,
pub activation_format: Option<crate::act_quant::ActivationQuantization>,
pub activation_granularity: crate::act_quant::ActGranularity,
pub activation_mode: crate::act_quant::ActQuantMode,
pub kv_format: Option<crate::kv_quant::KvQuantization>,
pub kv_granularity: crate::kv_quant::KvGranularity,
}
impl Default for QuantizeOptions {
fn default() -> Self {
Self {
method: QuantizeMethod::Rtn,
format: None,
bits: 8,
group_size: None,
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
calibration: None,
calibration_sequences: crate::calibration::DEFAULT_CALIBRATION_SEQUENCES,
calibration_batch_size: 1,
calibration_seed: crate::calibration::DEFAULT_CALIBRATION_SEED,
smoothquant_alpha: None,
exclude: DEFAULT_EXCLUDE.iter().map(|value| (*value).to_string()).collect(),
include: None,
eval_corpus: None,
eval_sequence_length: None,
eval_batch_size: 1,
max_ppl_delta: None,
max_kl: None,
json_out: None,
inference_mode: None,
activation_format: None,
activation_granularity: crate::act_quant::ActGranularity::PerToken,
activation_mode: crate::act_quant::ActQuantMode::Dynamic,
kv_format: None,
kv_granularity: crate::kv_quant::KvGranularity::PerHead,
}
}
}
impl QuantizeOptions {
pub fn weight_format(&self) -> Result<WeightQuantization, QuantizeError> {
if let Some(mode) = self.inference_mode {
return Ok(mode.weight_format());
}
if let Some(format) = self.format {
return Ok(format);
}
match self.bits {
8 => Ok(WeightQuantization::Int8),
4 => Ok(WeightQuantization::Int4),
3 => Ok(WeightQuantization::Int3),
2 => Ok(WeightQuantization::Int2),
other => Err(QuantizeError::Invalid(format!(
"unsupported --bits {other}; use 8|4|3|2 or --format nf4|fp8_e4m3|fp8_e5m2"
))),
}
}
pub fn resolved_activation_format(
&self,
) -> Option<crate::act_quant::ActivationQuantization> {
if let Some(mode) = self.inference_mode {
return mode
.activation_scheme()
.map(|scheme| scheme.format);
}
self.activation_format
}
pub fn activation_scheme(&self) -> Result<Option<crate::act_quant::ActQuantScheme>, QuantizeError> {
let Some(format) = self.resolved_activation_format() else {
return Ok(None);
};
let mut scheme = crate::act_quant::ActQuantScheme {
format,
granularity: self.activation_granularity,
mode: self.activation_mode,
};
if let Some(mode) = self.inference_mode {
if let Some(default_scheme) = mode.activation_scheme() {
if self.activation_format.is_none() {
scheme = default_scheme;
}
}
}
scheme.validate().map_err(QuantizeError::Invalid)?;
Ok(Some(scheme))
}
pub fn kv_scheme(&self) -> Option<crate::kv_quant::KvQuantScheme> {
self.kv_format.map(|format| crate::kv_quant::KvQuantScheme {
format,
granularity: self.kv_granularity,
})
}
pub fn requires_act_calibration(&self) -> bool {
matches!(
self.activation_scheme().ok().flatten().map(|s| s.mode),
Some(crate::act_quant::ActQuantMode::Static)
)
}
pub fn granularity(&self) -> Result<QuantGranularity, QuantizeError> {
match self.group_size {
None | Some(0) => Ok(QuantGranularity::PerChannel),
Some(1) => Ok(QuantGranularity::PerTensor),
Some(size) => Ok(QuantGranularity::PerGroup { size }),
}
}
pub fn scheme(&self) -> Result<QuantScheme, QuantizeError> {
let scheme = QuantScheme {
format: self.weight_format()?,
granularity: self.granularity()?,
symmetric: self.symmetric,
scale_method: self.scale_method,
percentile: self.percentile,
};
scheme.validate().map_err(QuantizeError::Invalid)?;
Ok(scheme)
}
pub fn validate(&self) -> Result<(), QuantizeError> {
let _ = self.scheme()?;
let _ = self.activation_scheme()?;
if self.method.requires_calibration() && self.calibration.is_none() {
return Err(QuantizeError::Invalid(format!(
"method={} requires --calibration <path>",
self.method.as_str()
)));
}
if self.requires_act_calibration() && self.calibration.is_none() {
return Err(QuantizeError::Invalid(
"static activation quantization requires --calibration <path>".into(),
));
}
if let Some(alpha) = self.smoothquant_alpha {
if !(0.0..=1.0).contains(&alpha) || !alpha.is_finite() {
return Err(QuantizeError::Invalid(
"--smoothquant-alpha must be in [0, 1]".into(),
));
}
}
if self.max_ppl_delta.is_some() || self.max_kl.is_some() {
if self.eval_corpus.is_none() {
return Err(QuantizeError::Invalid(
"--max-ppl-delta / --max-kl require --eval-corpus".into(),
));
}
}
Ok(())
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct QuantizeLayerReport {
pub name: String,
pub quantized: bool,
pub bits: Option<u32>,
pub packed_bytes: Option<usize>,
pub fp32_bytes: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct QuantizeQualityReport {
pub corpus_path: String,
pub fp32_perplexity: Option<f32>,
pub quantized_perplexity: Option<f32>,
pub kl_vs_fp32: Option<f32>,
pub tokens_compared: Option<usize>,
pub error: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct QuantizeReport {
pub schema_version: u32,
pub method: String,
pub format: String,
pub bits: u32,
pub group_size: Option<usize>,
pub granularity: String,
pub scale_method: String,
pub symmetric: bool,
pub excluded: Vec<String>,
pub included: Option<Vec<String>>,
pub input_path: String,
pub output_path: String,
pub layers_total: usize,
pub layers_quantized: usize,
pub before_bytes: usize,
pub after_bytes: usize,
pub compression_ratio: f64,
pub layers: Vec<QuantizeLayerReport>,
pub quality: Option<QuantizeQualityReport>,
pub warnings: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub inference_mode: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub activation: Option<ActivationQuantReport>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub kv_cache: Option<KvQuantReport>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ActivationQuantReport {
pub format: String,
pub granularity: String,
pub mode: String,
pub layers: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KvQuantReport {
pub format: String,
pub granularity: String,
}
#[derive(Debug)]
pub enum QuantizeError {
Invalid(String),
Io(std::io::Error),
Trainer(TrainerError),
Json(serde_json::Error),
Quality(String),
}
impl fmt::Display for QuantizeError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => write!(formatter, "{message}"),
Self::Io(error) => write!(formatter, "{error}"),
Self::Trainer(error) => write!(formatter, "{error}"),
Self::Json(error) => write!(formatter, "{error}"),
Self::Quality(message) => write!(formatter, "{message}"),
}
}
}
impl Error for QuantizeError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Trainer(error) => Some(error),
Self::Json(error) => Some(error),
_ => None,
}
}
}
impl From<std::io::Error> for QuantizeError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<TrainerError> for QuantizeError {
fn from(error: TrainerError) -> Self {
Self::Trainer(error)
}
}
impl From<serde_json::Error> for QuantizeError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
pub fn should_quantize_module(name: &str, options: &QuantizeOptions) -> bool {
if let Some(include) = &options.include {
if !include.is_empty() && !glob_list_matches(name, include) {
return false;
}
}
!glob_list_matches(name, &options.exclude)
}
fn glob_list_matches(name: &str, patterns: &[String]) -> bool {
patterns
.iter()
.any(|pattern| !pattern.is_empty() && glob_match(pattern, name))
}
pub fn glob_match(pattern: &str, text: &str) -> bool {
if pattern == "*" || pattern == "all" {
return true;
}
if !pattern.contains('*') {
return text.contains(pattern);
}
let parts: Vec<&str> = pattern.split('*').collect();
if parts.iter().all(|part| part.is_empty()) {
return true;
}
let mut cursor = 0usize;
for (index, part) in parts.iter().enumerate() {
if part.is_empty() {
continue;
}
if index == 0 && !pattern.starts_with('*') {
if !text[cursor..].starts_with(part) {
return false;
}
cursor += part.len();
continue;
}
match text[cursor..].find(part) {
Some(offset) => cursor += offset + part.len(),
None => return false,
}
}
if !pattern.ends_with('*') {
if let Some(last) = parts.iter().rev().find(|part| !part.is_empty()) {
return text.ends_with(last);
}
}
true
}
fn parse_csv_patterns(value: &str) -> Vec<String> {
value
.split(',')
.map(str::trim)
.filter(|part| !part.is_empty())
.map(str::to_string)
.collect()
}
pub fn apply_quantization(
model: &mut CausalLM,
options: &QuantizeOptions,
) -> Result<Vec<String>, QuantizeError> {
options.validate()?;
let scheme = options.scheme()?;
let mut quantized = Vec::new();
for (name, linear) in model.named_linears_mut() {
if should_quantize_module(&name, options) {
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
quantized.push(name);
}
}
Ok(quantized)
}
fn fp32_linear_bytes(linear: &crate::layers::Linear) -> usize {
linear.in_features * linear.out_features * 4
}
fn model_weight_bytes(model: &CausalLM) -> usize {
model
.named_parameters()
.into_iter()
.map(|(_, parameter)| parameter.numel() * 4)
.sum()
}
fn quantized_model_bytes(model: &CausalLM) -> usize {
let mut total = 0usize;
let mut covered = 0usize;
for (_, linear) in model.named_linears() {
if let Some(quant) = linear.quantized_weight() {
total = total.saturating_add(quant.storage_bytes());
covered = covered.saturating_add(fp32_linear_bytes(linear));
if let Some(bias) = &linear.bias {
total = total.saturating_add(bias.numel() * 4);
covered = covered.saturating_add(bias.numel() * 4);
}
}
}
let base = model_weight_bytes(model);
base.saturating_sub(covered).saturating_add(total)
}
pub fn build_quantization_metadata_for_model(
model: &CausalLM,
options: &QuantizeOptions,
quantized_names: &[String],
) -> Result<CheckpointQuantization, QuantizeError> {
build_quantization_metadata(model, options, quantized_names, &[])
}
fn build_quantization_metadata(
model: &CausalLM,
options: &QuantizeOptions,
quantized_names: &[String],
activation_scales: &[(String, f32)],
) -> Result<CheckpointQuantization, QuantizeError> {
let mut layers = Vec::new();
for name in quantized_names {
let linear = model
.named_linears()
.into_iter()
.find(|(module, _)| module == name)
.map(|(_, linear)| linear)
.ok_or_else(|| QuantizeError::Invalid(format!("missing quantized module {name}")))?;
let quant = linear.quantized_weight().ok_or_else(|| {
QuantizeError::Invalid(format!("module {name} was not quantized"))
})?;
let scheme = quant.scheme();
let zero_points_name = if scheme.symmetric {
None
} else {
Some(format!("{name}.weight.qzeros"))
};
layers.push(QuantizedLayerEntry {
name: name.clone(),
format: quant.format(),
in_features: quant.in_features(),
out_features: quant.out_features(),
packed_name: format!("{name}.weight.qpacked"),
scales_name: format!("{name}.weight.qscales"),
zero_points_name,
group_size: scheme.granularity.group_size(),
symmetric: scheme.symmetric,
scale_method: Some(scheme.scale_method.as_str().into()),
granularity: Some(scheme.granularity.as_str()),
});
}
let scheme = options.scheme()?;
let activation = options.activation_scheme()?.map(|act_scheme| {
let layers = if act_scheme.mode == crate::act_quant::ActQuantMode::Static {
activation_scales
.iter()
.map(|(name, scale)| crate::checkpoint::ActivationLayerEntry {
name: name.clone(),
scale: Some(*scale),
scales_name: Some(format!("{name}.act.qscale")),
})
.collect()
} else {
quantized_names
.iter()
.map(|name| crate::checkpoint::ActivationLayerEntry {
name: name.clone(),
scale: None,
scales_name: None,
})
.collect()
};
crate::checkpoint::CheckpointActivationQuant {
format: act_scheme.format.as_str().into(),
granularity: act_scheme.granularity.as_str().into(),
mode: act_scheme.mode.as_str().into(),
layers,
}
});
let kv_cache = options.kv_scheme().map(|kv| crate::checkpoint::CheckpointKvQuant {
format: kv.format.as_str().into(),
granularity: kv.granularity.as_str().into(),
});
Ok(CheckpointQuantization {
schema_version: QUANTIZATION_SCHEMA_VERSION,
method: options.method.as_str().into(),
bits: scheme.format.bits(),
group_size: scheme.granularity.group_size(),
symmetric: scheme.symmetric,
zero_points: !scheme.symmetric,
scale_method: Some(scheme.scale_method.as_str().into()),
granularity: Some(scheme.granularity.as_str()),
format: Some(scheme.format.as_str().into()),
inference_only: true,
excluded: options.exclude.clone(),
included: options.include.clone(),
layers,
activation,
kv_cache,
})
}
fn collect_static_activation_scales(
model: &mut CausalLM,
options: &QuantizeOptions,
tokenizer: &BpeTokenizer,
) -> Result<Vec<(String, f32)>, QuantizeError> {
let act_scheme = options
.activation_scheme()?
.filter(|scheme| scheme.mode == crate::act_quant::ActQuantMode::Static);
let Some(act_scheme) = act_scheme else {
return Ok(Vec::new());
};
let calibration_path = options.calibration.as_ref().ok_or_else(|| {
QuantizeError::Invalid("static activations require --calibration".into())
})?;
let sequence_length = options
.eval_sequence_length
.unwrap_or(model.config.max_sequence_length.min(32))
.min(model.config.max_sequence_length)
.max(2);
let cal_options = crate::calibration::CalibrationOptions {
path: calibration_path.clone(),
sequence_length,
max_sequences: options.calibration_sequences,
batch_size: options.calibration_batch_size,
seed: options.calibration_seed,
};
let calibration = crate::calibration::load_calibration(
&cal_options,
tokenizer,
model.config.vocab_size,
)
.map_err(|error| QuantizeError::Invalid(error.to_string()))?;
for (name, linear) in model.named_linears() {
if should_quantize_module(&name, options) {
let _ = name;
linear.begin_input_capture(crate::layers::InputCaptureMode::AbsMax);
}
}
crate::calibration::run_calibration_forwards(
&calibration,
options.calibration_batch_size,
|input| {
let _ = model.forward(input);
Ok(())
},
)
.map_err(|error| QuantizeError::Invalid(error.to_string()))?;
let mut scales = Vec::new();
for (name, linear) in model.named_linears() {
if !should_quantize_module(&name, options) {
linear.clear_input_capture();
continue;
}
if let Some(absmax) = linear.end_absmax_capture() {
let tensor_absmax = absmax.into_iter().fold(0.0f32, f32::max);
let scale = crate::act_quant::scale_from_absmax(tensor_absmax);
scales.push((name, scale));
}
}
let _ = act_scheme;
Ok(scales)
}
fn apply_activation_and_kv_quant(
model: &mut CausalLM,
options: &QuantizeOptions,
activation_scales: &[(String, f32)],
) -> Result<(), QuantizeError> {
if let Some(act_scheme) = options.activation_scheme()? {
let scale_map: std::collections::BTreeMap<&str, f32> = activation_scales
.iter()
.map(|(name, scale)| (name.as_str(), *scale))
.collect();
for (name, linear) in model.named_linears_mut() {
if !should_quantize_module(&name, options) {
continue;
}
let config = match act_scheme.mode {
crate::act_quant::ActQuantMode::Static => {
let scale = scale_map.get(name.as_str()).copied().unwrap_or(1.0);
crate::act_quant::ActQuantConfig::static_per_tensor(scale)
}
crate::act_quant::ActQuantMode::Dynamic => {
crate::act_quant::ActQuantConfig::dynamic(act_scheme)
.map_err(QuantizeError::Invalid)?
}
};
linear
.enable_activation_quant(config)
.map_err(QuantizeError::Invalid)?;
}
}
if let Some(kv) = options.kv_scheme() {
model.enable_kv_cache_quant(kv);
}
Ok(())
}
fn layer_reports(model: &CausalLM, _options: &QuantizeOptions) -> Vec<QuantizeLayerReport> {
model
.named_linears()
.into_iter()
.map(|(name, linear)| {
let fp32_bytes = fp32_linear_bytes(linear);
if let Some(quant) = linear.quantized_weight() {
QuantizeLayerReport {
name,
quantized: true,
bits: Some(quant.format().bits()),
packed_bytes: Some(quant.storage_bytes()),
fp32_bytes,
}
} else {
QuantizeLayerReport {
name,
quantized: false,
bits: None,
packed_bytes: None,
fp32_bytes,
}
}
})
.collect()
}
fn optional_quality(
fp32_model: &CausalLM,
quantized_model: &CausalLM,
tokenizer: &BpeTokenizer,
options: &QuantizeOptions,
) -> Option<QuantizeQualityReport> {
let corpus = options.eval_corpus.as_ref()?;
let text = match fs::read_to_string(corpus) {
Ok(text) => text,
Err(error) => {
return Some(QuantizeQualityReport {
corpus_path: corpus.display().to_string(),
fp32_perplexity: None,
quantized_perplexity: None,
kl_vs_fp32: None,
tokens_compared: None,
error: Some(error.to_string()),
});
}
};
let sequence_length = options
.eval_sequence_length
.unwrap_or(32)
.min(fp32_model.config.max_sequence_length)
.max(2);
let batch_size = options.eval_batch_size.max(1);
let fp32_ppl = evaluate_held_out_text(
fp32_model,
tokenizer,
&text,
sequence_length,
batch_size,
None,
);
let quant_ppl = evaluate_held_out_text(
quantized_model,
tokenizer,
&text,
sequence_length,
batch_size,
None,
);
let kl = evaluate_kl_held_out_text(
quantized_model,
fp32_model,
tokenizer,
&text,
sequence_length,
batch_size,
);
match (fp32_ppl, quant_ppl, kl) {
(Ok((fp32, _)), Ok((quant, _)), Ok(kl_report)) => Some(QuantizeQualityReport {
corpus_path: corpus.display().to_string(),
fp32_perplexity: Some(fp32.perplexity),
quantized_perplexity: Some(quant.perplexity),
kl_vs_fp32: Some(kl_report.mean_kl as f32),
tokens_compared: Some(kl_report.tokens_compared),
error: None,
}),
(fp32, quant, kl) => {
let error = fp32
.err()
.map(|error| error.to_string())
.or_else(|| quant.err().map(|error| error.to_string()))
.or_else(|| kl.err().map(|error| error.to_string()));
Some(QuantizeQualityReport {
corpus_path: corpus.display().to_string(),
fp32_perplexity: None,
quantized_perplexity: None,
kl_vs_fp32: None,
tokens_compared: None,
error,
})
}
}
}
pub fn quantize_checkpoint(
input: impl AsRef<Path>,
output: impl AsRef<Path>,
options: &QuantizeOptions,
) -> Result<QuantizeReport, QuantizeError> {
options.validate()?;
let input = input.as_ref();
let output = output.as_ref();
let mut warnings = Vec::new();
if options.calibration.is_some()
&& matches!(
options.method,
QuantizeMethod::Rtn
| QuantizeMethod::Hqq
| QuantizeMethod::Spqr
| QuantizeMethod::Quip
| QuantizeMethod::Aqlm
)
&& !options.requires_act_calibration()
{
warnings.push(
"--calibration is ignored for method without activation calibration (use gptq|awq|smoothquant or --activation-mode static)".into(),
);
}
if matches!(options.method, QuantizeMethod::Hqq) && options.symmetric {
return Err(QuantizeError::Invalid(
"method=hqq requires --asymmetric".into(),
));
}
let loaded = load_checkpoint(input)?;
let before_bytes = model_weight_bytes(&loaded.model);
let mut fp32_model = loaded.model;
let tokenizer = loaded.tokenizer;
let need_reference = options.eval_corpus.is_some()
|| options.max_ppl_delta.is_some()
|| options.max_kl.is_some();
let reference = if need_reference {
Some(clone_model_weights(&fp32_model)?)
} else {
None
};
let quantized_names = match options.method {
QuantizeMethod::Rtn => apply_quantization(&mut fp32_model, options)?,
QuantizeMethod::Hqq
| QuantizeMethod::Spqr
| QuantizeMethod::Quip
| QuantizeMethod::Aqlm => {
let scheme = options.scheme()?;
let select = |name: &str| should_quantize_module(name, options);
let result = match options.method {
QuantizeMethod::Hqq => {
crate::advanced_quant::apply_hqq(&mut fp32_model, scheme, &select)
}
QuantizeMethod::Spqr => {
crate::advanced_quant::apply_spqr(&mut fp32_model, scheme, &select)
}
QuantizeMethod::Quip => {
crate::advanced_quant::apply_quip(&mut fp32_model, scheme, &select)
}
QuantizeMethod::Aqlm => {
crate::advanced_quant::apply_aqlm(&mut fp32_model, scheme, &select)
}
_ => unreachable!(),
};
result.map_err(|error| QuantizeError::Invalid(error.to_string()))?
}
method => {
let calibration_path = options.calibration.as_ref().ok_or_else(|| {
QuantizeError::Invalid(format!(
"method={} requires --calibration",
method.as_str()
))
})?;
let sequence_length = options
.eval_sequence_length
.unwrap_or(fp32_model.config.max_sequence_length.min(32))
.min(fp32_model.config.max_sequence_length)
.max(2);
let cal_options = crate::calibration::CalibrationOptions {
path: calibration_path.clone(),
sequence_length,
max_sequences: options.calibration_sequences,
batch_size: options.calibration_batch_size,
seed: options.calibration_seed,
};
let calibration = crate::calibration::load_calibration(
&cal_options,
&tokenizer,
fp32_model.config.vocab_size,
)
.map_err(|error| QuantizeError::Invalid(error.to_string()))?;
let select = |name: &str| should_quantize_module(name, options);
let mut ptq = crate::ptq::PtqConfig::new(options.scheme()?, &select);
ptq.batch_size = options.calibration_batch_size;
if let Some(alpha) = options.smoothquant_alpha {
ptq.smoothquant_alpha = alpha;
}
let result = match method {
QuantizeMethod::Gptq => {
crate::ptq::apply_gptq(&mut fp32_model, &ptq, &calibration)
}
QuantizeMethod::Awq => crate::ptq::apply_awq(&mut fp32_model, &ptq, &calibration),
QuantizeMethod::SmoothQuant => {
crate::ptq::apply_smoothquant(&mut fp32_model, &ptq, &calibration)
}
QuantizeMethod::Rtn
| QuantizeMethod::Hqq
| QuantizeMethod::Spqr
| QuantizeMethod::Quip
| QuantizeMethod::Aqlm => unreachable!(),
};
result.map_err(|error| QuantizeError::Invalid(error.to_string()))?
}
};
let activation_scales = collect_static_activation_scales(&mut fp32_model, options, &tokenizer)?;
apply_activation_and_kv_quant(&mut fp32_model, options, &activation_scales)?;
let metadata =
build_quantization_metadata(&fp32_model, options, &quantized_names, &activation_scales)?;
save_inference_checkpoint(output, &fp32_model, &tokenizer, &metadata)?;
let after_bytes = quantized_model_bytes(&fp32_model);
let compression_ratio = if after_bytes == 0 {
0.0
} else {
before_bytes as f64 / after_bytes as f64
};
let quality = reference.as_ref().and_then(|fp32| {
optional_quality(fp32, &fp32_model, &tokenizer, options)
});
if let Some(quality) = &quality {
enforce_quality_gates(quality, options)?;
}
let scheme = options.scheme()?;
let activation_report = options.activation_scheme()?.map(|act| ActivationQuantReport {
format: act.format.as_str().into(),
granularity: act.granularity.as_str().into(),
mode: act.mode.as_str().into(),
layers: fp32_model
.named_linears()
.into_iter()
.filter(|(_, linear)| linear.activation_quant().is_some())
.count(),
});
let kv_report = options.kv_scheme().map(|kv| KvQuantReport {
format: kv.format.as_str().into(),
granularity: kv.granularity.as_str().into(),
});
let report = QuantizeReport {
schema_version: QUANTIZATION_SCHEMA_VERSION,
method: options.method.as_str().into(),
format: scheme.format.as_str().into(),
bits: scheme.format.bits(),
group_size: scheme.granularity.group_size(),
granularity: scheme.granularity.as_str(),
scale_method: scheme.scale_method.as_str().into(),
symmetric: scheme.symmetric,
excluded: options.exclude.clone(),
included: options.include.clone(),
input_path: input.display().to_string(),
output_path: output.display().to_string(),
layers_total: fp32_model.named_linears().len(),
layers_quantized: quantized_names.len(),
before_bytes,
after_bytes,
compression_ratio,
layers: layer_reports(&fp32_model, options),
quality,
warnings,
inference_mode: options.inference_mode.map(|mode| mode.as_str().into()),
activation: activation_report,
kv_cache: kv_report,
};
if let Some(path) = &options.json_out {
write_quantize_json(&report, path)?;
}
Ok(report)
}
fn enforce_quality_gates(
quality: &QuantizeQualityReport,
options: &QuantizeOptions,
) -> Result<(), QuantizeError> {
if let Some(error) = &quality.error {
if options.max_ppl_delta.is_some() || options.max_kl.is_some() {
return Err(QuantizeError::Quality(format!(
"quality gate requested but eval failed: {error}"
)));
}
return Ok(());
}
if let Some(max_delta) = options.max_ppl_delta {
match (quality.fp32_perplexity, quality.quantized_perplexity) {
(Some(fp32), Some(quant)) => {
let delta = quant - fp32;
if delta > max_delta {
return Err(QuantizeError::Quality(format!(
"perplexity delta {delta:.4} (fp32={fp32:.4}, quant={quant:.4}) exceeds --max-ppl-delta {max_delta}"
)));
}
}
_ => {
return Err(QuantizeError::Quality(
"quality gate --max-ppl-delta requires successful PPL eval".into(),
));
}
}
}
if let Some(max_kl) = options.max_kl {
match quality.kl_vs_fp32 {
Some(kl) if kl > max_kl => {
return Err(QuantizeError::Quality(format!(
"KL {kl:.6} exceeds --max-kl {max_kl}"
)));
}
Some(_) => {}
None => {
return Err(QuantizeError::Quality(
"quality gate --max-kl requires successful KL eval".into(),
));
}
}
}
Ok(())
}
fn clone_model_weights(model: &CausalLM) -> Result<CausalLM, QuantizeError> {
let clone = CausalLM::new(model.config.clone()).map_err(|error| {
QuantizeError::Invalid(format!("failed to clone model config: {error}"))
})?;
let source = model.named_parameters();
let destination = clone.named_parameters();
if source.len() != destination.len() {
return Err(QuantizeError::Invalid(
"model clone parameter count mismatch".into(),
));
}
for ((source_name, source), (dest_name, dest)) in source.iter().zip(destination.iter()) {
if source_name != dest_name {
return Err(QuantizeError::Invalid(format!(
"model clone parameter order mismatch: {source_name} vs {dest_name}"
)));
}
dest.set_data(source.data());
}
Ok(clone)
}
pub fn format_quantize_report(report: &QuantizeReport) -> String {
let mut lines = Vec::new();
lines.push(format!(
"quantize method={} format={} bits={} gran={} scale={} sym={} layers={}/{} compression={:.2}x",
report.method,
report.format,
report.bits,
report.granularity,
report.scale_method,
report.symmetric,
report.layers_quantized,
report.layers_total,
report.compression_ratio
));
lines.push(format!(
" size: {} → {} bytes",
report.before_bytes, report.after_bytes
));
lines.push(format!(
" exclude={} include={}",
report.excluded.join(","),
report
.included
.as_ref()
.map(|values| values.join(","))
.unwrap_or_else(|| "(all except exclude)".into())
));
if let Some(mode) = &report.inference_mode {
lines.push(format!(" inference_mode={mode}"));
}
if let Some(activation) = &report.activation {
lines.push(format!(
" activations format={} gran={} mode={} layers={}",
activation.format, activation.granularity, activation.mode, activation.layers
));
}
if let Some(kv) = &report.kv_cache {
lines.push(format!(
" kv_cache format={} gran={}",
kv.format, kv.granularity
));
}
for warning in &report.warnings {
lines.push(format!(" warning: {warning}"));
}
if let Some(quality) = &report.quality {
if let Some(error) = &quality.error {
lines.push(format!(" quality error: {error}"));
} else {
lines.push(format!(
" quality corpus={} fp32_ppl={:?} quant_ppl={:?} kl_vs_fp32={:?}",
quality.corpus_path,
quality.fp32_perplexity,
quality.quantized_perplexity,
quality.kl_vs_fp32
));
}
}
for layer in report.layers.iter().filter(|layer| layer.quantized) {
lines.push(format!(
" ✓ {} bits={} packed={} (fp32={})",
layer.name,
layer.bits.unwrap_or(0),
layer.packed_bytes.unwrap_or(0),
layer.fp32_bytes
));
}
lines.push(format!(" wrote {}", report.output_path));
lines.join("\n") + "\n"
}
pub fn write_quantize_json(report: &QuantizeReport, path: impl AsRef<Path>) -> Result<(), QuantizeError> {
let bytes = serde_json::to_vec_pretty(report)?;
fs::write(path, bytes)?;
Ok(())
}
pub fn parse_pattern_list(value: &str) -> Vec<String> {
parse_csv_patterns(value)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::optimizer::{AdamW, AdamWConfig};
use crate::tokenizer::FIRST_MERGE_ID;
use crate::trainer::{save_checkpoint, LearningRateSchedule, TrainerConfig};
fn tiny_config(vocab: usize) -> crate::model::CausalLmConfig {
crate::model::CausalLmConfig {
vocab_size: vocab,
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 8,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: vocab - 1,
seed: 7,
initializer_std: 0.02,
tie_word_embeddings: false,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
}
}
fn temp_dir(label: &str) -> PathBuf {
let path = std::env::temp_dir().join(format!(
"oxidelm-quantize-{label}-{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
fs::create_dir_all(&path).unwrap();
path
}
#[test]
fn glob_match_supports_substrings_and_stars() {
assert!(glob_match("lm_head", "lm_head"));
assert!(glob_match("q_proj", "layers.0.attention.q_proj"));
assert!(glob_match("*q_proj*", "layers.0.attention.q_proj"));
assert!(!glob_match("lm_head", "layers.0.attention.q_proj"));
}
#[test]
fn default_policy_skips_lm_head() {
let options = QuantizeOptions::default();
assert!(!should_quantize_module("lm_head", &options));
assert!(should_quantize_module(
"layers.0.attention.q_proj",
&options
));
}
#[test]
fn quantize_round_trips_inference_checkpoint() {
let directory = temp_dir("roundtrip");
let input = directory.join("fp32.bin");
let output = directory.join("int8.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config(tokenizer.vocab_size());
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
&input,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
0,
0,
0,
)
.unwrap();
let options = QuantizeOptions {
bits: 8,
..QuantizeOptions::default()
};
let report = quantize_checkpoint(&input, &output, &options).unwrap();
assert!(report.layers_quantized > 0);
assert!(report.compression_ratio > 1.0);
assert!(!report.layers.iter().any(|layer| layer.name == "lm_head" && layer.quantized));
let loaded = load_checkpoint(&output).unwrap();
assert!(loaded
.model
.named_linears()
.iter()
.any(|(_, linear)| linear.quantized_weight().is_some()));
assert!(loaded
.model
.named_linears()
.iter()
.find(|(name, _)| name == "lm_head")
.unwrap()
.1
.quantized_weight()
.is_none());
}
#[test]
fn calibration_ptq_methods_produce_quantized_checkpoint() {
let directory = temp_dir("cal-ptq");
let input = directory.join("fp32.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config(tokenizer.vocab_size());
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
&input,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
0,
0,
0,
)
.unwrap();
let cal_path = directory.join("cal.txt");
let mut text = String::new();
for _ in 0..80 {
text.push_str("hello world hello rust\n");
}
fs::write(&cal_path, text).unwrap();
for method in [
QuantizeMethod::Gptq,
QuantizeMethod::Awq,
QuantizeMethod::SmoothQuant,
] {
let output = directory.join(format!("{}.bin", method.as_str()));
let options = QuantizeOptions {
method,
bits: 8,
calibration: Some(cal_path.clone()),
calibration_sequences: 4,
calibration_seed: 0,
eval_sequence_length: Some(4),
..QuantizeOptions::default()
};
let report = quantize_checkpoint(&input, &output, &options).unwrap();
assert!(
report.layers_quantized > 0,
"{} quantized no layers",
method.as_str()
);
let loaded = load_checkpoint(&output).unwrap();
assert!(loaded
.model
.named_linears()
.iter()
.any(|(_, linear)| linear.quantized_weight().is_some()));
}
}
#[test]
fn advanced_methods_produce_quantized_checkpoint() {
let directory = temp_dir("adv-ptq");
let input = directory.join("fp32.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config(tokenizer.vocab_size());
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
save_checkpoint(
&input,
&model,
&optimizer,
&tokenizer,
&LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
},
&TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
},
0,
0,
0,
)
.unwrap();
for (method, asymmetric) in [
(QuantizeMethod::Hqq, true),
(QuantizeMethod::Spqr, false),
(QuantizeMethod::Quip, false),
(QuantizeMethod::Aqlm, false),
] {
let output = directory.join(format!("{}.bin", method.as_str()));
let options = QuantizeOptions {
method,
bits: 4,
group_size: Some(32),
symmetric: !asymmetric,
..QuantizeOptions::default()
};
let report = quantize_checkpoint(&input, &output, &options).unwrap();
assert!(
report.layers_quantized > 0,
"{} quantized no layers",
method.as_str()
);
}
}
#[test]
fn quality_gate_rejects_impossible_ppl_delta() {
let directory = temp_dir("gate");
let input = directory.join("fp32.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config(tokenizer.vocab_size());
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
&input,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
0,
0,
0,
)
.unwrap();
let corpus = directory.join("eval.txt");
let mut text = String::new();
for _ in 0..40 {
text.push_str("hello world hello rust\n");
}
fs::write(&corpus, text).unwrap();
let output = directory.join("int8.bin");
let options = QuantizeOptions {
bits: 8,
eval_corpus: Some(corpus),
eval_sequence_length: Some(4),
max_ppl_delta: Some(-1.0),
..QuantizeOptions::default()
};
let err = quantize_checkpoint(&input, &output, &options).unwrap_err();
assert!(
matches!(err, QuantizeError::Quality(_)),
"expected quality gate error, got {err}"
);
}
#[test]
fn w8a8_and_kv_quant_round_trip() {
let directory = temp_dir("w8a8-kv");
let input = directory.join("fp32.bin");
let output = directory.join("w8a8.bin");
let tokenizer = BpeTokenizer::train_from_bytes_with_min_frequency(
b"hello world hello rust",
FIRST_MERGE_ID + 8,
1,
)
.unwrap();
let mut config = tiny_config(tokenizer.vocab_size());
config.eos_token_id = tokenizer.special_tokens.eos;
let model = CausalLM::new(config).unwrap();
let optimizer = AdamW::new(&model.parameters(), AdamWConfig::default()).unwrap();
let schedule = LearningRateSchedule {
base_learning_rate: 3e-4,
minimum_learning_rate: 3e-5,
warmup_steps: 1,
total_steps: 10,
};
let trainer_config = TrainerConfig {
batch_size: 1,
sequence_length: 4,
max_gradient_norm: 1.0,
log_every: 1,
gradient_accumulation_steps: 1,
mixed_precision: crate::amp::MixedPrecisionConfig::disabled(),
activation_checkpointing: false,
data_parallel_workers: 1,
low_memory: Default::default(),
};
save_checkpoint(
&input,
&model,
&optimizer,
&tokenizer,
&schedule,
&trainer_config,
0,
0,
0,
)
.unwrap();
let options = QuantizeOptions {
inference_mode: Some(crate::act_quant::InferenceQuantMode::W8A8),
kv_format: Some(crate::kv_quant::KvQuantization::Int8),
kv_granularity: crate::kv_quant::KvGranularity::PerHead,
..QuantizeOptions::default()
};
let report = quantize_checkpoint(&input, &output, &options).unwrap();
assert_eq!(report.inference_mode.as_deref(), Some("W8A8"));
assert!(report.activation.is_some());
assert!(report.kv_cache.is_some());
let loaded = load_checkpoint(&output).unwrap();
assert!(loaded
.model
.named_linears()
.iter()
.filter(|(name, _)| name != "lm_head")
.all(|(_, linear)| {
linear.quantized_weight().is_some() && linear.activation_quant().is_some()
}));
assert_eq!(
loaded.model.kv_cache_quant().map(|s| s.format),
Some(crate::kv_quant::KvQuantization::Int8)
);
let ids = crate::IndexTensor::new(vec![1, 2, 3], vec![1, 3]);
let mut cache = crate::model::KvCache::new(loaded.model.layers.len());
let logits = loaded.model.forward_cached(&ids, &mut cache).unwrap();
assert!(logits.data().iter().all(|v| v.is_finite()));
assert!(cache.layers[0].as_ref().unwrap().is_quantized());
}
#[test]
fn w4a8_linear_matches_fp32_roughly() {
let mut linear = crate::layers::Linear::new(8, 4, 11);
linear.quantize_weight(WeightQuantization::Int4);
linear
.enable_activation_quant(
crate::act_quant::ActQuantConfig::dynamic(
crate::act_quant::ActQuantScheme::int8_dynamic_per_token(),
)
.unwrap(),
)
.unwrap();
let input = crate::Tensor::new(
(0..16).map(|i| (i as f32) * 0.05 - 0.4).collect(),
vec![2, 8],
false,
);
let output = linear.forward(&input);
assert!(output.data().iter().all(|v| v.is_finite()));
assert_eq!(output.shape(), vec![2, 4]);
}
}