use crate::calibration::{run_calibration_forwards, CalibrationData};
use crate::layers::{InputCaptureMode, Linear, QuantGranularity, QuantScheme};
use crate::model::CausalLM;
use crate::weight_quant::pack_weights;
use crate::IndexTensor;
use std::collections::HashMap;
const GPTQ_DAMP: f32 = 0.01;
const AWQ_GRID: usize = 20;
const DEFAULT_SMOOTHQUANT_ALPHA: f32 = 0.5;
#[derive(Debug)]
pub enum PtqError {
Invalid(String),
}
impl std::fmt::Display for PtqError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Invalid(message) => write!(formatter, "{message}"),
}
}
}
impl std::error::Error for PtqError {}
pub struct PtqConfig<'a> {
pub scheme: QuantScheme,
pub select: &'a dyn Fn(&str) -> bool,
pub batch_size: usize,
pub smoothquant_alpha: f32,
}
impl<'a> PtqConfig<'a> {
pub fn new(scheme: QuantScheme, select: &'a dyn Fn(&str) -> bool) -> Self {
Self {
scheme,
select,
batch_size: 1,
smoothquant_alpha: DEFAULT_SMOOTHQUANT_ALPHA,
}
}
}
pub fn apply_gptq(
model: &mut CausalLM,
config: &PtqConfig<'_>,
calibration: &CalibrationData,
) -> Result<Vec<String>, PtqError> {
let scheme = config.scheme;
if !scheme.format.is_integer() {
return Err(PtqError::Invalid(
"GPTQ currently supports integer weight formats (int8/int4/int3/int2)".into(),
));
}
let names = selected_names(model, config.select);
let groups = layer_module_groups(&names);
let batch_size = config.batch_size.max(1);
let block_size = scheme.granularity.group_size().unwrap_or(128).max(1);
for group in groups {
for name in &group {
if let Some(linear) = find_linear_mut(model, name) {
linear.begin_input_capture(InputCaptureMode::Hessian);
}
}
run_forwards(model, calibration, batch_size)?;
for name in &group {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear module {name}")))?;
let hessian = linear
.end_hessian_capture()
.ok_or_else(|| PtqError::Invalid(format!("missing Hessian capture for {name}")))?;
if hessian.rows == 0 {
return Err(PtqError::Invalid(format!(
"GPTQ captured zero activation rows for {name}"
)));
}
let mut weights = logical_weight_matrix(linear);
gptq_update_weights(
&mut weights,
linear.out_features,
linear.in_features,
&hessian.matrix,
scheme,
block_size,
)?;
write_logical_weights(linear, &weights);
pack_and_install(linear, scheme)?;
}
}
Ok(names)
}
pub fn apply_awq(
model: &mut CausalLM,
config: &PtqConfig<'_>,
calibration: &CalibrationData,
) -> Result<Vec<String>, PtqError> {
let scheme = config.scheme;
let names = selected_names(model, config.select);
let batch_size = config.batch_size.max(1);
for name in &names {
if let Some(linear) = find_linear_mut(model, name) {
linear.begin_input_capture(InputCaptureMode::AbsMax);
}
}
run_forwards(model, calibration, batch_size)?;
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear module {name}")))?;
let act_max = linear
.end_absmax_capture()
.ok_or_else(|| PtqError::Invalid(format!("missing absmax capture for {name}")))?;
let mut weights = logical_weight_matrix(linear);
let scales = search_awq_scales(
&weights,
linear.out_features,
linear.in_features,
&act_max,
scheme,
)?;
for input in 0..linear.in_features {
let scale = scales[input].max(1e-8);
for output in 0..linear.out_features {
weights[output * linear.in_features + input] *= scale;
}
}
let packed = pack_weights(
linear.in_features,
linear.out_features,
scheme,
|input, output| weights[output * linear.in_features + input],
)
.map_err(PtqError::Invalid)?;
for input in 0..linear.in_features {
let scale = scales[input].max(1e-8);
for output in 0..linear.out_features {
let dequant = packed.dequantized_value(input, output);
weights[output * linear.in_features + input] = dequant / scale;
}
}
write_logical_weights(linear, &weights);
pack_and_install(linear, scheme)?;
}
Ok(names)
}
pub fn apply_smoothquant(
model: &mut CausalLM,
config: &PtqConfig<'_>,
calibration: &CalibrationData,
) -> Result<Vec<String>, PtqError> {
let scheme = config.scheme;
let names = selected_names(model, config.select);
let batch_size = config.batch_size.max(1);
let alpha = config.smoothquant_alpha;
for name in &names {
if let Some(linear) = find_linear_mut(model, name) {
linear.begin_input_capture(InputCaptureMode::AbsMax);
}
}
run_forwards(model, calibration, batch_size)?;
let mut absmax_by_name = HashMap::new();
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear module {name}")))?;
let act_max = linear
.end_absmax_capture()
.ok_or_else(|| PtqError::Invalid(format!("missing absmax capture for {name}")))?;
absmax_by_name.insert(name.clone(), act_max);
}
let layer_count = model.layers.len();
for layer_index in 0..layer_count {
let q_name = format!("layers.{layer_index}.attention.q_proj");
let k_name = format!("layers.{layer_index}.attention.k_proj");
let v_name = format!("layers.{layer_index}.attention.v_proj");
let gate_name = format!("layers.{layer_index}.feed_forward.gate_proj");
let up_name = format!("layers.{layer_index}.feed_forward.up_proj");
if names.iter().any(|name| name == &q_name) {
let scales =
smooth_scales_for_modules(model, &[&q_name, &k_name, &v_name], &absmax_by_name, alpha)?;
scale_rms_norm(&mut model.layers[layer_index].attention_norm, &scales, true)?;
for name in [&q_name, &k_name, &v_name] {
if names.iter().any(|n| n == name) {
apply_weight_channel_scales(model, name, &scales)?;
}
}
}
if names.iter().any(|name| name == &gate_name || name == &up_name) {
let scales =
smooth_scales_for_modules(model, &[&gate_name, &up_name], &absmax_by_name, alpha)?;
scale_rms_norm(&mut model.layers[layer_index].ffn_norm, &scales, true)?;
for name in [&gate_name, &up_name] {
if names.iter().any(|n| n == name) {
apply_weight_channel_scales(model, name, &scales)?;
}
}
}
for name in [
format!("layers.{layer_index}.attention.o_proj"),
format!("layers.{layer_index}.feed_forward.down_proj"),
] {
if names.iter().any(|n| n == &name) {
let scales = smooth_scales_for_modules(model, &[&name], &absmax_by_name, alpha)?;
apply_weight_channel_scales(model, &name, &scales)?;
}
}
}
if names.iter().any(|name| name == "lm_head") {
let scales = smooth_scales_for_modules(model, &["lm_head"], &absmax_by_name, alpha)?;
apply_weight_channel_scales(model, "lm_head", &scales)?;
}
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear module {name}")))?;
pack_and_install(linear, scheme)?;
}
Ok(names)
}
fn selected_names(model: &CausalLM, select: &dyn Fn(&str) -> bool) -> Vec<String> {
model
.named_linears()
.into_iter()
.filter(|(name, _)| select(name))
.map(|(name, _)| name)
.collect()
}
fn find_linear_mut<'a>(model: &'a mut CausalLM, name: &str) -> Option<&'a mut Linear> {
model
.named_linears_mut()
.into_iter()
.find(|(module, _)| module == name)
.map(|(_, linear)| linear)
}
fn pack_and_install(linear: &mut Linear, scheme: QuantScheme) -> Result<(), PtqError> {
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
Ok(())
}
fn run_forwards(
model: &CausalLM,
calibration: &CalibrationData,
batch_size: usize,
) -> Result<(), PtqError> {
run_calibration_forwards(calibration, batch_size, |input: &IndexTensor| {
model
.forward(input)
.map(|_| ())
.map_err(|error| error.to_string())
})
.map_err(|error| PtqError::Invalid(error.to_string()))
}
fn layer_module_groups(names: &[String]) -> Vec<Vec<String>> {
let mut groups: Vec<(usize, Vec<String>)> = Vec::new();
let mut other = Vec::new();
for name in names {
if let Some(rest) = name.strip_prefix("layers.") {
if let Some((index_str, _)) = rest.split_once('.') {
if let Ok(index) = index_str.parse::<usize>() {
if let Some(group) = groups.iter_mut().find(|(idx, _)| *idx == index) {
group.1.push(name.clone());
} else {
groups.push((index, vec![name.clone()]));
}
continue;
}
}
}
other.push(name.clone());
}
groups.sort_by_key(|(index, _)| *index);
let mut ordered: Vec<Vec<String>> = groups.into_iter().map(|(_, names)| names).collect();
if !other.is_empty() {
ordered.push(other);
}
ordered
}
fn smooth_scales_for_modules(
model: &mut CausalLM,
module_names: &[&str],
absmax_by_name: &HashMap<String, Vec<f32>>,
alpha: f32,
) -> Result<Vec<f32>, PtqError> {
let mut act_max: Option<Vec<f32>> = None;
let mut weight_max: Option<Vec<f32>> = None;
for name in module_names {
if !absmax_by_name.contains_key(*name) {
continue;
}
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear module {name}")))?;
let captured = absmax_by_name.get(*name).unwrap();
act_max = Some(match act_max {
None => captured.clone(),
Some(mut current) => {
for (lhs, rhs) in current.iter_mut().zip(captured.iter()) {
*lhs = (*lhs).max(*rhs);
}
current
}
});
let w_max = weight_absmax_per_input(linear);
weight_max = Some(match weight_max {
None => w_max,
Some(mut current) => {
for (lhs, rhs) in current.iter_mut().zip(w_max.iter()) {
*lhs = (*lhs).max(*rhs);
}
current
}
});
}
let act_max = act_max.ok_or_else(|| PtqError::Invalid("SmoothQuant missing activation stats".into()))?;
let weight_max =
weight_max.ok_or_else(|| PtqError::Invalid("SmoothQuant missing weight stats".into()))?;
Ok(smooth_scales(&act_max, &weight_max, alpha))
}
fn smooth_scales(act_max: &[f32], weight_max: &[f32], alpha: f32) -> Vec<f32> {
act_max
.iter()
.zip(weight_max.iter())
.map(|(&a, &w)| {
let a = a.max(1e-5);
let w = w.max(1e-5);
(a.powf(alpha) / w.powf(1.0 - alpha)).max(1e-5)
})
.collect()
}
fn weight_absmax_per_input(linear: &Linear) -> Vec<f32> {
let values = linear.weight.shared_data();
let mut maxes = vec![0.0f32; linear.in_features];
for input in 0..linear.in_features {
let mut absmax = 0.0f32;
for output in 0..linear.out_features {
absmax = absmax.max(linear.logical_weight_value(&values, input, output).abs());
}
maxes[input] = absmax;
}
maxes
}
fn apply_weight_channel_scales(
model: &mut CausalLM,
name: &str,
scales: &[f32],
) -> Result<(), PtqError> {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear module {name}")))?;
if scales.len() != linear.in_features {
return Err(PtqError::Invalid(format!(
"scale length {} != in_features {}",
scales.len(),
linear.in_features
)));
}
let mut weights = logical_weight_matrix(linear);
for input in 0..linear.in_features {
let scale = scales[input];
for output in 0..linear.out_features {
weights[output * linear.in_features + input] *= scale;
}
}
write_logical_weights(linear, &weights);
Ok(())
}
fn scale_rms_norm(
norm: &mut crate::norm::RmsNorm,
scales: &[f32],
inverse: bool,
) -> Result<(), PtqError> {
if scales.len() != norm.hidden_size {
return Err(PtqError::Invalid(
"SmoothQuant RMSNorm scale width mismatch".into(),
));
}
let mut values = norm.weight.data();
for (value, scale) in values.iter_mut().zip(scales.iter()) {
let factor = if inverse {
1.0 / scale.max(1e-8)
} else {
*scale
};
*value *= factor;
}
norm.weight.set_data(values);
Ok(())
}
fn logical_weight_matrix(linear: &Linear) -> Vec<f32> {
let values = linear.weight.shared_data();
let mut matrix = vec![0.0; linear.in_features * linear.out_features];
for input in 0..linear.in_features {
for output in 0..linear.out_features {
matrix[output * linear.in_features + input] =
linear.logical_weight_value(&values, input, output);
}
}
matrix
}
fn write_logical_weights(linear: &mut Linear, matrix: &[f32]) {
let mut storage = vec![0.0; linear.in_features * linear.out_features];
for input in 0..linear.in_features {
for output in 0..linear.out_features {
let value = matrix[output * linear.in_features + input];
if linear.transpose_weight {
storage[output * linear.in_features + input] = value;
} else {
storage[input * linear.out_features + output] = value;
}
}
}
linear.weight.set_data(storage);
}
fn quantize_scalar(value: f32, scale: f32, zp: f32, scheme: QuantScheme) -> f32 {
let max_q = scheme.format.int_maximum().unwrap_or(127.0);
let q = if scheme.symmetric {
(value / scale).round().clamp(-max_q, max_q)
} else {
(value / scale + zp).round().clamp(-max_q, max_q)
};
if scheme.symmetric {
q * scale
} else {
(q - zp) * scale
}
}
fn group_scale_zp(values: &[f32], scheme: QuantScheme) -> (f32, f32) {
if values.is_empty() {
return (1.0, 0.0);
}
let max_q = scheme.format.int_maximum().unwrap_or(127.0);
if scheme.symmetric {
let absmax = values.iter().map(|v| v.abs()).fold(0.0f32, f32::max);
let scale = if absmax > 0.0 { absmax / max_q } else { 1.0 };
(scale, 0.0)
} else {
let min_v = values.iter().copied().fold(f32::INFINITY, f32::min);
let max_v = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let scale = ((max_v - min_v) / (2.0 * max_q)).max(1e-8);
let zp = (-max_q - min_v / scale).round().clamp(-max_q, max_q);
(scale, zp)
}
}
fn gptq_update_weights(
weights: &mut [f32],
out_features: usize,
in_features: usize,
hessian: &[f32],
scheme: QuantScheme,
block_size: usize,
) -> Result<(), PtqError> {
let mut h = hessian.to_vec();
let mut diag_mean = 0.0f32;
for i in 0..in_features {
diag_mean += h[i * in_features + i].abs();
}
diag_mean /= in_features.max(1) as f32;
let damp = GPTQ_DAMP * diag_mean.max(1e-5);
for i in 0..in_features {
h[i * in_features + i] += damp;
}
let hinv = cholesky_inverse(&h, in_features).map_err(PtqError::Invalid)?;
gptq_core(weights, out_features, in_features, &hinv, scheme, block_size)
}
fn gptq_core(
weights: &mut [f32],
out_features: usize,
in_features: usize,
hinv: &[f32],
scheme: QuantScheme,
block_size: usize,
) -> Result<(), PtqError> {
let group = match scheme.granularity {
QuantGranularity::PerGroup { size } => size.max(1),
QuantGranularity::PerChannel | QuantGranularity::PerTensor => in_features.max(1),
};
let mut col = 0usize;
while col < in_features {
let block_end = (col + block_size).min(in_features);
let mut errors = vec![0.0f32; out_features * (block_end - col)];
for i in col..block_end {
let group_start = (i / group) * group;
let group_end = (group_start + group).min(in_features);
let mut group_vals = Vec::with_capacity((group_end - group_start) * out_features);
for g in group_start..group_end {
for row in 0..out_features {
group_vals.push(weights[row * in_features + g]);
}
}
let (scale, zp) = group_scale_zp(&group_vals, scheme);
let inv_diag = hinv[i * in_features + i].abs().max(1e-8);
for row in 0..out_features {
let idx = row * in_features + i;
let w = weights[idx];
let q = quantize_scalar(w, scale, zp, scheme);
let err = (w - q) / inv_diag;
weights[idx] = q;
errors[row * (block_end - col) + (i - col)] = err;
for j in (i + 1)..block_end {
weights[row * in_features + j] -= err * hinv[i * in_features + j];
}
}
}
if block_end < in_features {
for row in 0..out_features {
for i in col..block_end {
let err = errors[row * (block_end - col) + (i - col)];
for j in block_end..in_features {
weights[row * in_features + j] -= err * hinv[i * in_features + j];
}
}
}
}
col = block_end;
}
Ok(())
}
fn cholesky_inverse(matrix: &[f32], n: usize) -> Result<Vec<f32>, String> {
let mut l = vec![0.0f32; n * n];
for i in 0..n {
for j in 0..=i {
let mut sum = matrix[i * n + j];
for k in 0..j {
sum -= l[i * n + k] * l[j * n + k];
}
if i == j {
if sum <= 1e-12 {
return Err(format!("GPTQ Hessian not SPD at diagonal {i}"));
}
l[i * n + j] = sum.sqrt();
} else {
let diag = l[j * n + j];
if diag.abs() < 1e-12 {
return Err(format!("GPTQ Hessian Cholesky failed at ({i},{j})"));
}
l[i * n + j] = sum / diag;
}
}
}
let mut inv = vec![0.0f32; n * n];
for col in 0..n {
let mut y = vec![0.0f32; n];
for i in 0..n {
let mut sum = if i == col { 1.0 } else { 0.0 };
for k in 0..i {
sum -= l[i * n + k] * y[k];
}
y[i] = sum / l[i * n + i];
}
let mut x = vec![0.0f32; n];
for i in (0..n).rev() {
let mut sum = y[i];
for k in (i + 1)..n {
sum -= l[k * n + i] * x[k];
}
x[i] = sum / l[i * n + i];
}
for row in 0..n {
inv[row * n + col] = x[row];
}
}
Ok(inv)
}
fn search_awq_scales(
weights: &[f32],
out_features: usize,
in_features: usize,
act_max: &[f32],
scheme: QuantScheme,
) -> Result<Vec<f32>, PtqError> {
let mut best_scales = vec![1.0f32; in_features];
let mut best_error = f32::INFINITY;
let mut w_max = vec![0.0f32; in_features];
for input in 0..in_features {
for output in 0..out_features {
w_max[input] = w_max[input].max(weights[output * in_features + input].abs());
}
}
for step in 0..=AWQ_GRID {
let ratio = step as f32 / AWQ_GRID as f32;
let scales: Vec<f32> = (0..in_features)
.map(|i| {
let a = act_max[i].max(1e-5);
let w = w_max[i].max(1e-5);
(a.powf(ratio) / w.powf(1.0 - ratio)).clamp(1e-4, 1e4)
})
.collect();
let error = awq_quant_error(weights, out_features, in_features, &scales, scheme);
if error < best_error {
best_error = error;
best_scales = scales;
}
}
Ok(best_scales)
}
fn awq_quant_error(
weights: &[f32],
out_features: usize,
in_features: usize,
scales: &[f32],
scheme: QuantScheme,
) -> f32 {
let mut scaled = weights.to_vec();
for input in 0..in_features {
let s = scales[input];
for output in 0..out_features {
scaled[output * in_features + input] *= s;
}
}
let packed = match pack_weights(in_features, out_features, scheme, |input, output| {
scaled[output * in_features + input]
}) {
Ok(packed) => packed,
Err(_) => return f32::INFINITY,
};
let mut error = 0.0f32;
for input in 0..in_features {
let s = scales[input].max(1e-8);
for output in 0..out_features {
let original = weights[output * in_features + input];
let restored = packed.dequantized_value(input, output) / s;
let diff = original - restored;
error += diff * diff;
}
}
error
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layers::{ScaleMethod, WeightQuantization};
#[test]
fn cholesky_inverse_recovers_identity_product() {
let matrix = vec![4.0, 1.0, 1.0, 3.0];
let inv = cholesky_inverse(&matrix, 2).unwrap();
let mut product = vec![0.0; 4];
for i in 0..2 {
for j in 0..2 {
product[i * 2 + j] =
matrix[i * 2 + 0] * inv[0 * 2 + j] + matrix[i * 2 + 1] * inv[1 * 2 + j];
}
}
assert!((product[0] - 1.0).abs() < 1e-4);
assert!(product[1].abs() < 1e-4);
assert!(product[2].abs() < 1e-4);
assert!((product[3] - 1.0).abs() < 1e-4);
}
#[test]
fn gptq_core_keeps_weights_finite() {
let out = 4usize;
let inn = 8usize;
let mut weights: Vec<f32> = (0..out * inn)
.map(|i| ((i % 7) as f32 - 3.0) * 0.1)
.collect();
let mut h = vec![0.0f32; inn * inn];
for i in 0..inn {
h[i * inn + i] = 1.0;
}
let hinv = cholesky_inverse(&h, inn).unwrap();
let scheme = QuantScheme {
format: WeightQuantization::Int8,
granularity: QuantGranularity::PerChannel,
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
};
gptq_core(&mut weights, out, inn, &hinv, scheme, 4).unwrap();
assert!(weights.iter().all(|v| v.is_finite()));
}
}