use crate::layers::{Linear, QuantScheme};
use crate::model::CausalLM;
use crate::ptq::PtqError;
use crate::weight_quant::pack_weights_with_scales;
const HQQ_ITERS: usize = 12;
const HQQ_BETA: f32 = 1e2;
const SPQR_OUTLIER_FRAC: f32 = 0.01;
const AQLM_CODEBOOKS: usize = 2;
const AQLM_CODES: usize = 16;
pub fn apply_hqq(
model: &mut CausalLM,
scheme: QuantScheme,
select: &dyn Fn(&str) -> bool,
) -> Result<Vec<String>, PtqError> {
if !scheme.format.is_integer() {
return Err(PtqError::Invalid(
"HQQ currently supports integer weight formats".into(),
));
}
if scheme.symmetric {
return Err(PtqError::Invalid(
"HQQ requires asymmetric quantization (--asymmetric)".into(),
));
}
let names = selected_names(model, select);
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear {name}")))?;
let matrix = logical_weight_matrix(linear);
let packed = hqq_pack(
linear.in_features,
linear.out_features,
scheme,
&matrix,
)
.map_err(PtqError::Invalid)?;
linear
.set_quantized_weight(crate::layers::QuantizedLinearWeight::from_packed(packed))
.map_err(PtqError::Invalid)?;
linear.materialize_dequantized_master();
}
Ok(names)
}
pub fn apply_spqr(
model: &mut CausalLM,
scheme: QuantScheme,
select: &dyn Fn(&str) -> bool,
) -> Result<Vec<String>, PtqError> {
let names = selected_names(model, select);
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear {name}")))?;
let mut matrix = logical_weight_matrix(linear);
let outliers = extract_outliers(&mut matrix, SPQR_OUTLIER_FRAC);
write_logical_weights(linear, &matrix);
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
let mut restored = logical_weight_matrix(linear);
for (index, value) in outliers {
restored[index] = value;
}
write_logical_weights(linear, &restored);
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
}
Ok(names)
}
pub fn apply_quip(
model: &mut CausalLM,
scheme: QuantScheme,
select: &dyn Fn(&str) -> bool,
) -> Result<Vec<String>, PtqError> {
let names = selected_names(model, select);
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear {name}")))?;
let matrix = logical_weight_matrix(linear);
let transformed = hadamard_rows(&matrix, linear.out_features, linear.in_features);
write_logical_weights(linear, &transformed);
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
let quantized = logical_weight_matrix(linear);
let folded = hadamard_rows(&quantized, linear.out_features, linear.in_features);
write_logical_weights(linear, &folded);
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
}
Ok(names)
}
pub fn apply_aqlm(
model: &mut CausalLM,
scheme: QuantScheme,
select: &dyn Fn(&str) -> bool,
) -> Result<Vec<String>, PtqError> {
let names = selected_names(model, select);
for name in &names {
let linear = find_linear_mut(model, name)
.ok_or_else(|| PtqError::Invalid(format!("missing linear {name}")))?;
let matrix = logical_weight_matrix(linear);
let approx = aqlm_approximate(&matrix, linear.in_features);
write_logical_weights(linear, &approx);
linear.quantize_weight_with(scheme);
linear.materialize_dequantized_master();
}
Ok(names)
}
fn hqq_pack(
in_features: usize,
out_features: usize,
scheme: QuantScheme,
matrix_out_in: &[f32],
) -> Result<crate::weight_quant::PackedLinearWeight, String> {
let scale_count = scheme.scale_count(in_features, out_features);
let mut scales = vec![1.0f32; scale_count];
let mut zero_points = vec![0.0f32; scale_count];
let max_q = scheme.format.int_maximum().unwrap_or(7.0);
let min_q = -max_q;
for scale_index in 0..scale_count {
let values = collect_group(in_features, out_features, scheme, scale_index, matrix_out_in);
if values.is_empty() {
continue;
}
let mut min_v = values.iter().copied().fold(f32::INFINITY, f32::min);
let mut max_v = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
if !min_v.is_finite() || !max_v.is_finite() || (max_v - min_v).abs() < 1e-12 {
scales[scale_index] = 1.0;
zero_points[scale_index] = 0.0;
continue;
}
let mut scale = ((max_v - min_v) / (max_q - min_q)).max(1e-8);
let mut zp = (min_q - min_v / scale).clamp(min_q, max_q);
for _ in 0..HQQ_ITERS {
let mut w_sum = 0.0f32;
let mut w_x = 0.0f32;
let mut w_q = 0.0f32;
let mut w_qq = 0.0f32;
for &x in &values {
let q = (x / scale + zp).round().clamp(min_q, max_q);
let recon = (q - zp) * scale;
let residual = (x - recon).abs();
let weight = 1.0 / (1.0 + HQQ_BETA * residual);
w_sum += weight;
w_x += weight * x;
w_q += weight * q;
w_qq += weight * q * q;
}
if w_sum < 1e-12 || w_qq < 1e-12 {
break;
}
zp = ((w_q / w_sum) - (w_x / (w_sum * scale))).clamp(min_q, max_q);
let mut num = 0.0f32;
let mut den = 0.0f32;
for &x in &values {
let q = (x / scale + zp).round().clamp(min_q, max_q);
let recon = (q - zp) * scale;
let residual = (x - recon).abs();
let weight = 1.0 / (1.0 + HQQ_BETA * residual);
num += weight * x * (q - zp);
den += weight * (q - zp) * (q - zp);
}
if den > 1e-12 {
scale = (num / den).abs().max(1e-8);
}
min_v = (min_q - zp) * scale;
max_v = (max_q - zp) * scale;
let _ = (min_v, max_v);
}
scales[scale_index] = scale;
zero_points[scale_index] = zp;
}
pack_weights_with_scales(
in_features,
out_features,
scheme,
&scales,
Some(&zero_points),
|input, output| matrix_out_in[output * in_features + input],
)
}
fn collect_group(
in_features: usize,
out_features: usize,
scheme: QuantScheme,
scale_index: usize,
matrix_out_in: &[f32],
) -> Vec<f32> {
let mut values = Vec::new();
match scheme.granularity {
crate::layers::QuantGranularity::PerTensor => {
values.extend_from_slice(matrix_out_in);
}
crate::layers::QuantGranularity::PerChannel => {
let output = scale_index;
for input in 0..in_features {
values.push(matrix_out_in[output * in_features + input]);
}
}
crate::layers::QuantGranularity::PerGroup { size } => {
let size = size.max(1);
let groups = in_features.div_ceil(size);
let output = scale_index / groups;
let group = scale_index % groups;
let start = group * size;
let end = (start + size).min(in_features);
for input in start..end {
values.push(matrix_out_in[output * in_features + input]);
}
}
}
let _ = out_features;
values
}
fn extract_outliers(matrix: &mut [f32], frac: f32) -> Vec<(usize, f32)> {
if matrix.is_empty() {
return Vec::new();
}
let keep = ((matrix.len() as f32) * frac.clamp(0.0, 0.25)).ceil() as usize;
let keep = keep.max(1).min(matrix.len());
let mut order: Vec<usize> = (0..matrix.len()).collect();
order.sort_by(|&a, &b| {
matrix[b]
.abs()
.partial_cmp(&matrix[a].abs())
.unwrap_or(std::cmp::Ordering::Equal)
});
let mut outliers = Vec::with_capacity(keep);
let median = {
let mut sorted = matrix.to_vec();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
sorted[sorted.len() / 2]
};
for &index in order.iter().take(keep) {
outliers.push((index, matrix[index]));
matrix[index] = median;
}
outliers
}
fn next_pow2(value: usize) -> usize {
value.next_power_of_two().max(1)
}
fn hadamard_rows(matrix: &[f32], rows: usize, cols: usize) -> Vec<f32> {
let width = next_pow2(cols);
let scale = 1.0 / (width as f32).sqrt();
let mut out = vec![0.0; rows * cols];
for row in 0..rows {
let mut buf = vec![0.0f32; width];
for col in 0..cols {
buf[col] = matrix[row * cols + col];
}
fwht(&mut buf);
for col in 0..cols {
out[row * cols + col] = buf[col] * scale;
}
}
out
}
fn fwht(values: &mut [f32]) {
let n = values.len();
let mut h = 1usize;
while h < n {
for i in (0..n).step_by(h * 2) {
for j in i..i + h {
let x = values[j];
let y = values[j + h];
values[j] = x + y;
values[j + h] = x - y;
}
}
h *= 2;
}
}
fn aqlm_approximate(matrix: &[f32], in_features: usize) -> Vec<f32> {
let mut residual = matrix.to_vec();
let mut approx = vec![0.0; matrix.len()];
for _ in 0..AQLM_CODEBOOKS {
let codebook = build_codebook(&residual, in_features, AQLM_CODES);
for (index, value) in residual.iter_mut().enumerate() {
let code = nearest_code(*value, &codebook);
approx[index] += code;
*value -= code;
}
}
approx
}
fn build_codebook(values: &[f32], stride_hint: usize, codes: usize) -> Vec<f32> {
let _ = stride_hint;
if values.is_empty() {
return vec![0.0; codes];
}
let mut sorted = values.to_vec();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let mut codebook = Vec::with_capacity(codes);
for i in 0..codes {
let idx = i * (sorted.len() - 1) / (codes - 1).max(1);
codebook.push(sorted[idx]);
}
codebook
}
fn nearest_code(value: f32, codebook: &[f32]) -> f32 {
let mut best = codebook[0];
let mut best_dist = (value - best).abs();
for &code in codebook.iter().skip(1) {
let dist = (value - code).abs();
if dist < best_dist {
best = code;
best_dist = dist;
}
}
best
}
fn selected_names(model: &CausalLM, select: &dyn Fn(&str) -> bool) -> Vec<String> {
model
.named_linears()
.into_iter()
.filter(|(name, _)| select(name))
.map(|(name, _)| name)
.collect()
}
fn find_linear_mut<'a>(model: &'a mut CausalLM, name: &str) -> Option<&'a mut Linear> {
model
.named_linears_mut()
.into_iter()
.find(|(module, _)| module == name)
.map(|(_, linear)| linear)
}
fn logical_weight_matrix(linear: &Linear) -> Vec<f32> {
let values = linear.weight.shared_data();
let mut matrix = vec![0.0; linear.in_features * linear.out_features];
for input in 0..linear.in_features {
for output in 0..linear.out_features {
matrix[output * linear.in_features + input] =
linear.logical_weight_value(&values, input, output);
}
}
matrix
}
fn write_logical_weights(linear: &mut Linear, matrix: &[f32]) {
let mut storage = vec![0.0; linear.in_features * linear.out_features];
for input in 0..linear.in_features {
for output in 0..linear.out_features {
let value = matrix[output * linear.in_features + input];
if linear.transpose_weight {
storage[output * linear.in_features + input] = value;
} else {
storage[input * linear.out_features + output] = value;
}
}
}
linear.weight.set_data(storage);
}
#[cfg(test)]
mod tests {
use super::*;
use crate::layers::{QuantGranularity, ScaleMethod, WeightQuantization};
#[test]
fn hqq_packs_asymmetric_int4() {
let mut model = CausalLM::new(crate::model::CausalLmConfig {
vocab_size: 32,
hidden_size: 16,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 2,
intermediate_size: 32,
max_sequence_length: 16,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: 0,
seed: 0,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap();
let scheme = QuantScheme {
format: WeightQuantization::Int4,
granularity: QuantGranularity::PerGroup { size: 32 },
symmetric: false,
scale_method: ScaleMethod::MinMax,
percentile: 99.99,
};
let names = apply_hqq(&mut model, scheme, &|name| name.contains("q_proj")).unwrap();
assert!(!names.is_empty());
assert!(model
.named_linears()
.iter()
.any(|(_, linear)| linear.quantized_weight().is_some()));
}
}