#[path = "common/mod.rs"]
mod common;
#[cfg(feature = "gpu")]
use common::gpu_or_skip;
use oxidelm::layers::{Linear, WeightQuantization};
use oxidelm::Tensor;
fn fixture() -> (Linear, Tensor) {
let linear = Linear::new(5, 3, 7);
linear.weight.set_data(vec![
-0.75, 0.10, 0.55, 0.25, -0.35, 0.80, 0.45, 0.90, -0.20, -0.60, 0.05, 0.30, 0.70, -0.85,
-0.40,
]);
let input = Tensor::new(
vec![
0.25, -0.50, 0.75, 1.00, -1.25, -0.30, 0.20, 0.90, -0.70, 0.40,
],
vec![2, 5],
true,
);
(linear, input)
}
fn maximum_difference(left: &[f32], right: &[f32]) -> f32 {
left.iter()
.zip(right)
.map(|(left, right)| (left - right).abs())
.fold(0.0, f32::max)
}
#[test]
fn int8_and_int4_cpu_inference_track_fp32_reference() {
let (linear, input) = fixture();
let reference = linear.forward(&input).data();
for &(format, tolerance) in &[
(WeightQuantization::Int8, 0.02),
(WeightQuantization::Int4, 0.25),
] {
let mut quantized = linear.clone();
quantized.quantize_weight(format);
let weight = quantized.quantized_weight().unwrap();
assert_eq!(weight.format(), format);
assert_eq!(weight.scales().len(), 3);
assert!(weight.scales().iter().all(|scale| *scale > 0.0));
assert!(weight.storage_bytes() < 5 * 3 * std::mem::size_of::<f32>());
let actual = quantized.forward(&input).data();
assert!(
maximum_difference(&actual, &reference) <= tolerance,
"{format:?} differs from FP32: actual={actual:?}, reference={reference:?}"
);
}
}
#[test]
fn weight_only_schemes_cpu_forward_are_finite() {
use oxidelm::layers::{QuantGranularity, QuantScheme, ScaleMethod};
let (linear, input) = fixture();
let schemes = [
QuantScheme {
format: WeightQuantization::Int8,
granularity: QuantGranularity::PerTensor,
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
},
QuantScheme {
format: WeightQuantization::Int4,
granularity: QuantGranularity::PerChannel,
symmetric: false,
scale_method: ScaleMethod::MinMax,
percentile: 99.99,
},
QuantScheme {
format: WeightQuantization::Int2,
granularity: QuantGranularity::PerChannel,
symmetric: true,
scale_method: ScaleMethod::Mse,
percentile: 99.99,
},
QuantScheme {
format: WeightQuantization::Int3,
granularity: QuantGranularity::PerChannel,
symmetric: true,
scale_method: ScaleMethod::Percentile,
percentile: 99.0,
},
QuantScheme::default_for(WeightQuantization::Nf4),
QuantScheme::default_for(WeightQuantization::Fp8E4M3),
QuantScheme::default_for(WeightQuantization::Fp8E5M2),
];
for scheme in schemes {
let mut quantized = linear.clone();
quantized.quantize_weight_with(scheme);
let actual = quantized.forward(&input).data();
assert!(
actual.iter().all(|value| value.is_finite()),
"{scheme:?} produced non-finite output: {actual:?}"
);
}
}
#[test]
fn grouped_int4_cpu_matches_dequant_matmul() {
use oxidelm::layers::{QuantGranularity, QuantScheme, ScaleMethod};
let linear = Linear::new(64, 4, 3);
linear.weight.set_data(
(0..64 * 4)
.map(|index| ((index % 17) as f32) * 0.01 - 0.08)
.collect(),
);
let input = Tensor::new(
(0..2 * 64)
.map(|index| ((index % 9) as f32) * 0.05 - 0.2)
.collect(),
vec![2, 64],
false,
);
let scheme = QuantScheme {
format: WeightQuantization::Int4,
granularity: QuantGranularity::PerGroup { size: 32 },
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
};
let mut quantized = linear.clone();
quantized.quantize_weight_with(scheme);
let weight = quantized.quantized_weight().unwrap();
assert_eq!(weight.scales().len(), 4 * 2);
let actual = quantized.forward(&input).data();
assert!(actual.iter().all(|value| value.is_finite()));
}
#[cfg(feature = "gpu")]
#[test]
fn quantized_gpu_linear_matches_cpu_reference() {
let Some(gpu) = gpu_or_skip() else {
return;
};
let (linear, input) = fixture();
for format in [WeightQuantization::Int8, WeightQuantization::Int4] {
let mut quantized = linear.clone();
quantized.quantize_weight(format);
let expected = quantized.forward(&input).data();
let actual = quantized
.forward_gpu(&input, &gpu)
.unwrap()
.to_cpu()
.unwrap()
.data();
assert!(
maximum_difference(&actual, &expected) <= 1e-5,
"{format:?} GPU differs from CPU: actual={actual:?}, expected={expected:?}"
);
}
}
#[cfg(feature = "gpu")]
#[test]
fn weight_only_gpu_parity_across_schemes() {
use oxidelm::layers::{QuantGranularity, QuantScheme, ScaleMethod};
let Some(gpu) = gpu_or_skip() else {
return;
};
let (linear, input) = fixture();
let schemes = [
QuantScheme::default_for(WeightQuantization::Int8),
QuantScheme {
format: WeightQuantization::Int4,
granularity: QuantGranularity::PerChannel,
symmetric: false,
scale_method: ScaleMethod::MinMax,
percentile: 99.99,
},
QuantScheme::default_for(WeightQuantization::Nf4),
QuantScheme::default_for(WeightQuantization::Fp8E4M3),
QuantScheme::default_for(WeightQuantization::Fp8E5M2),
QuantScheme {
format: WeightQuantization::Int8,
granularity: QuantGranularity::PerTensor,
symmetric: true,
scale_method: ScaleMethod::AbsMax,
percentile: 99.99,
},
];
for scheme in schemes {
let mut quantized = linear.clone();
quantized.quantize_weight_with(scheme);
let expected = quantized.forward(&input).data();
let actual = quantized
.forward_gpu(&input, &gpu)
.unwrap()
.to_cpu()
.unwrap()
.data();
assert!(
maximum_difference(&actual, &expected) <= 1e-4,
"{scheme:?} GPU differs from CPU: actual={actual:?}, expected={expected:?}"
);
}
}
#[test]
fn qat_forward_matches_packed_inference() {
let (linear, input) = fixture();
for format in [WeightQuantization::Int8, WeightQuantization::Int4] {
let mut packed = linear.clone();
packed.quantize_weight(format);
let expected = packed.forward(&input).data();
let mut qat = linear.clone();
qat.enable_qat(format);
assert_eq!(qat.qat_format(), Some(format));
assert!(qat.quantized_weight().is_none());
let actual = qat.forward(&input).data();
assert!(
maximum_difference(&actual, &expected) <= 1e-6,
"{format:?} QAT forward differs from packed: actual={actual:?}, expected={expected:?}"
);
}
}
#[test]
fn ste_replace_is_identity_on_backward() {
let source = Tensor::new(vec![0.5, -1.5, 2.0], vec![3], true);
let replaced = source.ste_replace(vec![9.0, 8.0, 7.0]);
assert_eq!(replaced.data(), vec![9.0, 8.0, 7.0]);
replaced.sum().backward();
assert_eq!(source.grad(), Some(vec![1.0, 1.0, 1.0]));
}
#[test]
fn qat_group_size_forward_is_finite() {
let (mut linear, input) = fixture();
linear
.enable_qat_with(oxidelm::qat::QatConfig {
format: WeightQuantization::Int4,
group_size: Some(32),
learnable_scales: false,
})
.unwrap();
let output = linear.forward(&input);
assert!(output.data().iter().all(|value| value.is_finite()));
}
#[test]
fn qat_lsq_trains_scales() {
let (mut linear, input) = fixture();
linear
.enable_qat_with(oxidelm::qat::QatConfig {
format: WeightQuantization::Int8,
group_size: None,
learnable_scales: true,
})
.unwrap();
let scales = linear.qat_scales().expect("LSQ scales").clone();
scales.zero_grad();
linear.weight.zero_grad();
let loss = linear.forward(&input).mean();
loss.backward();
let scale_grad = scales.grad().expect("LSQ scales should receive gradients");
assert!(
scale_grad.iter().any(|value| value.abs() > 0.0),
"expected non-zero LSQ scale gradients"
);
linear.export_qat_packed().unwrap();
assert!(linear.quantized_weight().is_some());
assert!(linear.qat_format().is_none());
}
#[test]
fn qlora_frozen_base_plus_adapter_forward() {
let mut model = oxidelm::model::CausalLM::new(
oxidelm::model::ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0),
)
.unwrap();
let config = oxidelm::peft::LoraConfig {
rank: 4,
alpha: 8.0,
dropout: 0.0,
dora: false,
targets: vec!["q_proj".into(), "v_proj".into()],
base_quant: Some(WeightQuantization::Nf4),
base_group_size: Some(64),
};
model.install_lora(&config).unwrap();
assert!(model.layers[0].attention.q_proj.quantized_weight().is_some());
assert!(!model.layers[0].attention.q_proj.weight.requires_grad());
assert!(model.layers[0].attention.q_proj.lora.is_some());
let input = oxidelm::IndexTensor::new(vec![1, 2, 3, 4], vec![1, 4]);
let logits = model.forward(&input).unwrap();
assert!(logits.data().iter().all(|value| value.is_finite()));
}
#[cfg(feature = "gpu")]
#[test]
fn qat_gpu_forward_matches_cpu() {
let Some(gpu) = gpu_or_skip() else {
return;
};
let (mut linear, input) = fixture();
linear.enable_qat(WeightQuantization::Int8);
let expected = linear.forward(&input).data();
let actual = linear
.forward_gpu(&input, &gpu)
.unwrap()
.to_cpu()
.unwrap()
.data();
assert!(
maximum_difference(&actual, &expected) <= 1e-5,
"QAT GPU differs from CPU: actual={actual:?}, expected={expected:?}"
);
}