oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
#[path = "common/mod.rs"]
mod common;

#[cfg(feature = "gpu")]
use common::gpu_or_skip;
use oxidelm::layers::{Linear, WeightQuantization};
use oxidelm::Tensor;

fn fixture() -> (Linear, Tensor) {
    let linear = Linear::new(5, 3, 7);
    linear.weight.set_data(vec![
        -0.75, 0.10, 0.55, 0.25, -0.35, 0.80, 0.45, 0.90, -0.20, -0.60, 0.05, 0.30, 0.70, -0.85,
        -0.40,
    ]);
    let input = Tensor::new(
        vec![
            0.25, -0.50, 0.75, 1.00, -1.25, -0.30, 0.20, 0.90, -0.70, 0.40,
        ],
        vec![2, 5],
        true,
    );
    (linear, input)
}

fn maximum_difference(left: &[f32], right: &[f32]) -> f32 {
    left.iter()
        .zip(right)
        .map(|(left, right)| (left - right).abs())
        .fold(0.0, f32::max)
}

#[test]
fn int8_and_int4_cpu_inference_track_fp32_reference() {
    let (linear, input) = fixture();
    let reference = linear.forward(&input).data();

    for &(format, tolerance) in &[
        (WeightQuantization::Int8, 0.02),
        (WeightQuantization::Int4, 0.25),
    ] {
        let mut quantized = linear.clone();
        quantized.quantize_weight(format);
        let weight = quantized.quantized_weight().unwrap();
        assert_eq!(weight.format(), format);
        assert_eq!(weight.scales().len(), 3);
        assert!(weight.scales().iter().all(|scale| *scale > 0.0));
        assert!(weight.storage_bytes() < 5 * 3 * std::mem::size_of::<f32>());

        let actual = quantized.forward(&input).data();
        assert!(
            maximum_difference(&actual, &reference) <= tolerance,
            "{format:?} differs from FP32: actual={actual:?}, reference={reference:?}"
        );
    }
}

#[test]
fn weight_only_schemes_cpu_forward_are_finite() {
    use oxidelm::layers::{QuantGranularity, QuantScheme, ScaleMethod};

    let (linear, input) = fixture();
    let schemes = [
        QuantScheme {
            format: WeightQuantization::Int8,
            granularity: QuantGranularity::PerTensor,
            symmetric: true,
            scale_method: ScaleMethod::AbsMax,
            percentile: 99.99,
        },
        QuantScheme {
            format: WeightQuantization::Int4,
            granularity: QuantGranularity::PerChannel,
            symmetric: false,
            scale_method: ScaleMethod::MinMax,
            percentile: 99.99,
        },
        QuantScheme {
            format: WeightQuantization::Int2,
            granularity: QuantGranularity::PerChannel,
            symmetric: true,
            scale_method: ScaleMethod::Mse,
            percentile: 99.99,
        },
        QuantScheme {
            format: WeightQuantization::Int3,
            granularity: QuantGranularity::PerChannel,
            symmetric: true,
            scale_method: ScaleMethod::Percentile,
            percentile: 99.0,
        },
        QuantScheme::default_for(WeightQuantization::Nf4),
        QuantScheme::default_for(WeightQuantization::Fp8E4M3),
        QuantScheme::default_for(WeightQuantization::Fp8E5M2),
    ];
    for scheme in schemes {
        let mut quantized = linear.clone();
        quantized.quantize_weight_with(scheme);
        let actual = quantized.forward(&input).data();
        assert!(
            actual.iter().all(|value| value.is_finite()),
            "{scheme:?} produced non-finite output: {actual:?}"
        );
    }
}

#[test]
fn grouped_int4_cpu_matches_dequant_matmul() {
    use oxidelm::layers::{QuantGranularity, QuantScheme, ScaleMethod};

    let linear = Linear::new(64, 4, 3);
    linear.weight.set_data(
        (0..64 * 4)
            .map(|index| ((index % 17) as f32) * 0.01 - 0.08)
            .collect(),
    );
    let input = Tensor::new(
        (0..2 * 64)
            .map(|index| ((index % 9) as f32) * 0.05 - 0.2)
            .collect(),
        vec![2, 64],
        false,
    );
    let scheme = QuantScheme {
        format: WeightQuantization::Int4,
        granularity: QuantGranularity::PerGroup { size: 32 },
        symmetric: true,
        scale_method: ScaleMethod::AbsMax,
        percentile: 99.99,
    };
    let mut quantized = linear.clone();
    quantized.quantize_weight_with(scheme);
    let weight = quantized.quantized_weight().unwrap();
    assert_eq!(weight.scales().len(), 4 * 2);
    let actual = quantized.forward(&input).data();
    assert!(actual.iter().all(|value| value.is_finite()));
}

#[cfg(feature = "gpu")]
#[test]
fn quantized_gpu_linear_matches_cpu_reference() {
    let Some(gpu) = gpu_or_skip() else {
        return;
    };
    let (linear, input) = fixture();

    for format in [WeightQuantization::Int8, WeightQuantization::Int4] {
        let mut quantized = linear.clone();
        quantized.quantize_weight(format);
        let expected = quantized.forward(&input).data();
        let actual = quantized
            .forward_gpu(&input, &gpu)
            .unwrap()
            .to_cpu()
            .unwrap()
            .data();
        assert!(
            maximum_difference(&actual, &expected) <= 1e-5,
            "{format:?} GPU differs from CPU: actual={actual:?}, expected={expected:?}"
        );
    }
}

#[cfg(feature = "gpu")]
#[test]
fn weight_only_gpu_parity_across_schemes() {
    use oxidelm::layers::{QuantGranularity, QuantScheme, ScaleMethod};

    let Some(gpu) = gpu_or_skip() else {
        return;
    };
    let (linear, input) = fixture();
    let schemes = [
        QuantScheme::default_for(WeightQuantization::Int8),
        QuantScheme {
            format: WeightQuantization::Int4,
            granularity: QuantGranularity::PerChannel,
            symmetric: false,
            scale_method: ScaleMethod::MinMax,
            percentile: 99.99,
        },
        QuantScheme::default_for(WeightQuantization::Nf4),
        QuantScheme::default_for(WeightQuantization::Fp8E4M3),
        QuantScheme::default_for(WeightQuantization::Fp8E5M2),
        QuantScheme {
            format: WeightQuantization::Int8,
            granularity: QuantGranularity::PerTensor,
            symmetric: true,
            scale_method: ScaleMethod::AbsMax,
            percentile: 99.99,
        },
    ];
    for scheme in schemes {
        let mut quantized = linear.clone();
        quantized.quantize_weight_with(scheme);
        let expected = quantized.forward(&input).data();
        let actual = quantized
            .forward_gpu(&input, &gpu)
            .unwrap()
            .to_cpu()
            .unwrap()
            .data();
        assert!(
            maximum_difference(&actual, &expected) <= 1e-4,
            "{scheme:?} GPU differs from CPU: actual={actual:?}, expected={expected:?}"
        );
    }
}

#[test]
fn qat_forward_matches_packed_inference() {
    let (linear, input) = fixture();
    for format in [WeightQuantization::Int8, WeightQuantization::Int4] {
        let mut packed = linear.clone();
        packed.quantize_weight(format);
        let expected = packed.forward(&input).data();

        let mut qat = linear.clone();
        qat.enable_qat(format);
        assert_eq!(qat.qat_format(), Some(format));
        assert!(qat.quantized_weight().is_none());
        let actual = qat.forward(&input).data();
        assert!(
            maximum_difference(&actual, &expected) <= 1e-6,
            "{format:?} QAT forward differs from packed: actual={actual:?}, expected={expected:?}"
        );
    }
}

#[test]
fn ste_replace_is_identity_on_backward() {
    let source = Tensor::new(vec![0.5, -1.5, 2.0], vec![3], true);
    let replaced = source.ste_replace(vec![9.0, 8.0, 7.0]);
    assert_eq!(replaced.data(), vec![9.0, 8.0, 7.0]);
    replaced.sum().backward();
    assert_eq!(source.grad(), Some(vec![1.0, 1.0, 1.0]));
}

#[test]
fn qat_group_size_forward_is_finite() {
    let (mut linear, input) = fixture();
    linear
        .enable_qat_with(oxidelm::qat::QatConfig {
            format: WeightQuantization::Int4,
            group_size: Some(32),
            learnable_scales: false,
        })
        .unwrap();
    let output = linear.forward(&input);
    assert!(output.data().iter().all(|value| value.is_finite()));
}

#[test]
fn qat_lsq_trains_scales() {
    let (mut linear, input) = fixture();
    linear
        .enable_qat_with(oxidelm::qat::QatConfig {
            format: WeightQuantization::Int8,
            group_size: None,
            learnable_scales: true,
        })
        .unwrap();
    let scales = linear.qat_scales().expect("LSQ scales").clone();
    scales.zero_grad();
    linear.weight.zero_grad();
    let loss = linear.forward(&input).mean();
    loss.backward();
    let scale_grad = scales.grad().expect("LSQ scales should receive gradients");
    assert!(
        scale_grad.iter().any(|value| value.abs() > 0.0),
        "expected non-zero LSQ scale gradients"
    );
    linear.export_qat_packed().unwrap();
    assert!(linear.quantized_weight().is_some());
    assert!(linear.qat_format().is_none());
}

#[test]
fn qlora_frozen_base_plus_adapter_forward() {
    let mut model = oxidelm::model::CausalLM::new(
        oxidelm::model::ArchitecturePreset::LlamaTiny.causal_lm_config(32, 2, 0),
    )
    .unwrap();
    let config = oxidelm::peft::LoraConfig {
        rank: 4,
        alpha: 8.0,
        dropout: 0.0,
        dora: false,
        targets: vec!["q_proj".into(), "v_proj".into()],
        base_quant: Some(WeightQuantization::Nf4),
        base_group_size: Some(64),
    };
    model.install_lora(&config).unwrap();
    assert!(model.layers[0].attention.q_proj.quantized_weight().is_some());
    assert!(!model.layers[0].attention.q_proj.weight.requires_grad());
    assert!(model.layers[0].attention.q_proj.lora.is_some());
    let input = oxidelm::IndexTensor::new(vec![1, 2, 3, 4], vec![1, 4]);
    let logits = model.forward(&input).unwrap();
    assert!(logits.data().iter().all(|value| value.is_finite()));
}

#[cfg(feature = "gpu")]
#[test]
fn qat_gpu_forward_matches_cpu() {
    let Some(gpu) = gpu_or_skip() else {
        return;
    };
    let (mut linear, input) = fixture();
    linear.enable_qat(WeightQuantization::Int8);
    let expected = linear.forward(&input).data();
    let actual = linear
        .forward_gpu(&input, &gpu)
        .unwrap()
        .to_cpu()
        .unwrap()
        .data();
    assert!(
        maximum_difference(&actual, &expected) <= 1e-5,
        "QAT GPU differs from CPU: actual={actual:?}, expected={expected:?}"
    );
}