Skip to main content

spec_stream_engine/
quantization.rs

1//! # GGUF Quantization Engine
2//!
3//! Provides pure-Rust, zero-dependency de-quantization and high-performance dot product
4//! kernels for standard GGUF quantization formats:
5//! - **Q8_0**: 8-bit quantized weights with FP16 block scales (34 bytes per 32 elements).
6//! - **Q4_0**: 4-bit quantized weights with FP16 block scales (18 bytes per 32 elements).
7
8use crate::error::EngineError;
9use crate::model::ModelEngine;
10use spec_stream_core::{Logits, TokenId};
11
12/// Number of elements per GGML quantization block.
13pub const QK_K: usize = 32;
14
15/// Converts an IEEE 754 half-precision (FP16) 16-bit integer to a single-precision (FP32) float.
16#[inline]
17pub fn f16_to_f32(h: u16) -> f32 {
18    let sign = ((h >> 15) & 1) as u32;
19    let exp = ((h >> 10) & 0x1F) as u32;
20    let mant = (h & 0x3FF) as u32;
21
22    let f_sign = sign << 31;
23    if exp == 0 {
24        if mant == 0 {
25            return f32::from_bits(f_sign);
26        }
27        // Subnormal number
28        let mut m = mant;
29        let mut e = 0;
30        while (m & 0x400) == 0 {
31            m <<= 1;
32            e += 1;
33        }
34        m &= 0x3FF;
35        let f_exp = (127 - 15 + 1 - e) << 23;
36        let f_mant = m << 13;
37        return f32::from_bits(f_sign | f_exp | f_mant);
38    } else if exp == 0x1F {
39        let f_exp = 0xFF << 23;
40        let f_mant = mant << 13;
41        return f32::from_bits(f_sign | f_exp | f_mant);
42    }
43
44    let f_exp = (exp + 127 - 15) << 23;
45    let f_mant = mant << 13;
46    f32::from_bits(f_sign | f_exp | f_mant)
47}
48
49/// Converts a single-precision float (FP32) into an IEEE 754 half-precision (FP16) integer.
50#[inline]
51pub fn f32_to_f16(val: f32) -> u16 {
52    let bits = val.to_bits();
53    let sign = (bits >> 31) & 1;
54    let exp = ((bits >> 23) & 0xFF) as i32;
55    let mant = bits & 0x7FFFFF;
56
57    if exp == 0xFF {
58        // Inf or NaN
59        let h_mant = if mant != 0 { 0x200 } else { 0 };
60        return ((sign << 15) | (0x1F << 10) | h_mant) as u16;
61    }
62
63    let new_exp = exp - 127 + 15;
64    if new_exp >= 31 {
65        return ((sign << 15) | (0x1F << 10)) as u16;
66    } else if new_exp <= 0 {
67        if new_exp < -10 {
68            return (sign << 15) as u16;
69        }
70        let m = (mant | 0x800000) >> (1 - new_exp);
71        return ((sign << 15) | (m >> 13)) as u16;
72    }
73
74    ((sign << 15) | ((new_exp as u32) << 10) | (mant >> 13)) as u16
75}
76
77/// Quantization format type.
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum QuantType {
80    /// 8-bit quantization with FP16 scale per 32 elements.
81    Q8_0,
82    /// 4-bit quantization with FP16 scale per 32 elements.
83    Q4_0,
84}
85
86/// Dequantizes a Q8_0 byte slice into an FP32 output buffer.
87pub fn dequantize_q8_0(src: &[u8], dst: &mut [f32]) {
88    assert_eq!(dst.len() % QK_K, 0, "Destination must be multiple of 32");
89    let num_blocks = dst.len() / QK_K;
90    assert_eq!(
91        src.len(),
92        num_blocks * 34,
93        "Source length mismatch for Q8_0"
94    );
95
96    for b in 0..num_blocks {
97        let src_offset = b * 34;
98        let scale_bytes = [src[src_offset], src[src_offset + 1]];
99        let scale = f16_to_f32(u16::from_le_bytes(scale_bytes));
100
101        let dst_offset = b * QK_K;
102        for i in 0..QK_K {
103            let q = src[src_offset + 2 + i] as i8;
104            dst[dst_offset + i] = q as f32 * scale;
105        }
106    }
107}
108
109/// Dequantizes a Q4_0 byte slice into an FP32 output buffer.
110pub fn dequantize_q4_0(src: &[u8], dst: &mut [f32]) {
111    assert_eq!(dst.len() % QK_K, 0, "Destination must be multiple of 32");
112    let num_blocks = dst.len() / QK_K;
113    assert_eq!(
114        src.len(),
115        num_blocks * 18,
116        "Source length mismatch for Q4_0"
117    );
118
119    for b in 0..num_blocks {
120        let src_offset = b * 18;
121        let scale_bytes = [src[src_offset], src[src_offset + 1]];
122        let scale = f16_to_f32(u16::from_le_bytes(scale_bytes));
123
124        let dst_offset = b * QK_K;
125        for i in 0..16 {
126            let byte = src[src_offset + 2 + i];
127            let q0 = (byte & 0x0F) as i8 - 8;
128            let q1 = ((byte >> 4) & 0x0F) as i8 - 8;
129
130            dst[dst_offset + i] = q0 as f32 * scale;
131            dst[dst_offset + i + 16] = q1 as f32 * scale;
132        }
133    }
134}
135
136/// Computes the dot product between a Q8_0 quantized weight row and an FP32 vector directly without full dequantization.
137pub fn vec_dot_q8_0(w_bytes: &[u8], x: &[f32]) -> f32 {
138    #[cfg(target_arch = "x86_64")]
139    {
140        if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
141            // SAFETY: Validated hardware feature presence via CPUID
142            return unsafe { vec_dot_q8_0_avx2(w_bytes, x) };
143        }
144    }
145
146    vec_dot_q8_0_scalar(w_bytes, x)
147}
148
149/// Fallback scalar implementation of Q8_0 dot product.
150pub fn vec_dot_q8_0_scalar(w_bytes: &[u8], x: &[f32]) -> f32 {
151    let num_blocks = x.len() / QK_K;
152    let mut sum = 0.0f32;
153
154    for b in 0..num_blocks {
155        let src_offset = b * 34;
156        let scale = f16_to_f32(u16::from_le_bytes([
157            w_bytes[src_offset],
158            w_bytes[src_offset + 1],
159        ]));
160        let x_offset = b * QK_K;
161
162        let mut block_acc = 0.0f32;
163        for i in 0..QK_K {
164            let q = w_bytes[src_offset + 2 + i] as i8;
165            block_acc += q as f32 * x[x_offset + i];
166        }
167        sum += block_acc * scale;
168    }
169
170    sum
171}
172
173#[cfg(target_arch = "x86_64")]
174#[target_feature(enable = "avx2,fma")]
175unsafe fn vec_dot_q8_0_avx2(w_bytes: &[u8], x: &[f32]) -> f32 {
176    use std::arch::x86_64::*;
177
178    let num_blocks = x.len() / QK_K;
179    let mut sum = 0.0f32;
180
181    for b in 0..num_blocks {
182        let src_offset = b * 34;
183        let scale = f16_to_f32(u16::from_le_bytes([
184            w_bytes[src_offset],
185            w_bytes[src_offset + 1],
186        ]));
187        let x_ptr = x.as_ptr().add(b * QK_K);
188        let q_ptr = w_bytes.as_ptr().add(src_offset + 2);
189
190        let mut acc = _mm256_setzero_ps();
191
192        for i in 0..4 {
193            let i8_slice = std::slice::from_raw_parts(q_ptr.add(i * 8), 8);
194            let mut i64_buf = [0u8; 8];
195            i64_buf.copy_from_slice(i8_slice);
196            let q_i64 = i64::from_ne_bytes(i64_buf);
197            let q128 = _mm_cvtsi64_si128(q_i64);
198
199            let q256_i32 = _mm256_cvtepi8_epi32(q128);
200            let q256_f32 = _mm256_cvtepi32_ps(q256_i32);
201            let x256_f32 = _mm256_loadu_ps(x_ptr.add(i * 8));
202
203            acc = _mm256_fmadd_ps(q256_f32, x256_f32, acc);
204        }
205
206        let h128 = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
207        let shuf = _mm_movehdup_ps(h128);
208        let sums = _mm_add_ps(h128, shuf);
209        let shuf2 = _mm_movehl_ps(sums, sums);
210        let total = _mm_add_ss(sums, shuf2);
211        let block_sum = _mm_cvtss_f32(total);
212
213        sum += block_sum * scale;
214    }
215
216    sum
217}
218
219/// Computes the dot product between a Q4_0 quantized weight row and an FP32 vector directly without full dequantization.
220pub fn vec_dot_q4_0(w_bytes: &[u8], x: &[f32]) -> f32 {
221    let num_blocks = x.len() / QK_K;
222    let mut sum = 0.0f32;
223
224    for b in 0..num_blocks {
225        let src_offset = b * 18;
226        let scale = f16_to_f32(u16::from_le_bytes([
227            w_bytes[src_offset],
228            w_bytes[src_offset + 1],
229        ]));
230        let x_offset = b * QK_K;
231
232        let mut block_acc = 0.0f32;
233        for i in 0..16 {
234            let byte = w_bytes[src_offset + 2 + i];
235            let q0 = (byte & 0x0F) as i8 - 8;
236            let q1 = ((byte >> 4) & 0x0F) as i8 - 8;
237
238            block_acc += q0 as f32 * x[x_offset + i];
239            block_acc += q1 as f32 * x[x_offset + i + 16];
240        }
241        sum += block_acc * scale;
242    }
243
244    sum
245}
246
247/// Quantizes an FP32 slice of 32 elements into a Q8_0 block (34 bytes).
248pub fn quantize_q8_0_block(src: &[f32], dst: &mut [u8]) {
249    assert_eq!(src.len(), QK_K);
250    assert_eq!(dst.len(), 34);
251
252    let mut amax = 0.0f32;
253    for &val in src {
254        amax = amax.max(val.abs());
255    }
256
257    let d = amax / 127.0;
258    let id = if d != 0.0 { 1.0 / d } else { 0.0 };
259
260    let f16_scale = f32_to_f16(d);
261    let scale_bytes = f16_scale.to_le_bytes();
262    dst[0] = scale_bytes[0];
263    dst[1] = scale_bytes[1];
264
265    for i in 0..QK_K {
266        let q = (src[i] * id).round().clamp(-128.0, 127.0) as i8;
267        dst[2 + i] = q as u8;
268    }
269}
270
271/// A memory-efficient quantized linear projection model (Q8_0 or Q4_0).
272/// Reduces LM head weight memory by up to 75%–87% compared to FP32.
273#[derive(Debug, Clone)]
274pub struct QuantizedLinearModel {
275    pub name: String,
276    pub vocab_size: usize,
277    pub hidden_dim: usize,
278    pub quant_type: QuantType,
279    /// Raw quantized bytes matrix of shape [vocab_size, bytes_per_row].
280    pub quantized_weights: Vec<u8>,
281    pub bytes_per_row: usize,
282    /// Unquantized or dequantized token embedding table.
283    pub embeddings: Vec<f32>,
284}
285
286impl QuantizedLinearModel {
287    /// Creates a Q8_0 quantized linear model from FP32 weights.
288    pub fn from_f32_weights(
289        name: impl Into<String>,
290        vocab_size: usize,
291        hidden_dim: usize,
292        weights: &[f32],
293        embeddings: Vec<f32>,
294    ) -> Result<Self, EngineError> {
295        if hidden_dim % QK_K != 0 {
296            return Err(EngineError::InvalidConfig(format!(
297                "Hidden dimension {} must be a multiple of {}",
298                hidden_dim, QK_K
299            )));
300        }
301
302        let num_blocks = hidden_dim / QK_K;
303        let bytes_per_row = num_blocks * 34; // Q8_0
304        let mut quantized_weights = vec![0u8; vocab_size * bytes_per_row];
305
306        for row in 0..vocab_size {
307            let row_offset_f32 = row * hidden_dim;
308            let row_offset_bytes = row * bytes_per_row;
309
310            for b in 0..num_blocks {
311                let src_slice =
312                    &weights[row_offset_f32 + b * QK_K..row_offset_f32 + (b + 1) * QK_K];
313                let dst_slice = &mut quantized_weights
314                    [row_offset_bytes + b * 34..row_offset_bytes + (b + 1) * 34];
315                quantize_q8_0_block(src_slice, dst_slice);
316            }
317        }
318
319        Ok(Self {
320            name: name.into(),
321            vocab_size,
322            hidden_dim,
323            quant_type: QuantType::Q8_0,
324            quantized_weights,
325            bytes_per_row,
326            embeddings,
327        })
328    }
329
330    /// Projects hidden states into logits using quantized dot products.
331    pub fn project_hidden(&self, hidden: &[f32]) -> Logits {
332        assert_eq!(hidden.len(), self.hidden_dim);
333        let mut logits = Vec::with_capacity(self.vocab_size);
334
335        for row in 0..self.vocab_size {
336            let offset = row * self.bytes_per_row;
337            let row_bytes = &self.quantized_weights[offset..offset + self.bytes_per_row];
338
339            let dot = match self.quant_type {
340                QuantType::Q8_0 => vec_dot_q8_0(row_bytes, hidden),
341                QuantType::Q4_0 => vec_dot_q4_0(row_bytes, hidden),
342            };
343            logits.push(dot);
344        }
345
346        Logits::new(logits)
347    }
348
349    /// Computes logits directly for an input token ID.
350    pub fn project_token(&self, token: TokenId) -> Logits {
351        let token_idx = (token as usize) % self.vocab_size;
352        let emb_offset = token_idx * self.hidden_dim;
353        let hidden = &self.embeddings[emb_offset..emb_offset + self.hidden_dim];
354        self.project_hidden(hidden)
355    }
356}
357
358impl ModelEngine for QuantizedLinearModel {
359    async fn forward_batch(&mut self, tokens: &[TokenId]) -> Result<Vec<Logits>, EngineError> {
360        let mut out = Vec::with_capacity(tokens.len());
361        for &tok in tokens {
362            out.push(self.project_token(tok));
363        }
364        Ok(out)
365    }
366
367    async fn forward_token(&mut self, token: TokenId) -> Result<Logits, EngineError> {
368        Ok(self.project_token(token))
369    }
370}
371
372#[cfg(test)]
373mod tests {
374    use super::*;
375
376    #[test]
377    fn test_f16_to_f32_roundtrip() {
378        let test_vals = [0.0f32, 1.0, -1.0, 0.5, 2.5, -15.25, 100.0];
379        for &v in &test_vals {
380            let h = f32_to_f16(v);
381            let back = f16_to_f32(h);
382            assert!((v - back).abs() < 1e-2, "Failed for {v}: got {back}");
383        }
384    }
385
386    #[test]
387    fn test_q8_0_quantize_dequantize() {
388        let mut original = [0.0f32; 32];
389        for (i, val) in original.iter_mut().enumerate() {
390            *val = (i as f32 - 16.0) * 0.25;
391        }
392
393        let mut block = [0u8; 34];
394        quantize_q8_0_block(&original, &mut block);
395
396        let mut dequantized = [0.0f32; 32];
397        dequantize_q8_0(&block, &mut dequantized);
398
399        for i in 0..32 {
400            assert!(
401                (original[i] - dequantized[i]).abs() < 0.05,
402                "Diff too large at {i}: orig {}, got {}",
403                original[i],
404                dequantized[i]
405            );
406        }
407    }
408
409    #[test]
410    fn test_quantized_linear_model() {
411        let vocab_size = 16;
412        let hidden_dim = 32;
413        let size = vocab_size * hidden_dim;
414
415        let weights: Vec<f32> = (0..size).map(|i| (i as f32 % 10.0) * 0.1).collect();
416        let embeddings: Vec<f32> = (0..size).map(|i| ((i + 5) as f32 % 10.0) * 0.1).collect();
417
418        let model = QuantizedLinearModel::from_f32_weights(
419            "test-q8", vocab_size, hidden_dim, &weights, embeddings,
420        )
421        .expect("Failed to build quantized model");
422
423        assert_eq!(model.bytes_per_row, 34);
424        assert_eq!(model.quantized_weights.len(), 16 * 34);
425
426        let logits = model.project_token(2);
427        assert_eq!(logits.vocab_size(), 16);
428    }
429
430    #[test]
431    fn test_vec_dot_q8_0_simd_equivalence() {
432        let mut original = [0.0f32; 64];
433        let mut x = [0.0f32; 64];
434        for i in 0..64 {
435            original[i] = (i as f32 - 32.0) * 0.15;
436            x[i] = (i as f32 % 7.0) * 0.3;
437        }
438
439        let mut w_bytes = vec![0u8; 68]; // 2 blocks of 34 bytes
440        quantize_q8_0_block(&original[0..32], &mut w_bytes[0..34]);
441        quantize_q8_0_block(&original[32..64], &mut w_bytes[34..68]);
442
443        let scalar_res = vec_dot_q8_0_scalar(&w_bytes, &x);
444        let simd_res = vec_dot_q8_0(&w_bytes, &x);
445
446        assert!(
447            (scalar_res - simd_res).abs() < 1e-4,
448            "SIMD dot product ({simd_res}) must match scalar dot product ({scalar_res})"
449        );
450    }
451}