entrenar/quant/gguf_quant/
q8_0.rs

1//! Q8_0 quantization format
2
3use super::GGUF_BLOCK_SIZE;
4use serde::{Deserialize, Serialize};
5
6/// Q8_0 quantized tensor (GGUF format)
7///
8/// 8-bit quantization with per-block f16 scale factors.
9/// Each block: 32 values → 34 bytes (2 scale + 32 data)
10#[derive(Clone, Debug, Serialize, Deserialize)]
11pub struct Q8_0 {
12    /// Per-block scale factors (stored as f32, converted to f16 on export)
13    pub scales: Vec<f32>,
14    /// 8-bit quantized data (1 byte per value)
15    pub data: Vec<i8>,
16    /// Original number of elements
17    pub len: usize,
18}
19
20impl Q8_0 {
21    /// Quantize f32 values to Q8_0 format
22    pub fn quantize(values: &[f32]) -> Self {
23        let len = values.len();
24        let num_blocks = len.div_ceil(GGUF_BLOCK_SIZE);
25
26        let mut scales = Vec::with_capacity(num_blocks);
27        let mut data = Vec::with_capacity(len);
28
29        for block_idx in 0..num_blocks {
30            let start = block_idx * GGUF_BLOCK_SIZE;
31            let end = (start + GGUF_BLOCK_SIZE).min(len);
32            let block = &values[start..end];
33
34            // Compute scale: max absolute value / 127 (8-bit signed: -128 to 127)
35            let max_abs = block
36                .iter()
37                .map(|v| v.abs())
38                .max_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
39                .unwrap_or(0.0);
40
41            let scale = if max_abs < 1e-10 { 1e-10 } else { max_abs / 127.0 };
42            scales.push(scale);
43
44            // Quantize block
45            for &val in block {
46                let q = (val / scale).round().clamp(-128.0, 127.0) as i8;
47                data.push(q);
48            }
49
50            // Pad incomplete blocks with zeros
51            let padding = GGUF_BLOCK_SIZE - block.len();
52            data.extend(std::iter::repeat_n(0i8, padding));
53        }
54
55        // Trim padding from last block
56        data.truncate(len);
57
58        Self { scales, data, len }
59    }
60
61    /// Dequantize Q8_0 back to f32
62    pub fn dequantize(&self) -> Vec<f32> {
63        let mut result = Vec::with_capacity(self.len);
64
65        for (i, &q) in self.data.iter().enumerate() {
66            let block_idx = i / GGUF_BLOCK_SIZE;
67            let scale = self.scales[block_idx];
68            result.push(f32::from(q) * scale);
69        }
70
71        result
72    }
73
74    /// Get memory usage in bytes
75    pub fn memory_bytes(&self) -> usize {
76        self.scales.len() * 4 + self.data.len()
77    }
78
79    /// Get GGUF-format memory (with f16 scales)
80    pub fn gguf_bytes(&self) -> usize {
81        self.scales.len() * 2 + self.data.len()
82    }
83
84    /// Get compression ratio vs f32
85    pub fn compression_ratio(&self) -> f32 {
86        let original = self.len * 4;
87        original as f32 / self.gguf_bytes() as f32
88    }
89
90    /// Number of blocks
91    pub fn num_blocks(&self) -> usize {
92        self.scales.len()
93    }
94}
entrenar/quant/gguf_quant/q8_0.rs

entrenar/quant/gguf_quant/
q8_0.rs
