fuel_ir/
quantized.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! The ggml block-format dtype tag (`GgmlDType`).
3//!
4//! As of B0.3 the backend-agnostic quantized *traits* (`DynQuantizedStorage`,
5//! `QuantizedDeviceKernels`) moved to the `fuel-backend-contract` crate. What
6//! stays here is the `GgmlDType` **data** tag — per-backend kernel crates and
7//! the contract traits both need to name it, so it lives in this bottom
8//! vocabulary crate.
9
10use crate::error::Result;
11
12/// The ggml block-format dtype tag. Mirrors llama.cpp's `ggml_type`
13/// **@`9d57ce456c`** for the subset fuel supports (the classic 15 — F32/F16/BF16
14/// plus Q4_0..Q8_1 and Q2K..Q8K; NOT the IQ*/TQ*/MXFP4/NVFP4 families).
15///
16/// Geometry checked against that commit (GAP-248 instance 5), and the referee
17/// differs by column:
18/// - `type_size` (bytes/block) is **ADJUDICATED** — upstream's own `static_assert`
19///   on each block struct is the referee; 12/12 for the quant types here.
20/// - `block_size` (elems/block) and the numeric codes are **CORROBORATED** — both
21///   are transcribed upstream (`type_traits[].blck_size`, `enum ggml_type`) and
22///   agree 12/12 with MLMF's INDEPENDENT transcription; no referee exists.
23/// - per-block **alignment** is UNANCHORED by both projects; fuel tracks none, so
24///   an upstream alignment change is undetectable here.
25///
26/// NO DETECTOR: this pin makes drift ANSWERABLE, not detected — it ages silently.
27/// Re-run the comparison on any llama.cpp bump that touches `ggml_type` or
28/// `type_traits[]`.
29///
30/// Lives here (rather than in `quantized/mod.rs`) because per-backend kernel
31/// crates need to name it without depending on fuel-core.
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
33pub enum GgmlDType {
34    F32,
35    F16,
36    BF16,
37    Q4_0,
38    Q4_1,
39    Q5_0,
40    Q5_1,
41    Q8_0,
42    Q8_1,
43    Q2K,
44    Q3K,
45    Q4K,
46    Q5K,
47    Q6K,
48    Q8K,
49}
50
51impl GgmlDType {
52    pub fn from_u32(u: u32) -> Result<Self> {
53        let dtype = match u {
54            0 => Self::F32,
55            1 => Self::F16,
56            2 => Self::Q4_0,
57            3 => Self::Q4_1,
58            6 => Self::Q5_0,
59            7 => Self::Q5_1,
60            8 => Self::Q8_0,
61            9 => Self::Q8_1,
62            10 => Self::Q2K,
63            11 => Self::Q3K,
64            12 => Self::Q4K,
65            13 => Self::Q5K,
66            14 => Self::Q6K,
67            15 => Self::Q8K,
68            30 => Self::BF16,
69            _ => return Err(crate::Error::Msg(format!("unknown dtype for tensor {u}")).bt()),
70        };
71        Ok(dtype)
72    }
73
74    pub fn to_u32(self) -> u32 {
75        match self {
76            Self::F32 => 0,
77            Self::F16 => 1,
78            Self::Q4_0 => 2,
79            Self::Q4_1 => 3,
80            Self::Q5_0 => 6,
81            Self::Q5_1 => 7,
82            Self::Q8_0 => 8,
83            Self::Q8_1 => 9,
84            Self::Q2K => 10,
85            Self::Q3K => 11,
86            Self::Q4K => 12,
87            Self::Q5K => 13,
88            Self::Q6K => 14,
89            Self::Q8K => 15,
90            Self::BF16 => 30,
91        }
92    }
93
94    pub fn type_size(&self) -> usize {
95        match self {
96            Self::F32 => 4,
97            Self::F16 | Self::BF16 => 2,
98            // ggml block sizes (must match k_quants::BlockQX struct sizes)
99            Self::Q4_0 => 18, // 2 + 16
100            Self::Q4_1 => 20, // 4 + 16
101            Self::Q5_0 => 22, // 2 + 4 + 16
102            Self::Q5_1 => 24, // 4 + 4 + 16
103            Self::Q8_0 => 34, // 2 + 32
104            Self::Q8_1 => 36, // 4 + 32
105            Self::Q2K => 84,  // QK_K/16 + QK_K/4 + 2 + 2
106            Self::Q3K => 110, // QK_K/8 + QK_K/4 + 12 + 2
107            Self::Q4K => 144, // 2 + 2 + 12 + QK_K/2
108            Self::Q5K => 176, // 2 + 2 + 12 + QK_K/8 + QK_K/2
109            Self::Q6K => 210, // QK_K/2 + QK_K/4 + QK_K/16 + 2
110            Self::Q8K => 292, // 4 + QK_K + QK_K/16 * 2
111        }
112    }
113
114    pub fn block_size(&self) -> usize {
115        match self {
116            Self::F32 | Self::F16 | Self::BF16 => 1,
117            Self::Q4_0 | Self::Q4_1 | Self::Q5_0 | Self::Q5_1 | Self::Q8_0 | Self::Q8_1 => 32,
118            Self::Q2K | Self::Q3K | Self::Q4K | Self::Q5K | Self::Q6K | Self::Q8K => 256,
119        }
120    }
121
122    /// GAP-333: `numel` dequantized elements are read from `numel / block_size`
123    /// whole blocks of `type_size` bytes. Declines a count that is not a whole
124    /// number of blocks, or whose blocks do not fit in `byte_len`.
125    ///
126    /// The one copy every backend's dequantize calls before it reads, so the
127    /// predicate cannot drift between them.
128    pub fn check_dequant_count(self, op_label: &str, numel: usize, byte_len: usize) -> Result<()> {
129        let block = self.block_size();
130        if !numel.is_multiple_of(block) {
131            return Err(crate::Error::Msg(format!(
132                "{op_label}: element count {numel} is not a multiple of {self:?}'s block size {block}",
133            ))
134            .bt());
135        }
136        match (numel / block).checked_mul(self.type_size()) {
137            Some(need) if need <= byte_len => Ok(()),
138            Some(need) => Err(crate::Error::Msg(format!(
139                "{op_label}: {numel} elements need {need} bytes of {self:?} blocks, but the \
140                 buffer holds {byte_len}",
141            ))
142            .bt()),
143            None => Err(crate::Error::Msg(format!(
144                "{op_label}: {numel} {self:?} elements overflow a byte count"
145            ))
146            .bt()),
147        }
148    }
149}
150
151#[cfg(test)]
152mod tests {
153    use super::GgmlDType;
154
155    #[track_caller]
156    fn declined(dtype: GgmlDType, numel: usize, byte_len: usize, needle: &str) {
157        let msg = dtype
158            .check_dequant_count("t", numel, byte_len)
159            .expect_err("expected a decline")
160            .to_string();
161        assert!(msg.contains(needle), "wrong decline: {msg}");
162    }
163
164    #[test]
165    fn whole_blocks_that_fit_are_accepted() {
166        // 2 Q4_0 blocks = 64 elements = 36 bytes; 1 Q4_K block = 256 = 144 bytes.
167        GgmlDType::Q4_0.check_dequant_count("t", 64, 36).unwrap();
168        GgmlDType::Q4K.check_dequant_count("t", 256, 144).unwrap();
169        GgmlDType::F32.check_dequant_count("t", 3, 12).unwrap();
170        GgmlDType::Q4_0.check_dequant_count("t", 0, 0).unwrap();
171    }
172
173    #[test]
174    fn a_partial_block_is_declined() {
175        declined(
176            GgmlDType::Q4_0,
177            65,
178            1_000,
179            "element count 65 is not a multiple of Q4_0's block size 32",
180        );
181        declined(
182            GgmlDType::Q4K,
183            128,
184            1_000,
185            "element count 128 is not a multiple of Q4K's block size 256",
186        );
187    }
188
189    #[test]
190    fn blocks_one_byte_past_the_buffer_are_declined() {
191        declined(
192            GgmlDType::Q4_0,
193            64,
194            35,
195            "64 elements need 36 bytes of Q4_0 blocks, but the buffer holds 35",
196        );
197        declined(
198            GgmlDType::F16,
199            3,
200            5,
201            "3 elements need 6 bytes of F16 blocks",
202        );
203        GgmlDType::F16.check_dequant_count("t", 3, 6).unwrap();
204    }
205
206    #[test]
207    fn an_overflowing_count_is_a_typed_error() {
208        declined(
209            GgmlDType::F32,
210            usize::MAX,
211            usize::MAX,
212            "overflow a byte count",
213        );
214    }
215}
fuel_ir/quantized.rs

fuel_ir/
quantized.rs
