fuel_ir/quantized.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! The ggml block-format dtype tag (`GgmlDType`).
3//!
4//! As of B0.3 the backend-agnostic quantized *traits* (`DynQuantizedStorage`,
5//! `QuantizedDeviceKernels`) moved to the `fuel-backend-contract` crate. What
6//! stays here is the `GgmlDType` **data** tag — per-backend kernel crates and
7//! the contract traits both need to name it, so it lives in this bottom
8//! vocabulary crate.
9
10use crate::error::Result;
11
12/// The ggml block-format dtype tag. Mirrors llama.cpp's `ggml_type`
13/// **@`9d57ce456c`** for the subset fuel supports (the classic 15 — F32/F16/BF16
14/// plus Q4_0..Q8_1 and Q2K..Q8K, plus every IQ family: IQ4_NL/IQ4_XS (GAP-211 B1)
15/// and the decode-only IQ1_S/IQ1_M/IQ2_XXS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S (GAP-211 B2);
16/// block geometry and the dequant formula verified against `ggml-common.h` and
17/// `ggml-quants.c` at that commit), plus the decode-only TQ1_0/TQ2_0/MXFP4/NVFP4 (GAP-211 C);
18/// NOT the other ggml types, which an unsupported-tensor GGUF load reports by name instead).
19///
20/// Geometry checked against that commit (GAP-248 instance 5), and the referee
21/// differs by column:
22/// - `type_size` (bytes/block) is **ADJUDICATED** — upstream's own `static_assert`
23/// on each block struct is the referee; 12/12 for the quant types here.
24/// - `block_size` (elems/block) and the numeric codes are **CORROBORATED** — both
25/// are transcribed upstream (`type_traits[].blck_size`, `enum ggml_type`) and
26/// agree 12/12 with MLMF's INDEPENDENT transcription; no referee exists.
27/// - per-block **alignment** is UNANCHORED by both projects; fuel tracks none, so
28/// an upstream alignment change is undetectable here.
29///
30/// NO DETECTOR: this pin makes drift ANSWERABLE, not detected — it ages silently.
31/// Re-run the comparison on any llama.cpp bump that touches `ggml_type` or
32/// `type_traits[]`.
33///
34/// Lives here (rather than in `quantized/mod.rs`) because per-backend kernel
35/// crates need to name it without depending on fuel-core.
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
37pub enum GgmlDType {
38 F32,
39 F16,
40 BF16,
41 Q4_0,
42 Q4_1,
43 Q5_0,
44 Q5_1,
45 Q8_0,
46 Q8_1,
47 Q2K,
48 Q3K,
49 Q4K,
50 Q5K,
51 Q6K,
52 Q8K,
53 /// llama.cpp `GGML_TYPE_IQ4_NL` (20): 32 elems/block, non-linear 4-bit codebook.
54 Iq4Nl,
55 /// llama.cpp `GGML_TYPE_IQ4_XS` (23): 256 elems/block, 4-bit codebook, 6-bit sub-block scales.
56 Iq4Xs,
57 /// llama.cpp `GGML_TYPE_IQ2_XXS` (16): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
58 Iq2Xxs,
59 /// llama.cpp `GGML_TYPE_IQ2_XS` (17): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
60 Iq2Xs,
61 /// llama.cpp `GGML_TYPE_IQ3_XXS` (18): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
62 Iq3Xxs,
63 /// llama.cpp `GGML_TYPE_IQ1_S` (19): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
64 Iq1S,
65 /// llama.cpp `GGML_TYPE_IQ3_S` (21): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
66 Iq3S,
67 /// llama.cpp `GGML_TYPE_IQ2_S` (22): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
68 Iq2S,
69 /// llama.cpp `GGML_TYPE_IQ1_M` (29): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
70 Iq1M,
71 /// llama.cpp `GGML_TYPE_TQ1_0` (34): 256 elems/block, ternary (5 trits/byte); DECODE-ONLY (GAP-211 C).
72 Tq1_0,
73 /// llama.cpp `GGML_TYPE_TQ2_0` (35): 256 elems/block, ternary (2 bits/elem); DECODE-ONLY (GAP-211 C).
74 Tq2_0,
75 /// llama.cpp `GGML_TYPE_MXFP4` (39): 32 elems/block, E2M1 with an E8M0 block scale; DECODE-ONLY (GAP-211 C).
76 Mxfp4,
77 /// llama.cpp `GGML_TYPE_NVFP4` (40): 64 elems/block, E2M1 with four UE4M3 sub-block scales; DECODE-ONLY (GAP-211 C).
78 Nvfp4,
79}
80
81impl GgmlDType {
82 pub fn from_u32(u: u32) -> Result<Self> {
83 let dtype = match u {
84 0 => Self::F32,
85 1 => Self::F16,
86 2 => Self::Q4_0,
87 3 => Self::Q4_1,
88 6 => Self::Q5_0,
89 7 => Self::Q5_1,
90 8 => Self::Q8_0,
91 9 => Self::Q8_1,
92 10 => Self::Q2K,
93 11 => Self::Q3K,
94 12 => Self::Q4K,
95 13 => Self::Q5K,
96 14 => Self::Q6K,
97 15 => Self::Q8K,
98 20 => Self::Iq4Nl,
99 23 => Self::Iq4Xs,
100 16 => Self::Iq2Xxs,
101 17 => Self::Iq2Xs,
102 18 => Self::Iq3Xxs,
103 19 => Self::Iq1S,
104 21 => Self::Iq3S,
105 22 => Self::Iq2S,
106 29 => Self::Iq1M,
107 34 => Self::Tq1_0,
108 35 => Self::Tq2_0,
109 39 => Self::Mxfp4,
110 40 => Self::Nvfp4,
111 30 => Self::BF16,
112 _ => return Err(crate::Error::Msg(format!("unknown dtype for tensor {u}")).bt()),
113 };
114 Ok(dtype)
115 }
116
117 pub fn to_u32(self) -> u32 {
118 match self {
119 Self::F32 => 0,
120 Self::F16 => 1,
121 Self::Q4_0 => 2,
122 Self::Q4_1 => 3,
123 Self::Q5_0 => 6,
124 Self::Q5_1 => 7,
125 Self::Q8_0 => 8,
126 Self::Q8_1 => 9,
127 Self::Q2K => 10,
128 Self::Q3K => 11,
129 Self::Q4K => 12,
130 Self::Q5K => 13,
131 Self::Q6K => 14,
132 Self::Q8K => 15,
133 Self::Iq4Nl => 20,
134 Self::Iq4Xs => 23,
135 Self::Iq2Xxs => 16,
136 Self::Iq2Xs => 17,
137 Self::Iq3Xxs => 18,
138 Self::Iq1S => 19,
139 Self::Iq3S => 21,
140 Self::Iq2S => 22,
141 Self::Iq1M => 29,
142 Self::Tq1_0 => 34,
143 Self::Tq2_0 => 35,
144 Self::Mxfp4 => 39,
145 Self::Nvfp4 => 40,
146 Self::BF16 => 30,
147 }
148 }
149
150 pub fn type_size(&self) -> usize {
151 match self {
152 Self::F32 => 4,
153 Self::F16 | Self::BF16 => 2,
154 // ggml block sizes (must match k_quants::BlockQX struct sizes)
155 Self::Q4_0 => 18, // 2 + 16
156 Self::Q4_1 => 20, // 4 + 16
157 Self::Q5_0 => 22, // 2 + 4 + 16
158 Self::Q5_1 => 24, // 4 + 4 + 16
159 Self::Q8_0 => 34, // 2 + 32
160 Self::Q8_1 => 36, // 4 + 32
161 Self::Q2K => 84, // QK_K/16 + QK_K/4 + 2 + 2
162 Self::Q3K => 110, // QK_K/8 + QK_K/4 + 12 + 2
163 Self::Q4K => 144, // 2 + 2 + 12 + QK_K/2
164 Self::Q5K => 176, // 2 + 2 + 12 + QK_K/8 + QK_K/2
165 Self::Q6K => 210, // QK_K/2 + QK_K/4 + QK_K/16 + 2
166 Self::Q8K => 292, // 4 + QK_K + QK_K/16 * 2
167 Self::Iq4Nl => 18, // 2 + QK4_NL/2
168 Self::Iq4Xs => 136, // 2 + 2 + QK_K/64 + QK_K/2
169 Self::Iq2Xxs => 66,
170 Self::Iq2Xs => 74,
171 Self::Iq3Xxs => 98,
172 Self::Iq1S => 50,
173 Self::Iq3S => 110,
174 Self::Iq2S => 82,
175 Self::Iq1M => 56,
176 Self::Tq1_0 => 54, // QK_K/64 + (QK_K - 4*QK_K/64)/5 + 2
177 Self::Tq2_0 => 66, // QK_K/4 + 2
178 Self::Mxfp4 => 17, // 1 + QK_MXFP4/2
179 Self::Nvfp4 => 36, // QK_NVFP4/16 + QK_NVFP4/2
180 }
181 }
182
183 pub fn block_size(&self) -> usize {
184 match self {
185 Self::F32 | Self::F16 | Self::BF16 => 1,
186 Self::Q4_0
187 | Self::Q4_1
188 | Self::Q5_0
189 | Self::Q5_1
190 | Self::Q8_0
191 | Self::Q8_1
192 | Self::Iq4Nl => 32,
193 Self::Q2K
194 | Self::Q3K
195 | Self::Q4K
196 | Self::Q5K
197 | Self::Q6K
198 | Self::Q8K
199 | Self::Iq4Xs
200 | Self::Iq2Xxs
201 | Self::Iq2Xs
202 | Self::Iq3Xxs
203 | Self::Iq1S
204 | Self::Iq3S
205 | Self::Iq2S
206 | Self::Iq1M
207 | Self::Tq1_0
208 | Self::Tq2_0 => 256,
209 Self::Mxfp4 => 32,
210 Self::Nvfp4 => 64,
211 }
212 }
213
214 /// `false` for the grid-table IQ formats (GAP-211 B2) and the TQ/MXFP4/NVFP4
215 /// formats (GAP-211 C): Fuel decodes them but has no quantizer for them.
216 pub fn supports_quantize(self) -> bool {
217 !matches!(
218 self,
219 Self::Iq2Xxs
220 | Self::Iq2Xs
221 | Self::Iq2S
222 | Self::Iq3Xxs
223 | Self::Iq3S
224 | Self::Iq1S
225 | Self::Iq1M
226 | Self::Tq1_0
227 | Self::Tq2_0
228 | Self::Mxfp4
229 | Self::Nvfp4
230 )
231 }
232
233 /// Declines quantizing to a decode-only format with a typed error that names it,
234 /// instead of reaching `GgmlType::from_float`, which has nothing to run.
235 pub fn check_quantize(self) -> Result<()> {
236 if self.supports_quantize() {
237 Ok(())
238 } else {
239 Err(crate::Error::Msg(format!(
240 "cannot quantize to {self:?}: Fuel decodes this format but has no quantizer for it (GAP-211 B2/C)"
241 ))
242 .bt())
243 }
244 }
245
246 /// GAP-333: `numel` dequantized elements are read from `numel / block_size`
247 /// whole blocks of `type_size` bytes. Declines a count that is not a whole
248 /// number of blocks, or whose blocks do not fit in `byte_len`.
249 ///
250 /// The one copy every backend's dequantize calls before it reads, so the
251 /// predicate cannot drift between them.
252 pub fn check_dequant_count(self, op_label: &str, numel: usize, byte_len: usize) -> Result<()> {
253 let block = self.block_size();
254 if !numel.is_multiple_of(block) {
255 return Err(crate::Error::Msg(format!(
256 "{op_label}: element count {numel} is not a multiple of {self:?}'s block size {block}",
257 ))
258 .bt());
259 }
260 match (numel / block).checked_mul(self.type_size()) {
261 Some(need) if need <= byte_len => Ok(()),
262 Some(need) => Err(crate::Error::Msg(format!(
263 "{op_label}: {numel} elements need {need} bytes of {self:?} blocks, but the \
264 buffer holds {byte_len}",
265 ))
266 .bt()),
267 None => Err(crate::Error::Msg(format!(
268 "{op_label}: {numel} {self:?} elements overflow a byte count"
269 ))
270 .bt()),
271 }
272 }
273}
274
275#[cfg(test)]
276mod tests {
277 use super::GgmlDType;
278
279 #[track_caller]
280 fn declined(dtype: GgmlDType, numel: usize, byte_len: usize, needle: &str) {
281 let msg = dtype
282 .check_dequant_count("t", numel, byte_len)
283 .expect_err("expected a decline")
284 .to_string();
285 assert!(msg.contains(needle), "wrong decline: {msg}");
286 }
287
288 #[test]
289 fn whole_blocks_that_fit_are_accepted() {
290 // 2 Q4_0 blocks = 64 elements = 36 bytes; 1 Q4_K block = 256 = 144 bytes.
291 GgmlDType::Q4_0.check_dequant_count("t", 64, 36).unwrap();
292 GgmlDType::Q4K.check_dequant_count("t", 256, 144).unwrap();
293 GgmlDType::F32.check_dequant_count("t", 3, 12).unwrap();
294 GgmlDType::Q4_0.check_dequant_count("t", 0, 0).unwrap();
295 }
296
297 #[test]
298 fn a_partial_block_is_declined() {
299 declined(
300 GgmlDType::Q4_0,
301 65,
302 1_000,
303 "element count 65 is not a multiple of Q4_0's block size 32",
304 );
305 declined(
306 GgmlDType::Q4K,
307 128,
308 1_000,
309 "element count 128 is not a multiple of Q4K's block size 256",
310 );
311 }
312
313 #[test]
314 fn blocks_one_byte_past_the_buffer_are_declined() {
315 declined(
316 GgmlDType::Q4_0,
317 64,
318 35,
319 "64 elements need 36 bytes of Q4_0 blocks, but the buffer holds 35",
320 );
321 declined(
322 GgmlDType::F16,
323 3,
324 5,
325 "3 elements need 6 bytes of F16 blocks",
326 );
327 GgmlDType::F16.check_dequant_count("t", 3, 6).unwrap();
328 }
329
330 #[test]
331 fn an_overflowing_count_is_a_typed_error() {
332 declined(
333 GgmlDType::F32,
334 usize::MAX,
335 usize::MAX,
336 "overflow a byte count",
337 );
338 }
339}