fuel_ir/quantized.rs
1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! The ggml block-format dtype tag (`GgmlDType`).
3//!
4//! As of B0.3 the backend-agnostic quantized *traits* (`DynQuantizedStorage`,
5//! `QuantizedDeviceKernels`) moved to the `fuel-backend-contract` crate. What
6//! stays here is the `GgmlDType` **data** tag — per-backend kernel crates and
7//! the contract traits both need to name it, so it lives in this bottom
8//! vocabulary crate.
9
10use crate::error::Result;
11
12/// The ggml block-format dtype tag. Mirrors llama.cpp's `ggml_type`
13/// **@`9d57ce456c`** for the subset fuel supports (the classic 15 — F32/F16/BF16
14/// plus Q4_0..Q8_1 and Q2K..Q8K, plus every IQ family: IQ4_NL/IQ4_XS (GAP-211 B1)
15/// and the decode-only IQ1_S/IQ1_M/IQ2_XXS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S (GAP-211 B2);
16/// block geometry and the dequant formula verified against `ggml-common.h` and
17/// `ggml-quants.c` at that commit); NOT the TQ*/MXFP4/NVFP4 families,
18/// which an unsupported-tensor GGUF load reports by name instead).
19///
20/// Geometry checked against that commit (GAP-248 instance 5), and the referee
21/// differs by column:
22/// - `type_size` (bytes/block) is **ADJUDICATED** — upstream's own `static_assert`
23/// on each block struct is the referee; 12/12 for the quant types here.
24/// - `block_size` (elems/block) and the numeric codes are **CORROBORATED** — both
25/// are transcribed upstream (`type_traits[].blck_size`, `enum ggml_type`) and
26/// agree 12/12 with MLMF's INDEPENDENT transcription; no referee exists.
27/// - per-block **alignment** is UNANCHORED by both projects; fuel tracks none, so
28/// an upstream alignment change is undetectable here.
29///
30/// NO DETECTOR: this pin makes drift ANSWERABLE, not detected — it ages silently.
31/// Re-run the comparison on any llama.cpp bump that touches `ggml_type` or
32/// `type_traits[]`.
33///
34/// Lives here (rather than in `quantized/mod.rs`) because per-backend kernel
35/// crates need to name it without depending on fuel-core.
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
37pub enum GgmlDType {
38 F32,
39 F16,
40 BF16,
41 Q4_0,
42 Q4_1,
43 Q5_0,
44 Q5_1,
45 Q8_0,
46 Q8_1,
47 Q2K,
48 Q3K,
49 Q4K,
50 Q5K,
51 Q6K,
52 Q8K,
53 /// llama.cpp `GGML_TYPE_IQ4_NL` (20): 32 elems/block, non-linear 4-bit codebook.
54 Iq4Nl,
55 /// llama.cpp `GGML_TYPE_IQ4_XS` (23): 256 elems/block, 4-bit codebook, 6-bit sub-block scales.
56 Iq4Xs,
57 /// llama.cpp `GGML_TYPE_IQ2_XXS` (16): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
58 Iq2Xxs,
59 /// llama.cpp `GGML_TYPE_IQ2_XS` (17): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
60 Iq2Xs,
61 /// llama.cpp `GGML_TYPE_IQ3_XXS` (18): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
62 Iq3Xxs,
63 /// llama.cpp `GGML_TYPE_IQ1_S` (19): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
64 Iq1S,
65 /// llama.cpp `GGML_TYPE_IQ3_S` (21): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
66 Iq3S,
67 /// llama.cpp `GGML_TYPE_IQ2_S` (22): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
68 Iq2S,
69 /// llama.cpp `GGML_TYPE_IQ1_M` (29): 256 elems/block, grid-table quantization; DECODE-ONLY (GAP-211 B2).
70 Iq1M,
71}
72
73impl GgmlDType {
74 pub fn from_u32(u: u32) -> Result<Self> {
75 let dtype = match u {
76 0 => Self::F32,
77 1 => Self::F16,
78 2 => Self::Q4_0,
79 3 => Self::Q4_1,
80 6 => Self::Q5_0,
81 7 => Self::Q5_1,
82 8 => Self::Q8_0,
83 9 => Self::Q8_1,
84 10 => Self::Q2K,
85 11 => Self::Q3K,
86 12 => Self::Q4K,
87 13 => Self::Q5K,
88 14 => Self::Q6K,
89 15 => Self::Q8K,
90 20 => Self::Iq4Nl,
91 23 => Self::Iq4Xs,
92 16 => Self::Iq2Xxs,
93 17 => Self::Iq2Xs,
94 18 => Self::Iq3Xxs,
95 19 => Self::Iq1S,
96 21 => Self::Iq3S,
97 22 => Self::Iq2S,
98 29 => Self::Iq1M,
99 30 => Self::BF16,
100 _ => return Err(crate::Error::Msg(format!("unknown dtype for tensor {u}")).bt()),
101 };
102 Ok(dtype)
103 }
104
105 pub fn to_u32(self) -> u32 {
106 match self {
107 Self::F32 => 0,
108 Self::F16 => 1,
109 Self::Q4_0 => 2,
110 Self::Q4_1 => 3,
111 Self::Q5_0 => 6,
112 Self::Q5_1 => 7,
113 Self::Q8_0 => 8,
114 Self::Q8_1 => 9,
115 Self::Q2K => 10,
116 Self::Q3K => 11,
117 Self::Q4K => 12,
118 Self::Q5K => 13,
119 Self::Q6K => 14,
120 Self::Q8K => 15,
121 Self::Iq4Nl => 20,
122 Self::Iq4Xs => 23,
123 Self::Iq2Xxs => 16,
124 Self::Iq2Xs => 17,
125 Self::Iq3Xxs => 18,
126 Self::Iq1S => 19,
127 Self::Iq3S => 21,
128 Self::Iq2S => 22,
129 Self::Iq1M => 29,
130 Self::BF16 => 30,
131 }
132 }
133
134 pub fn type_size(&self) -> usize {
135 match self {
136 Self::F32 => 4,
137 Self::F16 | Self::BF16 => 2,
138 // ggml block sizes (must match k_quants::BlockQX struct sizes)
139 Self::Q4_0 => 18, // 2 + 16
140 Self::Q4_1 => 20, // 4 + 16
141 Self::Q5_0 => 22, // 2 + 4 + 16
142 Self::Q5_1 => 24, // 4 + 4 + 16
143 Self::Q8_0 => 34, // 2 + 32
144 Self::Q8_1 => 36, // 4 + 32
145 Self::Q2K => 84, // QK_K/16 + QK_K/4 + 2 + 2
146 Self::Q3K => 110, // QK_K/8 + QK_K/4 + 12 + 2
147 Self::Q4K => 144, // 2 + 2 + 12 + QK_K/2
148 Self::Q5K => 176, // 2 + 2 + 12 + QK_K/8 + QK_K/2
149 Self::Q6K => 210, // QK_K/2 + QK_K/4 + QK_K/16 + 2
150 Self::Q8K => 292, // 4 + QK_K + QK_K/16 * 2
151 Self::Iq4Nl => 18, // 2 + QK4_NL/2
152 Self::Iq4Xs => 136, // 2 + 2 + QK_K/64 + QK_K/2
153 Self::Iq2Xxs => 66,
154 Self::Iq2Xs => 74,
155 Self::Iq3Xxs => 98,
156 Self::Iq1S => 50,
157 Self::Iq3S => 110,
158 Self::Iq2S => 82,
159 Self::Iq1M => 56,
160 }
161 }
162
163 pub fn block_size(&self) -> usize {
164 match self {
165 Self::F32 | Self::F16 | Self::BF16 => 1,
166 Self::Q4_0
167 | Self::Q4_1
168 | Self::Q5_0
169 | Self::Q5_1
170 | Self::Q8_0
171 | Self::Q8_1
172 | Self::Iq4Nl => 32,
173 Self::Q2K
174 | Self::Q3K
175 | Self::Q4K
176 | Self::Q5K
177 | Self::Q6K
178 | Self::Q8K
179 | Self::Iq4Xs
180 | Self::Iq2Xxs
181 | Self::Iq2Xs
182 | Self::Iq3Xxs
183 | Self::Iq1S
184 | Self::Iq3S
185 | Self::Iq2S
186 | Self::Iq1M => 256,
187 }
188 }
189
190 /// `false` for the grid-table IQ formats (GAP-211 B2): Fuel decodes them but
191 /// has no quantizer for them.
192 pub fn supports_quantize(self) -> bool {
193 !matches!(
194 self,
195 Self::Iq2Xxs
196 | Self::Iq2Xs
197 | Self::Iq2S
198 | Self::Iq3Xxs
199 | Self::Iq3S
200 | Self::Iq1S
201 | Self::Iq1M
202 )
203 }
204
205 /// Declines quantizing to a decode-only format with a typed error that names it,
206 /// instead of reaching `GgmlType::from_float`, which has nothing to run.
207 pub fn check_quantize(self) -> Result<()> {
208 if self.supports_quantize() {
209 Ok(())
210 } else {
211 Err(crate::Error::Msg(format!(
212 "cannot quantize to {self:?}: Fuel decodes this format but has no quantizer for it (GAP-211 B2)"
213 ))
214 .bt())
215 }
216 }
217
218 /// GAP-333: `numel` dequantized elements are read from `numel / block_size`
219 /// whole blocks of `type_size` bytes. Declines a count that is not a whole
220 /// number of blocks, or whose blocks do not fit in `byte_len`.
221 ///
222 /// The one copy every backend's dequantize calls before it reads, so the
223 /// predicate cannot drift between them.
224 pub fn check_dequant_count(self, op_label: &str, numel: usize, byte_len: usize) -> Result<()> {
225 let block = self.block_size();
226 if !numel.is_multiple_of(block) {
227 return Err(crate::Error::Msg(format!(
228 "{op_label}: element count {numel} is not a multiple of {self:?}'s block size {block}",
229 ))
230 .bt());
231 }
232 match (numel / block).checked_mul(self.type_size()) {
233 Some(need) if need <= byte_len => Ok(()),
234 Some(need) => Err(crate::Error::Msg(format!(
235 "{op_label}: {numel} elements need {need} bytes of {self:?} blocks, but the \
236 buffer holds {byte_len}",
237 ))
238 .bt()),
239 None => Err(crate::Error::Msg(format!(
240 "{op_label}: {numel} {self:?} elements overflow a byte count"
241 ))
242 .bt()),
243 }
244 }
245}
246
247#[cfg(test)]
248mod tests {
249 use super::GgmlDType;
250
251 #[track_caller]
252 fn declined(dtype: GgmlDType, numel: usize, byte_len: usize, needle: &str) {
253 let msg = dtype
254 .check_dequant_count("t", numel, byte_len)
255 .expect_err("expected a decline")
256 .to_string();
257 assert!(msg.contains(needle), "wrong decline: {msg}");
258 }
259
260 #[test]
261 fn whole_blocks_that_fit_are_accepted() {
262 // 2 Q4_0 blocks = 64 elements = 36 bytes; 1 Q4_K block = 256 = 144 bytes.
263 GgmlDType::Q4_0.check_dequant_count("t", 64, 36).unwrap();
264 GgmlDType::Q4K.check_dequant_count("t", 256, 144).unwrap();
265 GgmlDType::F32.check_dequant_count("t", 3, 12).unwrap();
266 GgmlDType::Q4_0.check_dequant_count("t", 0, 0).unwrap();
267 }
268
269 #[test]
270 fn a_partial_block_is_declined() {
271 declined(
272 GgmlDType::Q4_0,
273 65,
274 1_000,
275 "element count 65 is not a multiple of Q4_0's block size 32",
276 );
277 declined(
278 GgmlDType::Q4K,
279 128,
280 1_000,
281 "element count 128 is not a multiple of Q4K's block size 256",
282 );
283 }
284
285 #[test]
286 fn blocks_one_byte_past_the_buffer_are_declined() {
287 declined(
288 GgmlDType::Q4_0,
289 64,
290 35,
291 "64 elements need 36 bytes of Q4_0 blocks, but the buffer holds 35",
292 );
293 declined(
294 GgmlDType::F16,
295 3,
296 5,
297 "3 elements need 6 bytes of F16 blocks",
298 );
299 GgmlDType::F16.check_dequant_count("t", 3, 6).unwrap();
300 }
301
302 #[test]
303 fn an_overflowing_count_is_a_typed_error() {
304 declined(
305 GgmlDType::F32,
306 usize::MAX,
307 usize::MAX,
308 "overflow a byte count",
309 );
310 }
311}