llama_cpp_4/quantize.rs
1//! Quantization types and parameters for converting models to lower-bit precisions.
2//!
3//! # Quick start
4//!
5//! ```no_run
6//! use llama_cpp_4::quantize::{LlamaFtype, QuantizeParams};
7//!
8//! let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
9//! .with_nthread(8)
10//! .with_quantize_output_tensor(true);
11//!
12//! llama_cpp_4::model_quantize("model-f16.gguf", "model-q4km.gguf", ¶ms).unwrap();
13//! ```
14//!
15//! # `TurboQuant` – attention rotation (PR #21038)
16//!
17//! llama.cpp applies a Hadamard rotation to Q/K/V tensors before writing them into the KV cache.
18//! This significantly improves KV-cache quantization quality at near-zero cost, and is enabled by
19//! default for every model whose head dimension is a power of two. You can opt out per-context
20//! with [`LlamaContextParams::with_attn_rot_disabled`] or globally with
21//! [`set_attn_rot_disabled`].
22//!
23//! [`LlamaContextParams::with_attn_rot_disabled`]: crate::context::params::LlamaContextParams::with_attn_rot_disabled
24
25use std::ffi::{CString, NulError};
26use std::ptr::null;
27
28// ─────────────────────────────────────────────────────────────────────────────
29// LlamaFtype
30// ─────────────────────────────────────────────────────────────────────────────
31
32/// The quantization type used for the bulk of a model file (maps to `llama_ftype`).
33///
34/// Pass one of these variants to [`QuantizeParams::new`] to choose the target precision.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36#[non_exhaustive]
37#[allow(missing_docs)]
38pub enum LlamaFtype {
39 /// All tensors stored as full F32 (very large, for reference only)
40 AllF32 = 0,
41 /// F16 – 14 GB @ 7B, +0.0020 ppl vs Mistral-7B
42 MostlyF16 = 1,
43 /// `Q4_0` – 4.34 GB @ 8B, +0.4685 ppl
44 MostlyQ4_0 = 2,
45 /// `Q4_1` – 4.78 GB @ 8B, +0.4511 ppl
46 MostlyQ4_1 = 3,
47 /// `Q8_0` – 7.96 GB @ 8B, +0.0026 ppl
48 MostlyQ8_0 = 7,
49 /// `Q5_0` – 5.21 GB @ 8B, +0.1316 ppl
50 MostlyQ5_0 = 8,
51 /// `Q5_1` – 5.65 GB @ 8B, +0.1062 ppl
52 MostlyQ5_1 = 9,
53 /// `Q2_K` – 2.96 GB @ 8B, +3.5199 ppl
54 MostlyQ2K = 10,
55 /// `Q3_K` small – 3.41 GB @ 8B, +1.6321 ppl
56 MostlyQ3KS = 11,
57 /// `Q3_K` medium – 3.74 GB @ 8B, +0.6569 ppl
58 MostlyQ3KM = 12,
59 /// `Q3_K` large – 4.03 GB @ 8B, +0.5562 ppl
60 MostlyQ3KL = 13,
61 /// `Q4_K` small – 4.37 GB @ 8B, +0.2689 ppl
62 MostlyQ4KS = 14,
63 /// `Q4_K` medium – 4.58 GB @ 8B, +0.1754 ppl *(recommended default)*
64 MostlyQ4KM = 15,
65 /// `Q5_K` small – 5.21 GB @ 8B, +0.1049 ppl
66 MostlyQ5KS = 16,
67 /// `Q5_K` medium – 5.33 GB @ 8B, +0.0569 ppl
68 MostlyQ5KM = 17,
69 /// `Q6_K` – 6.14 GB @ 8B, +0.0217 ppl
70 MostlyQ6K = 18,
71 /// `IQ2_XXS` – 2.06 bpw
72 MostlyIQ2XXS = 19,
73 /// `IQ2_XS` – 2.31 bpw
74 MostlyIQ2XS = 20,
75 /// `Q2_K` small
76 MostlyQ2KS = 21,
77 /// `IQ3_XS` – 3.3 bpw
78 MostlyIQ3XS = 22,
79 /// `IQ3_XXS` – 3.06 bpw
80 MostlyIQ3XXS = 23,
81 /// `IQ1_S` – 1.56 bpw (extremely small, high loss)
82 MostlyIQ1S = 24,
83 /// `IQ4_NL` – 4.50 bpw non-linear
84 MostlyIQ4NL = 25,
85 /// `IQ3_S` – 3.44 bpw
86 MostlyIQ3S = 26,
87 /// `IQ3_M` – 3.66 bpw
88 MostlyIQ3M = 27,
89 /// `IQ2_S` – 2.5 bpw
90 MostlyIQ2S = 28,
91 /// `IQ2_M` – 2.7 bpw
92 MostlyIQ2M = 29,
93 /// `IQ4_XS` – 4.25 bpw non-linear
94 MostlyIQ4XS = 30,
95 /// `IQ1_M` – 1.75 bpw
96 MostlyIQ1M = 31,
97 /// BF16 – 14 GB @ 7B, −0.0050 ppl vs Mistral-7B
98 MostlyBF16 = 32,
99 /// `TQ1_0` – 1.69 bpw ternary
100 MostlyTQ1_0 = 36,
101 /// `TQ2_0` – 2.06 bpw ternary
102 MostlyTQ2_0 = 37,
103 /// MXFP4 (`MoE` layers)
104 MostlyMXFP4Moe = 38,
105 /// NVFP4
106 MostlyNVFP4 = 39,
107 /// `Q1_0` – 1.5 bpw binary (block size 32)
108 MostlyQ1_0 = 40,
109 /// `Q2_0` – 2 bpw binary
110 //
111 // 41 was `Q1_0_g128` behind the `q1` feature, from this crate's own
112 // quantization patch. Upstream uses 41 for `Q2_0`, and never had a
113 // `Q1_0_g128`, so the old variant named a type that does not exist while
114 // sitting on a value that means something else.
115 MostlyQ2_0 = 41,
116 /// `F8_E4M3` – 8-bit float (except 1d tensors)
117 MostlyF8E4M3 = 42,
118}
119
120impl LlamaFtype {
121 /// Short name suitable for filenames (e.g. `"Q4_K_M"`).
122 #[must_use]
123 pub fn name(self) -> &'static str {
124 match self {
125 Self::AllF32 => "F32",
126 Self::MostlyF16 => "F16",
127 Self::MostlyQ4_0 => "Q4_0",
128 Self::MostlyQ4_1 => "Q4_1",
129 Self::MostlyQ8_0 => "Q8_0",
130 Self::MostlyQ5_0 => "Q5_0",
131 Self::MostlyQ5_1 => "Q5_1",
132 Self::MostlyQ2K => "Q2_K",
133 Self::MostlyQ3KS => "Q3_K_S",
134 Self::MostlyQ3KM => "Q3_K_M",
135 Self::MostlyQ3KL => "Q3_K_L",
136 Self::MostlyQ4KS => "Q4_K_S",
137 Self::MostlyQ4KM => "Q4_K_M",
138 Self::MostlyQ5KS => "Q5_K_S",
139 Self::MostlyQ5KM => "Q5_K_M",
140 Self::MostlyQ6K => "Q6_K",
141 Self::MostlyIQ2XXS => "IQ2_XXS",
142 Self::MostlyIQ2XS => "IQ2_XS",
143 Self::MostlyQ2KS => "Q2_K_S",
144 Self::MostlyIQ3XS => "IQ3_XS",
145 Self::MostlyIQ3XXS => "IQ3_XXS",
146 Self::MostlyIQ1S => "IQ1_S",
147 Self::MostlyIQ4NL => "IQ4_NL",
148 Self::MostlyIQ3S => "IQ3_S",
149 Self::MostlyIQ3M => "IQ3_M",
150 Self::MostlyIQ2S => "IQ2_S",
151 Self::MostlyIQ2M => "IQ2_M",
152 Self::MostlyIQ4XS => "IQ4_XS",
153 Self::MostlyIQ1M => "IQ1_M",
154 Self::MostlyBF16 => "BF16",
155 Self::MostlyTQ1_0 => "TQ1_0",
156 Self::MostlyTQ2_0 => "TQ2_0",
157 Self::MostlyMXFP4Moe => "MXFP4_MOE",
158 Self::MostlyNVFP4 => "NVFP4",
159 Self::MostlyQ1_0 => "Q1_0",
160 Self::MostlyQ2_0 => "Q2_0",
161 Self::MostlyF8E4M3 => "F8_E4M3",
162 }
163 }
164
165 /// Human-readable description with approximate size and PPL delta.
166 #[must_use]
167 pub fn description(self) -> &'static str {
168 match self {
169 Self::AllF32 => "26.00 GB @ 7B — full precision reference",
170 Self::MostlyF16 => "14.00 GB @ 7B — +0.0020 ppl vs Mistral-7B",
171 Self::MostlyBF16 => "14.00 GB @ 7B — -0.0050 ppl vs Mistral-7B",
172 Self::MostlyQ8_0 => " 7.96 GB @ 8B — +0.0026 ppl",
173 Self::MostlyQ6K => " 6.14 GB @ 8B — +0.0217 ppl",
174 Self::MostlyQ5KM => " 5.33 GB @ 8B — +0.0569 ppl",
175 Self::MostlyQ5KS => " 5.21 GB @ 8B — +0.1049 ppl",
176 Self::MostlyQ5_1 => " 5.65 GB @ 8B — +0.1062 ppl",
177 Self::MostlyQ5_0 => " 5.21 GB @ 8B — +0.1316 ppl",
178 Self::MostlyQ4KM => " 4.58 GB @ 8B — +0.1754 ppl [recommended]",
179 Self::MostlyQ4KS => " 4.37 GB @ 8B — +0.2689 ppl",
180 Self::MostlyQ4_1 => " 4.78 GB @ 8B — +0.4511 ppl",
181 Self::MostlyQ4_0 => " 4.34 GB @ 8B — +0.4685 ppl",
182 Self::MostlyQ3KL => " 4.03 GB @ 8B — +0.5562 ppl",
183 Self::MostlyQ3KM => " 3.74 GB @ 8B — +0.6569 ppl",
184 Self::MostlyQ3KS => " 3.41 GB @ 8B — +1.6321 ppl",
185 Self::MostlyQ2KS => " 2.96 GB @ 8B — +3.1836 ppl",
186 Self::MostlyQ2K => " 2.96 GB @ 8B — +3.5199 ppl",
187 Self::MostlyIQ4XS => " 4.25 bpw non-linear",
188 Self::MostlyIQ4NL => " 4.50 bpw non-linear",
189 Self::MostlyIQ3S => " 3.44 bpw",
190 Self::MostlyIQ3M => " 3.66 bpw",
191 Self::MostlyIQ3XS => " 3.3 bpw",
192 Self::MostlyIQ3XXS => " 3.06 bpw",
193 Self::MostlyIQ2M => " 2.7 bpw",
194 Self::MostlyIQ2S => " 2.5 bpw",
195 Self::MostlyIQ2XS => " 2.31 bpw",
196 Self::MostlyIQ2XXS => " 2.06 bpw",
197 Self::MostlyIQ1M => " 1.75 bpw — extreme compression",
198 Self::MostlyIQ1S => " 1.56 bpw — extreme compression",
199 Self::MostlyTQ1_0 => " 1.69 bpw ternary",
200 Self::MostlyTQ2_0 => " 2.06 bpw ternary",
201 Self::MostlyMXFP4Moe => "MXFP4 MoE layers",
202 Self::MostlyNVFP4 => "NVFP4",
203 Self::MostlyQ1_0 => " 1.50 bpw — binary Q1_0 (block 32)",
204 Self::MostlyQ2_0 => " 2.00 bpw — binary Q2_0",
205 Self::MostlyF8E4M3 => " 8.00 bpw — float8 e4m3",
206 }
207 }
208
209 /// Look up a variant by its short name (case-insensitive).
210 ///
211 /// ```
212 /// use llama_cpp_4::quantize::LlamaFtype;
213 /// assert_eq!(LlamaFtype::from_name("Q4_K_M"), Some(LlamaFtype::MostlyQ4KM));
214 /// assert_eq!(LlamaFtype::from_name("q4_k_m"), Some(LlamaFtype::MostlyQ4KM));
215 /// assert_eq!(LlamaFtype::from_name("bogus"), None);
216 /// ```
217 #[must_use]
218 pub fn from_name(name: &str) -> Option<Self> {
219 let upper = name.to_uppercase();
220 match upper.as_str() {
221 "F32" => Some(Self::AllF32),
222 "F16" => Some(Self::MostlyF16),
223 "BF16" => Some(Self::MostlyBF16),
224 "Q4_0" => Some(Self::MostlyQ4_0),
225 "Q4_1" => Some(Self::MostlyQ4_1),
226 "Q8_0" => Some(Self::MostlyQ8_0),
227 "Q5_0" => Some(Self::MostlyQ5_0),
228 "Q5_1" => Some(Self::MostlyQ5_1),
229 "Q2_K" => Some(Self::MostlyQ2K),
230 "Q2_K_S" => Some(Self::MostlyQ2KS),
231 "Q3_K_S" => Some(Self::MostlyQ3KS),
232 "Q3_K_M" => Some(Self::MostlyQ3KM),
233 "Q3_K_L" => Some(Self::MostlyQ3KL),
234 "Q4_K_S" => Some(Self::MostlyQ4KS),
235 "Q4_K_M" => Some(Self::MostlyQ4KM),
236 "Q5_K_S" => Some(Self::MostlyQ5KS),
237 "Q5_K_M" => Some(Self::MostlyQ5KM),
238 "Q6_K" => Some(Self::MostlyQ6K),
239 "IQ1_S" => Some(Self::MostlyIQ1S),
240 "IQ1_M" => Some(Self::MostlyIQ1M),
241 "IQ2_XXS" => Some(Self::MostlyIQ2XXS),
242 "IQ2_XS" => Some(Self::MostlyIQ2XS),
243 "IQ2_S" => Some(Self::MostlyIQ2S),
244 "IQ2_M" => Some(Self::MostlyIQ2M),
245 "IQ3_XXS" => Some(Self::MostlyIQ3XXS),
246 "IQ3_XS" => Some(Self::MostlyIQ3XS),
247 "IQ3_S" => Some(Self::MostlyIQ3S),
248 "IQ3_M" => Some(Self::MostlyIQ3M),
249 "IQ4_NL" => Some(Self::MostlyIQ4NL),
250 "IQ4_XS" => Some(Self::MostlyIQ4XS),
251 "TQ1_0" => Some(Self::MostlyTQ1_0),
252 "TQ2_0" => Some(Self::MostlyTQ2_0),
253 "MXFP4_MOE" => Some(Self::MostlyMXFP4Moe),
254 "NVFP4" => Some(Self::MostlyNVFP4),
255 "Q1_0" => Some(Self::MostlyQ1_0),
256 "Q2_0" => Some(Self::MostlyQ2_0),
257 "F8_E4M3" => Some(Self::MostlyF8E4M3),
258 _ => None,
259 }
260 }
261
262 /// llama.cpp's own display name for this type, e.g. `"Q4_K - Medium"`.
263 ///
264 /// Where [`Self::name`] is a terse filename-safe token (`"Q4_K_M"`) kept in
265 /// a table in this crate, this asks llama.cpp, so it cannot drift from
266 /// upstream. Wraps `llama_ftype_name`.
267 ///
268 /// # Errors
269 ///
270 /// Returns [`FtypeNameError`] if llama.cpp returns null or non-UTF-8.
271 pub fn upstream_name(self) -> Result<&'static str, FtypeNameError> {
272 let ptr = unsafe { llama_cpp_sys_4::llama_ftype_name(self.into()) };
273 if ptr.is_null() {
274 return Err(FtypeNameError::Unnamed(self));
275 }
276 // SAFETY: llama.cpp returns a pointer to a string literal with static
277 // storage duration, so `'static` is sound here.
278 unsafe { std::ffi::CStr::from_ptr(ptr) }
279 .to_str()
280 .map_err(FtypeNameError::Utf8)
281 }
282
283 /// The `ggml` storage type most tensors get under this ftype.
284 ///
285 /// This is the *default*; the k-quant mixes deliberately store some tensors
286 /// (attention, output) at higher precision, so an individual tensor may not
287 /// use it. Wraps `llama_ftype_get_default_type`.
288 ///
289 /// Returns `None` for a `ggml` type this crate's [`GgmlType`] does not know.
290 #[must_use]
291 pub fn default_ggml_type(self) -> Option<GgmlType> {
292 let raw = unsafe { llama_cpp_sys_4::llama_ftype_get_default_type(self.into()) };
293 GgmlType::try_from(raw).ok()
294 }
295
296 /// All available types, ordered roughly from largest to smallest.
297 #[must_use]
298 pub fn all() -> &'static [Self] {
299 &[
300 Self::AllF32,
301 Self::MostlyF16,
302 Self::MostlyBF16,
303 Self::MostlyQ8_0,
304 Self::MostlyQ6K,
305 Self::MostlyQ5KM,
306 Self::MostlyQ5KS,
307 Self::MostlyQ5_1,
308 Self::MostlyQ5_0,
309 Self::MostlyQ4KM,
310 Self::MostlyQ4KS,
311 Self::MostlyQ4_1,
312 Self::MostlyQ4_0,
313 Self::MostlyQ3KL,
314 Self::MostlyQ3KM,
315 Self::MostlyQ3KS,
316 Self::MostlyQ2KS,
317 Self::MostlyQ2K,
318 Self::MostlyIQ4XS,
319 Self::MostlyIQ4NL,
320 Self::MostlyIQ3S,
321 Self::MostlyIQ3M,
322 Self::MostlyIQ3XS,
323 Self::MostlyIQ3XXS,
324 Self::MostlyIQ2M,
325 Self::MostlyIQ2S,
326 Self::MostlyIQ2XS,
327 Self::MostlyIQ2XXS,
328 Self::MostlyIQ1M,
329 Self::MostlyIQ1S,
330 Self::MostlyTQ1_0,
331 Self::MostlyTQ2_0,
332 Self::MostlyMXFP4Moe,
333 Self::MostlyNVFP4,
334 Self::MostlyQ1_0,
335 Self::MostlyQ2_0,
336 Self::MostlyF8E4M3,
337 ]
338 }
339}
340
341/// Failure reading llama.cpp's display name for an [`LlamaFtype`].
342#[derive(Debug, Eq, PartialEq, thiserror::Error)]
343pub enum FtypeNameError {
344 /// llama.cpp returned null for this type.
345 #[error("llama.cpp has no name for ftype {0:?}")]
346 Unnamed(LlamaFtype),
347 /// The returned bytes were not valid UTF-8.
348 #[error(transparent)]
349 Utf8(#[from] std::str::Utf8Error),
350}
351
352impl From<LlamaFtype> for llama_cpp_sys_4::llama_ftype {
353 fn from(t: LlamaFtype) -> Self {
354 t as llama_cpp_sys_4::llama_ftype
355 }
356}
357
358impl TryFrom<llama_cpp_sys_4::llama_ftype> for LlamaFtype {
359 type Error = llama_cpp_sys_4::llama_ftype;
360
361 /// Fails on a discriminant this build does not know — `LLAMA_FTYPE_GUESSED`
362 /// (1024, "the file did not say"), a `q1` type without that feature, or a
363 /// type upstream added since this release. The raw value is returned so the
364 /// caller can report it.
365 fn try_from(raw: llama_cpp_sys_4::llama_ftype) -> Result<Self, Self::Error> {
366 Self::all()
367 .iter()
368 .copied()
369 .find(|t| llama_cpp_sys_4::llama_ftype::from(*t) == raw)
370 .ok_or(raw)
371 }
372}
373
374impl std::fmt::Display for LlamaFtype {
375 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
376 write!(f, "{}", self.name())
377 }
378}
379
380// ─────────────────────────────────────────────────────────────────────────────
381// GgmlType
382// ─────────────────────────────────────────────────────────────────────────────
383
384/// GGML tensor storage type (maps to `ggml_type`).
385///
386/// Used to set [`QuantizeParams::output_tensor_type`] and
387/// [`QuantizeParams::token_embedding_type`], and for per-tensor type overrides
388/// in [`TensorTypeOverride`].
389#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
390#[non_exhaustive]
391#[allow(missing_docs)]
392pub enum GgmlType {
393 F32 = 0,
394 F16 = 1,
395 Q4_0 = 2,
396 Q4_1 = 3,
397 Q5_0 = 6,
398 Q5_1 = 7,
399 Q8_0 = 8,
400 Q8_1 = 9,
401 Q2K = 10,
402 Q3K = 11,
403 Q4K = 12,
404 Q5K = 13,
405 Q6K = 14,
406 Q8K = 15,
407 IQ2XXS = 16,
408 IQ2XS = 17,
409 IQ3XXS = 18,
410 IQ1S = 19,
411 IQ4NL = 20,
412 IQ3S = 21,
413 IQ2S = 22,
414 IQ4XS = 23,
415 I8 = 24,
416 I16 = 25,
417 I32 = 26,
418 I64 = 27,
419 F64 = 28,
420 IQ1M = 29,
421 BF16 = 30,
422 TQ1_0 = 34,
423 TQ2_0 = 35,
424 MXFP4 = 39,
425 /// NVFP4 (4 blocks, E4M3 scale).
426 NVFP4 = 40,
427 /// `Q1_0` — 1.5 bpw binary.
428 //
429 // These values used to be shuffled by the `q1` feature, which encoded the
430 // numbering of this crate's own `0001-q1-quantization.patch`: NVFP4 moved
431 // to 42 so Q1_0 could take 40 and a `Q1_0_G128` take 41. Q1_0 has since
432 // landed upstream at 41 — with Q2_0 at 42 and NVFP4 staying at 40 — and the
433 // patch is gone, so the feature was renumbering three variants away from
434 // what llama.cpp actually uses. Upstream's numbering is the only numbering.
435 Q1_0 = 41,
436 /// `Q2_0` — 2 bpw binary.
437 Q2_0 = 42,
438 /// `F8_E4M3` — 8-bit float, 4-bit exponent, 3-bit mantissa.
439 F8E4M3 = 43,
440}
441
442impl From<GgmlType> for llama_cpp_sys_4::ggml_type {
443 fn from(t: GgmlType) -> Self {
444 t as llama_cpp_sys_4::ggml_type
445 }
446}
447
448impl TryFrom<llama_cpp_sys_4::ggml_type> for GgmlType {
449 type Error = llama_cpp_sys_4::ggml_type;
450 fn try_from(v: llama_cpp_sys_4::ggml_type) -> Result<Self, Self::Error> {
451 match v {
452 0 => Ok(Self::F32),
453 1 => Ok(Self::F16),
454 2 => Ok(Self::Q4_0),
455 3 => Ok(Self::Q4_1),
456 6 => Ok(Self::Q5_0),
457 7 => Ok(Self::Q5_1),
458 8 => Ok(Self::Q8_0),
459 9 => Ok(Self::Q8_1),
460 10 => Ok(Self::Q2K),
461 11 => Ok(Self::Q3K),
462 12 => Ok(Self::Q4K),
463 13 => Ok(Self::Q5K),
464 14 => Ok(Self::Q6K),
465 15 => Ok(Self::Q8K),
466 16 => Ok(Self::IQ2XXS),
467 17 => Ok(Self::IQ2XS),
468 18 => Ok(Self::IQ3XXS),
469 19 => Ok(Self::IQ1S),
470 20 => Ok(Self::IQ4NL),
471 21 => Ok(Self::IQ3S),
472 22 => Ok(Self::IQ2S),
473 23 => Ok(Self::IQ4XS),
474 24 => Ok(Self::I8),
475 25 => Ok(Self::I16),
476 26 => Ok(Self::I32),
477 27 => Ok(Self::I64),
478 28 => Ok(Self::F64),
479 29 => Ok(Self::IQ1M),
480 30 => Ok(Self::BF16),
481 34 => Ok(Self::TQ1_0),
482 35 => Ok(Self::TQ2_0),
483 39 => Ok(Self::MXFP4),
484 40 => Ok(Self::NVFP4),
485 41 => Ok(Self::Q1_0),
486 42 => Ok(Self::Q2_0),
487 43 => Ok(Self::F8E4M3),
488 _ => Err(v),
489 }
490 }
491}
492
493// ─────────────────────────────────────────────────────────────────────────────
494// ImatrixEntry / Imatrix
495// ─────────────────────────────────────────────────────────────────────────────
496
497/// A single per-tensor importance matrix entry, as loaded from a `.imatrix` file.
498///
499/// Each entry contains activation statistics for one model tensor collected from
500/// a calibration dataset. When supplied to [`QuantizeParams::with_imatrix`] these
501/// statistics guide the quantizer to allocate more precision to weights that
502/// matter most.
503#[derive(Debug, Clone)]
504pub struct ImatrixEntry {
505 name: CString,
506 data: Vec<f32>,
507}
508
509impl ImatrixEntry {
510 /// Create a new entry from a tensor name and its importance scores.
511 ///
512 /// # Errors
513 ///
514 /// Returns [`NulError`] if `name` contains an interior null byte.
515 pub fn new(name: impl Into<Vec<u8>>, data: Vec<f32>) -> Result<Self, NulError> {
516 Ok(Self {
517 name: CString::new(name)?,
518 data,
519 })
520 }
521
522 /// Tensor name.
523 #[must_use]
524 pub fn name_str(&self) -> &str {
525 self.name.to_str().unwrap_or("")
526 }
527
528 /// Number of importance values.
529 #[must_use]
530 pub fn len(&self) -> usize {
531 self.data.len()
532 }
533
534 /// Returns `true` if the data slice is empty.
535 #[must_use]
536 pub fn is_empty(&self) -> bool {
537 self.data.is_empty()
538 }
539}
540
541/// A collection of importance matrix entries (one per quantized tensor).
542///
543/// Build one by pushing [`ImatrixEntry`] values, then pass it to
544/// [`QuantizeParams::with_imatrix`].
545#[derive(Debug, Clone, Default)]
546pub struct Imatrix {
547 entries: Vec<ImatrixEntry>,
548}
549
550impl Imatrix {
551 /// Create an empty imatrix.
552 #[must_use]
553 pub fn new() -> Self {
554 Self::default()
555 }
556
557 /// Add an entry.
558 pub fn push(&mut self, entry: ImatrixEntry) {
559 self.entries.push(entry);
560 }
561
562 /// Number of entries.
563 #[must_use]
564 pub fn len(&self) -> usize {
565 self.entries.len()
566 }
567
568 /// Returns `true` if no entries have been added.
569 #[must_use]
570 pub fn is_empty(&self) -> bool {
571 self.entries.is_empty()
572 }
573}
574
575// ─────────────────────────────────────────────────────────────────────────────
576// TensorTypeOverride
577// ─────────────────────────────────────────────────────────────────────────────
578
579/// Override the quantization type of every tensor whose name matches a glob `pattern`.
580///
581/// The pattern syntax is the same as used by the `--tensor-type` flag in
582/// `llama-quantize`, e.g. `"attn.*"` or `"blk.0.*"`.
583///
584/// # Example
585///
586/// ```
587/// use llama_cpp_4::quantize::{GgmlType, TensorTypeOverride};
588///
589/// // Keep the output projection in F16:
590/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
591/// ```
592#[derive(Debug, Clone)]
593pub struct TensorTypeOverride {
594 pattern: CString,
595 ty: GgmlType,
596}
597
598impl TensorTypeOverride {
599 /// Create a new override.
600 ///
601 /// # Errors
602 ///
603 /// Returns [`NulError`] if `pattern` contains an interior null byte.
604 pub fn new(pattern: impl Into<Vec<u8>>, ty: GgmlType) -> Result<Self, NulError> {
605 Ok(Self {
606 pattern: CString::new(pattern)?,
607 ty,
608 })
609 }
610
611 /// The glob pattern that selects tensors.
612 #[must_use]
613 pub fn pattern_str(&self) -> &str {
614 self.pattern.to_str().unwrap_or("")
615 }
616
617 /// The type to assign to matching tensors.
618 #[must_use]
619 pub fn ty(&self) -> GgmlType {
620 self.ty
621 }
622}
623
624// ─────────────────────────────────────────────────────────────────────────────
625// KvOverrideValue / KvOverride
626// ─────────────────────────────────────────────────────────────────────────────
627
628/// A value in a GGUF key-value metadata override.
629#[derive(Debug, Clone, PartialEq)]
630pub enum KvOverrideValue {
631 /// 64-bit integer
632 Int(i64),
633 /// 64-bit float
634 Float(f64),
635 /// Boolean
636 Bool(bool),
637 /// Fixed-length string (up to 127 bytes + NUL)
638 Str([std::os::raw::c_char; 128]),
639}
640
641/// A single GGUF metadata key-value override.
642///
643/// These are written into the output file's metadata when quantizing.
644#[derive(Debug, Clone)]
645pub struct KvOverride {
646 key: CString,
647 /// The value for this override.
648 pub value: KvOverrideValue,
649}
650
651impl KvOverride {
652 /// Create a new override.
653 ///
654 /// # Errors
655 ///
656 /// Returns [`NulError`] if `key` contains an interior null byte.
657 pub fn new(key: impl Into<Vec<u8>>, value: KvOverrideValue) -> Result<Self, NulError> {
658 Ok(Self {
659 key: CString::new(key)?,
660 value,
661 })
662 }
663}
664
665// ─────────────────────────────────────────────────────────────────────────────
666// QuantizeParams
667// ─────────────────────────────────────────────────────────────────────────────
668
669/// Parameters for quantizing a model.
670///
671/// Create with [`QuantizeParams::new`] and chain `with_*` builder methods to
672/// configure, then pass a reference to [`crate::model_quantize`].
673///
674/// # Example
675///
676/// ```no_run
677/// use llama_cpp_4::quantize::{GgmlType, LlamaFtype, QuantizeParams, TensorTypeOverride};
678///
679/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
680///
681/// let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
682/// .with_nthread(8)
683/// .with_allow_requantize(false)
684/// .with_quantize_output_tensor(true)
685/// .with_pure(false)
686/// .with_tensor_type_override(ov);
687///
688/// llama_cpp_4::model_quantize("in.gguf", "out.gguf", ¶ms).unwrap();
689/// ```
690#[derive(Debug, Clone)]
691#[allow(clippy::struct_excessive_bools)]
692pub struct QuantizeParams {
693 /// Number of threads (0 = auto-detect).
694 pub nthread: i32,
695 /// Target quantization type.
696 pub ftype: LlamaFtype,
697 /// Force this storage type for the output/lm-head tensor (`None` = use ftype default).
698 pub output_tensor_type: Option<GgmlType>,
699 /// Force this storage type for the token-embedding tensor (`None` = use ftype default).
700 pub token_embedding_type: Option<GgmlType>,
701 /// Allow re-quantizing tensors that are already quantized.
702 pub allow_requantize: bool,
703 /// Quantize the output/lm-head weight tensor.
704 pub quantize_output_tensor: bool,
705 /// Copy all tensors without quantizing (ignores `ftype`).
706 pub only_copy: bool,
707 /// Quantize every tensor to the same type (no mixed k-quant strategy).
708 pub pure: bool,
709 /// Keep the same number of shards as the input (for split models).
710 pub keep_split: bool,
711 /// Estimate output size without writing anything to disk.
712 pub dry_run: bool,
713 /// Cap, in bytes, on the tensor rows held in memory at once (`0` = the
714 /// upstream default of 8 GiB). Lower it to quantize a model larger than
715 /// available RAM at the cost of more I/O.
716 pub max_buf_size: usize,
717
718 imatrix: Vec<ImatrixEntry>,
719 kv_overrides: Vec<KvOverride>,
720 tt_overrides: Vec<TensorTypeOverride>,
721 prune_layers: Vec<i32>,
722}
723
724impl QuantizeParams {
725 /// Create a new params set targeting `ftype`.
726 ///
727 /// All other options are set to the same defaults as
728 /// `llama_model_quantize_default_params()`.
729 #[must_use]
730 pub fn new(ftype: LlamaFtype) -> Self {
731 // Read the C defaults so we match them exactly.
732 let d = unsafe { llama_cpp_sys_4::llama_model_quantize_default_params() };
733 Self {
734 nthread: d.nthread,
735 ftype,
736 output_tensor_type: GgmlType::try_from(d.output_tensor_type).ok(),
737 token_embedding_type: GgmlType::try_from(d.token_embedding_type).ok(),
738 allow_requantize: d.allow_requantize,
739 quantize_output_tensor: d.quantize_output_tensor,
740 only_copy: d.only_copy,
741 pure: d.pure_,
742 keep_split: d.keep_split,
743 dry_run: d.dry_run,
744 max_buf_size: d.max_buf_size,
745 imatrix: Vec::new(),
746 kv_overrides: Vec::new(),
747 tt_overrides: Vec::new(),
748 prune_layers: Vec::new(),
749 }
750 }
751
752 /// Set the number of quantization threads (`0` = auto).
753 #[must_use]
754 pub fn with_nthread(mut self, n: i32) -> Self {
755 self.nthread = n;
756 self
757 }
758
759 /// Override the output-tensor storage type.
760 #[must_use]
761 pub fn with_output_tensor_type(mut self, ty: GgmlType) -> Self {
762 self.output_tensor_type = Some(ty);
763 self
764 }
765
766 /// Override the token-embedding storage type.
767 #[must_use]
768 pub fn with_token_embedding_type(mut self, ty: GgmlType) -> Self {
769 self.token_embedding_type = Some(ty);
770 self
771 }
772
773 /// Allow (or disallow) re-quantizing already-quantized tensors.
774 #[must_use]
775 pub fn with_allow_requantize(mut self, v: bool) -> Self {
776 self.allow_requantize = v;
777 self
778 }
779
780 /// Quantize the output/lm-head weight (`true` by default).
781 #[must_use]
782 pub fn with_quantize_output_tensor(mut self, v: bool) -> Self {
783 self.quantize_output_tensor = v;
784 self
785 }
786
787 /// When `true`, only copy tensors verbatim (no quantization at all).
788 #[must_use]
789 pub fn with_only_copy(mut self, v: bool) -> Self {
790 self.only_copy = v;
791 self
792 }
793
794 /// When `true`, quantize all tensors to the same type (no mixed k-quant strategy).
795 #[must_use]
796 pub fn with_pure(mut self, v: bool) -> Self {
797 self.pure = v;
798 self
799 }
800
801 /// Preserve the number of shards when quantizing a split model.
802 #[must_use]
803 pub fn with_keep_split(mut self, v: bool) -> Self {
804 self.keep_split = v;
805 self
806 }
807
808 /// Only estimate the output size; do not write anything to disk.
809 #[must_use]
810 pub fn with_dry_run(mut self, v: bool) -> Self {
811 self.dry_run = v;
812 self
813 }
814
815 /// Cap the bytes of tensor rows kept in memory at once (`0` = upstream
816 /// default, 8 GiB).
817 #[must_use]
818 pub fn with_max_buf_size(mut self, bytes: usize) -> Self {
819 self.max_buf_size = bytes;
820 self
821 }
822
823 /// Supply importance matrix data to improve quantization quality.
824 ///
825 /// The imatrix is generated by the `imatrix` tool (or the `imatrix` example
826 /// in this crate) and contains per-tensor activation statistics collected
827 /// from a calibration dataset.
828 #[must_use]
829 pub fn with_imatrix(mut self, imatrix: Imatrix) -> Self {
830 self.imatrix = imatrix.entries;
831 self
832 }
833
834 /// Append a single imatrix entry.
835 #[must_use]
836 pub fn with_imatrix_entry(mut self, entry: ImatrixEntry) -> Self {
837 self.imatrix.push(entry);
838 self
839 }
840
841 /// Add (or replace) a GGUF metadata key-value pair in the output file.
842 #[must_use]
843 pub fn with_kv_override(mut self, kv: KvOverride) -> Self {
844 self.kv_overrides.push(kv);
845 self
846 }
847
848 /// Override the quantization type for tensors whose name matches `pattern`.
849 ///
850 /// Can be called multiple times; overrides are applied in order.
851 #[must_use]
852 pub fn with_tensor_type_override(mut self, ov: TensorTypeOverride) -> Self {
853 self.tt_overrides.push(ov);
854 self
855 }
856
857 /// Mark a layer index for pruning (removal) from the output model.
858 #[must_use]
859 pub fn with_pruned_layer(mut self, layer: i32) -> Self {
860 self.prune_layers.push(layer);
861 self
862 }
863
864 /// Mark multiple layer indices for pruning.
865 #[must_use]
866 pub fn with_pruned_layers(mut self, layers: impl IntoIterator<Item = i32>) -> Self {
867 self.prune_layers.extend(layers);
868 self
869 }
870
871 /// Build the raw C struct, together with the temporary backing storage
872 /// that must outlive the struct. Returns `(raw_params, _guards)`.
873 ///
874 /// This is `pub(crate)` so that `model_quantize` can call it safely while
875 /// holding all the guards alive.
876 #[allow(clippy::too_many_lines)]
877 pub(crate) fn to_raw(&self) -> RawQuantizeParamsGuard<'_> {
878 // ── imatrix ─────────────────────────────────────────────────────────
879 // Build a null-terminated array of llama_model_imatrix_data.
880 // The `name` and `data` pointers point directly into our owned Vecs.
881 let imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data> = self
882 .imatrix
883 .iter()
884 .map(|e| llama_cpp_sys_4::llama_model_imatrix_data {
885 name: e.name.as_ptr(),
886 data: e.data.as_ptr(),
887 size: e.data.len(),
888 })
889 .chain(std::iter::once(llama_cpp_sys_4::llama_model_imatrix_data {
890 name: null(),
891 data: null(),
892 size: 0,
893 }))
894 .collect();
895
896 // ── kv_overrides ────────────────────────────────────────────────────
897 // null-terminated by a sentinel with key[0] == 0
898 let kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override> = self
899 .kv_overrides
900 .iter()
901 .map(|kv| {
902 let mut raw = llama_cpp_sys_4::llama_model_kv_override {
903 key: [0; 128],
904 tag: 0,
905 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
906 val_i64: 0,
907 },
908 };
909 // Copy key bytes (up to 127 chars + NUL).
910 let bytes = kv.key.to_bytes_with_nul();
911 let copy_len = bytes.len().min(128);
912 for (dst, &src) in raw.key.iter_mut().zip(bytes[..copy_len].iter()) {
913 // `c_char` is `i8` on x86_64 but `u8` on ARM64/Android; `as _`
914 // infers the target signedness on every platform (issue #306).
915 // The wrap on `i8` targets is intentional: llama.cpp reads the
916 // key back as bytes, so the bit pattern is what matters.
917 #[allow(clippy::cast_possible_wrap)]
918 {
919 *dst = src as _;
920 }
921 }
922 match &kv.value {
923 KvOverrideValue::Int(v) => {
924 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_INT;
925 raw.__bindgen_anon_1 =
926 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_i64: *v };
927 }
928 KvOverrideValue::Float(v) => {
929 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_FLOAT;
930 raw.__bindgen_anon_1 =
931 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_f64: *v };
932 }
933 KvOverrideValue::Bool(v) => {
934 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_BOOL;
935 raw.__bindgen_anon_1 =
936 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_bool: *v };
937 }
938 KvOverrideValue::Str(s) => {
939 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_STR;
940 raw.__bindgen_anon_1 =
941 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_str: *s };
942 }
943 }
944 raw
945 })
946 .chain(std::iter::once(llama_cpp_sys_4::llama_model_kv_override {
947 key: [0; 128],
948 tag: 0,
949 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
950 val_i64: 0,
951 },
952 }))
953 .collect();
954
955 // ── tt_overrides ────────────────────────────────────────────────────
956 // null-terminated by { null, GGML_TYPE_COUNT }
957 let tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override> = self
958 .tt_overrides
959 .iter()
960 .map(|ov| llama_cpp_sys_4::llama_model_tensor_override {
961 pattern: ov.pattern.as_ptr(),
962 type_: ov.ty as llama_cpp_sys_4::ggml_type,
963 })
964 .chain(std::iter::once(
965 llama_cpp_sys_4::llama_model_tensor_override {
966 pattern: null(),
967 type_: llama_cpp_sys_4::GGML_TYPE_COUNT,
968 },
969 ))
970 .collect();
971
972 // ── prune_layers ─────────────────────────────────────────────────────
973 // -1-terminated
974 let mut prune_c = self.prune_layers.clone();
975 prune_c.push(-1);
976
977 // ── assemble ────────────────────────────────────────────────────────
978 let raw = llama_cpp_sys_4::llama_model_quantize_params {
979 nthread: self.nthread,
980 ftype: self.ftype as llama_cpp_sys_4::llama_ftype,
981 output_tensor_type: self
982 .output_tensor_type
983 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
984 t as llama_cpp_sys_4::ggml_type
985 }),
986 token_embedding_type: self
987 .token_embedding_type
988 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
989 t as llama_cpp_sys_4::ggml_type
990 }),
991 allow_requantize: self.allow_requantize,
992 quantize_output_tensor: self.quantize_output_tensor,
993 only_copy: self.only_copy,
994 pure_: self.pure,
995 keep_split: self.keep_split,
996 dry_run: self.dry_run,
997 max_buf_size: self.max_buf_size,
998 imatrix: if self.imatrix.is_empty() {
999 null()
1000 } else {
1001 imatrix_c.as_ptr()
1002 },
1003 kv_overrides: if self.kv_overrides.is_empty() {
1004 null()
1005 } else {
1006 kv_c.as_ptr()
1007 },
1008 tt_overrides: if self.tt_overrides.is_empty() {
1009 null()
1010 } else {
1011 tt_c.as_ptr()
1012 },
1013 prune_layers: if self.prune_layers.is_empty() {
1014 null()
1015 } else {
1016 prune_c.as_ptr()
1017 },
1018 };
1019
1020 RawQuantizeParamsGuard {
1021 raw,
1022 _imatrix_c: imatrix_c,
1023 _kv_c: kv_c,
1024 _tt_c: tt_c,
1025 _prune_c: prune_c,
1026 _marker: std::marker::PhantomData,
1027 }
1028 }
1029}
1030
1031/// Temporary storage that keeps the C pointers inside a raw
1032/// `llama_model_quantize_params` valid. Dropped after the quantize call.
1033pub(crate) struct RawQuantizeParamsGuard<'a> {
1034 pub(crate) raw: llama_cpp_sys_4::llama_model_quantize_params,
1035 _imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data>,
1036 _kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override>,
1037 _tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override>,
1038 _prune_c: Vec<i32>,
1039 // tie lifetime to the source QuantizeParams so the string/data
1040 // pointers inside imatrix_c and tt_c stay valid
1041 _marker: std::marker::PhantomData<&'a QuantizeParams>,
1042}
1043
1044// ─────────────────────────────────────────────────────────────────────────────
1045// TurboQuant – attention rotation
1046// ─────────────────────────────────────────────────────────────────────────────
1047
1048/// Control the `TurboQuant` attention-rotation feature globally.
1049///
1050/// When enabled (the default), llama.cpp applies a Hadamard rotation to Q/K/V
1051/// tensors before storing them in the KV cache. This significantly improves
1052/// quantization quality of the KV cache with near-zero overhead, as described
1053/// in llama.cpp PR #21038.
1054///
1055/// This function sets or clears the `LLAMA_ATTN_ROT_DISABLE` environment
1056/// variable, which llama.cpp reads once when a context (and its KV cache) is
1057/// first created. Call it **before** creating any [`LlamaContext`] on the
1058/// current process.
1059///
1060/// # Thread safety
1061///
1062/// Mutating environment variables while other threads may be reading them is
1063/// undefined behaviour. Call this function before spawning any threads that
1064/// use llama contexts, or ensure no contexts are being created concurrently.
1065///
1066/// # Example
1067///
1068/// ```no_run
1069/// // Disable the rotation for benchmarking purposes:
1070/// llama_cpp_4::quantize::set_attn_rot_disabled(true);
1071///
1072/// // Re-enable (default behaviour):
1073/// llama_cpp_4::quantize::set_attn_rot_disabled(false);
1074/// ```
1075///
1076/// [`LlamaContext`]: crate::context::LlamaContext
1077pub fn set_attn_rot_disabled(disabled: bool) {
1078 if disabled {
1079 // SAFETY: single-threaded context required by the caller.
1080 #[allow(unused_unsafe)]
1081 unsafe {
1082 std::env::set_var("LLAMA_ATTN_ROT_DISABLE", "1");
1083 }
1084 } else {
1085 #[allow(unused_unsafe)]
1086 unsafe {
1087 std::env::remove_var("LLAMA_ATTN_ROT_DISABLE");
1088 }
1089 }
1090}
1091
1092/// Returns `true` if `TurboQuant` attention rotation is currently disabled.
1093#[must_use]
1094pub fn attn_rot_disabled() -> bool {
1095 std::env::var("LLAMA_ATTN_ROT_DISABLE")
1096 .ok()
1097 .and_then(|v| v.parse::<i32>().ok())
1098 .is_some_and(|v| v != 0)
1099}
1100
1101// ─────────────────────────────────────────────────────────────────────────────
1102// Quantization preview
1103// ─────────────────────────────────────────────────────────────────────────────
1104
1105/// Ask llama.cpp what it *would* do, without writing a file.
1106///
1107/// [`crate::model_quantize`] is all-or-nothing: it reads a model, quantizes
1108/// every tensor, and writes the result. This exposes the decision layer
1109/// underneath — which tensors get quantized at all, and to which `ggml` type —
1110/// so a tool can show the plan, or estimate output size, before committing to
1111/// a run that may take minutes and tens of gigabytes.
1112///
1113/// The k-quant mixes are why this is not simply [`LlamaFtype::default_ggml_type`]:
1114/// they deliberately keep attention and output tensors at higher precision, so
1115/// the per-tensor answer differs from the ftype's nominal type.
1116///
1117/// Wraps `llama_quant_init` / `llama_quant_free`.
1118#[derive(Debug)]
1119pub struct QuantPreview<'model> {
1120 qs: std::ptr::NonNull<llama_cpp_sys_4::quantize_state_impl>,
1121 // The state borrows the model it was built from.
1122 _model: std::marker::PhantomData<&'model crate::model::LlamaModel>,
1123}
1124
1125impl Drop for QuantPreview<'_> {
1126 fn drop(&mut self) {
1127 unsafe { llama_cpp_sys_4::llama_quant_free(self.qs.as_ptr()) }
1128 }
1129}
1130
1131impl<'model> QuantPreview<'model> {
1132 /// Build a preview for `model` under `params`.
1133 ///
1134 /// # Errors
1135 ///
1136 /// Returns [`QuantPreviewError::Init`] if llama.cpp could not build the
1137 /// quantization state, which happens for a model it cannot quantize.
1138 pub fn new(
1139 model: &'model crate::model::LlamaModel,
1140 params: &QuantizeParams,
1141 ) -> Result<Self, QuantPreviewError> {
1142 let guard = params.to_raw();
1143 // The guarded wrapper: `llama_quant_init` throws for a model it cannot
1144 // quantize, and that unwinding into Rust aborts the process.
1145 let qs = unsafe {
1146 llama_cpp_sys_4::llama_quant_init_guarded(model.model.as_ptr(), &raw const guard.raw)
1147 };
1148 std::ptr::NonNull::new(qs)
1149 .map(|qs| Self {
1150 qs,
1151 _model: std::marker::PhantomData,
1152 })
1153 .ok_or(QuantPreviewError::Init)
1154 }
1155
1156 /// Whether this tensor would be quantized at all.
1157 ///
1158 /// llama.cpp skips 1-D tensors, tensors below a size threshold, and ones
1159 /// whose name marks them as needing full precision — so a `false` here is
1160 /// the usual reason a tensor keeps its original type.
1161 ///
1162 /// Requires the `ggml` feature, which is what exposes [`crate::ggml::GgmlTensor`].
1163 #[cfg(feature = "ggml")]
1164 #[must_use]
1165 pub fn allows_quantization(&self, tensor: &crate::ggml::GgmlTensor) -> bool {
1166 // 1 = yes, 0 = no, -1 = the underlying call threw and was caught.
1167 let rc = unsafe {
1168 llama_cpp_sys_4::llama_quant_tensor_allows_quantization_guarded(
1169 self.qs.as_ptr(),
1170 tensor.as_ptr(),
1171 )
1172 };
1173 rc == 1
1174 }
1175
1176 /// Compute the storage type each tensor would be assigned under `ftype`.
1177 ///
1178 /// Every tensor passed must already satisfy [`Self::allows_quantization`] —
1179 /// upstream states the caller filters first, and does not re-check.
1180 ///
1181 /// An entry is `None` when llama.cpp picks a `ggml` type this crate's
1182 /// [`GgmlType`] does not know.
1183 ///
1184 /// # Errors
1185 ///
1186 /// Returns [`QuantPreviewError::NotQuantizable`] naming the first tensor
1187 /// that fails the filter, rather than letting llama.cpp decide what to do
1188 /// with it.
1189 ///
1190 /// Requires the `ggml` feature.
1191 #[cfg(feature = "ggml")]
1192 pub fn compute_types(
1193 &self,
1194 tensors: &[&crate::ggml::GgmlTensor],
1195 ftype: LlamaFtype,
1196 ) -> Result<Vec<Option<GgmlType>>, QuantPreviewError> {
1197 if tensors.is_empty() {
1198 return Ok(Vec::new());
1199 }
1200 for tensor in tensors {
1201 if !self.allows_quantization(tensor) {
1202 return Err(QuantPreviewError::NotQuantizable(tensor.name().to_owned()));
1203 }
1204 }
1205
1206 let mut raw_tensors: Vec<*mut llama_cpp_sys_4::ggml_tensor> =
1207 tensors.iter().map(|t| t.as_ptr()).collect();
1208 let mut out = vec![0 as llama_cpp_sys_4::ggml_type; tensors.len()];
1209 let rc = unsafe {
1210 llama_cpp_sys_4::llama_quant_compute_types_guarded(
1211 self.qs.as_ptr(),
1212 ftype.into(),
1213 raw_tensors.as_mut_ptr(),
1214 out.as_mut_ptr(),
1215 out.len(),
1216 )
1217 };
1218 if rc != 0 {
1219 return Err(QuantPreviewError::Init);
1220 }
1221 Ok(out
1222 .into_iter()
1223 .map(|t| GgmlType::try_from(t).ok())
1224 .collect())
1225 }
1226}
1227
1228/// Failure building or using a [`QuantPreview`].
1229#[derive(Debug, Eq, PartialEq, thiserror::Error)]
1230pub enum QuantPreviewError {
1231 /// llama.cpp could not build a quantization state for this model.
1232 #[error("could not initialize quantization state")]
1233 Init,
1234 /// A tensor that would not be quantized was passed to
1235 /// `QuantPreview::compute_types` (which requires the `ggml` feature).
1236 #[error("tensor {0:?} is not quantizable; filter with allows_quantization first")]
1237 NotQuantizable(String),
1238 /// The mock-model descriptor contained an interior NUL byte.
1239 #[error("architecture name contained an interior NUL byte")]
1240 Nul(#[from] NulError),
1241 /// llama.cpp could not build a model from the descriptor.
1242 #[error("could not build a mock model from the descriptor")]
1243 MockModel,
1244}
1245
1246/// Shape of a synthetic model, for previewing a quantization plan without a
1247/// real checkpoint on disk.
1248///
1249/// Upstream marks `llama_quant_model_from_metadata` as being for testing; it is
1250/// exposed here for the same reason — it lets you ask "what would a `Q4_K_M` of a
1251/// model this shape look like?" without downloading one.
1252#[derive(Debug, Clone, PartialEq, Eq)]
1253pub struct QuantModelDesc {
1254 /// Architecture name, e.g. `"llama"`.
1255 pub architecture: String,
1256 /// Embedding dimension.
1257 pub n_embd: u32,
1258 /// Feed-forward dimension.
1259 pub n_ff: u32,
1260 /// Number of layers.
1261 pub n_layer: u32,
1262 /// Number of attention heads.
1263 pub n_head: u32,
1264 /// Number of key/value heads.
1265 pub n_head_kv: u32,
1266 /// Number of experts (0 for a dense model).
1267 pub n_expert: u32,
1268 /// Per-head key dimension.
1269 pub n_embd_head_k: u32,
1270 /// Per-head value dimension.
1271 pub n_embd_head_v: u32,
1272}
1273
1274impl QuantModelDesc {
1275 /// A dense llama-shaped descriptor with the given dimensions.
1276 #[must_use]
1277 pub fn llama(n_embd: u32, n_ff: u32, n_layer: u32, n_head: u32) -> Self {
1278 Self {
1279 architecture: "llama".to_owned(),
1280 n_embd,
1281 n_ff,
1282 n_layer,
1283 n_head,
1284 n_head_kv: n_head,
1285 n_expert: 0,
1286 n_embd_head_k: n_embd / n_head.max(1),
1287 n_embd_head_v: n_embd / n_head.max(1),
1288 }
1289 }
1290
1291 /// Build a synthetic model from this descriptor.
1292 ///
1293 /// The returned model owns llama.cpp memory and frees it on drop, exactly
1294 /// like a loaded one — it simply has no weights.
1295 ///
1296 /// # Errors
1297 ///
1298 /// Returns [`QuantPreviewError::MockModel`] if llama.cpp rejects the
1299 /// descriptor, or [`QuantPreviewError::Nul`] for a bad architecture name.
1300 pub fn build(&self) -> Result<crate::model::LlamaModel, QuantPreviewError> {
1301 let arch = CString::new(self.architecture.as_str())?;
1302 let desc = llama_cpp_sys_4::llama_quant_model_desc {
1303 architecture: arch.as_ptr(),
1304 n_embd: self.n_embd,
1305 n_ff: self.n_ff,
1306 n_layer: self.n_layer,
1307 n_head: self.n_head,
1308 n_head_kv: self.n_head_kv,
1309 n_expert: self.n_expert,
1310 n_embd_head_k: self.n_embd_head_k,
1311 n_embd_head_v: self.n_embd_head_v,
1312 };
1313 // Guarded: an architecture llama.cpp does not know makes the raw entry
1314 // point throw, which aborts the process on the way back into Rust.
1315 let raw =
1316 unsafe { llama_cpp_sys_4::llama_quant_model_from_metadata_guarded(&raw const desc) };
1317 let ptr = std::ptr::NonNull::new(raw).ok_or(QuantPreviewError::MockModel)?;
1318 // SAFETY: upstream documents the result as owned by the caller and
1319 // freed with `llama_model_free`, which is what `LlamaModel` does.
1320 Ok(unsafe { crate::model::LlamaModel::from_raw(ptr) })
1321 }
1322}