llama_cpp_4/quantize.rs
1//! Quantization types and parameters for converting models to lower-bit precisions.
2//!
3//! # Quick start
4//!
5//! ```no_run
6//! use llama_cpp_4::quantize::{LlamaFtype, QuantizeParams};
7//!
8//! let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
9//! .with_nthread(8)
10//! .with_quantize_output_tensor(true);
11//!
12//! llama_cpp_4::model_quantize("model-f16.gguf", "model-q4km.gguf", ¶ms).unwrap();
13//! ```
14//!
15//! # `TurboQuant` – attention rotation (PR #21038)
16//!
17//! llama.cpp applies a Hadamard rotation to Q/K/V tensors before writing them into the KV cache.
18//! This significantly improves KV-cache quantization quality at near-zero cost, and is enabled by
19//! default for every model whose head dimension is a power of two. You can opt out per-context
20//! with [`LlamaContextParams::with_attn_rot_disabled`] or globally with
21//! [`set_attn_rot_disabled`].
22//!
23//! [`LlamaContextParams::with_attn_rot_disabled`]: crate::context::params::LlamaContextParams::with_attn_rot_disabled
24
25use std::ffi::{CString, NulError};
26use std::ptr::null;
27
28// ─────────────────────────────────────────────────────────────────────────────
29// LlamaFtype
30// ─────────────────────────────────────────────────────────────────────────────
31
32/// The quantization type used for the bulk of a model file (maps to `llama_ftype`).
33///
34/// Pass one of these variants to [`QuantizeParams::new`] to choose the target precision.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36#[non_exhaustive]
37#[allow(missing_docs)]
38pub enum LlamaFtype {
39 /// All tensors stored as full F32 (very large, for reference only)
40 AllF32 = 0,
41 /// F16 – 14 GB @ 7B, +0.0020 ppl vs Mistral-7B
42 MostlyF16 = 1,
43 /// `Q4_0` – 4.34 GB @ 8B, +0.4685 ppl
44 MostlyQ4_0 = 2,
45 /// `Q4_1` – 4.78 GB @ 8B, +0.4511 ppl
46 MostlyQ4_1 = 3,
47 /// `Q8_0` – 7.96 GB @ 8B, +0.0026 ppl
48 MostlyQ8_0 = 7,
49 /// `Q5_0` – 5.21 GB @ 8B, +0.1316 ppl
50 MostlyQ5_0 = 8,
51 /// `Q5_1` – 5.65 GB @ 8B, +0.1062 ppl
52 MostlyQ5_1 = 9,
53 /// `Q2_K` – 2.96 GB @ 8B, +3.5199 ppl
54 MostlyQ2K = 10,
55 /// `Q3_K` small – 3.41 GB @ 8B, +1.6321 ppl
56 MostlyQ3KS = 11,
57 /// `Q3_K` medium – 3.74 GB @ 8B, +0.6569 ppl
58 MostlyQ3KM = 12,
59 /// `Q3_K` large – 4.03 GB @ 8B, +0.5562 ppl
60 MostlyQ3KL = 13,
61 /// `Q4_K` small – 4.37 GB @ 8B, +0.2689 ppl
62 MostlyQ4KS = 14,
63 /// `Q4_K` medium – 4.58 GB @ 8B, +0.1754 ppl *(recommended default)*
64 MostlyQ4KM = 15,
65 /// `Q5_K` small – 5.21 GB @ 8B, +0.1049 ppl
66 MostlyQ5KS = 16,
67 /// `Q5_K` medium – 5.33 GB @ 8B, +0.0569 ppl
68 MostlyQ5KM = 17,
69 /// `Q6_K` – 6.14 GB @ 8B, +0.0217 ppl
70 MostlyQ6K = 18,
71 /// `IQ2_XXS` – 2.06 bpw
72 MostlyIQ2XXS = 19,
73 /// `IQ2_XS` – 2.31 bpw
74 MostlyIQ2XS = 20,
75 /// `Q2_K` small
76 MostlyQ2KS = 21,
77 /// `IQ3_XS` – 3.3 bpw
78 MostlyIQ3XS = 22,
79 /// `IQ3_XXS` – 3.06 bpw
80 MostlyIQ3XXS = 23,
81 /// `IQ1_S` – 1.56 bpw (extremely small, high loss)
82 MostlyIQ1S = 24,
83 /// `IQ4_NL` – 4.50 bpw non-linear
84 MostlyIQ4NL = 25,
85 /// `IQ3_S` – 3.44 bpw
86 MostlyIQ3S = 26,
87 /// `IQ3_M` – 3.66 bpw
88 MostlyIQ3M = 27,
89 /// `IQ2_S` – 2.5 bpw
90 MostlyIQ2S = 28,
91 /// `IQ2_M` – 2.7 bpw
92 MostlyIQ2M = 29,
93 /// `IQ4_XS` – 4.25 bpw non-linear
94 MostlyIQ4XS = 30,
95 /// `IQ1_M` – 1.75 bpw
96 MostlyIQ1M = 31,
97 /// BF16 – 14 GB @ 7B, −0.0050 ppl vs Mistral-7B
98 MostlyBF16 = 32,
99 /// `TQ1_0` – 1.69 bpw ternary
100 MostlyTQ1_0 = 36,
101 /// `TQ2_0` – 2.06 bpw ternary
102 MostlyTQ2_0 = 37,
103 /// MXFP4 (`MoE` layers)
104 MostlyMXFP4Moe = 38,
105 /// NVFP4
106 MostlyNVFP4 = 39,
107 /// `Q1_0` – 1.5 bpw binary (block size 32)
108 MostlyQ1_0 = 40,
109 /// `Q2_0` – 2 bpw binary
110 //
111 // 41 was `Q1_0_g128` behind the `q1` feature, from this crate's own
112 // quantization patch. Upstream uses 41 for `Q2_0`, and never had a
113 // `Q1_0_g128`, so the old variant named a type that does not exist while
114 // sitting on a value that means something else.
115 MostlyQ2_0 = 41,
116 /// `F8_E4M3` – 8-bit float (except 1d tensors)
117 MostlyF8E4M3 = 42,
118}
119
120impl LlamaFtype {
121 /// Short name suitable for filenames (e.g. `"Q4_K_M"`).
122 #[must_use]
123 pub fn name(self) -> &'static str {
124 match self {
125 Self::AllF32 => "F32",
126 Self::MostlyF16 => "F16",
127 Self::MostlyQ4_0 => "Q4_0",
128 Self::MostlyQ4_1 => "Q4_1",
129 Self::MostlyQ8_0 => "Q8_0",
130 Self::MostlyQ5_0 => "Q5_0",
131 Self::MostlyQ5_1 => "Q5_1",
132 Self::MostlyQ2K => "Q2_K",
133 Self::MostlyQ3KS => "Q3_K_S",
134 Self::MostlyQ3KM => "Q3_K_M",
135 Self::MostlyQ3KL => "Q3_K_L",
136 Self::MostlyQ4KS => "Q4_K_S",
137 Self::MostlyQ4KM => "Q4_K_M",
138 Self::MostlyQ5KS => "Q5_K_S",
139 Self::MostlyQ5KM => "Q5_K_M",
140 Self::MostlyQ6K => "Q6_K",
141 Self::MostlyIQ2XXS => "IQ2_XXS",
142 Self::MostlyIQ2XS => "IQ2_XS",
143 Self::MostlyQ2KS => "Q2_K_S",
144 Self::MostlyIQ3XS => "IQ3_XS",
145 Self::MostlyIQ3XXS => "IQ3_XXS",
146 Self::MostlyIQ1S => "IQ1_S",
147 Self::MostlyIQ4NL => "IQ4_NL",
148 Self::MostlyIQ3S => "IQ3_S",
149 Self::MostlyIQ3M => "IQ3_M",
150 Self::MostlyIQ2S => "IQ2_S",
151 Self::MostlyIQ2M => "IQ2_M",
152 Self::MostlyIQ4XS => "IQ4_XS",
153 Self::MostlyIQ1M => "IQ1_M",
154 Self::MostlyBF16 => "BF16",
155 Self::MostlyTQ1_0 => "TQ1_0",
156 Self::MostlyTQ2_0 => "TQ2_0",
157 Self::MostlyMXFP4Moe => "MXFP4_MOE",
158 Self::MostlyNVFP4 => "NVFP4",
159 Self::MostlyQ1_0 => "Q1_0",
160 Self::MostlyQ2_0 => "Q2_0",
161 Self::MostlyF8E4M3 => "F8_E4M3",
162 }
163 }
164
165 /// Human-readable description with approximate size and PPL delta.
166 #[must_use]
167 pub fn description(self) -> &'static str {
168 match self {
169 Self::AllF32 => "26.00 GB @ 7B — full precision reference",
170 Self::MostlyF16 => "14.00 GB @ 7B — +0.0020 ppl vs Mistral-7B",
171 Self::MostlyBF16 => "14.00 GB @ 7B — -0.0050 ppl vs Mistral-7B",
172 Self::MostlyQ8_0 => " 7.96 GB @ 8B — +0.0026 ppl",
173 Self::MostlyQ6K => " 6.14 GB @ 8B — +0.0217 ppl",
174 Self::MostlyQ5KM => " 5.33 GB @ 8B — +0.0569 ppl",
175 Self::MostlyQ5KS => " 5.21 GB @ 8B — +0.1049 ppl",
176 Self::MostlyQ5_1 => " 5.65 GB @ 8B — +0.1062 ppl",
177 Self::MostlyQ5_0 => " 5.21 GB @ 8B — +0.1316 ppl",
178 Self::MostlyQ4KM => " 4.58 GB @ 8B — +0.1754 ppl [recommended]",
179 Self::MostlyQ4KS => " 4.37 GB @ 8B — +0.2689 ppl",
180 Self::MostlyQ4_1 => " 4.78 GB @ 8B — +0.4511 ppl",
181 Self::MostlyQ4_0 => " 4.34 GB @ 8B — +0.4685 ppl",
182 Self::MostlyQ3KL => " 4.03 GB @ 8B — +0.5562 ppl",
183 Self::MostlyQ3KM => " 3.74 GB @ 8B — +0.6569 ppl",
184 Self::MostlyQ3KS => " 3.41 GB @ 8B — +1.6321 ppl",
185 Self::MostlyQ2KS => " 2.96 GB @ 8B — +3.1836 ppl",
186 Self::MostlyQ2K => " 2.96 GB @ 8B — +3.5199 ppl",
187 Self::MostlyIQ4XS => " 4.25 bpw non-linear",
188 Self::MostlyIQ4NL => " 4.50 bpw non-linear",
189 Self::MostlyIQ3S => " 3.44 bpw",
190 Self::MostlyIQ3M => " 3.66 bpw",
191 Self::MostlyIQ3XS => " 3.3 bpw",
192 Self::MostlyIQ3XXS => " 3.06 bpw",
193 Self::MostlyIQ2M => " 2.7 bpw",
194 Self::MostlyIQ2S => " 2.5 bpw",
195 Self::MostlyIQ2XS => " 2.31 bpw",
196 Self::MostlyIQ2XXS => " 2.06 bpw",
197 Self::MostlyIQ1M => " 1.75 bpw — extreme compression",
198 Self::MostlyIQ1S => " 1.56 bpw — extreme compression",
199 Self::MostlyTQ1_0 => " 1.69 bpw ternary",
200 Self::MostlyTQ2_0 => " 2.06 bpw ternary",
201 Self::MostlyMXFP4Moe => "MXFP4 MoE layers",
202 Self::MostlyNVFP4 => "NVFP4",
203 Self::MostlyQ1_0 => " 1.50 bpw — binary Q1_0 (block 32)",
204 Self::MostlyQ2_0 => " 2.00 bpw — binary Q2_0",
205 Self::MostlyF8E4M3 => " 8.00 bpw — float8 e4m3",
206 }
207 }
208
209 /// Look up a variant by its short name (case-insensitive).
210 ///
211 /// ```
212 /// use llama_cpp_4::quantize::LlamaFtype;
213 /// assert_eq!(LlamaFtype::from_name("Q4_K_M"), Some(LlamaFtype::MostlyQ4KM));
214 /// assert_eq!(LlamaFtype::from_name("q4_k_m"), Some(LlamaFtype::MostlyQ4KM));
215 /// assert_eq!(LlamaFtype::from_name("bogus"), None);
216 /// ```
217 #[must_use]
218 pub fn from_name(name: &str) -> Option<Self> {
219 let upper = name.to_uppercase();
220 match upper.as_str() {
221 "F32" => Some(Self::AllF32),
222 "F16" => Some(Self::MostlyF16),
223 "BF16" => Some(Self::MostlyBF16),
224 "Q4_0" => Some(Self::MostlyQ4_0),
225 "Q4_1" => Some(Self::MostlyQ4_1),
226 "Q8_0" => Some(Self::MostlyQ8_0),
227 "Q5_0" => Some(Self::MostlyQ5_0),
228 "Q5_1" => Some(Self::MostlyQ5_1),
229 "Q2_K" => Some(Self::MostlyQ2K),
230 "Q2_K_S" => Some(Self::MostlyQ2KS),
231 "Q3_K_S" => Some(Self::MostlyQ3KS),
232 "Q3_K_M" => Some(Self::MostlyQ3KM),
233 "Q3_K_L" => Some(Self::MostlyQ3KL),
234 "Q4_K_S" => Some(Self::MostlyQ4KS),
235 "Q4_K_M" => Some(Self::MostlyQ4KM),
236 "Q5_K_S" => Some(Self::MostlyQ5KS),
237 "Q5_K_M" => Some(Self::MostlyQ5KM),
238 "Q6_K" => Some(Self::MostlyQ6K),
239 "IQ1_S" => Some(Self::MostlyIQ1S),
240 "IQ1_M" => Some(Self::MostlyIQ1M),
241 "IQ2_XXS" => Some(Self::MostlyIQ2XXS),
242 "IQ2_XS" => Some(Self::MostlyIQ2XS),
243 "IQ2_S" => Some(Self::MostlyIQ2S),
244 "IQ2_M" => Some(Self::MostlyIQ2M),
245 "IQ3_XXS" => Some(Self::MostlyIQ3XXS),
246 "IQ3_XS" => Some(Self::MostlyIQ3XS),
247 "IQ3_S" => Some(Self::MostlyIQ3S),
248 "IQ3_M" => Some(Self::MostlyIQ3M),
249 "IQ4_NL" => Some(Self::MostlyIQ4NL),
250 "IQ4_XS" => Some(Self::MostlyIQ4XS),
251 "TQ1_0" => Some(Self::MostlyTQ1_0),
252 "TQ2_0" => Some(Self::MostlyTQ2_0),
253 "MXFP4_MOE" => Some(Self::MostlyMXFP4Moe),
254 "NVFP4" => Some(Self::MostlyNVFP4),
255 "Q1_0" => Some(Self::MostlyQ1_0),
256 "Q2_0" => Some(Self::MostlyQ2_0),
257 "F8_E4M3" => Some(Self::MostlyF8E4M3),
258 _ => None,
259 }
260 }
261
262 /// llama.cpp's own display name for this type, e.g. `"Q4_K - Medium"`.
263 ///
264 /// Where [`Self::name`] is a terse filename-safe token (`"Q4_K_M"`) kept in
265 /// a table in this crate, this asks llama.cpp, so it cannot drift from
266 /// upstream. Wraps `llama_ftype_name`.
267 ///
268 /// # Errors
269 ///
270 /// Returns [`FtypeNameError`] if llama.cpp returns null or non-UTF-8.
271 pub fn upstream_name(self) -> Result<&'static str, FtypeNameError> {
272 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
273 // the library owns whatever it returns.
274 let ptr = unsafe { llama_cpp_sys_4::llama_ftype_name(self.into()) };
275 if ptr.is_null() {
276 return Err(FtypeNameError::Unnamed(self));
277 }
278 // SAFETY: llama.cpp returns a pointer to a string literal with static
279 // storage duration, so `'static` is sound here.
280 unsafe { std::ffi::CStr::from_ptr(ptr) }
281 .to_str()
282 .map_err(FtypeNameError::Utf8)
283 }
284
285 /// The `ggml` storage type most tensors get under this ftype.
286 ///
287 /// This is the *default*; the k-quant mixes deliberately store some tensors
288 /// (attention, output) at higher precision, so an individual tensor may not
289 /// use it. Wraps `llama_ftype_get_default_type`.
290 ///
291 /// Returns `None` for a `ggml` type this crate's [`GgmlType`] does not know.
292 #[must_use]
293 pub fn default_ggml_type(self) -> Option<GgmlType> {
294 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
295 // the library owns whatever it returns.
296 let raw = unsafe { llama_cpp_sys_4::llama_ftype_get_default_type(self.into()) };
297 GgmlType::try_from(raw).ok()
298 }
299
300 /// All available types, ordered roughly from largest to smallest.
301 #[must_use]
302 pub fn all() -> &'static [Self] {
303 &[
304 Self::AllF32,
305 Self::MostlyF16,
306 Self::MostlyBF16,
307 Self::MostlyQ8_0,
308 Self::MostlyQ6K,
309 Self::MostlyQ5KM,
310 Self::MostlyQ5KS,
311 Self::MostlyQ5_1,
312 Self::MostlyQ5_0,
313 Self::MostlyQ4KM,
314 Self::MostlyQ4KS,
315 Self::MostlyQ4_1,
316 Self::MostlyQ4_0,
317 Self::MostlyQ3KL,
318 Self::MostlyQ3KM,
319 Self::MostlyQ3KS,
320 Self::MostlyQ2KS,
321 Self::MostlyQ2K,
322 Self::MostlyIQ4XS,
323 Self::MostlyIQ4NL,
324 Self::MostlyIQ3S,
325 Self::MostlyIQ3M,
326 Self::MostlyIQ3XS,
327 Self::MostlyIQ3XXS,
328 Self::MostlyIQ2M,
329 Self::MostlyIQ2S,
330 Self::MostlyIQ2XS,
331 Self::MostlyIQ2XXS,
332 Self::MostlyIQ1M,
333 Self::MostlyIQ1S,
334 Self::MostlyTQ1_0,
335 Self::MostlyTQ2_0,
336 Self::MostlyMXFP4Moe,
337 Self::MostlyNVFP4,
338 Self::MostlyQ1_0,
339 Self::MostlyQ2_0,
340 Self::MostlyF8E4M3,
341 ]
342 }
343}
344
345/// Failure reading llama.cpp's display name for an [`LlamaFtype`].
346#[derive(Debug, Eq, PartialEq, thiserror::Error)]
347pub enum FtypeNameError {
348 /// llama.cpp returned null for this type.
349 #[error("llama.cpp has no name for ftype {0:?}")]
350 Unnamed(LlamaFtype),
351 /// The returned bytes were not valid UTF-8.
352 #[error(transparent)]
353 Utf8(#[from] std::str::Utf8Error),
354}
355
356impl From<LlamaFtype> for llama_cpp_sys_4::llama_ftype {
357 fn from(t: LlamaFtype) -> Self {
358 t as llama_cpp_sys_4::llama_ftype
359 }
360}
361
362impl TryFrom<llama_cpp_sys_4::llama_ftype> for LlamaFtype {
363 type Error = llama_cpp_sys_4::llama_ftype;
364
365 /// Fails on a discriminant this build does not know — `LLAMA_FTYPE_GUESSED`
366 /// (1024, "the file did not say"), a `q1` type without that feature, or a
367 /// type upstream added since this release. The raw value is returned so the
368 /// caller can report it.
369 fn try_from(raw: llama_cpp_sys_4::llama_ftype) -> Result<Self, Self::Error> {
370 Self::all()
371 .iter()
372 .copied()
373 .find(|t| llama_cpp_sys_4::llama_ftype::from(*t) == raw)
374 .ok_or(raw)
375 }
376}
377
378impl std::fmt::Display for LlamaFtype {
379 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
380 write!(f, "{}", self.name())
381 }
382}
383
384// ─────────────────────────────────────────────────────────────────────────────
385// GgmlType
386// ─────────────────────────────────────────────────────────────────────────────
387
388/// GGML tensor storage type (maps to `ggml_type`).
389///
390/// Used to set [`QuantizeParams::output_tensor_type`] and
391/// [`QuantizeParams::token_embedding_type`], and for per-tensor type overrides
392/// in [`TensorTypeOverride`].
393#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
394#[non_exhaustive]
395#[allow(missing_docs)]
396pub enum GgmlType {
397 F32 = 0,
398 F16 = 1,
399 Q4_0 = 2,
400 Q4_1 = 3,
401 Q5_0 = 6,
402 Q5_1 = 7,
403 Q8_0 = 8,
404 Q8_1 = 9,
405 Q2K = 10,
406 Q3K = 11,
407 Q4K = 12,
408 Q5K = 13,
409 Q6K = 14,
410 Q8K = 15,
411 IQ2XXS = 16,
412 IQ2XS = 17,
413 IQ3XXS = 18,
414 IQ1S = 19,
415 IQ4NL = 20,
416 IQ3S = 21,
417 IQ2S = 22,
418 IQ4XS = 23,
419 I8 = 24,
420 I16 = 25,
421 I32 = 26,
422 I64 = 27,
423 F64 = 28,
424 IQ1M = 29,
425 BF16 = 30,
426 TQ1_0 = 34,
427 TQ2_0 = 35,
428 MXFP4 = 39,
429 /// NVFP4 (4 blocks, E4M3 scale).
430 NVFP4 = 40,
431 /// `Q1_0` — 1.5 bpw binary.
432 //
433 // These values used to be shuffled by the `q1` feature, which encoded the
434 // numbering of this crate's own `0001-q1-quantization.patch`: NVFP4 moved
435 // to 42 so Q1_0 could take 40 and a `Q1_0_G128` take 41. Q1_0 has since
436 // landed upstream at 41 — with Q2_0 at 42 and NVFP4 staying at 40 — and the
437 // patch is gone, so the feature was renumbering three variants away from
438 // what llama.cpp actually uses. Upstream's numbering is the only numbering.
439 Q1_0 = 41,
440 /// `Q2_0` — 2 bpw binary.
441 Q2_0 = 42,
442 /// `F8_E4M3` — 8-bit float, 4-bit exponent, 3-bit mantissa.
443 F8E4M3 = 43,
444}
445
446impl From<GgmlType> for llama_cpp_sys_4::ggml_type {
447 fn from(t: GgmlType) -> Self {
448 t as llama_cpp_sys_4::ggml_type
449 }
450}
451
452impl TryFrom<llama_cpp_sys_4::ggml_type> for GgmlType {
453 type Error = llama_cpp_sys_4::ggml_type;
454 fn try_from(v: llama_cpp_sys_4::ggml_type) -> Result<Self, Self::Error> {
455 match v {
456 0 => Ok(Self::F32),
457 1 => Ok(Self::F16),
458 2 => Ok(Self::Q4_0),
459 3 => Ok(Self::Q4_1),
460 6 => Ok(Self::Q5_0),
461 7 => Ok(Self::Q5_1),
462 8 => Ok(Self::Q8_0),
463 9 => Ok(Self::Q8_1),
464 10 => Ok(Self::Q2K),
465 11 => Ok(Self::Q3K),
466 12 => Ok(Self::Q4K),
467 13 => Ok(Self::Q5K),
468 14 => Ok(Self::Q6K),
469 15 => Ok(Self::Q8K),
470 16 => Ok(Self::IQ2XXS),
471 17 => Ok(Self::IQ2XS),
472 18 => Ok(Self::IQ3XXS),
473 19 => Ok(Self::IQ1S),
474 20 => Ok(Self::IQ4NL),
475 21 => Ok(Self::IQ3S),
476 22 => Ok(Self::IQ2S),
477 23 => Ok(Self::IQ4XS),
478 24 => Ok(Self::I8),
479 25 => Ok(Self::I16),
480 26 => Ok(Self::I32),
481 27 => Ok(Self::I64),
482 28 => Ok(Self::F64),
483 29 => Ok(Self::IQ1M),
484 30 => Ok(Self::BF16),
485 34 => Ok(Self::TQ1_0),
486 35 => Ok(Self::TQ2_0),
487 39 => Ok(Self::MXFP4),
488 40 => Ok(Self::NVFP4),
489 41 => Ok(Self::Q1_0),
490 42 => Ok(Self::Q2_0),
491 43 => Ok(Self::F8E4M3),
492 _ => Err(v),
493 }
494 }
495}
496
497// ─────────────────────────────────────────────────────────────────────────────
498// ImatrixEntry / Imatrix
499// ─────────────────────────────────────────────────────────────────────────────
500
501/// A single per-tensor importance matrix entry, as loaded from a `.imatrix` file.
502///
503/// Each entry contains activation statistics for one model tensor collected from
504/// a calibration dataset. When supplied to [`QuantizeParams::with_imatrix`] these
505/// statistics guide the quantizer to allocate more precision to weights that
506/// matter most.
507#[derive(Debug, Clone)]
508pub struct ImatrixEntry {
509 name: CString,
510 data: Vec<f32>,
511}
512
513impl ImatrixEntry {
514 /// Create a new entry from a tensor name and its importance scores.
515 ///
516 /// # Errors
517 ///
518 /// Returns [`NulError`] if `name` contains an interior null byte.
519 pub fn new(name: impl Into<Vec<u8>>, data: Vec<f32>) -> Result<Self, NulError> {
520 Ok(Self {
521 name: CString::new(name)?,
522 data,
523 })
524 }
525
526 /// Tensor name.
527 #[must_use]
528 pub fn name_str(&self) -> &str {
529 self.name.to_str().unwrap_or("")
530 }
531
532 /// Number of importance values.
533 #[must_use]
534 pub fn len(&self) -> usize {
535 self.data.len()
536 }
537
538 /// Returns `true` if the data slice is empty.
539 #[must_use]
540 pub fn is_empty(&self) -> bool {
541 self.data.is_empty()
542 }
543}
544
545/// A collection of importance matrix entries (one per quantized tensor).
546///
547/// Build one by pushing [`ImatrixEntry`] values, then pass it to
548/// [`QuantizeParams::with_imatrix`].
549#[derive(Debug, Clone, Default)]
550pub struct Imatrix {
551 entries: Vec<ImatrixEntry>,
552}
553
554impl Imatrix {
555 /// Create an empty imatrix.
556 #[must_use]
557 pub fn new() -> Self {
558 Self::default()
559 }
560
561 /// Add an entry.
562 pub fn push(&mut self, entry: ImatrixEntry) {
563 self.entries.push(entry);
564 }
565
566 /// Number of entries.
567 #[must_use]
568 pub fn len(&self) -> usize {
569 self.entries.len()
570 }
571
572 /// Returns `true` if no entries have been added.
573 #[must_use]
574 pub fn is_empty(&self) -> bool {
575 self.entries.is_empty()
576 }
577}
578
579// ─────────────────────────────────────────────────────────────────────────────
580// TensorTypeOverride
581// ─────────────────────────────────────────────────────────────────────────────
582
583/// Override the quantization type of every tensor whose name matches a glob `pattern`.
584///
585/// The pattern syntax is the same as used by the `--tensor-type` flag in
586/// `llama-quantize`, e.g. `"attn.*"` or `"blk.0.*"`.
587///
588/// # Example
589///
590/// ```
591/// use llama_cpp_4::quantize::{GgmlType, TensorTypeOverride};
592///
593/// // Keep the output projection in F16:
594/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
595/// ```
596#[derive(Debug, Clone)]
597pub struct TensorTypeOverride {
598 pattern: CString,
599 ty: GgmlType,
600}
601
602impl TensorTypeOverride {
603 /// Create a new override.
604 ///
605 /// # Errors
606 ///
607 /// Returns [`NulError`] if `pattern` contains an interior null byte.
608 pub fn new(pattern: impl Into<Vec<u8>>, ty: GgmlType) -> Result<Self, NulError> {
609 Ok(Self {
610 pattern: CString::new(pattern)?,
611 ty,
612 })
613 }
614
615 /// The glob pattern that selects tensors.
616 #[must_use]
617 pub fn pattern_str(&self) -> &str {
618 self.pattern.to_str().unwrap_or("")
619 }
620
621 /// The type to assign to matching tensors.
622 #[must_use]
623 pub fn ty(&self) -> GgmlType {
624 self.ty
625 }
626}
627
628// ─────────────────────────────────────────────────────────────────────────────
629// KvOverrideValue / KvOverride
630// ─────────────────────────────────────────────────────────────────────────────
631
632/// A value in a GGUF key-value metadata override.
633#[derive(Debug, Clone, PartialEq)]
634pub enum KvOverrideValue {
635 /// 64-bit integer
636 Int(i64),
637 /// 64-bit float
638 Float(f64),
639 /// Boolean
640 Bool(bool),
641 /// Fixed-length string (up to 127 bytes + NUL)
642 Str([std::os::raw::c_char; 128]),
643}
644
645/// A single GGUF metadata key-value override.
646///
647/// These are written into the output file's metadata when quantizing.
648#[derive(Debug, Clone)]
649pub struct KvOverride {
650 key: CString,
651 /// The value for this override.
652 pub value: KvOverrideValue,
653}
654
655impl KvOverride {
656 /// Create a new override.
657 ///
658 /// # Errors
659 ///
660 /// Returns [`NulError`] if `key` contains an interior null byte.
661 pub fn new(key: impl Into<Vec<u8>>, value: KvOverrideValue) -> Result<Self, NulError> {
662 Ok(Self {
663 key: CString::new(key)?,
664 value,
665 })
666 }
667}
668
669// ─────────────────────────────────────────────────────────────────────────────
670// QuantizeParams
671// ─────────────────────────────────────────────────────────────────────────────
672
673/// Parameters for quantizing a model.
674///
675/// Create with [`QuantizeParams::new`] and chain `with_*` builder methods to
676/// configure, then pass a reference to [`crate::model_quantize`].
677///
678/// # Example
679///
680/// ```no_run
681/// use llama_cpp_4::quantize::{GgmlType, LlamaFtype, QuantizeParams, TensorTypeOverride};
682///
683/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
684///
685/// let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
686/// .with_nthread(8)
687/// .with_allow_requantize(false)
688/// .with_quantize_output_tensor(true)
689/// .with_pure(false)
690/// .with_tensor_type_override(ov);
691///
692/// llama_cpp_4::model_quantize("in.gguf", "out.gguf", ¶ms).unwrap();
693/// ```
694#[derive(Debug, Clone)]
695#[allow(clippy::struct_excessive_bools)]
696pub struct QuantizeParams {
697 /// Number of threads (0 = auto-detect).
698 pub nthread: i32,
699 /// Target quantization type.
700 pub ftype: LlamaFtype,
701 /// Force this storage type for the output/lm-head tensor (`None` = use ftype default).
702 pub output_tensor_type: Option<GgmlType>,
703 /// Force this storage type for the token-embedding tensor (`None` = use ftype default).
704 pub token_embedding_type: Option<GgmlType>,
705 /// Allow re-quantizing tensors that are already quantized.
706 pub allow_requantize: bool,
707 /// Quantize the output/lm-head weight tensor.
708 pub quantize_output_tensor: bool,
709 /// Copy all tensors without quantizing (ignores `ftype`).
710 pub only_copy: bool,
711 /// Quantize every tensor to the same type (no mixed k-quant strategy).
712 pub pure: bool,
713 /// Keep the same number of shards as the input (for split models).
714 pub keep_split: bool,
715 /// Estimate output size without writing anything to disk.
716 pub dry_run: bool,
717 /// Cap, in bytes, on the tensor rows held in memory at once (`0` = the
718 /// upstream default of 8 GiB). Lower it to quantize a model larger than
719 /// available RAM at the cost of more I/O.
720 pub max_buf_size: usize,
721
722 imatrix: Vec<ImatrixEntry>,
723 kv_overrides: Vec<KvOverride>,
724 tt_overrides: Vec<TensorTypeOverride>,
725 prune_layers: Vec<i32>,
726}
727
728impl QuantizeParams {
729 /// Create a new params set targeting `ftype`.
730 ///
731 /// All other options are set to the same defaults as
732 /// `llama_model_quantize_default_params()`.
733 #[must_use]
734 pub fn new(ftype: LlamaFtype) -> Self {
735 // Read the C defaults so we match them exactly.
736 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
737 // the library owns whatever it returns.
738 let d = unsafe { llama_cpp_sys_4::llama_model_quantize_default_params() };
739 Self {
740 nthread: d.nthread,
741 ftype,
742 output_tensor_type: GgmlType::try_from(d.output_tensor_type).ok(),
743 token_embedding_type: GgmlType::try_from(d.token_embedding_type).ok(),
744 allow_requantize: d.allow_requantize,
745 quantize_output_tensor: d.quantize_output_tensor,
746 only_copy: d.only_copy,
747 pure: d.pure_,
748 keep_split: d.keep_split,
749 dry_run: d.dry_run,
750 max_buf_size: d.max_buf_size,
751 imatrix: Vec::new(),
752 kv_overrides: Vec::new(),
753 tt_overrides: Vec::new(),
754 prune_layers: Vec::new(),
755 }
756 }
757
758 /// Set the number of quantization threads (`0` = auto).
759 #[must_use]
760 pub fn with_nthread(mut self, n: i32) -> Self {
761 self.nthread = n;
762 self
763 }
764
765 /// Override the output-tensor storage type.
766 #[must_use]
767 pub fn with_output_tensor_type(mut self, ty: GgmlType) -> Self {
768 self.output_tensor_type = Some(ty);
769 self
770 }
771
772 /// Override the token-embedding storage type.
773 #[must_use]
774 pub fn with_token_embedding_type(mut self, ty: GgmlType) -> Self {
775 self.token_embedding_type = Some(ty);
776 self
777 }
778
779 /// Allow (or disallow) re-quantizing already-quantized tensors.
780 #[must_use]
781 pub fn with_allow_requantize(mut self, v: bool) -> Self {
782 self.allow_requantize = v;
783 self
784 }
785
786 /// Quantize the output/lm-head weight (`true` by default).
787 #[must_use]
788 pub fn with_quantize_output_tensor(mut self, v: bool) -> Self {
789 self.quantize_output_tensor = v;
790 self
791 }
792
793 /// When `true`, only copy tensors verbatim (no quantization at all).
794 #[must_use]
795 pub fn with_only_copy(mut self, v: bool) -> Self {
796 self.only_copy = v;
797 self
798 }
799
800 /// When `true`, quantize all tensors to the same type (no mixed k-quant strategy).
801 #[must_use]
802 pub fn with_pure(mut self, v: bool) -> Self {
803 self.pure = v;
804 self
805 }
806
807 /// Preserve the number of shards when quantizing a split model.
808 #[must_use]
809 pub fn with_keep_split(mut self, v: bool) -> Self {
810 self.keep_split = v;
811 self
812 }
813
814 /// Only estimate the output size; do not write anything to disk.
815 #[must_use]
816 pub fn with_dry_run(mut self, v: bool) -> Self {
817 self.dry_run = v;
818 self
819 }
820
821 /// Cap the bytes of tensor rows kept in memory at once (`0` = upstream
822 /// default, 8 GiB).
823 #[must_use]
824 pub fn with_max_buf_size(mut self, bytes: usize) -> Self {
825 self.max_buf_size = bytes;
826 self
827 }
828
829 /// Supply importance matrix data to improve quantization quality.
830 ///
831 /// The imatrix is generated by the `imatrix` tool (or the `imatrix` example
832 /// in this crate) and contains per-tensor activation statistics collected
833 /// from a calibration dataset.
834 #[must_use]
835 pub fn with_imatrix(mut self, imatrix: Imatrix) -> Self {
836 self.imatrix = imatrix.entries;
837 self
838 }
839
840 /// Append a single imatrix entry.
841 #[must_use]
842 pub fn with_imatrix_entry(mut self, entry: ImatrixEntry) -> Self {
843 self.imatrix.push(entry);
844 self
845 }
846
847 /// Add (or replace) a GGUF metadata key-value pair in the output file.
848 #[must_use]
849 pub fn with_kv_override(mut self, kv: KvOverride) -> Self {
850 self.kv_overrides.push(kv);
851 self
852 }
853
854 /// Override the quantization type for tensors whose name matches `pattern`.
855 ///
856 /// Can be called multiple times; overrides are applied in order.
857 #[must_use]
858 pub fn with_tensor_type_override(mut self, ov: TensorTypeOverride) -> Self {
859 self.tt_overrides.push(ov);
860 self
861 }
862
863 /// Mark a layer index for pruning (removal) from the output model.
864 #[must_use]
865 pub fn with_pruned_layer(mut self, layer: i32) -> Self {
866 self.prune_layers.push(layer);
867 self
868 }
869
870 /// Mark multiple layer indices for pruning.
871 #[must_use]
872 pub fn with_pruned_layers(mut self, layers: impl IntoIterator<Item = i32>) -> Self {
873 self.prune_layers.extend(layers);
874 self
875 }
876
877 /// Build the raw C struct, together with the temporary backing storage
878 /// that must outlive the struct. Returns `(raw_params, _guards)`.
879 ///
880 /// This is `pub(crate)` so that `model_quantize` can call it safely while
881 /// holding all the guards alive.
882 #[allow(clippy::too_many_lines)]
883 pub(crate) fn to_raw(&self) -> RawQuantizeParamsGuard<'_> {
884 // ── imatrix ─────────────────────────────────────────────────────────
885 // Build a null-terminated array of llama_model_imatrix_data.
886 // The `name` and `data` pointers point directly into our owned Vecs.
887 let imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data> = self
888 .imatrix
889 .iter()
890 .map(|e| llama_cpp_sys_4::llama_model_imatrix_data {
891 name: e.name.as_ptr(),
892 data: e.data.as_ptr(),
893 size: e.data.len(),
894 })
895 .chain(std::iter::once(llama_cpp_sys_4::llama_model_imatrix_data {
896 name: null(),
897 data: null(),
898 size: 0,
899 }))
900 .collect();
901
902 // ── kv_overrides ────────────────────────────────────────────────────
903 // null-terminated by a sentinel with key[0] == 0
904 let kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override> = self
905 .kv_overrides
906 .iter()
907 .map(|kv| {
908 let mut raw = llama_cpp_sys_4::llama_model_kv_override {
909 key: [0; 128],
910 tag: 0,
911 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
912 val_i64: 0,
913 },
914 };
915 // Copy key bytes (up to 127 chars + NUL).
916 let bytes = kv.key.to_bytes_with_nul();
917 let copy_len = bytes.len().min(128);
918 for (dst, &src) in raw.key.iter_mut().zip(bytes[..copy_len].iter()) {
919 // `c_char` is `i8` on x86_64 but `u8` on ARM64/Android; `as _`
920 // infers the target signedness on every platform (issue #306).
921 // The wrap on `i8` targets is intentional: llama.cpp reads the
922 // key back as bytes, so the bit pattern is what matters.
923 #[allow(clippy::cast_possible_wrap)]
924 {
925 *dst = src as _;
926 }
927 }
928 match &kv.value {
929 KvOverrideValue::Int(v) => {
930 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_INT;
931 raw.__bindgen_anon_1 =
932 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_i64: *v };
933 }
934 KvOverrideValue::Float(v) => {
935 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_FLOAT;
936 raw.__bindgen_anon_1 =
937 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_f64: *v };
938 }
939 KvOverrideValue::Bool(v) => {
940 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_BOOL;
941 raw.__bindgen_anon_1 =
942 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_bool: *v };
943 }
944 KvOverrideValue::Str(s) => {
945 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_STR;
946 raw.__bindgen_anon_1 =
947 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_str: *s };
948 }
949 }
950 raw
951 })
952 .chain(std::iter::once(llama_cpp_sys_4::llama_model_kv_override {
953 key: [0; 128],
954 tag: 0,
955 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
956 val_i64: 0,
957 },
958 }))
959 .collect();
960
961 // ── tt_overrides ────────────────────────────────────────────────────
962 // null-terminated by { null, GGML_TYPE_COUNT }
963 let tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override> = self
964 .tt_overrides
965 .iter()
966 .map(|ov| llama_cpp_sys_4::llama_model_tensor_override {
967 pattern: ov.pattern.as_ptr(),
968 type_: ov.ty as llama_cpp_sys_4::ggml_type,
969 })
970 .chain(std::iter::once(
971 llama_cpp_sys_4::llama_model_tensor_override {
972 pattern: null(),
973 type_: llama_cpp_sys_4::GGML_TYPE_COUNT,
974 },
975 ))
976 .collect();
977
978 // ── prune_layers ─────────────────────────────────────────────────────
979 // -1-terminated
980 let mut prune_c = self.prune_layers.clone();
981 prune_c.push(-1);
982
983 // ── assemble ────────────────────────────────────────────────────────
984 let raw = llama_cpp_sys_4::llama_model_quantize_params {
985 nthread: self.nthread,
986 ftype: self.ftype as llama_cpp_sys_4::llama_ftype,
987 output_tensor_type: self
988 .output_tensor_type
989 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
990 t as llama_cpp_sys_4::ggml_type
991 }),
992 token_embedding_type: self
993 .token_embedding_type
994 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
995 t as llama_cpp_sys_4::ggml_type
996 }),
997 allow_requantize: self.allow_requantize,
998 quantize_output_tensor: self.quantize_output_tensor,
999 only_copy: self.only_copy,
1000 pure_: self.pure,
1001 keep_split: self.keep_split,
1002 dry_run: self.dry_run,
1003 max_buf_size: self.max_buf_size,
1004 imatrix: if self.imatrix.is_empty() {
1005 null()
1006 } else {
1007 imatrix_c.as_ptr()
1008 },
1009 kv_overrides: if self.kv_overrides.is_empty() {
1010 null()
1011 } else {
1012 kv_c.as_ptr()
1013 },
1014 tt_overrides: if self.tt_overrides.is_empty() {
1015 null()
1016 } else {
1017 tt_c.as_ptr()
1018 },
1019 prune_layers: if self.prune_layers.is_empty() {
1020 null()
1021 } else {
1022 prune_c.as_ptr()
1023 },
1024 };
1025
1026 RawQuantizeParamsGuard {
1027 raw,
1028 _imatrix_c: imatrix_c,
1029 _kv_c: kv_c,
1030 _tt_c: tt_c,
1031 _prune_c: prune_c,
1032 _marker: std::marker::PhantomData,
1033 }
1034 }
1035}
1036
1037/// Temporary storage that keeps the C pointers inside a raw
1038/// `llama_model_quantize_params` valid. Dropped after the quantize call.
1039pub(crate) struct RawQuantizeParamsGuard<'a> {
1040 pub(crate) raw: llama_cpp_sys_4::llama_model_quantize_params,
1041 _imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data>,
1042 _kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override>,
1043 _tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override>,
1044 _prune_c: Vec<i32>,
1045 // tie lifetime to the source QuantizeParams so the string/data
1046 // pointers inside imatrix_c and tt_c stay valid
1047 _marker: std::marker::PhantomData<&'a QuantizeParams>,
1048}
1049
1050// ─────────────────────────────────────────────────────────────────────────────
1051// TurboQuant – attention rotation
1052// ─────────────────────────────────────────────────────────────────────────────
1053
1054/// Control the `TurboQuant` attention-rotation feature globally.
1055///
1056/// When enabled (the default), llama.cpp applies a Hadamard rotation to Q/K/V
1057/// tensors before storing them in the KV cache. This significantly improves
1058/// quantization quality of the KV cache with near-zero overhead, as described
1059/// in llama.cpp PR #21038.
1060///
1061/// This function sets or clears the `LLAMA_ATTN_ROT_DISABLE` environment
1062/// variable, which llama.cpp reads once when a context (and its KV cache) is
1063/// first created. Call it **before** creating any [`LlamaContext`] on the
1064/// current process.
1065///
1066/// # Thread safety
1067///
1068/// Mutating environment variables while other threads may be reading them is
1069/// undefined behaviour. Call this function before spawning any threads that
1070/// use llama contexts, or ensure no contexts are being created concurrently.
1071///
1072/// # Example
1073///
1074/// ```no_run
1075/// // Disable the rotation for benchmarking purposes:
1076/// llama_cpp_4::quantize::set_attn_rot_disabled(true);
1077///
1078/// // Re-enable (default behaviour):
1079/// llama_cpp_4::quantize::set_attn_rot_disabled(false);
1080/// ```
1081///
1082/// [`LlamaContext`]: crate::context::LlamaContext
1083pub fn set_attn_rot_disabled(disabled: bool) {
1084 if disabled {
1085 // SAFETY: single-threaded context required by the caller.
1086 #[allow(unused_unsafe)]
1087 // SAFETY: setting an environment variable races with any concurrent
1088 // `getenv` elsewhere in the process. This is a deliberate opt-in
1089 // toggle meant to be set during setup, before threads that read it
1090 // exist; the function's docs say so.
1091 unsafe {
1092 std::env::set_var("LLAMA_ATTN_ROT_DISABLE", "1");
1093 }
1094 } else {
1095 #[allow(unused_unsafe)]
1096 // SAFETY: setting an environment variable races with any concurrent
1097 // `getenv` elsewhere in the process. This is a deliberate opt-in
1098 // toggle meant to be set during setup, before threads that read it
1099 // exist; the function's docs say so.
1100 unsafe {
1101 std::env::remove_var("LLAMA_ATTN_ROT_DISABLE");
1102 }
1103 }
1104}
1105
1106/// Returns `true` if `TurboQuant` attention rotation is currently disabled.
1107#[must_use]
1108pub fn attn_rot_disabled() -> bool {
1109 std::env::var("LLAMA_ATTN_ROT_DISABLE")
1110 .ok()
1111 .and_then(|v| v.parse::<i32>().ok())
1112 .is_some_and(|v| v != 0)
1113}
1114
1115// ─────────────────────────────────────────────────────────────────────────────
1116// Quantization preview
1117// ─────────────────────────────────────────────────────────────────────────────
1118
1119/// Ask llama.cpp what it *would* do, without writing a file.
1120///
1121/// [`crate::model_quantize`] is all-or-nothing: it reads a model, quantizes
1122/// every tensor, and writes the result. This exposes the decision layer
1123/// underneath — which tensors get quantized at all, and to which `ggml` type —
1124/// so a tool can show the plan, or estimate output size, before committing to
1125/// a run that may take minutes and tens of gigabytes.
1126///
1127/// The k-quant mixes are why this is not simply [`LlamaFtype::default_ggml_type`]:
1128/// they deliberately keep attention and output tensors at higher precision, so
1129/// the per-tensor answer differs from the ftype's nominal type.
1130///
1131/// Wraps `llama_quant_init` / `llama_quant_free`.
1132#[derive(Debug)]
1133pub struct QuantPreview<'model> {
1134 qs: std::ptr::NonNull<llama_cpp_sys_4::quantize_state_impl>,
1135 // The state borrows the model it was built from.
1136 _model: std::marker::PhantomData<&'model crate::model::LlamaModel>,
1137}
1138
1139impl Drop for QuantPreview<'_> {
1140 fn drop(&mut self) {
1141 // SAFETY: this runs once from `Drop` on a non-null handle, and nothing reads it
1142 // afterwards.
1143 unsafe { llama_cpp_sys_4::llama_quant_free(self.qs.as_ptr()) }
1144 }
1145}
1146
1147impl<'model> QuantPreview<'model> {
1148 /// Build a preview for `model` under `params`.
1149 ///
1150 /// # Errors
1151 ///
1152 /// Returns [`QuantPreviewError::Init`] if llama.cpp could not build the
1153 /// quantization state, which happens for a model it cannot quantize.
1154 pub fn new(
1155 model: &'model crate::model::LlamaModel,
1156 params: &QuantizeParams,
1157 ) -> Result<Self, QuantPreviewError> {
1158 let guard = params.to_raw();
1159 // The guarded wrapper: `llama_quant_init` throws for a model it cannot
1160 // quantize, and that unwinding into Rust aborts the process.
1161 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
1162 // the library owns whatever it returns.
1163 let qs = unsafe {
1164 llama_cpp_sys_4::llama_quant_init_guarded(model.model.as_ptr(), &raw const guard.raw)
1165 };
1166 std::ptr::NonNull::new(qs)
1167 .map(|qs| Self {
1168 qs,
1169 _model: std::marker::PhantomData,
1170 })
1171 .ok_or(QuantPreviewError::Init)
1172 }
1173
1174 /// Whether this tensor would be quantized at all.
1175 ///
1176 /// llama.cpp skips 1-D tensors, tensors below a size threshold, and ones
1177 /// whose name marks them as needing full precision — so a `false` here is
1178 /// the usual reason a tensor keeps its original type.
1179 ///
1180 /// Requires the `ggml` feature, which is what exposes [`crate::ggml::GgmlTensor`].
1181 #[cfg(feature = "ggml")]
1182 #[must_use]
1183 pub fn allows_quantization(&self, tensor: &crate::ggml::GgmlTensor) -> bool {
1184 // 1 = yes, 0 = no, -1 = the underlying call threw and was caught.
1185 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
1186 // the library owns whatever it returns.
1187 let rc = unsafe {
1188 llama_cpp_sys_4::llama_quant_tensor_allows_quantization_guarded(
1189 self.qs.as_ptr(),
1190 tensor.as_ptr(),
1191 )
1192 };
1193 rc == 1
1194 }
1195
1196 /// Compute the storage type each tensor would be assigned under `ftype`.
1197 ///
1198 /// Every tensor passed must already satisfy [`Self::allows_quantization`] —
1199 /// upstream states the caller filters first, and does not re-check.
1200 ///
1201 /// An entry is `None` when llama.cpp picks a `ggml` type this crate's
1202 /// [`GgmlType`] does not know.
1203 ///
1204 /// # Errors
1205 ///
1206 /// Returns [`QuantPreviewError::NotQuantizable`] naming the first tensor
1207 /// that fails the filter, rather than letting llama.cpp decide what to do
1208 /// with it.
1209 ///
1210 /// Requires the `ggml` feature.
1211 #[cfg(feature = "ggml")]
1212 pub fn compute_types(
1213 &self,
1214 tensors: &[&crate::ggml::GgmlTensor],
1215 ftype: LlamaFtype,
1216 ) -> Result<Vec<Option<GgmlType>>, QuantPreviewError> {
1217 if tensors.is_empty() {
1218 return Ok(Vec::new());
1219 }
1220 for tensor in tensors {
1221 if !self.allows_quantization(tensor) {
1222 return Err(QuantPreviewError::NotQuantizable(tensor.name().to_owned()));
1223 }
1224 }
1225
1226 let mut raw_tensors: Vec<*mut llama_cpp_sys_4::ggml_tensor> =
1227 tensors.iter().map(|t| t.as_ptr()).collect();
1228 let mut out = vec![0 as llama_cpp_sys_4::ggml_type; tensors.len()];
1229 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
1230 // the library owns whatever it returns.
1231 let rc = unsafe {
1232 llama_cpp_sys_4::llama_quant_compute_types_guarded(
1233 self.qs.as_ptr(),
1234 ftype.into(),
1235 raw_tensors.as_mut_ptr(),
1236 out.as_mut_ptr(),
1237 out.len(),
1238 )
1239 };
1240 if rc != 0 {
1241 return Err(QuantPreviewError::Init);
1242 }
1243 Ok(out
1244 .into_iter()
1245 .map(|t| GgmlType::try_from(t).ok())
1246 .collect())
1247 }
1248}
1249
1250/// Failure building or using a [`QuantPreview`].
1251#[derive(Debug, Eq, PartialEq, thiserror::Error)]
1252pub enum QuantPreviewError {
1253 /// llama.cpp could not build a quantization state for this model.
1254 #[error("could not initialize quantization state")]
1255 Init,
1256 /// A tensor that would not be quantized was passed to
1257 /// `QuantPreview::compute_types` (which requires the `ggml` feature).
1258 #[error("tensor {0:?} is not quantizable; filter with allows_quantization first")]
1259 NotQuantizable(String),
1260 /// The mock-model descriptor contained an interior NUL byte.
1261 #[error("architecture name contained an interior NUL byte")]
1262 Nul(#[from] NulError),
1263 /// llama.cpp could not build a model from the descriptor.
1264 #[error("could not build a mock model from the descriptor")]
1265 MockModel,
1266}
1267
1268/// Shape of a synthetic model, for previewing a quantization plan without a
1269/// real checkpoint on disk.
1270///
1271/// Upstream marks `llama_quant_model_from_metadata` as being for testing; it is
1272/// exposed here for the same reason — it lets you ask "what would a `Q4_K_M` of a
1273/// model this shape look like?" without downloading one.
1274#[derive(Debug, Clone, PartialEq, Eq)]
1275pub struct QuantModelDesc {
1276 /// Architecture name, e.g. `"llama"`.
1277 pub architecture: String,
1278 /// Embedding dimension.
1279 pub n_embd: u32,
1280 /// Feed-forward dimension.
1281 pub n_ff: u32,
1282 /// Number of layers.
1283 pub n_layer: u32,
1284 /// Number of attention heads.
1285 pub n_head: u32,
1286 /// Number of key/value heads.
1287 pub n_head_kv: u32,
1288 /// Number of experts (0 for a dense model).
1289 pub n_expert: u32,
1290 /// Per-head key dimension.
1291 pub n_embd_head_k: u32,
1292 /// Per-head value dimension.
1293 pub n_embd_head_v: u32,
1294}
1295
1296impl QuantModelDesc {
1297 /// A dense llama-shaped descriptor with the given dimensions.
1298 #[must_use]
1299 pub fn llama(n_embd: u32, n_ff: u32, n_layer: u32, n_head: u32) -> Self {
1300 Self {
1301 architecture: "llama".to_owned(),
1302 n_embd,
1303 n_ff,
1304 n_layer,
1305 n_head,
1306 n_head_kv: n_head,
1307 n_expert: 0,
1308 n_embd_head_k: n_embd / n_head.max(1),
1309 n_embd_head_v: n_embd / n_head.max(1),
1310 }
1311 }
1312
1313 /// Build a synthetic model from this descriptor.
1314 ///
1315 /// The returned model owns llama.cpp memory and frees it on drop, exactly
1316 /// like a loaded one — it simply has no weights.
1317 ///
1318 /// # Errors
1319 ///
1320 /// Returns [`QuantPreviewError::MockModel`] if llama.cpp rejects the
1321 /// descriptor, or [`QuantPreviewError::Nul`] for a bad architecture name.
1322 pub fn build(&self) -> Result<crate::model::LlamaModel, QuantPreviewError> {
1323 let arch = CString::new(self.architecture.as_str())?;
1324 let desc = llama_cpp_sys_4::llama_quant_model_desc {
1325 architecture: arch.as_ptr(),
1326 n_embd: self.n_embd,
1327 n_ff: self.n_ff,
1328 n_layer: self.n_layer,
1329 n_head: self.n_head,
1330 n_head_kv: self.n_head_kv,
1331 n_expert: self.n_expert,
1332 n_embd_head_k: self.n_embd_head_k,
1333 n_embd_head_v: self.n_embd_head_v,
1334 };
1335 // Guarded: an architecture llama.cpp does not know makes the raw entry
1336 // point throw, which aborts the process on the way back into Rust.
1337 let raw =
1338 // SAFETY: the arguments are either scalars or pointers that outlive the call, and
1339 // the library owns whatever it returns.
1340 unsafe { llama_cpp_sys_4::llama_quant_model_from_metadata_guarded(&raw const desc) };
1341 let ptr = std::ptr::NonNull::new(raw).ok_or(QuantPreviewError::MockModel)?;
1342 // SAFETY: upstream documents the result as owned by the caller and
1343 // freed with `llama_model_free`, which is what `LlamaModel` does.
1344 Ok(unsafe { crate::model::LlamaModel::from_raw(ptr) })
1345 }
1346}