llama_cpp_4/quantize.rs
1//! Quantization types and parameters for converting models to lower-bit precisions.
2//!
3//! # Quick start
4//!
5//! ```no_run
6//! use llama_cpp_4::quantize::{LlamaFtype, QuantizeParams};
7//!
8//! let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
9//! .with_nthread(8)
10//! .with_quantize_output_tensor(true);
11//!
12//! llama_cpp_4::model_quantize("model-f16.gguf", "model-q4km.gguf", ¶ms).unwrap();
13//! ```
14//!
15//! # `TurboQuant` – attention rotation (PR #21038)
16//!
17//! llama.cpp applies a Hadamard rotation to Q/K/V tensors before writing them into the KV cache.
18//! This significantly improves KV-cache quantization quality at near-zero cost, and is enabled by
19//! default for every model whose head dimension is a power of two. You can opt out per-context
20//! with [`LlamaContextParams::with_attn_rot_disabled`] or globally with
21//! [`set_attn_rot_disabled`].
22//!
23//! [`LlamaContextParams::with_attn_rot_disabled`]: crate::context::params::LlamaContextParams::with_attn_rot_disabled
24
25use std::ffi::{CString, NulError};
26use std::ptr::null;
27
28// ─────────────────────────────────────────────────────────────────────────────
29// LlamaFtype
30// ─────────────────────────────────────────────────────────────────────────────
31
32/// The quantization type used for the bulk of a model file (maps to `llama_ftype`).
33///
34/// Pass one of these variants to [`QuantizeParams::new`] to choose the target precision.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36#[non_exhaustive]
37#[allow(missing_docs)]
38pub enum LlamaFtype {
39 /// All tensors stored as full F32 (very large, for reference only)
40 AllF32 = 0,
41 /// F16 – 14 GB @ 7B, +0.0020 ppl vs Mistral-7B
42 MostlyF16 = 1,
43 /// `Q4_0` – 4.34 GB @ 8B, +0.4685 ppl
44 MostlyQ4_0 = 2,
45 /// `Q4_1` – 4.78 GB @ 8B, +0.4511 ppl
46 MostlyQ4_1 = 3,
47 /// `Q8_0` – 7.96 GB @ 8B, +0.0026 ppl
48 MostlyQ8_0 = 7,
49 /// `Q5_0` – 5.21 GB @ 8B, +0.1316 ppl
50 MostlyQ5_0 = 8,
51 /// `Q5_1` – 5.65 GB @ 8B, +0.1062 ppl
52 MostlyQ5_1 = 9,
53 /// `Q2_K` – 2.96 GB @ 8B, +3.5199 ppl
54 MostlyQ2K = 10,
55 /// `Q3_K` small – 3.41 GB @ 8B, +1.6321 ppl
56 MostlyQ3KS = 11,
57 /// `Q3_K` medium – 3.74 GB @ 8B, +0.6569 ppl
58 MostlyQ3KM = 12,
59 /// `Q3_K` large – 4.03 GB @ 8B, +0.5562 ppl
60 MostlyQ3KL = 13,
61 /// `Q4_K` small – 4.37 GB @ 8B, +0.2689 ppl
62 MostlyQ4KS = 14,
63 /// `Q4_K` medium – 4.58 GB @ 8B, +0.1754 ppl *(recommended default)*
64 MostlyQ4KM = 15,
65 /// `Q5_K` small – 5.21 GB @ 8B, +0.1049 ppl
66 MostlyQ5KS = 16,
67 /// `Q5_K` medium – 5.33 GB @ 8B, +0.0569 ppl
68 MostlyQ5KM = 17,
69 /// `Q6_K` – 6.14 GB @ 8B, +0.0217 ppl
70 MostlyQ6K = 18,
71 /// `IQ2_XXS` – 2.06 bpw
72 MostlyIQ2XXS = 19,
73 /// `IQ2_XS` – 2.31 bpw
74 MostlyIQ2XS = 20,
75 /// `Q2_K` small
76 MostlyQ2KS = 21,
77 /// `IQ3_XS` – 3.3 bpw
78 MostlyIQ3XS = 22,
79 /// `IQ3_XXS` – 3.06 bpw
80 MostlyIQ3XXS = 23,
81 /// `IQ1_S` – 1.56 bpw (extremely small, high loss)
82 MostlyIQ1S = 24,
83 /// `IQ4_NL` – 4.50 bpw non-linear
84 MostlyIQ4NL = 25,
85 /// `IQ3_S` – 3.44 bpw
86 MostlyIQ3S = 26,
87 /// `IQ3_M` – 3.66 bpw
88 MostlyIQ3M = 27,
89 /// `IQ2_S` – 2.5 bpw
90 MostlyIQ2S = 28,
91 /// `IQ2_M` – 2.7 bpw
92 MostlyIQ2M = 29,
93 /// `IQ4_XS` – 4.25 bpw non-linear
94 MostlyIQ4XS = 30,
95 /// `IQ1_M` – 1.75 bpw
96 MostlyIQ1M = 31,
97 /// BF16 – 14 GB @ 7B, −0.0050 ppl vs Mistral-7B
98 MostlyBF16 = 32,
99 /// `TQ1_0` – 1.69 bpw ternary
100 MostlyTQ1_0 = 36,
101 /// `TQ2_0` – 2.06 bpw ternary
102 MostlyTQ2_0 = 37,
103 /// MXFP4 (`MoE` layers)
104 MostlyMXFP4Moe = 38,
105 /// NVFP4
106 MostlyNVFP4 = 39,
107 /// `Q1_0` – 1.5 bpw binary (block size 32)
108 #[cfg(feature = "q1")]
109 MostlyQ1_0 = 40,
110 /// `Q1_0_g128` – 1.125 bpw binary (block size 128)
111 //
112 // Named after upstream's `LLAMA_FTYPE_MOSTLY_Q1_0_G128`; renaming to
113 // satisfy the casing lint would break the correspondence.
114 #[allow(non_camel_case_types)]
115 #[cfg(feature = "q1")]
116 MostlyQ1_0_G128 = 41,
117}
118
119impl LlamaFtype {
120 /// Short name suitable for filenames (e.g. `"Q4_K_M"`).
121 #[must_use]
122 pub fn name(self) -> &'static str {
123 match self {
124 Self::AllF32 => "F32",
125 Self::MostlyF16 => "F16",
126 Self::MostlyQ4_0 => "Q4_0",
127 Self::MostlyQ4_1 => "Q4_1",
128 Self::MostlyQ8_0 => "Q8_0",
129 Self::MostlyQ5_0 => "Q5_0",
130 Self::MostlyQ5_1 => "Q5_1",
131 Self::MostlyQ2K => "Q2_K",
132 Self::MostlyQ3KS => "Q3_K_S",
133 Self::MostlyQ3KM => "Q3_K_M",
134 Self::MostlyQ3KL => "Q3_K_L",
135 Self::MostlyQ4KS => "Q4_K_S",
136 Self::MostlyQ4KM => "Q4_K_M",
137 Self::MostlyQ5KS => "Q5_K_S",
138 Self::MostlyQ5KM => "Q5_K_M",
139 Self::MostlyQ6K => "Q6_K",
140 Self::MostlyIQ2XXS => "IQ2_XXS",
141 Self::MostlyIQ2XS => "IQ2_XS",
142 Self::MostlyQ2KS => "Q2_K_S",
143 Self::MostlyIQ3XS => "IQ3_XS",
144 Self::MostlyIQ3XXS => "IQ3_XXS",
145 Self::MostlyIQ1S => "IQ1_S",
146 Self::MostlyIQ4NL => "IQ4_NL",
147 Self::MostlyIQ3S => "IQ3_S",
148 Self::MostlyIQ3M => "IQ3_M",
149 Self::MostlyIQ2S => "IQ2_S",
150 Self::MostlyIQ2M => "IQ2_M",
151 Self::MostlyIQ4XS => "IQ4_XS",
152 Self::MostlyIQ1M => "IQ1_M",
153 Self::MostlyBF16 => "BF16",
154 Self::MostlyTQ1_0 => "TQ1_0",
155 Self::MostlyTQ2_0 => "TQ2_0",
156 Self::MostlyMXFP4Moe => "MXFP4_MOE",
157 Self::MostlyNVFP4 => "NVFP4",
158 #[cfg(feature = "q1")]
159 Self::MostlyQ1_0 => "Q1_0",
160 #[cfg(feature = "q1")]
161 Self::MostlyQ1_0_G128 => "Q1_0_g128",
162 }
163 }
164
165 /// Human-readable description with approximate size and PPL delta.
166 #[must_use]
167 pub fn description(self) -> &'static str {
168 match self {
169 Self::AllF32 => "26.00 GB @ 7B — full precision reference",
170 Self::MostlyF16 => "14.00 GB @ 7B — +0.0020 ppl vs Mistral-7B",
171 Self::MostlyBF16 => "14.00 GB @ 7B — -0.0050 ppl vs Mistral-7B",
172 Self::MostlyQ8_0 => " 7.96 GB @ 8B — +0.0026 ppl",
173 Self::MostlyQ6K => " 6.14 GB @ 8B — +0.0217 ppl",
174 Self::MostlyQ5KM => " 5.33 GB @ 8B — +0.0569 ppl",
175 Self::MostlyQ5KS => " 5.21 GB @ 8B — +0.1049 ppl",
176 Self::MostlyQ5_1 => " 5.65 GB @ 8B — +0.1062 ppl",
177 Self::MostlyQ5_0 => " 5.21 GB @ 8B — +0.1316 ppl",
178 Self::MostlyQ4KM => " 4.58 GB @ 8B — +0.1754 ppl [recommended]",
179 Self::MostlyQ4KS => " 4.37 GB @ 8B — +0.2689 ppl",
180 Self::MostlyQ4_1 => " 4.78 GB @ 8B — +0.4511 ppl",
181 Self::MostlyQ4_0 => " 4.34 GB @ 8B — +0.4685 ppl",
182 Self::MostlyQ3KL => " 4.03 GB @ 8B — +0.5562 ppl",
183 Self::MostlyQ3KM => " 3.74 GB @ 8B — +0.6569 ppl",
184 Self::MostlyQ3KS => " 3.41 GB @ 8B — +1.6321 ppl",
185 Self::MostlyQ2KS => " 2.96 GB @ 8B — +3.1836 ppl",
186 Self::MostlyQ2K => " 2.96 GB @ 8B — +3.5199 ppl",
187 Self::MostlyIQ4XS => " 4.25 bpw non-linear",
188 Self::MostlyIQ4NL => " 4.50 bpw non-linear",
189 Self::MostlyIQ3S => " 3.44 bpw",
190 Self::MostlyIQ3M => " 3.66 bpw",
191 Self::MostlyIQ3XS => " 3.3 bpw",
192 Self::MostlyIQ3XXS => " 3.06 bpw",
193 Self::MostlyIQ2M => " 2.7 bpw",
194 Self::MostlyIQ2S => " 2.5 bpw",
195 Self::MostlyIQ2XS => " 2.31 bpw",
196 Self::MostlyIQ2XXS => " 2.06 bpw",
197 Self::MostlyIQ1M => " 1.75 bpw — extreme compression",
198 Self::MostlyIQ1S => " 1.56 bpw — extreme compression",
199 Self::MostlyTQ1_0 => " 1.69 bpw ternary",
200 Self::MostlyTQ2_0 => " 2.06 bpw ternary",
201 Self::MostlyMXFP4Moe => "MXFP4 MoE layers",
202 Self::MostlyNVFP4 => "NVFP4",
203 #[cfg(feature = "q1")]
204 Self::MostlyQ1_0 => " 1.50 bpw — binary Q1_0 (block 32)",
205 #[cfg(feature = "q1")]
206 Self::MostlyQ1_0_G128 => " 1.125 bpw — binary Q1_0_g128 (block 128)",
207 }
208 }
209
210 /// Look up a variant by its short name (case-insensitive).
211 ///
212 /// ```
213 /// use llama_cpp_4::quantize::LlamaFtype;
214 /// assert_eq!(LlamaFtype::from_name("Q4_K_M"), Some(LlamaFtype::MostlyQ4KM));
215 /// assert_eq!(LlamaFtype::from_name("q4_k_m"), Some(LlamaFtype::MostlyQ4KM));
216 /// assert_eq!(LlamaFtype::from_name("bogus"), None);
217 /// ```
218 #[must_use]
219 pub fn from_name(name: &str) -> Option<Self> {
220 let upper = name.to_uppercase();
221 match upper.as_str() {
222 "F32" => Some(Self::AllF32),
223 "F16" => Some(Self::MostlyF16),
224 "BF16" => Some(Self::MostlyBF16),
225 "Q4_0" => Some(Self::MostlyQ4_0),
226 "Q4_1" => Some(Self::MostlyQ4_1),
227 "Q8_0" => Some(Self::MostlyQ8_0),
228 "Q5_0" => Some(Self::MostlyQ5_0),
229 "Q5_1" => Some(Self::MostlyQ5_1),
230 "Q2_K" => Some(Self::MostlyQ2K),
231 "Q2_K_S" => Some(Self::MostlyQ2KS),
232 "Q3_K_S" => Some(Self::MostlyQ3KS),
233 "Q3_K_M" => Some(Self::MostlyQ3KM),
234 "Q3_K_L" => Some(Self::MostlyQ3KL),
235 "Q4_K_S" => Some(Self::MostlyQ4KS),
236 "Q4_K_M" => Some(Self::MostlyQ4KM),
237 "Q5_K_S" => Some(Self::MostlyQ5KS),
238 "Q5_K_M" => Some(Self::MostlyQ5KM),
239 "Q6_K" => Some(Self::MostlyQ6K),
240 "IQ1_S" => Some(Self::MostlyIQ1S),
241 "IQ1_M" => Some(Self::MostlyIQ1M),
242 "IQ2_XXS" => Some(Self::MostlyIQ2XXS),
243 "IQ2_XS" => Some(Self::MostlyIQ2XS),
244 "IQ2_S" => Some(Self::MostlyIQ2S),
245 "IQ2_M" => Some(Self::MostlyIQ2M),
246 "IQ3_XXS" => Some(Self::MostlyIQ3XXS),
247 "IQ3_XS" => Some(Self::MostlyIQ3XS),
248 "IQ3_S" => Some(Self::MostlyIQ3S),
249 "IQ3_M" => Some(Self::MostlyIQ3M),
250 "IQ4_NL" => Some(Self::MostlyIQ4NL),
251 "IQ4_XS" => Some(Self::MostlyIQ4XS),
252 "TQ1_0" => Some(Self::MostlyTQ1_0),
253 "TQ2_0" => Some(Self::MostlyTQ2_0),
254 "MXFP4_MOE" => Some(Self::MostlyMXFP4Moe),
255 "NVFP4" => Some(Self::MostlyNVFP4),
256 #[cfg(feature = "q1")]
257 "Q1_0" => Some(Self::MostlyQ1_0),
258 #[cfg(feature = "q1")]
259 "Q1_0_G128" | "Q1_0_g128" => Some(Self::MostlyQ1_0_G128),
260 _ => None,
261 }
262 }
263
264 /// llama.cpp's own display name for this type, e.g. `"Q4_K - Medium"`.
265 ///
266 /// Where [`Self::name`] is a terse filename-safe token (`"Q4_K_M"`) kept in
267 /// a table in this crate, this asks llama.cpp, so it cannot drift from
268 /// upstream. Wraps `llama_ftype_name`.
269 ///
270 /// # Errors
271 ///
272 /// Returns [`FtypeNameError`] if llama.cpp returns null or non-UTF-8.
273 pub fn upstream_name(self) -> Result<&'static str, FtypeNameError> {
274 let ptr = unsafe { llama_cpp_sys_4::llama_ftype_name(self.into()) };
275 if ptr.is_null() {
276 return Err(FtypeNameError::Unnamed(self));
277 }
278 // SAFETY: llama.cpp returns a pointer to a string literal with static
279 // storage duration, so `'static` is sound here.
280 unsafe { std::ffi::CStr::from_ptr(ptr) }
281 .to_str()
282 .map_err(FtypeNameError::Utf8)
283 }
284
285 /// The `ggml` storage type most tensors get under this ftype.
286 ///
287 /// This is the *default*; the k-quant mixes deliberately store some tensors
288 /// (attention, output) at higher precision, so an individual tensor may not
289 /// use it. Wraps `llama_ftype_get_default_type`.
290 ///
291 /// Returns `None` for a `ggml` type this crate's [`GgmlType`] does not know.
292 #[must_use]
293 pub fn default_ggml_type(self) -> Option<GgmlType> {
294 let raw = unsafe { llama_cpp_sys_4::llama_ftype_get_default_type(self.into()) };
295 GgmlType::try_from(raw).ok()
296 }
297
298 /// All available types, ordered roughly from largest to smallest.
299 #[must_use]
300 pub fn all() -> &'static [Self] {
301 &[
302 Self::AllF32,
303 Self::MostlyF16,
304 Self::MostlyBF16,
305 Self::MostlyQ8_0,
306 Self::MostlyQ6K,
307 Self::MostlyQ5KM,
308 Self::MostlyQ5KS,
309 Self::MostlyQ5_1,
310 Self::MostlyQ5_0,
311 Self::MostlyQ4KM,
312 Self::MostlyQ4KS,
313 Self::MostlyQ4_1,
314 Self::MostlyQ4_0,
315 Self::MostlyQ3KL,
316 Self::MostlyQ3KM,
317 Self::MostlyQ3KS,
318 Self::MostlyQ2KS,
319 Self::MostlyQ2K,
320 Self::MostlyIQ4XS,
321 Self::MostlyIQ4NL,
322 Self::MostlyIQ3S,
323 Self::MostlyIQ3M,
324 Self::MostlyIQ3XS,
325 Self::MostlyIQ3XXS,
326 Self::MostlyIQ2M,
327 Self::MostlyIQ2S,
328 Self::MostlyIQ2XS,
329 Self::MostlyIQ2XXS,
330 Self::MostlyIQ1M,
331 Self::MostlyIQ1S,
332 Self::MostlyTQ1_0,
333 Self::MostlyTQ2_0,
334 Self::MostlyMXFP4Moe,
335 Self::MostlyNVFP4,
336 #[cfg(feature = "q1")]
337 Self::MostlyQ1_0,
338 #[cfg(feature = "q1")]
339 Self::MostlyQ1_0_G128,
340 ]
341 }
342}
343
344/// Failure reading llama.cpp's display name for an [`LlamaFtype`].
345#[derive(Debug, Eq, PartialEq, thiserror::Error)]
346pub enum FtypeNameError {
347 /// llama.cpp returned null for this type.
348 #[error("llama.cpp has no name for ftype {0:?}")]
349 Unnamed(LlamaFtype),
350 /// The returned bytes were not valid UTF-8.
351 #[error(transparent)]
352 Utf8(#[from] std::str::Utf8Error),
353}
354
355impl From<LlamaFtype> for llama_cpp_sys_4::llama_ftype {
356 fn from(t: LlamaFtype) -> Self {
357 t as llama_cpp_sys_4::llama_ftype
358 }
359}
360
361impl TryFrom<llama_cpp_sys_4::llama_ftype> for LlamaFtype {
362 type Error = llama_cpp_sys_4::llama_ftype;
363
364 /// Fails on a discriminant this build does not know — `LLAMA_FTYPE_GUESSED`
365 /// (1024, "the file did not say"), a `q1` type without that feature, or a
366 /// type upstream added since this release. The raw value is returned so the
367 /// caller can report it.
368 fn try_from(raw: llama_cpp_sys_4::llama_ftype) -> Result<Self, Self::Error> {
369 Self::all()
370 .iter()
371 .copied()
372 .find(|t| llama_cpp_sys_4::llama_ftype::from(*t) == raw)
373 .ok_or(raw)
374 }
375}
376
377impl std::fmt::Display for LlamaFtype {
378 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
379 write!(f, "{}", self.name())
380 }
381}
382
383// ─────────────────────────────────────────────────────────────────────────────
384// GgmlType
385// ─────────────────────────────────────────────────────────────────────────────
386
387/// GGML tensor storage type (maps to `ggml_type`).
388///
389/// Used to set [`QuantizeParams::output_tensor_type`] and
390/// [`QuantizeParams::token_embedding_type`], and for per-tensor type overrides
391/// in [`TensorTypeOverride`].
392#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
393#[non_exhaustive]
394#[allow(missing_docs)]
395pub enum GgmlType {
396 F32 = 0,
397 F16 = 1,
398 Q4_0 = 2,
399 Q4_1 = 3,
400 Q5_0 = 6,
401 Q5_1 = 7,
402 Q8_0 = 8,
403 Q8_1 = 9,
404 Q2K = 10,
405 Q3K = 11,
406 Q4K = 12,
407 Q5K = 13,
408 Q6K = 14,
409 Q8K = 15,
410 IQ2XXS = 16,
411 IQ2XS = 17,
412 IQ3XXS = 18,
413 IQ1S = 19,
414 IQ4NL = 20,
415 IQ3S = 21,
416 IQ2S = 22,
417 IQ4XS = 23,
418 I8 = 24,
419 I16 = 25,
420 I32 = 26,
421 I64 = 27,
422 F64 = 28,
423 IQ1M = 29,
424 BF16 = 30,
425 TQ1_0 = 34,
426 TQ2_0 = 35,
427 MXFP4 = 39,
428 /// NVFP4 — renumbered to 42 when the `q1` feature is active (40 and 41
429 /// are taken by `Q1_0` / `Q1_0_g128` for `PrismML` GGUF compatibility).
430 #[cfg(not(feature = "q1"))]
431 NVFP4 = 40,
432 #[cfg(feature = "q1")]
433 Q1_0 = 40,
434 #[cfg(feature = "q1")]
435 // Mirrors ggml's `GGML_TYPE_Q1_0_G128`; see the note on
436 // `LlamaFtype::MostlyQ1_0_G128`.
437 #[allow(non_camel_case_types)]
438 Q1_0_G128 = 41,
439 #[cfg(feature = "q1")]
440 NVFP4 = 42,
441}
442
443impl From<GgmlType> for llama_cpp_sys_4::ggml_type {
444 fn from(t: GgmlType) -> Self {
445 t as llama_cpp_sys_4::ggml_type
446 }
447}
448
449impl TryFrom<llama_cpp_sys_4::ggml_type> for GgmlType {
450 type Error = llama_cpp_sys_4::ggml_type;
451 fn try_from(v: llama_cpp_sys_4::ggml_type) -> Result<Self, Self::Error> {
452 match v {
453 0 => Ok(Self::F32),
454 1 => Ok(Self::F16),
455 2 => Ok(Self::Q4_0),
456 3 => Ok(Self::Q4_1),
457 6 => Ok(Self::Q5_0),
458 7 => Ok(Self::Q5_1),
459 8 => Ok(Self::Q8_0),
460 9 => Ok(Self::Q8_1),
461 10 => Ok(Self::Q2K),
462 11 => Ok(Self::Q3K),
463 12 => Ok(Self::Q4K),
464 13 => Ok(Self::Q5K),
465 14 => Ok(Self::Q6K),
466 15 => Ok(Self::Q8K),
467 16 => Ok(Self::IQ2XXS),
468 17 => Ok(Self::IQ2XS),
469 18 => Ok(Self::IQ3XXS),
470 19 => Ok(Self::IQ1S),
471 20 => Ok(Self::IQ4NL),
472 21 => Ok(Self::IQ3S),
473 22 => Ok(Self::IQ2S),
474 23 => Ok(Self::IQ4XS),
475 24 => Ok(Self::I8),
476 25 => Ok(Self::I16),
477 26 => Ok(Self::I32),
478 27 => Ok(Self::I64),
479 28 => Ok(Self::F64),
480 29 => Ok(Self::IQ1M),
481 30 => Ok(Self::BF16),
482 34 => Ok(Self::TQ1_0),
483 35 => Ok(Self::TQ2_0),
484 39 => Ok(Self::MXFP4),
485 #[cfg(not(feature = "q1"))]
486 40 => Ok(Self::NVFP4),
487 #[cfg(feature = "q1")]
488 40 => Ok(Self::Q1_0),
489 #[cfg(feature = "q1")]
490 41 => Ok(Self::Q1_0_G128),
491 #[cfg(feature = "q1")]
492 42 => Ok(Self::NVFP4),
493 _ => Err(v),
494 }
495 }
496}
497
498// ─────────────────────────────────────────────────────────────────────────────
499// ImatrixEntry / Imatrix
500// ─────────────────────────────────────────────────────────────────────────────
501
502/// A single per-tensor importance matrix entry, as loaded from a `.imatrix` file.
503///
504/// Each entry contains activation statistics for one model tensor collected from
505/// a calibration dataset. When supplied to [`QuantizeParams::with_imatrix`] these
506/// statistics guide the quantizer to allocate more precision to weights that
507/// matter most.
508#[derive(Debug, Clone)]
509pub struct ImatrixEntry {
510 name: CString,
511 data: Vec<f32>,
512}
513
514impl ImatrixEntry {
515 /// Create a new entry from a tensor name and its importance scores.
516 ///
517 /// # Errors
518 ///
519 /// Returns [`NulError`] if `name` contains an interior null byte.
520 pub fn new(name: impl Into<Vec<u8>>, data: Vec<f32>) -> Result<Self, NulError> {
521 Ok(Self {
522 name: CString::new(name)?,
523 data,
524 })
525 }
526
527 /// Tensor name.
528 #[must_use]
529 pub fn name_str(&self) -> &str {
530 self.name.to_str().unwrap_or("")
531 }
532
533 /// Number of importance values.
534 #[must_use]
535 pub fn len(&self) -> usize {
536 self.data.len()
537 }
538
539 /// Returns `true` if the data slice is empty.
540 #[must_use]
541 pub fn is_empty(&self) -> bool {
542 self.data.is_empty()
543 }
544}
545
546/// A collection of importance matrix entries (one per quantized tensor).
547///
548/// Build one by pushing [`ImatrixEntry`] values, then pass it to
549/// [`QuantizeParams::with_imatrix`].
550#[derive(Debug, Clone, Default)]
551pub struct Imatrix {
552 entries: Vec<ImatrixEntry>,
553}
554
555impl Imatrix {
556 /// Create an empty imatrix.
557 #[must_use]
558 pub fn new() -> Self {
559 Self::default()
560 }
561
562 /// Add an entry.
563 pub fn push(&mut self, entry: ImatrixEntry) {
564 self.entries.push(entry);
565 }
566
567 /// Number of entries.
568 #[must_use]
569 pub fn len(&self) -> usize {
570 self.entries.len()
571 }
572
573 /// Returns `true` if no entries have been added.
574 #[must_use]
575 pub fn is_empty(&self) -> bool {
576 self.entries.is_empty()
577 }
578}
579
580// ─────────────────────────────────────────────────────────────────────────────
581// TensorTypeOverride
582// ─────────────────────────────────────────────────────────────────────────────
583
584/// Override the quantization type of every tensor whose name matches a glob `pattern`.
585///
586/// The pattern syntax is the same as used by the `--tensor-type` flag in
587/// `llama-quantize`, e.g. `"attn.*"` or `"blk.0.*"`.
588///
589/// # Example
590///
591/// ```
592/// use llama_cpp_4::quantize::{GgmlType, TensorTypeOverride};
593///
594/// // Keep the output projection in F16:
595/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
596/// ```
597#[derive(Debug, Clone)]
598pub struct TensorTypeOverride {
599 pattern: CString,
600 ty: GgmlType,
601}
602
603impl TensorTypeOverride {
604 /// Create a new override.
605 ///
606 /// # Errors
607 ///
608 /// Returns [`NulError`] if `pattern` contains an interior null byte.
609 pub fn new(pattern: impl Into<Vec<u8>>, ty: GgmlType) -> Result<Self, NulError> {
610 Ok(Self {
611 pattern: CString::new(pattern)?,
612 ty,
613 })
614 }
615
616 /// The glob pattern that selects tensors.
617 #[must_use]
618 pub fn pattern_str(&self) -> &str {
619 self.pattern.to_str().unwrap_or("")
620 }
621
622 /// The type to assign to matching tensors.
623 #[must_use]
624 pub fn ty(&self) -> GgmlType {
625 self.ty
626 }
627}
628
629// ─────────────────────────────────────────────────────────────────────────────
630// KvOverrideValue / KvOverride
631// ─────────────────────────────────────────────────────────────────────────────
632
633/// A value in a GGUF key-value metadata override.
634#[derive(Debug, Clone, PartialEq)]
635pub enum KvOverrideValue {
636 /// 64-bit integer
637 Int(i64),
638 /// 64-bit float
639 Float(f64),
640 /// Boolean
641 Bool(bool),
642 /// Fixed-length string (up to 127 bytes + NUL)
643 Str([std::os::raw::c_char; 128]),
644}
645
646/// A single GGUF metadata key-value override.
647///
648/// These are written into the output file's metadata when quantizing.
649#[derive(Debug, Clone)]
650pub struct KvOverride {
651 key: CString,
652 /// The value for this override.
653 pub value: KvOverrideValue,
654}
655
656impl KvOverride {
657 /// Create a new override.
658 ///
659 /// # Errors
660 ///
661 /// Returns [`NulError`] if `key` contains an interior null byte.
662 pub fn new(key: impl Into<Vec<u8>>, value: KvOverrideValue) -> Result<Self, NulError> {
663 Ok(Self {
664 key: CString::new(key)?,
665 value,
666 })
667 }
668}
669
670// ─────────────────────────────────────────────────────────────────────────────
671// QuantizeParams
672// ─────────────────────────────────────────────────────────────────────────────
673
674/// Parameters for quantizing a model.
675///
676/// Create with [`QuantizeParams::new`] and chain `with_*` builder methods to
677/// configure, then pass a reference to [`crate::model_quantize`].
678///
679/// # Example
680///
681/// ```no_run
682/// use llama_cpp_4::quantize::{GgmlType, LlamaFtype, QuantizeParams, TensorTypeOverride};
683///
684/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
685///
686/// let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
687/// .with_nthread(8)
688/// .with_allow_requantize(false)
689/// .with_quantize_output_tensor(true)
690/// .with_pure(false)
691/// .with_tensor_type_override(ov);
692///
693/// llama_cpp_4::model_quantize("in.gguf", "out.gguf", ¶ms).unwrap();
694/// ```
695#[derive(Debug, Clone)]
696#[allow(clippy::struct_excessive_bools)]
697pub struct QuantizeParams {
698 /// Number of threads (0 = auto-detect).
699 pub nthread: i32,
700 /// Target quantization type.
701 pub ftype: LlamaFtype,
702 /// Force this storage type for the output/lm-head tensor (`None` = use ftype default).
703 pub output_tensor_type: Option<GgmlType>,
704 /// Force this storage type for the token-embedding tensor (`None` = use ftype default).
705 pub token_embedding_type: Option<GgmlType>,
706 /// Allow re-quantizing tensors that are already quantized.
707 pub allow_requantize: bool,
708 /// Quantize the output/lm-head weight tensor.
709 pub quantize_output_tensor: bool,
710 /// Copy all tensors without quantizing (ignores `ftype`).
711 pub only_copy: bool,
712 /// Quantize every tensor to the same type (no mixed k-quant strategy).
713 pub pure: bool,
714 /// Keep the same number of shards as the input (for split models).
715 pub keep_split: bool,
716 /// Estimate output size without writing anything to disk.
717 pub dry_run: bool,
718 /// Cap, in bytes, on the tensor rows held in memory at once (`0` = the
719 /// upstream default of 8 GiB). Lower it to quantize a model larger than
720 /// available RAM at the cost of more I/O.
721 pub max_buf_size: usize,
722
723 imatrix: Vec<ImatrixEntry>,
724 kv_overrides: Vec<KvOverride>,
725 tt_overrides: Vec<TensorTypeOverride>,
726 prune_layers: Vec<i32>,
727}
728
729impl QuantizeParams {
730 /// Create a new params set targeting `ftype`.
731 ///
732 /// All other options are set to the same defaults as
733 /// `llama_model_quantize_default_params()`.
734 #[must_use]
735 pub fn new(ftype: LlamaFtype) -> Self {
736 // Read the C defaults so we match them exactly.
737 let d = unsafe { llama_cpp_sys_4::llama_model_quantize_default_params() };
738 Self {
739 nthread: d.nthread,
740 ftype,
741 output_tensor_type: GgmlType::try_from(d.output_tensor_type).ok(),
742 token_embedding_type: GgmlType::try_from(d.token_embedding_type).ok(),
743 allow_requantize: d.allow_requantize,
744 quantize_output_tensor: d.quantize_output_tensor,
745 only_copy: d.only_copy,
746 pure: d.pure_,
747 keep_split: d.keep_split,
748 dry_run: d.dry_run,
749 max_buf_size: d.max_buf_size,
750 imatrix: Vec::new(),
751 kv_overrides: Vec::new(),
752 tt_overrides: Vec::new(),
753 prune_layers: Vec::new(),
754 }
755 }
756
757 /// Set the number of quantization threads (`0` = auto).
758 #[must_use]
759 pub fn with_nthread(mut self, n: i32) -> Self {
760 self.nthread = n;
761 self
762 }
763
764 /// Override the output-tensor storage type.
765 #[must_use]
766 pub fn with_output_tensor_type(mut self, ty: GgmlType) -> Self {
767 self.output_tensor_type = Some(ty);
768 self
769 }
770
771 /// Override the token-embedding storage type.
772 #[must_use]
773 pub fn with_token_embedding_type(mut self, ty: GgmlType) -> Self {
774 self.token_embedding_type = Some(ty);
775 self
776 }
777
778 /// Allow (or disallow) re-quantizing already-quantized tensors.
779 #[must_use]
780 pub fn with_allow_requantize(mut self, v: bool) -> Self {
781 self.allow_requantize = v;
782 self
783 }
784
785 /// Quantize the output/lm-head weight (`true` by default).
786 #[must_use]
787 pub fn with_quantize_output_tensor(mut self, v: bool) -> Self {
788 self.quantize_output_tensor = v;
789 self
790 }
791
792 /// When `true`, only copy tensors verbatim (no quantization at all).
793 #[must_use]
794 pub fn with_only_copy(mut self, v: bool) -> Self {
795 self.only_copy = v;
796 self
797 }
798
799 /// When `true`, quantize all tensors to the same type (no mixed k-quant strategy).
800 #[must_use]
801 pub fn with_pure(mut self, v: bool) -> Self {
802 self.pure = v;
803 self
804 }
805
806 /// Preserve the number of shards when quantizing a split model.
807 #[must_use]
808 pub fn with_keep_split(mut self, v: bool) -> Self {
809 self.keep_split = v;
810 self
811 }
812
813 /// Only estimate the output size; do not write anything to disk.
814 #[must_use]
815 pub fn with_dry_run(mut self, v: bool) -> Self {
816 self.dry_run = v;
817 self
818 }
819
820 /// Cap the bytes of tensor rows kept in memory at once (`0` = upstream
821 /// default, 8 GiB).
822 #[must_use]
823 pub fn with_max_buf_size(mut self, bytes: usize) -> Self {
824 self.max_buf_size = bytes;
825 self
826 }
827
828 /// Supply importance matrix data to improve quantization quality.
829 ///
830 /// The imatrix is generated by the `imatrix` tool (or the `imatrix` example
831 /// in this crate) and contains per-tensor activation statistics collected
832 /// from a calibration dataset.
833 #[must_use]
834 pub fn with_imatrix(mut self, imatrix: Imatrix) -> Self {
835 self.imatrix = imatrix.entries;
836 self
837 }
838
839 /// Append a single imatrix entry.
840 #[must_use]
841 pub fn with_imatrix_entry(mut self, entry: ImatrixEntry) -> Self {
842 self.imatrix.push(entry);
843 self
844 }
845
846 /// Add (or replace) a GGUF metadata key-value pair in the output file.
847 #[must_use]
848 pub fn with_kv_override(mut self, kv: KvOverride) -> Self {
849 self.kv_overrides.push(kv);
850 self
851 }
852
853 /// Override the quantization type for tensors whose name matches `pattern`.
854 ///
855 /// Can be called multiple times; overrides are applied in order.
856 #[must_use]
857 pub fn with_tensor_type_override(mut self, ov: TensorTypeOverride) -> Self {
858 self.tt_overrides.push(ov);
859 self
860 }
861
862 /// Mark a layer index for pruning (removal) from the output model.
863 #[must_use]
864 pub fn with_pruned_layer(mut self, layer: i32) -> Self {
865 self.prune_layers.push(layer);
866 self
867 }
868
869 /// Mark multiple layer indices for pruning.
870 #[must_use]
871 pub fn with_pruned_layers(mut self, layers: impl IntoIterator<Item = i32>) -> Self {
872 self.prune_layers.extend(layers);
873 self
874 }
875
876 /// Build the raw C struct, together with the temporary backing storage
877 /// that must outlive the struct. Returns `(raw_params, _guards)`.
878 ///
879 /// This is `pub(crate)` so that `model_quantize` can call it safely while
880 /// holding all the guards alive.
881 #[allow(clippy::too_many_lines)]
882 pub(crate) fn to_raw(&self) -> RawQuantizeParamsGuard<'_> {
883 // ── imatrix ─────────────────────────────────────────────────────────
884 // Build a null-terminated array of llama_model_imatrix_data.
885 // The `name` and `data` pointers point directly into our owned Vecs.
886 let imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data> = self
887 .imatrix
888 .iter()
889 .map(|e| llama_cpp_sys_4::llama_model_imatrix_data {
890 name: e.name.as_ptr(),
891 data: e.data.as_ptr(),
892 size: e.data.len(),
893 })
894 .chain(std::iter::once(llama_cpp_sys_4::llama_model_imatrix_data {
895 name: null(),
896 data: null(),
897 size: 0,
898 }))
899 .collect();
900
901 // ── kv_overrides ────────────────────────────────────────────────────
902 // null-terminated by a sentinel with key[0] == 0
903 let kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override> = self
904 .kv_overrides
905 .iter()
906 .map(|kv| {
907 let mut raw = llama_cpp_sys_4::llama_model_kv_override {
908 key: [0; 128],
909 tag: 0,
910 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
911 val_i64: 0,
912 },
913 };
914 // Copy key bytes (up to 127 chars + NUL).
915 let bytes = kv.key.to_bytes_with_nul();
916 let copy_len = bytes.len().min(128);
917 for (dst, &src) in raw.key.iter_mut().zip(bytes[..copy_len].iter()) {
918 // `c_char` is `i8` on x86_64 but `u8` on ARM64/Android; `as _`
919 // infers the target signedness on every platform (issue #306).
920 // The wrap on `i8` targets is intentional: llama.cpp reads the
921 // key back as bytes, so the bit pattern is what matters.
922 #[allow(clippy::cast_possible_wrap)]
923 {
924 *dst = src as _;
925 }
926 }
927 match &kv.value {
928 KvOverrideValue::Int(v) => {
929 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_INT;
930 raw.__bindgen_anon_1 =
931 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_i64: *v };
932 }
933 KvOverrideValue::Float(v) => {
934 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_FLOAT;
935 raw.__bindgen_anon_1 =
936 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_f64: *v };
937 }
938 KvOverrideValue::Bool(v) => {
939 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_BOOL;
940 raw.__bindgen_anon_1 =
941 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_bool: *v };
942 }
943 KvOverrideValue::Str(s) => {
944 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_STR;
945 raw.__bindgen_anon_1 =
946 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_str: *s };
947 }
948 }
949 raw
950 })
951 .chain(std::iter::once(llama_cpp_sys_4::llama_model_kv_override {
952 key: [0; 128],
953 tag: 0,
954 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
955 val_i64: 0,
956 },
957 }))
958 .collect();
959
960 // ── tt_overrides ────────────────────────────────────────────────────
961 // null-terminated by { null, GGML_TYPE_COUNT }
962 let tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override> = self
963 .tt_overrides
964 .iter()
965 .map(|ov| llama_cpp_sys_4::llama_model_tensor_override {
966 pattern: ov.pattern.as_ptr(),
967 type_: ov.ty as llama_cpp_sys_4::ggml_type,
968 })
969 .chain(std::iter::once(
970 llama_cpp_sys_4::llama_model_tensor_override {
971 pattern: null(),
972 type_: llama_cpp_sys_4::GGML_TYPE_COUNT,
973 },
974 ))
975 .collect();
976
977 // ── prune_layers ─────────────────────────────────────────────────────
978 // -1-terminated
979 let mut prune_c = self.prune_layers.clone();
980 prune_c.push(-1);
981
982 // ── assemble ────────────────────────────────────────────────────────
983 let raw = llama_cpp_sys_4::llama_model_quantize_params {
984 nthread: self.nthread,
985 ftype: self.ftype as llama_cpp_sys_4::llama_ftype,
986 output_tensor_type: self
987 .output_tensor_type
988 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
989 t as llama_cpp_sys_4::ggml_type
990 }),
991 token_embedding_type: self
992 .token_embedding_type
993 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
994 t as llama_cpp_sys_4::ggml_type
995 }),
996 allow_requantize: self.allow_requantize,
997 quantize_output_tensor: self.quantize_output_tensor,
998 only_copy: self.only_copy,
999 pure_: self.pure,
1000 keep_split: self.keep_split,
1001 dry_run: self.dry_run,
1002 max_buf_size: self.max_buf_size,
1003 imatrix: if self.imatrix.is_empty() {
1004 null()
1005 } else {
1006 imatrix_c.as_ptr()
1007 },
1008 kv_overrides: if self.kv_overrides.is_empty() {
1009 null()
1010 } else {
1011 kv_c.as_ptr()
1012 },
1013 tt_overrides: if self.tt_overrides.is_empty() {
1014 null()
1015 } else {
1016 tt_c.as_ptr()
1017 },
1018 prune_layers: if self.prune_layers.is_empty() {
1019 null()
1020 } else {
1021 prune_c.as_ptr()
1022 },
1023 };
1024
1025 RawQuantizeParamsGuard {
1026 raw,
1027 _imatrix_c: imatrix_c,
1028 _kv_c: kv_c,
1029 _tt_c: tt_c,
1030 _prune_c: prune_c,
1031 _marker: std::marker::PhantomData,
1032 }
1033 }
1034}
1035
1036/// Temporary storage that keeps the C pointers inside a raw
1037/// `llama_model_quantize_params` valid. Dropped after the quantize call.
1038pub(crate) struct RawQuantizeParamsGuard<'a> {
1039 pub(crate) raw: llama_cpp_sys_4::llama_model_quantize_params,
1040 _imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data>,
1041 _kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override>,
1042 _tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override>,
1043 _prune_c: Vec<i32>,
1044 // tie lifetime to the source QuantizeParams so the string/data
1045 // pointers inside imatrix_c and tt_c stay valid
1046 _marker: std::marker::PhantomData<&'a QuantizeParams>,
1047}
1048
1049// ─────────────────────────────────────────────────────────────────────────────
1050// TurboQuant – attention rotation
1051// ─────────────────────────────────────────────────────────────────────────────
1052
1053/// Control the `TurboQuant` attention-rotation feature globally.
1054///
1055/// When enabled (the default), llama.cpp applies a Hadamard rotation to Q/K/V
1056/// tensors before storing them in the KV cache. This significantly improves
1057/// quantization quality of the KV cache with near-zero overhead, as described
1058/// in llama.cpp PR #21038.
1059///
1060/// This function sets or clears the `LLAMA_ATTN_ROT_DISABLE` environment
1061/// variable, which llama.cpp reads once when a context (and its KV cache) is
1062/// first created. Call it **before** creating any [`LlamaContext`] on the
1063/// current process.
1064///
1065/// # Thread safety
1066///
1067/// Mutating environment variables while other threads may be reading them is
1068/// undefined behaviour. Call this function before spawning any threads that
1069/// use llama contexts, or ensure no contexts are being created concurrently.
1070///
1071/// # Example
1072///
1073/// ```no_run
1074/// // Disable the rotation for benchmarking purposes:
1075/// llama_cpp_4::quantize::set_attn_rot_disabled(true);
1076///
1077/// // Re-enable (default behaviour):
1078/// llama_cpp_4::quantize::set_attn_rot_disabled(false);
1079/// ```
1080///
1081/// [`LlamaContext`]: crate::context::LlamaContext
1082pub fn set_attn_rot_disabled(disabled: bool) {
1083 if disabled {
1084 // SAFETY: single-threaded context required by the caller.
1085 #[allow(unused_unsafe)]
1086 unsafe {
1087 std::env::set_var("LLAMA_ATTN_ROT_DISABLE", "1");
1088 }
1089 } else {
1090 #[allow(unused_unsafe)]
1091 unsafe {
1092 std::env::remove_var("LLAMA_ATTN_ROT_DISABLE");
1093 }
1094 }
1095}
1096
1097/// Returns `true` if `TurboQuant` attention rotation is currently disabled.
1098#[must_use]
1099pub fn attn_rot_disabled() -> bool {
1100 std::env::var("LLAMA_ATTN_ROT_DISABLE")
1101 .ok()
1102 .and_then(|v| v.parse::<i32>().ok())
1103 .is_some_and(|v| v != 0)
1104}
1105
1106// ─────────────────────────────────────────────────────────────────────────────
1107// Quantization preview
1108// ─────────────────────────────────────────────────────────────────────────────
1109
1110/// Ask llama.cpp what it *would* do, without writing a file.
1111///
1112/// [`crate::model_quantize`] is all-or-nothing: it reads a model, quantizes
1113/// every tensor, and writes the result. This exposes the decision layer
1114/// underneath — which tensors get quantized at all, and to which `ggml` type —
1115/// so a tool can show the plan, or estimate output size, before committing to
1116/// a run that may take minutes and tens of gigabytes.
1117///
1118/// The k-quant mixes are why this is not simply [`LlamaFtype::default_ggml_type`]:
1119/// they deliberately keep attention and output tensors at higher precision, so
1120/// the per-tensor answer differs from the ftype's nominal type.
1121///
1122/// Wraps `llama_quant_init` / `llama_quant_free`.
1123#[derive(Debug)]
1124pub struct QuantPreview<'model> {
1125 qs: std::ptr::NonNull<llama_cpp_sys_4::quantize_state_impl>,
1126 // The state borrows the model it was built from.
1127 _model: std::marker::PhantomData<&'model crate::model::LlamaModel>,
1128}
1129
1130impl Drop for QuantPreview<'_> {
1131 fn drop(&mut self) {
1132 unsafe { llama_cpp_sys_4::llama_quant_free(self.qs.as_ptr()) }
1133 }
1134}
1135
1136impl<'model> QuantPreview<'model> {
1137 /// Build a preview for `model` under `params`.
1138 ///
1139 /// # Errors
1140 ///
1141 /// Returns [`QuantPreviewError::Init`] if llama.cpp could not build the
1142 /// quantization state, which happens for a model it cannot quantize.
1143 pub fn new(
1144 model: &'model crate::model::LlamaModel,
1145 params: &QuantizeParams,
1146 ) -> Result<Self, QuantPreviewError> {
1147 let guard = params.to_raw();
1148 // The guarded wrapper: `llama_quant_init` throws for a model it cannot
1149 // quantize, and that unwinding into Rust aborts the process.
1150 let qs = unsafe {
1151 llama_cpp_sys_4::llama_quant_init_guarded(model.model.as_ptr(), &raw const guard.raw)
1152 };
1153 std::ptr::NonNull::new(qs)
1154 .map(|qs| Self {
1155 qs,
1156 _model: std::marker::PhantomData,
1157 })
1158 .ok_or(QuantPreviewError::Init)
1159 }
1160
1161 /// Whether this tensor would be quantized at all.
1162 ///
1163 /// llama.cpp skips 1-D tensors, tensors below a size threshold, and ones
1164 /// whose name marks them as needing full precision — so a `false` here is
1165 /// the usual reason a tensor keeps its original type.
1166 ///
1167 /// Requires the `ggml` feature, which is what exposes [`crate::ggml::GgmlTensor`].
1168 #[cfg(feature = "ggml")]
1169 #[must_use]
1170 pub fn allows_quantization(&self, tensor: &crate::ggml::GgmlTensor) -> bool {
1171 // 1 = yes, 0 = no, -1 = the underlying call threw and was caught.
1172 let rc = unsafe {
1173 llama_cpp_sys_4::llama_quant_tensor_allows_quantization_guarded(
1174 self.qs.as_ptr(),
1175 tensor.as_ptr(),
1176 )
1177 };
1178 rc == 1
1179 }
1180
1181 /// Compute the storage type each tensor would be assigned under `ftype`.
1182 ///
1183 /// Every tensor passed must already satisfy [`Self::allows_quantization`] —
1184 /// upstream states the caller filters first, and does not re-check.
1185 ///
1186 /// An entry is `None` when llama.cpp picks a `ggml` type this crate's
1187 /// [`GgmlType`] does not know.
1188 ///
1189 /// # Errors
1190 ///
1191 /// Returns [`QuantPreviewError::NotQuantizable`] naming the first tensor
1192 /// that fails the filter, rather than letting llama.cpp decide what to do
1193 /// with it.
1194 ///
1195 /// Requires the `ggml` feature.
1196 #[cfg(feature = "ggml")]
1197 pub fn compute_types(
1198 &self,
1199 tensors: &[&crate::ggml::GgmlTensor],
1200 ftype: LlamaFtype,
1201 ) -> Result<Vec<Option<GgmlType>>, QuantPreviewError> {
1202 if tensors.is_empty() {
1203 return Ok(Vec::new());
1204 }
1205 for tensor in tensors {
1206 if !self.allows_quantization(tensor) {
1207 return Err(QuantPreviewError::NotQuantizable(tensor.name().to_owned()));
1208 }
1209 }
1210
1211 let mut raw_tensors: Vec<*mut llama_cpp_sys_4::ggml_tensor> =
1212 tensors.iter().map(|t| t.as_ptr()).collect();
1213 let mut out = vec![0 as llama_cpp_sys_4::ggml_type; tensors.len()];
1214 let rc = unsafe {
1215 llama_cpp_sys_4::llama_quant_compute_types_guarded(
1216 self.qs.as_ptr(),
1217 ftype.into(),
1218 raw_tensors.as_mut_ptr(),
1219 out.as_mut_ptr(),
1220 out.len(),
1221 )
1222 };
1223 if rc != 0 {
1224 return Err(QuantPreviewError::Init);
1225 }
1226 Ok(out.into_iter().map(|t| GgmlType::try_from(t).ok()).collect())
1227 }
1228}
1229
1230/// Failure building or using a [`QuantPreview`].
1231#[derive(Debug, Eq, PartialEq, thiserror::Error)]
1232pub enum QuantPreviewError {
1233 /// llama.cpp could not build a quantization state for this model.
1234 #[error("could not initialize quantization state")]
1235 Init,
1236 /// A tensor that would not be quantized was passed to
1237 /// [`QuantPreview::compute_types`].
1238 #[error("tensor {0:?} is not quantizable; filter with allows_quantization first")]
1239 NotQuantizable(String),
1240 /// The mock-model descriptor contained an interior NUL byte.
1241 #[error("architecture name contained an interior NUL byte")]
1242 Nul(#[from] NulError),
1243 /// llama.cpp could not build a model from the descriptor.
1244 #[error("could not build a mock model from the descriptor")]
1245 MockModel,
1246}
1247
1248/// Shape of a synthetic model, for previewing a quantization plan without a
1249/// real checkpoint on disk.
1250///
1251/// Upstream marks `llama_quant_model_from_metadata` as being for testing; it is
1252/// exposed here for the same reason — it lets you ask "what would a `Q4_K_M` of a
1253/// model this shape look like?" without downloading one.
1254#[derive(Debug, Clone, PartialEq, Eq)]
1255pub struct QuantModelDesc {
1256 /// Architecture name, e.g. `"llama"`.
1257 pub architecture: String,
1258 /// Embedding dimension.
1259 pub n_embd: u32,
1260 /// Feed-forward dimension.
1261 pub n_ff: u32,
1262 /// Number of layers.
1263 pub n_layer: u32,
1264 /// Number of attention heads.
1265 pub n_head: u32,
1266 /// Number of key/value heads.
1267 pub n_head_kv: u32,
1268 /// Number of experts (0 for a dense model).
1269 pub n_expert: u32,
1270 /// Per-head key dimension.
1271 pub n_embd_head_k: u32,
1272 /// Per-head value dimension.
1273 pub n_embd_head_v: u32,
1274}
1275
1276impl QuantModelDesc {
1277 /// A dense llama-shaped descriptor with the given dimensions.
1278 #[must_use]
1279 pub fn llama(n_embd: u32, n_ff: u32, n_layer: u32, n_head: u32) -> Self {
1280 Self {
1281 architecture: "llama".to_owned(),
1282 n_embd,
1283 n_ff,
1284 n_layer,
1285 n_head,
1286 n_head_kv: n_head,
1287 n_expert: 0,
1288 n_embd_head_k: n_embd / n_head.max(1),
1289 n_embd_head_v: n_embd / n_head.max(1),
1290 }
1291 }
1292
1293 /// Build a synthetic model from this descriptor.
1294 ///
1295 /// The returned model owns llama.cpp memory and frees it on drop, exactly
1296 /// like a loaded one — it simply has no weights.
1297 ///
1298 /// # Errors
1299 ///
1300 /// Returns [`QuantPreviewError::MockModel`] if llama.cpp rejects the
1301 /// descriptor, or [`QuantPreviewError::Nul`] for a bad architecture name.
1302 pub fn build(&self) -> Result<crate::model::LlamaModel, QuantPreviewError> {
1303 let arch = CString::new(self.architecture.as_str())?;
1304 let desc = llama_cpp_sys_4::llama_quant_model_desc {
1305 architecture: arch.as_ptr(),
1306 n_embd: self.n_embd,
1307 n_ff: self.n_ff,
1308 n_layer: self.n_layer,
1309 n_head: self.n_head,
1310 n_head_kv: self.n_head_kv,
1311 n_expert: self.n_expert,
1312 n_embd_head_k: self.n_embd_head_k,
1313 n_embd_head_v: self.n_embd_head_v,
1314 };
1315 // Guarded: an architecture llama.cpp does not know makes the raw entry
1316 // point throw, which aborts the process on the way back into Rust.
1317 let raw =
1318 unsafe { llama_cpp_sys_4::llama_quant_model_from_metadata_guarded(&raw const desc) };
1319 let ptr = std::ptr::NonNull::new(raw).ok_or(QuantPreviewError::MockModel)?;
1320 // SAFETY: upstream documents the result as owned by the caller and
1321 // freed with `llama_model_free`, which is what `LlamaModel` does.
1322 Ok(unsafe { crate::model::LlamaModel::from_raw(ptr) })
1323 }
1324}