llama_cpp_4/quantize.rs
1//! Quantization types and parameters for converting models to lower-bit precisions.
2//!
3//! # Quick start
4//!
5//! ```no_run
6//! use llama_cpp_4::quantize::{LlamaFtype, QuantizeParams};
7//!
8//! let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
9//! .with_nthread(8)
10//! .with_quantize_output_tensor(true);
11//!
12//! llama_cpp_4::model_quantize("model-f16.gguf", "model-q4km.gguf", ¶ms).unwrap();
13//! ```
14//!
15//! # `TurboQuant` – attention rotation (PR #21038)
16//!
17//! llama.cpp applies a Hadamard rotation to Q/K/V tensors before writing them into the KV cache.
18//! This significantly improves KV-cache quantization quality at near-zero cost, and is enabled by
19//! default for every model whose head dimension is a power of two. You can opt out per-context
20//! with [`LlamaContextParams::with_attn_rot_disabled`] or globally with
21//! [`set_attn_rot_disabled`].
22//!
23//! [`LlamaContextParams::with_attn_rot_disabled`]: crate::context::params::LlamaContextParams::with_attn_rot_disabled
24
25use std::ffi::{CString, NulError};
26use std::ptr::null;
27
28// ─────────────────────────────────────────────────────────────────────────────
29// LlamaFtype
30// ─────────────────────────────────────────────────────────────────────────────
31
32/// The quantization type used for the bulk of a model file (maps to `llama_ftype`).
33///
34/// Pass one of these variants to [`QuantizeParams::new`] to choose the target precision.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
36#[non_exhaustive]
37#[allow(missing_docs)]
38pub enum LlamaFtype {
39 /// All tensors stored as full F32 (very large, for reference only)
40 AllF32 = 0,
41 /// F16 – 14 GB @ 7B, +0.0020 ppl vs Mistral-7B
42 MostlyF16 = 1,
43 /// `Q4_0` – 4.34 GB @ 8B, +0.4685 ppl
44 MostlyQ4_0 = 2,
45 /// `Q4_1` – 4.78 GB @ 8B, +0.4511 ppl
46 MostlyQ4_1 = 3,
47 /// `Q8_0` – 7.96 GB @ 8B, +0.0026 ppl
48 MostlyQ8_0 = 7,
49 /// `Q5_0` – 5.21 GB @ 8B, +0.1316 ppl
50 MostlyQ5_0 = 8,
51 /// `Q5_1` – 5.65 GB @ 8B, +0.1062 ppl
52 MostlyQ5_1 = 9,
53 /// `Q2_K` – 2.96 GB @ 8B, +3.5199 ppl
54 MostlyQ2K = 10,
55 /// `Q3_K` small – 3.41 GB @ 8B, +1.6321 ppl
56 MostlyQ3KS = 11,
57 /// `Q3_K` medium – 3.74 GB @ 8B, +0.6569 ppl
58 MostlyQ3KM = 12,
59 /// `Q3_K` large – 4.03 GB @ 8B, +0.5562 ppl
60 MostlyQ3KL = 13,
61 /// `Q4_K` small – 4.37 GB @ 8B, +0.2689 ppl
62 MostlyQ4KS = 14,
63 /// `Q4_K` medium – 4.58 GB @ 8B, +0.1754 ppl *(recommended default)*
64 MostlyQ4KM = 15,
65 /// `Q5_K` small – 5.21 GB @ 8B, +0.1049 ppl
66 MostlyQ5KS = 16,
67 /// `Q5_K` medium – 5.33 GB @ 8B, +0.0569 ppl
68 MostlyQ5KM = 17,
69 /// `Q6_K` – 6.14 GB @ 8B, +0.0217 ppl
70 MostlyQ6K = 18,
71 /// `IQ2_XXS` – 2.06 bpw
72 MostlyIQ2XXS = 19,
73 /// `IQ2_XS` – 2.31 bpw
74 MostlyIQ2XS = 20,
75 /// `Q2_K` small
76 MostlyQ2KS = 21,
77 /// `IQ3_XS` – 3.3 bpw
78 MostlyIQ3XS = 22,
79 /// `IQ3_XXS` – 3.06 bpw
80 MostlyIQ3XXS = 23,
81 /// `IQ1_S` – 1.56 bpw (extremely small, high loss)
82 MostlyIQ1S = 24,
83 /// `IQ4_NL` – 4.50 bpw non-linear
84 MostlyIQ4NL = 25,
85 /// `IQ3_S` – 3.44 bpw
86 MostlyIQ3S = 26,
87 /// `IQ3_M` – 3.66 bpw
88 MostlyIQ3M = 27,
89 /// `IQ2_S` – 2.5 bpw
90 MostlyIQ2S = 28,
91 /// `IQ2_M` – 2.7 bpw
92 MostlyIQ2M = 29,
93 /// `IQ4_XS` – 4.25 bpw non-linear
94 MostlyIQ4XS = 30,
95 /// `IQ1_M` – 1.75 bpw
96 MostlyIQ1M = 31,
97 /// BF16 – 14 GB @ 7B, −0.0050 ppl vs Mistral-7B
98 MostlyBF16 = 32,
99 /// `TQ1_0` – 1.69 bpw ternary
100 MostlyTQ1_0 = 36,
101 /// `TQ2_0` – 2.06 bpw ternary
102 MostlyTQ2_0 = 37,
103 /// MXFP4 (`MoE` layers)
104 MostlyMXFP4Moe = 38,
105 /// NVFP4
106 MostlyNVFP4 = 39,
107 /// `Q1_0` – 1.5 bpw binary (block size 32)
108 MostlyQ1_0 = 40,
109 /// `Q2_0` – 2 bpw binary
110 //
111 // 41 was `Q1_0_g128` behind the `q1` feature, from this crate's own
112 // quantization patch. Upstream uses 41 for `Q2_0`, and never had a
113 // `Q1_0_g128`, so the old variant named a type that does not exist while
114 // sitting on a value that means something else.
115 MostlyQ2_0 = 41,
116}
117
118impl LlamaFtype {
119 /// Short name suitable for filenames (e.g. `"Q4_K_M"`).
120 #[must_use]
121 pub fn name(self) -> &'static str {
122 match self {
123 Self::AllF32 => "F32",
124 Self::MostlyF16 => "F16",
125 Self::MostlyQ4_0 => "Q4_0",
126 Self::MostlyQ4_1 => "Q4_1",
127 Self::MostlyQ8_0 => "Q8_0",
128 Self::MostlyQ5_0 => "Q5_0",
129 Self::MostlyQ5_1 => "Q5_1",
130 Self::MostlyQ2K => "Q2_K",
131 Self::MostlyQ3KS => "Q3_K_S",
132 Self::MostlyQ3KM => "Q3_K_M",
133 Self::MostlyQ3KL => "Q3_K_L",
134 Self::MostlyQ4KS => "Q4_K_S",
135 Self::MostlyQ4KM => "Q4_K_M",
136 Self::MostlyQ5KS => "Q5_K_S",
137 Self::MostlyQ5KM => "Q5_K_M",
138 Self::MostlyQ6K => "Q6_K",
139 Self::MostlyIQ2XXS => "IQ2_XXS",
140 Self::MostlyIQ2XS => "IQ2_XS",
141 Self::MostlyQ2KS => "Q2_K_S",
142 Self::MostlyIQ3XS => "IQ3_XS",
143 Self::MostlyIQ3XXS => "IQ3_XXS",
144 Self::MostlyIQ1S => "IQ1_S",
145 Self::MostlyIQ4NL => "IQ4_NL",
146 Self::MostlyIQ3S => "IQ3_S",
147 Self::MostlyIQ3M => "IQ3_M",
148 Self::MostlyIQ2S => "IQ2_S",
149 Self::MostlyIQ2M => "IQ2_M",
150 Self::MostlyIQ4XS => "IQ4_XS",
151 Self::MostlyIQ1M => "IQ1_M",
152 Self::MostlyBF16 => "BF16",
153 Self::MostlyTQ1_0 => "TQ1_0",
154 Self::MostlyTQ2_0 => "TQ2_0",
155 Self::MostlyMXFP4Moe => "MXFP4_MOE",
156 Self::MostlyNVFP4 => "NVFP4",
157 Self::MostlyQ1_0 => "Q1_0",
158 Self::MostlyQ2_0 => "Q2_0",
159 }
160 }
161
162 /// Human-readable description with approximate size and PPL delta.
163 #[must_use]
164 pub fn description(self) -> &'static str {
165 match self {
166 Self::AllF32 => "26.00 GB @ 7B — full precision reference",
167 Self::MostlyF16 => "14.00 GB @ 7B — +0.0020 ppl vs Mistral-7B",
168 Self::MostlyBF16 => "14.00 GB @ 7B — -0.0050 ppl vs Mistral-7B",
169 Self::MostlyQ8_0 => " 7.96 GB @ 8B — +0.0026 ppl",
170 Self::MostlyQ6K => " 6.14 GB @ 8B — +0.0217 ppl",
171 Self::MostlyQ5KM => " 5.33 GB @ 8B — +0.0569 ppl",
172 Self::MostlyQ5KS => " 5.21 GB @ 8B — +0.1049 ppl",
173 Self::MostlyQ5_1 => " 5.65 GB @ 8B — +0.1062 ppl",
174 Self::MostlyQ5_0 => " 5.21 GB @ 8B — +0.1316 ppl",
175 Self::MostlyQ4KM => " 4.58 GB @ 8B — +0.1754 ppl [recommended]",
176 Self::MostlyQ4KS => " 4.37 GB @ 8B — +0.2689 ppl",
177 Self::MostlyQ4_1 => " 4.78 GB @ 8B — +0.4511 ppl",
178 Self::MostlyQ4_0 => " 4.34 GB @ 8B — +0.4685 ppl",
179 Self::MostlyQ3KL => " 4.03 GB @ 8B — +0.5562 ppl",
180 Self::MostlyQ3KM => " 3.74 GB @ 8B — +0.6569 ppl",
181 Self::MostlyQ3KS => " 3.41 GB @ 8B — +1.6321 ppl",
182 Self::MostlyQ2KS => " 2.96 GB @ 8B — +3.1836 ppl",
183 Self::MostlyQ2K => " 2.96 GB @ 8B — +3.5199 ppl",
184 Self::MostlyIQ4XS => " 4.25 bpw non-linear",
185 Self::MostlyIQ4NL => " 4.50 bpw non-linear",
186 Self::MostlyIQ3S => " 3.44 bpw",
187 Self::MostlyIQ3M => " 3.66 bpw",
188 Self::MostlyIQ3XS => " 3.3 bpw",
189 Self::MostlyIQ3XXS => " 3.06 bpw",
190 Self::MostlyIQ2M => " 2.7 bpw",
191 Self::MostlyIQ2S => " 2.5 bpw",
192 Self::MostlyIQ2XS => " 2.31 bpw",
193 Self::MostlyIQ2XXS => " 2.06 bpw",
194 Self::MostlyIQ1M => " 1.75 bpw — extreme compression",
195 Self::MostlyIQ1S => " 1.56 bpw — extreme compression",
196 Self::MostlyTQ1_0 => " 1.69 bpw ternary",
197 Self::MostlyTQ2_0 => " 2.06 bpw ternary",
198 Self::MostlyMXFP4Moe => "MXFP4 MoE layers",
199 Self::MostlyNVFP4 => "NVFP4",
200 Self::MostlyQ1_0 => " 1.50 bpw — binary Q1_0 (block 32)",
201 Self::MostlyQ2_0 => " 2.00 bpw — binary Q2_0",
202 }
203 }
204
205 /// Look up a variant by its short name (case-insensitive).
206 ///
207 /// ```
208 /// use llama_cpp_4::quantize::LlamaFtype;
209 /// assert_eq!(LlamaFtype::from_name("Q4_K_M"), Some(LlamaFtype::MostlyQ4KM));
210 /// assert_eq!(LlamaFtype::from_name("q4_k_m"), Some(LlamaFtype::MostlyQ4KM));
211 /// assert_eq!(LlamaFtype::from_name("bogus"), None);
212 /// ```
213 #[must_use]
214 pub fn from_name(name: &str) -> Option<Self> {
215 let upper = name.to_uppercase();
216 match upper.as_str() {
217 "F32" => Some(Self::AllF32),
218 "F16" => Some(Self::MostlyF16),
219 "BF16" => Some(Self::MostlyBF16),
220 "Q4_0" => Some(Self::MostlyQ4_0),
221 "Q4_1" => Some(Self::MostlyQ4_1),
222 "Q8_0" => Some(Self::MostlyQ8_0),
223 "Q5_0" => Some(Self::MostlyQ5_0),
224 "Q5_1" => Some(Self::MostlyQ5_1),
225 "Q2_K" => Some(Self::MostlyQ2K),
226 "Q2_K_S" => Some(Self::MostlyQ2KS),
227 "Q3_K_S" => Some(Self::MostlyQ3KS),
228 "Q3_K_M" => Some(Self::MostlyQ3KM),
229 "Q3_K_L" => Some(Self::MostlyQ3KL),
230 "Q4_K_S" => Some(Self::MostlyQ4KS),
231 "Q4_K_M" => Some(Self::MostlyQ4KM),
232 "Q5_K_S" => Some(Self::MostlyQ5KS),
233 "Q5_K_M" => Some(Self::MostlyQ5KM),
234 "Q6_K" => Some(Self::MostlyQ6K),
235 "IQ1_S" => Some(Self::MostlyIQ1S),
236 "IQ1_M" => Some(Self::MostlyIQ1M),
237 "IQ2_XXS" => Some(Self::MostlyIQ2XXS),
238 "IQ2_XS" => Some(Self::MostlyIQ2XS),
239 "IQ2_S" => Some(Self::MostlyIQ2S),
240 "IQ2_M" => Some(Self::MostlyIQ2M),
241 "IQ3_XXS" => Some(Self::MostlyIQ3XXS),
242 "IQ3_XS" => Some(Self::MostlyIQ3XS),
243 "IQ3_S" => Some(Self::MostlyIQ3S),
244 "IQ3_M" => Some(Self::MostlyIQ3M),
245 "IQ4_NL" => Some(Self::MostlyIQ4NL),
246 "IQ4_XS" => Some(Self::MostlyIQ4XS),
247 "TQ1_0" => Some(Self::MostlyTQ1_0),
248 "TQ2_0" => Some(Self::MostlyTQ2_0),
249 "MXFP4_MOE" => Some(Self::MostlyMXFP4Moe),
250 "NVFP4" => Some(Self::MostlyNVFP4),
251 "Q1_0" => Some(Self::MostlyQ1_0),
252 "Q2_0" => Some(Self::MostlyQ2_0),
253 _ => None,
254 }
255 }
256
257 /// llama.cpp's own display name for this type, e.g. `"Q4_K - Medium"`.
258 ///
259 /// Where [`Self::name`] is a terse filename-safe token (`"Q4_K_M"`) kept in
260 /// a table in this crate, this asks llama.cpp, so it cannot drift from
261 /// upstream. Wraps `llama_ftype_name`.
262 ///
263 /// # Errors
264 ///
265 /// Returns [`FtypeNameError`] if llama.cpp returns null or non-UTF-8.
266 pub fn upstream_name(self) -> Result<&'static str, FtypeNameError> {
267 let ptr = unsafe { llama_cpp_sys_4::llama_ftype_name(self.into()) };
268 if ptr.is_null() {
269 return Err(FtypeNameError::Unnamed(self));
270 }
271 // SAFETY: llama.cpp returns a pointer to a string literal with static
272 // storage duration, so `'static` is sound here.
273 unsafe { std::ffi::CStr::from_ptr(ptr) }
274 .to_str()
275 .map_err(FtypeNameError::Utf8)
276 }
277
278 /// The `ggml` storage type most tensors get under this ftype.
279 ///
280 /// This is the *default*; the k-quant mixes deliberately store some tensors
281 /// (attention, output) at higher precision, so an individual tensor may not
282 /// use it. Wraps `llama_ftype_get_default_type`.
283 ///
284 /// Returns `None` for a `ggml` type this crate's [`GgmlType`] does not know.
285 #[must_use]
286 pub fn default_ggml_type(self) -> Option<GgmlType> {
287 let raw = unsafe { llama_cpp_sys_4::llama_ftype_get_default_type(self.into()) };
288 GgmlType::try_from(raw).ok()
289 }
290
291 /// All available types, ordered roughly from largest to smallest.
292 #[must_use]
293 pub fn all() -> &'static [Self] {
294 &[
295 Self::AllF32,
296 Self::MostlyF16,
297 Self::MostlyBF16,
298 Self::MostlyQ8_0,
299 Self::MostlyQ6K,
300 Self::MostlyQ5KM,
301 Self::MostlyQ5KS,
302 Self::MostlyQ5_1,
303 Self::MostlyQ5_0,
304 Self::MostlyQ4KM,
305 Self::MostlyQ4KS,
306 Self::MostlyQ4_1,
307 Self::MostlyQ4_0,
308 Self::MostlyQ3KL,
309 Self::MostlyQ3KM,
310 Self::MostlyQ3KS,
311 Self::MostlyQ2KS,
312 Self::MostlyQ2K,
313 Self::MostlyIQ4XS,
314 Self::MostlyIQ4NL,
315 Self::MostlyIQ3S,
316 Self::MostlyIQ3M,
317 Self::MostlyIQ3XS,
318 Self::MostlyIQ3XXS,
319 Self::MostlyIQ2M,
320 Self::MostlyIQ2S,
321 Self::MostlyIQ2XS,
322 Self::MostlyIQ2XXS,
323 Self::MostlyIQ1M,
324 Self::MostlyIQ1S,
325 Self::MostlyTQ1_0,
326 Self::MostlyTQ2_0,
327 Self::MostlyMXFP4Moe,
328 Self::MostlyNVFP4,
329 Self::MostlyQ1_0,
330 Self::MostlyQ2_0,
331 ]
332 }
333}
334
335/// Failure reading llama.cpp's display name for an [`LlamaFtype`].
336#[derive(Debug, Eq, PartialEq, thiserror::Error)]
337pub enum FtypeNameError {
338 /// llama.cpp returned null for this type.
339 #[error("llama.cpp has no name for ftype {0:?}")]
340 Unnamed(LlamaFtype),
341 /// The returned bytes were not valid UTF-8.
342 #[error(transparent)]
343 Utf8(#[from] std::str::Utf8Error),
344}
345
346impl From<LlamaFtype> for llama_cpp_sys_4::llama_ftype {
347 fn from(t: LlamaFtype) -> Self {
348 t as llama_cpp_sys_4::llama_ftype
349 }
350}
351
352impl TryFrom<llama_cpp_sys_4::llama_ftype> for LlamaFtype {
353 type Error = llama_cpp_sys_4::llama_ftype;
354
355 /// Fails on a discriminant this build does not know — `LLAMA_FTYPE_GUESSED`
356 /// (1024, "the file did not say"), a `q1` type without that feature, or a
357 /// type upstream added since this release. The raw value is returned so the
358 /// caller can report it.
359 fn try_from(raw: llama_cpp_sys_4::llama_ftype) -> Result<Self, Self::Error> {
360 Self::all()
361 .iter()
362 .copied()
363 .find(|t| llama_cpp_sys_4::llama_ftype::from(*t) == raw)
364 .ok_or(raw)
365 }
366}
367
368impl std::fmt::Display for LlamaFtype {
369 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
370 write!(f, "{}", self.name())
371 }
372}
373
374// ─────────────────────────────────────────────────────────────────────────────
375// GgmlType
376// ─────────────────────────────────────────────────────────────────────────────
377
378/// GGML tensor storage type (maps to `ggml_type`).
379///
380/// Used to set [`QuantizeParams::output_tensor_type`] and
381/// [`QuantizeParams::token_embedding_type`], and for per-tensor type overrides
382/// in [`TensorTypeOverride`].
383#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
384#[non_exhaustive]
385#[allow(missing_docs)]
386pub enum GgmlType {
387 F32 = 0,
388 F16 = 1,
389 Q4_0 = 2,
390 Q4_1 = 3,
391 Q5_0 = 6,
392 Q5_1 = 7,
393 Q8_0 = 8,
394 Q8_1 = 9,
395 Q2K = 10,
396 Q3K = 11,
397 Q4K = 12,
398 Q5K = 13,
399 Q6K = 14,
400 Q8K = 15,
401 IQ2XXS = 16,
402 IQ2XS = 17,
403 IQ3XXS = 18,
404 IQ1S = 19,
405 IQ4NL = 20,
406 IQ3S = 21,
407 IQ2S = 22,
408 IQ4XS = 23,
409 I8 = 24,
410 I16 = 25,
411 I32 = 26,
412 I64 = 27,
413 F64 = 28,
414 IQ1M = 29,
415 BF16 = 30,
416 TQ1_0 = 34,
417 TQ2_0 = 35,
418 MXFP4 = 39,
419 /// NVFP4 (4 blocks, E4M3 scale).
420 NVFP4 = 40,
421 /// `Q1_0` — 1.5 bpw binary.
422 //
423 // These values used to be shuffled by the `q1` feature, which encoded the
424 // numbering of this crate's own `0001-q1-quantization.patch`: NVFP4 moved
425 // to 42 so Q1_0 could take 40 and a `Q1_0_G128` take 41. Q1_0 has since
426 // landed upstream at 41 — with Q2_0 at 42 and NVFP4 staying at 40 — and the
427 // patch is gone, so the feature was renumbering three variants away from
428 // what llama.cpp actually uses. Upstream's numbering is the only numbering.
429 Q1_0 = 41,
430 /// `Q2_0` — 2 bpw binary.
431 Q2_0 = 42,
432}
433
434impl From<GgmlType> for llama_cpp_sys_4::ggml_type {
435 fn from(t: GgmlType) -> Self {
436 t as llama_cpp_sys_4::ggml_type
437 }
438}
439
440impl TryFrom<llama_cpp_sys_4::ggml_type> for GgmlType {
441 type Error = llama_cpp_sys_4::ggml_type;
442 fn try_from(v: llama_cpp_sys_4::ggml_type) -> Result<Self, Self::Error> {
443 match v {
444 0 => Ok(Self::F32),
445 1 => Ok(Self::F16),
446 2 => Ok(Self::Q4_0),
447 3 => Ok(Self::Q4_1),
448 6 => Ok(Self::Q5_0),
449 7 => Ok(Self::Q5_1),
450 8 => Ok(Self::Q8_0),
451 9 => Ok(Self::Q8_1),
452 10 => Ok(Self::Q2K),
453 11 => Ok(Self::Q3K),
454 12 => Ok(Self::Q4K),
455 13 => Ok(Self::Q5K),
456 14 => Ok(Self::Q6K),
457 15 => Ok(Self::Q8K),
458 16 => Ok(Self::IQ2XXS),
459 17 => Ok(Self::IQ2XS),
460 18 => Ok(Self::IQ3XXS),
461 19 => Ok(Self::IQ1S),
462 20 => Ok(Self::IQ4NL),
463 21 => Ok(Self::IQ3S),
464 22 => Ok(Self::IQ2S),
465 23 => Ok(Self::IQ4XS),
466 24 => Ok(Self::I8),
467 25 => Ok(Self::I16),
468 26 => Ok(Self::I32),
469 27 => Ok(Self::I64),
470 28 => Ok(Self::F64),
471 29 => Ok(Self::IQ1M),
472 30 => Ok(Self::BF16),
473 34 => Ok(Self::TQ1_0),
474 35 => Ok(Self::TQ2_0),
475 39 => Ok(Self::MXFP4),
476 40 => Ok(Self::NVFP4),
477 41 => Ok(Self::Q1_0),
478 42 => Ok(Self::Q2_0),
479 _ => Err(v),
480 }
481 }
482}
483
484// ─────────────────────────────────────────────────────────────────────────────
485// ImatrixEntry / Imatrix
486// ─────────────────────────────────────────────────────────────────────────────
487
488/// A single per-tensor importance matrix entry, as loaded from a `.imatrix` file.
489///
490/// Each entry contains activation statistics for one model tensor collected from
491/// a calibration dataset. When supplied to [`QuantizeParams::with_imatrix`] these
492/// statistics guide the quantizer to allocate more precision to weights that
493/// matter most.
494#[derive(Debug, Clone)]
495pub struct ImatrixEntry {
496 name: CString,
497 data: Vec<f32>,
498}
499
500impl ImatrixEntry {
501 /// Create a new entry from a tensor name and its importance scores.
502 ///
503 /// # Errors
504 ///
505 /// Returns [`NulError`] if `name` contains an interior null byte.
506 pub fn new(name: impl Into<Vec<u8>>, data: Vec<f32>) -> Result<Self, NulError> {
507 Ok(Self {
508 name: CString::new(name)?,
509 data,
510 })
511 }
512
513 /// Tensor name.
514 #[must_use]
515 pub fn name_str(&self) -> &str {
516 self.name.to_str().unwrap_or("")
517 }
518
519 /// Number of importance values.
520 #[must_use]
521 pub fn len(&self) -> usize {
522 self.data.len()
523 }
524
525 /// Returns `true` if the data slice is empty.
526 #[must_use]
527 pub fn is_empty(&self) -> bool {
528 self.data.is_empty()
529 }
530}
531
532/// A collection of importance matrix entries (one per quantized tensor).
533///
534/// Build one by pushing [`ImatrixEntry`] values, then pass it to
535/// [`QuantizeParams::with_imatrix`].
536#[derive(Debug, Clone, Default)]
537pub struct Imatrix {
538 entries: Vec<ImatrixEntry>,
539}
540
541impl Imatrix {
542 /// Create an empty imatrix.
543 #[must_use]
544 pub fn new() -> Self {
545 Self::default()
546 }
547
548 /// Add an entry.
549 pub fn push(&mut self, entry: ImatrixEntry) {
550 self.entries.push(entry);
551 }
552
553 /// Number of entries.
554 #[must_use]
555 pub fn len(&self) -> usize {
556 self.entries.len()
557 }
558
559 /// Returns `true` if no entries have been added.
560 #[must_use]
561 pub fn is_empty(&self) -> bool {
562 self.entries.is_empty()
563 }
564}
565
566// ─────────────────────────────────────────────────────────────────────────────
567// TensorTypeOverride
568// ─────────────────────────────────────────────────────────────────────────────
569
570/// Override the quantization type of every tensor whose name matches a glob `pattern`.
571///
572/// The pattern syntax is the same as used by the `--tensor-type` flag in
573/// `llama-quantize`, e.g. `"attn.*"` or `"blk.0.*"`.
574///
575/// # Example
576///
577/// ```
578/// use llama_cpp_4::quantize::{GgmlType, TensorTypeOverride};
579///
580/// // Keep the output projection in F16:
581/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
582/// ```
583#[derive(Debug, Clone)]
584pub struct TensorTypeOverride {
585 pattern: CString,
586 ty: GgmlType,
587}
588
589impl TensorTypeOverride {
590 /// Create a new override.
591 ///
592 /// # Errors
593 ///
594 /// Returns [`NulError`] if `pattern` contains an interior null byte.
595 pub fn new(pattern: impl Into<Vec<u8>>, ty: GgmlType) -> Result<Self, NulError> {
596 Ok(Self {
597 pattern: CString::new(pattern)?,
598 ty,
599 })
600 }
601
602 /// The glob pattern that selects tensors.
603 #[must_use]
604 pub fn pattern_str(&self) -> &str {
605 self.pattern.to_str().unwrap_or("")
606 }
607
608 /// The type to assign to matching tensors.
609 #[must_use]
610 pub fn ty(&self) -> GgmlType {
611 self.ty
612 }
613}
614
615// ─────────────────────────────────────────────────────────────────────────────
616// KvOverrideValue / KvOverride
617// ─────────────────────────────────────────────────────────────────────────────
618
619/// A value in a GGUF key-value metadata override.
620#[derive(Debug, Clone, PartialEq)]
621pub enum KvOverrideValue {
622 /// 64-bit integer
623 Int(i64),
624 /// 64-bit float
625 Float(f64),
626 /// Boolean
627 Bool(bool),
628 /// Fixed-length string (up to 127 bytes + NUL)
629 Str([std::os::raw::c_char; 128]),
630}
631
632/// A single GGUF metadata key-value override.
633///
634/// These are written into the output file's metadata when quantizing.
635#[derive(Debug, Clone)]
636pub struct KvOverride {
637 key: CString,
638 /// The value for this override.
639 pub value: KvOverrideValue,
640}
641
642impl KvOverride {
643 /// Create a new override.
644 ///
645 /// # Errors
646 ///
647 /// Returns [`NulError`] if `key` contains an interior null byte.
648 pub fn new(key: impl Into<Vec<u8>>, value: KvOverrideValue) -> Result<Self, NulError> {
649 Ok(Self {
650 key: CString::new(key)?,
651 value,
652 })
653 }
654}
655
656// ─────────────────────────────────────────────────────────────────────────────
657// QuantizeParams
658// ─────────────────────────────────────────────────────────────────────────────
659
660/// Parameters for quantizing a model.
661///
662/// Create with [`QuantizeParams::new`] and chain `with_*` builder methods to
663/// configure, then pass a reference to [`crate::model_quantize`].
664///
665/// # Example
666///
667/// ```no_run
668/// use llama_cpp_4::quantize::{GgmlType, LlamaFtype, QuantizeParams, TensorTypeOverride};
669///
670/// let ov = TensorTypeOverride::new("output", GgmlType::F16).unwrap();
671///
672/// let params = QuantizeParams::new(LlamaFtype::MostlyQ4KM)
673/// .with_nthread(8)
674/// .with_allow_requantize(false)
675/// .with_quantize_output_tensor(true)
676/// .with_pure(false)
677/// .with_tensor_type_override(ov);
678///
679/// llama_cpp_4::model_quantize("in.gguf", "out.gguf", ¶ms).unwrap();
680/// ```
681#[derive(Debug, Clone)]
682#[allow(clippy::struct_excessive_bools)]
683pub struct QuantizeParams {
684 /// Number of threads (0 = auto-detect).
685 pub nthread: i32,
686 /// Target quantization type.
687 pub ftype: LlamaFtype,
688 /// Force this storage type for the output/lm-head tensor (`None` = use ftype default).
689 pub output_tensor_type: Option<GgmlType>,
690 /// Force this storage type for the token-embedding tensor (`None` = use ftype default).
691 pub token_embedding_type: Option<GgmlType>,
692 /// Allow re-quantizing tensors that are already quantized.
693 pub allow_requantize: bool,
694 /// Quantize the output/lm-head weight tensor.
695 pub quantize_output_tensor: bool,
696 /// Copy all tensors without quantizing (ignores `ftype`).
697 pub only_copy: bool,
698 /// Quantize every tensor to the same type (no mixed k-quant strategy).
699 pub pure: bool,
700 /// Keep the same number of shards as the input (for split models).
701 pub keep_split: bool,
702 /// Estimate output size without writing anything to disk.
703 pub dry_run: bool,
704 /// Cap, in bytes, on the tensor rows held in memory at once (`0` = the
705 /// upstream default of 8 GiB). Lower it to quantize a model larger than
706 /// available RAM at the cost of more I/O.
707 pub max_buf_size: usize,
708
709 imatrix: Vec<ImatrixEntry>,
710 kv_overrides: Vec<KvOverride>,
711 tt_overrides: Vec<TensorTypeOverride>,
712 prune_layers: Vec<i32>,
713}
714
715impl QuantizeParams {
716 /// Create a new params set targeting `ftype`.
717 ///
718 /// All other options are set to the same defaults as
719 /// `llama_model_quantize_default_params()`.
720 #[must_use]
721 pub fn new(ftype: LlamaFtype) -> Self {
722 // Read the C defaults so we match them exactly.
723 let d = unsafe { llama_cpp_sys_4::llama_model_quantize_default_params() };
724 Self {
725 nthread: d.nthread,
726 ftype,
727 output_tensor_type: GgmlType::try_from(d.output_tensor_type).ok(),
728 token_embedding_type: GgmlType::try_from(d.token_embedding_type).ok(),
729 allow_requantize: d.allow_requantize,
730 quantize_output_tensor: d.quantize_output_tensor,
731 only_copy: d.only_copy,
732 pure: d.pure_,
733 keep_split: d.keep_split,
734 dry_run: d.dry_run,
735 max_buf_size: d.max_buf_size,
736 imatrix: Vec::new(),
737 kv_overrides: Vec::new(),
738 tt_overrides: Vec::new(),
739 prune_layers: Vec::new(),
740 }
741 }
742
743 /// Set the number of quantization threads (`0` = auto).
744 #[must_use]
745 pub fn with_nthread(mut self, n: i32) -> Self {
746 self.nthread = n;
747 self
748 }
749
750 /// Override the output-tensor storage type.
751 #[must_use]
752 pub fn with_output_tensor_type(mut self, ty: GgmlType) -> Self {
753 self.output_tensor_type = Some(ty);
754 self
755 }
756
757 /// Override the token-embedding storage type.
758 #[must_use]
759 pub fn with_token_embedding_type(mut self, ty: GgmlType) -> Self {
760 self.token_embedding_type = Some(ty);
761 self
762 }
763
764 /// Allow (or disallow) re-quantizing already-quantized tensors.
765 #[must_use]
766 pub fn with_allow_requantize(mut self, v: bool) -> Self {
767 self.allow_requantize = v;
768 self
769 }
770
771 /// Quantize the output/lm-head weight (`true` by default).
772 #[must_use]
773 pub fn with_quantize_output_tensor(mut self, v: bool) -> Self {
774 self.quantize_output_tensor = v;
775 self
776 }
777
778 /// When `true`, only copy tensors verbatim (no quantization at all).
779 #[must_use]
780 pub fn with_only_copy(mut self, v: bool) -> Self {
781 self.only_copy = v;
782 self
783 }
784
785 /// When `true`, quantize all tensors to the same type (no mixed k-quant strategy).
786 #[must_use]
787 pub fn with_pure(mut self, v: bool) -> Self {
788 self.pure = v;
789 self
790 }
791
792 /// Preserve the number of shards when quantizing a split model.
793 #[must_use]
794 pub fn with_keep_split(mut self, v: bool) -> Self {
795 self.keep_split = v;
796 self
797 }
798
799 /// Only estimate the output size; do not write anything to disk.
800 #[must_use]
801 pub fn with_dry_run(mut self, v: bool) -> Self {
802 self.dry_run = v;
803 self
804 }
805
806 /// Cap the bytes of tensor rows kept in memory at once (`0` = upstream
807 /// default, 8 GiB).
808 #[must_use]
809 pub fn with_max_buf_size(mut self, bytes: usize) -> Self {
810 self.max_buf_size = bytes;
811 self
812 }
813
814 /// Supply importance matrix data to improve quantization quality.
815 ///
816 /// The imatrix is generated by the `imatrix` tool (or the `imatrix` example
817 /// in this crate) and contains per-tensor activation statistics collected
818 /// from a calibration dataset.
819 #[must_use]
820 pub fn with_imatrix(mut self, imatrix: Imatrix) -> Self {
821 self.imatrix = imatrix.entries;
822 self
823 }
824
825 /// Append a single imatrix entry.
826 #[must_use]
827 pub fn with_imatrix_entry(mut self, entry: ImatrixEntry) -> Self {
828 self.imatrix.push(entry);
829 self
830 }
831
832 /// Add (or replace) a GGUF metadata key-value pair in the output file.
833 #[must_use]
834 pub fn with_kv_override(mut self, kv: KvOverride) -> Self {
835 self.kv_overrides.push(kv);
836 self
837 }
838
839 /// Override the quantization type for tensors whose name matches `pattern`.
840 ///
841 /// Can be called multiple times; overrides are applied in order.
842 #[must_use]
843 pub fn with_tensor_type_override(mut self, ov: TensorTypeOverride) -> Self {
844 self.tt_overrides.push(ov);
845 self
846 }
847
848 /// Mark a layer index for pruning (removal) from the output model.
849 #[must_use]
850 pub fn with_pruned_layer(mut self, layer: i32) -> Self {
851 self.prune_layers.push(layer);
852 self
853 }
854
855 /// Mark multiple layer indices for pruning.
856 #[must_use]
857 pub fn with_pruned_layers(mut self, layers: impl IntoIterator<Item = i32>) -> Self {
858 self.prune_layers.extend(layers);
859 self
860 }
861
862 /// Build the raw C struct, together with the temporary backing storage
863 /// that must outlive the struct. Returns `(raw_params, _guards)`.
864 ///
865 /// This is `pub(crate)` so that `model_quantize` can call it safely while
866 /// holding all the guards alive.
867 #[allow(clippy::too_many_lines)]
868 pub(crate) fn to_raw(&self) -> RawQuantizeParamsGuard<'_> {
869 // ── imatrix ─────────────────────────────────────────────────────────
870 // Build a null-terminated array of llama_model_imatrix_data.
871 // The `name` and `data` pointers point directly into our owned Vecs.
872 let imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data> = self
873 .imatrix
874 .iter()
875 .map(|e| llama_cpp_sys_4::llama_model_imatrix_data {
876 name: e.name.as_ptr(),
877 data: e.data.as_ptr(),
878 size: e.data.len(),
879 })
880 .chain(std::iter::once(llama_cpp_sys_4::llama_model_imatrix_data {
881 name: null(),
882 data: null(),
883 size: 0,
884 }))
885 .collect();
886
887 // ── kv_overrides ────────────────────────────────────────────────────
888 // null-terminated by a sentinel with key[0] == 0
889 let kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override> = self
890 .kv_overrides
891 .iter()
892 .map(|kv| {
893 let mut raw = llama_cpp_sys_4::llama_model_kv_override {
894 key: [0; 128],
895 tag: 0,
896 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
897 val_i64: 0,
898 },
899 };
900 // Copy key bytes (up to 127 chars + NUL).
901 let bytes = kv.key.to_bytes_with_nul();
902 let copy_len = bytes.len().min(128);
903 for (dst, &src) in raw.key.iter_mut().zip(bytes[..copy_len].iter()) {
904 // `c_char` is `i8` on x86_64 but `u8` on ARM64/Android; `as _`
905 // infers the target signedness on every platform (issue #306).
906 // The wrap on `i8` targets is intentional: llama.cpp reads the
907 // key back as bytes, so the bit pattern is what matters.
908 #[allow(clippy::cast_possible_wrap)]
909 {
910 *dst = src as _;
911 }
912 }
913 match &kv.value {
914 KvOverrideValue::Int(v) => {
915 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_INT;
916 raw.__bindgen_anon_1 =
917 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_i64: *v };
918 }
919 KvOverrideValue::Float(v) => {
920 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_FLOAT;
921 raw.__bindgen_anon_1 =
922 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_f64: *v };
923 }
924 KvOverrideValue::Bool(v) => {
925 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_BOOL;
926 raw.__bindgen_anon_1 =
927 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_bool: *v };
928 }
929 KvOverrideValue::Str(s) => {
930 raw.tag = llama_cpp_sys_4::LLAMA_KV_OVERRIDE_TYPE_STR;
931 raw.__bindgen_anon_1 =
932 llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 { val_str: *s };
933 }
934 }
935 raw
936 })
937 .chain(std::iter::once(llama_cpp_sys_4::llama_model_kv_override {
938 key: [0; 128],
939 tag: 0,
940 __bindgen_anon_1: llama_cpp_sys_4::llama_model_kv_override__bindgen_ty_1 {
941 val_i64: 0,
942 },
943 }))
944 .collect();
945
946 // ── tt_overrides ────────────────────────────────────────────────────
947 // null-terminated by { null, GGML_TYPE_COUNT }
948 let tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override> = self
949 .tt_overrides
950 .iter()
951 .map(|ov| llama_cpp_sys_4::llama_model_tensor_override {
952 pattern: ov.pattern.as_ptr(),
953 type_: ov.ty as llama_cpp_sys_4::ggml_type,
954 })
955 .chain(std::iter::once(
956 llama_cpp_sys_4::llama_model_tensor_override {
957 pattern: null(),
958 type_: llama_cpp_sys_4::GGML_TYPE_COUNT,
959 },
960 ))
961 .collect();
962
963 // ── prune_layers ─────────────────────────────────────────────────────
964 // -1-terminated
965 let mut prune_c = self.prune_layers.clone();
966 prune_c.push(-1);
967
968 // ── assemble ────────────────────────────────────────────────────────
969 let raw = llama_cpp_sys_4::llama_model_quantize_params {
970 nthread: self.nthread,
971 ftype: self.ftype as llama_cpp_sys_4::llama_ftype,
972 output_tensor_type: self
973 .output_tensor_type
974 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
975 t as llama_cpp_sys_4::ggml_type
976 }),
977 token_embedding_type: self
978 .token_embedding_type
979 .map_or(llama_cpp_sys_4::GGML_TYPE_COUNT, |t| {
980 t as llama_cpp_sys_4::ggml_type
981 }),
982 allow_requantize: self.allow_requantize,
983 quantize_output_tensor: self.quantize_output_tensor,
984 only_copy: self.only_copy,
985 pure_: self.pure,
986 keep_split: self.keep_split,
987 dry_run: self.dry_run,
988 max_buf_size: self.max_buf_size,
989 imatrix: if self.imatrix.is_empty() {
990 null()
991 } else {
992 imatrix_c.as_ptr()
993 },
994 kv_overrides: if self.kv_overrides.is_empty() {
995 null()
996 } else {
997 kv_c.as_ptr()
998 },
999 tt_overrides: if self.tt_overrides.is_empty() {
1000 null()
1001 } else {
1002 tt_c.as_ptr()
1003 },
1004 prune_layers: if self.prune_layers.is_empty() {
1005 null()
1006 } else {
1007 prune_c.as_ptr()
1008 },
1009 };
1010
1011 RawQuantizeParamsGuard {
1012 raw,
1013 _imatrix_c: imatrix_c,
1014 _kv_c: kv_c,
1015 _tt_c: tt_c,
1016 _prune_c: prune_c,
1017 _marker: std::marker::PhantomData,
1018 }
1019 }
1020}
1021
1022/// Temporary storage that keeps the C pointers inside a raw
1023/// `llama_model_quantize_params` valid. Dropped after the quantize call.
1024pub(crate) struct RawQuantizeParamsGuard<'a> {
1025 pub(crate) raw: llama_cpp_sys_4::llama_model_quantize_params,
1026 _imatrix_c: Vec<llama_cpp_sys_4::llama_model_imatrix_data>,
1027 _kv_c: Vec<llama_cpp_sys_4::llama_model_kv_override>,
1028 _tt_c: Vec<llama_cpp_sys_4::llama_model_tensor_override>,
1029 _prune_c: Vec<i32>,
1030 // tie lifetime to the source QuantizeParams so the string/data
1031 // pointers inside imatrix_c and tt_c stay valid
1032 _marker: std::marker::PhantomData<&'a QuantizeParams>,
1033}
1034
1035// ─────────────────────────────────────────────────────────────────────────────
1036// TurboQuant – attention rotation
1037// ─────────────────────────────────────────────────────────────────────────────
1038
1039/// Control the `TurboQuant` attention-rotation feature globally.
1040///
1041/// When enabled (the default), llama.cpp applies a Hadamard rotation to Q/K/V
1042/// tensors before storing them in the KV cache. This significantly improves
1043/// quantization quality of the KV cache with near-zero overhead, as described
1044/// in llama.cpp PR #21038.
1045///
1046/// This function sets or clears the `LLAMA_ATTN_ROT_DISABLE` environment
1047/// variable, which llama.cpp reads once when a context (and its KV cache) is
1048/// first created. Call it **before** creating any [`LlamaContext`] on the
1049/// current process.
1050///
1051/// # Thread safety
1052///
1053/// Mutating environment variables while other threads may be reading them is
1054/// undefined behaviour. Call this function before spawning any threads that
1055/// use llama contexts, or ensure no contexts are being created concurrently.
1056///
1057/// # Example
1058///
1059/// ```no_run
1060/// // Disable the rotation for benchmarking purposes:
1061/// llama_cpp_4::quantize::set_attn_rot_disabled(true);
1062///
1063/// // Re-enable (default behaviour):
1064/// llama_cpp_4::quantize::set_attn_rot_disabled(false);
1065/// ```
1066///
1067/// [`LlamaContext`]: crate::context::LlamaContext
1068pub fn set_attn_rot_disabled(disabled: bool) {
1069 if disabled {
1070 // SAFETY: single-threaded context required by the caller.
1071 #[allow(unused_unsafe)]
1072 unsafe {
1073 std::env::set_var("LLAMA_ATTN_ROT_DISABLE", "1");
1074 }
1075 } else {
1076 #[allow(unused_unsafe)]
1077 unsafe {
1078 std::env::remove_var("LLAMA_ATTN_ROT_DISABLE");
1079 }
1080 }
1081}
1082
1083/// Returns `true` if `TurboQuant` attention rotation is currently disabled.
1084#[must_use]
1085pub fn attn_rot_disabled() -> bool {
1086 std::env::var("LLAMA_ATTN_ROT_DISABLE")
1087 .ok()
1088 .and_then(|v| v.parse::<i32>().ok())
1089 .is_some_and(|v| v != 0)
1090}
1091
1092// ─────────────────────────────────────────────────────────────────────────────
1093// Quantization preview
1094// ─────────────────────────────────────────────────────────────────────────────
1095
1096/// Ask llama.cpp what it *would* do, without writing a file.
1097///
1098/// [`crate::model_quantize`] is all-or-nothing: it reads a model, quantizes
1099/// every tensor, and writes the result. This exposes the decision layer
1100/// underneath — which tensors get quantized at all, and to which `ggml` type —
1101/// so a tool can show the plan, or estimate output size, before committing to
1102/// a run that may take minutes and tens of gigabytes.
1103///
1104/// The k-quant mixes are why this is not simply [`LlamaFtype::default_ggml_type`]:
1105/// they deliberately keep attention and output tensors at higher precision, so
1106/// the per-tensor answer differs from the ftype's nominal type.
1107///
1108/// Wraps `llama_quant_init` / `llama_quant_free`.
1109#[derive(Debug)]
1110pub struct QuantPreview<'model> {
1111 qs: std::ptr::NonNull<llama_cpp_sys_4::quantize_state_impl>,
1112 // The state borrows the model it was built from.
1113 _model: std::marker::PhantomData<&'model crate::model::LlamaModel>,
1114}
1115
1116impl Drop for QuantPreview<'_> {
1117 fn drop(&mut self) {
1118 unsafe { llama_cpp_sys_4::llama_quant_free(self.qs.as_ptr()) }
1119 }
1120}
1121
1122impl<'model> QuantPreview<'model> {
1123 /// Build a preview for `model` under `params`.
1124 ///
1125 /// # Errors
1126 ///
1127 /// Returns [`QuantPreviewError::Init`] if llama.cpp could not build the
1128 /// quantization state, which happens for a model it cannot quantize.
1129 pub fn new(
1130 model: &'model crate::model::LlamaModel,
1131 params: &QuantizeParams,
1132 ) -> Result<Self, QuantPreviewError> {
1133 let guard = params.to_raw();
1134 // The guarded wrapper: `llama_quant_init` throws for a model it cannot
1135 // quantize, and that unwinding into Rust aborts the process.
1136 let qs = unsafe {
1137 llama_cpp_sys_4::llama_quant_init_guarded(model.model.as_ptr(), &raw const guard.raw)
1138 };
1139 std::ptr::NonNull::new(qs)
1140 .map(|qs| Self {
1141 qs,
1142 _model: std::marker::PhantomData,
1143 })
1144 .ok_or(QuantPreviewError::Init)
1145 }
1146
1147 /// Whether this tensor would be quantized at all.
1148 ///
1149 /// llama.cpp skips 1-D tensors, tensors below a size threshold, and ones
1150 /// whose name marks them as needing full precision — so a `false` here is
1151 /// the usual reason a tensor keeps its original type.
1152 ///
1153 /// Requires the `ggml` feature, which is what exposes [`crate::ggml::GgmlTensor`].
1154 #[cfg(feature = "ggml")]
1155 #[must_use]
1156 pub fn allows_quantization(&self, tensor: &crate::ggml::GgmlTensor) -> bool {
1157 // 1 = yes, 0 = no, -1 = the underlying call threw and was caught.
1158 let rc = unsafe {
1159 llama_cpp_sys_4::llama_quant_tensor_allows_quantization_guarded(
1160 self.qs.as_ptr(),
1161 tensor.as_ptr(),
1162 )
1163 };
1164 rc == 1
1165 }
1166
1167 /// Compute the storage type each tensor would be assigned under `ftype`.
1168 ///
1169 /// Every tensor passed must already satisfy [`Self::allows_quantization`] —
1170 /// upstream states the caller filters first, and does not re-check.
1171 ///
1172 /// An entry is `None` when llama.cpp picks a `ggml` type this crate's
1173 /// [`GgmlType`] does not know.
1174 ///
1175 /// # Errors
1176 ///
1177 /// Returns [`QuantPreviewError::NotQuantizable`] naming the first tensor
1178 /// that fails the filter, rather than letting llama.cpp decide what to do
1179 /// with it.
1180 ///
1181 /// Requires the `ggml` feature.
1182 #[cfg(feature = "ggml")]
1183 pub fn compute_types(
1184 &self,
1185 tensors: &[&crate::ggml::GgmlTensor],
1186 ftype: LlamaFtype,
1187 ) -> Result<Vec<Option<GgmlType>>, QuantPreviewError> {
1188 if tensors.is_empty() {
1189 return Ok(Vec::new());
1190 }
1191 for tensor in tensors {
1192 if !self.allows_quantization(tensor) {
1193 return Err(QuantPreviewError::NotQuantizable(tensor.name().to_owned()));
1194 }
1195 }
1196
1197 let mut raw_tensors: Vec<*mut llama_cpp_sys_4::ggml_tensor> =
1198 tensors.iter().map(|t| t.as_ptr()).collect();
1199 let mut out = vec![0 as llama_cpp_sys_4::ggml_type; tensors.len()];
1200 let rc = unsafe {
1201 llama_cpp_sys_4::llama_quant_compute_types_guarded(
1202 self.qs.as_ptr(),
1203 ftype.into(),
1204 raw_tensors.as_mut_ptr(),
1205 out.as_mut_ptr(),
1206 out.len(),
1207 )
1208 };
1209 if rc != 0 {
1210 return Err(QuantPreviewError::Init);
1211 }
1212 Ok(out.into_iter().map(|t| GgmlType::try_from(t).ok()).collect())
1213 }
1214}
1215
1216/// Failure building or using a [`QuantPreview`].
1217#[derive(Debug, Eq, PartialEq, thiserror::Error)]
1218pub enum QuantPreviewError {
1219 /// llama.cpp could not build a quantization state for this model.
1220 #[error("could not initialize quantization state")]
1221 Init,
1222 /// A tensor that would not be quantized was passed to
1223 /// `QuantPreview::compute_types` (which requires the `ggml` feature).
1224 #[error("tensor {0:?} is not quantizable; filter with allows_quantization first")]
1225 NotQuantizable(String),
1226 /// The mock-model descriptor contained an interior NUL byte.
1227 #[error("architecture name contained an interior NUL byte")]
1228 Nul(#[from] NulError),
1229 /// llama.cpp could not build a model from the descriptor.
1230 #[error("could not build a mock model from the descriptor")]
1231 MockModel,
1232}
1233
1234/// Shape of a synthetic model, for previewing a quantization plan without a
1235/// real checkpoint on disk.
1236///
1237/// Upstream marks `llama_quant_model_from_metadata` as being for testing; it is
1238/// exposed here for the same reason — it lets you ask "what would a `Q4_K_M` of a
1239/// model this shape look like?" without downloading one.
1240#[derive(Debug, Clone, PartialEq, Eq)]
1241pub struct QuantModelDesc {
1242 /// Architecture name, e.g. `"llama"`.
1243 pub architecture: String,
1244 /// Embedding dimension.
1245 pub n_embd: u32,
1246 /// Feed-forward dimension.
1247 pub n_ff: u32,
1248 /// Number of layers.
1249 pub n_layer: u32,
1250 /// Number of attention heads.
1251 pub n_head: u32,
1252 /// Number of key/value heads.
1253 pub n_head_kv: u32,
1254 /// Number of experts (0 for a dense model).
1255 pub n_expert: u32,
1256 /// Per-head key dimension.
1257 pub n_embd_head_k: u32,
1258 /// Per-head value dimension.
1259 pub n_embd_head_v: u32,
1260}
1261
1262impl QuantModelDesc {
1263 /// A dense llama-shaped descriptor with the given dimensions.
1264 #[must_use]
1265 pub fn llama(n_embd: u32, n_ff: u32, n_layer: u32, n_head: u32) -> Self {
1266 Self {
1267 architecture: "llama".to_owned(),
1268 n_embd,
1269 n_ff,
1270 n_layer,
1271 n_head,
1272 n_head_kv: n_head,
1273 n_expert: 0,
1274 n_embd_head_k: n_embd / n_head.max(1),
1275 n_embd_head_v: n_embd / n_head.max(1),
1276 }
1277 }
1278
1279 /// Build a synthetic model from this descriptor.
1280 ///
1281 /// The returned model owns llama.cpp memory and frees it on drop, exactly
1282 /// like a loaded one — it simply has no weights.
1283 ///
1284 /// # Errors
1285 ///
1286 /// Returns [`QuantPreviewError::MockModel`] if llama.cpp rejects the
1287 /// descriptor, or [`QuantPreviewError::Nul`] for a bad architecture name.
1288 pub fn build(&self) -> Result<crate::model::LlamaModel, QuantPreviewError> {
1289 let arch = CString::new(self.architecture.as_str())?;
1290 let desc = llama_cpp_sys_4::llama_quant_model_desc {
1291 architecture: arch.as_ptr(),
1292 n_embd: self.n_embd,
1293 n_ff: self.n_ff,
1294 n_layer: self.n_layer,
1295 n_head: self.n_head,
1296 n_head_kv: self.n_head_kv,
1297 n_expert: self.n_expert,
1298 n_embd_head_k: self.n_embd_head_k,
1299 n_embd_head_v: self.n_embd_head_v,
1300 };
1301 // Guarded: an architecture llama.cpp does not know makes the raw entry
1302 // point throw, which aborts the process on the way back into Rust.
1303 let raw =
1304 unsafe { llama_cpp_sys_4::llama_quant_model_from_metadata_guarded(&raw const desc) };
1305 let ptr = std::ptr::NonNull::new(raw).ok_or(QuantPreviewError::MockModel)?;
1306 // SAFETY: upstream documents the result as owned by the caller and
1307 // freed with `llama_model_free`, which is what `LlamaModel` does.
1308 Ok(unsafe { crate::model::LlamaModel::from_raw(ptr) })
1309 }
1310}