llama_cpp_4/context/params/advanced.rs
1use super::{LlamaAttentionType, LlamaContextParams, LlamaFlashAttnType};
2use crate::sampling::LlamaSampler;
3
4impl LlamaContextParams {
5 /// Set the flash-attention mode (`Auto`, `Enabled`, or `Disabled`).
6 ///
7 /// Maps to `llama_context_params.flash_attn_type`. Use
8 /// [`LlamaFlashAttnType::Auto`] to match llama.cpp defaults.
9 ///
10 /// # Examples
11 ///
12 /// ```rust
13 /// use llama_cpp_4::context::params::{LlamaContextParams, LlamaFlashAttnType};
14 /// let params = LlamaContextParams::default()
15 /// .with_flash_attn_type(LlamaFlashAttnType::Auto);
16 /// assert_eq!(params.flash_attn_type(), LlamaFlashAttnType::Auto);
17 /// ```
18 #[must_use]
19 pub fn with_flash_attn_type(mut self, flash_attn_type: LlamaFlashAttnType) -> Self {
20 self.context_params.flash_attn_type = flash_attn_type.into();
21 self
22 }
23
24 /// Get the configured flash-attention mode.
25 #[must_use]
26 pub fn flash_attn_type(&self) -> LlamaFlashAttnType {
27 LlamaFlashAttnType::from(self.context_params.flash_attn_type)
28 }
29
30 /// Set the attention type used when extracting embeddings.
31 ///
32 /// Maps to `llama_context_params.attention_type`. Embedding models often
33 /// need [`LlamaAttentionType::NonCausal`]; generative decoding uses
34 /// [`LlamaAttentionType::Causal`].
35 ///
36 /// # Examples
37 ///
38 /// ```rust
39 /// use llama_cpp_4::context::params::{LlamaAttentionType, LlamaContextParams};
40 /// let params = LlamaContextParams::default()
41 /// .with_attention_type(LlamaAttentionType::Causal);
42 /// assert_eq!(params.attention_type(), LlamaAttentionType::Causal);
43 /// ```
44 #[must_use]
45 pub fn with_attention_type(mut self, attention_type: LlamaAttentionType) -> Self {
46 self.context_params.attention_type = attention_type.into();
47 self
48 }
49
50 /// Get the attention type used when extracting embeddings.
51 #[must_use]
52 pub fn attention_type(&self) -> LlamaAttentionType {
53 LlamaAttentionType::from(self.context_params.attention_type)
54 }
55
56 /// Set the maximum number of outputs per micro-batch.
57 ///
58 /// Maps to `llama_context_params.n_outputs_max`. When `0`, llama.cpp uses
59 /// `n_batch` as the cap.
60 ///
61 /// # Examples
62 ///
63 /// ```rust
64 /// use llama_cpp_4::context::params::LlamaContextParams;
65 /// let params = LlamaContextParams::default().with_n_outputs_max(256);
66 /// assert_eq!(params.n_outputs_max(), 256);
67 /// ```
68 #[must_use]
69 pub fn with_n_outputs_max(mut self, n_outputs_max: u32) -> Self {
70 self.context_params.n_outputs_max = n_outputs_max;
71 self
72 }
73
74 /// Get the maximum number of outputs per micro-batch.
75 #[must_use]
76 pub fn n_outputs_max(&self) -> u32 {
77 self.context_params.n_outputs_max
78 }
79
80 /// Set the maximum number of outputs per sequence.
81 ///
82 /// Maps to `llama_context_params.n_outputs_max_per_seq`. **Defaults to `1`**
83 /// — a single output per sequence; pass `0` to fall back to
84 /// [`Self::n_outputs_max`] instead. Backend samplers are initialized for
85 /// this many outputs per sequence, so multi-output backend sampling (e.g.
86 /// speculative decoding) must raise it above `1`. llama.cpp clamps the
87 /// value to `n_outputs_max`.
88 ///
89 /// # Examples
90 ///
91 /// ```rust
92 /// use llama_cpp_4::context::params::LlamaContextParams;
93 /// let params = LlamaContextParams::default().with_n_outputs_max_per_seq(16);
94 /// assert_eq!(params.n_outputs_max_per_seq(), 16);
95 /// ```
96 #[must_use]
97 pub fn with_n_outputs_max_per_seq(mut self, n_outputs_max_per_seq: u32) -> Self {
98 self.context_params.n_outputs_max_per_seq = n_outputs_max_per_seq;
99 self
100 }
101
102 /// Get the maximum number of outputs per sequence.
103 #[must_use]
104 pub fn n_outputs_max_per_seq(&self) -> u32 {
105 self.context_params.n_outputs_max_per_seq
106 }
107
108 /// Set the device-side cache for `MoE` experts that live in host memory, in
109 /// bytes. `0` (the default) disables it.
110 ///
111 /// Maps to `llama_context_params.moe_cache_size`
112 /// ([llama.cpp #29887](https://github.com/ggml-org/llama.cpp/pull/29887),
113 /// spread across devices in
114 /// [#30112](https://github.com/ggml-org/llama.cpp/pull/30112)). This is for
115 /// the case where a mixture-of-experts model does not fit in VRAM and some
116 /// expert tensors stay in RAM: instead of streaming every expert across the
117 /// bus on every token, llama.cpp keeps the hottest ones in a GPU cache of
118 /// this size. The budget is split across devices the same way layers are,
119 /// so it is a total and not a per-device figure.
120 ///
121 /// # A non-zero budget is a requirement, not a hint
122 ///
123 /// llama.cpp refuses to build a context it cannot honour the budget on, so
124 /// [`new_context`](crate::model::LlamaModel::new_context) then fails with
125 /// [`LlamaContextLoadError::NullReturn`](crate::LlamaContextLoadError::NullReturn)
126 /// and the reason in the llama.cpp log. All of these must hold:
127 ///
128 /// - a GPU or integrated-GPU backend is present — a CPU-only build, or no
129 /// offloaded layers, has nowhere to put the cache;
130 /// - the model is a mixture of experts;
131 /// - the split mode is not tensor/row split;
132 /// - the budget is big enough for one token's experts in at least one layer
133 /// whose experts are all host-resident.
134 ///
135 /// So leave it at `0` unless you know the model and device layout call for
136 /// it; it is not safe to set speculatively.
137 ///
138 /// # Examples
139 ///
140 /// ```rust
141 /// use llama_cpp_4::context::params::LlamaContextParams;
142 /// let params = LlamaContextParams::default()
143 /// .with_moe_cache_size(2 * 1024 * 1024 * 1024);
144 /// assert_eq!(params.moe_cache_size(), 2 * 1024 * 1024 * 1024);
145 /// ```
146 #[must_use]
147 pub fn with_moe_cache_size(mut self, bytes: usize) -> Self {
148 self.context_params.moe_cache_size = bytes;
149 self
150 }
151
152 /// Get the device-side `MoE` expert cache size in bytes. `0` means disabled.
153 #[must_use]
154 pub fn moe_cache_size(&self) -> usize {
155 self.context_params.moe_cache_size
156 }
157
158 /// Use a unified KV buffer across input sequences.
159 ///
160 /// Maps to `llama_context_params.kv_unified`. Disabling can improve
161 /// throughput for batched decoding when sequences do not share a long prefix.
162 ///
163 /// # Examples
164 ///
165 /// ```rust
166 /// use llama_cpp_4::context::params::LlamaContextParams;
167 /// let params = LlamaContextParams::default().with_kv_unified(false);
168 /// assert!(!params.kv_unified());
169 /// ```
170 #[must_use]
171 pub fn with_kv_unified(mut self, kv_unified: bool) -> Self {
172 self.context_params.kv_unified = kv_unified;
173 self
174 }
175
176 /// Returns `true` when a unified KV buffer is enabled.
177 #[must_use]
178 pub fn kv_unified(&self) -> bool {
179 self.context_params.kv_unified
180 }
181
182 /// Use a full-size sliding-window-attention (SWA) KV cache.
183 ///
184 /// Maps to `llama_context_params.swa_full`. When `false` and `n_seq_max > 1`,
185 /// llama.cpp may use a smaller per-sequence SWA window for better performance.
186 ///
187 /// # Examples
188 ///
189 /// ```rust
190 /// use llama_cpp_4::context::params::LlamaContextParams;
191 /// let params = LlamaContextParams::default().with_swa_full(true);
192 /// assert!(params.swa_full());
193 /// ```
194 #[must_use]
195 pub fn with_swa_full(mut self, swa_full: bool) -> Self {
196 self.context_params.swa_full = swa_full;
197 self
198 }
199
200 /// Returns `true` when full SWA cache is enabled.
201 #[must_use]
202 pub fn swa_full(&self) -> bool {
203 self.context_params.swa_full
204 }
205
206 /// Offload eligible host tensor operations to the active device.
207 ///
208 /// Maps to `llama_context_params.op_offload`.
209 ///
210 /// # Examples
211 ///
212 /// ```rust
213 /// use llama_cpp_4::context::params::LlamaContextParams;
214 /// let params = LlamaContextParams::default().with_op_offload(true);
215 /// assert!(params.op_offload());
216 /// ```
217 #[must_use]
218 pub fn with_op_offload(mut self, op_offload: bool) -> Self {
219 self.context_params.op_offload = op_offload;
220 self
221 }
222
223 /// Returns `true` when host tensor ops are offloaded to device.
224 #[must_use]
225 pub fn op_offload(&self) -> bool {
226 self.context_params.op_offload
227 }
228
229 /// Pair this context with another for shared memory or cross-context results.
230 ///
231 /// Maps to `llama_context_params.ctx_other`. The paired context is returned
232 /// by [`crate::context::LlamaContext::ctx_other`] after creation.
233 ///
234 /// `other` must remain alive until [`crate::model::LlamaModel::new_context`]
235 /// returns.
236 ///
237 /// # Examples
238 ///
239 /// ```no_run
240 /// # use llama_cpp_4::context::params::LlamaContextParams;
241 /// # use llama_cpp_4::llama_backend::LlamaBackend;
242 /// # use llama_cpp_4::model::LlamaModel;
243 /// # fn f(backend: &LlamaBackend, model: &LlamaModel)
244 /// # -> Result<(), Box<dyn std::error::Error>> {
245 /// let target = model.new_context(&backend, LlamaContextParams::default())?;
246 /// let draft = model.new_context(
247 /// &backend,
248 /// LlamaContextParams::default().with_ctx_other(&target),
249 /// )?;
250 /// # Ok(())
251 /// # }
252 /// ```
253 #[must_use]
254 pub fn with_ctx_other(mut self, other: &crate::context::LlamaContext<'_>) -> Self {
255 self.context_params.ctx_other = other.context.as_ptr();
256 self
257 }
258
259 /// Set `YaRN` extrapolation mix factor.
260 ///
261 /// Maps to `llama_context_params.yarn_ext_factor`. Negative values use the
262 /// model default. Only meaningful when [`super::RopeScalingType::Yarn`] is active.
263 ///
264 /// # Examples
265 ///
266 /// ```rust
267 /// use llama_cpp_4::context::params::LlamaContextParams;
268 /// let params = LlamaContextParams::default().with_yarn_ext_factor(1.0);
269 /// assert_eq!(params.yarn_ext_factor(), 1.0);
270 /// ```
271 #[must_use]
272 pub fn with_yarn_ext_factor(mut self, yarn_ext_factor: f32) -> Self {
273 self.context_params.yarn_ext_factor = yarn_ext_factor;
274 self
275 }
276
277 /// Get `YaRN` extrapolation mix factor (`yarn_ext_factor`).
278 #[must_use]
279 pub fn yarn_ext_factor(&self) -> f32 {
280 self.context_params.yarn_ext_factor
281 }
282
283 /// Set `YaRN` magnitude scaling factor.
284 ///
285 /// Maps to `llama_context_params.yarn_attn_factor`.
286 ///
287 /// # Examples
288 ///
289 /// ```rust
290 /// use llama_cpp_4::context::params::LlamaContextParams;
291 /// let params = LlamaContextParams::default().with_yarn_attn_factor(1.0);
292 /// assert_eq!(params.yarn_attn_factor(), 1.0);
293 /// ```
294 #[must_use]
295 pub fn with_yarn_attn_factor(mut self, yarn_attn_factor: f32) -> Self {
296 self.context_params.yarn_attn_factor = yarn_attn_factor;
297 self
298 }
299
300 /// Get `YaRN` magnitude scaling factor (`yarn_attn_factor`).
301 #[must_use]
302 pub fn yarn_attn_factor(&self) -> f32 {
303 self.context_params.yarn_attn_factor
304 }
305
306 /// Set `YaRN` low correction dimension (`yarn_beta_fast`).
307 ///
308 /// Maps to `llama_context_params.yarn_beta_fast`.
309 #[must_use]
310 pub fn with_yarn_beta_fast(mut self, yarn_beta_fast: f32) -> Self {
311 self.context_params.yarn_beta_fast = yarn_beta_fast;
312 self
313 }
314
315 /// Get `YaRN` low correction dimension.
316 #[must_use]
317 pub fn yarn_beta_fast(&self) -> f32 {
318 self.context_params.yarn_beta_fast
319 }
320
321 /// Set `YaRN` high correction dimension (`yarn_beta_slow`).
322 ///
323 /// Maps to `llama_context_params.yarn_beta_slow`.
324 #[must_use]
325 pub fn with_yarn_beta_slow(mut self, yarn_beta_slow: f32) -> Self {
326 self.context_params.yarn_beta_slow = yarn_beta_slow;
327 self
328 }
329
330 /// Get `YaRN` high correction dimension.
331 #[must_use]
332 pub fn yarn_beta_slow(&self) -> f32 {
333 self.context_params.yarn_beta_slow
334 }
335
336 /// Set `YaRN` original context size.
337 ///
338 /// Maps to `llama_context_params.yarn_orig_ctx`. `0` uses the model default.
339 ///
340 /// # Examples
341 ///
342 /// ```rust
343 /// use llama_cpp_4::context::params::LlamaContextParams;
344 /// let params = LlamaContextParams::default().with_yarn_orig_ctx(8192);
345 /// assert_eq!(params.yarn_orig_ctx(), 8192);
346 /// ```
347 #[must_use]
348 pub fn with_yarn_orig_ctx(mut self, yarn_orig_ctx: u32) -> Self {
349 self.context_params.yarn_orig_ctx = yarn_orig_ctx;
350 self
351 }
352
353 /// Get `YaRN` original context size (`yarn_orig_ctx`).
354 #[must_use]
355 pub fn yarn_orig_ctx(&self) -> u32 {
356 self.context_params.yarn_orig_ctx
357 }
358
359 /// Disable performance timing collection for this context.
360 ///
361 /// Maps to `llama_context_params.no_perf`. When `true`, calls such as
362 /// [`crate::context::LlamaContext::timings`] return empty counters.
363 ///
364 /// # Examples
365 ///
366 /// ```rust
367 /// use llama_cpp_4::context::params::LlamaContextParams;
368 /// let params = LlamaContextParams::default().with_no_perf(true);
369 /// assert!(params.no_perf());
370 /// ```
371 #[must_use]
372 pub fn with_no_perf(mut self, no_perf: bool) -> Self {
373 self.context_params.no_perf = no_perf;
374 self
375 }
376
377 /// Returns `true` when perf timings are disabled for this context.
378 #[must_use]
379 pub fn no_perf(&self) -> bool {
380 self.context_params.no_perf
381 }
382
383 /// Register an abort callback checked during `decode()` on CPU backends.
384 ///
385 /// Maps to `llama_context_params.abort_callback` / `abort_callback_data`.
386 /// The callback is invoked periodically during long decodes; return a
387 /// non-zero value to stop the current operation.
388 ///
389 /// `user_data` is passed through unchanged and must remain valid for the
390 /// lifetime of any context created from these params.
391 #[must_use]
392 pub fn with_abort_callback(
393 mut self,
394 callback: llama_cpp_sys_4::ggml_abort_callback,
395 user_data: *mut std::ffi::c_void,
396 ) -> Self {
397 self.context_params.abort_callback = callback;
398 self.context_params.abort_callback_data = user_data;
399 self
400 }
401
402 /// Assign per-sequence backend sampler chains.
403 ///
404 /// Maps to `llama_context_params.samplers` / `n_samplers`. Each
405 /// [`LlamaSampler`] must be a sampler **chain** created with
406 /// `llama_sampler_chain_init`. The samplers are kept alive inside these
407 /// params until [`crate::model::LlamaModel::new_context`] returns.
408 ///
409 /// Pair sequence ids with the chains that should run when decoding those
410 /// sequences on the backend.
411 ///
412 /// # Examples
413 ///
414 /// ```no_run
415 /// use llama_cpp_4::context::params::LlamaContextParams;
416 /// use llama_cpp_4::sampling::LlamaSampler;
417 /// # use llama_cpp_4::model::LlamaModel;
418 /// # fn f(model: &LlamaModel) -> Result<(), Box<dyn std::error::Error>> {
419 /// let chain = LlamaSampler::chain_simple([LlamaSampler::greedy()]);
420 /// let params = LlamaContextParams::default()
421 /// .with_sampler_seq_configs([(0, chain)]);
422 /// assert_eq!(params.n_sampler_seq_configs(), 1);
423 /// # Ok(())
424 /// # }
425 /// ```
426 #[must_use]
427 pub fn with_sampler_seq_configs(
428 mut self,
429 configs: impl IntoIterator<Item = (i32, LlamaSampler)>,
430 ) -> Self {
431 self.owned_samplers.clear();
432 self.sampler_configs.clear();
433
434 for (seq_id, sampler) in configs {
435 self.sampler_configs
436 .push(llama_cpp_sys_4::llama_sampler_seq_config {
437 seq_id,
438 sampler: sampler.sampler.as_ptr(),
439 });
440 self.owned_samplers.push(sampler);
441 }
442
443 if self.sampler_configs.is_empty() {
444 self.context_params.samplers = std::ptr::null_mut();
445 self.context_params.n_samplers = 0;
446 } else {
447 self.context_params.samplers = self.sampler_configs.as_mut_ptr();
448 self.context_params.n_samplers = self.sampler_configs.len();
449 }
450
451 self
452 }
453
454 /// Number of per-sequence sampler configs attached to these params.
455 ///
456 /// Returns `0` when no chains were set or after [`Clone`] (sampler chains
457 /// are not duplicated).
458 #[must_use]
459 pub fn n_sampler_seq_configs(&self) -> usize {
460 self.sampler_configs.len()
461 }
462}