llama_cpp_4/context/params/advanced.rs
1use super::{LlamaAttentionType, LlamaContextParams, LlamaFlashAttnType};
2use crate::sampling::LlamaSampler;
3
4impl LlamaContextParams {
5 /// Set the flash-attention mode (`Auto`, `Enabled`, or `Disabled`).
6 ///
7 /// Maps to `llama_context_params.flash_attn_type`. Use
8 /// [`LlamaFlashAttnType::Auto`] to match llama.cpp defaults.
9 ///
10 /// # Examples
11 ///
12 /// ```rust
13 /// use llama_cpp_4::context::params::{LlamaContextParams, LlamaFlashAttnType};
14 /// let params = LlamaContextParams::default()
15 /// .with_flash_attn_type(LlamaFlashAttnType::Auto);
16 /// assert_eq!(params.flash_attn_type(), LlamaFlashAttnType::Auto);
17 /// ```
18 #[must_use]
19 pub fn with_flash_attn_type(mut self, flash_attn_type: LlamaFlashAttnType) -> Self {
20 self.context_params.flash_attn_type = flash_attn_type.into();
21 self
22 }
23
24 /// Get the configured flash-attention mode.
25 #[must_use]
26 pub fn flash_attn_type(&self) -> LlamaFlashAttnType {
27 LlamaFlashAttnType::from(self.context_params.flash_attn_type)
28 }
29
30 /// Set the attention type used when extracting embeddings.
31 ///
32 /// Maps to `llama_context_params.attention_type`. Embedding models often
33 /// need [`LlamaAttentionType::NonCausal`]; generative decoding uses
34 /// [`LlamaAttentionType::Causal`].
35 ///
36 /// # Examples
37 ///
38 /// ```rust
39 /// use llama_cpp_4::context::params::{LlamaAttentionType, LlamaContextParams};
40 /// let params = LlamaContextParams::default()
41 /// .with_attention_type(LlamaAttentionType::Causal);
42 /// assert_eq!(params.attention_type(), LlamaAttentionType::Causal);
43 /// ```
44 #[must_use]
45 pub fn with_attention_type(mut self, attention_type: LlamaAttentionType) -> Self {
46 self.context_params.attention_type = attention_type.into();
47 self
48 }
49
50 /// Get the attention type used when extracting embeddings.
51 #[must_use]
52 pub fn attention_type(&self) -> LlamaAttentionType {
53 LlamaAttentionType::from(self.context_params.attention_type)
54 }
55
56 /// Set the maximum number of outputs per micro-batch.
57 ///
58 /// Maps to `llama_context_params.n_outputs_max`. When `0`, llama.cpp uses
59 /// `n_batch` as the cap.
60 ///
61 /// # Examples
62 ///
63 /// ```rust
64 /// use llama_cpp_4::context::params::LlamaContextParams;
65 /// let params = LlamaContextParams::default().with_n_outputs_max(256);
66 /// assert_eq!(params.n_outputs_max(), 256);
67 /// ```
68 #[must_use]
69 pub fn with_n_outputs_max(mut self, n_outputs_max: u32) -> Self {
70 self.context_params.n_outputs_max = n_outputs_max;
71 self
72 }
73
74 /// Get the maximum number of outputs per micro-batch.
75 #[must_use]
76 pub fn n_outputs_max(&self) -> u32 {
77 self.context_params.n_outputs_max
78 }
79
80 /// Set the maximum number of outputs per sequence.
81 ///
82 /// Maps to `llama_context_params.n_outputs_max_per_seq`. **Defaults to `1`**
83 /// — a single output per sequence; pass `0` to fall back to
84 /// [`Self::n_outputs_max`] instead. Backend samplers are initialized for
85 /// this many outputs per sequence, so multi-output backend sampling (e.g.
86 /// speculative decoding) must raise it above `1`. llama.cpp clamps the
87 /// value to `n_outputs_max`.
88 ///
89 /// # Examples
90 ///
91 /// ```rust
92 /// use llama_cpp_4::context::params::LlamaContextParams;
93 /// let params = LlamaContextParams::default().with_n_outputs_max_per_seq(16);
94 /// assert_eq!(params.n_outputs_max_per_seq(), 16);
95 /// ```
96 #[must_use]
97 pub fn with_n_outputs_max_per_seq(mut self, n_outputs_max_per_seq: u32) -> Self {
98 self.context_params.n_outputs_max_per_seq = n_outputs_max_per_seq;
99 self
100 }
101
102 /// Get the maximum number of outputs per sequence.
103 #[must_use]
104 pub fn n_outputs_max_per_seq(&self) -> u32 {
105 self.context_params.n_outputs_max_per_seq
106 }
107
108 /// Use a unified KV buffer across input sequences.
109 ///
110 /// Maps to `llama_context_params.kv_unified`. Disabling can improve
111 /// throughput for batched decoding when sequences do not share a long prefix.
112 ///
113 /// # Examples
114 ///
115 /// ```rust
116 /// use llama_cpp_4::context::params::LlamaContextParams;
117 /// let params = LlamaContextParams::default().with_kv_unified(false);
118 /// assert!(!params.kv_unified());
119 /// ```
120 #[must_use]
121 pub fn with_kv_unified(mut self, kv_unified: bool) -> Self {
122 self.context_params.kv_unified = kv_unified;
123 self
124 }
125
126 /// Returns `true` when a unified KV buffer is enabled.
127 #[must_use]
128 pub fn kv_unified(&self) -> bool {
129 self.context_params.kv_unified
130 }
131
132 /// Use a full-size sliding-window-attention (SWA) KV cache.
133 ///
134 /// Maps to `llama_context_params.swa_full`. When `false` and `n_seq_max > 1`,
135 /// llama.cpp may use a smaller per-sequence SWA window for better performance.
136 ///
137 /// # Examples
138 ///
139 /// ```rust
140 /// use llama_cpp_4::context::params::LlamaContextParams;
141 /// let params = LlamaContextParams::default().with_swa_full(true);
142 /// assert!(params.swa_full());
143 /// ```
144 #[must_use]
145 pub fn with_swa_full(mut self, swa_full: bool) -> Self {
146 self.context_params.swa_full = swa_full;
147 self
148 }
149
150 /// Returns `true` when full SWA cache is enabled.
151 #[must_use]
152 pub fn swa_full(&self) -> bool {
153 self.context_params.swa_full
154 }
155
156 /// Offload eligible host tensor operations to the active device.
157 ///
158 /// Maps to `llama_context_params.op_offload`.
159 ///
160 /// # Examples
161 ///
162 /// ```rust
163 /// use llama_cpp_4::context::params::LlamaContextParams;
164 /// let params = LlamaContextParams::default().with_op_offload(true);
165 /// assert!(params.op_offload());
166 /// ```
167 #[must_use]
168 pub fn with_op_offload(mut self, op_offload: bool) -> Self {
169 self.context_params.op_offload = op_offload;
170 self
171 }
172
173 /// Returns `true` when host tensor ops are offloaded to device.
174 #[must_use]
175 pub fn op_offload(&self) -> bool {
176 self.context_params.op_offload
177 }
178
179 /// Pair this context with another for shared memory or cross-context results.
180 ///
181 /// Maps to `llama_context_params.ctx_other`. The paired context is returned
182 /// by [`crate::context::LlamaContext::ctx_other`] after creation.
183 ///
184 /// `other` must remain alive until [`crate::model::LlamaModel::new_context`]
185 /// returns.
186 ///
187 /// # Examples
188 ///
189 /// ```ignore
190 /// let target = model.new_context(&backend, LlamaContextParams::default())?;
191 /// let draft = model.new_context(
192 /// &backend,
193 /// LlamaContextParams::default().with_ctx_other(&target),
194 /// )?;
195 /// ```
196 #[must_use]
197 pub fn with_ctx_other(mut self, other: &crate::context::LlamaContext<'_>) -> Self {
198 self.context_params.ctx_other = other.context.as_ptr();
199 self
200 }
201
202 /// Set `YaRN` extrapolation mix factor.
203 ///
204 /// Maps to `llama_context_params.yarn_ext_factor`. Negative values use the
205 /// model default. Only meaningful when [`super::RopeScalingType::Yarn`] is active.
206 ///
207 /// # Examples
208 ///
209 /// ```rust
210 /// use llama_cpp_4::context::params::LlamaContextParams;
211 /// let params = LlamaContextParams::default().with_yarn_ext_factor(1.0);
212 /// assert_eq!(params.yarn_ext_factor(), 1.0);
213 /// ```
214 #[must_use]
215 pub fn with_yarn_ext_factor(mut self, yarn_ext_factor: f32) -> Self {
216 self.context_params.yarn_ext_factor = yarn_ext_factor;
217 self
218 }
219
220 /// Get `YaRN` extrapolation mix factor (`yarn_ext_factor`).
221 #[must_use]
222 pub fn yarn_ext_factor(&self) -> f32 {
223 self.context_params.yarn_ext_factor
224 }
225
226 /// Set `YaRN` magnitude scaling factor.
227 ///
228 /// Maps to `llama_context_params.yarn_attn_factor`.
229 ///
230 /// # Examples
231 ///
232 /// ```rust
233 /// use llama_cpp_4::context::params::LlamaContextParams;
234 /// let params = LlamaContextParams::default().with_yarn_attn_factor(1.0);
235 /// assert_eq!(params.yarn_attn_factor(), 1.0);
236 /// ```
237 #[must_use]
238 pub fn with_yarn_attn_factor(mut self, yarn_attn_factor: f32) -> Self {
239 self.context_params.yarn_attn_factor = yarn_attn_factor;
240 self
241 }
242
243 /// Get `YaRN` magnitude scaling factor (`yarn_attn_factor`).
244 #[must_use]
245 pub fn yarn_attn_factor(&self) -> f32 {
246 self.context_params.yarn_attn_factor
247 }
248
249 /// Set `YaRN` low correction dimension (`yarn_beta_fast`).
250 ///
251 /// Maps to `llama_context_params.yarn_beta_fast`.
252 #[must_use]
253 pub fn with_yarn_beta_fast(mut self, yarn_beta_fast: f32) -> Self {
254 self.context_params.yarn_beta_fast = yarn_beta_fast;
255 self
256 }
257
258 /// Get `YaRN` low correction dimension.
259 #[must_use]
260 pub fn yarn_beta_fast(&self) -> f32 {
261 self.context_params.yarn_beta_fast
262 }
263
264 /// Set `YaRN` high correction dimension (`yarn_beta_slow`).
265 ///
266 /// Maps to `llama_context_params.yarn_beta_slow`.
267 #[must_use]
268 pub fn with_yarn_beta_slow(mut self, yarn_beta_slow: f32) -> Self {
269 self.context_params.yarn_beta_slow = yarn_beta_slow;
270 self
271 }
272
273 /// Get `YaRN` high correction dimension.
274 #[must_use]
275 pub fn yarn_beta_slow(&self) -> f32 {
276 self.context_params.yarn_beta_slow
277 }
278
279 /// Set `YaRN` original context size.
280 ///
281 /// Maps to `llama_context_params.yarn_orig_ctx`. `0` uses the model default.
282 ///
283 /// # Examples
284 ///
285 /// ```rust
286 /// use llama_cpp_4::context::params::LlamaContextParams;
287 /// let params = LlamaContextParams::default().with_yarn_orig_ctx(8192);
288 /// assert_eq!(params.yarn_orig_ctx(), 8192);
289 /// ```
290 #[must_use]
291 pub fn with_yarn_orig_ctx(mut self, yarn_orig_ctx: u32) -> Self {
292 self.context_params.yarn_orig_ctx = yarn_orig_ctx;
293 self
294 }
295
296 /// Get `YaRN` original context size (`yarn_orig_ctx`).
297 #[must_use]
298 pub fn yarn_orig_ctx(&self) -> u32 {
299 self.context_params.yarn_orig_ctx
300 }
301
302 /// Disable performance timing collection for this context.
303 ///
304 /// Maps to `llama_context_params.no_perf`. When `true`, calls such as
305 /// [`crate::context::LlamaContext::timings`] return empty counters.
306 ///
307 /// # Examples
308 ///
309 /// ```rust
310 /// use llama_cpp_4::context::params::LlamaContextParams;
311 /// let params = LlamaContextParams::default().with_no_perf(true);
312 /// assert!(params.no_perf());
313 /// ```
314 #[must_use]
315 pub fn with_no_perf(mut self, no_perf: bool) -> Self {
316 self.context_params.no_perf = no_perf;
317 self
318 }
319
320 /// Returns `true` when perf timings are disabled for this context.
321 #[must_use]
322 pub fn no_perf(&self) -> bool {
323 self.context_params.no_perf
324 }
325
326 /// Register an abort callback checked during `decode()` on CPU backends.
327 ///
328 /// Maps to `llama_context_params.abort_callback` / `abort_callback_data`.
329 /// The callback is invoked periodically during long decodes; return a
330 /// non-zero value to stop the current operation.
331 ///
332 /// `user_data` is passed through unchanged and must remain valid for the
333 /// lifetime of any context created from these params.
334 #[must_use]
335 pub fn with_abort_callback(
336 mut self,
337 callback: llama_cpp_sys_4::ggml_abort_callback,
338 user_data: *mut std::ffi::c_void,
339 ) -> Self {
340 self.context_params.abort_callback = callback;
341 self.context_params.abort_callback_data = user_data;
342 self
343 }
344
345 /// Assign per-sequence backend sampler chains.
346 ///
347 /// Maps to `llama_context_params.samplers` / `n_samplers`. Each
348 /// [`LlamaSampler`] must be a sampler **chain** created with
349 /// `llama_sampler_chain_init`. The samplers are kept alive inside these
350 /// params until [`crate::model::LlamaModel::new_context`] returns.
351 ///
352 /// Pair sequence ids with the chains that should run when decoding those
353 /// sequences on the backend.
354 ///
355 /// # Examples
356 ///
357 /// ```ignore
358 /// use llama_cpp_4::context::params::LlamaContextParams;
359 /// use llama_cpp_4::sampling::LlamaSampler;
360 ///
361 /// let chain = LlamaSampler::chain_default(&model)?;
362 /// let params = LlamaContextParams::default()
363 /// .with_sampler_seq_configs([(0, chain)]);
364 /// assert_eq!(params.n_sampler_seq_configs(), 1);
365 /// ```
366 #[must_use]
367 pub fn with_sampler_seq_configs(
368 mut self,
369 configs: impl IntoIterator<Item = (i32, LlamaSampler)>,
370 ) -> Self {
371 self.owned_samplers.clear();
372 self.sampler_configs.clear();
373
374 for (seq_id, sampler) in configs {
375 self.sampler_configs
376 .push(llama_cpp_sys_4::llama_sampler_seq_config {
377 seq_id,
378 sampler: sampler.sampler.as_ptr(),
379 });
380 self.owned_samplers.push(sampler);
381 }
382
383 if self.sampler_configs.is_empty() {
384 self.context_params.samplers = std::ptr::null_mut();
385 self.context_params.n_samplers = 0;
386 } else {
387 self.context_params.samplers = self.sampler_configs.as_mut_ptr();
388 self.context_params.n_samplers = self.sampler_configs.len();
389 }
390
391 self
392 }
393
394 /// Number of per-sequence sampler configs attached to these params.
395 ///
396 /// Returns `0` when no chains were set or after [`Clone`] (sampler chains
397 /// are not duplicated).
398 #[must_use]
399 pub fn n_sampler_seq_configs(&self) -> usize {
400 self.sampler_configs.len()
401 }
402}