llama_cpp_2/context/params/get_set.rs
1use std::num::NonZeroU32;
2
3use super::{
4 KvCacheType, LlamaAttentionType, LlamaContextParams, LlamaContextType, LlamaPoolingType,
5 RopeScalingType,
6};
7
8impl LlamaContextParams {
9 /// Set the size of the context
10 ///
11 /// # Examples
12 ///
13 /// ```rust
14 /// # use std::num::NonZeroU32;
15 /// # use llama_cpp_2::context::params::LlamaContextParams;
16 /// let params = LlamaContextParams::default();
17 /// let params = params.with_n_ctx(NonZeroU32::new(2048));
18 /// assert_eq!(params.n_ctx(), NonZeroU32::new(2048));
19 /// ```
20 #[must_use]
21 pub fn with_n_ctx(mut self, n_ctx: Option<NonZeroU32>) -> Self {
22 self.context_params.n_ctx = n_ctx.map_or(0, std::num::NonZeroU32::get);
23 self
24 }
25
26 /// Get the size of the context.
27 ///
28 /// [`None`] if the context size is specified by the model and not the context.
29 ///
30 /// # Examples
31 ///
32 /// ```rust
33 /// # use llama_cpp_2::context::params::LlamaContextParams;
34 /// let params = LlamaContextParams::default();
35 /// assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(512));
36 /// ```
37 #[must_use]
38 pub fn n_ctx(&self) -> Option<NonZeroU32> {
39 NonZeroU32::new(self.context_params.n_ctx)
40 }
41
42 /// Set the `n_batch`
43 ///
44 /// # Examples
45 ///
46 /// ```rust
47 /// # use llama_cpp_2::context::params::LlamaContextParams;
48 /// let params = LlamaContextParams::default()
49 /// .with_n_batch(2048);
50 /// assert_eq!(params.n_batch(), 2048);
51 /// ```
52 #[must_use]
53 pub fn with_n_batch(mut self, n_batch: u32) -> Self {
54 self.context_params.n_batch = n_batch;
55 self
56 }
57
58 /// Get the `n_batch`
59 ///
60 /// # Examples
61 ///
62 /// ```rust
63 /// # use llama_cpp_2::context::params::LlamaContextParams;
64 /// let params = LlamaContextParams::default();
65 /// assert_eq!(params.n_batch(), 2048);
66 /// ```
67 #[must_use]
68 pub fn n_batch(&self) -> u32 {
69 self.context_params.n_batch
70 }
71
72 /// Set the `n_ubatch`
73 ///
74 /// # Examples
75 ///
76 /// ```rust
77 /// # use llama_cpp_2::context::params::LlamaContextParams;
78 /// let params = LlamaContextParams::default()
79 /// .with_n_ubatch(512);
80 /// assert_eq!(params.n_ubatch(), 512);
81 /// ```
82 #[must_use]
83 pub fn with_n_ubatch(mut self, n_ubatch: u32) -> Self {
84 self.context_params.n_ubatch = n_ubatch;
85 self
86 }
87
88 /// Get the `n_ubatch`
89 ///
90 /// # Examples
91 ///
92 /// ```rust
93 /// # use llama_cpp_2::context::params::LlamaContextParams;
94 /// let params = LlamaContextParams::default();
95 /// assert_eq!(params.n_ubatch(), 512);
96 /// ```
97 #[must_use]
98 pub fn n_ubatch(&self) -> u32 {
99 self.context_params.n_ubatch
100 }
101
102 /// Set the max number of sequences (i.e. distinct states for recurrent models)
103 ///
104 /// # Examples
105 ///
106 /// ```rust
107 /// # use llama_cpp_2::context::params::LlamaContextParams;
108 /// let params = LlamaContextParams::default()
109 /// .with_n_seq_max(64);
110 /// assert_eq!(params.n_seq_max(), 64);
111 /// ```
112 #[must_use]
113 pub fn with_n_seq_max(mut self, n_seq_max: u32) -> Self {
114 self.context_params.n_seq_max = n_seq_max;
115 self
116 }
117
118 /// Get the max number of sequences (i.e. distinct states for recurrent models)
119 ///
120 /// # Examples
121 ///
122 /// ```rust
123 /// # use llama_cpp_2::context::params::LlamaContextParams;
124 /// let params = LlamaContextParams::default();
125 /// assert_eq!(params.n_seq_max(), 1);
126 /// ```
127 #[must_use]
128 pub fn n_seq_max(&self) -> u32 {
129 self.context_params.n_seq_max
130 }
131
132 /// Set the number of recurrent-state rollback snapshots per sequence.
133 ///
134 /// MTP speculative decoding uses this on the target context so llama.cpp can
135 /// roll recurrent state back after partially accepted drafts.
136 #[must_use]
137 pub fn with_n_rs_seq(mut self, n_rs_seq: u32) -> Self {
138 self.context_params.n_rs_seq = n_rs_seq;
139 self
140 }
141
142 /// Get the number of recurrent-state rollback snapshots per sequence.
143 #[must_use]
144 pub fn n_rs_seq(&self) -> u32 {
145 self.context_params.n_rs_seq
146 }
147
148 /// Set the llama.cpp context type.
149 #[must_use]
150 pub fn with_context_type(mut self, context_type: LlamaContextType) -> Self {
151 self.context_params.ctx_type = context_type.into();
152 self
153 }
154
155 /// Get the llama.cpp context type.
156 #[must_use]
157 pub fn context_type(&self) -> LlamaContextType {
158 self.context_params.ctx_type.into()
159 }
160
161 /// Set the number of threads
162 ///
163 /// # Examples
164 ///
165 /// ```rust
166 /// # use llama_cpp_2::context::params::LlamaContextParams;
167 /// let params = LlamaContextParams::default()
168 /// .with_n_threads(8);
169 /// assert_eq!(params.n_threads(), 8);
170 /// ```
171 #[must_use]
172 pub fn with_n_threads(mut self, n_threads: i32) -> Self {
173 self.context_params.n_threads = n_threads;
174 self
175 }
176
177 /// Get the number of threads
178 ///
179 /// # Examples
180 ///
181 /// ```rust
182 /// # use llama_cpp_2::context::params::LlamaContextParams;
183 /// let params = LlamaContextParams::default();
184 /// assert_eq!(params.n_threads(), 4);
185 /// ```
186 #[must_use]
187 pub fn n_threads(&self) -> i32 {
188 self.context_params.n_threads
189 }
190
191 /// Set the number of threads allocated for batches
192 ///
193 /// # Examples
194 ///
195 /// ```rust
196 /// # use llama_cpp_2::context::params::LlamaContextParams;
197 /// let params = LlamaContextParams::default()
198 /// .with_n_threads_batch(8);
199 /// assert_eq!(params.n_threads_batch(), 8);
200 /// ```
201 #[must_use]
202 pub fn with_n_threads_batch(mut self, n_threads: i32) -> Self {
203 self.context_params.n_threads_batch = n_threads;
204 self
205 }
206
207 /// Get the number of threads allocated for batches
208 ///
209 /// # Examples
210 ///
211 /// ```rust
212 /// # use llama_cpp_2::context::params::LlamaContextParams;
213 /// let params = LlamaContextParams::default();
214 /// assert_eq!(params.n_threads_batch(), 4);
215 /// ```
216 #[must_use]
217 pub fn n_threads_batch(&self) -> i32 {
218 self.context_params.n_threads_batch
219 }
220
221 /// Set the type of rope scaling
222 ///
223 /// # Examples
224 ///
225 /// ```rust
226 /// # use llama_cpp_2::context::params::{LlamaContextParams, RopeScalingType};
227 /// let params = LlamaContextParams::default()
228 /// .with_rope_scaling_type(RopeScalingType::Linear);
229 /// assert_eq!(params.rope_scaling_type(), RopeScalingType::Linear);
230 /// ```
231 #[must_use]
232 pub fn with_rope_scaling_type(mut self, rope_scaling_type: RopeScalingType) -> Self {
233 self.context_params.rope_scaling_type = i32::from(rope_scaling_type);
234 self
235 }
236
237 /// Get the type of rope scaling
238 ///
239 /// # Examples
240 ///
241 /// ```rust
242 /// # use llama_cpp_2::context::params::{LlamaContextParams, RopeScalingType};
243 /// let params = LlamaContextParams::default();
244 /// assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified);
245 /// ```
246 #[must_use]
247 pub fn rope_scaling_type(&self) -> RopeScalingType {
248 RopeScalingType::from(self.context_params.rope_scaling_type)
249 }
250
251 /// Set the type of pooling
252 ///
253 /// # Examples
254 ///
255 /// ```rust
256 /// # use llama_cpp_2::context::params::{LlamaContextParams, LlamaPoolingType};
257 /// let params = LlamaContextParams::default()
258 /// .with_pooling_type(LlamaPoolingType::Last);
259 /// assert_eq!(params.pooling_type(), LlamaPoolingType::Last);
260 /// ```
261 #[must_use]
262 pub fn with_pooling_type(mut self, pooling_type: LlamaPoolingType) -> Self {
263 self.context_params.pooling_type = i32::from(pooling_type);
264 self
265 }
266
267 /// Get the type of pooling
268 ///
269 /// # Examples
270 ///
271 /// ```rust
272 /// # use llama_cpp_2::context::params::{LlamaContextParams, LlamaPoolingType};
273 /// let params = LlamaContextParams::default();
274 /// assert_eq!(params.pooling_type(), LlamaPoolingType::Unspecified);
275 /// ```
276 #[must_use]
277 pub fn pooling_type(&self) -> LlamaPoolingType {
278 LlamaPoolingType::from(self.context_params.pooling_type)
279 }
280
281 /// Set the attention type for embeddings
282 ///
283 /// # Examples
284 ///
285 /// ```rust
286 /// # use llama_cpp_2::context::params::{LlamaContextParams, LlamaAttentionType};
287 /// let params = LlamaContextParams::default()
288 /// .with_attention_type(LlamaAttentionType::Causal);
289 /// assert_eq!(params.attention_type(), LlamaAttentionType::Causal);
290 /// ```
291 #[must_use]
292 pub fn with_attention_type(mut self, attention_type: LlamaAttentionType) -> Self {
293 self.context_params.attention_type = i32::from(attention_type);
294 self
295 }
296
297 /// Get the attention type for embeddings
298 ///
299 /// # Examples
300 ///
301 /// ```rust
302 /// # use llama_cpp_2::context::params::{LlamaContextParams, LlamaAttentionType};
303 /// let params = LlamaContextParams::default();
304 /// assert_eq!(params.attention_type(), LlamaAttentionType::Unspecified);
305 /// ```
306 #[must_use]
307 pub fn attention_type(&self) -> LlamaAttentionType {
308 LlamaAttentionType::from(self.context_params.attention_type)
309 }
310
311 /// Set the flash attention policy using llama.cpp enum
312 #[must_use]
313 pub fn with_flash_attention_policy(
314 mut self,
315 policy: llama_cpp_sys_2::llama_flash_attn_type,
316 ) -> Self {
317 self.context_params.flash_attn_type = policy;
318 self
319 }
320
321 /// Get the flash attention policy
322 #[must_use]
323 pub fn flash_attention_policy(&self) -> llama_cpp_sys_2::llama_flash_attn_type {
324 self.context_params.flash_attn_type
325 }
326
327 /// Set the rope frequency base
328 ///
329 /// # Examples
330 ///
331 /// ```rust
332 /// # use llama_cpp_2::context::params::LlamaContextParams;
333 /// let params = LlamaContextParams::default()
334 /// .with_rope_freq_base(0.5);
335 /// assert_eq!(params.rope_freq_base(), 0.5);
336 /// ```
337 #[must_use]
338 pub fn with_rope_freq_base(mut self, rope_freq_base: f32) -> Self {
339 self.context_params.rope_freq_base = rope_freq_base;
340 self
341 }
342
343 /// Get the rope frequency base
344 ///
345 /// # Examples
346 ///
347 /// ```rust
348 /// # use llama_cpp_2::context::params::LlamaContextParams;
349 /// let params = LlamaContextParams::default();
350 /// assert_eq!(params.rope_freq_base(), 0.0);
351 /// ```
352 #[must_use]
353 pub fn rope_freq_base(&self) -> f32 {
354 self.context_params.rope_freq_base
355 }
356
357 /// Set the rope frequency scale
358 ///
359 /// # Examples
360 ///
361 /// ```rust
362 /// # use llama_cpp_2::context::params::LlamaContextParams;
363 /// let params = LlamaContextParams::default()
364 /// .with_rope_freq_scale(0.5);
365 /// assert_eq!(params.rope_freq_scale(), 0.5);
366 /// ```
367 #[must_use]
368 pub fn with_rope_freq_scale(mut self, rope_freq_scale: f32) -> Self {
369 self.context_params.rope_freq_scale = rope_freq_scale;
370 self
371 }
372
373 /// Get the rope frequency scale
374 ///
375 /// # Examples
376 ///
377 /// ```rust
378 /// # use llama_cpp_2::context::params::LlamaContextParams;
379 /// let params = LlamaContextParams::default();
380 /// assert_eq!(params.rope_freq_scale(), 0.0);
381 /// ```
382 #[must_use]
383 pub fn rope_freq_scale(&self) -> f32 {
384 self.context_params.rope_freq_scale
385 }
386
387 /// Set the YaRN extrapolation mix factor
388 ///
389 /// # Examples
390 ///
391 /// ```rust
392 /// # use llama_cpp_2::context::params::LlamaContextParams;
393 /// let params = LlamaContextParams::default().with_yarn_ext_factor(1.0);
394 /// assert_eq!(params.yarn_ext_factor(), 1.0);
395 /// ```
396 #[must_use]
397 pub fn with_yarn_ext_factor(mut self, yarn_ext_factor: f32) -> Self {
398 self.context_params.yarn_ext_factor = yarn_ext_factor;
399 self
400 }
401
402 /// Get the YaRN extrapolation mix factor
403 #[must_use]
404 pub fn yarn_ext_factor(&self) -> f32 {
405 self.context_params.yarn_ext_factor
406 }
407
408 /// Set the YaRN magnitude scaling factor
409 ///
410 /// # Examples
411 ///
412 /// ```rust
413 /// # use llama_cpp_2::context::params::LlamaContextParams;
414 /// let params = LlamaContextParams::default().with_yarn_attn_factor(2.0);
415 /// assert_eq!(params.yarn_attn_factor(), 2.0);
416 /// ```
417 #[must_use]
418 pub fn with_yarn_attn_factor(mut self, yarn_attn_factor: f32) -> Self {
419 self.context_params.yarn_attn_factor = yarn_attn_factor;
420 self
421 }
422
423 /// Get the YaRN magnitude scaling factor
424 #[must_use]
425 pub fn yarn_attn_factor(&self) -> f32 {
426 self.context_params.yarn_attn_factor
427 }
428
429 /// Set the YaRN low correction dim
430 ///
431 /// # Examples
432 ///
433 /// ```rust
434 /// # use llama_cpp_2::context::params::LlamaContextParams;
435 /// let params = LlamaContextParams::default().with_yarn_beta_fast(16.0);
436 /// assert_eq!(params.yarn_beta_fast(), 16.0);
437 /// ```
438 #[must_use]
439 pub fn with_yarn_beta_fast(mut self, yarn_beta_fast: f32) -> Self {
440 self.context_params.yarn_beta_fast = yarn_beta_fast;
441 self
442 }
443
444 /// Get the YaRN low correction dim
445 #[must_use]
446 pub fn yarn_beta_fast(&self) -> f32 {
447 self.context_params.yarn_beta_fast
448 }
449
450 /// Set the YaRN high correction dim
451 ///
452 /// # Examples
453 ///
454 /// ```rust
455 /// # use llama_cpp_2::context::params::LlamaContextParams;
456 /// let params = LlamaContextParams::default().with_yarn_beta_slow(2.0);
457 /// assert_eq!(params.yarn_beta_slow(), 2.0);
458 /// ```
459 #[must_use]
460 pub fn with_yarn_beta_slow(mut self, yarn_beta_slow: f32) -> Self {
461 self.context_params.yarn_beta_slow = yarn_beta_slow;
462 self
463 }
464
465 /// Get the YaRN high correction dim
466 #[must_use]
467 pub fn yarn_beta_slow(&self) -> f32 {
468 self.context_params.yarn_beta_slow
469 }
470
471 /// Set the YaRN original context size
472 ///
473 /// # Examples
474 ///
475 /// ```rust
476 /// # use llama_cpp_2::context::params::LlamaContextParams;
477 /// let params = LlamaContextParams::default().with_yarn_orig_ctx(4096);
478 /// assert_eq!(params.yarn_orig_ctx(), 4096);
479 /// ```
480 #[must_use]
481 pub fn with_yarn_orig_ctx(mut self, yarn_orig_ctx: u32) -> Self {
482 self.context_params.yarn_orig_ctx = yarn_orig_ctx;
483 self
484 }
485
486 /// Get the YaRN original context size
487 #[must_use]
488 pub fn yarn_orig_ctx(&self) -> u32 {
489 self.context_params.yarn_orig_ctx
490 }
491
492 /// Set the KV cache defragmentation threshold
493 ///
494 /// # Examples
495 ///
496 /// ```rust
497 /// # use llama_cpp_2::context::params::LlamaContextParams;
498 /// let params = LlamaContextParams::default().with_defrag_thold(0.1);
499 /// assert_eq!(params.defrag_thold(), 0.1);
500 /// ```
501 #[must_use]
502 pub fn with_defrag_thold(mut self, defrag_thold: f32) -> Self {
503 self.context_params.defrag_thold = defrag_thold;
504 self
505 }
506
507 /// Get the KV cache defragmentation threshold
508 #[must_use]
509 pub fn defrag_thold(&self) -> f32 {
510 self.context_params.defrag_thold
511 }
512
513 /// Set the KV cache data type for K
514 ///
515 /// # Examples
516 ///
517 /// ```rust
518 /// # use llama_cpp_2::context::params::{LlamaContextParams, KvCacheType};
519 /// let params = LlamaContextParams::default().with_type_k(KvCacheType::Q4_0);
520 /// assert_eq!(params.type_k(), KvCacheType::Q4_0);
521 /// ```
522 #[must_use]
523 pub fn with_type_k(mut self, type_k: KvCacheType) -> Self {
524 self.context_params.type_k = type_k.into();
525 self
526 }
527
528 /// Get the KV cache data type for K
529 ///
530 /// # Examples
531 ///
532 /// ```rust
533 /// # use llama_cpp_2::context::params::LlamaContextParams;
534 /// let params = LlamaContextParams::default();
535 /// let _ = params.type_k();
536 /// ```
537 #[must_use]
538 pub fn type_k(&self) -> KvCacheType {
539 KvCacheType::from(self.context_params.type_k)
540 }
541
542 /// Set the KV cache data type for V
543 ///
544 /// # Examples
545 ///
546 /// ```rust
547 /// # use llama_cpp_2::context::params::{LlamaContextParams, KvCacheType};
548 /// let params = LlamaContextParams::default().with_type_v(KvCacheType::Q4_1);
549 /// assert_eq!(params.type_v(), KvCacheType::Q4_1);
550 /// ```
551 #[must_use]
552 pub fn with_type_v(mut self, type_v: KvCacheType) -> Self {
553 self.context_params.type_v = type_v.into();
554 self
555 }
556
557 /// Get the KV cache data type for V
558 ///
559 /// # Examples
560 ///
561 /// ```rust
562 /// # use llama_cpp_2::context::params::LlamaContextParams;
563 /// let params = LlamaContextParams::default();
564 /// let _ = params.type_v();
565 /// ```
566 #[must_use]
567 pub fn type_v(&self) -> KvCacheType {
568 KvCacheType::from(self.context_params.type_v)
569 }
570
571 /// Set whether embeddings are enabled
572 ///
573 /// # Examples
574 ///
575 /// ```rust
576 /// # use llama_cpp_2::context::params::LlamaContextParams;
577 /// let params = LlamaContextParams::default()
578 /// .with_embeddings(true);
579 /// assert!(params.embeddings());
580 /// ```
581 #[must_use]
582 pub fn with_embeddings(mut self, embedding: bool) -> Self {
583 self.context_params.embeddings = embedding;
584 self
585 }
586
587 /// Get whether embeddings are enabled
588 ///
589 /// # Examples
590 ///
591 /// ```rust
592 /// # use llama_cpp_2::context::params::LlamaContextParams;
593 /// let params = LlamaContextParams::default();
594 /// assert!(!params.embeddings());
595 /// ```
596 #[must_use]
597 pub fn embeddings(&self) -> bool {
598 self.context_params.embeddings
599 }
600
601 /// Set whether to offload KQV ops to GPU
602 ///
603 /// # Examples
604 ///
605 /// ```rust
606 /// # use llama_cpp_2::context::params::LlamaContextParams;
607 /// let params = LlamaContextParams::default()
608 /// .with_offload_kqv(false);
609 /// assert_eq!(params.offload_kqv(), false);
610 /// ```
611 #[must_use]
612 pub fn with_offload_kqv(mut self, enabled: bool) -> Self {
613 self.context_params.offload_kqv = enabled;
614 self
615 }
616
617 /// Get whether KQV ops are offloaded to GPU
618 ///
619 /// # Examples
620 ///
621 /// ```rust
622 /// # use llama_cpp_2::context::params::LlamaContextParams;
623 /// let params = LlamaContextParams::default();
624 /// assert_eq!(params.offload_kqv(), true);
625 /// ```
626 #[must_use]
627 pub fn offload_kqv(&self) -> bool {
628 self.context_params.offload_kqv
629 }
630
631 /// Set whether to disable performance timings
632 ///
633 /// # Examples
634 ///
635 /// ```rust
636 /// # use llama_cpp_2::context::params::LlamaContextParams;
637 /// let params = LlamaContextParams::default().with_no_perf(true);
638 /// assert!(params.no_perf());
639 /// ```
640 #[must_use]
641 pub fn with_no_perf(mut self, no_perf: bool) -> Self {
642 self.context_params.no_perf = no_perf;
643 self
644 }
645
646 /// Get whether performance timings are disabled
647 #[must_use]
648 pub fn no_perf(&self) -> bool {
649 self.context_params.no_perf
650 }
651
652 /// Set whether to offload ops to GPU
653 ///
654 /// # Examples
655 ///
656 /// ```rust
657 /// # use llama_cpp_2::context::params::LlamaContextParams;
658 /// let params = LlamaContextParams::default().with_op_offload(false);
659 /// assert_eq!(params.op_offload(), false);
660 /// ```
661 #[must_use]
662 pub fn with_op_offload(mut self, op_offload: bool) -> Self {
663 self.context_params.op_offload = op_offload;
664 self
665 }
666
667 /// Get whether ops are offloaded to GPU
668 #[must_use]
669 pub fn op_offload(&self) -> bool {
670 self.context_params.op_offload
671 }
672
673 /// Set whether to use full sliding window attention
674 ///
675 /// # Examples
676 ///
677 /// ```rust
678 /// # use llama_cpp_2::context::params::LlamaContextParams;
679 /// let params = LlamaContextParams::default()
680 /// .with_swa_full(false);
681 /// assert_eq!(params.swa_full(), false);
682 /// ```
683 #[must_use]
684 pub fn with_swa_full(mut self, enabled: bool) -> Self {
685 self.context_params.swa_full = enabled;
686 self
687 }
688
689 /// Get whether full sliding window attention is enabled
690 ///
691 /// # Examples
692 ///
693 /// ```rust
694 /// # use llama_cpp_2::context::params::LlamaContextParams;
695 /// let params = LlamaContextParams::default();
696 /// assert_eq!(params.swa_full(), true);
697 /// ```
698 #[must_use]
699 pub fn swa_full(&self) -> bool {
700 self.context_params.swa_full
701 }
702
703 /// Set whether to use a unified KV cache buffer across input sequences
704 ///
705 /// # Examples
706 ///
707 /// ```rust
708 /// # use llama_cpp_2::context::params::LlamaContextParams;
709 /// let params = LlamaContextParams::default().with_kv_unified(true);
710 /// assert!(params.kv_unified());
711 /// ```
712 #[must_use]
713 pub fn with_kv_unified(mut self, kv_unified: bool) -> Self {
714 self.context_params.kv_unified = kv_unified;
715 self
716 }
717
718 /// Get whether a unified KV cache buffer is used across input sequences
719 ///
720 /// # Examples
721 ///
722 /// ```rust
723 /// # use llama_cpp_2::context::params::LlamaContextParams;
724 /// let params = LlamaContextParams::default();
725 /// let _ = params.kv_unified();
726 /// ```
727 #[must_use]
728 pub fn kv_unified(&self) -> bool {
729 self.context_params.kv_unified
730 }
731}