llama_cpp_bindings/context/
params.rs1use std::fmt::Debug;
2use std::num::NonZeroU32;
3
4pub use crate::context::kv_cache_type::KvCacheType;
5pub use crate::context::llama_attention_type::LlamaAttentionType;
6pub use crate::context::llama_pooling_type::LlamaPoolingType;
7pub use crate::context::rope_scaling_type::RopeScalingType;
8
9#[derive(Debug, Clone, Copy)]
10#[expect(
11 missing_docs,
12 reason = "field meanings mirror llama.cpp's `llama_context_params` C struct; restating each \
13 one inline would risk drift from the upstream spec — the doc-comment on the struct \
14 points at the canonical reference"
15)]
16pub struct LlamaContextParams {
17 pub context_params: llama_cpp_bindings_sys::llama_context_params,
18}
19
20unsafe impl Send for LlamaContextParams {}
21unsafe impl Sync for LlamaContextParams {}
22
23impl LlamaContextParams {
24 #[must_use]
25 pub fn with_n_ctx(mut self, n_ctx: Option<NonZeroU32>) -> Self {
26 self.context_params.n_ctx = n_ctx.map_or(0, NonZeroU32::get);
27 self
28 }
29
30 #[must_use]
31 pub const fn n_ctx(&self) -> Option<NonZeroU32> {
32 NonZeroU32::new(self.context_params.n_ctx)
33 }
34
35 #[must_use]
36 pub const fn with_n_batch(mut self, n_batch: u32) -> Self {
37 self.context_params.n_batch = n_batch;
38 self
39 }
40
41 #[must_use]
42 pub const fn n_batch(&self) -> u32 {
43 self.context_params.n_batch
44 }
45
46 #[must_use]
47 pub const fn with_n_ubatch(mut self, n_ubatch: u32) -> Self {
48 self.context_params.n_ubatch = n_ubatch;
49 self
50 }
51
52 #[must_use]
53 pub const fn n_ubatch(&self) -> u32 {
54 self.context_params.n_ubatch
55 }
56
57 #[must_use]
58 pub const fn with_flash_attention_policy(
59 mut self,
60 policy: llama_cpp_bindings_sys::llama_flash_attn_type,
61 ) -> Self {
62 self.context_params.flash_attn_type = policy;
63 self
64 }
65
66 #[must_use]
67 pub const fn flash_attention_policy(&self) -> llama_cpp_bindings_sys::llama_flash_attn_type {
68 self.context_params.flash_attn_type
69 }
70
71 #[must_use]
72 pub const fn with_offload_kqv(mut self, enabled: bool) -> Self {
73 self.context_params.offload_kqv = enabled;
74 self
75 }
76
77 #[must_use]
78 pub const fn offload_kqv(&self) -> bool {
79 self.context_params.offload_kqv
80 }
81
82 #[must_use]
83 pub fn with_rope_scaling_type(mut self, rope_scaling_type: RopeScalingType) -> Self {
84 self.context_params.rope_scaling_type = i32::from(rope_scaling_type);
85 self
86 }
87
88 #[must_use]
89 pub fn rope_scaling_type(&self) -> RopeScalingType {
90 RopeScalingType::from(self.context_params.rope_scaling_type)
91 }
92
93 #[must_use]
94 pub const fn with_rope_freq_base(mut self, rope_freq_base: f32) -> Self {
95 self.context_params.rope_freq_base = rope_freq_base;
96 self
97 }
98
99 #[must_use]
100 pub const fn rope_freq_base(&self) -> f32 {
101 self.context_params.rope_freq_base
102 }
103
104 #[must_use]
105 pub const fn with_rope_freq_scale(mut self, rope_freq_scale: f32) -> Self {
106 self.context_params.rope_freq_scale = rope_freq_scale;
107 self
108 }
109
110 #[must_use]
111 pub const fn rope_freq_scale(&self) -> f32 {
112 self.context_params.rope_freq_scale
113 }
114
115 #[must_use]
116 pub const fn n_threads(&self) -> i32 {
117 self.context_params.n_threads
118 }
119
120 #[must_use]
121 pub const fn n_threads_batch(&self) -> i32 {
122 self.context_params.n_threads_batch
123 }
124
125 #[must_use]
126 pub const fn with_n_threads(mut self, n_threads: i32) -> Self {
127 self.context_params.n_threads = n_threads;
128 self
129 }
130
131 #[must_use]
132 pub const fn with_n_threads_batch(mut self, n_threads: i32) -> Self {
133 self.context_params.n_threads_batch = n_threads;
134 self
135 }
136
137 #[must_use]
138 pub const fn embeddings(&self) -> bool {
139 self.context_params.embeddings
140 }
141
142 #[must_use]
143 pub const fn with_embeddings(mut self, embedding: bool) -> Self {
144 self.context_params.embeddings = embedding;
145 self
146 }
147
148 #[must_use]
149 pub fn with_cb_eval(
150 mut self,
151 cb_eval: llama_cpp_bindings_sys::ggml_backend_sched_eval_callback,
152 ) -> Self {
153 self.context_params.cb_eval = cb_eval;
154 self
155 }
156
157 #[must_use]
158 pub const fn with_cb_eval_user_data(
159 mut self,
160 cb_eval_user_data: *mut std::ffi::c_void,
161 ) -> Self {
162 self.context_params.cb_eval_user_data = cb_eval_user_data;
163 self
164 }
165
166 #[must_use]
167 pub fn with_pooling_type(mut self, pooling_type: LlamaPoolingType) -> Self {
168 self.context_params.pooling_type = i32::from(pooling_type);
169 self
170 }
171
172 #[must_use]
173 pub fn pooling_type(&self) -> LlamaPoolingType {
174 LlamaPoolingType::from(self.context_params.pooling_type)
175 }
176
177 #[must_use]
178 pub const fn with_swa_full(mut self, enabled: bool) -> Self {
179 self.context_params.swa_full = enabled;
180 self
181 }
182
183 #[must_use]
184 pub const fn swa_full(&self) -> bool {
185 self.context_params.swa_full
186 }
187
188 #[must_use]
189 pub const fn with_n_seq_max(mut self, n_seq_max: u32) -> Self {
190 self.context_params.n_seq_max = n_seq_max;
191 self
192 }
193
194 #[must_use]
195 pub const fn n_seq_max(&self) -> u32 {
196 self.context_params.n_seq_max
197 }
198 #[must_use]
199 pub fn with_type_k(mut self, type_k: KvCacheType) -> Self {
200 self.context_params.type_k = type_k.into();
201 self
202 }
203
204 #[must_use]
205 pub fn type_k(&self) -> KvCacheType {
206 KvCacheType::from(self.context_params.type_k)
207 }
208
209 #[must_use]
210 pub fn with_type_v(mut self, type_v: KvCacheType) -> Self {
211 self.context_params.type_v = type_v.into();
212 self
213 }
214
215 #[must_use]
216 pub fn type_v(&self) -> KvCacheType {
217 KvCacheType::from(self.context_params.type_v)
218 }
219
220 #[must_use]
221 pub fn with_attention_type(mut self, attention_type: LlamaAttentionType) -> Self {
222 self.context_params.attention_type = i32::from(attention_type);
223 self
224 }
225
226 #[must_use]
227 pub fn attention_type(&self) -> LlamaAttentionType {
228 LlamaAttentionType::from(self.context_params.attention_type)
229 }
230
231 #[must_use]
232 pub const fn with_yarn_ext_factor(mut self, yarn_ext_factor: f32) -> Self {
233 self.context_params.yarn_ext_factor = yarn_ext_factor;
234 self
235 }
236
237 #[must_use]
238 pub const fn yarn_ext_factor(&self) -> f32 {
239 self.context_params.yarn_ext_factor
240 }
241
242 #[must_use]
243 pub const fn with_yarn_attn_factor(mut self, yarn_attn_factor: f32) -> Self {
244 self.context_params.yarn_attn_factor = yarn_attn_factor;
245 self
246 }
247
248 #[must_use]
249 pub const fn yarn_attn_factor(&self) -> f32 {
250 self.context_params.yarn_attn_factor
251 }
252
253 #[must_use]
254 pub const fn with_yarn_beta_fast(mut self, yarn_beta_fast: f32) -> Self {
255 self.context_params.yarn_beta_fast = yarn_beta_fast;
256 self
257 }
258
259 #[must_use]
260 pub const fn yarn_beta_fast(&self) -> f32 {
261 self.context_params.yarn_beta_fast
262 }
263
264 #[must_use]
265 pub const fn with_yarn_beta_slow(mut self, yarn_beta_slow: f32) -> Self {
266 self.context_params.yarn_beta_slow = yarn_beta_slow;
267 self
268 }
269
270 #[must_use]
271 pub const fn yarn_beta_slow(&self) -> f32 {
272 self.context_params.yarn_beta_slow
273 }
274
275 #[must_use]
276 pub const fn with_yarn_orig_ctx(mut self, yarn_orig_ctx: u32) -> Self {
277 self.context_params.yarn_orig_ctx = yarn_orig_ctx;
278 self
279 }
280
281 #[must_use]
282 pub const fn yarn_orig_ctx(&self) -> u32 {
283 self.context_params.yarn_orig_ctx
284 }
285
286 #[must_use]
287 pub const fn with_defrag_thold(mut self, defrag_thold: f32) -> Self {
288 self.context_params.defrag_thold = defrag_thold;
289 self
290 }
291
292 #[must_use]
293 pub const fn defrag_thold(&self) -> f32 {
294 self.context_params.defrag_thold
295 }
296
297 #[must_use]
298 pub const fn with_no_perf(mut self, no_perf: bool) -> Self {
299 self.context_params.no_perf = no_perf;
300 self
301 }
302
303 #[must_use]
304 pub const fn no_perf(&self) -> bool {
305 self.context_params.no_perf
306 }
307
308 #[must_use]
309 pub const fn with_op_offload(mut self, op_offload: bool) -> Self {
310 self.context_params.op_offload = op_offload;
311 self
312 }
313
314 #[must_use]
315 pub const fn op_offload(&self) -> bool {
316 self.context_params.op_offload
317 }
318
319 #[must_use]
320 pub const fn with_kv_unified(mut self, kv_unified: bool) -> Self {
321 self.context_params.kv_unified = kv_unified;
322 self
323 }
324
325 #[must_use]
326 pub const fn kv_unified(&self) -> bool {
327 self.context_params.kv_unified
328 }
329}
330
331impl Default for LlamaContextParams {
332 fn default() -> Self {
333 let context_params = unsafe { llama_cpp_bindings_sys::llama_context_default_params() };
334 Self { context_params }
335 }
336}
337
338#[cfg(test)]
339mod tests {
340 use super::{KvCacheType, LlamaAttentionType, LlamaPoolingType, RopeScalingType};
341
342 #[test]
343 fn default_params_have_expected_values() {
344 let params = super::LlamaContextParams::default();
345
346 assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(512));
347 assert_eq!(params.n_batch(), 2048);
348 assert_eq!(params.n_ubatch(), 512);
349 assert_eq!(params.rope_scaling_type(), RopeScalingType::Unspecified);
350 assert_eq!(params.pooling_type(), LlamaPoolingType::Unspecified);
351 }
352
353 #[test]
354 fn with_n_ctx_sets_value() {
355 let params =
356 super::LlamaContextParams::default().with_n_ctx(std::num::NonZeroU32::new(2048));
357
358 assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(2048));
359 }
360
361 #[test]
362 fn with_n_ctx_none_sets_zero() {
363 let params = super::LlamaContextParams::default().with_n_ctx(None);
364
365 assert_eq!(params.n_ctx(), None);
366 }
367
368 #[test]
369 fn with_n_batch_sets_value() {
370 let params = super::LlamaContextParams::default().with_n_batch(4096);
371
372 assert_eq!(params.n_batch(), 4096);
373 }
374
375 #[test]
376 fn with_n_ubatch_sets_value() {
377 let params = super::LlamaContextParams::default().with_n_ubatch(1024);
378
379 assert_eq!(params.n_ubatch(), 1024);
380 }
381
382 #[test]
383 fn with_n_seq_max_sets_value() {
384 let params = super::LlamaContextParams::default().with_n_seq_max(64);
385
386 assert_eq!(params.n_seq_max(), 64);
387 }
388
389 #[test]
390 fn with_embeddings_enables() {
391 let params = super::LlamaContextParams::default().with_embeddings(true);
392
393 assert!(params.embeddings());
394 }
395
396 #[test]
397 fn with_embeddings_disables() {
398 let params = super::LlamaContextParams::default().with_embeddings(false);
399
400 assert!(!params.embeddings());
401 }
402
403 #[test]
404 fn with_offload_kqv_disables() {
405 let params = super::LlamaContextParams::default().with_offload_kqv(false);
406
407 assert!(!params.offload_kqv());
408 }
409
410 #[test]
411 fn with_offload_kqv_enables() {
412 let params = super::LlamaContextParams::default().with_offload_kqv(true);
413
414 assert!(params.offload_kqv());
415 }
416
417 #[test]
418 fn with_swa_full_disables() {
419 let params = super::LlamaContextParams::default().with_swa_full(false);
420
421 assert!(!params.swa_full());
422 }
423
424 #[test]
425 fn with_swa_full_enables() {
426 let params = super::LlamaContextParams::default().with_swa_full(true);
427
428 assert!(params.swa_full());
429 }
430
431 #[test]
432 fn with_rope_scaling_type_linear() {
433 let params =
434 super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::Linear);
435
436 assert_eq!(params.rope_scaling_type(), RopeScalingType::Linear);
437 }
438
439 #[test]
440 fn with_rope_scaling_type_yarn() {
441 let params =
442 super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::Yarn);
443
444 assert_eq!(params.rope_scaling_type(), RopeScalingType::Yarn);
445 }
446
447 #[test]
448 fn with_rope_scaling_type_none() {
449 let params =
450 super::LlamaContextParams::default().with_rope_scaling_type(RopeScalingType::None);
451
452 assert_eq!(params.rope_scaling_type(), RopeScalingType::None);
453 }
454
455 #[test]
456 fn with_rope_freq_base_sets_value() {
457 let params = super::LlamaContextParams::default().with_rope_freq_base(10000.0);
458
459 assert!((params.rope_freq_base() - 10000.0).abs() < f32::EPSILON);
460 }
461
462 #[test]
463 fn with_rope_freq_scale_sets_value() {
464 let params = super::LlamaContextParams::default().with_rope_freq_scale(0.5);
465
466 assert!((params.rope_freq_scale() - 0.5).abs() < f32::EPSILON);
467 }
468
469 #[test]
470 fn with_n_threads_sets_value() {
471 let params = super::LlamaContextParams::default().with_n_threads(16);
472
473 assert_eq!(params.n_threads(), 16);
474 }
475
476 #[test]
477 fn with_n_threads_batch_sets_value() {
478 let params = super::LlamaContextParams::default().with_n_threads_batch(16);
479
480 assert_eq!(params.n_threads_batch(), 16);
481 }
482
483 #[test]
484 fn with_pooling_type_mean() {
485 let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Mean);
486
487 assert_eq!(params.pooling_type(), LlamaPoolingType::Mean);
488 }
489
490 #[test]
491 fn with_pooling_type_cls() {
492 let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Cls);
493
494 assert_eq!(params.pooling_type(), LlamaPoolingType::Cls);
495 }
496
497 #[test]
498 fn with_pooling_type_last() {
499 let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Last);
500
501 assert_eq!(params.pooling_type(), LlamaPoolingType::Last);
502 }
503
504 #[test]
505 fn with_pooling_type_rank() {
506 let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::Rank);
507
508 assert_eq!(params.pooling_type(), LlamaPoolingType::Rank);
509 }
510
511 #[test]
512 fn with_pooling_type_none() {
513 let params = super::LlamaContextParams::default().with_pooling_type(LlamaPoolingType::None);
514
515 assert_eq!(params.pooling_type(), LlamaPoolingType::None);
516 }
517
518 #[test]
519 fn with_type_k_sets_value() {
520 let params = super::LlamaContextParams::default().with_type_k(KvCacheType::Q4_0);
521
522 assert_eq!(params.type_k(), KvCacheType::Q4_0);
523 }
524
525 #[test]
526 fn with_type_v_sets_value() {
527 let params = super::LlamaContextParams::default().with_type_v(KvCacheType::Q4_1);
528
529 assert_eq!(params.type_v(), KvCacheType::Q4_1);
530 }
531
532 #[test]
533 fn with_flash_attention_policy_sets_value() {
534 let params = super::LlamaContextParams::default()
535 .with_flash_attention_policy(llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED);
536
537 assert_eq!(
538 params.flash_attention_policy(),
539 llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_ENABLED
540 );
541 }
542
543 #[test]
544 fn builder_chaining_preserves_all_values() {
545 let params = super::LlamaContextParams::default()
546 .with_n_ctx(std::num::NonZeroU32::new(1024))
547 .with_n_batch(4096)
548 .with_n_ubatch(256)
549 .with_n_threads(8)
550 .with_n_threads_batch(12)
551 .with_embeddings(true)
552 .with_offload_kqv(false)
553 .with_rope_scaling_type(RopeScalingType::Yarn)
554 .with_rope_freq_base(5000.0)
555 .with_rope_freq_scale(0.25);
556
557 assert_eq!(params.n_ctx(), std::num::NonZeroU32::new(1024));
558 assert_eq!(params.n_batch(), 4096);
559 assert_eq!(params.n_ubatch(), 256);
560 assert_eq!(params.n_threads(), 8);
561 assert_eq!(params.n_threads_batch(), 12);
562 assert!(params.embeddings());
563 assert!(!params.offload_kqv());
564 assert_eq!(params.rope_scaling_type(), RopeScalingType::Yarn);
565 assert!((params.rope_freq_base() - 5000.0).abs() < f32::EPSILON);
566 assert!((params.rope_freq_scale() - 0.25).abs() < f32::EPSILON);
567 }
568
569 #[test]
570 fn with_cb_eval_sets_callback() {
571 extern "C" fn test_cb_eval(
572 _tensor: *mut llama_cpp_bindings_sys::ggml_tensor,
573 _ask: bool,
574 _user_data: *mut std::ffi::c_void,
575 ) -> bool {
576 false
577 }
578
579 let result = test_cb_eval(std::ptr::null_mut(), false, std::ptr::null_mut());
580
581 assert!(!result);
582
583 let params = super::LlamaContextParams::default().with_cb_eval(Some(test_cb_eval));
584
585 assert!(params.context_params.cb_eval.is_some());
586 }
587
588 #[test]
589 fn with_cb_eval_user_data_sets_pointer() {
590 let mut value: i32 = 42;
591 let user_data = (&raw mut value).cast::<std::ffi::c_void>();
592 let params = super::LlamaContextParams::default().with_cb_eval_user_data(user_data);
593
594 assert_eq!(params.context_params.cb_eval_user_data, user_data);
595 }
596
597 #[test]
598 fn with_flash_attention_policy_disabled() {
599 let params = super::LlamaContextParams::default()
600 .with_flash_attention_policy(llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED);
601
602 assert_eq!(
603 params.flash_attention_policy(),
604 llama_cpp_bindings_sys::LLAMA_FLASH_ATTN_TYPE_DISABLED
605 );
606 }
607
608 #[test]
609 fn with_attention_type_causal() {
610 let params =
611 super::LlamaContextParams::default().with_attention_type(LlamaAttentionType::Causal);
612
613 assert_eq!(params.attention_type(), LlamaAttentionType::Causal);
614 }
615
616 #[test]
617 fn with_attention_type_non_causal() {
618 let params =
619 super::LlamaContextParams::default().with_attention_type(LlamaAttentionType::NonCausal);
620
621 assert_eq!(params.attention_type(), LlamaAttentionType::NonCausal);
622 }
623
624 #[test]
625 fn with_yarn_ext_factor_sets_value() {
626 let params = super::LlamaContextParams::default().with_yarn_ext_factor(1.5);
627
628 assert!((params.yarn_ext_factor() - 1.5).abs() < f32::EPSILON);
629 }
630
631 #[test]
632 fn with_yarn_attn_factor_sets_value() {
633 let params = super::LlamaContextParams::default().with_yarn_attn_factor(2.0);
634
635 assert!((params.yarn_attn_factor() - 2.0).abs() < f32::EPSILON);
636 }
637
638 #[test]
639 fn with_yarn_beta_fast_sets_value() {
640 let params = super::LlamaContextParams::default().with_yarn_beta_fast(32.0);
641
642 assert!((params.yarn_beta_fast() - 32.0).abs() < f32::EPSILON);
643 }
644
645 #[test]
646 fn with_yarn_beta_slow_sets_value() {
647 let params = super::LlamaContextParams::default().with_yarn_beta_slow(1.0);
648
649 assert!((params.yarn_beta_slow() - 1.0).abs() < f32::EPSILON);
650 }
651
652 #[test]
653 fn with_yarn_orig_ctx_sets_value() {
654 let params = super::LlamaContextParams::default().with_yarn_orig_ctx(4096);
655
656 assert_eq!(params.yarn_orig_ctx(), 4096);
657 }
658
659 #[test]
660 fn with_defrag_thold_sets_value() {
661 let params = super::LlamaContextParams::default().with_defrag_thold(0.1);
662
663 assert!((params.defrag_thold() - 0.1).abs() < f32::EPSILON);
664 }
665
666 #[test]
667 fn with_no_perf_enables() {
668 let params = super::LlamaContextParams::default().with_no_perf(true);
669
670 assert!(params.no_perf());
671 }
672
673 #[test]
674 fn with_no_perf_disables() {
675 let params = super::LlamaContextParams::default().with_no_perf(false);
676
677 assert!(!params.no_perf());
678 }
679
680 #[test]
681 fn with_op_offload_enables() {
682 let params = super::LlamaContextParams::default().with_op_offload(true);
683
684 assert!(params.op_offload());
685 }
686
687 #[test]
688 fn with_op_offload_disables() {
689 let params = super::LlamaContextParams::default().with_op_offload(false);
690
691 assert!(!params.op_offload());
692 }
693
694 #[test]
695 fn with_kv_unified_enables() {
696 let params = super::LlamaContextParams::default().with_kv_unified(true);
697
698 assert!(params.kv_unified());
699 }
700
701 #[test]
702 fn with_kv_unified_disables() {
703 let params = super::LlamaContextParams::default().with_kv_unified(false);
704
705 assert!(!params.kv_unified());
706 }
707}