1use std::marker::PhantomData;
6use std::num::NonZeroU32;
7use std::panic::{AssertUnwindSafe, catch_unwind};
8use std::rc::Rc;
9
10use llama_cpp_4::context::LlamaContext;
11use llama_cpp_4::context::params::{LlamaContextParams, LlamaContextType};
12use llama_cpp_4::llama_batch::LlamaBatch;
13use llama_cpp_4::sampling::LlamaSampler;
14use llama_cpp_4::token::LlamaToken;
15use llama_cpp_4::token::data::LlamaTokenData;
16use llama_cpp_4::token::data_array::LlamaTokenDataArray;
17use logit_loom::{
18 ActivationPhaseV1, ActivationTelemetryDispositionV1, CheckpointReceipt, ControlFlow, Digest,
19 GenerationFinish, GenerationPlan, GenerationPlanV2, GenerationReceipt, ObservedToken,
20 ObserverSet, Pipeline, PrefillFinish, PrefillMonitor, PrefillProgress, PrefillReceipt,
21 SteeringKind, TokenId,
22};
23
24use crate::{
25 ActivationCaptureOutput, ActivationConfiguration, ActivationProgramOutput, Error, Model,
26 Runtime,
27 activation::ActivationController,
28 error::native,
29 sampler::{build_sampler, build_sampler_v2},
30};
31
32const STATE_BYTES_DOMAIN: &str = "llamacpp-state-bytes-v1";
33const TOKEN_HISTORY_DOMAIN: &str = "llamacpp-state-token-history-v1";
34
35#[derive(Clone, Copy, Debug, PartialEq, Eq)]
37pub struct SessionOptions {
38 pub context_size: NonZeroU32,
40 pub batch_size: u32,
42 pub micro_batch_size: u32,
44 pub threads: i32,
46}
47
48impl Default for SessionOptions {
49 fn default() -> Self {
50 Self {
51 context_size: NonZeroU32::new(4_096).expect("nonzero constant"),
52 batch_size: 512,
53 micro_batch_size: 512,
54 threads: 4,
55 }
56 }
57}
58
59impl SessionOptions {
60 fn validate(self) -> Result<(), Error> {
61 let maximum_native_context = u32::try_from(i32::MAX).unwrap_or(u32::MAX);
62 if self.batch_size == 0
63 || self.micro_batch_size == 0
64 || self.micro_batch_size > self.batch_size
65 || self.batch_size > self.context_size.get()
66 || self.context_size.get() > maximum_native_context
67 || self.threads <= 0
68 {
69 return Err(Error::Invalid(
70 "session requires 0 < micro_batch <= batch <= context and positive threads"
71 .to_owned(),
72 ));
73 }
74 Ok(())
75 }
76}
77
78#[cfg(test)]
79mod tests {
80 use super::*;
81
82 #[test]
83 fn session_options_reject_native_position_overflow() {
84 let options = SessionOptions {
85 context_size: NonZeroU32::new(u32::MAX).unwrap(),
86 ..SessionOptions::default()
87 };
88 assert!(options.validate().is_err());
89 }
90
91 #[test]
92 fn session_options_enforce_batch_and_thread_bounds() {
93 let defaults = SessionOptions::default();
94 assert!(defaults.validate().is_ok());
95 assert!(
96 SessionOptions {
97 micro_batch_size: defaults.batch_size + 1,
98 ..defaults
99 }
100 .validate()
101 .is_err()
102 );
103 assert!(
104 SessionOptions {
105 threads: 0,
106 ..defaults
107 }
108 .validate()
109 .is_err()
110 );
111 }
112
113 #[test]
114 fn checkpoint_compatibility_binds_session_options() {
115 let runtime = Digest::of_bytes("test-runtime", b"one");
116 let defaults = SessionOptions::default();
117 let different_context = SessionOptions {
118 context_size: NonZeroU32::new(defaults.context_size.get() + 1).unwrap(),
119 ..defaults
120 };
121 let different_threads = SessionOptions {
122 threads: defaults.threads + 1,
123 ..defaults
124 };
125 let identity =
126 session_compatibility_identity(&runtime, defaults, LlamaContextType::Default, 0);
127 assert_ne!(
128 identity,
129 session_compatibility_identity(
130 &runtime,
131 different_context,
132 LlamaContextType::Default,
133 0
134 )
135 );
136 assert_ne!(
137 identity,
138 session_compatibility_identity(
139 &runtime,
140 different_threads,
141 LlamaContextType::Default,
142 0
143 )
144 );
145 assert_ne!(
146 identity,
147 session_compatibility_identity(&runtime, defaults, LlamaContextType::Mtp, 4)
148 );
149 }
150
151 #[test]
152 fn generation_output_preserves_arbitrary_bytes() {
153 let output = GenerationOutput {
154 bytes: vec![0xff],
155 tokens: vec![TokenId::new(1).unwrap()],
156 receipt: GenerationReceipt {
157 plan: Digest::of_bytes("test-plan", b"bytes"),
158 initial_position: 0,
159 admitted_tokens: 1,
160 admitted_bytes: 1,
161 final_position: 1,
162 finish: GenerationFinish::TokenLimit,
163 transform_receipt: None,
164 observer_receipts: Vec::new(),
165 },
166 };
167 assert!(output.text().is_err());
168 assert_eq!(output.bytes, [0xff]);
169 }
170
171 #[test]
172 fn checkpoint_parts_validate_before_native_restore() {
173 let bytes = vec![1, 2, 3];
174 let tokens = vec![TokenId::new(4).unwrap()];
175 let receipt = CheckpointReceipt {
176 model: Digest::of_bytes("test-model", b"one"),
177 backend: Digest::of_bytes("test-backend", b"one"),
178 state: Digest::of_bytes(STATE_BYTES_DOMAIN, &bytes),
179 state_bytes: 3,
180 token_history: Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &tokens).unwrap(),
181 position: 1,
182 };
183 let snapshot =
184 StateSnapshot::from_parts(bytes.clone(), tokens.clone(), receipt.clone()).unwrap();
185 assert_eq!(snapshot.bytes(), bytes);
186 assert_eq!(snapshot.tokens(), tokens);
187 assert_eq!(snapshot.receipt(), &receipt);
188 assert_eq!(snapshot.into_parts(), (bytes, tokens, receipt));
189 }
190
191 #[test]
192 fn checkpoint_parts_reject_tampered_accounting() {
193 let bytes = vec![1, 2, 3];
194 let tokens = vec![TokenId::new(4).unwrap()];
195 let mut receipt = CheckpointReceipt {
196 model: Digest::of_bytes("test-model", b"one"),
197 backend: Digest::of_bytes("test-backend", b"one"),
198 state: Digest::of_bytes(STATE_BYTES_DOMAIN, &bytes),
199 state_bytes: 4,
200 token_history: Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &tokens).unwrap(),
201 position: 1,
202 };
203 assert!(StateSnapshot::from_parts(bytes.clone(), tokens.clone(), receipt.clone()).is_err());
204
205 receipt.state_bytes = 3;
206 receipt.position = 2;
207 assert!(StateSnapshot::from_parts(bytes, tokens, receipt).is_err());
208 }
209
210 #[test]
211 fn overlapping_stops_choose_the_lowest_declared_index() {
212 let stops = vec![b"bc".to_vec(), b"abc".to_vec(), b"c".to_vec()];
213 assert_eq!(matching_stop(&stops, b"zabc"), Some(0));
214 assert_eq!(matching_stop(&stops, b"zab"), None);
215 }
216
217 #[test]
218 fn ordinary_checkpoint_stop_tail_preserves_cross_operation_matches() {
219 let plan = GenerationPlan {
220 sampling: logit_loom::SamplingPlan::default(),
221 max_tokens: 4,
222 biases: Vec::new(),
223 grammar: None,
224 stops: vec![b"abcd".to_vec()],
225 };
226 let mut window = b"ab".to_vec();
227 extend_stop_window(&plan, &mut window, b"cd");
228 assert_eq!(matching_stop(&plan.stops, &window), Some(0));
229 assert_eq!(checkpoint_stop_tail(&plan, b"zabc"), b"abc");
230 assert!(validate_stop_tail(&plan, b"abc").is_ok());
231 assert!(validate_stop_tail(&plan, b"abcd").is_err());
232 }
233
234 #[test]
235 fn checkpoint_logit_refresh_targets_only_the_last_position() {
236 assert_eq!(checkpoint_logit_refresh_range(0).unwrap(), None);
237 assert_eq!(checkpoint_logit_refresh_range(1).unwrap(), Some((0, 1)));
238 assert_eq!(
239 checkpoint_logit_refresh_range(i32::MAX as u64).unwrap(),
240 Some((i32::MAX as u32 - 1, i32::MAX as u32))
241 );
242 assert!(checkpoint_logit_refresh_range(u64::MAX).is_err());
243 }
244}
245
246#[derive(Clone, Debug, PartialEq, Eq)]
248pub struct PrefillOutput {
249 pub admitted_tokens: u64,
251 pub position: u64,
253 pub receipt: Option<PrefillReceipt>,
255}
256
257#[derive(Clone, Debug, PartialEq, Eq)]
259pub struct GenerationOutput {
260 pub bytes: Vec<u8>,
262 pub tokens: Vec<TokenId>,
264 pub receipt: GenerationReceipt,
266}
267
268pub(crate) struct StatefulGenerationOutput {
269 pub(crate) output: GenerationOutput,
270 pub(crate) sampler: LlamaSampler,
271 pub(crate) stop_tail: Vec<u8>,
272}
273
274impl GenerationOutput {
275 pub fn text(&self) -> Result<&str, std::str::Utf8Error> {
281 std::str::from_utf8(&self.bytes)
282 }
283}
284
285#[derive(Clone, Debug)]
287pub struct StateSnapshot {
288 bytes: Vec<u8>,
289 tokens: Vec<TokenId>,
290 receipt: CheckpointReceipt,
291}
292
293impl StateSnapshot {
294 pub fn from_parts(
305 bytes: Vec<u8>,
306 tokens: Vec<TokenId>,
307 receipt: CheckpointReceipt,
308 ) -> Result<Self, Error> {
309 let snapshot = Self {
310 bytes,
311 tokens,
312 receipt,
313 };
314 snapshot.validate_contents()?;
315 Ok(snapshot)
316 }
317
318 pub fn bytes(&self) -> &[u8] {
320 &self.bytes
321 }
322
323 pub fn tokens(&self) -> &[TokenId] {
325 &self.tokens
326 }
327
328 pub const fn receipt(&self) -> &CheckpointReceipt {
330 &self.receipt
331 }
332
333 pub fn into_parts(self) -> (Vec<u8>, Vec<TokenId>, CheckpointReceipt) {
335 (self.bytes, self.tokens, self.receipt)
336 }
337
338 pub(crate) fn validate_contents(&self) -> Result<(), Error> {
339 if self.bytes.is_empty() {
340 return Err(Error::Incompatible(
341 "checkpoint state bytes must not be empty".to_owned(),
342 ));
343 }
344 let state_bytes = u64::try_from(self.bytes.len())
345 .map_err(|_| Error::Incompatible("checkpoint state size exceeds u64".to_owned()))?;
346 let token_count = u64::try_from(self.tokens.len())
347 .map_err(|_| Error::Incompatible("checkpoint token count exceeds u64".to_owned()))?;
348 if self.receipt.state != Digest::of_bytes(STATE_BYTES_DOMAIN, &self.bytes)
349 || self.receipt.state_bytes != state_bytes
350 || self.receipt.token_history
351 != Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &self.tokens)?
352 || self.receipt.position != token_count
353 {
354 return Err(Error::Incompatible(
355 "checkpoint metadata does not match state bytes or token lineage".to_owned(),
356 ));
357 }
358 self.receipt.digest()?;
359 Ok(())
360 }
361}
362
363pub struct Session<'model> {
368 pub(crate) context: LlamaContext<'model>,
369 pub(crate) model: &'model Model,
370 pub(crate) options: SessionOptions,
371 pub(crate) backend: Digest,
372 pub(crate) token_history: Vec<TokenId>,
373 pub(crate) position: u64,
374 pub(crate) active_steering: Vec<SteeringKind>,
375 pub(crate) activation: Option<ActivationController>,
376 pub(crate) poison_reason: Option<String>,
377 thread_affinity: PhantomData<Rc<()>>,
378}
379
380impl std::fmt::Debug for Session<'_> {
381 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
382 formatter
383 .debug_struct("Session")
384 .field("options", &self.options)
385 .field("position", &self.position)
386 .field("tokens", &self.token_history.len())
387 .field("active_steering", &self.active_steering)
388 .field("has_activation", &self.activation.is_some())
389 .field("healthy", &self.poison_reason.is_none())
390 .finish_non_exhaustive()
391 }
392}
393
394impl<'model> Session<'model> {
395 pub(crate) fn new(
396 model: &'model Model,
397 runtime: &Runtime,
398 options: SessionOptions,
399 ) -> Result<Self, Error> {
400 Self::new_inner(model, runtime, options, LlamaContextType::Default, 0, None)
401 }
402
403 pub(crate) fn new_with_activation(
404 model: &'model Model,
405 runtime: &Runtime,
406 options: SessionOptions,
407 activation: ActivationConfiguration,
408 ) -> Result<Self, Error> {
409 Self::new_inner(
410 model,
411 runtime,
412 options,
413 LlamaContextType::Default,
414 0,
415 Some(activation),
416 )
417 }
418
419 pub(crate) fn new_speculative(
420 model: &'model Model,
421 runtime: &Runtime,
422 options: SessionOptions,
423 context_type: LlamaContextType,
424 recurrent_state_slots: u32,
425 activation: Option<ActivationConfiguration>,
426 ) -> Result<Self, Error> {
427 Self::new_inner(
428 model,
429 runtime,
430 options,
431 context_type,
432 recurrent_state_slots,
433 activation,
434 )
435 }
436
437 fn new_inner(
438 model: &'model Model,
439 runtime: &Runtime,
440 options: SessionOptions,
441 context_type: LlamaContextType,
442 recurrent_state_slots: u32,
443 activation: Option<ActivationConfiguration>,
444 ) -> Result<Self, Error> {
445 options.validate()?;
446 let (activation, transactions) = activation
447 .map(|configuration| configuration.lower(options.batch_size))
448 .transpose()?
449 .map_or((None, None), |(controller, transactions)| {
450 (Some(controller), Some(transactions))
451 });
452 let mut params = LlamaContextParams::default()
453 .with_n_ctx(Some(options.context_size))
454 .with_n_batch(options.batch_size)
455 .with_n_ubatch(options.micro_batch_size)
456 .with_n_threads(options.threads)
457 .with_n_threads_batch(options.threads)
458 .with_ctx_type(context_type)
459 .with_n_rs_seq(recurrent_state_slots)
460 .with_offload_kqv(true);
461 if let Some(transactions) = transactions {
462 params = params.with_tensor_transactions(transactions);
463 }
464 let context = model
465 .native
466 .new_context(&runtime.native, params)
467 .map_err(native)?;
468 Ok(Self {
469 context,
470 model,
471 options,
472 backend: session_compatibility_identity(
473 runtime.identity(),
474 options,
475 context_type,
476 recurrent_state_slots,
477 ),
478 token_history: Vec::new(),
479 position: 0,
480 active_steering: Vec::new(),
481 activation,
482 poison_reason: None,
483 thread_affinity: PhantomData,
484 })
485 }
486
487 pub const fn options(&self) -> SessionOptions {
489 self.options
490 }
491
492 pub const fn position(&self) -> u64 {
494 self.position
495 }
496
497 pub fn token_history(&self) -> &[TokenId] {
499 &self.token_history
500 }
501
502 pub const fn is_healthy(&self) -> bool {
504 self.poison_reason.is_none()
505 }
506
507 pub fn poison_reason(&self) -> Option<&str> {
509 self.poison_reason.as_deref()
510 }
511
512 pub fn active_steering(&self) -> &[SteeringKind] {
514 &self.active_steering
515 }
516
517 pub const fn has_activation_runtime(&self) -> bool {
519 self.activation.is_some()
520 }
521
522 pub fn take_activation_captures(&mut self) -> Result<ActivationCaptureOutput, Error> {
529 self.ensure_healthy()?;
530 self.activation
531 .as_mut()
532 .ok_or_else(|| Error::Invalid("session has no activation runtime".to_owned()))?
533 .take_capture_output()
534 }
535
536 pub fn take_activation_program_output(&mut self) -> Result<ActivationProgramOutput, Error> {
544 self.ensure_healthy()?;
545 self.activation
546 .as_ref()
547 .ok_or_else(|| Error::Invalid("session has no activation runtime".to_owned()))?
548 .take_program_output()
549 }
550
551 pub fn clear(&mut self) -> Result<(), Error> {
557 self.ensure_healthy()?;
558 if let Some(activation) = self.activation.as_mut() {
559 activation.reset()?;
560 }
561 self.context.clear_kv_cache();
562 self.token_history.clear();
563 self.position = 0;
564 Ok(())
565 }
566
567 pub fn prefill(
574 &mut self,
575 tokens: &[TokenId],
576 clear_first: bool,
577 ) -> Result<PrefillOutput, Error> {
578 self.prefill_with_monitor(tokens, clear_first, None)
579 }
580
581 pub fn prefill_observed(
591 &mut self,
592 tokens: &[TokenId],
593 clear_first: bool,
594 monitor: &mut PrefillMonitor,
595 ) -> Result<PrefillOutput, Error> {
596 self.prefill_with_monitor(tokens, clear_first, Some(monitor))
597 }
598
599 fn prefill_with_monitor(
600 &mut self,
601 tokens: &[TokenId],
602 clear_first: bool,
603 mut monitor: Option<&mut PrefillMonitor>,
604 ) -> Result<PrefillOutput, Error> {
605 self.ensure_healthy()?;
606 if tokens.is_empty() {
607 return Err(Error::Invalid(
608 "prefill tokens must not be empty".to_owned(),
609 ));
610 }
611 self.model.validate_tokens(tokens)?;
612 let initial_position = if clear_first { 0 } else { self.position };
613 let requested_tokens = u64::try_from(tokens.len())
614 .map_err(|_| Error::Invalid("prefill token count overflowed".to_owned()))?;
615 let final_position = initial_position
616 .checked_add(requested_tokens)
617 .ok_or_else(|| Error::Invalid("prefill position overflowed".to_owned()))?;
618 if final_position > u64::from(self.options.context_size.get()) {
619 return Err(Error::Invalid(format!(
620 "prefill would exceed context size {}",
621 self.options.context_size
622 )));
623 }
624 let mut progress = PrefillProgress {
625 initial_position,
626 requested_tokens,
627 admitted_tokens: 0,
628 admitted_chunks: 0,
629 position: initial_position,
630 };
631 if let Some(active) = monitor.as_deref_mut() {
632 active.begin(progress)?;
633 }
634 if clear_first {
635 self.clear()?;
636 }
637
638 let chunk_limit = usize::try_from(self.options.batch_size)
639 .map_err(|_| Error::Invalid("batch size exceeds usize".to_owned()))?;
640 let isolate_last = self
641 .activation
642 .as_ref()
643 .is_some_and(ActivationController::has_last_prefill_capture);
644 let prefix_end = if isolate_last {
645 tokens.len().saturating_sub(1)
646 } else {
647 tokens.len()
648 };
649 let mut offset = 0_usize;
650 while offset < tokens.len() {
651 let end = if offset < prefix_end {
652 offset.saturating_add(chunk_limit).min(prefix_end)
653 } else {
654 tokens.len()
655 };
656 let chunk = &tokens[offset..end];
657 if let Some(active) = monitor.as_deref_mut()
658 && active.poll(progress)? == ControlFlow::Stop
659 {
660 let receipt = active.finish(PrefillFinish::Stopped)?;
661 return Ok(PrefillOutput {
662 admitted_tokens: progress.admitted_tokens,
663 position: self.position,
664 receipt: Some(receipt),
665 });
666 }
667 self.decode_tokens(
668 chunk,
669 ActivationPhaseV1::Prefill,
670 isolate_last && end == tokens.len(),
671 ActivationTelemetryDispositionV1::Admitted,
672 )?;
673 let chunk_tokens = u64::try_from(chunk.len())
674 .map_err(|_| Error::Invalid("prefill chunk size overflowed".to_owned()))?;
675 progress.admitted_tokens = progress
676 .admitted_tokens
677 .checked_add(chunk_tokens)
678 .ok_or_else(|| Error::Invalid("prefill token accounting overflowed".to_owned()))?;
679 progress.admitted_chunks = progress
680 .admitted_chunks
681 .checked_add(1)
682 .ok_or_else(|| Error::Invalid("prefill chunk accounting overflowed".to_owned()))?;
683 progress.position = self.position;
684 if let Some(active) = monitor.as_deref_mut()
685 && active.observe_chunk(progress)? == ControlFlow::Stop
686 && progress.admitted_tokens < progress.requested_tokens
687 {
688 let receipt = active.finish(PrefillFinish::Stopped)?;
689 return Ok(PrefillOutput {
690 admitted_tokens: progress.admitted_tokens,
691 position: self.position,
692 receipt: Some(receipt),
693 });
694 }
695 offset = end;
696 }
697 let receipt = monitor
698 .map(|active| active.finish(PrefillFinish::Complete))
699 .transpose()?;
700 Ok(PrefillOutput {
701 admitted_tokens: progress.admitted_tokens,
702 position: self.position,
703 receipt,
704 })
705 }
706
707 pub fn generate(
720 &mut self,
721 plan: &GenerationPlan,
722 pipeline: Option<&mut Pipeline>,
723 observers: Option<&mut ObserverSet>,
724 ) -> Result<GenerationOutput, Error> {
725 plan.validate()?;
726 self.ensure_generation_ready()?;
727 let plan_identity = plan.digest()?;
728 let sampler = build_sampler(&self.model.native, plan, &self.token_history)?;
729 self.generate_inner(
730 plan,
731 plan_identity,
732 sampler,
733 Vec::new(),
734 pipeline,
735 observers,
736 )
737 .map(|output| output.output)
738 }
739
740 pub fn generate_v2(
751 &mut self,
752 plan: &GenerationPlanV2,
753 pipeline: Option<&mut Pipeline>,
754 observers: Option<&mut ObserverSet>,
755 ) -> Result<GenerationOutput, Error> {
756 plan.validate()?;
757 self.ensure_generation_ready()?;
758 let plan_identity = plan.digest()?;
759 let sampler = build_sampler_v2(&self.model.native, plan, &self.token_history)?;
760 self.generate_inner(
761 &plan.generation,
762 plan_identity,
763 sampler,
764 Vec::new(),
765 pipeline,
766 observers,
767 )
768 .map(|output| output.output)
769 }
770
771 pub(crate) fn generate_stateful(
772 &mut self,
773 plan: &GenerationPlan,
774 continuation: Option<(LlamaSampler, Vec<u8>)>,
775 pipeline: Option<&mut Pipeline>,
776 observers: Option<&mut ObserverSet>,
777 ) -> Result<StatefulGenerationOutput, Error> {
778 plan.validate()?;
779 self.ensure_generation_ready()?;
780 let plan_identity = plan.digest()?;
781 let (sampler, stop_tail) = match continuation {
782 Some((sampler, stop_tail)) => {
783 validate_stop_tail(plan, &stop_tail)?;
784 (sampler, stop_tail)
785 }
786 None => (
787 build_sampler(&self.model.native, plan, &self.token_history)?,
788 Vec::new(),
789 ),
790 };
791 self.generate_inner(plan, plan_identity, sampler, stop_tail, pipeline, observers)
792 }
793
794 fn ensure_generation_ready(&self) -> Result<(), Error> {
795 self.ensure_healthy()?;
796 if self.token_history.is_empty() {
797 return Err(Error::Invalid(
798 "generation requires at least one prefilled token".to_owned(),
799 ));
800 }
801 Ok(())
802 }
803
804 fn generate_inner(
805 &mut self,
806 plan: &GenerationPlan,
807 plan_identity: Digest,
808 mut sampler: LlamaSampler,
809 mut stop_window: Vec<u8>,
810 mut pipeline: Option<&mut Pipeline>,
811 mut observers: Option<&mut ObserverSet>,
812 ) -> Result<StatefulGenerationOutput, Error> {
813 let initial_position = self.position;
814 if let Some(active) = pipeline.as_deref_mut() {
815 active.begin(&self.token_history)?;
816 }
817 if let Some(active) = observers.as_deref_mut() {
818 active.begin(initial_position, plan.max_tokens)?;
819 }
820 let mut output_tokens = Vec::new();
821 let mut output_bytes = Vec::new();
822 let mut finish = GenerationFinish::TokenLimit;
823
824 for step in 0..plan.max_tokens {
825 if let Some(active) = observers.as_deref_mut()
826 && active.poll()? == ControlFlow::Stop
827 {
828 finish = GenerationFinish::ObserverStop;
829 break;
830 }
831 if self.position >= u64::from(self.options.context_size.get()) {
832 return Err(Error::Invalid(
833 "generation reached the configured context bound".to_owned(),
834 ));
835 }
836 let token = self.sample_token(step, &mut sampler, pipeline.as_deref_mut())?;
837 if self.model.is_end_of_generation(token) {
838 finish = GenerationFinish::EndOfGeneration { token };
839 break;
840 }
841
842 let piece = self.model.token_piece(token)?;
843 self.decode_tokens(
844 std::slice::from_ref(&token),
845 ActivationPhaseV1::Generation,
846 false,
847 ActivationTelemetryDispositionV1::Admitted,
848 )?;
849 sampler.accept(LlamaToken::new(token.get()));
850 if let Some(active) = pipeline.as_deref_mut() {
851 active.accept(token)?;
852 }
853 output_tokens.push(token);
854 output_bytes.extend_from_slice(&piece);
855 extend_stop_window(plan, &mut stop_window, &piece);
856 let mut observer_stop = false;
857 if let Some(active) = observers.as_deref_mut()
858 && active.observe(ObservedToken {
859 token,
860 piece: &piece,
861 position: self.position,
862 })? == ControlFlow::Stop
863 {
864 observer_stop = true;
865 }
866 if observer_stop {
867 finish = GenerationFinish::ObserverStop;
868 break;
869 }
870 if let Some(index) = matching_stop(&plan.stops, &stop_window) {
871 let index = u32::try_from(index)
872 .map_err(|_| Error::Invalid("stop sequence index overflowed".to_owned()))?;
873 finish = GenerationFinish::StopSequence { index };
874 break;
875 }
876 }
877
878 let transform_receipt = pipeline
879 .as_deref()
880 .map(|active| active.receipt().digest())
881 .transpose()?;
882 let observer_receipts = observers
883 .as_deref()
884 .map_or_else(Vec::new, ObserverSet::receipts)
885 .into_iter()
886 .map(|receipt| receipt.digest())
887 .collect::<Result<Vec<_>, _>>()?;
888 let receipt = GenerationReceipt {
889 plan: plan_identity,
890 initial_position,
891 admitted_tokens: u32::try_from(output_tokens.len())
892 .map_err(|_| Error::Invalid("generated token accounting overflowed".to_owned()))?,
893 admitted_bytes: u64::try_from(output_bytes.len())
894 .map_err(|_| Error::Invalid("generated byte accounting overflowed".to_owned()))?,
895 final_position: self.position,
896 finish,
897 transform_receipt,
898 observer_receipts,
899 };
900 receipt.digest()?;
901 Ok(StatefulGenerationOutput {
902 output: GenerationOutput {
903 bytes: output_bytes,
904 tokens: output_tokens,
905 receipt,
906 },
907 sampler,
908 stop_tail: checkpoint_stop_tail(plan, &stop_window),
909 })
910 }
911
912 fn sample_token(
913 &mut self,
914 step: u32,
915 sampler: &mut LlamaSampler,
916 pipeline: Option<&mut Pipeline>,
917 ) -> Result<TokenId, Error> {
918 let mut logits = catch_unwind(AssertUnwindSafe(|| self.context.get_logits().to_vec()))
919 .map_err(|_| Error::Native("llama.cpp logits were unavailable".to_owned()))?;
920 if let Some(active) = pipeline {
921 active.apply_to_vocabulary(step, &self.token_history, &mut logits)?;
922 }
923 let data = logits
924 .into_iter()
925 .enumerate()
926 .map(|(index, logit)| {
927 let token = i32::try_from(index).map_err(|_| {
928 Error::Native("vocabulary token identifier overflowed".to_owned())
929 })?;
930 Ok(LlamaTokenData::new(LlamaToken::new(token), logit, 0.0))
931 })
932 .collect::<Result<Vec<_>, Error>>()?;
933 let mut candidates = LlamaTokenDataArray::new(data, false);
934 sampler.apply(&mut candidates);
935 let selected = candidates
936 .selected_token()
937 .ok_or_else(|| Error::Native("native sampler did not select a token".to_owned()))?;
938 TokenId::new(selected.0).map_err(Error::from)
939 }
940
941 pub fn capture_state(&mut self) -> Result<StateSnapshot, Error> {
947 self.ensure_healthy()?;
948 if self.activation.is_some() {
949 return Err(Error::Invalid(
950 "activation sessions require an activation-bound checkpoint envelope".to_owned(),
951 ));
952 }
953 if !self.active_steering.is_empty() {
954 return Err(Error::Invalid(
955 "clear active steering before capturing a checkpoint".to_owned(),
956 ));
957 }
958 self.capture_state_inner()
959 }
960
961 pub(crate) fn capture_envelope_state(&mut self) -> Result<StateSnapshot, Error> {
962 self.ensure_healthy()?;
963 if !self.active_steering.is_empty() {
964 return Err(Error::Invalid(
965 "clear active steering before capturing an aggregate checkpoint".to_owned(),
966 ));
967 }
968 if let Some(activation) = self.activation.as_mut() {
969 activation.reset()?;
970 }
971 self.capture_state_inner()
972 }
973
974 fn capture_state_inner(&mut self) -> Result<StateSnapshot, Error> {
975 let size = self.context.state_get_size();
976 if size == 0 {
977 return Err(Error::Native(
978 "llama.cpp returned a zero-sized state".to_owned(),
979 ));
980 }
981 let mut bytes = vec![0_u8; size];
982 let written = self.context.state_get_data(&mut bytes);
983 if written == 0 || written > bytes.len() {
984 return Err(Error::Native(
985 "llama.cpp returned invalid state accounting".to_owned(),
986 ));
987 }
988 bytes.truncate(written);
989 let receipt = CheckpointReceipt {
990 model: self.model.artifact_digest().clone(),
991 backend: self.backend.clone(),
992 state: Digest::of_bytes(STATE_BYTES_DOMAIN, &bytes),
993 state_bytes: u64::try_from(bytes.len())
994 .map_err(|_| Error::Native("native state size exceeds u64".to_owned()))?,
995 token_history: Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &self.token_history)?,
996 position: self.position,
997 };
998 StateSnapshot::from_parts(bytes, self.token_history.clone(), receipt)
999 }
1000
1001 pub fn restore_state(&mut self, snapshot: &StateSnapshot) -> Result<(), Error> {
1008 self.ensure_healthy()?;
1009 if self.activation.is_some() {
1010 return Err(Error::Invalid(
1011 "activation sessions require an activation-bound checkpoint envelope".to_owned(),
1012 ));
1013 }
1014 if !self.active_steering.is_empty() {
1015 return Err(Error::Invalid(
1016 "clear active steering before restoring a checkpoint".to_owned(),
1017 ));
1018 }
1019 self.restore_state_inner(snapshot)?;
1020 if let Some(last_token) = snapshot.tokens.last().copied()
1021 && let Err(error) =
1022 self.refresh_logits_after_restore(last_token, snapshot.receipt.position, None)
1023 {
1024 self.record_poison(&error);
1025 return Err(error);
1026 }
1027 Ok(())
1028 }
1029
1030 pub(crate) fn restore_envelope_state(
1031 &mut self,
1032 snapshot: &StateSnapshot,
1033 refresh_phase: Option<ActivationPhaseV1>,
1034 ) -> Result<(), Error> {
1035 self.ensure_healthy()?;
1036 if !self.active_steering.is_empty() {
1037 return Err(Error::Invalid(
1038 "clear active steering before restoring an aggregate checkpoint".to_owned(),
1039 ));
1040 }
1041 self.restore_state_inner(snapshot)?;
1042 if let Some(last_token) = snapshot.tokens.last().copied()
1043 && let Err(error) = self.refresh_logits_after_restore(
1044 last_token,
1045 snapshot.receipt.position,
1046 refresh_phase,
1047 )
1048 {
1049 self.record_poison(&error);
1050 return Err(error);
1051 }
1052 if let Some(activation) = self.activation.as_mut() {
1053 activation.reset()?;
1054 }
1055 Ok(())
1056 }
1057
1058 pub(crate) fn refresh_restored_logits_with_active_steering(
1059 &mut self,
1060 activation_phase: Option<ActivationPhaseV1>,
1061 ) -> Result<(), Error> {
1062 self.ensure_healthy()?;
1063 if self.active_steering.is_empty() {
1064 return Ok(());
1065 }
1066 let Some(last_token) = self.token_history.last().copied() else {
1067 return Err(Error::Incompatible(
1068 "an actively steered checkpoint must contain a causal token".to_owned(),
1069 ));
1070 };
1071 if let Err(error) =
1072 self.refresh_logits_after_restore(last_token, self.position, activation_phase)
1073 {
1074 self.record_poison(&error);
1075 return Err(error);
1076 }
1077 if let Some(activation) = self.activation.as_mut()
1078 && let Err(error) = activation.reset()
1079 {
1080 self.record_poison(&error);
1081 return Err(error);
1082 }
1083 Ok(())
1084 }
1085
1086 fn restore_state_inner(&mut self, snapshot: &StateSnapshot) -> Result<(), Error> {
1087 snapshot.validate_contents()?;
1088 if snapshot.receipt.model != *self.model.artifact_digest()
1089 || snapshot.receipt.backend != self.backend
1090 {
1091 return Err(Error::Incompatible(
1092 "checkpoint model or backend identity does not match".to_owned(),
1093 ));
1094 }
1095 if snapshot.receipt.position > u64::from(self.options.context_size.get()) {
1096 return Err(Error::Incompatible(
1097 "checkpoint position exceeds the destination context".to_owned(),
1098 ));
1099 }
1100 self.model.validate_tokens(&snapshot.tokens)?;
1101 let read = self.context.state_set_data(&snapshot.bytes);
1102 if read != snapshot.bytes.len() {
1103 let error = Error::Native(format!(
1104 "llama.cpp restored {read} of {} state bytes",
1105 snapshot.bytes.len()
1106 ));
1107 self.record_poison(&error);
1108 return Err(error);
1109 }
1110 self.token_history.clone_from(&snapshot.tokens);
1111 self.position = snapshot.receipt.position;
1112 Ok(())
1113 }
1114
1115 fn refresh_logits_after_restore(
1116 &mut self,
1117 last_token: TokenId,
1118 position: u64,
1119 activation_phase: Option<ActivationPhaseV1>,
1120 ) -> Result<(), Error> {
1121 let Some((last_position, end_position)) = checkpoint_logit_refresh_range(position)? else {
1122 return Ok(());
1123 };
1124 let removed = self
1125 .context
1126 .clear_kv_cache_seq(Some(0), Some(last_position), Some(end_position))
1127 .map_err(native)?;
1128 if !removed {
1129 return Err(Error::Incompatible(
1130 "native memory cannot remove the final checkpoint position to refresh logits"
1131 .to_owned(),
1132 ));
1133 }
1134
1135 let native_position = i32::try_from(last_position)
1136 .map_err(|_| Error::Incompatible("checkpoint position exceeds i32".to_owned()))?;
1137 let mut batch = LlamaBatch::new(1, 1);
1138 batch
1139 .add(
1140 LlamaToken::new(last_token.get()),
1141 native_position,
1142 &[0],
1143 true,
1144 )
1145 .map_err(native)?;
1146 if let (Some(activation), Some(phase)) = (self.activation.as_ref(), activation_phase) {
1147 activation.begin_decode(phase, false, ActivationTelemetryDispositionV1::Admitted)?;
1148 }
1149 let decode_result = self.context.decode(&mut batch);
1150 let callback_state = match (self.activation.as_ref(), activation_phase) {
1151 (Some(activation), Some(_)) => Some(activation.end_decode()),
1152 _ => None,
1153 };
1154 if let Err(error) = decode_result {
1155 return Err(native(error));
1156 }
1157 let active_decode = match callback_state {
1158 Some(result) => Some(result?),
1159 None => None,
1160 };
1161 if let Some(active_decode) = active_decode {
1162 let captures = self
1163 .context
1164 .tensor_transactions_mut()
1165 .ok_or_else(|| {
1166 Error::Poisoned(
1167 "activation context lost its tensor transaction state".to_owned(),
1168 )
1169 })?
1170 .take_captures();
1171 self.activation
1172 .as_mut()
1173 .ok_or_else(|| {
1174 Error::Poisoned("activation controller became unavailable".to_owned())
1175 })?
1176 .consume_captures(captures, active_decode)?;
1177 }
1178 Ok(())
1179 }
1180
1181 pub(crate) fn decode_tokens(
1182 &mut self,
1183 tokens: &[TokenId],
1184 phase: ActivationPhaseV1,
1185 last_prefill: bool,
1186 disposition: ActivationTelemetryDispositionV1,
1187 ) -> Result<(), Error> {
1188 self.model.validate_tokens(tokens)?;
1189 let token_count = u64::try_from(tokens.len())
1190 .map_err(|_| Error::Invalid("native batch token count overflowed".to_owned()))?;
1191 let mut batch = LlamaBatch::new(tokens.len(), 1);
1192 for (offset, token) in tokens.iter().enumerate() {
1193 let offset = u64::try_from(offset)
1194 .map_err(|_| Error::Invalid("native batch offset overflowed".to_owned()))?;
1195 let position = self
1196 .position
1197 .checked_add(offset)
1198 .ok_or_else(|| Error::Invalid("native position overflowed".to_owned()))?;
1199 let position = i32::try_from(position)
1200 .map_err(|_| Error::Invalid("native position exceeds i32".to_owned()))?;
1201 batch
1202 .add(
1203 LlamaToken::new(token.get()),
1204 position,
1205 &[0],
1206 offset + 1 == token_count,
1207 )
1208 .map_err(native)?;
1209 }
1210 if let Some(activation) = self.activation.as_ref() {
1211 activation.begin_decode(phase, last_prefill, disposition)?;
1212 }
1213 let decode_result = self.context.decode(&mut batch);
1214 let callback_state = self
1215 .activation
1216 .as_ref()
1217 .map(ActivationController::end_decode);
1218 if let Err(error) = decode_result {
1219 let error = native(error);
1220 if self.activation.is_some() {
1221 self.record_poison(&error);
1222 }
1223 return Err(error);
1224 }
1225 self.token_history.extend_from_slice(tokens);
1226 self.position = self
1227 .position
1228 .checked_add(token_count)
1229 .ok_or_else(|| Error::Invalid("causal position overflowed".to_owned()))?;
1230 let active_decode = match callback_state {
1231 Some(Ok(active)) => Some(active),
1232 Some(Err(error)) => {
1233 self.record_poison(&error);
1234 return Err(error);
1235 }
1236 None => None,
1237 };
1238 if let Some(active_decode) = active_decode {
1239 let captures = self
1240 .context
1241 .tensor_transactions_mut()
1242 .ok_or_else(|| {
1243 Error::Poisoned(
1244 "activation context lost its tensor transaction state".to_owned(),
1245 )
1246 })?
1247 .take_captures();
1248 let result = self
1249 .activation
1250 .as_mut()
1251 .ok_or_else(|| {
1252 Error::Poisoned("activation controller became unavailable".to_owned())
1253 })?
1254 .consume_captures(captures, active_decode);
1255 if let Err(error) = result {
1256 self.record_poison(&error);
1257 return Err(error);
1258 }
1259 }
1260 Ok(())
1261 }
1262
1263 pub(crate) fn ensure_steering_available(&self) -> Result<(), Error> {
1264 self.ensure_healthy()?;
1265 if !self.active_steering.is_empty() {
1266 return Err(Error::Invalid(
1267 "clear active steering before applying another steering scope".to_owned(),
1268 ));
1269 }
1270 Ok(())
1271 }
1272
1273 pub(crate) fn mark_steering_active(&mut self, kind: SteeringKind) {
1274 self.active_steering.push(kind);
1275 }
1276
1277 pub(crate) fn mark_steering_stack_active(&mut self, kinds: Vec<SteeringKind>) {
1278 debug_assert!(!kinds.is_empty());
1279 debug_assert!(self.active_steering.is_empty());
1280 self.active_steering = kinds;
1281 }
1282
1283 pub(crate) fn mark_steering_cleared(&mut self) {
1284 self.active_steering.clear();
1285 }
1286
1287 pub(crate) fn record_cleanup_failure(&mut self, error: &Error) {
1288 self.record_poison(error);
1289 }
1290
1291 pub(crate) fn record_poison(&mut self, error: &Error) {
1292 self.poison_reason.get_or_insert_with(|| error.to_string());
1293 }
1294
1295 pub(crate) fn ensure_healthy(&self) -> Result<(), Error> {
1296 if let Some(reason) = &self.poison_reason {
1297 return Err(Error::Poisoned(reason.clone()));
1298 }
1299 Ok(())
1300 }
1301}
1302
1303fn matching_stop(stops: &[Vec<u8>], output: &[u8]) -> Option<usize> {
1304 stops.iter().position(|stop| output.ends_with(stop))
1305}
1306
1307fn validate_stop_tail(plan: &GenerationPlan, tail: &[u8]) -> Result<(), Error> {
1308 let maximum = maximum_stop_bytes(plan).saturating_sub(1);
1309 if tail.len() > maximum {
1310 return Err(Error::Incompatible(
1311 "ordinary checkpoint stop-prefix state exceeds the generation plan".to_owned(),
1312 ));
1313 }
1314 Ok(())
1315}
1316
1317fn extend_stop_window(plan: &GenerationPlan, window: &mut Vec<u8>, piece: &[u8]) {
1318 let maximum = maximum_stop_bytes(plan);
1319 if maximum == 0 {
1320 window.clear();
1321 return;
1322 }
1323 if piece.len() >= maximum {
1324 window.clear();
1325 window.extend_from_slice(&piece[piece.len() - maximum..]);
1326 return;
1327 }
1328 window.extend_from_slice(piece);
1329 if window.len() > maximum {
1330 window.drain(..window.len() - maximum);
1331 }
1332}
1333
1334fn checkpoint_stop_tail(plan: &GenerationPlan, window: &[u8]) -> Vec<u8> {
1335 let maximum = maximum_stop_bytes(plan).saturating_sub(1);
1336 window[window.len().saturating_sub(maximum)..].to_vec()
1337}
1338
1339fn maximum_stop_bytes(plan: &GenerationPlan) -> usize {
1340 plan.stops.iter().map(Vec::len).max().unwrap_or(0)
1341}
1342
1343fn checkpoint_logit_refresh_range(position: u64) -> Result<Option<(u32, u32)>, Error> {
1344 let Some(last_position) = position.checked_sub(1) else {
1345 return Ok(None);
1346 };
1347 let last_position = u32::try_from(last_position)
1348 .map_err(|_| Error::Incompatible("checkpoint position exceeds u32".to_owned()))?;
1349 let end_position = last_position
1350 .checked_add(1)
1351 .ok_or_else(|| Error::Incompatible("checkpoint position overflowed".to_owned()))?;
1352 Ok(Some((last_position, end_position)))
1353}
1354
1355fn session_compatibility_identity(
1356 runtime: &Digest,
1357 options: SessionOptions,
1358 context_type: LlamaContextType,
1359 recurrent_state_slots: u32,
1360) -> Digest {
1361 let mut bytes = Vec::with_capacity(runtime.as_str().len() + 24);
1362 bytes.extend_from_slice(runtime.as_str().as_bytes());
1363 bytes.extend_from_slice(&options.context_size.get().to_le_bytes());
1364 bytes.extend_from_slice(&options.batch_size.to_le_bytes());
1365 bytes.extend_from_slice(&options.micro_batch_size.to_le_bytes());
1366 bytes.extend_from_slice(&options.threads.to_le_bytes());
1367 let context_type = match context_type {
1368 LlamaContextType::Default => 0_u32,
1369 LlamaContextType::Mtp => 1_u32,
1370 };
1371 bytes.extend_from_slice(&context_type.to_le_bytes());
1372 bytes.extend_from_slice(&recurrent_state_slots.to_le_bytes());
1373 Digest::of_bytes("llamacpp-session-compatibility-v3", &bytes)
1374}