Skip to main content

logit_loom_llamacpp/
session.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! Single-owner causal sessions, generation, and checkpoints.
4
5use std::marker::PhantomData;
6use std::num::NonZeroU32;
7use std::panic::{AssertUnwindSafe, catch_unwind};
8use std::rc::Rc;
9
10use llama_cpp_4::context::LlamaContext;
11use llama_cpp_4::context::params::{LlamaContextParams, LlamaContextType};
12use llama_cpp_4::llama_batch::LlamaBatch;
13use llama_cpp_4::sampling::LlamaSampler;
14use llama_cpp_4::token::LlamaToken;
15use llama_cpp_4::token::data::LlamaTokenData;
16use llama_cpp_4::token::data_array::LlamaTokenDataArray;
17use logit_loom::{
18    ActivationPhaseV1, ActivationTelemetryDispositionV1, CheckpointReceipt, ControlFlow, Digest,
19    GenerationFinish, GenerationPlan, GenerationPlanV2, GenerationReceipt, ObservedToken,
20    ObserverSet, Pipeline, PrefillFinish, PrefillMonitor, PrefillProgress, PrefillReceipt,
21    SteeringKind, TokenId,
22};
23
24use crate::{
25    ActivationCaptureOutput, ActivationConfiguration, ActivationProgramOutput, Error, Model,
26    Runtime,
27    activation::ActivationController,
28    error::native,
29    sampler::{build_sampler, build_sampler_v2},
30};
31
32const STATE_BYTES_DOMAIN: &str = "llamacpp-state-bytes-v1";
33const TOKEN_HISTORY_DOMAIN: &str = "llamacpp-state-token-history-v1";
34
35/// Context-allocation options.
36#[derive(Clone, Copy, Debug, PartialEq, Eq)]
37pub struct SessionOptions {
38    /// Maximum causal context size.
39    pub context_size: NonZeroU32,
40    /// Maximum logical tokens per decode batch.
41    pub batch_size: u32,
42    /// Maximum physical tokens per decode micro-batch.
43    pub micro_batch_size: u32,
44    /// Host orchestration threads used by llama.cpp.
45    pub threads: i32,
46}
47
48impl Default for SessionOptions {
49    fn default() -> Self {
50        Self {
51            context_size: NonZeroU32::new(4_096).expect("nonzero constant"),
52            batch_size: 512,
53            micro_batch_size: 512,
54            threads: 4,
55        }
56    }
57}
58
59impl SessionOptions {
60    fn validate(self) -> Result<(), Error> {
61        let maximum_native_context = u32::try_from(i32::MAX).unwrap_or(u32::MAX);
62        if self.batch_size == 0
63            || self.micro_batch_size == 0
64            || self.micro_batch_size > self.batch_size
65            || self.batch_size > self.context_size.get()
66            || self.context_size.get() > maximum_native_context
67            || self.threads <= 0
68        {
69            return Err(Error::Invalid(
70                "session requires 0 < micro_batch <= batch <= context and positive threads"
71                    .to_owned(),
72            ));
73        }
74        Ok(())
75    }
76}
77
78#[cfg(test)]
79mod tests {
80    use super::*;
81
82    #[test]
83    fn session_options_reject_native_position_overflow() {
84        let options = SessionOptions {
85            context_size: NonZeroU32::new(u32::MAX).unwrap(),
86            ..SessionOptions::default()
87        };
88        assert!(options.validate().is_err());
89    }
90
91    #[test]
92    fn session_options_enforce_batch_and_thread_bounds() {
93        let defaults = SessionOptions::default();
94        assert!(defaults.validate().is_ok());
95        assert!(
96            SessionOptions {
97                micro_batch_size: defaults.batch_size + 1,
98                ..defaults
99            }
100            .validate()
101            .is_err()
102        );
103        assert!(
104            SessionOptions {
105                threads: 0,
106                ..defaults
107            }
108            .validate()
109            .is_err()
110        );
111    }
112
113    #[test]
114    fn checkpoint_compatibility_binds_session_options() {
115        let runtime = Digest::of_bytes("test-runtime", b"one");
116        let defaults = SessionOptions::default();
117        let different_context = SessionOptions {
118            context_size: NonZeroU32::new(defaults.context_size.get() + 1).unwrap(),
119            ..defaults
120        };
121        let different_threads = SessionOptions {
122            threads: defaults.threads + 1,
123            ..defaults
124        };
125        let identity =
126            session_compatibility_identity(&runtime, defaults, LlamaContextType::Default, 0);
127        assert_ne!(
128            identity,
129            session_compatibility_identity(
130                &runtime,
131                different_context,
132                LlamaContextType::Default,
133                0
134            )
135        );
136        assert_ne!(
137            identity,
138            session_compatibility_identity(
139                &runtime,
140                different_threads,
141                LlamaContextType::Default,
142                0
143            )
144        );
145        assert_ne!(
146            identity,
147            session_compatibility_identity(&runtime, defaults, LlamaContextType::Mtp, 4)
148        );
149    }
150
151    #[test]
152    fn generation_output_preserves_arbitrary_bytes() {
153        let output = GenerationOutput {
154            bytes: vec![0xff],
155            tokens: vec![TokenId::new(1).unwrap()],
156            receipt: GenerationReceipt {
157                plan: Digest::of_bytes("test-plan", b"bytes"),
158                initial_position: 0,
159                admitted_tokens: 1,
160                admitted_bytes: 1,
161                final_position: 1,
162                finish: GenerationFinish::TokenLimit,
163                transform_receipt: None,
164                observer_receipts: Vec::new(),
165            },
166        };
167        assert!(output.text().is_err());
168        assert_eq!(output.bytes, [0xff]);
169    }
170
171    #[test]
172    fn checkpoint_parts_validate_before_native_restore() {
173        let bytes = vec![1, 2, 3];
174        let tokens = vec![TokenId::new(4).unwrap()];
175        let receipt = CheckpointReceipt {
176            model: Digest::of_bytes("test-model", b"one"),
177            backend: Digest::of_bytes("test-backend", b"one"),
178            state: Digest::of_bytes(STATE_BYTES_DOMAIN, &bytes),
179            state_bytes: 3,
180            token_history: Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &tokens).unwrap(),
181            position: 1,
182        };
183        let snapshot =
184            StateSnapshot::from_parts(bytes.clone(), tokens.clone(), receipt.clone()).unwrap();
185        assert_eq!(snapshot.bytes(), bytes);
186        assert_eq!(snapshot.tokens(), tokens);
187        assert_eq!(snapshot.receipt(), &receipt);
188        assert_eq!(snapshot.into_parts(), (bytes, tokens, receipt));
189    }
190
191    #[test]
192    fn checkpoint_parts_reject_tampered_accounting() {
193        let bytes = vec![1, 2, 3];
194        let tokens = vec![TokenId::new(4).unwrap()];
195        let mut receipt = CheckpointReceipt {
196            model: Digest::of_bytes("test-model", b"one"),
197            backend: Digest::of_bytes("test-backend", b"one"),
198            state: Digest::of_bytes(STATE_BYTES_DOMAIN, &bytes),
199            state_bytes: 4,
200            token_history: Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &tokens).unwrap(),
201            position: 1,
202        };
203        assert!(StateSnapshot::from_parts(bytes.clone(), tokens.clone(), receipt.clone()).is_err());
204
205        receipt.state_bytes = 3;
206        receipt.position = 2;
207        assert!(StateSnapshot::from_parts(bytes, tokens, receipt).is_err());
208    }
209
210    #[test]
211    fn overlapping_stops_choose_the_lowest_declared_index() {
212        let stops = vec![b"bc".to_vec(), b"abc".to_vec(), b"c".to_vec()];
213        assert_eq!(matching_stop(&stops, b"zabc"), Some(0));
214        assert_eq!(matching_stop(&stops, b"zab"), None);
215    }
216
217    #[test]
218    fn ordinary_checkpoint_stop_tail_preserves_cross_operation_matches() {
219        let plan = GenerationPlan {
220            sampling: logit_loom::SamplingPlan::default(),
221            max_tokens: 4,
222            biases: Vec::new(),
223            grammar: None,
224            stops: vec![b"abcd".to_vec()],
225        };
226        let mut window = b"ab".to_vec();
227        extend_stop_window(&plan, &mut window, b"cd");
228        assert_eq!(matching_stop(&plan.stops, &window), Some(0));
229        assert_eq!(checkpoint_stop_tail(&plan, b"zabc"), b"abc");
230        assert!(validate_stop_tail(&plan, b"abc").is_ok());
231        assert!(validate_stop_tail(&plan, b"abcd").is_err());
232    }
233
234    #[test]
235    fn checkpoint_logit_refresh_targets_only_the_last_position() {
236        assert_eq!(checkpoint_logit_refresh_range(0).unwrap(), None);
237        assert_eq!(checkpoint_logit_refresh_range(1).unwrap(), Some((0, 1)));
238        assert_eq!(
239            checkpoint_logit_refresh_range(i32::MAX as u64).unwrap(),
240            Some((i32::MAX as u32 - 1, i32::MAX as u32))
241        );
242        assert!(checkpoint_logit_refresh_range(u64::MAX).is_err());
243    }
244}
245
246/// Result of one plain or controlled prefill call.
247#[derive(Clone, Debug, PartialEq, Eq)]
248pub struct PrefillOutput {
249    /// Tokens admitted by this call.
250    pub admitted_tokens: u64,
251    /// Final causal position.
252    pub position: u64,
253    /// Controlled-prefill accounting when an observer was supplied.
254    pub receipt: Option<PrefillReceipt>,
255}
256
257/// Successful bounded generation output.
258#[derive(Clone, Debug, PartialEq, Eq)]
259pub struct GenerationOutput {
260    /// Exact, potentially non-UTF-8 generated bytes.
261    pub bytes: Vec<u8>,
262    /// Causally admitted generated token IDs.
263    pub tokens: Vec<TokenId>,
264    /// Complete mechanical receipt.
265    pub receipt: GenerationReceipt,
266}
267
268pub(crate) struct StatefulGenerationOutput {
269    pub(crate) output: GenerationOutput,
270    pub(crate) sampler: LlamaSampler,
271    pub(crate) stop_tail: Vec<u8>,
272}
273
274impl GenerationOutput {
275    /// Returns generated bytes as UTF-8 when the complete output is valid.
276    ///
277    /// # Errors
278    ///
279    /// Returns the standard UTF-8 error for arbitrary model bytes.
280    pub fn text(&self) -> Result<&str, std::str::Utf8Error> {
281        std::str::from_utf8(&self.bytes)
282    }
283}
284
285/// Opaque llama.cpp causal state with compatibility-bound identity metadata.
286#[derive(Clone, Debug)]
287pub struct StateSnapshot {
288    bytes: Vec<u8>,
289    tokens: Vec<TokenId>,
290    receipt: CheckpointReceipt,
291}
292
293impl StateSnapshot {
294    /// Reconstructs a checkpoint from caller-owned storage.
295    ///
296    /// This validates internal byte count, state identity, token lineage, and
297    /// position. [`Session::restore_state`] separately verifies the model,
298    /// backend build, and session allocation identities before passing bytes to
299    /// llama.cpp.
300    ///
301    /// # Errors
302    ///
303    /// Returns an error for empty state bytes or inconsistent metadata.
304    pub fn from_parts(
305        bytes: Vec<u8>,
306        tokens: Vec<TokenId>,
307        receipt: CheckpointReceipt,
308    ) -> Result<Self, Error> {
309        let snapshot = Self {
310            bytes,
311            tokens,
312            receipt,
313        };
314        snapshot.validate_contents()?;
315        Ok(snapshot)
316    }
317
318    /// Returns opaque native state bytes covered by the checkpoint receipt.
319    pub fn bytes(&self) -> &[u8] {
320        &self.bytes
321    }
322
323    /// Returns the exact causal tokens represented by the state.
324    pub fn tokens(&self) -> &[TokenId] {
325        &self.tokens
326    }
327
328    /// Returns serializable, compatibility-bound checkpoint metadata.
329    pub const fn receipt(&self) -> &CheckpointReceipt {
330        &self.receipt
331    }
332
333    /// Returns all caller-owned parts for application-defined persistence.
334    pub fn into_parts(self) -> (Vec<u8>, Vec<TokenId>, CheckpointReceipt) {
335        (self.bytes, self.tokens, self.receipt)
336    }
337
338    pub(crate) fn validate_contents(&self) -> Result<(), Error> {
339        if self.bytes.is_empty() {
340            return Err(Error::Incompatible(
341                "checkpoint state bytes must not be empty".to_owned(),
342            ));
343        }
344        let state_bytes = u64::try_from(self.bytes.len())
345            .map_err(|_| Error::Incompatible("checkpoint state size exceeds u64".to_owned()))?;
346        let token_count = u64::try_from(self.tokens.len())
347            .map_err(|_| Error::Incompatible("checkpoint token count exceeds u64".to_owned()))?;
348        if self.receipt.state != Digest::of_bytes(STATE_BYTES_DOMAIN, &self.bytes)
349            || self.receipt.state_bytes != state_bytes
350            || self.receipt.token_history
351                != Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &self.tokens)?
352            || self.receipt.position != token_count
353        {
354            return Err(Error::Incompatible(
355                "checkpoint metadata does not match state bytes or token lineage".to_owned(),
356            ));
357        }
358        self.receipt.digest()?;
359        Ok(())
360    }
361}
362
363/// Single-owner llama.cpp context with explicit causal lineage.
364///
365/// `Session` is deliberately neither `Send` nor `Sync`. Create it inside the
366/// thread that will own all mutation.
367pub struct Session<'model> {
368    pub(crate) context: LlamaContext<'model>,
369    pub(crate) model: &'model Model,
370    pub(crate) options: SessionOptions,
371    pub(crate) backend: Digest,
372    pub(crate) token_history: Vec<TokenId>,
373    pub(crate) position: u64,
374    pub(crate) active_steering: Vec<SteeringKind>,
375    pub(crate) activation: Option<ActivationController>,
376    pub(crate) poison_reason: Option<String>,
377    thread_affinity: PhantomData<Rc<()>>,
378}
379
380impl std::fmt::Debug for Session<'_> {
381    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
382        formatter
383            .debug_struct("Session")
384            .field("options", &self.options)
385            .field("position", &self.position)
386            .field("tokens", &self.token_history.len())
387            .field("active_steering", &self.active_steering)
388            .field("has_activation", &self.activation.is_some())
389            .field("healthy", &self.poison_reason.is_none())
390            .finish_non_exhaustive()
391    }
392}
393
394impl<'model> Session<'model> {
395    pub(crate) fn new(
396        model: &'model Model,
397        runtime: &Runtime,
398        options: SessionOptions,
399    ) -> Result<Self, Error> {
400        Self::new_inner(model, runtime, options, LlamaContextType::Default, 0, None)
401    }
402
403    pub(crate) fn new_with_activation(
404        model: &'model Model,
405        runtime: &Runtime,
406        options: SessionOptions,
407        activation: ActivationConfiguration,
408    ) -> Result<Self, Error> {
409        Self::new_inner(
410            model,
411            runtime,
412            options,
413            LlamaContextType::Default,
414            0,
415            Some(activation),
416        )
417    }
418
419    pub(crate) fn new_speculative(
420        model: &'model Model,
421        runtime: &Runtime,
422        options: SessionOptions,
423        context_type: LlamaContextType,
424        recurrent_state_slots: u32,
425        activation: Option<ActivationConfiguration>,
426    ) -> Result<Self, Error> {
427        Self::new_inner(
428            model,
429            runtime,
430            options,
431            context_type,
432            recurrent_state_slots,
433            activation,
434        )
435    }
436
437    fn new_inner(
438        model: &'model Model,
439        runtime: &Runtime,
440        options: SessionOptions,
441        context_type: LlamaContextType,
442        recurrent_state_slots: u32,
443        activation: Option<ActivationConfiguration>,
444    ) -> Result<Self, Error> {
445        options.validate()?;
446        let (activation, transactions) = activation
447            .map(|configuration| configuration.lower(options.batch_size))
448            .transpose()?
449            .map_or((None, None), |(controller, transactions)| {
450                (Some(controller), Some(transactions))
451            });
452        let mut params = LlamaContextParams::default()
453            .with_n_ctx(Some(options.context_size))
454            .with_n_batch(options.batch_size)
455            .with_n_ubatch(options.micro_batch_size)
456            .with_n_threads(options.threads)
457            .with_n_threads_batch(options.threads)
458            .with_ctx_type(context_type)
459            .with_n_rs_seq(recurrent_state_slots)
460            .with_offload_kqv(true);
461        if let Some(transactions) = transactions {
462            params = params.with_tensor_transactions(transactions);
463        }
464        let context = model
465            .native
466            .new_context(&runtime.native, params)
467            .map_err(native)?;
468        Ok(Self {
469            context,
470            model,
471            options,
472            backend: session_compatibility_identity(
473                runtime.identity(),
474                options,
475                context_type,
476                recurrent_state_slots,
477            ),
478            token_history: Vec::new(),
479            position: 0,
480            active_steering: Vec::new(),
481            activation,
482            poison_reason: None,
483            thread_affinity: PhantomData,
484        })
485    }
486
487    /// Returns the context-allocation options.
488    pub const fn options(&self) -> SessionOptions {
489        self.options
490    }
491
492    /// Returns the current causal position.
493    pub const fn position(&self) -> u64 {
494        self.position
495    }
496
497    /// Returns exact admitted token history.
498    pub fn token_history(&self) -> &[TokenId] {
499        &self.token_history
500    }
501
502    /// Returns whether no operation has left native state uncertain.
503    pub const fn is_healthy(&self) -> bool {
504        self.poison_reason.is_none()
505    }
506
507    /// Returns the retained reason native state became uncertain, if any.
508    pub fn poison_reason(&self) -> Option<&str> {
509        self.poison_reason.as_deref()
510    }
511
512    /// Returns the ordered session-scoped steering resources currently active.
513    pub fn active_steering(&self) -> &[SteeringKind] {
514        &self.active_steering
515    }
516
517    /// Returns whether this session owns a transactional activation runtime.
518    pub const fn has_activation_runtime(&self) -> bool {
519        self.activation.is_some()
520    }
521
522    /// Drains captured activation records and aggregate per-plan receipts.
523    ///
524    /// # Errors
525    ///
526    /// Returns an error when no activation runtime is installed or callback
527    /// evidence is unavailable.
528    pub fn take_activation_captures(&mut self) -> Result<ActivationCaptureOutput, Error> {
529        self.ensure_healthy()?;
530        self.activation
531            .as_mut()
532            .ok_or_else(|| Error::Invalid("session has no activation runtime".to_owned()))?
533            .take_capture_output()
534    }
535
536    /// Drains successful activation write-back records and aggregate program
537    /// accounting.
538    ///
539    /// # Errors
540    ///
541    /// Returns an error when no activation runtime is installed or callback
542    /// evidence is unavailable.
543    pub fn take_activation_program_output(&mut self) -> Result<ActivationProgramOutput, Error> {
544        self.ensure_healthy()?;
545        self.activation
546            .as_ref()
547            .ok_or_else(|| Error::Invalid("session has no activation runtime".to_owned()))?
548            .take_program_output()
549    }
550
551    /// Clears native causal memory and local lineage.
552    ///
553    /// # Errors
554    ///
555    /// Returns the retained reason when the session is poisoned.
556    pub fn clear(&mut self) -> Result<(), Error> {
557        self.ensure_healthy()?;
558        if let Some(activation) = self.activation.as_mut() {
559            activation.reset()?;
560        }
561        self.context.clear_kv_cache();
562        self.token_history.clear();
563        self.position = 0;
564        Ok(())
565    }
566
567    /// Admits exact tokens in bounded native batches.
568    ///
569    /// # Errors
570    ///
571    /// Returns an error for an empty or overflowing request or native decode
572    /// failure. Complete chunks admitted before a native failure remain causal.
573    pub fn prefill(
574        &mut self,
575        tokens: &[TokenId],
576        clear_first: bool,
577    ) -> Result<PrefillOutput, Error> {
578        self.prefill_with_monitor(tokens, clear_first, None)
579    }
580
581    /// Admits exact tokens with polling and observation at complete chunks.
582    ///
583    /// Cooperative stop retains every previously completed chunk. Callback
584    /// failure likewise leaves already admitted native work visible so callers
585    /// can inspect or checkpoint the exact prefix.
586    ///
587    /// # Errors
588    ///
589    /// Returns a validation, observer, or native decode error.
590    pub fn prefill_observed(
591        &mut self,
592        tokens: &[TokenId],
593        clear_first: bool,
594        monitor: &mut PrefillMonitor,
595    ) -> Result<PrefillOutput, Error> {
596        self.prefill_with_monitor(tokens, clear_first, Some(monitor))
597    }
598
599    fn prefill_with_monitor(
600        &mut self,
601        tokens: &[TokenId],
602        clear_first: bool,
603        mut monitor: Option<&mut PrefillMonitor>,
604    ) -> Result<PrefillOutput, Error> {
605        self.ensure_healthy()?;
606        if tokens.is_empty() {
607            return Err(Error::Invalid(
608                "prefill tokens must not be empty".to_owned(),
609            ));
610        }
611        self.model.validate_tokens(tokens)?;
612        let initial_position = if clear_first { 0 } else { self.position };
613        let requested_tokens = u64::try_from(tokens.len())
614            .map_err(|_| Error::Invalid("prefill token count overflowed".to_owned()))?;
615        let final_position = initial_position
616            .checked_add(requested_tokens)
617            .ok_or_else(|| Error::Invalid("prefill position overflowed".to_owned()))?;
618        if final_position > u64::from(self.options.context_size.get()) {
619            return Err(Error::Invalid(format!(
620                "prefill would exceed context size {}",
621                self.options.context_size
622            )));
623        }
624        let mut progress = PrefillProgress {
625            initial_position,
626            requested_tokens,
627            admitted_tokens: 0,
628            admitted_chunks: 0,
629            position: initial_position,
630        };
631        if let Some(active) = monitor.as_deref_mut() {
632            active.begin(progress)?;
633        }
634        if clear_first {
635            self.clear()?;
636        }
637
638        let chunk_limit = usize::try_from(self.options.batch_size)
639            .map_err(|_| Error::Invalid("batch size exceeds usize".to_owned()))?;
640        let isolate_last = self
641            .activation
642            .as_ref()
643            .is_some_and(ActivationController::has_last_prefill_capture);
644        let prefix_end = if isolate_last {
645            tokens.len().saturating_sub(1)
646        } else {
647            tokens.len()
648        };
649        let mut offset = 0_usize;
650        while offset < tokens.len() {
651            let end = if offset < prefix_end {
652                offset.saturating_add(chunk_limit).min(prefix_end)
653            } else {
654                tokens.len()
655            };
656            let chunk = &tokens[offset..end];
657            if let Some(active) = monitor.as_deref_mut()
658                && active.poll(progress)? == ControlFlow::Stop
659            {
660                let receipt = active.finish(PrefillFinish::Stopped)?;
661                return Ok(PrefillOutput {
662                    admitted_tokens: progress.admitted_tokens,
663                    position: self.position,
664                    receipt: Some(receipt),
665                });
666            }
667            self.decode_tokens(
668                chunk,
669                ActivationPhaseV1::Prefill,
670                isolate_last && end == tokens.len(),
671                ActivationTelemetryDispositionV1::Admitted,
672            )?;
673            let chunk_tokens = u64::try_from(chunk.len())
674                .map_err(|_| Error::Invalid("prefill chunk size overflowed".to_owned()))?;
675            progress.admitted_tokens = progress
676                .admitted_tokens
677                .checked_add(chunk_tokens)
678                .ok_or_else(|| Error::Invalid("prefill token accounting overflowed".to_owned()))?;
679            progress.admitted_chunks = progress
680                .admitted_chunks
681                .checked_add(1)
682                .ok_or_else(|| Error::Invalid("prefill chunk accounting overflowed".to_owned()))?;
683            progress.position = self.position;
684            if let Some(active) = monitor.as_deref_mut()
685                && active.observe_chunk(progress)? == ControlFlow::Stop
686                && progress.admitted_tokens < progress.requested_tokens
687            {
688                let receipt = active.finish(PrefillFinish::Stopped)?;
689                return Ok(PrefillOutput {
690                    admitted_tokens: progress.admitted_tokens,
691                    position: self.position,
692                    receipt: Some(receipt),
693                });
694            }
695            offset = end;
696        }
697        let receipt = monitor
698            .map(|active| active.finish(PrefillFinish::Complete))
699            .transpose()?;
700        Ok(PrefillOutput {
701            admitted_tokens: progress.admitted_tokens,
702            position: self.position,
703            receipt,
704        })
705    }
706
707    /// Generates through optional Logit Loom transforms and observers.
708    ///
709    /// Transform stages see raw candidate logits before native grammar, bias,
710    /// filtering, temperature, and terminal sampling. The selected token is
711    /// decoded before `accept` and observer callbacks, so those callbacks see
712    /// only causal admissions. An end-of-generation token is not decoded.
713    ///
714    /// # Errors
715    ///
716    /// Returns a plan, pipeline, observer, detokenization, sampling, capacity,
717    /// or decode error. Native state retains every token successfully decoded
718    /// before the error.
719    pub fn generate(
720        &mut self,
721        plan: &GenerationPlan,
722        pipeline: Option<&mut Pipeline>,
723        observers: Option<&mut ObserverSet>,
724    ) -> Result<GenerationOutput, Error> {
725        plan.validate()?;
726        self.ensure_generation_ready()?;
727        let plan_identity = plan.digest()?;
728        let sampler = build_sampler(&self.model.native, plan, &self.token_history)?;
729        self.generate_inner(
730            plan,
731            plan_identity,
732            sampler,
733            Vec::new(),
734            pipeline,
735            observers,
736        )
737        .map(|output| output.output)
738    }
739
740    /// Generates with an explicit eager or lazy version-two grammar.
741    ///
742    /// This preserves every v1 sampler, bias, stop, transform, observer, and
743    /// causal-accounting rule. The only additional mechanic is the separately
744    /// versioned grammar activation contract.
745    ///
746    /// # Errors
747    ///
748    /// Returns the same errors as [`Self::generate`], plus invalid or
749    /// model-incompatible lazy-grammar triggers.
750    pub fn generate_v2(
751        &mut self,
752        plan: &GenerationPlanV2,
753        pipeline: Option<&mut Pipeline>,
754        observers: Option<&mut ObserverSet>,
755    ) -> Result<GenerationOutput, Error> {
756        plan.validate()?;
757        self.ensure_generation_ready()?;
758        let plan_identity = plan.digest()?;
759        let sampler = build_sampler_v2(&self.model.native, plan, &self.token_history)?;
760        self.generate_inner(
761            &plan.generation,
762            plan_identity,
763            sampler,
764            Vec::new(),
765            pipeline,
766            observers,
767        )
768        .map(|output| output.output)
769    }
770
771    pub(crate) fn generate_stateful(
772        &mut self,
773        plan: &GenerationPlan,
774        continuation: Option<(LlamaSampler, Vec<u8>)>,
775        pipeline: Option<&mut Pipeline>,
776        observers: Option<&mut ObserverSet>,
777    ) -> Result<StatefulGenerationOutput, Error> {
778        plan.validate()?;
779        self.ensure_generation_ready()?;
780        let plan_identity = plan.digest()?;
781        let (sampler, stop_tail) = match continuation {
782            Some((sampler, stop_tail)) => {
783                validate_stop_tail(plan, &stop_tail)?;
784                (sampler, stop_tail)
785            }
786            None => (
787                build_sampler(&self.model.native, plan, &self.token_history)?,
788                Vec::new(),
789            ),
790        };
791        self.generate_inner(plan, plan_identity, sampler, stop_tail, pipeline, observers)
792    }
793
794    fn ensure_generation_ready(&self) -> Result<(), Error> {
795        self.ensure_healthy()?;
796        if self.token_history.is_empty() {
797            return Err(Error::Invalid(
798                "generation requires at least one prefilled token".to_owned(),
799            ));
800        }
801        Ok(())
802    }
803
804    fn generate_inner(
805        &mut self,
806        plan: &GenerationPlan,
807        plan_identity: Digest,
808        mut sampler: LlamaSampler,
809        mut stop_window: Vec<u8>,
810        mut pipeline: Option<&mut Pipeline>,
811        mut observers: Option<&mut ObserverSet>,
812    ) -> Result<StatefulGenerationOutput, Error> {
813        let initial_position = self.position;
814        if let Some(active) = pipeline.as_deref_mut() {
815            active.begin(&self.token_history)?;
816        }
817        if let Some(active) = observers.as_deref_mut() {
818            active.begin(initial_position, plan.max_tokens)?;
819        }
820        let mut output_tokens = Vec::new();
821        let mut output_bytes = Vec::new();
822        let mut finish = GenerationFinish::TokenLimit;
823
824        for step in 0..plan.max_tokens {
825            if let Some(active) = observers.as_deref_mut()
826                && active.poll()? == ControlFlow::Stop
827            {
828                finish = GenerationFinish::ObserverStop;
829                break;
830            }
831            if self.position >= u64::from(self.options.context_size.get()) {
832                return Err(Error::Invalid(
833                    "generation reached the configured context bound".to_owned(),
834                ));
835            }
836            let token = self.sample_token(step, &mut sampler, pipeline.as_deref_mut())?;
837            if self.model.is_end_of_generation(token) {
838                finish = GenerationFinish::EndOfGeneration { token };
839                break;
840            }
841
842            let piece = self.model.token_piece(token)?;
843            self.decode_tokens(
844                std::slice::from_ref(&token),
845                ActivationPhaseV1::Generation,
846                false,
847                ActivationTelemetryDispositionV1::Admitted,
848            )?;
849            sampler.accept(LlamaToken::new(token.get()));
850            if let Some(active) = pipeline.as_deref_mut() {
851                active.accept(token)?;
852            }
853            output_tokens.push(token);
854            output_bytes.extend_from_slice(&piece);
855            extend_stop_window(plan, &mut stop_window, &piece);
856            let mut observer_stop = false;
857            if let Some(active) = observers.as_deref_mut()
858                && active.observe(ObservedToken {
859                    token,
860                    piece: &piece,
861                    position: self.position,
862                })? == ControlFlow::Stop
863            {
864                observer_stop = true;
865            }
866            if observer_stop {
867                finish = GenerationFinish::ObserverStop;
868                break;
869            }
870            if let Some(index) = matching_stop(&plan.stops, &stop_window) {
871                let index = u32::try_from(index)
872                    .map_err(|_| Error::Invalid("stop sequence index overflowed".to_owned()))?;
873                finish = GenerationFinish::StopSequence { index };
874                break;
875            }
876        }
877
878        let transform_receipt = pipeline
879            .as_deref()
880            .map(|active| active.receipt().digest())
881            .transpose()?;
882        let observer_receipts = observers
883            .as_deref()
884            .map_or_else(Vec::new, ObserverSet::receipts)
885            .into_iter()
886            .map(|receipt| receipt.digest())
887            .collect::<Result<Vec<_>, _>>()?;
888        let receipt = GenerationReceipt {
889            plan: plan_identity,
890            initial_position,
891            admitted_tokens: u32::try_from(output_tokens.len())
892                .map_err(|_| Error::Invalid("generated token accounting overflowed".to_owned()))?,
893            admitted_bytes: u64::try_from(output_bytes.len())
894                .map_err(|_| Error::Invalid("generated byte accounting overflowed".to_owned()))?,
895            final_position: self.position,
896            finish,
897            transform_receipt,
898            observer_receipts,
899        };
900        receipt.digest()?;
901        Ok(StatefulGenerationOutput {
902            output: GenerationOutput {
903                bytes: output_bytes,
904                tokens: output_tokens,
905                receipt,
906            },
907            sampler,
908            stop_tail: checkpoint_stop_tail(plan, &stop_window),
909        })
910    }
911
912    fn sample_token(
913        &mut self,
914        step: u32,
915        sampler: &mut LlamaSampler,
916        pipeline: Option<&mut Pipeline>,
917    ) -> Result<TokenId, Error> {
918        let mut logits = catch_unwind(AssertUnwindSafe(|| self.context.get_logits().to_vec()))
919            .map_err(|_| Error::Native("llama.cpp logits were unavailable".to_owned()))?;
920        if let Some(active) = pipeline {
921            active.apply_to_vocabulary(step, &self.token_history, &mut logits)?;
922        }
923        let data = logits
924            .into_iter()
925            .enumerate()
926            .map(|(index, logit)| {
927                let token = i32::try_from(index).map_err(|_| {
928                    Error::Native("vocabulary token identifier overflowed".to_owned())
929                })?;
930                Ok(LlamaTokenData::new(LlamaToken::new(token), logit, 0.0))
931            })
932            .collect::<Result<Vec<_>, Error>>()?;
933        let mut candidates = LlamaTokenDataArray::new(data, false);
934        sampler.apply(&mut candidates);
935        let selected = candidates
936            .selected_token()
937            .ok_or_else(|| Error::Native("native sampler did not select a token".to_owned()))?;
938        TokenId::new(selected.0).map_err(Error::from)
939    }
940
941    /// Captures opaque native state and exact causal lineage.
942    ///
943    /// # Errors
944    ///
945    /// Returns an error when native state capture or identity encoding fails.
946    pub fn capture_state(&mut self) -> Result<StateSnapshot, Error> {
947        self.ensure_healthy()?;
948        if self.activation.is_some() {
949            return Err(Error::Invalid(
950                "activation sessions require an activation-bound checkpoint envelope".to_owned(),
951            ));
952        }
953        if !self.active_steering.is_empty() {
954            return Err(Error::Invalid(
955                "clear active steering before capturing a checkpoint".to_owned(),
956            ));
957        }
958        self.capture_state_inner()
959    }
960
961    pub(crate) fn capture_envelope_state(&mut self) -> Result<StateSnapshot, Error> {
962        self.ensure_healthy()?;
963        if !self.active_steering.is_empty() {
964            return Err(Error::Invalid(
965                "clear active steering before capturing an aggregate checkpoint".to_owned(),
966            ));
967        }
968        if let Some(activation) = self.activation.as_mut() {
969            activation.reset()?;
970        }
971        self.capture_state_inner()
972    }
973
974    fn capture_state_inner(&mut self) -> Result<StateSnapshot, Error> {
975        let size = self.context.state_get_size();
976        if size == 0 {
977            return Err(Error::Native(
978                "llama.cpp returned a zero-sized state".to_owned(),
979            ));
980        }
981        let mut bytes = vec![0_u8; size];
982        let written = self.context.state_get_data(&mut bytes);
983        if written == 0 || written > bytes.len() {
984            return Err(Error::Native(
985                "llama.cpp returned invalid state accounting".to_owned(),
986            ));
987        }
988        bytes.truncate(written);
989        let receipt = CheckpointReceipt {
990            model: self.model.artifact_digest().clone(),
991            backend: self.backend.clone(),
992            state: Digest::of_bytes(STATE_BYTES_DOMAIN, &bytes),
993            state_bytes: u64::try_from(bytes.len())
994                .map_err(|_| Error::Native("native state size exceeds u64".to_owned()))?,
995            token_history: Digest::of_serializable(TOKEN_HISTORY_DOMAIN, &self.token_history)?,
996            position: self.position,
997        };
998        StateSnapshot::from_parts(bytes, self.token_history.clone(), receipt)
999    }
1000
1001    /// Restores an exact compatible checkpoint.
1002    ///
1003    /// # Errors
1004    ///
1005    /// Returns an identity or native restore error. Local lineage changes only
1006    /// after llama.cpp accepts the complete state bytes.
1007    pub fn restore_state(&mut self, snapshot: &StateSnapshot) -> Result<(), Error> {
1008        self.ensure_healthy()?;
1009        if self.activation.is_some() {
1010            return Err(Error::Invalid(
1011                "activation sessions require an activation-bound checkpoint envelope".to_owned(),
1012            ));
1013        }
1014        if !self.active_steering.is_empty() {
1015            return Err(Error::Invalid(
1016                "clear active steering before restoring a checkpoint".to_owned(),
1017            ));
1018        }
1019        self.restore_state_inner(snapshot)?;
1020        if let Some(last_token) = snapshot.tokens.last().copied()
1021            && let Err(error) =
1022                self.refresh_logits_after_restore(last_token, snapshot.receipt.position, None)
1023        {
1024            self.record_poison(&error);
1025            return Err(error);
1026        }
1027        Ok(())
1028    }
1029
1030    pub(crate) fn restore_envelope_state(
1031        &mut self,
1032        snapshot: &StateSnapshot,
1033        refresh_phase: Option<ActivationPhaseV1>,
1034    ) -> Result<(), Error> {
1035        self.ensure_healthy()?;
1036        if !self.active_steering.is_empty() {
1037            return Err(Error::Invalid(
1038                "clear active steering before restoring an aggregate checkpoint".to_owned(),
1039            ));
1040        }
1041        self.restore_state_inner(snapshot)?;
1042        if let Some(last_token) = snapshot.tokens.last().copied()
1043            && let Err(error) = self.refresh_logits_after_restore(
1044                last_token,
1045                snapshot.receipt.position,
1046                refresh_phase,
1047            )
1048        {
1049            self.record_poison(&error);
1050            return Err(error);
1051        }
1052        if let Some(activation) = self.activation.as_mut() {
1053            activation.reset()?;
1054        }
1055        Ok(())
1056    }
1057
1058    pub(crate) fn refresh_restored_logits_with_active_steering(
1059        &mut self,
1060        activation_phase: Option<ActivationPhaseV1>,
1061    ) -> Result<(), Error> {
1062        self.ensure_healthy()?;
1063        if self.active_steering.is_empty() {
1064            return Ok(());
1065        }
1066        let Some(last_token) = self.token_history.last().copied() else {
1067            return Err(Error::Incompatible(
1068                "an actively steered checkpoint must contain a causal token".to_owned(),
1069            ));
1070        };
1071        if let Err(error) =
1072            self.refresh_logits_after_restore(last_token, self.position, activation_phase)
1073        {
1074            self.record_poison(&error);
1075            return Err(error);
1076        }
1077        if let Some(activation) = self.activation.as_mut()
1078            && let Err(error) = activation.reset()
1079        {
1080            self.record_poison(&error);
1081            return Err(error);
1082        }
1083        Ok(())
1084    }
1085
1086    fn restore_state_inner(&mut self, snapshot: &StateSnapshot) -> Result<(), Error> {
1087        snapshot.validate_contents()?;
1088        if snapshot.receipt.model != *self.model.artifact_digest()
1089            || snapshot.receipt.backend != self.backend
1090        {
1091            return Err(Error::Incompatible(
1092                "checkpoint model or backend identity does not match".to_owned(),
1093            ));
1094        }
1095        if snapshot.receipt.position > u64::from(self.options.context_size.get()) {
1096            return Err(Error::Incompatible(
1097                "checkpoint position exceeds the destination context".to_owned(),
1098            ));
1099        }
1100        self.model.validate_tokens(&snapshot.tokens)?;
1101        let read = self.context.state_set_data(&snapshot.bytes);
1102        if read != snapshot.bytes.len() {
1103            let error = Error::Native(format!(
1104                "llama.cpp restored {read} of {} state bytes",
1105                snapshot.bytes.len()
1106            ));
1107            self.record_poison(&error);
1108            return Err(error);
1109        }
1110        self.token_history.clone_from(&snapshot.tokens);
1111        self.position = snapshot.receipt.position;
1112        Ok(())
1113    }
1114
1115    fn refresh_logits_after_restore(
1116        &mut self,
1117        last_token: TokenId,
1118        position: u64,
1119        activation_phase: Option<ActivationPhaseV1>,
1120    ) -> Result<(), Error> {
1121        let Some((last_position, end_position)) = checkpoint_logit_refresh_range(position)? else {
1122            return Ok(());
1123        };
1124        let removed = self
1125            .context
1126            .clear_kv_cache_seq(Some(0), Some(last_position), Some(end_position))
1127            .map_err(native)?;
1128        if !removed {
1129            return Err(Error::Incompatible(
1130                "native memory cannot remove the final checkpoint position to refresh logits"
1131                    .to_owned(),
1132            ));
1133        }
1134
1135        let native_position = i32::try_from(last_position)
1136            .map_err(|_| Error::Incompatible("checkpoint position exceeds i32".to_owned()))?;
1137        let mut batch = LlamaBatch::new(1, 1);
1138        batch
1139            .add(
1140                LlamaToken::new(last_token.get()),
1141                native_position,
1142                &[0],
1143                true,
1144            )
1145            .map_err(native)?;
1146        if let (Some(activation), Some(phase)) = (self.activation.as_ref(), activation_phase) {
1147            activation.begin_decode(phase, false, ActivationTelemetryDispositionV1::Admitted)?;
1148        }
1149        let decode_result = self.context.decode(&mut batch);
1150        let callback_state = match (self.activation.as_ref(), activation_phase) {
1151            (Some(activation), Some(_)) => Some(activation.end_decode()),
1152            _ => None,
1153        };
1154        if let Err(error) = decode_result {
1155            return Err(native(error));
1156        }
1157        let active_decode = match callback_state {
1158            Some(result) => Some(result?),
1159            None => None,
1160        };
1161        if let Some(active_decode) = active_decode {
1162            let captures = self
1163                .context
1164                .tensor_transactions_mut()
1165                .ok_or_else(|| {
1166                    Error::Poisoned(
1167                        "activation context lost its tensor transaction state".to_owned(),
1168                    )
1169                })?
1170                .take_captures();
1171            self.activation
1172                .as_mut()
1173                .ok_or_else(|| {
1174                    Error::Poisoned("activation controller became unavailable".to_owned())
1175                })?
1176                .consume_captures(captures, active_decode)?;
1177        }
1178        Ok(())
1179    }
1180
1181    pub(crate) fn decode_tokens(
1182        &mut self,
1183        tokens: &[TokenId],
1184        phase: ActivationPhaseV1,
1185        last_prefill: bool,
1186        disposition: ActivationTelemetryDispositionV1,
1187    ) -> Result<(), Error> {
1188        self.model.validate_tokens(tokens)?;
1189        let token_count = u64::try_from(tokens.len())
1190            .map_err(|_| Error::Invalid("native batch token count overflowed".to_owned()))?;
1191        let mut batch = LlamaBatch::new(tokens.len(), 1);
1192        for (offset, token) in tokens.iter().enumerate() {
1193            let offset = u64::try_from(offset)
1194                .map_err(|_| Error::Invalid("native batch offset overflowed".to_owned()))?;
1195            let position = self
1196                .position
1197                .checked_add(offset)
1198                .ok_or_else(|| Error::Invalid("native position overflowed".to_owned()))?;
1199            let position = i32::try_from(position)
1200                .map_err(|_| Error::Invalid("native position exceeds i32".to_owned()))?;
1201            batch
1202                .add(
1203                    LlamaToken::new(token.get()),
1204                    position,
1205                    &[0],
1206                    offset + 1 == token_count,
1207                )
1208                .map_err(native)?;
1209        }
1210        if let Some(activation) = self.activation.as_ref() {
1211            activation.begin_decode(phase, last_prefill, disposition)?;
1212        }
1213        let decode_result = self.context.decode(&mut batch);
1214        let callback_state = self
1215            .activation
1216            .as_ref()
1217            .map(ActivationController::end_decode);
1218        if let Err(error) = decode_result {
1219            let error = native(error);
1220            if self.activation.is_some() {
1221                self.record_poison(&error);
1222            }
1223            return Err(error);
1224        }
1225        self.token_history.extend_from_slice(tokens);
1226        self.position = self
1227            .position
1228            .checked_add(token_count)
1229            .ok_or_else(|| Error::Invalid("causal position overflowed".to_owned()))?;
1230        let active_decode = match callback_state {
1231            Some(Ok(active)) => Some(active),
1232            Some(Err(error)) => {
1233                self.record_poison(&error);
1234                return Err(error);
1235            }
1236            None => None,
1237        };
1238        if let Some(active_decode) = active_decode {
1239            let captures = self
1240                .context
1241                .tensor_transactions_mut()
1242                .ok_or_else(|| {
1243                    Error::Poisoned(
1244                        "activation context lost its tensor transaction state".to_owned(),
1245                    )
1246                })?
1247                .take_captures();
1248            let result = self
1249                .activation
1250                .as_mut()
1251                .ok_or_else(|| {
1252                    Error::Poisoned("activation controller became unavailable".to_owned())
1253                })?
1254                .consume_captures(captures, active_decode);
1255            if let Err(error) = result {
1256                self.record_poison(&error);
1257                return Err(error);
1258            }
1259        }
1260        Ok(())
1261    }
1262
1263    pub(crate) fn ensure_steering_available(&self) -> Result<(), Error> {
1264        self.ensure_healthy()?;
1265        if !self.active_steering.is_empty() {
1266            return Err(Error::Invalid(
1267                "clear active steering before applying another steering scope".to_owned(),
1268            ));
1269        }
1270        Ok(())
1271    }
1272
1273    pub(crate) fn mark_steering_active(&mut self, kind: SteeringKind) {
1274        self.active_steering.push(kind);
1275    }
1276
1277    pub(crate) fn mark_steering_stack_active(&mut self, kinds: Vec<SteeringKind>) {
1278        debug_assert!(!kinds.is_empty());
1279        debug_assert!(self.active_steering.is_empty());
1280        self.active_steering = kinds;
1281    }
1282
1283    pub(crate) fn mark_steering_cleared(&mut self) {
1284        self.active_steering.clear();
1285    }
1286
1287    pub(crate) fn record_cleanup_failure(&mut self, error: &Error) {
1288        self.record_poison(error);
1289    }
1290
1291    pub(crate) fn record_poison(&mut self, error: &Error) {
1292        self.poison_reason.get_or_insert_with(|| error.to_string());
1293    }
1294
1295    pub(crate) fn ensure_healthy(&self) -> Result<(), Error> {
1296        if let Some(reason) = &self.poison_reason {
1297            return Err(Error::Poisoned(reason.clone()));
1298        }
1299        Ok(())
1300    }
1301}
1302
1303fn matching_stop(stops: &[Vec<u8>], output: &[u8]) -> Option<usize> {
1304    stops.iter().position(|stop| output.ends_with(stop))
1305}
1306
1307fn validate_stop_tail(plan: &GenerationPlan, tail: &[u8]) -> Result<(), Error> {
1308    let maximum = maximum_stop_bytes(plan).saturating_sub(1);
1309    if tail.len() > maximum {
1310        return Err(Error::Incompatible(
1311            "ordinary checkpoint stop-prefix state exceeds the generation plan".to_owned(),
1312        ));
1313    }
1314    Ok(())
1315}
1316
1317fn extend_stop_window(plan: &GenerationPlan, window: &mut Vec<u8>, piece: &[u8]) {
1318    let maximum = maximum_stop_bytes(plan);
1319    if maximum == 0 {
1320        window.clear();
1321        return;
1322    }
1323    if piece.len() >= maximum {
1324        window.clear();
1325        window.extend_from_slice(&piece[piece.len() - maximum..]);
1326        return;
1327    }
1328    window.extend_from_slice(piece);
1329    if window.len() > maximum {
1330        window.drain(..window.len() - maximum);
1331    }
1332}
1333
1334fn checkpoint_stop_tail(plan: &GenerationPlan, window: &[u8]) -> Vec<u8> {
1335    let maximum = maximum_stop_bytes(plan).saturating_sub(1);
1336    window[window.len().saturating_sub(maximum)..].to_vec()
1337}
1338
1339fn maximum_stop_bytes(plan: &GenerationPlan) -> usize {
1340    plan.stops.iter().map(Vec::len).max().unwrap_or(0)
1341}
1342
1343fn checkpoint_logit_refresh_range(position: u64) -> Result<Option<(u32, u32)>, Error> {
1344    let Some(last_position) = position.checked_sub(1) else {
1345        return Ok(None);
1346    };
1347    let last_position = u32::try_from(last_position)
1348        .map_err(|_| Error::Incompatible("checkpoint position exceeds u32".to_owned()))?;
1349    let end_position = last_position
1350        .checked_add(1)
1351        .ok_or_else(|| Error::Incompatible("checkpoint position overflowed".to_owned()))?;
1352    Ok(Some((last_position, end_position)))
1353}
1354
1355fn session_compatibility_identity(
1356    runtime: &Digest,
1357    options: SessionOptions,
1358    context_type: LlamaContextType,
1359    recurrent_state_slots: u32,
1360) -> Digest {
1361    let mut bytes = Vec::with_capacity(runtime.as_str().len() + 24);
1362    bytes.extend_from_slice(runtime.as_str().as_bytes());
1363    bytes.extend_from_slice(&options.context_size.get().to_le_bytes());
1364    bytes.extend_from_slice(&options.batch_size.to_le_bytes());
1365    bytes.extend_from_slice(&options.micro_batch_size.to_le_bytes());
1366    bytes.extend_from_slice(&options.threads.to_le_bytes());
1367    let context_type = match context_type {
1368        LlamaContextType::Default => 0_u32,
1369        LlamaContextType::Mtp => 1_u32,
1370    };
1371    bytes.extend_from_slice(&context_type.to_le_bytes());
1372    bytes.extend_from_slice(&recurrent_state_slots.to_le_bytes());
1373    Digest::of_bytes("llamacpp-session-compatibility-v3", &bytes)
1374}