logit-loom-runtime 0.2.1

Explicit high-level local runtime for Logit Loom and llama.cpp
Documentation
// SPDX-License-Identifier: MIT OR Apache-2.0

//! Exact text admission, generation, and checkpoint access.

use logit_loom::{PrefillMonitor, TokenId};
use logit_loom_llamacpp::{
    GenerationOutput, Model, PrefillOutput, Session, SessionOptions, StateSnapshot, Tokenization,
};

use crate::{GenerationRequest, Result};

/// Result of tokenizing exact caller text and its completed or stopped prefill.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionOutput {
    /// Exact token IDs produced by the model tokenizer.
    ///
    /// A controlled prefill stop may admit only a prefix; compare this length
    /// with [`PrefillOutput::admitted_tokens`].
    pub tokens: Vec<TokenId>,
    /// Native prefill position and optional observer receipt.
    pub prefill: PrefillOutput,
}

/// Result of one fresh text admission followed by bounded generation.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CompletionOutput {
    /// Exact prompt tokenization and causal prefill result.
    pub admission: AdmissionOutput,
    /// Exact generated bytes, token IDs, and mechanical receipt.
    pub generation: GenerationOutput,
}

impl CompletionOutput {
    /// Returns the exact, potentially non-UTF-8 generated bytes.
    pub fn bytes(&self) -> &[u8] {
        &self.generation.bytes
    }

    /// Returns the causally admitted generated token IDs.
    pub fn tokens(&self) -> &[TokenId] {
        &self.generation.tokens
    }

    /// Returns generated bytes as UTF-8 when the complete output is valid.
    ///
    /// # Errors
    ///
    /// Returns the standard UTF-8 error for arbitrary model bytes.
    pub fn text(&self) -> std::result::Result<&str, std::str::Utf8Error> {
        self.generation.text()
    }
}

/// Higher-level borrowing façade over one single-owner causal session.
pub struct LoomSession<'model> {
    pub(crate) model: &'model Model,
    pub(crate) inner: Session<'model>,
}

impl std::fmt::Debug for LoomSession<'_> {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("LoomSession")
            .field("session", &self.inner)
            .finish()
    }
}

impl<'model> LoomSession<'model> {
    pub(crate) const fn new(model: &'model Model, inner: Session<'model>) -> Self {
        Self { model, inner }
    }

    /// Tokenizes exact text and replaces prior causal state before admission.
    ///
    /// # Errors
    ///
    /// Returns a tokenization, capacity, validation, or native prefill error.
    pub fn replace_text(
        &mut self,
        text: &str,
        tokenization: Tokenization,
    ) -> Result<AdmissionOutput> {
        admit_text(&mut self.inner, self.model, text, tokenization, true, None)
    }

    /// Tokenizes exact text and appends it to existing causal state.
    ///
    /// # Errors
    ///
    /// Returns a tokenization, capacity, validation, or native prefill error.
    pub fn append_text(
        &mut self,
        text: &str,
        tokenization: Tokenization,
    ) -> Result<AdmissionOutput> {
        admit_text(&mut self.inner, self.model, text, tokenization, false, None)
    }

    /// Replaces prior causal state while observing complete prefill chunks.
    ///
    /// A cooperative stop returns successful partial admission. Inspect the
    /// prefill receipt before deciding whether to generate.
    ///
    /// # Errors
    ///
    /// Returns a tokenization, capacity, observer, or native prefill error.
    pub fn replace_text_observed(
        &mut self,
        text: &str,
        tokenization: Tokenization,
        monitor: &mut PrefillMonitor,
    ) -> Result<AdmissionOutput> {
        admit_text(
            &mut self.inner,
            self.model,
            text,
            tokenization,
            true,
            Some(monitor),
        )
    }

    /// Appends exact text while observing complete prefill chunks.
    ///
    /// A cooperative stop returns successful partial admission. Inspect the
    /// prefill receipt before deciding whether to generate.
    ///
    /// # Errors
    ///
    /// Returns a tokenization, capacity, observer, or native prefill error.
    pub fn append_text_observed(
        &mut self,
        text: &str,
        tokenization: Tokenization,
        monitor: &mut PrefillMonitor,
    ) -> Result<AdmissionOutput> {
        admit_text(
            &mut self.inner,
            self.model,
            text,
            tokenization,
            false,
            Some(monitor),
        )
    }

    /// Executes one bounded generation request against current causal state.
    ///
    /// # Errors
    ///
    /// Returns a plan, pipeline, observer, sampling, capacity, or native error.
    pub fn generate(&mut self, request: GenerationRequest<'_>) -> Result<GenerationOutput> {
        generate(&mut self.inner, request)
    }

    /// Clears current causal state and exact local lineage.
    ///
    /// # Errors
    ///
    /// Returns the retained reason when native state is poisoned.
    pub fn clear(&mut self) -> Result<()> {
        self.inner.clear().map_err(Into::into)
    }

    /// Captures opaque native state and exact causal lineage.
    ///
    /// # Errors
    ///
    /// Returns an identity, allocation, or native state error.
    pub fn capture_state(&mut self) -> Result<StateSnapshot> {
        self.inner.capture_state().map_err(Into::into)
    }

    /// Restores a compatible checkpoint.
    ///
    /// # Errors
    ///
    /// Returns an identity, compatibility, or native restore error.
    pub fn restore_state(&mut self, snapshot: &StateSnapshot) -> Result<()> {
        self.inner.restore_state(snapshot).map_err(Into::into)
    }

    /// Returns the current causal position.
    pub const fn position(&self) -> u64 {
        self.inner.position()
    }

    /// Returns the exact allocation contract for this session.
    pub const fn options(&self) -> SessionOptions {
        self.inner.options()
    }

    /// Returns exact admitted token history.
    pub fn token_history(&self) -> &[TokenId] {
        self.inner.token_history()
    }

    /// Returns whether native state is not poisoned.
    pub const fn is_healthy(&self) -> bool {
        self.inner.is_healthy()
    }

    /// Returns the retained poison reason, when present.
    pub fn poison_reason(&self) -> Option<&str> {
        self.inner.poison_reason()
    }

    /// Returns the underlying low-level session.
    pub const fn raw(&self) -> &Session<'model> {
        &self.inner
    }

    /// Returns the underlying mutable low-level session.
    pub fn raw_mut(&mut self) -> &mut Session<'model> {
        &mut self.inner
    }
}

pub(crate) fn admit_text(
    session: &mut Session<'_>,
    model: &Model,
    text: &str,
    tokenization: Tokenization,
    clear_first: bool,
    monitor: Option<&mut PrefillMonitor>,
) -> Result<AdmissionOutput> {
    let tokens = model.tokenize(text, tokenization)?;
    let prefill = match monitor {
        Some(active) => session.prefill_observed(&tokens, clear_first, active)?,
        None => session.prefill(&tokens, clear_first)?,
    };
    Ok(AdmissionOutput { tokens, prefill })
}

pub(crate) fn generate(
    session: &mut Session<'_>,
    request: GenerationRequest<'_>,
) -> Result<GenerationOutput> {
    let (plan, pipeline, observers) = request.into_parts();
    session
        .generate(&plan, pipeline, observers)
        .map_err(Into::into)
}

#[cfg(test)]
mod tests {
    use logit_loom::{Digest, GenerationFinish, GenerationReceipt};

    use super::*;

    fn completion(bytes: Vec<u8>) -> CompletionOutput {
        let admitted_bytes = u64::try_from(bytes.len()).unwrap();
        CompletionOutput {
            admission: AdmissionOutput {
                tokens: vec![TokenId::new(1).unwrap()],
                prefill: PrefillOutput {
                    admitted_tokens: 1,
                    position: 1,
                    receipt: None,
                },
            },
            generation: GenerationOutput {
                bytes,
                tokens: vec![TokenId::new(2).unwrap()],
                receipt: GenerationReceipt {
                    plan: Digest::of_bytes("test-plan", b"runtime-output"),
                    initial_position: 1,
                    admitted_tokens: 1,
                    admitted_bytes,
                    final_position: 2,
                    finish: GenerationFinish::TokenLimit,
                    transform_receipt: None,
                    observer_receipts: Vec::new(),
                },
            },
        }
    }

    #[test]
    fn completion_preserves_exact_bytes_and_tokens() {
        let output = completion(vec![0xff]);
        assert_eq!(output.bytes(), [0xff]);
        assert_eq!(output.tokens(), [TokenId::new(2).unwrap()]);
        assert!(output.text().is_err());
    }

    #[test]
    fn completion_exposes_valid_complete_utf8() {
        let output = completion("hello".as_bytes().to_vec());
        assert_eq!(output.text().unwrap(), "hello");
    }
}