use logit_loom::{PrefillMonitor, TokenId};
use logit_loom_llamacpp::{
GenerationOutput, Model, PrefillOutput, Session, SessionOptions, StateSnapshot, Tokenization,
};
use crate::{GenerationRequest, Result};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AdmissionOutput {
pub tokens: Vec<TokenId>,
pub prefill: PrefillOutput,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CompletionOutput {
pub admission: AdmissionOutput,
pub generation: GenerationOutput,
}
impl CompletionOutput {
pub fn bytes(&self) -> &[u8] {
&self.generation.bytes
}
pub fn tokens(&self) -> &[TokenId] {
&self.generation.tokens
}
pub fn text(&self) -> std::result::Result<&str, std::str::Utf8Error> {
self.generation.text()
}
}
pub struct LoomSession<'model> {
pub(crate) model: &'model Model,
pub(crate) inner: Session<'model>,
}
impl std::fmt::Debug for LoomSession<'_> {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("LoomSession")
.field("session", &self.inner)
.finish()
}
}
impl<'model> LoomSession<'model> {
pub(crate) const fn new(model: &'model Model, inner: Session<'model>) -> Self {
Self { model, inner }
}
pub fn replace_text(
&mut self,
text: &str,
tokenization: Tokenization,
) -> Result<AdmissionOutput> {
admit_text(&mut self.inner, self.model, text, tokenization, true, None)
}
pub fn append_text(
&mut self,
text: &str,
tokenization: Tokenization,
) -> Result<AdmissionOutput> {
admit_text(&mut self.inner, self.model, text, tokenization, false, None)
}
pub fn replace_text_observed(
&mut self,
text: &str,
tokenization: Tokenization,
monitor: &mut PrefillMonitor,
) -> Result<AdmissionOutput> {
admit_text(
&mut self.inner,
self.model,
text,
tokenization,
true,
Some(monitor),
)
}
pub fn append_text_observed(
&mut self,
text: &str,
tokenization: Tokenization,
monitor: &mut PrefillMonitor,
) -> Result<AdmissionOutput> {
admit_text(
&mut self.inner,
self.model,
text,
tokenization,
false,
Some(monitor),
)
}
pub fn generate(&mut self, request: GenerationRequest<'_>) -> Result<GenerationOutput> {
generate(&mut self.inner, request)
}
pub fn clear(&mut self) -> Result<()> {
self.inner.clear().map_err(Into::into)
}
pub fn capture_state(&mut self) -> Result<StateSnapshot> {
self.inner.capture_state().map_err(Into::into)
}
pub fn restore_state(&mut self, snapshot: &StateSnapshot) -> Result<()> {
self.inner.restore_state(snapshot).map_err(Into::into)
}
pub const fn position(&self) -> u64 {
self.inner.position()
}
pub const fn options(&self) -> SessionOptions {
self.inner.options()
}
pub fn token_history(&self) -> &[TokenId] {
self.inner.token_history()
}
pub const fn is_healthy(&self) -> bool {
self.inner.is_healthy()
}
pub fn poison_reason(&self) -> Option<&str> {
self.inner.poison_reason()
}
pub const fn raw(&self) -> &Session<'model> {
&self.inner
}
pub fn raw_mut(&mut self) -> &mut Session<'model> {
&mut self.inner
}
}
pub(crate) fn admit_text(
session: &mut Session<'_>,
model: &Model,
text: &str,
tokenization: Tokenization,
clear_first: bool,
monitor: Option<&mut PrefillMonitor>,
) -> Result<AdmissionOutput> {
let tokens = model.tokenize(text, tokenization)?;
let prefill = match monitor {
Some(active) => session.prefill_observed(&tokens, clear_first, active)?,
None => session.prefill(&tokens, clear_first)?,
};
Ok(AdmissionOutput { tokens, prefill })
}
pub(crate) fn generate(
session: &mut Session<'_>,
request: GenerationRequest<'_>,
) -> Result<GenerationOutput> {
let (plan, pipeline, observers) = request.into_parts();
session
.generate(&plan, pipeline, observers)
.map_err(Into::into)
}
#[cfg(test)]
mod tests {
use logit_loom::{Digest, GenerationFinish, GenerationReceipt};
use super::*;
fn completion(bytes: Vec<u8>) -> CompletionOutput {
let admitted_bytes = u64::try_from(bytes.len()).unwrap();
CompletionOutput {
admission: AdmissionOutput {
tokens: vec![TokenId::new(1).unwrap()],
prefill: PrefillOutput {
admitted_tokens: 1,
position: 1,
receipt: None,
},
},
generation: GenerationOutput {
bytes,
tokens: vec![TokenId::new(2).unwrap()],
receipt: GenerationReceipt {
plan: Digest::of_bytes("test-plan", b"runtime-output"),
initial_position: 1,
admitted_tokens: 1,
admitted_bytes,
final_position: 2,
finish: GenerationFinish::TokenLimit,
transform_receipt: None,
observer_receipts: Vec::new(),
},
},
}
}
#[test]
fn completion_preserves_exact_bytes_and_tokens() {
let output = completion(vec![0xff]);
assert_eq!(output.bytes(), [0xff]);
assert_eq!(output.tokens(), [TokenId::new(2).unwrap()]);
assert!(output.text().is_err());
}
#[test]
fn completion_exposes_valid_complete_utf8() {
let output = completion("hello".as_bytes().to_vec());
assert_eq!(output.text().unwrap(), "hello");
}
}