1use logit_loom::{PrefillMonitor, TokenId};
6use logit_loom_llamacpp::{
7 GenerationOutput, Model, PrefillOutput, Session, SessionOptions, StateSnapshot, Tokenization,
8};
9
10use crate::{GenerationRequest, Result};
11
12#[derive(Clone, Debug, PartialEq, Eq)]
14pub struct AdmissionOutput {
15 pub tokens: Vec<TokenId>,
20 pub prefill: PrefillOutput,
22}
23
24#[derive(Clone, Debug, PartialEq, Eq)]
26pub struct CompletionOutput {
27 pub admission: AdmissionOutput,
29 pub generation: GenerationOutput,
31}
32
33impl CompletionOutput {
34 pub fn bytes(&self) -> &[u8] {
36 &self.generation.bytes
37 }
38
39 pub fn tokens(&self) -> &[TokenId] {
41 &self.generation.tokens
42 }
43
44 pub fn text(&self) -> std::result::Result<&str, std::str::Utf8Error> {
50 self.generation.text()
51 }
52}
53
54pub struct LoomSession<'model> {
56 pub(crate) model: &'model Model,
57 pub(crate) inner: Session<'model>,
58}
59
60impl std::fmt::Debug for LoomSession<'_> {
61 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62 formatter
63 .debug_struct("LoomSession")
64 .field("session", &self.inner)
65 .finish()
66 }
67}
68
69impl<'model> LoomSession<'model> {
70 pub(crate) const fn new(model: &'model Model, inner: Session<'model>) -> Self {
71 Self { model, inner }
72 }
73
74 pub fn replace_text(
80 &mut self,
81 text: &str,
82 tokenization: Tokenization,
83 ) -> Result<AdmissionOutput> {
84 admit_text(&mut self.inner, self.model, text, tokenization, true, None)
85 }
86
87 pub fn append_text(
93 &mut self,
94 text: &str,
95 tokenization: Tokenization,
96 ) -> Result<AdmissionOutput> {
97 admit_text(&mut self.inner, self.model, text, tokenization, false, None)
98 }
99
100 pub fn replace_text_observed(
109 &mut self,
110 text: &str,
111 tokenization: Tokenization,
112 monitor: &mut PrefillMonitor,
113 ) -> Result<AdmissionOutput> {
114 admit_text(
115 &mut self.inner,
116 self.model,
117 text,
118 tokenization,
119 true,
120 Some(monitor),
121 )
122 }
123
124 pub fn append_text_observed(
133 &mut self,
134 text: &str,
135 tokenization: Tokenization,
136 monitor: &mut PrefillMonitor,
137 ) -> Result<AdmissionOutput> {
138 admit_text(
139 &mut self.inner,
140 self.model,
141 text,
142 tokenization,
143 false,
144 Some(monitor),
145 )
146 }
147
148 pub fn generate(&mut self, request: GenerationRequest<'_>) -> Result<GenerationOutput> {
154 generate(&mut self.inner, request)
155 }
156
157 pub fn clear(&mut self) -> Result<()> {
163 self.inner.clear().map_err(Into::into)
164 }
165
166 pub fn capture_state(&mut self) -> Result<StateSnapshot> {
172 self.inner.capture_state().map_err(Into::into)
173 }
174
175 pub fn restore_state(&mut self, snapshot: &StateSnapshot) -> Result<()> {
181 self.inner.restore_state(snapshot).map_err(Into::into)
182 }
183
184 pub const fn position(&self) -> u64 {
186 self.inner.position()
187 }
188
189 pub const fn options(&self) -> SessionOptions {
191 self.inner.options()
192 }
193
194 pub fn token_history(&self) -> &[TokenId] {
196 self.inner.token_history()
197 }
198
199 pub const fn is_healthy(&self) -> bool {
201 self.inner.is_healthy()
202 }
203
204 pub fn poison_reason(&self) -> Option<&str> {
206 self.inner.poison_reason()
207 }
208
209 pub const fn raw(&self) -> &Session<'model> {
211 &self.inner
212 }
213
214 pub fn raw_mut(&mut self) -> &mut Session<'model> {
216 &mut self.inner
217 }
218}
219
220pub(crate) fn admit_text(
221 session: &mut Session<'_>,
222 model: &Model,
223 text: &str,
224 tokenization: Tokenization,
225 clear_first: bool,
226 monitor: Option<&mut PrefillMonitor>,
227) -> Result<AdmissionOutput> {
228 let tokens = model.tokenize(text, tokenization)?;
229 let prefill = match monitor {
230 Some(active) => session.prefill_observed(&tokens, clear_first, active)?,
231 None => session.prefill(&tokens, clear_first)?,
232 };
233 Ok(AdmissionOutput { tokens, prefill })
234}
235
236pub(crate) fn generate(
237 session: &mut Session<'_>,
238 request: GenerationRequest<'_>,
239) -> Result<GenerationOutput> {
240 let (plan, pipeline, observers) = request.into_parts();
241 session
242 .generate(&plan, pipeline, observers)
243 .map_err(Into::into)
244}
245
246#[cfg(test)]
247mod tests {
248 use logit_loom::{Digest, GenerationFinish, GenerationReceipt};
249
250 use super::*;
251
252 fn completion(bytes: Vec<u8>) -> CompletionOutput {
253 let admitted_bytes = u64::try_from(bytes.len()).unwrap();
254 CompletionOutput {
255 admission: AdmissionOutput {
256 tokens: vec![TokenId::new(1).unwrap()],
257 prefill: PrefillOutput {
258 admitted_tokens: 1,
259 position: 1,
260 receipt: None,
261 },
262 },
263 generation: GenerationOutput {
264 bytes,
265 tokens: vec![TokenId::new(2).unwrap()],
266 receipt: GenerationReceipt {
267 plan: Digest::of_bytes("test-plan", b"runtime-output"),
268 initial_position: 1,
269 admitted_tokens: 1,
270 admitted_bytes,
271 final_position: 2,
272 finish: GenerationFinish::TokenLimit,
273 transform_receipt: None,
274 observer_receipts: Vec::new(),
275 },
276 },
277 }
278 }
279
280 #[test]
281 fn completion_preserves_exact_bytes_and_tokens() {
282 let output = completion(vec![0xff]);
283 assert_eq!(output.bytes(), [0xff]);
284 assert_eq!(output.tokens(), [TokenId::new(2).unwrap()]);
285 assert!(output.text().is_err());
286 }
287
288 #[test]
289 fn completion_exposes_valid_complete_utf8() {
290 let output = completion("hello".as_bytes().to_vec());
291 assert_eq!(output.text().unwrap(), "hello");
292 }
293}