Skip to main content

logit_loom_runtime/
runtime.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! Loaded model/runtime ownership and one-shot generation.
4
5use std::path::Path;
6
7use logit_loom::Digest;
8use logit_loom_llamacpp::{
9    LoraAdapter, Model, ModelOptions, Runtime, SessionOptions, Tokenization,
10};
11use logit_loom_models::{
12    ArtifactReceipt, Catalog, QWEN3_SMALL_ARTIFACT_PATH, QWEN3_SMALL_PROFILE_ID,
13    QWEN3_SMALL_SOURCE_ID,
14};
15
16use crate::{
17    CompletionOutput, GenerationRequest, LoomSession, Result, session::admit_text,
18    session::generate,
19};
20
21/// Native log handling selected before model loading.
22#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
23pub enum NativeLogPolicy {
24    /// Preserve llama.cpp's configured native logging.
25    #[default]
26    Preserve,
27    /// Suppress llama.cpp log output for this process runtime.
28    Silence,
29}
30
31/// Model loading, default session allocation, and native log options.
32#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
33pub struct LoomOptions {
34    /// Native model placement and device requirements.
35    pub model: ModelOptions,
36    /// Allocation used by [`Loom::session`] and one-shot completion.
37    pub session: SessionOptions,
38    /// Native logging behavior applied before model loading.
39    pub native_logs: NativeLogPolicy,
40}
41
42/// One loaded local model and its process-wide llama.cpp runtime.
43///
44/// The model field intentionally precedes the runtime field so native model
45/// destruction occurs before backend shutdown.
46pub struct Loom {
47    model: Model,
48    runtime: Runtime,
49    options: LoomOptions,
50    profile_artifact: Option<ArtifactReceipt>,
51}
52
53impl std::fmt::Debug for Loom {
54    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55        formatter
56            .debug_struct("Loom")
57            .field("model", &self.model)
58            .field("runtime", &self.runtime.compatibility_label())
59            .field("options", &self.options)
60            .field("profile_artifact", &self.profile_artifact)
61            .finish()
62    }
63}
64
65impl Loom {
66    /// Initializes llama.cpp and loads one caller-supplied GGUF model.
67    ///
68    /// This method never downloads a model or retries a rejected placement as
69    /// CPU-only inference.
70    ///
71    /// # Errors
72    ///
73    /// Returns an initialization, I/O, native load, identity, or device-policy
74    /// error.
75    pub fn load(path: impl AsRef<Path>, options: LoomOptions) -> Result<Self> {
76        Self::load_inner(path.as_ref(), options, None)
77    }
78
79    /// Verifies and loads the catalogued Qwen3 0.6B `Q8_0` GGUF.
80    ///
81    /// The caller still selects the exact local path and every [`LoomOptions`]
82    /// placement and allocation field. This method does not format a chat
83    /// prompt, choose tokenization flags, allocate a session, or download
84    /// artifacts.
85    ///
86    /// # Errors
87    ///
88    /// Returns a catalog, size, SHA-256, initialization, native load, identity,
89    /// or device-policy error.
90    pub fn load_qwen3_small(path: impl AsRef<Path>, options: LoomOptions) -> Result<Self> {
91        let catalog = Catalog::embedded()?;
92        let catalog_sha256 = catalog.packaged_sha256();
93        let profile = catalog
94            .find_profile(QWEN3_SMALL_PROFILE_ID)
95            .ok_or_else(|| logit_loom_models::ArtifactError::Unknown {
96                profile: QWEN3_SMALL_PROFILE_ID.to_owned(),
97                source_id: QWEN3_SMALL_SOURCE_ID.to_owned(),
98                path: QWEN3_SMALL_ARTIFACT_PATH.to_owned(),
99            })?;
100        let verified = profile.verify_artifact(
101            &catalog_sha256,
102            QWEN3_SMALL_SOURCE_ID,
103            QWEN3_SMALL_ARTIFACT_PATH,
104            path,
105        )?;
106        let loaded = Self::load_inner(verified.path(), options, Some(verified.receipt().clone()))?;
107        profile.verify_artifact(
108            &catalog_sha256,
109            QWEN3_SMALL_SOURCE_ID,
110            QWEN3_SMALL_ARTIFACT_PATH,
111            verified.path(),
112        )?;
113        Ok(loaded)
114    }
115
116    fn load_inner(
117        path: &Path,
118        options: LoomOptions,
119        profile_artifact: Option<ArtifactReceipt>,
120    ) -> Result<Self> {
121        let mut runtime = Runtime::initialize()?;
122        if options.native_logs == NativeLogPolicy::Silence {
123            runtime.silence_native_logs();
124        }
125        let model = Model::load(&runtime, path, options.model)?;
126        Ok(Self {
127            model,
128            runtime,
129            options,
130            profile_artifact,
131        })
132    }
133
134    /// Returns the exact options used for model loading and default sessions.
135    pub const fn options(&self) -> LoomOptions {
136        self.options
137    }
138
139    /// Returns the model's reported native device descriptions.
140    pub fn devices(&self) -> &[String] {
141        self.model.devices()
142    }
143
144    /// Returns the content identity of the exact loaded GGUF bytes.
145    pub const fn model_identity(&self) -> &Digest {
146        self.model.artifact_digest()
147    }
148
149    /// Returns the native build identity bound into checkpoints.
150    pub const fn backend_identity(&self) -> &Digest {
151        self.runtime.identity()
152    }
153
154    /// Returns the readable native build label bound into checkpoints.
155    pub fn backend_compatibility(&self) -> &str {
156        self.runtime.compatibility_label()
157    }
158
159    /// Returns exact profile verification evidence when loaded through a
160    /// profile-specific constructor.
161    pub const fn profile_artifact(&self) -> Option<&ArtifactReceipt> {
162        self.profile_artifact.as_ref()
163    }
164
165    /// Returns the underlying loaded model.
166    pub const fn raw_model(&self) -> &Model {
167        &self.model
168    }
169
170    /// Returns the underlying process runtime.
171    pub const fn raw_runtime(&self) -> &Runtime {
172        &self.runtime
173    }
174
175    /// Creates a borrowing session with the configured default allocation.
176    ///
177    /// # Errors
178    ///
179    /// Returns an allocation or native context error.
180    pub fn session(&self) -> Result<LoomSession<'_>> {
181        self.session_with(self.options.session)
182    }
183
184    /// Creates a borrowing session with an explicit allocation contract.
185    ///
186    /// # Errors
187    ///
188    /// Returns a validation, allocation, or native context error.
189    pub fn session_with(&self, options: SessionOptions) -> Result<LoomSession<'_>> {
190        let session = self.model.session(&self.runtime, options)?;
191        Ok(LoomSession::new(&self.model, session))
192    }
193
194    /// Loads a model-compatible `LoRA` without applying it.
195    ///
196    /// # Errors
197    ///
198    /// Returns an I/O, identity, or native compatibility error.
199    pub fn load_lora(&self, path: impl AsRef<Path>) -> Result<LoraAdapter> {
200        self.model.load_lora(path).map_err(Into::into)
201    }
202
203    /// Runs one fresh prompt replacement and bounded generation call.
204    ///
205    /// # Errors
206    ///
207    /// Returns a tokenization, prefill, request-control, sampling, callback, or
208    /// native execution error.
209    pub fn complete(
210        &self,
211        text: &str,
212        tokenization: Tokenization,
213        request: GenerationRequest<'_>,
214    ) -> Result<CompletionOutput> {
215        let mut session = self.model.session(&self.runtime, self.options.session)?;
216        let admission = admit_text(&mut session, &self.model, text, tokenization, true, None)?;
217        let generation = generate(&mut session, request)?;
218        Ok(CompletionOutput {
219            admission,
220            generation,
221        })
222    }
223}