logit_loom_runtime/
runtime.rs1use std::path::Path;
6
7use logit_loom::Digest;
8use logit_loom_llamacpp::{
9 LoraAdapter, Model, ModelOptions, Runtime, SessionOptions, Tokenization,
10};
11use logit_loom_models::{
12 ArtifactReceipt, Catalog, QWEN3_SMALL_ARTIFACT_PATH, QWEN3_SMALL_PROFILE_ID,
13 QWEN3_SMALL_SOURCE_ID,
14};
15
16use crate::{
17 CompletionOutput, GenerationRequest, LoomSession, Result, session::admit_text,
18 session::generate,
19};
20
21#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
23pub enum NativeLogPolicy {
24 #[default]
26 Preserve,
27 Silence,
29}
30
31#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
33pub struct LoomOptions {
34 pub model: ModelOptions,
36 pub session: SessionOptions,
38 pub native_logs: NativeLogPolicy,
40}
41
42pub struct Loom {
47 model: Model,
48 runtime: Runtime,
49 options: LoomOptions,
50 profile_artifact: Option<ArtifactReceipt>,
51}
52
53impl std::fmt::Debug for Loom {
54 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55 formatter
56 .debug_struct("Loom")
57 .field("model", &self.model)
58 .field("runtime", &self.runtime.compatibility_label())
59 .field("options", &self.options)
60 .field("profile_artifact", &self.profile_artifact)
61 .finish()
62 }
63}
64
65impl Loom {
66 pub fn load(path: impl AsRef<Path>, options: LoomOptions) -> Result<Self> {
76 Self::load_inner(path.as_ref(), options, None)
77 }
78
79 pub fn load_qwen3_small(path: impl AsRef<Path>, options: LoomOptions) -> Result<Self> {
91 let catalog = Catalog::embedded()?;
92 let catalog_sha256 = catalog.packaged_sha256();
93 let profile = catalog
94 .find_profile(QWEN3_SMALL_PROFILE_ID)
95 .ok_or_else(|| logit_loom_models::ArtifactError::Unknown {
96 profile: QWEN3_SMALL_PROFILE_ID.to_owned(),
97 source_id: QWEN3_SMALL_SOURCE_ID.to_owned(),
98 path: QWEN3_SMALL_ARTIFACT_PATH.to_owned(),
99 })?;
100 let verified = profile.verify_artifact(
101 &catalog_sha256,
102 QWEN3_SMALL_SOURCE_ID,
103 QWEN3_SMALL_ARTIFACT_PATH,
104 path,
105 )?;
106 let loaded = Self::load_inner(verified.path(), options, Some(verified.receipt().clone()))?;
107 profile.verify_artifact(
108 &catalog_sha256,
109 QWEN3_SMALL_SOURCE_ID,
110 QWEN3_SMALL_ARTIFACT_PATH,
111 verified.path(),
112 )?;
113 Ok(loaded)
114 }
115
116 fn load_inner(
117 path: &Path,
118 options: LoomOptions,
119 profile_artifact: Option<ArtifactReceipt>,
120 ) -> Result<Self> {
121 let mut runtime = Runtime::initialize()?;
122 if options.native_logs == NativeLogPolicy::Silence {
123 runtime.silence_native_logs();
124 }
125 let model = Model::load(&runtime, path, options.model)?;
126 Ok(Self {
127 model,
128 runtime,
129 options,
130 profile_artifact,
131 })
132 }
133
134 pub const fn options(&self) -> LoomOptions {
136 self.options
137 }
138
139 pub fn devices(&self) -> &[String] {
141 self.model.devices()
142 }
143
144 pub const fn model_identity(&self) -> &Digest {
146 self.model.artifact_digest()
147 }
148
149 pub const fn backend_identity(&self) -> &Digest {
151 self.runtime.identity()
152 }
153
154 pub fn backend_compatibility(&self) -> &str {
156 self.runtime.compatibility_label()
157 }
158
159 pub const fn profile_artifact(&self) -> Option<&ArtifactReceipt> {
162 self.profile_artifact.as_ref()
163 }
164
165 pub const fn raw_model(&self) -> &Model {
167 &self.model
168 }
169
170 pub const fn raw_runtime(&self) -> &Runtime {
172 &self.runtime
173 }
174
175 pub fn session(&self) -> Result<LoomSession<'_>> {
181 self.session_with(self.options.session)
182 }
183
184 pub fn session_with(&self, options: SessionOptions) -> Result<LoomSession<'_>> {
190 let session = self.model.session(&self.runtime, options)?;
191 Ok(LoomSession::new(&self.model, session))
192 }
193
194 pub fn load_lora(&self, path: impl AsRef<Path>) -> Result<LoraAdapter> {
200 self.model.load_lora(path).map_err(Into::into)
201 }
202
203 pub fn complete(
210 &self,
211 text: &str,
212 tokenization: Tokenization,
213 request: GenerationRequest<'_>,
214 ) -> Result<CompletionOutput> {
215 let mut session = self.model.session(&self.runtime, self.options.session)?;
216 let admission = admit_text(&mut session, &self.model, text, tokenization, true, None)?;
217 let generation = generate(&mut session, request)?;
218 Ok(CompletionOutput {
219 admission,
220 generation,
221 })
222 }
223}