1use std::fs::File;
6use std::io::Read;
7use std::path::Path;
8
9use llama_cpp_4::llama_backend::LlamaBackend;
10use llama_cpp_4::model::params::LlamaModelParams;
11use llama_cpp_4::model::{AddBos, LlamaBackendDeviceType, LlamaModel, Special};
12use llama_cpp_4::token::LlamaToken;
13use logit_loom::{Digest, TextModelTopologyV1, TextSpeculativeMechanismV1, TokenId};
14
15use crate::{
16 ActivationConfiguration, Error, LLAMA_CPP_BINDING_SOURCE_REVISION, LLAMA_CPP_BINDING_VERSION,
17 LLAMA_CPP_REVISION, Session, SessionOptions, error::native,
18};
19
20const MODEL_ARTIFACT_DOMAIN: &str = "llamacpp-model-file-blake3-v1";
21pub(crate) const LORA_ARTIFACT_DOMAIN: &str = "llamacpp-lora-file-blake3-v1";
22
23pub const MAX_TOKENIZATION_BYTES: usize = 16 * 1024 * 1024;
25
26#[derive(Debug)]
28pub struct Runtime {
29 pub(crate) native: LlamaBackend,
30 identity: Digest,
31 compatibility: String,
32}
33
34impl Runtime {
35 pub fn initialize() -> Result<Self, Error> {
42 let native = LlamaBackend::init().map_err(native)?;
43 let compatibility = compatibility_label();
44 Ok(Self {
45 native,
46 identity: Digest::of_bytes("llamacpp-binding-identity-v2", compatibility.as_bytes()),
47 compatibility,
48 })
49 }
50
51 pub const fn identity(&self) -> &Digest {
53 &self.identity
54 }
55
56 pub fn compatibility_label(&self) -> &str {
58 &self.compatibility
59 }
60
61 pub const fn native_backend(&self) -> &LlamaBackend {
68 &self.native
69 }
70
71 pub fn silence_native_logs(&mut self) {
73 self.native.void_logs();
74 }
75}
76
77#[non_exhaustive]
79#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
80pub enum DevicePolicy {
81 #[default]
83 RequireAccelerator,
84 Any,
86}
87
88#[derive(Clone, Copy, Debug, PartialEq, Eq)]
90pub struct ModelOptions {
91 pub gpu_layers: u32,
93 pub main_gpu: i32,
95 pub device_policy: DevicePolicy,
97}
98
99#[derive(Clone, Copy, Debug, PartialEq, Eq)]
106pub struct PreverifiedModelArtifact {
107 content_blake3: [u8; 32],
108 byte_length: u64,
109}
110
111#[derive(Clone, Debug, PartialEq, Eq)]
117pub struct AuthorizedModelArtifact {
118 identity: Digest,
119 byte_length: u64,
120}
121
122impl AuthorizedModelArtifact {
123 #[must_use]
125 pub const fn new(identity: Digest, byte_length: u64) -> Self {
126 Self {
127 identity,
128 byte_length,
129 }
130 }
131
132 #[must_use]
134 pub const fn identity(&self) -> &Digest {
135 &self.identity
136 }
137
138 #[must_use]
140 pub const fn byte_length(&self) -> u64 {
141 self.byte_length
142 }
143}
144
145impl PreverifiedModelArtifact {
146 #[must_use]
148 pub const fn new(content_blake3: [u8; 32], byte_length: u64) -> Self {
149 Self {
150 content_blake3,
151 byte_length,
152 }
153 }
154
155 #[must_use]
157 pub const fn content_blake3(self) -> [u8; 32] {
158 self.content_blake3
159 }
160
161 #[must_use]
163 pub const fn byte_length(self) -> u64 {
164 self.byte_length
165 }
166
167 fn artifact_digest(self) -> Digest {
168 model_artifact_digest(self.content_blake3)
169 }
170}
171
172impl Default for ModelOptions {
173 fn default() -> Self {
174 Self {
175 gpu_layers: u32::MAX,
176 main_gpu: 0,
177 device_policy: DevicePolicy::RequireAccelerator,
178 }
179 }
180}
181
182#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
184pub struct Tokenization {
185 pub add_bos: bool,
187}
188
189pub struct Model {
191 pub(crate) native: LlamaModel,
192 artifact: Digest,
193 topology: TextModelTopologyV1,
194 tensor_selector_implementation: Digest,
195 architecture: String,
196 devices: Vec<String>,
197}
198
199impl std::fmt::Debug for Model {
200 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
201 formatter
202 .debug_struct("Model")
203 .field("artifact", &self.artifact)
204 .field("architecture", &self.architecture)
205 .field("topology", &self.topology)
206 .field("devices", &self.devices)
207 .finish_non_exhaustive()
208 }
209}
210
211impl Model {
212 pub fn load(
218 runtime: &Runtime,
219 path: impl AsRef<Path>,
220 options: ModelOptions,
221 ) -> Result<Self, Error> {
222 validate_model_options(options)?;
223 let path = path.as_ref();
224 let artifact_before_load = digest_file(path, MODEL_ARTIFACT_DOMAIN)?;
225 let model = Self::load_with_artifact(runtime, path, options, artifact_before_load.clone())?;
226 let artifact = digest_file(path, MODEL_ARTIFACT_DOMAIN)?;
227 if artifact != artifact_before_load {
228 return Err(Error::Incompatible(
229 "model file changed while llama.cpp was loading it".to_owned(),
230 ));
231 }
232 Ok(model)
233 }
234
235 pub fn load_preverified(
248 runtime: &Runtime,
249 path: impl AsRef<Path>,
250 artifact: PreverifiedModelArtifact,
251 options: ModelOptions,
252 ) -> Result<Self, Error> {
253 validate_model_options(options)?;
254 let path = path.as_ref();
255 verify_artifact_length(path, artifact.byte_length())?;
256 let model = Self::load_with_artifact(runtime, path, options, artifact.artifact_digest())?;
257 verify_artifact_length(path, artifact.byte_length())?;
258 Ok(model)
259 }
260
261 pub fn load_authorized(
272 runtime: &Runtime,
273 path: impl AsRef<Path>,
274 artifact: &AuthorizedModelArtifact,
275 options: ModelOptions,
276 ) -> Result<Self, Error> {
277 validate_model_options(options)?;
278 let path = path.as_ref();
279 verify_artifact_length(path, artifact.byte_length())?;
280 let model = Self::load_with_artifact(runtime, path, options, artifact.identity().clone())?;
281 verify_artifact_length(path, artifact.byte_length())?;
282 Ok(model)
283 }
284
285 fn load_with_artifact(
286 runtime: &Runtime,
287 path: &Path,
288 options: ModelOptions,
289 artifact: Digest,
290 ) -> Result<Self, Error> {
291 let params = LlamaModelParams::default()
292 .with_n_gpu_layers(options.gpu_layers)
293 .with_main_gpu(options.main_gpu);
294 let native_model =
295 LlamaModel::load_from_file(&runtime.native, path, ¶ms).map_err(native)?;
296 validate_vocabulary_size(native_model.n_vocab())?;
297 let mut has_accelerator = false;
298 let devices = native_model
299 .devices()
300 .map(|device| {
301 let device_type = device.device_type();
302 has_accelerator |= matches!(
303 device_type,
304 LlamaBackendDeviceType::Gpu
305 | LlamaBackendDeviceType::IntegratedGpu
306 | LlamaBackendDeviceType::Accel
307 );
308 let name = device.name().unwrap_or("unknown");
309 format!("{name}:{device_type:?}")
310 })
311 .collect::<Vec<_>>();
312 if options.device_policy == DevicePolicy::RequireAccelerator && !has_accelerator {
313 return Err(Error::Native(accelerator_placement_error(&devices)));
314 }
315 let architecture = native_model
316 .meta_val_str("general.architecture", 256)
317 .map_err(native)?;
318 if architecture.is_empty() || architecture.len() > 256 {
319 return Err(Error::Incompatible(
320 "model architecture metadata is empty or excessive".to_owned(),
321 ));
322 }
323 let architecture_implementation =
324 architecture_implementation(runtime.identity(), &architecture)?;
325 let tensor_selector_implementation =
326 tensor_selector_implementation(&architecture_implementation)?;
327 let layers = positive_native_dimension(native_model.n_layer(), "model layer count")?;
328 let embedding_width =
329 positive_native_dimension(native_model.n_embd(), "model embedding width")?;
330 let experts = optional_native_dimension(native_model.n_expert(), "model expert count")?;
331 let experts_used = experts.and_then(|_| {
332 native_model
333 .meta_val_str(&format!("{architecture}.expert_used_count"), 64)
334 .ok()
335 .and_then(|value| value.parse::<u32>().ok())
336 });
337 let nextn_layers =
338 optional_native_dimension(native_model.n_layer_nextn(), "model NextN layer count")?
339 .unwrap_or(0);
340 let mut supported_speculation = Vec::with_capacity(2);
341 if nextn_layers > 0 {
342 supported_speculation.push(TextSpeculativeMechanismV1::Mtp);
343 }
344 if native_model.has_decoder() && !native_model.is_diffusion() {
345 supported_speculation.push(TextSpeculativeMechanismV1::Eagle3);
346 }
347 let topology = TextModelTopologyV1 {
348 model: artifact.clone(),
349 backend: runtime.identity().clone(),
350 architecture_implementation,
351 layers,
352 embedding_width,
353 experts,
354 experts_used,
355 nextn_layers,
356 supported_speculation,
357 };
358 topology.digest()?;
359 Ok(Self {
360 native: native_model,
361 artifact,
362 topology,
363 tensor_selector_implementation,
364 architecture,
365 devices,
366 })
367 }
368
369 pub const fn artifact_digest(&self) -> &Digest {
371 &self.artifact
372 }
373
374 pub fn architecture(&self) -> &str {
376 &self.architecture
377 }
378
379 pub const fn topology(&self) -> &TextModelTopologyV1 {
381 &self.topology
382 }
383
384 pub const fn native_model(&self) -> &LlamaModel {
391 &self.native
392 }
393
394 pub const fn tensor_selector_implementation(&self) -> &Digest {
399 &self.tensor_selector_implementation
400 }
401
402 pub fn devices(&self) -> &[String] {
404 &self.devices
405 }
406
407 pub fn vocabulary_size(&self) -> Result<u32, Error> {
413 validate_vocabulary_size(self.native.n_vocab())?;
414 u32::try_from(self.native.n_vocab())
415 .map_err(|_| Error::Native("model vocabulary size exceeds u32".to_owned()))
416 }
417
418 pub fn tokenize(&self, text: &str, options: Tokenization) -> Result<Vec<TokenId>, Error> {
425 validate_tokenization_input(text)?;
426 let add_bos = if options.add_bos {
427 AddBos::Always
428 } else {
429 AddBos::Never
430 };
431 let tokens = self
432 .native
433 .str_to_token(text, add_bos)
434 .map_err(native)?
435 .into_iter()
436 .map(|token| TokenId::new(token.0).map_err(Error::from))
437 .collect::<Result<Vec<_>, _>>()?;
438 self.validate_tokens(&tokens)?;
439 Ok(tokens)
440 }
441
442 pub fn token_piece(&self, token: TokenId) -> Result<Vec<u8>, Error> {
449 self.validate_tokens(std::slice::from_ref(&token))?;
450 self.native
451 .token_to_raw_bytes(LlamaToken::new(token.get()), Special::Tokenize)
452 .map_err(native)
453 }
454
455 pub fn is_end_of_generation(&self, token: TokenId) -> bool {
457 self.native.is_eog_token(LlamaToken::new(token.get()))
458 }
459
460 pub(crate) fn validate_tokens(&self, tokens: &[TokenId]) -> Result<(), Error> {
461 validate_token_ids(self.native.n_vocab(), tokens)
462 }
463
464 pub fn session<'model>(
470 &'model self,
471 runtime: &Runtime,
472 options: SessionOptions,
473 ) -> Result<Session<'model>, Error> {
474 Session::new(self, runtime, options)
475 }
476
477 pub fn session_with_activation<'model>(
485 &'model self,
486 runtime: &Runtime,
487 options: SessionOptions,
488 activation: ActivationConfiguration,
489 ) -> Result<Session<'model>, Error> {
490 Session::new_with_activation(self, runtime, options, activation)
491 }
492}
493
494fn accelerator_placement_error(devices: &[String]) -> String {
495 let observed = if devices.is_empty() {
496 "none".to_owned()
497 } else {
498 devices.join(", ")
499 };
500 format!("model placement contains no accelerator device; observed devices: {observed}")
501}
502
503fn validate_token_ids(n_vocab: i32, tokens: &[TokenId]) -> Result<(), Error> {
504 validate_vocabulary_size(n_vocab)?;
505 if let Some(token) = tokens.iter().find(|token| token.get() >= n_vocab) {
506 return Err(Error::Invalid(format!(
507 "token {} is outside vocabulary size {n_vocab}",
508 token.get()
509 )));
510 }
511 Ok(())
512}
513
514pub(crate) fn digest_file(path: &Path, domain: &str) -> Result<Digest, Error> {
515 let mut file = File::open(path)?;
516 digest_reader(&mut file, domain)
517}
518
519fn digest_reader(reader: &mut impl Read, domain: &str) -> Result<Digest, Error> {
520 let mut hasher = blake3::Hasher::new();
521 let mut buffer = vec![0_u8; 1024 * 1024];
522 loop {
523 let read = reader.read(&mut buffer)?;
524 if read == 0 {
525 break;
526 }
527 hasher.update(&buffer[..read]);
528 }
529 Ok(Digest::of_bytes(domain, hasher.finalize().as_bytes()))
530}
531
532fn model_artifact_digest(content_blake3: [u8; 32]) -> Digest {
533 Digest::of_bytes(MODEL_ARTIFACT_DOMAIN, &content_blake3)
534}
535
536pub(crate) fn verify_artifact_length(path: &Path, expected: u64) -> Result<(), Error> {
537 let actual = std::fs::metadata(path)?.len();
538 if actual != expected {
539 return Err(Error::Incompatible(format!(
540 "authorized artifact length differs: expected {expected}, observed {actual}"
541 )));
542 }
543 Ok(())
544}
545
546fn validate_model_options(options: ModelOptions) -> Result<(), Error> {
547 if options.main_gpu < 0 {
548 return Err(Error::Invalid(
549 "main GPU index must be non-negative".to_owned(),
550 ));
551 }
552 Ok(())
553}
554
555fn validate_tokenization_input(text: &str) -> Result<(), Error> {
556 validate_tokenization_shape(text.len(), text.as_bytes().contains(&0))
557}
558
559fn validate_tokenization_shape(bytes: usize, contains_nul: bool) -> Result<(), Error> {
560 if bytes > MAX_TOKENIZATION_BYTES {
561 return Err(Error::Invalid(format!(
562 "tokenization input exceeds {MAX_TOKENIZATION_BYTES} UTF-8 bytes"
563 )));
564 }
565 if contains_nul {
566 return Err(Error::Invalid(
567 "tokenization input must not contain NUL bytes".to_owned(),
568 ));
569 }
570 Ok(())
571}
572
573fn compatibility_label() -> String {
574 let features = [
575 ("blas", cfg!(feature = "blas")),
576 ("cuda", cfg!(feature = "cuda")),
577 ("dynamic-link", cfg!(feature = "dynamic-link")),
578 ("hip", cfg!(feature = "hip")),
579 ("metal", cfg!(feature = "metal")),
580 ("native-cpu", cfg!(feature = "native-cpu")),
581 ("opencl", cfg!(feature = "opencl")),
582 ("openmp", cfg!(feature = "openmp")),
583 ("prebuilt", cfg!(feature = "prebuilt")),
584 ("rpc", cfg!(feature = "rpc")),
585 ("vulkan", cfg!(feature = "vulkan")),
586 ("webgpu", cfg!(feature = "webgpu")),
587 ]
588 .into_iter()
589 .filter_map(|(name, enabled)| enabled.then_some(name))
590 .collect::<Vec<_>>();
591 let features = if features.is_empty() {
592 "baseline".to_owned()
593 } else {
594 features.join(",")
595 };
596 format!(
597 "{LLAMA_CPP_BINDING_VERSION};binding-revision={LLAMA_CPP_BINDING_SOURCE_REVISION};llama-revision={LLAMA_CPP_REVISION};logit-loom={};target={}-{}-{};features={features}",
598 env!("CARGO_PKG_VERSION"),
599 std::env::consts::ARCH,
600 std::env::consts::OS,
601 if cfg!(target_endian = "little") {
602 "little"
603 } else {
604 "big"
605 },
606 )
607}
608
609fn validate_vocabulary_size(n_vocab: i32) -> Result<(), Error> {
610 if n_vocab <= 0 {
611 return Err(Error::Native(
612 "model returned a non-positive vocabulary size".to_owned(),
613 ));
614 }
615 Ok(())
616}
617
618fn positive_native_dimension(value: i32, field: &'static str) -> Result<u32, Error> {
619 if value <= 0 {
620 return Err(Error::Incompatible(format!(
621 "{field} must be positive, got {value}"
622 )));
623 }
624 u32::try_from(value)
625 .map_err(|_| Error::Incompatible(format!("{field} exceeds the u32 contract")))
626}
627
628fn optional_native_dimension(value: i32, field: &'static str) -> Result<Option<u32>, Error> {
629 if value < 0 {
630 return Err(Error::Incompatible(format!(
631 "{field} must not be negative, got {value}"
632 )));
633 }
634 if value == 0 {
635 return Ok(None);
636 }
637 u32::try_from(value)
638 .map(Some)
639 .map_err(|_| Error::Incompatible(format!("{field} exceeds the u32 contract")))
640}
641
642fn architecture_implementation(runtime: &Digest, architecture: &str) -> Result<Digest, Error> {
643 Ok(Digest::of_serializable(
644 "llamacpp-architecture-implementation-v1",
645 &(
646 runtime,
647 architecture,
648 LLAMA_CPP_BINDING_VERSION,
649 LLAMA_CPP_BINDING_SOURCE_REVISION,
650 LLAMA_CPP_REVISION,
651 "l_out-{layer};ffn_moe_logits-{layer};ffn_moe_probs-{layer};ffn_moe_topk-{layer}",
652 ),
653 )?)
654}
655
656fn tensor_selector_implementation(architecture: &Digest) -> Result<Digest, Error> {
657 Ok(Digest::of_serializable(
658 "llamacpp-tensor-selector-implementation-v1",
659 &(
660 architecture,
661 LLAMA_CPP_BINDING_VERSION,
662 LLAMA_CPP_BINDING_SOURCE_REVISION,
663 LLAMA_CPP_REVISION,
664 "exact-name;f32-or-i32;batch-token-rows-v1",
665 ),
666 )?)
667}
668
669#[cfg(test)]
670mod tests {
671 use std::io::Cursor;
672
673 use super::*;
674
675 #[test]
676 fn compatibility_label_binds_version_target_and_features() {
677 let label = compatibility_label();
678 assert!(label.contains(LLAMA_CPP_BINDING_VERSION));
679 assert!(label.contains(LLAMA_CPP_BINDING_SOURCE_REVISION));
680 assert!(label.contains(LLAMA_CPP_REVISION));
681 assert!(label.contains(std::env::consts::ARCH));
682 assert!(label.contains(std::env::consts::OS));
683 assert!(label.contains("features="));
684 }
685
686 #[test]
687 fn artifact_domains_separate_models_and_lora_adapters() {
688 let bytes = b"same artifact bytes";
689 let model = digest_reader(&mut Cursor::new(bytes), MODEL_ARTIFACT_DOMAIN).unwrap();
690 let lora = digest_reader(&mut Cursor::new(bytes), LORA_ARTIFACT_DOMAIN).unwrap();
691 assert_ne!(model, lora);
692 }
693
694 #[test]
695 fn preverified_artifact_preserves_model_identity() {
696 let bytes = b"same immutable model bytes";
697 let streamed = digest_reader(&mut Cursor::new(bytes), MODEL_ARTIFACT_DOMAIN).unwrap();
698 let raw = *blake3::hash(bytes).as_bytes();
699 let preverified = PreverifiedModelArtifact::new(raw, u64::try_from(bytes.len()).unwrap());
700 assert_eq!(preverified.artifact_digest(), streamed);
701 assert_eq!(preverified.byte_length(), bytes.len() as u64);
702 assert_eq!(preverified.content_blake3(), raw);
703 }
704
705 #[test]
706 fn authorized_artifact_preserves_provider_identity_without_payload() {
707 let identity = Digest::of_bytes("provider-artifact-id-v1", b"director-primary");
708 let artifact = AuthorizedModelArtifact::new(identity.clone(), 17_392_736_384);
709 assert_eq!(artifact.identity(), &identity);
710 assert_eq!(artifact.byte_length(), 17_392_736_384);
711 }
712
713 #[test]
714 fn tokenization_inputs_are_bounded_before_native_calls() {
715 assert!(validate_tokenization_input("hello").is_ok());
716 assert!(validate_tokenization_input("hello\0world").is_err());
717 assert!(validate_tokenization_shape(MAX_TOKENIZATION_BYTES + 1, false).is_err());
718 }
719
720 #[test]
721 fn vocabulary_size_must_be_positive() {
722 assert!(validate_vocabulary_size(1).is_ok());
723 assert!(validate_vocabulary_size(0).is_err());
724 assert!(validate_vocabulary_size(-1).is_err());
725 }
726
727 #[test]
728 fn token_ids_must_fit_the_loaded_vocabulary() {
729 let inside = TokenId::new(9).unwrap();
730 let outside = TokenId::new(10).unwrap();
731 assert!(validate_token_ids(10, &[inside]).is_ok());
732 assert!(validate_token_ids(10, &[outside]).is_err());
733 }
734
735 #[test]
736 fn accelerator_placement_errors_report_observed_devices() {
737 assert_eq!(
738 accelerator_placement_error(&[]),
739 "model placement contains no accelerator device; observed devices: none"
740 );
741 assert_eq!(
742 accelerator_placement_error(&["CPU:Cpu".to_owned(), "NPU:Unknown".to_owned()]),
743 "model placement contains no accelerator device; observed devices: CPU:Cpu, NPU:Unknown"
744 );
745 }
746}