ik_llama_cpp_2/
lib.rs

1//! Safe Rust bindings for `ik_llama.cpp` (ikawrakow's SOTA-quant fork).
2//!
3//! Mirrors the API and codestyle of the `llama-cpp-2` crate
4//! (`utilityai/llama-cpp-rs`), adapted for ik_llama.cpp's divergent C API:
5//! legacy `llama_sample_*` sampling, model-pointer tokenizer, ik-specific
6//! model/context params, and the MTP / NextN speculative path.
7//!
8//! v1 covers the general generation path (load → tokenize → decode → sample)
9//! plus the MTP scaffolding.
10#![allow(clippy::pedantic, clippy::module_name_repetitions)]
11
12use std::ffi::NulError;
13
14pub mod context;
15pub mod gguf;
16pub mod grammar;
17pub mod llama_backend;
18pub mod llama_batch;
19pub mod model;
20#[cfg(feature = "mtmd")]
21pub mod mtmd;
22pub mod quantize;
23pub mod sampling;
24pub mod speculative;
25pub mod timing;
26pub mod token;
27
28pub use context::params::{LlamaContextParams, LlamaContextType};
29pub use context::LlamaContext;
30#[cfg(feature = "common")]
31pub use grammar::{json_schema_to_grammar, JsonSchemaError};
32pub use grammar::{DryInitError, DryParams, GrammarInitError, LlamaDrySampler, LlamaGrammar};
33pub use llama_backend::LlamaBackend;
34pub use llama_batch::LlamaBatch;
35pub use model::params::LlamaModelParams;
36pub use model::{AddBos, LlamaModel};
37#[cfg(feature = "mtmd")]
38pub use mtmd::{
39    mtmd_default_marker, MtmdBitmap, MtmdBitmapError, MtmdContext, MtmdContextParams,
40    MtmdEncodeError, MtmdEvalError, MtmdInitError, MtmdInputChunk, MtmdInputChunkError,
41    MtmdInputChunkType, MtmdInputChunks, MtmdInputChunksError, MtmdInputText, MtmdTokenizeError,
42};
43pub use sampling::LlamaSampler;
44pub use speculative::{MtpOpType, MtpSpeculativeParams};
45#[cfg(feature = "common")]
46pub use speculative::{MtpSpeculative, MtpStep};
47pub use token::data::LlamaTokenData;
48pub use token::data_array::LlamaTokenDataArray;
49pub use token::LlamaToken;
50
51/// Errors returned by the safe wrapper.
52#[derive(Debug, thiserror::Error)]
53pub enum LlamaError {
54    /// The global backend was already initialized (it is a process-wide singleton).
55    #[error("llama backend already initialized")]
56    BackendAlreadyInitialized,
57    /// `llama_model_load_from_file` returned null for the given path.
58    #[error("failed to load model from {0}")]
59    ModelLoad(String),
60    /// `llama_init_from_model` returned null.
61    #[error("failed to create llama context")]
62    ContextCreation,
63    /// A path or prompt contained an interior NUL byte.
64    #[error("string contained an interior NUL byte")]
65    Nul(#[from] NulError),
66    /// The model path was not valid UTF-8 / could not be converted to a C string.
67    #[error("invalid model path")]
68    InvalidPath,
69    /// Tokenization failed (negative count from `llama_tokenize`).
70    #[error("tokenization failed")]
71    Tokenize,
72    /// Detokenization produced invalid UTF-8 that could not be recovered.
73    #[error("token could not be converted to text")]
74    TokenToPiece,
75    /// `llama_decode` returned a non-zero status.
76    #[error("llama_decode failed with status {0}")]
77    Decode(i32),
78    /// A batch operation exceeded the batch's allocated capacity.
79    #[error("batch capacity {capacity} exceeded (tried to add token {index})")]
80    BatchOverflow {
81        /// Allocated token capacity of the batch.
82        capacity: usize,
83        /// Index that overflowed.
84        index: usize,
85    },
86    /// More sequence ids were supplied for a token than the batch's `n_seq_max`.
87    #[error("too many seq_ids for a batch token: got {got}, max {max}")]
88    TooManySeqIds {
89        /// Number of seq_ids supplied.
90        got: usize,
91        /// Batch's configured `n_seq_max`.
92        max: usize,
93    },
94    /// Requested logits for a position that was not computed / out of range.
95    #[error("no logits available at index {0}")]
96    NoLogits(i32),
97    /// MTP speculative driver initialization failed (e.g. 0 NextN layers, or an
98    /// openPangu/recurrent target, or a context not created with `.with_mtp(true)`).
99    #[error("failed to initialize MTP speculative driver")]
100    MtpInit,
101    /// MTP begin (prompt warmup) failed.
102    #[error("MTP begin failed with status {0}")]
103    MtpBegin(i32),
104    /// MTP step (draft/verify/accept/commit) failed.
105    #[error("MTP step failed with status {0}")]
106    MtpStep(i32),
107}
ik_llama_cpp_2/lib.rs

ik_llama_cpp_2/
lib.rs
