llama_cpp_2/
lib.rs

1//! Bindings to the llama.cpp library.
2//!
3//! As llama.cpp is a very fast moving target, this crate does not attempt to create a stable API
4//! with all the rust idioms. Instead it provided safe wrappers around nearly direct bindings to
5//! llama.cpp. This makes it easier to keep up with the changes in llama.cpp, but does mean that
6//! the API is not as nice as it could be.
7//!
8//! # Examples
9//!
10//! - [simple](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/simple)
11//! - [tools](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/tools)
12//!
13//! # Feature Flags
14//!
15//! - `cuda` enables CUDA gpu support.
16//! - `sampler` adds the [`context::sample::sampler`] struct for a more rusty way of sampling.
17use std::ffi::{c_char, CStr, CString, NulError};
18use std::fmt::Debug;
19use std::num::NonZeroI32;
20use std::sync::OnceLock;
21
22use crate::llama_batch::BatchAddError;
23use std::os::raw::c_int;
24use std::path::PathBuf;
25use std::string::FromUtf8Error;
26
27pub mod context;
28pub mod gguf;
29pub mod llama_backend;
30pub mod llama_batch;
31#[cfg(feature = "llguidance")]
32pub(crate) mod llguidance_sampler;
33mod log;
34pub mod model;
35#[cfg(feature = "mtmd")]
36pub mod mtmd;
37mod ptr;
38pub mod sampling;
39#[cfg(feature = "common")]
40pub mod speculative;
41pub mod timing;
42pub mod token;
43pub mod token_type;
44pub mod vocab;
45
46pub use crate::context::session::{LlamaStateSeqFlags, SeqState};
47
48#[cfg(feature = "common")]
49pub(crate) fn status_is_ok(status: llama_cpp_sys_2::llama_rs_status) -> bool {
50    status == llama_cpp_sys_2::LLAMA_RS_STATUS_OK
51}
52
53/// A failable result from a llama.cpp function.
54pub type Result<T> = std::result::Result<T, LlamaCppError>;
55
56/// All errors that can occur in the llama-cpp crate.
57#[derive(Debug, Eq, PartialEq, thiserror::Error)]
58pub enum LlamaCppError {
59    /// The backend was already initialized. This can generally be ignored as initializing the backend
60    /// is idempotent.
61    #[error("BackendAlreadyInitialized")]
62    BackendAlreadyInitialized,
63    /// There was an error while get the chat template from model.
64    #[error("{0}")]
65    ChatTemplateError(#[from] ChatTemplateError),
66    /// There was an error while decoding a batch.
67    #[error("{0}")]
68    DecodeError(#[from] DecodeError),
69    /// There was an error while encoding a batch.
70    #[error("{0}")]
71    EncodeError(#[from] EncodeError),
72    /// There was an error loading a model.
73    #[error("{0}")]
74    LlamaModelLoadError(#[from] LlamaModelLoadError),
75    /// There was an error creating a new model context.
76    #[error("{0}")]
77    LlamaContextLoadError(#[from] LlamaContextLoadError),
78    /// There was an error adding a token to a batch.
79    #[error["{0}"]]
80    BatchAddError(#[from] BatchAddError),
81    /// see [`EmbeddingsError`]
82    #[error(transparent)]
83    EmbeddingError(#[from] EmbeddingsError),
84    // See [`LlamaSamplerError`]
85    /// Backend device not found
86    #[error("Backend device {0} not found")]
87    BackendDeviceNotFound(usize),
88    /// Max devices exceeded
89    #[error("Max devices exceeded. Max devices is {0}")]
90    MaxDevicesExceeded(usize),
91    /// Failed to convert JSON schema to grammar.
92    #[cfg(feature = "common")]
93    #[error("JsonSchemaToGrammarError: {0}")]
94    JsonSchemaToGrammarError(String),
95    /// There was an error fitting model parameters to available memory.
96    #[cfg(feature = "common")]
97    #[error("{0}")]
98    FitError(#[from] crate::model::params::FitError),
99}
100
101/// There was an error while getting the chat template from a model.
102#[derive(Debug, Eq, PartialEq, thiserror::Error)]
103pub enum ChatTemplateError {
104    /// gguf has no chat template (by that name)
105    #[error("chat template not found - returned null pointer")]
106    MissingTemplate,
107
108    /// chat template contained a null byte
109    #[error("null byte in string {0}")]
110    NullError(#[from] NulError),
111
112    /// The chat template was not valid utf8.
113    #[error(transparent)]
114    Utf8Error(#[from] std::str::Utf8Error),
115}
116
117/// Failed fetching metadata value
118#[derive(Debug, Eq, PartialEq, thiserror::Error)]
119pub enum MetaValError {
120    /// The provided string contains an unexpected null-byte
121    #[error("null byte in string {0}")]
122    NullError(#[from] NulError),
123
124    /// The returned data contains invalid UTF8 data
125    #[error("FromUtf8Error {0}")]
126    FromUtf8Error(#[from] FromUtf8Error),
127
128    /// Got negative return value. This happens if the key or index queried does not exist.
129    #[error("Negative return value. Likely due to a missing index or key. Got return value: {0}")]
130    NegativeReturn(i32),
131}
132
133/// Failed to Load context
134#[derive(Debug, Eq, PartialEq, thiserror::Error)]
135pub enum LlamaContextLoadError {
136    /// llama.cpp returned null
137    #[error("null reference from llama.cpp")]
138    NullReturn,
139}
140
141/// Errors from the sequence-state save/restore API.
142#[derive(Debug, Eq, PartialEq, thiserror::Error)]
143pub enum StateSeqError {
144    /// llama.cpp read or wrote a different number of bytes than expected.
145    #[error("state seq size mismatch: expected {expected}, actual {actual}")]
146    SizeMismatch {
147        /// Number of bytes the caller expected to transfer.
148        expected: usize,
149        /// Number of bytes llama.cpp actually transferred.
150        actual: usize,
151    },
152}
153
154/// Failed to decode a batch.
155#[derive(Debug, Eq, PartialEq, thiserror::Error)]
156pub enum DecodeError {
157    /// No kv cache slot was available.
158    #[error("Decode Error 1: NoKvCacheSlot")]
159    NoKvCacheSlot,
160    /// The number of tokens in the batch was 0.
161    #[error("Decode Error -1: n_tokens == 0")]
162    NTokensZero,
163    /// An unknown error occurred.
164    #[error("Decode Error {0}: unknown")]
165    Unknown(c_int),
166}
167
168/// Failed to decode a batch.
169#[derive(Debug, Eq, PartialEq, thiserror::Error)]
170pub enum EncodeError {
171    /// No kv cache slot was available.
172    #[error("Encode Error 1: NoKvCacheSlot")]
173    NoKvCacheSlot,
174    /// The number of tokens in the batch was 0.
175    #[error("Encode Error -1: n_tokens == 0")]
176    NTokensZero,
177    /// An unknown error occurred.
178    #[error("Encode Error {0}: unknown")]
179    Unknown(c_int),
180}
181
182/// Fetching embeddings failed.
183#[derive(Debug, Eq, PartialEq, thiserror::Error)]
184#[error("Embeddings or logits weren't enabled")]
185pub struct EmbeddingsError(());
186
187/// Retrieving embeddings sequences failed.
188#[derive(Debug, Eq, PartialEq, thiserror::Error)]
189#[error("Embeddings or logits weren't enabled, or the model was configured with LLAMA_POOLING_TYPE_NONE")]
190pub struct EmbeddingsSeqError(());
191
192/// Errors that can occur when initializing a grammar sampler
193#[derive(Debug, Eq, PartialEq, thiserror::Error)]
194pub enum GrammarError {
195    /// The grammar root was not found in the grammar string
196    #[error("Grammar root not found in grammar string")]
197    RootNotFound,
198    /// The trigger word contains null bytes
199    #[error("Trigger word contains null bytes")]
200    TriggerWordNullBytes,
201    /// The grammar string or root contains null bytes
202    #[error("Grammar string or root contains null bytes")]
203    GrammarNullBytes,
204    /// The grammar call returned null
205    #[error("Grammar call returned null")]
206    NullGrammar,
207}
208
209/// Decode a error from llama.cpp into a [`DecodeError`].
210impl From<NonZeroI32> for DecodeError {
211    fn from(value: NonZeroI32) -> Self {
212        match value.get() {
213            1 => DecodeError::NoKvCacheSlot,
214            -1 => DecodeError::NTokensZero,
215            i => DecodeError::Unknown(i),
216        }
217    }
218}
219
220/// Encode a error from llama.cpp into a [`EncodeError`].
221impl From<NonZeroI32> for EncodeError {
222    fn from(value: NonZeroI32) -> Self {
223        match value.get() {
224            1 => EncodeError::NoKvCacheSlot,
225            -1 => EncodeError::NTokensZero,
226            i => EncodeError::Unknown(i),
227        }
228    }
229}
230
231/// An error that can occur when loading a model.
232#[derive(Debug, Eq, PartialEq, thiserror::Error)]
233pub enum LlamaModelLoadError {
234    /// There was a null byte in a provided string and thus it could not be converted to a C string.
235    #[error("null byte in string {0}")]
236    NullError(#[from] NulError),
237    /// llama.cpp returned a nullptr - this could be many different causes.
238    #[error("null result from llama cpp")]
239    NullResult,
240    /// Failed to convert the path to a rust str. This means the path was not valid unicode
241    #[error("failed to convert path {0} to str")]
242    PathToStrError(PathBuf),
243}
244
245/// An error that can occur when loading a model.
246#[derive(Debug, Eq, PartialEq, thiserror::Error)]
247pub enum LlamaLoraAdapterInitError {
248    /// There was a null byte in a provided string and thus it could not be converted to a C string.
249    #[error("null byte in string {0}")]
250    NullError(#[from] NulError),
251    /// llama.cpp returned a nullptr - this could be many different causes.
252    #[error("null result from llama cpp")]
253    NullResult,
254    /// Failed to convert the path to a rust str. This means the path was not valid unicode
255    #[error("failed to convert path {0} to str")]
256    PathToStrError(PathBuf),
257}
258
259/// An error that can occur when loading a model.
260#[derive(Debug, Eq, PartialEq, thiserror::Error)]
261pub enum LlamaLoraAdapterSetError {
262    /// llama.cpp returned a non-zero error code.
263    #[error("error code from llama cpp")]
264    ErrorResult(i32),
265}
266
267/// An error that can occur when loading a model.
268#[derive(Debug, Eq, PartialEq, thiserror::Error)]
269pub enum LlamaLoraAdapterRemoveError {
270    /// llama.cpp returned a non-zero error code.
271    #[error("error code from llama cpp")]
272    ErrorResult(i32),
273}
274
275/// get the time (in microseconds) according to llama.cpp
276/// ```
277/// # use llama_cpp_2::llama_time_us;
278/// # use llama_cpp_2::llama_backend::LlamaBackend;
279/// let backend = LlamaBackend::init().unwrap();
280/// let time = llama_time_us();
281/// assert!(time > 0);
282/// ```
283#[must_use]
284pub fn llama_time_us() -> i64 {
285    unsafe { llama_cpp_sys_2::llama_time_us() }
286}
287
288/// get the max number of devices according to llama.cpp (this is generally cuda devices)
289/// ```
290/// # use llama_cpp_2::max_devices;
291/// let max_devices = max_devices();
292/// assert!(max_devices >= 0);
293/// ```
294#[must_use]
295pub fn max_devices() -> usize {
296    unsafe { llama_cpp_sys_2::llama_max_devices() }
297}
298
299/// is memory mapping supported according to llama.cpp
300/// ```
301/// # use llama_cpp_2::mmap_supported;
302/// let mmap_supported = mmap_supported();
303/// if mmap_supported {
304///   println!("mmap_supported!");
305/// }
306/// ```
307#[must_use]
308pub fn mmap_supported() -> bool {
309    unsafe { llama_cpp_sys_2::llama_supports_mmap() }
310}
311
312/// is memory locking supported according to llama.cpp
313/// ```
314/// # use llama_cpp_2::mlock_supported;
315/// let mlock_supported = mlock_supported();
316/// if mlock_supported {
317///    println!("mlock_supported!");
318/// }
319/// ```
320#[must_use]
321pub fn mlock_supported() -> bool {
322    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
323}
324
325/// Convert a JSON schema string into a llama.cpp grammar string.
326#[cfg(feature = "common")]
327pub fn json_schema_to_grammar(schema_json: &str) -> Result<String> {
328    let schema_cstr = CString::new(schema_json)
329        .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
330    let mut out = std::ptr::null_mut();
331    let rc = unsafe {
332        llama_cpp_sys_2::llama_rs_json_schema_to_grammar(schema_cstr.as_ptr(), false, &mut out)
333    };
334
335    let result = {
336        if !status_is_ok(rc) || out.is_null() {
337            return Err(LlamaCppError::JsonSchemaToGrammarError(format!(
338                "ffi error {}",
339                rc
340            )));
341        }
342        let grammar_bytes = unsafe { CStr::from_ptr(out) }.to_bytes().to_vec();
343        let grammar = String::from_utf8(grammar_bytes)
344            .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
345        Ok(grammar)
346    };
347
348    unsafe { llama_cpp_sys_2::llama_rs_string_free(out) };
349    result
350}
351
352#[cfg(all(test, feature = "common"))]
353mod tests {
354    use super::json_schema_to_grammar;
355
356    #[test]
357    fn json_schema_string_api_returns_grammar() {
358        let schema = r#"{
359            "type": "object",
360            "properties": {
361                "city": { "type": "string" },
362                "unit": { "enum": ["c", "f"] }
363            },
364            "required": ["city"]
365        }"#;
366
367        let grammar =
368            json_schema_to_grammar(schema).expect("string-based schema conversion should succeed");
369
370        assert!(grammar.contains("root ::="));
371    }
372}
373
374/// Failed to apply model chat template.
375#[derive(Debug, thiserror::Error)]
376pub enum NewLlamaChatMessageError {
377    /// the string contained a null byte and thus could not be converted to a c string.
378    #[error("{0}")]
379    NulError(#[from] NulError),
380}
381
382/// Failed to apply model chat template.
383#[derive(Debug, thiserror::Error)]
384pub enum ApplyChatTemplateError {
385    /// the string contained a null byte and thus could not be converted to a c string.
386    #[error("{0}")]
387    NulError(#[from] NulError),
388    /// the string could not be converted to utf8.
389    #[error("{0}")]
390    FromUtf8Error(#[from] FromUtf8Error),
391    /// llama.cpp returned a null pointer for the template result.
392    #[error("null result from llama.cpp")]
393    NullResult,
394    /// llama.cpp returned an error code.
395    #[error("ffi error {0}")]
396    FfiError(i32),
397}
398
399/// Failed to accept a token in a sampler.
400#[derive(Debug, thiserror::Error)]
401pub enum SamplerAcceptError {
402    /// llama.cpp returned an error code.
403    #[error("ffi error {0}")]
404    FfiError(i32),
405}
406
407/// Get the time in microseconds according to ggml
408///
409/// ```
410/// # use std::time::Duration;
411/// # use llama_cpp_2::llama_backend::LlamaBackend;
412/// let backend = LlamaBackend::init().unwrap();
413/// use llama_cpp_2::ggml_time_us;
414///
415/// let start = ggml_time_us();
416///
417/// std::thread::sleep(Duration::from_micros(10));
418///
419/// let end = ggml_time_us();
420///
421/// let elapsed = end - start;
422///
423/// assert!(elapsed >= 10)
424#[must_use]
425pub fn ggml_time_us() -> i64 {
426    unsafe { llama_cpp_sys_2::ggml_time_us() }
427}
428
429/// checks if mlock is supported
430///
431/// ```
432/// # use llama_cpp_2::llama_supports_mlock;
433///
434/// if llama_supports_mlock() {
435///   println!("mlock is supported!");
436/// } else {
437///   println!("mlock is not supported!");
438/// }
439/// ```
440#[must_use]
441pub fn llama_supports_mlock() -> bool {
442    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
443}
444
445/// Backend device type
446#[derive(Debug, Clone, Copy, PartialEq, Eq)]
447pub enum LlamaBackendDeviceType {
448    /// CPU device
449    Cpu,
450    /// ACCEL device
451    Accelerator,
452    /// GPU device
453    Gpu,
454    /// iGPU device
455    IntegratedGpu,
456    /// Unknown device type
457    Unknown,
458}
459
460/// A ggml backend device
461///
462/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
463#[derive(Debug, Clone)]
464pub struct LlamaBackendDevice {
465    /// The index of the device
466    ///
467    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
468    pub index: usize,
469    /// The name of the device (e.g. "Vulkan0")
470    pub name: String,
471    /// A description of the device (e.g. "NVIDIA GeForce RTX 3080")
472    pub description: String,
473    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
474    pub backend: String,
475    /// Total memory of the device in bytes
476    pub memory_total: usize,
477    /// Free memory of the device in bytes
478    pub memory_free: usize,
479    /// Device type
480    pub device_type: LlamaBackendDeviceType,
481}
482
483/// List ggml backend devices
484#[must_use]
485pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
486    let mut devices = Vec::new();
487    for i in 0..unsafe { llama_cpp_sys_2::ggml_backend_dev_count() } {
488        fn cstr_to_string(ptr: *const c_char) -> String {
489            if ptr.is_null() {
490                String::new()
491            } else {
492                unsafe { std::ffi::CStr::from_ptr(ptr) }
493                    .to_string_lossy()
494                    .to_string()
495            }
496        }
497        let dev = unsafe { llama_cpp_sys_2::ggml_backend_dev_get(i) };
498        let props = unsafe {
499            let mut props = std::mem::zeroed();
500            llama_cpp_sys_2::ggml_backend_dev_get_props(dev, &raw mut props);
501            props
502        };
503        let name = cstr_to_string(props.name);
504        let description = cstr_to_string(props.description);
505        let backend = unsafe { llama_cpp_sys_2::ggml_backend_dev_backend_reg(dev) };
506        let backend_name = unsafe { llama_cpp_sys_2::ggml_backend_reg_name(backend) };
507        let backend = cstr_to_string(backend_name);
508        let memory_total = props.memory_total;
509        let memory_free = props.memory_free;
510        let device_type = match props.type_ {
511            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
512            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_ACCEL => LlamaBackendDeviceType::Accelerator,
513            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
514            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_IGPU => LlamaBackendDeviceType::IntegratedGpu,
515            _ => LlamaBackendDeviceType::Unknown,
516        };
517        devices.push(LlamaBackendDevice {
518            index: i,
519            name,
520            description,
521            backend,
522            memory_total,
523            memory_free,
524            device_type,
525        });
526    }
527    devices
528}
529
530/// Options to configure how llama.cpp logs are intercepted.
531#[derive(Default, Debug, Clone)]
532pub struct LogOptions {
533    disabled: bool,
534}
535
536impl LogOptions {
537    /// If enabled, logs are sent to tracing. If disabled, all logs are suppressed. Default is for
538    /// logs to be sent to tracing.
539    #[must_use]
540    pub fn with_logs_enabled(mut self, enabled: bool) -> Self {
541        self.disabled = !enabled;
542        self
543    }
544}
545
546extern "C" fn logs_to_trace(
547    level: llama_cpp_sys_2::ggml_log_level,
548    text: *const ::std::os::raw::c_char,
549    data: *mut ::std::os::raw::c_void,
550) {
551    // In the "fast-path" (i.e. the vast majority of logs) we want to avoid needing to take the log state
552    // lock at all. Similarly, we try to avoid any heap allocations within this function. This is accomplished
553    // by being a dummy pass-through to tracing in the normal case of DEBUG/INFO/WARN/ERROR logs that are
554    // newline terminated and limiting the slow-path of locks and/or heap allocations for other cases.
555    use std::borrow::Borrow;
556
557    let log_state = unsafe { &*(data as *const log::State) };
558
559    if log_state.options.disabled {
560        return;
561    }
562
563    // If the log level is disabled, we can just return early
564    if !log_state.is_enabled_for_level(level) {
565        log_state.update_previous_level_for_disabled_log(level);
566        return;
567    }
568
569    let text = unsafe { std::ffi::CStr::from_ptr(text) };
570    let text = text.to_string_lossy();
571    let text: &str = text.borrow();
572
573    // As best I can tell llama.cpp / ggml require all log format strings at call sites to have the '\n'.
574    // If it's missing, it means that you expect more logs via CONT (or there's a typo in the codebase). To
575    // distinguish typo from intentional support for CONT, we have to buffer until the next message comes in
576    // to know how to flush it.
577
578    if level == llama_cpp_sys_2::GGML_LOG_LEVEL_CONT {
579        log_state.cont_buffered_log(text);
580    } else if text.ends_with('\n') {
581        log_state.emit_non_cont_line(level, text);
582    } else {
583        log_state.buffer_non_cont(level, text);
584    }
585}
586
587/// Redirect llama.cpp logs into tracing.
588pub fn send_logs_to_tracing(options: LogOptions) {
589    // TODO: Reinitialize the state to support calling send_logs_to_tracing multiple times.
590
591    // We set up separate log states for each backend to make sure that CONT
592    // logs between them can't possibly interfere with each other.
593    //
594    // In other words, if llama.cpp emits a log without a trailing newline and
595    // calls a GGML function, the logs won't be weirdly intermixed and instead
596    // we'll llama.cpp logs will CONT previous llama.cpp logs and GGML logs
597    // will CONT previous ggml logs.
598    static LLAMA_STATE: OnceLock<Box<log::State>> = OnceLock::new();
599    #[cfg(feature = "mtmd")]
600    static MTMD_STATE: OnceLock<Box<log::State>> = OnceLock::new();
601    static GGML_STATE: OnceLock<Box<log::State>> = OnceLock::new();
602
603    let llama_heap_state = Box::as_ref(
604        LLAMA_STATE
605            .get_or_init(|| Box::new(log::State::new(log::Module::LlamaCpp, options.clone()))),
606    ) as *const _;
607
608    #[cfg(feature = "mtmd")]
609    let mtmd_heap_state = Box::as_ref(
610        MTMD_STATE.get_or_init(|| Box::new(log::State::new(log::Module::Mtmd, options.clone()))),
611    ) as *const _;
612
613    let ggml_heap_state = Box::as_ref(
614        GGML_STATE.get_or_init(|| Box::new(log::State::new(log::Module::GGML, options))),
615    ) as *const _;
616
617    unsafe {
618        llama_cpp_sys_2::llama_log_set(Some(logs_to_trace), llama_heap_state as *mut _);
619
620        // There is both `mtmd_log_set` and `mtmd_helper_log_set`. We use the
621        // helper, since we don't particularly want to differentiate these
622        // two, and the helper calls the normal `mtmd_log_set` as well.
623        #[cfg(feature = "mtmd")]
624        llama_cpp_sys_2::mtmd_helper_log_set(Some(logs_to_trace), mtmd_heap_state as *mut _);
625
626        // GGML has to be set after llama since setting llama logs sets ggml logs as well.
627        llama_cpp_sys_2::ggml_log_set(Some(logs_to_trace), ggml_heap_state as *mut _);
628    }
629}
llama_cpp_2/lib.rs

llama_cpp_2/
lib.rs
