llama_cpp_2/
lib.rs

1//! Bindings to the llama.cpp library.
2//!
3//! As llama.cpp is a very fast moving target, this crate does not attempt to create a stable API
4//! with all the rust idioms. Instead it provided safe wrappers around nearly direct bindings to
5//! llama.cpp. This makes it easier to keep up with the changes in llama.cpp, but does mean that
6//! the API is not as nice as it could be.
7//!
8//! # Examples
9//!
10//! - [simple](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/simple)
11//! - [tools](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/tools)
12//!
13//! # Feature Flags
14//!
15//! - `cuda` enables CUDA gpu support.
16//! - `sampler` adds the [`context::sample::sampler`] struct for a more rusty way of sampling.
17use std::ffi::{c_char, CStr, CString, NulError};
18use std::fmt::Debug;
19use std::num::NonZeroI32;
20use std::sync::OnceLock;
21
22use crate::llama_batch::BatchAddError;
23use std::os::raw::c_int;
24use std::path::PathBuf;
25use std::string::FromUtf8Error;
26
27pub mod context;
28pub mod gguf;
29pub mod llama_backend;
30pub mod llama_batch;
31#[cfg(feature = "llguidance")]
32pub(crate) mod llguidance_sampler;
33mod log;
34pub mod model;
35#[cfg(feature = "mtmd")]
36pub mod mtmd;
37pub mod sampling;
38#[cfg(feature = "common")]
39pub mod speculative;
40pub mod timing;
41pub mod token;
42pub mod token_type;
43pub mod vocab;
44
45pub use crate::context::session::{LlamaStateSeqFlags, SeqState};
46
47#[cfg(feature = "common")]
48pub(crate) fn status_is_ok(status: llama_cpp_sys_2::llama_rs_status) -> bool {
49    status == llama_cpp_sys_2::LLAMA_RS_STATUS_OK
50}
51
52/// A failable result from a llama.cpp function.
53pub type Result<T> = std::result::Result<T, LlamaCppError>;
54
55/// All errors that can occur in the llama-cpp crate.
56#[derive(Debug, Eq, PartialEq, thiserror::Error)]
57pub enum LlamaCppError {
58    /// The backend was already initialized. This can generally be ignored as initializing the backend
59    /// is idempotent.
60    #[error("BackendAlreadyInitialized")]
61    BackendAlreadyInitialized,
62    /// There was an error while get the chat template from model.
63    #[error("{0}")]
64    ChatTemplateError(#[from] ChatTemplateError),
65    /// There was an error while decoding a batch.
66    #[error("{0}")]
67    DecodeError(#[from] DecodeError),
68    /// There was an error while encoding a batch.
69    #[error("{0}")]
70    EncodeError(#[from] EncodeError),
71    /// There was an error loading a model.
72    #[error("{0}")]
73    LlamaModelLoadError(#[from] LlamaModelLoadError),
74    /// There was an error creating a new model context.
75    #[error("{0}")]
76    LlamaContextLoadError(#[from] LlamaContextLoadError),
77    /// There was an error adding a token to a batch.
78    #[error["{0}"]]
79    BatchAddError(#[from] BatchAddError),
80    /// see [`EmbeddingsError`]
81    #[error(transparent)]
82    EmbeddingError(#[from] EmbeddingsError),
83    // See [`LlamaSamplerError`]
84    /// Backend device not found
85    #[error("Backend device {0} not found")]
86    BackendDeviceNotFound(usize),
87    /// Max devices exceeded
88    #[error("Max devices exceeded. Max devices is {0}")]
89    MaxDevicesExceeded(usize),
90    /// Failed to convert JSON schema to grammar.
91    #[cfg(feature = "common")]
92    #[error("JsonSchemaToGrammarError: {0}")]
93    JsonSchemaToGrammarError(String),
94    /// There was an error fitting model parameters to available memory.
95    #[cfg(feature = "common")]
96    #[error("{0}")]
97    FitError(#[from] crate::model::params::FitError),
98}
99
100/// There was an error while getting the chat template from a model.
101#[derive(Debug, Eq, PartialEq, thiserror::Error)]
102pub enum ChatTemplateError {
103    /// gguf has no chat template (by that name)
104    #[error("chat template not found - returned null pointer")]
105    MissingTemplate,
106
107    /// chat template contained a null byte
108    #[error("null byte in string {0}")]
109    NullError(#[from] NulError),
110
111    /// The chat template was not valid utf8.
112    #[error(transparent)]
113    Utf8Error(#[from] std::str::Utf8Error),
114}
115
116/// Failed fetching metadata value
117#[derive(Debug, Eq, PartialEq, thiserror::Error)]
118pub enum MetaValError {
119    /// The provided string contains an unexpected null-byte
120    #[error("null byte in string {0}")]
121    NullError(#[from] NulError),
122
123    /// The returned data contains invalid UTF8 data
124    #[error("FromUtf8Error {0}")]
125    FromUtf8Error(#[from] FromUtf8Error),
126
127    /// Got negative return value. This happens if the key or index queried does not exist.
128    #[error("Negative return value. Likely due to a missing index or key. Got return value: {0}")]
129    NegativeReturn(i32),
130}
131
132/// Failed to Load context
133#[derive(Debug, Eq, PartialEq, thiserror::Error)]
134pub enum LlamaContextLoadError {
135    /// llama.cpp returned null
136    #[error("null reference from llama.cpp")]
137    NullReturn,
138}
139
140/// Errors from the sequence-state save/restore API.
141#[derive(Debug, Eq, PartialEq, thiserror::Error)]
142pub enum StateSeqError {
143    /// llama.cpp read or wrote a different number of bytes than expected.
144    #[error("state seq size mismatch: expected {expected}, actual {actual}")]
145    SizeMismatch {
146        /// Number of bytes the caller expected to transfer.
147        expected: usize,
148        /// Number of bytes llama.cpp actually transferred.
149        actual: usize,
150    },
151}
152
153/// Failed to decode a batch.
154#[derive(Debug, Eq, PartialEq, thiserror::Error)]
155pub enum DecodeError {
156    /// No kv cache slot was available.
157    #[error("Decode Error 1: NoKvCacheSlot")]
158    NoKvCacheSlot,
159    /// The number of tokens in the batch was 0.
160    #[error("Decode Error -1: n_tokens == 0")]
161    NTokensZero,
162    /// An unknown error occurred.
163    #[error("Decode Error {0}: unknown")]
164    Unknown(c_int),
165}
166
167/// Failed to decode a batch.
168#[derive(Debug, Eq, PartialEq, thiserror::Error)]
169pub enum EncodeError {
170    /// No kv cache slot was available.
171    #[error("Encode Error 1: NoKvCacheSlot")]
172    NoKvCacheSlot,
173    /// The number of tokens in the batch was 0.
174    #[error("Encode Error -1: n_tokens == 0")]
175    NTokensZero,
176    /// An unknown error occurred.
177    #[error("Encode Error {0}: unknown")]
178    Unknown(c_int),
179}
180
181/// When embedding related functions fail
182#[derive(Debug, Eq, PartialEq, thiserror::Error)]
183pub enum EmbeddingsError {
184    /// Embeddings weren't enabled in the context options
185    #[error("Embeddings weren't enabled in the context options")]
186    NotEnabled,
187    /// Logits weren't enabled for the given token
188    #[error("Logits were not enabled for the given token")]
189    LogitsNotEnabled,
190    /// The given sequence index exceeds the max sequence id
191    #[error("Can't use sequence embeddings with a model supporting only LLAMA_POOLING_TYPE_NONE")]
192    NonePoolType,
193}
194
195/// Errors that can occur when initializing a grammar sampler
196#[derive(Debug, Eq, PartialEq, thiserror::Error)]
197pub enum GrammarError {
198    /// The grammar root was not found in the grammar string
199    #[error("Grammar root not found in grammar string")]
200    RootNotFound,
201    /// The trigger word contains null bytes
202    #[error("Trigger word contains null bytes")]
203    TriggerWordNullBytes,
204    /// The grammar string or root contains null bytes
205    #[error("Grammar string or root contains null bytes")]
206    GrammarNullBytes,
207    /// The grammar call returned null
208    #[error("Grammar call returned null")]
209    NullGrammar,
210}
211
212/// Decode a error from llama.cpp into a [`DecodeError`].
213impl From<NonZeroI32> for DecodeError {
214    fn from(value: NonZeroI32) -> Self {
215        match value.get() {
216            1 => DecodeError::NoKvCacheSlot,
217            -1 => DecodeError::NTokensZero,
218            i => DecodeError::Unknown(i),
219        }
220    }
221}
222
223/// Encode a error from llama.cpp into a [`EncodeError`].
224impl From<NonZeroI32> for EncodeError {
225    fn from(value: NonZeroI32) -> Self {
226        match value.get() {
227            1 => EncodeError::NoKvCacheSlot,
228            -1 => EncodeError::NTokensZero,
229            i => EncodeError::Unknown(i),
230        }
231    }
232}
233
234/// An error that can occur when loading a model.
235#[derive(Debug, Eq, PartialEq, thiserror::Error)]
236pub enum LlamaModelLoadError {
237    /// There was a null byte in a provided string and thus it could not be converted to a C string.
238    #[error("null byte in string {0}")]
239    NullError(#[from] NulError),
240    /// llama.cpp returned a nullptr - this could be many different causes.
241    #[error("null result from llama cpp")]
242    NullResult,
243    /// Failed to convert the path to a rust str. This means the path was not valid unicode
244    #[error("failed to convert path {0} to str")]
245    PathToStrError(PathBuf),
246}
247
248/// An error that can occur when loading a model.
249#[derive(Debug, Eq, PartialEq, thiserror::Error)]
250pub enum LlamaLoraAdapterInitError {
251    /// There was a null byte in a provided string and thus it could not be converted to a C string.
252    #[error("null byte in string {0}")]
253    NullError(#[from] NulError),
254    /// llama.cpp returned a nullptr - this could be many different causes.
255    #[error("null result from llama cpp")]
256    NullResult,
257    /// Failed to convert the path to a rust str. This means the path was not valid unicode
258    #[error("failed to convert path {0} to str")]
259    PathToStrError(PathBuf),
260}
261
262/// An error that can occur when loading a model.
263#[derive(Debug, Eq, PartialEq, thiserror::Error)]
264pub enum LlamaLoraAdapterSetError {
265    /// llama.cpp returned a non-zero error code.
266    #[error("error code from llama cpp")]
267    ErrorResult(i32),
268}
269
270/// An error that can occur when loading a model.
271#[derive(Debug, Eq, PartialEq, thiserror::Error)]
272pub enum LlamaLoraAdapterRemoveError {
273    /// llama.cpp returned a non-zero error code.
274    #[error("error code from llama cpp")]
275    ErrorResult(i32),
276}
277
278/// get the time (in microseconds) according to llama.cpp
279/// ```
280/// # use llama_cpp_2::llama_time_us;
281/// # use llama_cpp_2::llama_backend::LlamaBackend;
282/// let backend = LlamaBackend::init().unwrap();
283/// let time = llama_time_us();
284/// assert!(time > 0);
285/// ```
286#[must_use]
287pub fn llama_time_us() -> i64 {
288    unsafe { llama_cpp_sys_2::llama_time_us() }
289}
290
291/// get the max number of devices according to llama.cpp (this is generally cuda devices)
292/// ```
293/// # use llama_cpp_2::max_devices;
294/// let max_devices = max_devices();
295/// assert!(max_devices >= 0);
296/// ```
297#[must_use]
298pub fn max_devices() -> usize {
299    unsafe { llama_cpp_sys_2::llama_max_devices() }
300}
301
302/// is memory mapping supported according to llama.cpp
303/// ```
304/// # use llama_cpp_2::mmap_supported;
305/// let mmap_supported = mmap_supported();
306/// if mmap_supported {
307///   println!("mmap_supported!");
308/// }
309/// ```
310#[must_use]
311pub fn mmap_supported() -> bool {
312    unsafe { llama_cpp_sys_2::llama_supports_mmap() }
313}
314
315/// is memory locking supported according to llama.cpp
316/// ```
317/// # use llama_cpp_2::mlock_supported;
318/// let mlock_supported = mlock_supported();
319/// if mlock_supported {
320///    println!("mlock_supported!");
321/// }
322/// ```
323#[must_use]
324pub fn mlock_supported() -> bool {
325    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
326}
327
328/// Convert a JSON schema string into a llama.cpp grammar string.
329#[cfg(feature = "common")]
330pub fn json_schema_to_grammar(schema_json: &str) -> Result<String> {
331    let schema_cstr = CString::new(schema_json)
332        .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
333    let mut out = std::ptr::null_mut();
334    let rc = unsafe {
335        llama_cpp_sys_2::llama_rs_json_schema_to_grammar(schema_cstr.as_ptr(), false, &mut out)
336    };
337
338    let result = {
339        if !status_is_ok(rc) || out.is_null() {
340            return Err(LlamaCppError::JsonSchemaToGrammarError(format!(
341                "ffi error {}",
342                rc
343            )));
344        }
345        let grammar_bytes = unsafe { CStr::from_ptr(out) }.to_bytes().to_vec();
346        let grammar = String::from_utf8(grammar_bytes)
347            .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
348        Ok(grammar)
349    };
350
351    unsafe { llama_cpp_sys_2::llama_rs_string_free(out) };
352    result
353}
354
355#[cfg(all(test, feature = "common"))]
356mod tests {
357    use super::json_schema_to_grammar;
358
359    #[test]
360    fn json_schema_string_api_returns_grammar() {
361        let schema = r#"{
362            "type": "object",
363            "properties": {
364                "city": { "type": "string" },
365                "unit": { "enum": ["c", "f"] }
366            },
367            "required": ["city"]
368        }"#;
369
370        let grammar =
371            json_schema_to_grammar(schema).expect("string-based schema conversion should succeed");
372
373        assert!(grammar.contains("root ::="));
374    }
375}
376
377/// Failed to apply model chat template.
378#[derive(Debug, thiserror::Error)]
379pub enum NewLlamaChatMessageError {
380    /// the string contained a null byte and thus could not be converted to a c string.
381    #[error("{0}")]
382    NulError(#[from] NulError),
383}
384
385/// Failed to apply model chat template.
386#[derive(Debug, thiserror::Error)]
387pub enum ApplyChatTemplateError {
388    /// the string contained a null byte and thus could not be converted to a c string.
389    #[error("{0}")]
390    NulError(#[from] NulError),
391    /// the string could not be converted to utf8.
392    #[error("{0}")]
393    FromUtf8Error(#[from] FromUtf8Error),
394    /// llama.cpp returned a null pointer for the template result.
395    #[error("null result from llama.cpp")]
396    NullResult,
397    /// llama.cpp returned an error code.
398    #[error("ffi error {0}")]
399    FfiError(i32),
400}
401
402/// Failed to accept a token in a sampler.
403#[derive(Debug, thiserror::Error)]
404pub enum SamplerAcceptError {
405    /// llama.cpp returned an error code.
406    #[error("ffi error {0}")]
407    FfiError(i32),
408}
409
410/// Get the time in microseconds according to ggml
411///
412/// ```
413/// # use std::time::Duration;
414/// # use llama_cpp_2::llama_backend::LlamaBackend;
415/// let backend = LlamaBackend::init().unwrap();
416/// use llama_cpp_2::ggml_time_us;
417///
418/// let start = ggml_time_us();
419///
420/// std::thread::sleep(Duration::from_micros(10));
421///
422/// let end = ggml_time_us();
423///
424/// let elapsed = end - start;
425///
426/// assert!(elapsed >= 10)
427#[must_use]
428pub fn ggml_time_us() -> i64 {
429    unsafe { llama_cpp_sys_2::ggml_time_us() }
430}
431
432/// checks if mlock is supported
433///
434/// ```
435/// # use llama_cpp_2::llama_supports_mlock;
436///
437/// if llama_supports_mlock() {
438///   println!("mlock is supported!");
439/// } else {
440///   println!("mlock is not supported!");
441/// }
442/// ```
443#[must_use]
444pub fn llama_supports_mlock() -> bool {
445    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
446}
447
448/// Backend device type
449#[derive(Debug, Clone, Copy, PartialEq, Eq)]
450pub enum LlamaBackendDeviceType {
451    /// CPU device
452    Cpu,
453    /// ACCEL device
454    Accelerator,
455    /// GPU device
456    Gpu,
457    /// iGPU device
458    IntegratedGpu,
459    /// Unknown device type
460    Unknown,
461}
462
463/// A ggml backend device
464///
465/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
466#[derive(Debug, Clone)]
467pub struct LlamaBackendDevice {
468    /// The index of the device
469    ///
470    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
471    pub index: usize,
472    /// The name of the device (e.g. "Vulkan0")
473    pub name: String,
474    /// A description of the device (e.g. "NVIDIA GeForce RTX 3080")
475    pub description: String,
476    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
477    pub backend: String,
478    /// Total memory of the device in bytes
479    pub memory_total: usize,
480    /// Free memory of the device in bytes
481    pub memory_free: usize,
482    /// Device type
483    pub device_type: LlamaBackendDeviceType,
484}
485
486/// List ggml backend devices
487#[must_use]
488pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
489    let mut devices = Vec::new();
490    for i in 0..unsafe { llama_cpp_sys_2::ggml_backend_dev_count() } {
491        fn cstr_to_string(ptr: *const c_char) -> String {
492            if ptr.is_null() {
493                String::new()
494            } else {
495                unsafe { std::ffi::CStr::from_ptr(ptr) }
496                    .to_string_lossy()
497                    .to_string()
498            }
499        }
500        let dev = unsafe { llama_cpp_sys_2::ggml_backend_dev_get(i) };
501        let props = unsafe {
502            let mut props = std::mem::zeroed();
503            llama_cpp_sys_2::ggml_backend_dev_get_props(dev, &raw mut props);
504            props
505        };
506        let name = cstr_to_string(props.name);
507        let description = cstr_to_string(props.description);
508        let backend = unsafe { llama_cpp_sys_2::ggml_backend_dev_backend_reg(dev) };
509        let backend_name = unsafe { llama_cpp_sys_2::ggml_backend_reg_name(backend) };
510        let backend = cstr_to_string(backend_name);
511        let memory_total = props.memory_total;
512        let memory_free = props.memory_free;
513        let device_type = match props.type_ {
514            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
515            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_ACCEL => LlamaBackendDeviceType::Accelerator,
516            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
517            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_IGPU => LlamaBackendDeviceType::IntegratedGpu,
518            _ => LlamaBackendDeviceType::Unknown,
519        };
520        devices.push(LlamaBackendDevice {
521            index: i,
522            name,
523            description,
524            backend,
525            memory_total,
526            memory_free,
527            device_type,
528        });
529    }
530    devices
531}
532
533/// Options to configure how llama.cpp logs are intercepted.
534#[derive(Default, Debug, Clone)]
535pub struct LogOptions {
536    disabled: bool,
537}
538
539impl LogOptions {
540    /// If enabled, logs are sent to tracing. If disabled, all logs are suppressed. Default is for
541    /// logs to be sent to tracing.
542    #[must_use]
543    pub fn with_logs_enabled(mut self, enabled: bool) -> Self {
544        self.disabled = !enabled;
545        self
546    }
547}
548
549extern "C" fn logs_to_trace(
550    level: llama_cpp_sys_2::ggml_log_level,
551    text: *const ::std::os::raw::c_char,
552    data: *mut ::std::os::raw::c_void,
553) {
554    // In the "fast-path" (i.e. the vast majority of logs) we want to avoid needing to take the log state
555    // lock at all. Similarly, we try to avoid any heap allocations within this function. This is accomplished
556    // by being a dummy pass-through to tracing in the normal case of DEBUG/INFO/WARN/ERROR logs that are
557    // newline terminated and limiting the slow-path of locks and/or heap allocations for other cases.
558    use std::borrow::Borrow;
559
560    let log_state = unsafe { &*(data as *const log::State) };
561
562    if log_state.options.disabled {
563        return;
564    }
565
566    // If the log level is disabled, we can just return early
567    if !log_state.is_enabled_for_level(level) {
568        log_state.update_previous_level_for_disabled_log(level);
569        return;
570    }
571
572    let text = unsafe { std::ffi::CStr::from_ptr(text) };
573    let text = text.to_string_lossy();
574    let text: &str = text.borrow();
575
576    // As best I can tell llama.cpp / ggml require all log format strings at call sites to have the '\n'.
577    // If it's missing, it means that you expect more logs via CONT (or there's a typo in the codebase). To
578    // distinguish typo from intentional support for CONT, we have to buffer until the next message comes in
579    // to know how to flush it.
580
581    if level == llama_cpp_sys_2::GGML_LOG_LEVEL_CONT {
582        log_state.cont_buffered_log(text);
583    } else if text.ends_with('\n') {
584        log_state.emit_non_cont_line(level, text);
585    } else {
586        log_state.buffer_non_cont(level, text);
587    }
588}
589
590/// Redirect llama.cpp logs into tracing.
591pub fn send_logs_to_tracing(options: LogOptions) {
592    // TODO: Reinitialize the state to support calling send_logs_to_tracing multiple times.
593
594    // We set up separate log states for each backend to make sure that CONT
595    // logs between them can't possibly interfere with each other.
596    //
597    // In other words, if llama.cpp emits a log without a trailing newline and
598    // calls a GGML function, the logs won't be weirdly intermixed and instead
599    // we'll llama.cpp logs will CONT previous llama.cpp logs and GGML logs
600    // will CONT previous ggml logs.
601    static LLAMA_STATE: OnceLock<Box<log::State>> = OnceLock::new();
602    #[cfg(feature = "mtmd")]
603    static MTMD_STATE: OnceLock<Box<log::State>> = OnceLock::new();
604    static GGML_STATE: OnceLock<Box<log::State>> = OnceLock::new();
605
606    let llama_heap_state = Box::as_ref(
607        LLAMA_STATE
608            .get_or_init(|| Box::new(log::State::new(log::Module::LlamaCpp, options.clone()))),
609    ) as *const _;
610
611    #[cfg(feature = "mtmd")]
612    let mtmd_heap_state = Box::as_ref(
613        MTMD_STATE.get_or_init(|| Box::new(log::State::new(log::Module::Mtmd, options.clone()))),
614    ) as *const _;
615
616    let ggml_heap_state = Box::as_ref(
617        GGML_STATE.get_or_init(|| Box::new(log::State::new(log::Module::GGML, options))),
618    ) as *const _;
619
620    unsafe {
621        llama_cpp_sys_2::llama_log_set(Some(logs_to_trace), llama_heap_state as *mut _);
622
623        // There is both `mtmd_log_set` and `mtmd_helper_log_set`. We use the
624        // helper, since we don't particularly want to differentiate these
625        // two, and the helper calls the normal `mtmd_log_set` as well.
626        #[cfg(feature = "mtmd")]
627        llama_cpp_sys_2::mtmd_helper_log_set(Some(logs_to_trace), mtmd_heap_state as *mut _);
628
629        // GGML has to be set after llama since setting llama logs sets ggml logs as well.
630        llama_cpp_sys_2::ggml_log_set(Some(logs_to_trace), ggml_heap_state as *mut _);
631    }
632}
llama_cpp_2/lib.rs

llama_cpp_2/
lib.rs
