llama_cpp_2/
lib.rs

1//! Bindings to the llama.cpp library.
2//!
3//! As llama.cpp is a very fast moving target, this crate does not attempt to create a stable API
4//! with all the rust idioms. Instead it provided safe wrappers around nearly direct bindings to
5//! llama.cpp. This makes it easier to keep up with the changes in llama.cpp, but does mean that
6//! the API is not as nice as it could be.
7//!
8//! # Examples
9//!
10//! - [simple](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/simple)
11//!
12//! # Feature Flags
13//!
14//! - `cuda` enables CUDA gpu support.
15//! - `sampler` adds the [`context::sample::sampler`] struct for a more rusty way of sampling.
16use std::ffi::{c_char, CStr, CString, NulError};
17use std::fmt::Debug;
18use std::num::NonZeroI32;
19
20use crate::llama_batch::BatchAddError;
21use std::os::raw::c_int;
22use std::path::PathBuf;
23use std::string::FromUtf8Error;
24
25pub mod context;
26pub mod gguf;
27pub mod llama_backend;
28pub mod llama_batch;
29#[cfg(feature = "llguidance")]
30pub(crate) mod llguidance_sampler;
31mod log;
32pub mod model;
33#[cfg(feature = "mtmd")]
34pub mod mtmd;
35pub mod openai;
36pub mod sampling;
37pub mod timing;
38pub mod token;
39pub mod token_type;
40
41pub use crate::context::session::LlamaStateSeqFlags;
42
43pub(crate) fn status_is_ok(status: llama_cpp_sys_2::llama_rs_status) -> bool {
44    status == llama_cpp_sys_2::LLAMA_RS_STATUS_OK
45}
46
47pub(crate) fn status_to_i32(status: llama_cpp_sys_2::llama_rs_status) -> i32 {
48    status as i32
49}
50
51/// A failable result from a llama.cpp function.
52pub type Result<T> = std::result::Result<T, LlamaCppError>;
53
54/// All errors that can occur in the llama-cpp crate.
55#[derive(Debug, Eq, PartialEq, thiserror::Error)]
56pub enum LlamaCppError {
57    /// The backend was already initialized. This can generally be ignored as initializing the backend
58    /// is idempotent.
59    #[error("BackendAlreadyInitialized")]
60    BackendAlreadyInitialized,
61    /// There was an error while get the chat template from model.
62    #[error("{0}")]
63    ChatTemplateError(#[from] ChatTemplateError),
64    /// There was an error while decoding a batch.
65    #[error("{0}")]
66    DecodeError(#[from] DecodeError),
67    /// There was an error while encoding a batch.
68    #[error("{0}")]
69    EncodeError(#[from] EncodeError),
70    /// There was an error loading a model.
71    #[error("{0}")]
72    LlamaModelLoadError(#[from] LlamaModelLoadError),
73    /// There was an error creating a new model context.
74    #[error("{0}")]
75    LlamaContextLoadError(#[from] LlamaContextLoadError),
76    /// There was an error adding a token to a batch.
77    #[error["{0}"]]
78    BatchAddError(#[from] BatchAddError),
79    /// see [`EmbeddingsError`]
80    #[error(transparent)]
81    EmbeddingError(#[from] EmbeddingsError),
82    // See [`LlamaSamplerError`]
83    /// Backend device not found
84    #[error("Backend device {0} not found")]
85    BackendDeviceNotFound(usize),
86    /// Max devices exceeded
87    #[error("Max devices exceeded. Max devices is {0}")]
88    MaxDevicesExceeded(usize),
89    /// Failed to convert JSON schema to grammar.
90    #[error("JsonSchemaToGrammarError: {0}")]
91    JsonSchemaToGrammarError(String),
92}
93
94/// There was an error while getting the chat template from a model.
95#[derive(Debug, Eq, PartialEq, thiserror::Error)]
96pub enum ChatTemplateError {
97    /// gguf has no chat template (by that name)
98    #[error("chat template not found - returned null pointer")]
99    MissingTemplate,
100
101    /// chat template contained a null byte
102    #[error("null byte in string {0}")]
103    NullError(#[from] NulError),
104
105    /// The chat template was not valid utf8.
106    #[error(transparent)]
107    Utf8Error(#[from] std::str::Utf8Error),
108}
109
110/// Failed fetching metadata value
111#[derive(Debug, Eq, PartialEq, thiserror::Error)]
112pub enum MetaValError {
113    /// The provided string contains an unexpected null-byte
114    #[error("null byte in string {0}")]
115    NullError(#[from] NulError),
116
117    /// The returned data contains invalid UTF8 data
118    #[error("FromUtf8Error {0}")]
119    FromUtf8Error(#[from] FromUtf8Error),
120
121    /// Got negative return value. This happens if the key or index queried does not exist.
122    #[error("Negative return value. Likely due to a missing index or key. Got return value: {0}")]
123    NegativeReturn(i32),
124}
125
126/// Failed to Load context
127#[derive(Debug, Eq, PartialEq, thiserror::Error)]
128pub enum LlamaContextLoadError {
129    /// llama.cpp returned null
130    #[error("null reference from llama.cpp")]
131    NullReturn,
132}
133
134/// Failed to decode a batch.
135#[derive(Debug, Eq, PartialEq, thiserror::Error)]
136pub enum DecodeError {
137    /// No kv cache slot was available.
138    #[error("Decode Error 1: NoKvCacheSlot")]
139    NoKvCacheSlot,
140    /// The number of tokens in the batch was 0.
141    #[error("Decode Error -1: n_tokens == 0")]
142    NTokensZero,
143    /// An unknown error occurred.
144    #[error("Decode Error {0}: unknown")]
145    Unknown(c_int),
146}
147
148/// Failed to decode a batch.
149#[derive(Debug, Eq, PartialEq, thiserror::Error)]
150pub enum EncodeError {
151    /// No kv cache slot was available.
152    #[error("Encode Error 1: NoKvCacheSlot")]
153    NoKvCacheSlot,
154    /// The number of tokens in the batch was 0.
155    #[error("Encode Error -1: n_tokens == 0")]
156    NTokensZero,
157    /// An unknown error occurred.
158    #[error("Encode Error {0}: unknown")]
159    Unknown(c_int),
160}
161
162/// When embedding related functions fail
163#[derive(Debug, Eq, PartialEq, thiserror::Error)]
164pub enum EmbeddingsError {
165    /// Embeddings weren't enabled in the context options
166    #[error("Embeddings weren't enabled in the context options")]
167    NotEnabled,
168    /// Logits weren't enabled for the given token
169    #[error("Logits were not enabled for the given token")]
170    LogitsNotEnabled,
171    /// The given sequence index exceeds the max sequence id
172    #[error("Can't use sequence embeddings with a model supporting only LLAMA_POOLING_TYPE_NONE")]
173    NonePoolType,
174}
175
176/// Errors that can occur when initializing a grammar sampler
177#[derive(Debug, Eq, PartialEq, thiserror::Error)]
178pub enum GrammarError {
179    /// The grammar root was not found in the grammar string
180    #[error("Grammar root not found in grammar string")]
181    RootNotFound,
182    /// The trigger word contains null bytes
183    #[error("Trigger word contains null bytes")]
184    TriggerWordNullBytes,
185    /// The grammar string or root contains null bytes
186    #[error("Grammar string or root contains null bytes")]
187    GrammarNullBytes,
188    /// The grammar call returned null
189    #[error("Grammar call returned null")]
190    NullGrammar,
191}
192
193/// Decode a error from llama.cpp into a [`DecodeError`].
194impl From<NonZeroI32> for DecodeError {
195    fn from(value: NonZeroI32) -> Self {
196        match value.get() {
197            1 => DecodeError::NoKvCacheSlot,
198            -1 => DecodeError::NTokensZero,
199            i => DecodeError::Unknown(i),
200        }
201    }
202}
203
204/// Encode a error from llama.cpp into a [`EncodeError`].
205impl From<NonZeroI32> for EncodeError {
206    fn from(value: NonZeroI32) -> Self {
207        match value.get() {
208            1 => EncodeError::NoKvCacheSlot,
209            -1 => EncodeError::NTokensZero,
210            i => EncodeError::Unknown(i),
211        }
212    }
213}
214
215/// An error that can occur when loading a model.
216#[derive(Debug, Eq, PartialEq, thiserror::Error)]
217pub enum LlamaModelLoadError {
218    /// There was a null byte in a provided string and thus it could not be converted to a C string.
219    #[error("null byte in string {0}")]
220    NullError(#[from] NulError),
221    /// llama.cpp returned a nullptr - this could be many different causes.
222    #[error("null result from llama cpp")]
223    NullResult,
224    /// Failed to convert the path to a rust str. This means the path was not valid unicode
225    #[error("failed to convert path {0} to str")]
226    PathToStrError(PathBuf),
227}
228
229/// An error that can occur when loading a model.
230#[derive(Debug, Eq, PartialEq, thiserror::Error)]
231pub enum LlamaLoraAdapterInitError {
232    /// There was a null byte in a provided string and thus it could not be converted to a C string.
233    #[error("null byte in string {0}")]
234    NullError(#[from] NulError),
235    /// llama.cpp returned a nullptr - this could be many different causes.
236    #[error("null result from llama cpp")]
237    NullResult,
238    /// Failed to convert the path to a rust str. This means the path was not valid unicode
239    #[error("failed to convert path {0} to str")]
240    PathToStrError(PathBuf),
241}
242
243/// An error that can occur when loading a model.
244#[derive(Debug, Eq, PartialEq, thiserror::Error)]
245pub enum LlamaLoraAdapterSetError {
246    /// llama.cpp returned a non-zero error code.
247    #[error("error code from llama cpp")]
248    ErrorResult(i32),
249}
250
251/// An error that can occur when loading a model.
252#[derive(Debug, Eq, PartialEq, thiserror::Error)]
253pub enum LlamaLoraAdapterRemoveError {
254    /// llama.cpp returned a non-zero error code.
255    #[error("error code from llama cpp")]
256    ErrorResult(i32),
257}
258
259/// get the time (in microseconds) according to llama.cpp
260/// ```
261/// # use llama_cpp_2::llama_time_us;
262/// # use llama_cpp_2::llama_backend::LlamaBackend;
263/// let backend = LlamaBackend::init().unwrap();
264/// let time = llama_time_us();
265/// assert!(time > 0);
266/// ```
267#[must_use]
268pub fn llama_time_us() -> i64 {
269    unsafe { llama_cpp_sys_2::llama_time_us() }
270}
271
272/// get the max number of devices according to llama.cpp (this is generally cuda devices)
273/// ```
274/// # use llama_cpp_2::max_devices;
275/// let max_devices = max_devices();
276/// assert!(max_devices >= 0);
277/// ```
278#[must_use]
279pub fn max_devices() -> usize {
280    unsafe { llama_cpp_sys_2::llama_max_devices() }
281}
282
283/// is memory mapping supported according to llama.cpp
284/// ```
285/// # use llama_cpp_2::mmap_supported;
286/// let mmap_supported = mmap_supported();
287/// if mmap_supported {
288///   println!("mmap_supported!");
289/// }
290/// ```
291#[must_use]
292pub fn mmap_supported() -> bool {
293    unsafe { llama_cpp_sys_2::llama_supports_mmap() }
294}
295
296/// is memory locking supported according to llama.cpp
297/// ```
298/// # use llama_cpp_2::mlock_supported;
299/// let mlock_supported = mlock_supported();
300/// if mlock_supported {
301///    println!("mlock_supported!");
302/// }
303/// ```
304#[must_use]
305pub fn mlock_supported() -> bool {
306    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
307}
308
309/// Convert a JSON schema string into a llama.cpp grammar string.
310pub fn json_schema_to_grammar(schema_json: &str) -> Result<String> {
311    let schema_cstr = CString::new(schema_json)
312        .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
313    let mut out = std::ptr::null_mut();
314    let rc = unsafe {
315        llama_cpp_sys_2::llama_rs_json_schema_to_grammar(schema_cstr.as_ptr(), false, &mut out)
316    };
317
318    let result = {
319        if !status_is_ok(rc) || out.is_null() {
320            return Err(LlamaCppError::JsonSchemaToGrammarError(format!(
321                "ffi error {}",
322                status_to_i32(rc)
323            )));
324        }
325        let grammar_bytes = unsafe { CStr::from_ptr(out) }.to_bytes().to_vec();
326        let grammar = String::from_utf8(grammar_bytes)
327            .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
328        Ok(grammar)
329    };
330
331    unsafe { llama_cpp_sys_2::llama_rs_string_free(out) };
332    result
333}
334
335/// An error that can occur when converting a token to a string.
336#[derive(Debug, thiserror::Error, Clone)]
337#[non_exhaustive]
338pub enum TokenToStringError {
339    /// the token type was unknown
340    #[error("Unknown Token Type")]
341    UnknownTokenType,
342    /// There was insufficient buffer space to convert the token to a string.
343    #[error("Insufficient Buffer Space {0}")]
344    InsufficientBufferSpace(c_int),
345    /// The token was not valid utf8.
346    #[error("FromUtf8Error {0}")]
347    FromUtf8Error(#[from] FromUtf8Error),
348}
349
350/// Failed to convert a string to a token sequence.
351#[derive(Debug, thiserror::Error)]
352pub enum StringToTokenError {
353    /// the string contained a null byte and thus could not be converted to a c string.
354    #[error("{0}")]
355    NulError(#[from] NulError),
356    #[error("{0}")]
357    /// Failed to convert a provided integer to a [`c_int`].
358    CIntConversionError(#[from] std::num::TryFromIntError),
359}
360
361/// Failed to apply model chat template.
362#[derive(Debug, thiserror::Error)]
363pub enum NewLlamaChatMessageError {
364    /// the string contained a null byte and thus could not be converted to a c string.
365    #[error("{0}")]
366    NulError(#[from] NulError),
367}
368
369/// Failed to apply model chat template.
370#[derive(Debug, thiserror::Error)]
371pub enum ApplyChatTemplateError {
372    /// the string contained a null byte and thus could not be converted to a c string.
373    #[error("{0}")]
374    NulError(#[from] NulError),
375    /// the string could not be converted to utf8.
376    #[error("{0}")]
377    FromUtf8Error(#[from] FromUtf8Error),
378    /// llama.cpp returned a null pointer for the template result.
379    #[error("null result from llama.cpp")]
380    NullResult,
381    /// llama.cpp returned an error code.
382    #[error("ffi error {0}")]
383    FfiError(i32),
384    /// invalid grammar trigger data returned by llama.cpp.
385    #[error("invalid grammar trigger data")]
386    InvalidGrammarTriggerType,
387}
388
389/// Failed to parse a chat response.
390#[derive(Debug, thiserror::Error)]
391pub enum ChatParseError {
392    /// the string contained a null byte and thus could not be converted to a c string.
393    #[error("{0}")]
394    NulError(#[from] NulError),
395    /// the string could not be converted to utf8.
396    #[error("{0}")]
397    Utf8Error(#[from] FromUtf8Error),
398    /// llama.cpp returned a null pointer for the parse result.
399    #[error("null result from llama.cpp")]
400    NullResult,
401    /// llama.cpp returned an error code.
402    #[error("ffi error {0}")]
403    FfiError(i32),
404}
405
406/// Failed to accept a token in a sampler.
407#[derive(Debug, thiserror::Error)]
408pub enum SamplerAcceptError {
409    /// llama.cpp returned an error code.
410    #[error("ffi error {0}")]
411    FfiError(i32),
412}
413
414/// Get the time in microseconds according to ggml
415///
416/// ```
417/// # use std::time::Duration;
418/// # use llama_cpp_2::llama_backend::LlamaBackend;
419/// let backend = LlamaBackend::init().unwrap();
420/// use llama_cpp_2::ggml_time_us;
421///
422/// let start = ggml_time_us();
423///
424/// std::thread::sleep(Duration::from_micros(10));
425///
426/// let end = ggml_time_us();
427///
428/// let elapsed = end - start;
429///
430/// assert!(elapsed >= 10)
431#[must_use]
432pub fn ggml_time_us() -> i64 {
433    unsafe { llama_cpp_sys_2::ggml_time_us() }
434}
435
436/// checks if mlock is supported
437///
438/// ```
439/// # use llama_cpp_2::llama_supports_mlock;
440///
441/// if llama_supports_mlock() {
442///   println!("mlock is supported!");
443/// } else {
444///   println!("mlock is not supported!");
445/// }
446/// ```
447#[must_use]
448pub fn llama_supports_mlock() -> bool {
449    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
450}
451
452/// Backend device type
453#[derive(Debug, Clone, Copy, PartialEq, Eq)]
454pub enum LlamaBackendDeviceType {
455    /// CPU device
456    Cpu,
457    /// ACCEL device
458    Accelerator,
459    /// GPU device
460    Gpu,
461    /// iGPU device
462    IntegratedGpu,
463    /// Unknown device type
464    Unknown,
465}
466
467/// A ggml backend device
468///
469/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
470#[derive(Debug, Clone)]
471pub struct LlamaBackendDevice {
472    /// The index of the device
473    ///
474    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
475    pub index: usize,
476    /// The name of the device (e.g. "Vulkan0")
477    pub name: String,
478    /// A description of the device (e.g. "NVIDIA GeForce RTX 3080")
479    pub description: String,
480    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
481    pub backend: String,
482    /// Total memory of the device in bytes
483    pub memory_total: usize,
484    /// Free memory of the device in bytes
485    pub memory_free: usize,
486    /// Device type
487    pub device_type: LlamaBackendDeviceType,
488}
489
490/// List ggml backend devices
491#[must_use]
492pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
493    let mut devices = Vec::new();
494    for i in 0..unsafe { llama_cpp_sys_2::ggml_backend_dev_count() } {
495        fn cstr_to_string(ptr: *const c_char) -> String {
496            if ptr.is_null() {
497                String::new()
498            } else {
499                unsafe { std::ffi::CStr::from_ptr(ptr) }
500                    .to_string_lossy()
501                    .to_string()
502            }
503        }
504        let dev = unsafe { llama_cpp_sys_2::ggml_backend_dev_get(i) };
505        let props = unsafe {
506            let mut props = std::mem::zeroed();
507            llama_cpp_sys_2::ggml_backend_dev_get_props(dev, &raw mut props);
508            props
509        };
510        let name = cstr_to_string(props.name);
511        let description = cstr_to_string(props.description);
512        let backend = unsafe { llama_cpp_sys_2::ggml_backend_dev_backend_reg(dev) };
513        let backend_name = unsafe { llama_cpp_sys_2::ggml_backend_reg_name(backend) };
514        let backend = cstr_to_string(backend_name);
515        let memory_total = props.memory_total;
516        let memory_free = props.memory_free;
517        let device_type = match props.type_ {
518            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
519            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_ACCEL => LlamaBackendDeviceType::Accelerator,
520            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
521            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_IGPU => LlamaBackendDeviceType::IntegratedGpu,
522            _ => LlamaBackendDeviceType::Unknown,
523        };
524        devices.push(LlamaBackendDevice {
525            index: i,
526            name,
527            description,
528            backend,
529            memory_total,
530            memory_free,
531            device_type,
532        });
533    }
534    devices
535}
536
537/// Options to configure how llama.cpp logs are intercepted.
538#[derive(Default, Debug, Clone)]
539pub struct LogOptions {
540    disabled: bool,
541}
542
543impl LogOptions {
544    /// If enabled, logs are sent to tracing. If disabled, all logs are suppressed. Default is for
545    /// logs to be sent to tracing.
546    #[must_use]
547    pub fn with_logs_enabled(mut self, enabled: bool) -> Self {
548        self.disabled = !enabled;
549        self
550    }
551}
552
553extern "C" fn logs_to_trace(
554    level: llama_cpp_sys_2::ggml_log_level,
555    text: *const ::std::os::raw::c_char,
556    data: *mut ::std::os::raw::c_void,
557) {
558    // In the "fast-path" (i.e. the vast majority of logs) we want to avoid needing to take the log state
559    // lock at all. Similarly, we try to avoid any heap allocations within this function. This is accomplished
560    // by being a dummy pass-through to tracing in the normal case of DEBUG/INFO/WARN/ERROR logs that are
561    // newline terminated and limiting the slow-path of locks and/or heap allocations for other cases.
562    use std::borrow::Borrow;
563
564    let log_state = unsafe { &*(data as *const log::State) };
565
566    if log_state.options.disabled {
567        return;
568    }
569
570    // If the log level is disabled, we can just return early
571    if !log_state.is_enabled_for_level(level) {
572        log_state.update_previous_level_for_disabled_log(level);
573        return;
574    }
575
576    let text = unsafe { std::ffi::CStr::from_ptr(text) };
577    let text = text.to_string_lossy();
578    let text: &str = text.borrow();
579
580    // As best I can tell llama.cpp / ggml require all log format strings at call sites to have the '\n'.
581    // If it's missing, it means that you expect more logs via CONT (or there's a typo in the codebase). To
582    // distinguish typo from intentional support for CONT, we have to buffer until the next message comes in
583    // to know how to flush it.
584
585    if level == llama_cpp_sys_2::GGML_LOG_LEVEL_CONT {
586        log_state.cont_buffered_log(text);
587    } else if text.ends_with('\n') {
588        log_state.emit_non_cont_line(level, text);
589    } else {
590        log_state.buffer_non_cont(level, text);
591    }
592}
593
594/// Redirect llama.cpp logs into tracing.
595pub fn send_logs_to_tracing(options: LogOptions) {
596    // TODO: Reinitialize the state to support calling send_logs_to_tracing multiple times.
597
598    // We set up separate log states for llama.cpp and ggml to make sure that CONT logs between the two
599    // can't possibly interfere with each other. In other words, if llama.cpp emits a log without a trailing
600    // newline and calls a GGML function, the logs won't be weirdly intermixed and instead we'll llama.cpp logs
601    // will CONT previous llama.cpp logs and GGML logs will CONT previous ggml logs.
602    let llama_heap_state = Box::as_ref(
603        log::LLAMA_STATE
604            .get_or_init(|| Box::new(log::State::new(log::Module::LlamaCpp, options.clone()))),
605    ) as *const _;
606    let ggml_heap_state = Box::as_ref(
607        log::GGML_STATE.get_or_init(|| Box::new(log::State::new(log::Module::GGML, options))),
608    ) as *const _;
609
610    unsafe {
611        // GGML has to be set after llama since setting llama sets ggml as well.
612        llama_cpp_sys_2::llama_log_set(Some(logs_to_trace), llama_heap_state as *mut _);
613        llama_cpp_sys_2::ggml_log_set(Some(logs_to_trace), ggml_heap_state as *mut _);
614    }
615}
llama_cpp_2/lib.rs

llama_cpp_2/
lib.rs
