llama_cpp_2/
lib.rs

1//! Bindings to the llama.cpp library.
2//!
3//! As llama.cpp is a very fast moving target, this crate does not attempt to create a stable API
4//! with all the rust idioms. Instead it provided safe wrappers around nearly direct bindings to
5//! llama.cpp. This makes it easier to keep up with the changes in llama.cpp, but does mean that
6//! the API is not as nice as it could be.
7//!
8//! # Examples
9//!
10//! - [simple](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/simple)
11//! - [tools](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/tools)
12//!
13//! # Feature Flags
14//!
15//! - `cuda` enables CUDA gpu support.
16//! - `sampler` adds the [`context::sample::sampler`] struct for a more rusty way of sampling.
17use std::ffi::{c_char, CStr, CString, NulError};
18use std::fmt::Debug;
19use std::num::NonZeroI32;
20
21use crate::llama_batch::BatchAddError;
22use std::os::raw::c_int;
23use std::path::PathBuf;
24use std::string::FromUtf8Error;
25
26pub mod context;
27pub mod gguf;
28pub mod llama_backend;
29pub mod llama_batch;
30#[cfg(feature = "llguidance")]
31pub(crate) mod llguidance_sampler;
32mod log;
33pub mod model;
34#[cfg(feature = "mtmd")]
35pub mod mtmd;
36pub mod sampling;
37#[cfg(feature = "common")]
38pub mod speculative;
39pub mod timing;
40pub mod token;
41pub mod token_type;
42
43pub use crate::context::session::{LlamaStateSeqFlags, SeqState};
44
45#[cfg(feature = "common")]
46pub(crate) fn status_is_ok(status: llama_cpp_sys_2::llama_rs_status) -> bool {
47    status == llama_cpp_sys_2::LLAMA_RS_STATUS_OK
48}
49
50/// A failable result from a llama.cpp function.
51pub type Result<T> = std::result::Result<T, LlamaCppError>;
52
53/// All errors that can occur in the llama-cpp crate.
54#[derive(Debug, Eq, PartialEq, thiserror::Error)]
55pub enum LlamaCppError {
56    /// The backend was already initialized. This can generally be ignored as initializing the backend
57    /// is idempotent.
58    #[error("BackendAlreadyInitialized")]
59    BackendAlreadyInitialized,
60    /// There was an error while get the chat template from model.
61    #[error("{0}")]
62    ChatTemplateError(#[from] ChatTemplateError),
63    /// There was an error while decoding a batch.
64    #[error("{0}")]
65    DecodeError(#[from] DecodeError),
66    /// There was an error while encoding a batch.
67    #[error("{0}")]
68    EncodeError(#[from] EncodeError),
69    /// There was an error loading a model.
70    #[error("{0}")]
71    LlamaModelLoadError(#[from] LlamaModelLoadError),
72    /// There was an error creating a new model context.
73    #[error("{0}")]
74    LlamaContextLoadError(#[from] LlamaContextLoadError),
75    /// There was an error adding a token to a batch.
76    #[error["{0}"]]
77    BatchAddError(#[from] BatchAddError),
78    /// see [`EmbeddingsError`]
79    #[error(transparent)]
80    EmbeddingError(#[from] EmbeddingsError),
81    // See [`LlamaSamplerError`]
82    /// Backend device not found
83    #[error("Backend device {0} not found")]
84    BackendDeviceNotFound(usize),
85    /// Max devices exceeded
86    #[error("Max devices exceeded. Max devices is {0}")]
87    MaxDevicesExceeded(usize),
88    /// Failed to convert JSON schema to grammar.
89    #[cfg(feature = "common")]
90    #[error("JsonSchemaToGrammarError: {0}")]
91    JsonSchemaToGrammarError(String),
92    /// There was an error fitting model parameters to available memory.
93    #[cfg(feature = "common")]
94    #[error("{0}")]
95    FitError(#[from] crate::model::params::FitError),
96}
97
98/// There was an error while getting the chat template from a model.
99#[derive(Debug, Eq, PartialEq, thiserror::Error)]
100pub enum ChatTemplateError {
101    /// gguf has no chat template (by that name)
102    #[error("chat template not found - returned null pointer")]
103    MissingTemplate,
104
105    /// chat template contained a null byte
106    #[error("null byte in string {0}")]
107    NullError(#[from] NulError),
108
109    /// The chat template was not valid utf8.
110    #[error(transparent)]
111    Utf8Error(#[from] std::str::Utf8Error),
112}
113
114/// Failed fetching metadata value
115#[derive(Debug, Eq, PartialEq, thiserror::Error)]
116pub enum MetaValError {
117    /// The provided string contains an unexpected null-byte
118    #[error("null byte in string {0}")]
119    NullError(#[from] NulError),
120
121    /// The returned data contains invalid UTF8 data
122    #[error("FromUtf8Error {0}")]
123    FromUtf8Error(#[from] FromUtf8Error),
124
125    /// Got negative return value. This happens if the key or index queried does not exist.
126    #[error("Negative return value. Likely due to a missing index or key. Got return value: {0}")]
127    NegativeReturn(i32),
128}
129
130/// Failed to Load context
131#[derive(Debug, Eq, PartialEq, thiserror::Error)]
132pub enum LlamaContextLoadError {
133    /// llama.cpp returned null
134    #[error("null reference from llama.cpp")]
135    NullReturn,
136}
137
138/// Errors from the sequence-state save/restore API.
139#[derive(Debug, Eq, PartialEq, thiserror::Error)]
140pub enum StateSeqError {
141    /// llama.cpp read or wrote a different number of bytes than expected.
142    #[error("state seq size mismatch: expected {expected}, actual {actual}")]
143    SizeMismatch {
144        /// Number of bytes the caller expected to transfer.
145        expected: usize,
146        /// Number of bytes llama.cpp actually transferred.
147        actual: usize,
148    },
149}
150
151/// Failed to decode a batch.
152#[derive(Debug, Eq, PartialEq, thiserror::Error)]
153pub enum DecodeError {
154    /// No kv cache slot was available.
155    #[error("Decode Error 1: NoKvCacheSlot")]
156    NoKvCacheSlot,
157    /// The number of tokens in the batch was 0.
158    #[error("Decode Error -1: n_tokens == 0")]
159    NTokensZero,
160    /// An unknown error occurred.
161    #[error("Decode Error {0}: unknown")]
162    Unknown(c_int),
163}
164
165/// Failed to decode a batch.
166#[derive(Debug, Eq, PartialEq, thiserror::Error)]
167pub enum EncodeError {
168    /// No kv cache slot was available.
169    #[error("Encode Error 1: NoKvCacheSlot")]
170    NoKvCacheSlot,
171    /// The number of tokens in the batch was 0.
172    #[error("Encode Error -1: n_tokens == 0")]
173    NTokensZero,
174    /// An unknown error occurred.
175    #[error("Encode Error {0}: unknown")]
176    Unknown(c_int),
177}
178
179/// When embedding related functions fail
180#[derive(Debug, Eq, PartialEq, thiserror::Error)]
181pub enum EmbeddingsError {
182    /// Embeddings weren't enabled in the context options
183    #[error("Embeddings weren't enabled in the context options")]
184    NotEnabled,
185    /// Logits weren't enabled for the given token
186    #[error("Logits were not enabled for the given token")]
187    LogitsNotEnabled,
188    /// The given sequence index exceeds the max sequence id
189    #[error("Can't use sequence embeddings with a model supporting only LLAMA_POOLING_TYPE_NONE")]
190    NonePoolType,
191}
192
193/// Errors that can occur when initializing a grammar sampler
194#[derive(Debug, Eq, PartialEq, thiserror::Error)]
195pub enum GrammarError {
196    /// The grammar root was not found in the grammar string
197    #[error("Grammar root not found in grammar string")]
198    RootNotFound,
199    /// The trigger word contains null bytes
200    #[error("Trigger word contains null bytes")]
201    TriggerWordNullBytes,
202    /// The grammar string or root contains null bytes
203    #[error("Grammar string or root contains null bytes")]
204    GrammarNullBytes,
205    /// The grammar call returned null
206    #[error("Grammar call returned null")]
207    NullGrammar,
208}
209
210/// Decode a error from llama.cpp into a [`DecodeError`].
211impl From<NonZeroI32> for DecodeError {
212    fn from(value: NonZeroI32) -> Self {
213        match value.get() {
214            1 => DecodeError::NoKvCacheSlot,
215            -1 => DecodeError::NTokensZero,
216            i => DecodeError::Unknown(i),
217        }
218    }
219}
220
221/// Encode a error from llama.cpp into a [`EncodeError`].
222impl From<NonZeroI32> for EncodeError {
223    fn from(value: NonZeroI32) -> Self {
224        match value.get() {
225            1 => EncodeError::NoKvCacheSlot,
226            -1 => EncodeError::NTokensZero,
227            i => EncodeError::Unknown(i),
228        }
229    }
230}
231
232/// An error that can occur when loading a model.
233#[derive(Debug, Eq, PartialEq, thiserror::Error)]
234pub enum LlamaModelLoadError {
235    /// There was a null byte in a provided string and thus it could not be converted to a C string.
236    #[error("null byte in string {0}")]
237    NullError(#[from] NulError),
238    /// llama.cpp returned a nullptr - this could be many different causes.
239    #[error("null result from llama cpp")]
240    NullResult,
241    /// Failed to convert the path to a rust str. This means the path was not valid unicode
242    #[error("failed to convert path {0} to str")]
243    PathToStrError(PathBuf),
244}
245
246/// An error that can occur when loading a model.
247#[derive(Debug, Eq, PartialEq, thiserror::Error)]
248pub enum LlamaLoraAdapterInitError {
249    /// There was a null byte in a provided string and thus it could not be converted to a C string.
250    #[error("null byte in string {0}")]
251    NullError(#[from] NulError),
252    /// llama.cpp returned a nullptr - this could be many different causes.
253    #[error("null result from llama cpp")]
254    NullResult,
255    /// Failed to convert the path to a rust str. This means the path was not valid unicode
256    #[error("failed to convert path {0} to str")]
257    PathToStrError(PathBuf),
258}
259
260/// An error that can occur when loading a model.
261#[derive(Debug, Eq, PartialEq, thiserror::Error)]
262pub enum LlamaLoraAdapterSetError {
263    /// llama.cpp returned a non-zero error code.
264    #[error("error code from llama cpp")]
265    ErrorResult(i32),
266}
267
268/// An error that can occur when loading a model.
269#[derive(Debug, Eq, PartialEq, thiserror::Error)]
270pub enum LlamaLoraAdapterRemoveError {
271    /// llama.cpp returned a non-zero error code.
272    #[error("error code from llama cpp")]
273    ErrorResult(i32),
274}
275
276/// get the time (in microseconds) according to llama.cpp
277/// ```
278/// # use llama_cpp_2::llama_time_us;
279/// # use llama_cpp_2::llama_backend::LlamaBackend;
280/// let backend = LlamaBackend::init().unwrap();
281/// let time = llama_time_us();
282/// assert!(time > 0);
283/// ```
284#[must_use]
285pub fn llama_time_us() -> i64 {
286    unsafe { llama_cpp_sys_2::llama_time_us() }
287}
288
289/// get the max number of devices according to llama.cpp (this is generally cuda devices)
290/// ```
291/// # use llama_cpp_2::max_devices;
292/// let max_devices = max_devices();
293/// assert!(max_devices >= 0);
294/// ```
295#[must_use]
296pub fn max_devices() -> usize {
297    unsafe { llama_cpp_sys_2::llama_max_devices() }
298}
299
300/// is memory mapping supported according to llama.cpp
301/// ```
302/// # use llama_cpp_2::mmap_supported;
303/// let mmap_supported = mmap_supported();
304/// if mmap_supported {
305///   println!("mmap_supported!");
306/// }
307/// ```
308#[must_use]
309pub fn mmap_supported() -> bool {
310    unsafe { llama_cpp_sys_2::llama_supports_mmap() }
311}
312
313/// is memory locking supported according to llama.cpp
314/// ```
315/// # use llama_cpp_2::mlock_supported;
316/// let mlock_supported = mlock_supported();
317/// if mlock_supported {
318///    println!("mlock_supported!");
319/// }
320/// ```
321#[must_use]
322pub fn mlock_supported() -> bool {
323    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
324}
325
326/// Convert a JSON schema string into a llama.cpp grammar string.
327#[cfg(feature = "common")]
328pub fn json_schema_to_grammar(schema_json: &str) -> Result<String> {
329    let schema_cstr = CString::new(schema_json)
330        .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
331    let mut out = std::ptr::null_mut();
332    let rc = unsafe {
333        llama_cpp_sys_2::llama_rs_json_schema_to_grammar(schema_cstr.as_ptr(), false, &mut out)
334    };
335
336    let result = {
337        if !status_is_ok(rc) || out.is_null() {
338            return Err(LlamaCppError::JsonSchemaToGrammarError(format!(
339                "ffi error {}",
340                rc
341            )));
342        }
343        let grammar_bytes = unsafe { CStr::from_ptr(out) }.to_bytes().to_vec();
344        let grammar = String::from_utf8(grammar_bytes)
345            .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
346        Ok(grammar)
347    };
348
349    unsafe { llama_cpp_sys_2::llama_rs_string_free(out) };
350    result
351}
352
353#[cfg(all(test, feature = "common"))]
354mod tests {
355    use super::json_schema_to_grammar;
356
357    #[test]
358    fn json_schema_string_api_returns_grammar() {
359        let schema = r#"{
360            "type": "object",
361            "properties": {
362                "city": { "type": "string" },
363                "unit": { "enum": ["c", "f"] }
364            },
365            "required": ["city"]
366        }"#;
367
368        let grammar =
369            json_schema_to_grammar(schema).expect("string-based schema conversion should succeed");
370
371        assert!(grammar.contains("root ::="));
372    }
373}
374
375/// An error that can occur when converting a token to a string.
376#[derive(Debug, thiserror::Error, Clone)]
377#[non_exhaustive]
378pub enum TokenToStringError {
379    /// the token type was unknown
380    #[error("Unknown Token Type")]
381    UnknownTokenType,
382    /// There was insufficient buffer space to convert the token to a string.
383    #[error("Insufficient Buffer Space {0}")]
384    InsufficientBufferSpace(c_int),
385    /// The token was not valid utf8.
386    #[error("FromUtf8Error {0}")]
387    FromUtf8Error(#[from] FromUtf8Error),
388}
389
390/// Failed to convert a string to a token sequence.
391#[derive(Debug, thiserror::Error)]
392pub enum StringToTokenError {
393    /// the string contained a null byte and thus could not be converted to a c string.
394    #[error("{0}")]
395    NulError(#[from] NulError),
396    #[error("{0}")]
397    /// Failed to convert a provided integer to a [`c_int`].
398    CIntConversionError(#[from] std::num::TryFromIntError),
399}
400
401/// Failed to apply model chat template.
402#[derive(Debug, thiserror::Error)]
403pub enum NewLlamaChatMessageError {
404    /// the string contained a null byte and thus could not be converted to a c string.
405    #[error("{0}")]
406    NulError(#[from] NulError),
407}
408
409/// Failed to apply model chat template.
410#[derive(Debug, thiserror::Error)]
411pub enum ApplyChatTemplateError {
412    /// the string contained a null byte and thus could not be converted to a c string.
413    #[error("{0}")]
414    NulError(#[from] NulError),
415    /// the string could not be converted to utf8.
416    #[error("{0}")]
417    FromUtf8Error(#[from] FromUtf8Error),
418    /// llama.cpp returned a null pointer for the template result.
419    #[error("null result from llama.cpp")]
420    NullResult,
421    /// llama.cpp returned an error code.
422    #[error("ffi error {0}")]
423    FfiError(i32),
424}
425
426/// Failed to accept a token in a sampler.
427#[derive(Debug, thiserror::Error)]
428pub enum SamplerAcceptError {
429    /// llama.cpp returned an error code.
430    #[error("ffi error {0}")]
431    FfiError(i32),
432}
433
434/// Get the time in microseconds according to ggml
435///
436/// ```
437/// # use std::time::Duration;
438/// # use llama_cpp_2::llama_backend::LlamaBackend;
439/// let backend = LlamaBackend::init().unwrap();
440/// use llama_cpp_2::ggml_time_us;
441///
442/// let start = ggml_time_us();
443///
444/// std::thread::sleep(Duration::from_micros(10));
445///
446/// let end = ggml_time_us();
447///
448/// let elapsed = end - start;
449///
450/// assert!(elapsed >= 10)
451#[must_use]
452pub fn ggml_time_us() -> i64 {
453    unsafe { llama_cpp_sys_2::ggml_time_us() }
454}
455
456/// checks if mlock is supported
457///
458/// ```
459/// # use llama_cpp_2::llama_supports_mlock;
460///
461/// if llama_supports_mlock() {
462///   println!("mlock is supported!");
463/// } else {
464///   println!("mlock is not supported!");
465/// }
466/// ```
467#[must_use]
468pub fn llama_supports_mlock() -> bool {
469    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
470}
471
472/// Backend device type
473#[derive(Debug, Clone, Copy, PartialEq, Eq)]
474pub enum LlamaBackendDeviceType {
475    /// CPU device
476    Cpu,
477    /// ACCEL device
478    Accelerator,
479    /// GPU device
480    Gpu,
481    /// iGPU device
482    IntegratedGpu,
483    /// Unknown device type
484    Unknown,
485}
486
487/// A ggml backend device
488///
489/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
490#[derive(Debug, Clone)]
491pub struct LlamaBackendDevice {
492    /// The index of the device
493    ///
494    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
495    pub index: usize,
496    /// The name of the device (e.g. "Vulkan0")
497    pub name: String,
498    /// A description of the device (e.g. "NVIDIA GeForce RTX 3080")
499    pub description: String,
500    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
501    pub backend: String,
502    /// Total memory of the device in bytes
503    pub memory_total: usize,
504    /// Free memory of the device in bytes
505    pub memory_free: usize,
506    /// Device type
507    pub device_type: LlamaBackendDeviceType,
508}
509
510/// List ggml backend devices
511#[must_use]
512pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
513    let mut devices = Vec::new();
514    for i in 0..unsafe { llama_cpp_sys_2::ggml_backend_dev_count() } {
515        fn cstr_to_string(ptr: *const c_char) -> String {
516            if ptr.is_null() {
517                String::new()
518            } else {
519                unsafe { std::ffi::CStr::from_ptr(ptr) }
520                    .to_string_lossy()
521                    .to_string()
522            }
523        }
524        let dev = unsafe { llama_cpp_sys_2::ggml_backend_dev_get(i) };
525        let props = unsafe {
526            let mut props = std::mem::zeroed();
527            llama_cpp_sys_2::ggml_backend_dev_get_props(dev, &raw mut props);
528            props
529        };
530        let name = cstr_to_string(props.name);
531        let description = cstr_to_string(props.description);
532        let backend = unsafe { llama_cpp_sys_2::ggml_backend_dev_backend_reg(dev) };
533        let backend_name = unsafe { llama_cpp_sys_2::ggml_backend_reg_name(backend) };
534        let backend = cstr_to_string(backend_name);
535        let memory_total = props.memory_total;
536        let memory_free = props.memory_free;
537        let device_type = match props.type_ {
538            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
539            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_ACCEL => LlamaBackendDeviceType::Accelerator,
540            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
541            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_IGPU => LlamaBackendDeviceType::IntegratedGpu,
542            _ => LlamaBackendDeviceType::Unknown,
543        };
544        devices.push(LlamaBackendDevice {
545            index: i,
546            name,
547            description,
548            backend,
549            memory_total,
550            memory_free,
551            device_type,
552        });
553    }
554    devices
555}
556
557/// Options to configure how llama.cpp logs are intercepted.
558#[derive(Default, Debug, Clone)]
559pub struct LogOptions {
560    disabled: bool,
561}
562
563impl LogOptions {
564    /// If enabled, logs are sent to tracing. If disabled, all logs are suppressed. Default is for
565    /// logs to be sent to tracing.
566    #[must_use]
567    pub fn with_logs_enabled(mut self, enabled: bool) -> Self {
568        self.disabled = !enabled;
569        self
570    }
571}
572
573extern "C" fn logs_to_trace(
574    level: llama_cpp_sys_2::ggml_log_level,
575    text: *const ::std::os::raw::c_char,
576    data: *mut ::std::os::raw::c_void,
577) {
578    // In the "fast-path" (i.e. the vast majority of logs) we want to avoid needing to take the log state
579    // lock at all. Similarly, we try to avoid any heap allocations within this function. This is accomplished
580    // by being a dummy pass-through to tracing in the normal case of DEBUG/INFO/WARN/ERROR logs that are
581    // newline terminated and limiting the slow-path of locks and/or heap allocations for other cases.
582    use std::borrow::Borrow;
583
584    let log_state = unsafe { &*(data as *const log::State) };
585
586    if log_state.options.disabled {
587        return;
588    }
589
590    // If the log level is disabled, we can just return early
591    if !log_state.is_enabled_for_level(level) {
592        log_state.update_previous_level_for_disabled_log(level);
593        return;
594    }
595
596    let text = unsafe { std::ffi::CStr::from_ptr(text) };
597    let text = text.to_string_lossy();
598    let text: &str = text.borrow();
599
600    // As best I can tell llama.cpp / ggml require all log format strings at call sites to have the '\n'.
601    // If it's missing, it means that you expect more logs via CONT (or there's a typo in the codebase). To
602    // distinguish typo from intentional support for CONT, we have to buffer until the next message comes in
603    // to know how to flush it.
604
605    if level == llama_cpp_sys_2::GGML_LOG_LEVEL_CONT {
606        log_state.cont_buffered_log(text);
607    } else if text.ends_with('\n') {
608        log_state.emit_non_cont_line(level, text);
609    } else {
610        log_state.buffer_non_cont(level, text);
611    }
612}
613
614/// Redirect llama.cpp logs into tracing.
615pub fn send_logs_to_tracing(options: LogOptions) {
616    // TODO: Reinitialize the state to support calling send_logs_to_tracing multiple times.
617
618    // We set up separate log states for llama.cpp and ggml to make sure that CONT logs between the two
619    // can't possibly interfere with each other. In other words, if llama.cpp emits a log without a trailing
620    // newline and calls a GGML function, the logs won't be weirdly intermixed and instead we'll llama.cpp logs
621    // will CONT previous llama.cpp logs and GGML logs will CONT previous ggml logs.
622    let llama_heap_state = Box::as_ref(
623        log::LLAMA_STATE
624            .get_or_init(|| Box::new(log::State::new(log::Module::LlamaCpp, options.clone()))),
625    ) as *const _;
626    let ggml_heap_state = Box::as_ref(
627        log::GGML_STATE.get_or_init(|| Box::new(log::State::new(log::Module::GGML, options))),
628    ) as *const _;
629
630    unsafe {
631        // GGML has to be set after llama since setting llama sets ggml as well.
632        llama_cpp_sys_2::llama_log_set(Some(logs_to_trace), llama_heap_state as *mut _);
633        llama_cpp_sys_2::ggml_log_set(Some(logs_to_trace), ggml_heap_state as *mut _);
634    }
635}
llama_cpp_2/lib.rs

llama_cpp_2/
lib.rs
