llama_cpp_2/
lib.rs

1//! Bindings to the llama.cpp library.
2//!
3//! As llama.cpp is a very fast moving target, this crate does not attempt to create a stable API
4//! with all the rust idioms. Instead it provided safe wrappers around nearly direct bindings to
5//! llama.cpp. This makes it easier to keep up with the changes in llama.cpp, but does mean that
6//! the API is not as nice as it could be.
7//!
8//! # Examples
9//!
10//! - [simple](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/simple)
11//! - [tools](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/tools)
12//!
13//! # Feature Flags
14//!
15//! - `cuda` enables CUDA gpu support.
16//! - `sampler` adds the [`context::sample::sampler`] struct for a more rusty way of sampling.
17use std::ffi::{c_char, CStr, CString, NulError};
18use std::fmt::Debug;
19use std::num::NonZeroI32;
20
21use crate::llama_batch::BatchAddError;
22use std::os::raw::c_int;
23use std::path::PathBuf;
24use std::string::FromUtf8Error;
25
26pub mod context;
27pub mod gguf;
28pub mod llama_backend;
29pub mod llama_batch;
30#[cfg(feature = "llguidance")]
31pub(crate) mod llguidance_sampler;
32mod log;
33pub mod model;
34#[cfg(feature = "mtmd")]
35pub mod mtmd;
36pub mod sampling;
37#[cfg(feature = "common")]
38pub mod speculative;
39pub mod timing;
40pub mod token;
41pub mod token_type;
42
43pub use crate::context::session::LlamaStateSeqFlags;
44
45#[cfg(feature = "common")]
46pub(crate) fn status_is_ok(status: llama_cpp_sys_2::llama_rs_status) -> bool {
47    status == llama_cpp_sys_2::LLAMA_RS_STATUS_OK
48}
49
50/// A failable result from a llama.cpp function.
51pub type Result<T> = std::result::Result<T, LlamaCppError>;
52
53/// All errors that can occur in the llama-cpp crate.
54#[derive(Debug, Eq, PartialEq, thiserror::Error)]
55pub enum LlamaCppError {
56    /// The backend was already initialized. This can generally be ignored as initializing the backend
57    /// is idempotent.
58    #[error("BackendAlreadyInitialized")]
59    BackendAlreadyInitialized,
60    /// There was an error while get the chat template from model.
61    #[error("{0}")]
62    ChatTemplateError(#[from] ChatTemplateError),
63    /// There was an error while decoding a batch.
64    #[error("{0}")]
65    DecodeError(#[from] DecodeError),
66    /// There was an error while encoding a batch.
67    #[error("{0}")]
68    EncodeError(#[from] EncodeError),
69    /// There was an error loading a model.
70    #[error("{0}")]
71    LlamaModelLoadError(#[from] LlamaModelLoadError),
72    /// There was an error creating a new model context.
73    #[error("{0}")]
74    LlamaContextLoadError(#[from] LlamaContextLoadError),
75    /// There was an error adding a token to a batch.
76    #[error["{0}"]]
77    BatchAddError(#[from] BatchAddError),
78    /// see [`EmbeddingsError`]
79    #[error(transparent)]
80    EmbeddingError(#[from] EmbeddingsError),
81    // See [`LlamaSamplerError`]
82    /// Backend device not found
83    #[error("Backend device {0} not found")]
84    BackendDeviceNotFound(usize),
85    /// Max devices exceeded
86    #[error("Max devices exceeded. Max devices is {0}")]
87    MaxDevicesExceeded(usize),
88    /// Failed to convert JSON schema to grammar.
89    #[cfg(feature = "common")]
90    #[error("JsonSchemaToGrammarError: {0}")]
91    JsonSchemaToGrammarError(String),
92    /// There was an error fitting model parameters to available memory.
93    #[cfg(feature = "common")]
94    #[error("{0}")]
95    FitError(#[from] crate::model::params::FitError),
96}
97
98/// There was an error while getting the chat template from a model.
99#[derive(Debug, Eq, PartialEq, thiserror::Error)]
100pub enum ChatTemplateError {
101    /// gguf has no chat template (by that name)
102    #[error("chat template not found - returned null pointer")]
103    MissingTemplate,
104
105    /// chat template contained a null byte
106    #[error("null byte in string {0}")]
107    NullError(#[from] NulError),
108
109    /// The chat template was not valid utf8.
110    #[error(transparent)]
111    Utf8Error(#[from] std::str::Utf8Error),
112}
113
114/// Failed fetching metadata value
115#[derive(Debug, Eq, PartialEq, thiserror::Error)]
116pub enum MetaValError {
117    /// The provided string contains an unexpected null-byte
118    #[error("null byte in string {0}")]
119    NullError(#[from] NulError),
120
121    /// The returned data contains invalid UTF8 data
122    #[error("FromUtf8Error {0}")]
123    FromUtf8Error(#[from] FromUtf8Error),
124
125    /// Got negative return value. This happens if the key or index queried does not exist.
126    #[error("Negative return value. Likely due to a missing index or key. Got return value: {0}")]
127    NegativeReturn(i32),
128}
129
130/// Failed to Load context
131#[derive(Debug, Eq, PartialEq, thiserror::Error)]
132pub enum LlamaContextLoadError {
133    /// llama.cpp returned null
134    #[error("null reference from llama.cpp")]
135    NullReturn,
136}
137
138/// Failed to decode a batch.
139#[derive(Debug, Eq, PartialEq, thiserror::Error)]
140pub enum DecodeError {
141    /// No kv cache slot was available.
142    #[error("Decode Error 1: NoKvCacheSlot")]
143    NoKvCacheSlot,
144    /// The number of tokens in the batch was 0.
145    #[error("Decode Error -1: n_tokens == 0")]
146    NTokensZero,
147    /// An unknown error occurred.
148    #[error("Decode Error {0}: unknown")]
149    Unknown(c_int),
150}
151
152/// Failed to decode a batch.
153#[derive(Debug, Eq, PartialEq, thiserror::Error)]
154pub enum EncodeError {
155    /// No kv cache slot was available.
156    #[error("Encode Error 1: NoKvCacheSlot")]
157    NoKvCacheSlot,
158    /// The number of tokens in the batch was 0.
159    #[error("Encode Error -1: n_tokens == 0")]
160    NTokensZero,
161    /// An unknown error occurred.
162    #[error("Encode Error {0}: unknown")]
163    Unknown(c_int),
164}
165
166/// When embedding related functions fail
167#[derive(Debug, Eq, PartialEq, thiserror::Error)]
168pub enum EmbeddingsError {
169    /// Embeddings weren't enabled in the context options
170    #[error("Embeddings weren't enabled in the context options")]
171    NotEnabled,
172    /// Logits weren't enabled for the given token
173    #[error("Logits were not enabled for the given token")]
174    LogitsNotEnabled,
175    /// The given sequence index exceeds the max sequence id
176    #[error("Can't use sequence embeddings with a model supporting only LLAMA_POOLING_TYPE_NONE")]
177    NonePoolType,
178}
179
180/// Errors that can occur when initializing a grammar sampler
181#[derive(Debug, Eq, PartialEq, thiserror::Error)]
182pub enum GrammarError {
183    /// The grammar root was not found in the grammar string
184    #[error("Grammar root not found in grammar string")]
185    RootNotFound,
186    /// The trigger word contains null bytes
187    #[error("Trigger word contains null bytes")]
188    TriggerWordNullBytes,
189    /// The grammar string or root contains null bytes
190    #[error("Grammar string or root contains null bytes")]
191    GrammarNullBytes,
192    /// The grammar call returned null
193    #[error("Grammar call returned null")]
194    NullGrammar,
195}
196
197/// Decode a error from llama.cpp into a [`DecodeError`].
198impl From<NonZeroI32> for DecodeError {
199    fn from(value: NonZeroI32) -> Self {
200        match value.get() {
201            1 => DecodeError::NoKvCacheSlot,
202            -1 => DecodeError::NTokensZero,
203            i => DecodeError::Unknown(i),
204        }
205    }
206}
207
208/// Encode a error from llama.cpp into a [`EncodeError`].
209impl From<NonZeroI32> for EncodeError {
210    fn from(value: NonZeroI32) -> Self {
211        match value.get() {
212            1 => EncodeError::NoKvCacheSlot,
213            -1 => EncodeError::NTokensZero,
214            i => EncodeError::Unknown(i),
215        }
216    }
217}
218
219/// An error that can occur when loading a model.
220#[derive(Debug, Eq, PartialEq, thiserror::Error)]
221pub enum LlamaModelLoadError {
222    /// There was a null byte in a provided string and thus it could not be converted to a C string.
223    #[error("null byte in string {0}")]
224    NullError(#[from] NulError),
225    /// llama.cpp returned a nullptr - this could be many different causes.
226    #[error("null result from llama cpp")]
227    NullResult,
228    /// Failed to convert the path to a rust str. This means the path was not valid unicode
229    #[error("failed to convert path {0} to str")]
230    PathToStrError(PathBuf),
231}
232
233/// An error that can occur when loading a model.
234#[derive(Debug, Eq, PartialEq, thiserror::Error)]
235pub enum LlamaLoraAdapterInitError {
236    /// There was a null byte in a provided string and thus it could not be converted to a C string.
237    #[error("null byte in string {0}")]
238    NullError(#[from] NulError),
239    /// llama.cpp returned a nullptr - this could be many different causes.
240    #[error("null result from llama cpp")]
241    NullResult,
242    /// Failed to convert the path to a rust str. This means the path was not valid unicode
243    #[error("failed to convert path {0} to str")]
244    PathToStrError(PathBuf),
245}
246
247/// An error that can occur when loading a model.
248#[derive(Debug, Eq, PartialEq, thiserror::Error)]
249pub enum LlamaLoraAdapterSetError {
250    /// llama.cpp returned a non-zero error code.
251    #[error("error code from llama cpp")]
252    ErrorResult(i32),
253}
254
255/// An error that can occur when loading a model.
256#[derive(Debug, Eq, PartialEq, thiserror::Error)]
257pub enum LlamaLoraAdapterRemoveError {
258    /// llama.cpp returned a non-zero error code.
259    #[error("error code from llama cpp")]
260    ErrorResult(i32),
261}
262
263/// get the time (in microseconds) according to llama.cpp
264/// ```
265/// # use llama_cpp_2::llama_time_us;
266/// # use llama_cpp_2::llama_backend::LlamaBackend;
267/// let backend = LlamaBackend::init().unwrap();
268/// let time = llama_time_us();
269/// assert!(time > 0);
270/// ```
271#[must_use]
272pub fn llama_time_us() -> i64 {
273    unsafe { llama_cpp_sys_2::llama_time_us() }
274}
275
276/// get the max number of devices according to llama.cpp (this is generally cuda devices)
277/// ```
278/// # use llama_cpp_2::max_devices;
279/// let max_devices = max_devices();
280/// assert!(max_devices >= 0);
281/// ```
282#[must_use]
283pub fn max_devices() -> usize {
284    unsafe { llama_cpp_sys_2::llama_max_devices() }
285}
286
287/// is memory mapping supported according to llama.cpp
288/// ```
289/// # use llama_cpp_2::mmap_supported;
290/// let mmap_supported = mmap_supported();
291/// if mmap_supported {
292///   println!("mmap_supported!");
293/// }
294/// ```
295#[must_use]
296pub fn mmap_supported() -> bool {
297    unsafe { llama_cpp_sys_2::llama_supports_mmap() }
298}
299
300/// is memory locking supported according to llama.cpp
301/// ```
302/// # use llama_cpp_2::mlock_supported;
303/// let mlock_supported = mlock_supported();
304/// if mlock_supported {
305///    println!("mlock_supported!");
306/// }
307/// ```
308#[must_use]
309pub fn mlock_supported() -> bool {
310    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
311}
312
313/// Convert a JSON schema string into a llama.cpp grammar string.
314#[cfg(feature = "common")]
315pub fn json_schema_to_grammar(schema_json: &str) -> Result<String> {
316    let schema_cstr = CString::new(schema_json)
317        .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
318    let mut out = std::ptr::null_mut();
319    let rc = unsafe {
320        llama_cpp_sys_2::llama_rs_json_schema_to_grammar(schema_cstr.as_ptr(), false, &mut out)
321    };
322
323    let result = {
324        if !status_is_ok(rc) || out.is_null() {
325            return Err(LlamaCppError::JsonSchemaToGrammarError(format!(
326                "ffi error {}",
327                rc
328            )));
329        }
330        let grammar_bytes = unsafe { CStr::from_ptr(out) }.to_bytes().to_vec();
331        let grammar = String::from_utf8(grammar_bytes)
332            .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
333        Ok(grammar)
334    };
335
336    unsafe { llama_cpp_sys_2::llama_rs_string_free(out) };
337    result
338}
339
340#[cfg(all(test, feature = "common"))]
341mod tests {
342    use super::json_schema_to_grammar;
343
344    #[test]
345    fn json_schema_string_api_returns_grammar() {
346        let schema = r#"{
347            "type": "object",
348            "properties": {
349                "city": { "type": "string" },
350                "unit": { "enum": ["c", "f"] }
351            },
352            "required": ["city"]
353        }"#;
354
355        let grammar =
356            json_schema_to_grammar(schema).expect("string-based schema conversion should succeed");
357
358        assert!(grammar.contains("root ::="));
359    }
360}
361
362/// An error that can occur when converting a token to a string.
363#[derive(Debug, thiserror::Error, Clone)]
364#[non_exhaustive]
365pub enum TokenToStringError {
366    /// the token type was unknown
367    #[error("Unknown Token Type")]
368    UnknownTokenType,
369    /// There was insufficient buffer space to convert the token to a string.
370    #[error("Insufficient Buffer Space {0}")]
371    InsufficientBufferSpace(c_int),
372    /// The token was not valid utf8.
373    #[error("FromUtf8Error {0}")]
374    FromUtf8Error(#[from] FromUtf8Error),
375}
376
377/// Failed to convert a string to a token sequence.
378#[derive(Debug, thiserror::Error)]
379pub enum StringToTokenError {
380    /// the string contained a null byte and thus could not be converted to a c string.
381    #[error("{0}")]
382    NulError(#[from] NulError),
383    #[error("{0}")]
384    /// Failed to convert a provided integer to a [`c_int`].
385    CIntConversionError(#[from] std::num::TryFromIntError),
386}
387
388/// Failed to apply model chat template.
389#[derive(Debug, thiserror::Error)]
390pub enum NewLlamaChatMessageError {
391    /// the string contained a null byte and thus could not be converted to a c string.
392    #[error("{0}")]
393    NulError(#[from] NulError),
394}
395
396/// Failed to apply model chat template.
397#[derive(Debug, thiserror::Error)]
398pub enum ApplyChatTemplateError {
399    /// the string contained a null byte and thus could not be converted to a c string.
400    #[error("{0}")]
401    NulError(#[from] NulError),
402    /// the string could not be converted to utf8.
403    #[error("{0}")]
404    FromUtf8Error(#[from] FromUtf8Error),
405    /// llama.cpp returned a null pointer for the template result.
406    #[error("null result from llama.cpp")]
407    NullResult,
408    /// llama.cpp returned an error code.
409    #[error("ffi error {0}")]
410    FfiError(i32),
411}
412
413/// Failed to accept a token in a sampler.
414#[derive(Debug, thiserror::Error)]
415pub enum SamplerAcceptError {
416    /// llama.cpp returned an error code.
417    #[error("ffi error {0}")]
418    FfiError(i32),
419}
420
421/// Get the time in microseconds according to ggml
422///
423/// ```
424/// # use std::time::Duration;
425/// # use llama_cpp_2::llama_backend::LlamaBackend;
426/// let backend = LlamaBackend::init().unwrap();
427/// use llama_cpp_2::ggml_time_us;
428///
429/// let start = ggml_time_us();
430///
431/// std::thread::sleep(Duration::from_micros(10));
432///
433/// let end = ggml_time_us();
434///
435/// let elapsed = end - start;
436///
437/// assert!(elapsed >= 10)
438#[must_use]
439pub fn ggml_time_us() -> i64 {
440    unsafe { llama_cpp_sys_2::ggml_time_us() }
441}
442
443/// checks if mlock is supported
444///
445/// ```
446/// # use llama_cpp_2::llama_supports_mlock;
447///
448/// if llama_supports_mlock() {
449///   println!("mlock is supported!");
450/// } else {
451///   println!("mlock is not supported!");
452/// }
453/// ```
454#[must_use]
455pub fn llama_supports_mlock() -> bool {
456    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
457}
458
459/// Backend device type
460#[derive(Debug, Clone, Copy, PartialEq, Eq)]
461pub enum LlamaBackendDeviceType {
462    /// CPU device
463    Cpu,
464    /// ACCEL device
465    Accelerator,
466    /// GPU device
467    Gpu,
468    /// iGPU device
469    IntegratedGpu,
470    /// Unknown device type
471    Unknown,
472}
473
474/// A ggml backend device
475///
476/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
477#[derive(Debug, Clone)]
478pub struct LlamaBackendDevice {
479    /// The index of the device
480    ///
481    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
482    pub index: usize,
483    /// The name of the device (e.g. "Vulkan0")
484    pub name: String,
485    /// A description of the device (e.g. "NVIDIA GeForce RTX 3080")
486    pub description: String,
487    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
488    pub backend: String,
489    /// Total memory of the device in bytes
490    pub memory_total: usize,
491    /// Free memory of the device in bytes
492    pub memory_free: usize,
493    /// Device type
494    pub device_type: LlamaBackendDeviceType,
495}
496
497/// List ggml backend devices
498#[must_use]
499pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
500    let mut devices = Vec::new();
501    for i in 0..unsafe { llama_cpp_sys_2::ggml_backend_dev_count() } {
502        fn cstr_to_string(ptr: *const c_char) -> String {
503            if ptr.is_null() {
504                String::new()
505            } else {
506                unsafe { std::ffi::CStr::from_ptr(ptr) }
507                    .to_string_lossy()
508                    .to_string()
509            }
510        }
511        let dev = unsafe { llama_cpp_sys_2::ggml_backend_dev_get(i) };
512        let props = unsafe {
513            let mut props = std::mem::zeroed();
514            llama_cpp_sys_2::ggml_backend_dev_get_props(dev, &raw mut props);
515            props
516        };
517        let name = cstr_to_string(props.name);
518        let description = cstr_to_string(props.description);
519        let backend = unsafe { llama_cpp_sys_2::ggml_backend_dev_backend_reg(dev) };
520        let backend_name = unsafe { llama_cpp_sys_2::ggml_backend_reg_name(backend) };
521        let backend = cstr_to_string(backend_name);
522        let memory_total = props.memory_total;
523        let memory_free = props.memory_free;
524        let device_type = match props.type_ {
525            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
526            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_ACCEL => LlamaBackendDeviceType::Accelerator,
527            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
528            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_IGPU => LlamaBackendDeviceType::IntegratedGpu,
529            _ => LlamaBackendDeviceType::Unknown,
530        };
531        devices.push(LlamaBackendDevice {
532            index: i,
533            name,
534            description,
535            backend,
536            memory_total,
537            memory_free,
538            device_type,
539        });
540    }
541    devices
542}
543
544/// Options to configure how llama.cpp logs are intercepted.
545#[derive(Default, Debug, Clone)]
546pub struct LogOptions {
547    disabled: bool,
548}
549
550impl LogOptions {
551    /// If enabled, logs are sent to tracing. If disabled, all logs are suppressed. Default is for
552    /// logs to be sent to tracing.
553    #[must_use]
554    pub fn with_logs_enabled(mut self, enabled: bool) -> Self {
555        self.disabled = !enabled;
556        self
557    }
558}
559
560extern "C" fn logs_to_trace(
561    level: llama_cpp_sys_2::ggml_log_level,
562    text: *const ::std::os::raw::c_char,
563    data: *mut ::std::os::raw::c_void,
564) {
565    // In the "fast-path" (i.e. the vast majority of logs) we want to avoid needing to take the log state
566    // lock at all. Similarly, we try to avoid any heap allocations within this function. This is accomplished
567    // by being a dummy pass-through to tracing in the normal case of DEBUG/INFO/WARN/ERROR logs that are
568    // newline terminated and limiting the slow-path of locks and/or heap allocations for other cases.
569    use std::borrow::Borrow;
570
571    let log_state = unsafe { &*(data as *const log::State) };
572
573    if log_state.options.disabled {
574        return;
575    }
576
577    // If the log level is disabled, we can just return early
578    if !log_state.is_enabled_for_level(level) {
579        log_state.update_previous_level_for_disabled_log(level);
580        return;
581    }
582
583    let text = unsafe { std::ffi::CStr::from_ptr(text) };
584    let text = text.to_string_lossy();
585    let text: &str = text.borrow();
586
587    // As best I can tell llama.cpp / ggml require all log format strings at call sites to have the '\n'.
588    // If it's missing, it means that you expect more logs via CONT (or there's a typo in the codebase). To
589    // distinguish typo from intentional support for CONT, we have to buffer until the next message comes in
590    // to know how to flush it.
591
592    if level == llama_cpp_sys_2::GGML_LOG_LEVEL_CONT {
593        log_state.cont_buffered_log(text);
594    } else if text.ends_with('\n') {
595        log_state.emit_non_cont_line(level, text);
596    } else {
597        log_state.buffer_non_cont(level, text);
598    }
599}
600
601/// Redirect llama.cpp logs into tracing.
602pub fn send_logs_to_tracing(options: LogOptions) {
603    // TODO: Reinitialize the state to support calling send_logs_to_tracing multiple times.
604
605    // We set up separate log states for llama.cpp and ggml to make sure that CONT logs between the two
606    // can't possibly interfere with each other. In other words, if llama.cpp emits a log without a trailing
607    // newline and calls a GGML function, the logs won't be weirdly intermixed and instead we'll llama.cpp logs
608    // will CONT previous llama.cpp logs and GGML logs will CONT previous ggml logs.
609    let llama_heap_state = Box::as_ref(
610        log::LLAMA_STATE
611            .get_or_init(|| Box::new(log::State::new(log::Module::LlamaCpp, options.clone()))),
612    ) as *const _;
613    let ggml_heap_state = Box::as_ref(
614        log::GGML_STATE.get_or_init(|| Box::new(log::State::new(log::Module::GGML, options))),
615    ) as *const _;
616
617    unsafe {
618        // GGML has to be set after llama since setting llama sets ggml as well.
619        llama_cpp_sys_2::llama_log_set(Some(logs_to_trace), llama_heap_state as *mut _);
620        llama_cpp_sys_2::ggml_log_set(Some(logs_to_trace), ggml_heap_state as *mut _);
621    }
622}
llama_cpp_2/lib.rs

llama_cpp_2/
lib.rs
