llama_cpp_2/
lib.rs

1//! Bindings to the llama.cpp library.
2//!
3//! As llama.cpp is a very fast moving target, this crate does not attempt to create a stable API
4//! with all the rust idioms. Instead it provided safe wrappers around nearly direct bindings to
5//! llama.cpp. This makes it easier to keep up with the changes in llama.cpp, but does mean that
6//! the API is not as nice as it could be.
7//!
8//! # Examples
9//!
10//! - [simple](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/simple)
11//! - [tools](https://github.com/utilityai/llama-cpp-rs/tree/main/examples/tools)
12//!
13//! # Feature Flags
14//!
15//! - `cuda` enables CUDA gpu support.
16//! - `sampler` adds the [`context::sample::sampler`] struct for a more rusty way of sampling.
17use std::ffi::{c_char, CStr, CString, NulError};
18use std::fmt::Debug;
19use std::num::NonZeroI32;
20use std::sync::OnceLock;
21
22use crate::llama_batch::BatchAddError;
23use std::os::raw::c_int;
24use std::path::PathBuf;
25use std::string::FromUtf8Error;
26
27pub mod context;
28pub mod gguf;
29pub mod llama_backend;
30pub mod llama_batch;
31#[cfg(feature = "llguidance")]
32pub(crate) mod llguidance_sampler;
33mod log;
34pub mod model;
35#[cfg(feature = "mtmd")]
36pub mod mtmd;
37pub mod sampling;
38#[cfg(feature = "common")]
39pub mod speculative;
40pub mod timing;
41pub mod token;
42pub mod token_type;
43
44pub use crate::context::session::{LlamaStateSeqFlags, SeqState};
45
46#[cfg(feature = "common")]
47pub(crate) fn status_is_ok(status: llama_cpp_sys_2::llama_rs_status) -> bool {
48    status == llama_cpp_sys_2::LLAMA_RS_STATUS_OK
49}
50
51/// A failable result from a llama.cpp function.
52pub type Result<T> = std::result::Result<T, LlamaCppError>;
53
54/// All errors that can occur in the llama-cpp crate.
55#[derive(Debug, Eq, PartialEq, thiserror::Error)]
56pub enum LlamaCppError {
57    /// The backend was already initialized. This can generally be ignored as initializing the backend
58    /// is idempotent.
59    #[error("BackendAlreadyInitialized")]
60    BackendAlreadyInitialized,
61    /// There was an error while get the chat template from model.
62    #[error("{0}")]
63    ChatTemplateError(#[from] ChatTemplateError),
64    /// There was an error while decoding a batch.
65    #[error("{0}")]
66    DecodeError(#[from] DecodeError),
67    /// There was an error while encoding a batch.
68    #[error("{0}")]
69    EncodeError(#[from] EncodeError),
70    /// There was an error loading a model.
71    #[error("{0}")]
72    LlamaModelLoadError(#[from] LlamaModelLoadError),
73    /// There was an error creating a new model context.
74    #[error("{0}")]
75    LlamaContextLoadError(#[from] LlamaContextLoadError),
76    /// There was an error adding a token to a batch.
77    #[error["{0}"]]
78    BatchAddError(#[from] BatchAddError),
79    /// see [`EmbeddingsError`]
80    #[error(transparent)]
81    EmbeddingError(#[from] EmbeddingsError),
82    // See [`LlamaSamplerError`]
83    /// Backend device not found
84    #[error("Backend device {0} not found")]
85    BackendDeviceNotFound(usize),
86    /// Max devices exceeded
87    #[error("Max devices exceeded. Max devices is {0}")]
88    MaxDevicesExceeded(usize),
89    /// Failed to convert JSON schema to grammar.
90    #[cfg(feature = "common")]
91    #[error("JsonSchemaToGrammarError: {0}")]
92    JsonSchemaToGrammarError(String),
93    /// There was an error fitting model parameters to available memory.
94    #[cfg(feature = "common")]
95    #[error("{0}")]
96    FitError(#[from] crate::model::params::FitError),
97}
98
99/// There was an error while getting the chat template from a model.
100#[derive(Debug, Eq, PartialEq, thiserror::Error)]
101pub enum ChatTemplateError {
102    /// gguf has no chat template (by that name)
103    #[error("chat template not found - returned null pointer")]
104    MissingTemplate,
105
106    /// chat template contained a null byte
107    #[error("null byte in string {0}")]
108    NullError(#[from] NulError),
109
110    /// The chat template was not valid utf8.
111    #[error(transparent)]
112    Utf8Error(#[from] std::str::Utf8Error),
113}
114
115/// Failed fetching metadata value
116#[derive(Debug, Eq, PartialEq, thiserror::Error)]
117pub enum MetaValError {
118    /// The provided string contains an unexpected null-byte
119    #[error("null byte in string {0}")]
120    NullError(#[from] NulError),
121
122    /// The returned data contains invalid UTF8 data
123    #[error("FromUtf8Error {0}")]
124    FromUtf8Error(#[from] FromUtf8Error),
125
126    /// Got negative return value. This happens if the key or index queried does not exist.
127    #[error("Negative return value. Likely due to a missing index or key. Got return value: {0}")]
128    NegativeReturn(i32),
129}
130
131/// Failed to Load context
132#[derive(Debug, Eq, PartialEq, thiserror::Error)]
133pub enum LlamaContextLoadError {
134    /// llama.cpp returned null
135    #[error("null reference from llama.cpp")]
136    NullReturn,
137}
138
139/// Errors from the sequence-state save/restore API.
140#[derive(Debug, Eq, PartialEq, thiserror::Error)]
141pub enum StateSeqError {
142    /// llama.cpp read or wrote a different number of bytes than expected.
143    #[error("state seq size mismatch: expected {expected}, actual {actual}")]
144    SizeMismatch {
145        /// Number of bytes the caller expected to transfer.
146        expected: usize,
147        /// Number of bytes llama.cpp actually transferred.
148        actual: usize,
149    },
150}
151
152/// Failed to decode a batch.
153#[derive(Debug, Eq, PartialEq, thiserror::Error)]
154pub enum DecodeError {
155    /// No kv cache slot was available.
156    #[error("Decode Error 1: NoKvCacheSlot")]
157    NoKvCacheSlot,
158    /// The number of tokens in the batch was 0.
159    #[error("Decode Error -1: n_tokens == 0")]
160    NTokensZero,
161    /// An unknown error occurred.
162    #[error("Decode Error {0}: unknown")]
163    Unknown(c_int),
164}
165
166/// Failed to decode a batch.
167#[derive(Debug, Eq, PartialEq, thiserror::Error)]
168pub enum EncodeError {
169    /// No kv cache slot was available.
170    #[error("Encode Error 1: NoKvCacheSlot")]
171    NoKvCacheSlot,
172    /// The number of tokens in the batch was 0.
173    #[error("Encode Error -1: n_tokens == 0")]
174    NTokensZero,
175    /// An unknown error occurred.
176    #[error("Encode Error {0}: unknown")]
177    Unknown(c_int),
178}
179
180/// When embedding related functions fail
181#[derive(Debug, Eq, PartialEq, thiserror::Error)]
182pub enum EmbeddingsError {
183    /// Embeddings weren't enabled in the context options
184    #[error("Embeddings weren't enabled in the context options")]
185    NotEnabled,
186    /// Logits weren't enabled for the given token
187    #[error("Logits were not enabled for the given token")]
188    LogitsNotEnabled,
189    /// The given sequence index exceeds the max sequence id
190    #[error("Can't use sequence embeddings with a model supporting only LLAMA_POOLING_TYPE_NONE")]
191    NonePoolType,
192}
193
194/// Errors that can occur when initializing a grammar sampler
195#[derive(Debug, Eq, PartialEq, thiserror::Error)]
196pub enum GrammarError {
197    /// The grammar root was not found in the grammar string
198    #[error("Grammar root not found in grammar string")]
199    RootNotFound,
200    /// The trigger word contains null bytes
201    #[error("Trigger word contains null bytes")]
202    TriggerWordNullBytes,
203    /// The grammar string or root contains null bytes
204    #[error("Grammar string or root contains null bytes")]
205    GrammarNullBytes,
206    /// The grammar call returned null
207    #[error("Grammar call returned null")]
208    NullGrammar,
209}
210
211/// Decode a error from llama.cpp into a [`DecodeError`].
212impl From<NonZeroI32> for DecodeError {
213    fn from(value: NonZeroI32) -> Self {
214        match value.get() {
215            1 => DecodeError::NoKvCacheSlot,
216            -1 => DecodeError::NTokensZero,
217            i => DecodeError::Unknown(i),
218        }
219    }
220}
221
222/// Encode a error from llama.cpp into a [`EncodeError`].
223impl From<NonZeroI32> for EncodeError {
224    fn from(value: NonZeroI32) -> Self {
225        match value.get() {
226            1 => EncodeError::NoKvCacheSlot,
227            -1 => EncodeError::NTokensZero,
228            i => EncodeError::Unknown(i),
229        }
230    }
231}
232
233/// An error that can occur when loading a model.
234#[derive(Debug, Eq, PartialEq, thiserror::Error)]
235pub enum LlamaModelLoadError {
236    /// There was a null byte in a provided string and thus it could not be converted to a C string.
237    #[error("null byte in string {0}")]
238    NullError(#[from] NulError),
239    /// llama.cpp returned a nullptr - this could be many different causes.
240    #[error("null result from llama cpp")]
241    NullResult,
242    /// Failed to convert the path to a rust str. This means the path was not valid unicode
243    #[error("failed to convert path {0} to str")]
244    PathToStrError(PathBuf),
245}
246
247/// An error that can occur when loading a model.
248#[derive(Debug, Eq, PartialEq, thiserror::Error)]
249pub enum LlamaLoraAdapterInitError {
250    /// There was a null byte in a provided string and thus it could not be converted to a C string.
251    #[error("null byte in string {0}")]
252    NullError(#[from] NulError),
253    /// llama.cpp returned a nullptr - this could be many different causes.
254    #[error("null result from llama cpp")]
255    NullResult,
256    /// Failed to convert the path to a rust str. This means the path was not valid unicode
257    #[error("failed to convert path {0} to str")]
258    PathToStrError(PathBuf),
259}
260
261/// An error that can occur when loading a model.
262#[derive(Debug, Eq, PartialEq, thiserror::Error)]
263pub enum LlamaLoraAdapterSetError {
264    /// llama.cpp returned a non-zero error code.
265    #[error("error code from llama cpp")]
266    ErrorResult(i32),
267}
268
269/// An error that can occur when loading a model.
270#[derive(Debug, Eq, PartialEq, thiserror::Error)]
271pub enum LlamaLoraAdapterRemoveError {
272    /// llama.cpp returned a non-zero error code.
273    #[error("error code from llama cpp")]
274    ErrorResult(i32),
275}
276
277/// get the time (in microseconds) according to llama.cpp
278/// ```
279/// # use llama_cpp_2::llama_time_us;
280/// # use llama_cpp_2::llama_backend::LlamaBackend;
281/// let backend = LlamaBackend::init().unwrap();
282/// let time = llama_time_us();
283/// assert!(time > 0);
284/// ```
285#[must_use]
286pub fn llama_time_us() -> i64 {
287    unsafe { llama_cpp_sys_2::llama_time_us() }
288}
289
290/// get the max number of devices according to llama.cpp (this is generally cuda devices)
291/// ```
292/// # use llama_cpp_2::max_devices;
293/// let max_devices = max_devices();
294/// assert!(max_devices >= 0);
295/// ```
296#[must_use]
297pub fn max_devices() -> usize {
298    unsafe { llama_cpp_sys_2::llama_max_devices() }
299}
300
301/// is memory mapping supported according to llama.cpp
302/// ```
303/// # use llama_cpp_2::mmap_supported;
304/// let mmap_supported = mmap_supported();
305/// if mmap_supported {
306///   println!("mmap_supported!");
307/// }
308/// ```
309#[must_use]
310pub fn mmap_supported() -> bool {
311    unsafe { llama_cpp_sys_2::llama_supports_mmap() }
312}
313
314/// is memory locking supported according to llama.cpp
315/// ```
316/// # use llama_cpp_2::mlock_supported;
317/// let mlock_supported = mlock_supported();
318/// if mlock_supported {
319///    println!("mlock_supported!");
320/// }
321/// ```
322#[must_use]
323pub fn mlock_supported() -> bool {
324    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
325}
326
327/// Convert a JSON schema string into a llama.cpp grammar string.
328#[cfg(feature = "common")]
329pub fn json_schema_to_grammar(schema_json: &str) -> Result<String> {
330    let schema_cstr = CString::new(schema_json)
331        .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
332    let mut out = std::ptr::null_mut();
333    let rc = unsafe {
334        llama_cpp_sys_2::llama_rs_json_schema_to_grammar(schema_cstr.as_ptr(), false, &mut out)
335    };
336
337    let result = {
338        if !status_is_ok(rc) || out.is_null() {
339            return Err(LlamaCppError::JsonSchemaToGrammarError(format!(
340                "ffi error {}",
341                rc
342            )));
343        }
344        let grammar_bytes = unsafe { CStr::from_ptr(out) }.to_bytes().to_vec();
345        let grammar = String::from_utf8(grammar_bytes)
346            .map_err(|err| LlamaCppError::JsonSchemaToGrammarError(err.to_string()))?;
347        Ok(grammar)
348    };
349
350    unsafe { llama_cpp_sys_2::llama_rs_string_free(out) };
351    result
352}
353
354#[cfg(all(test, feature = "common"))]
355mod tests {
356    use super::json_schema_to_grammar;
357
358    #[test]
359    fn json_schema_string_api_returns_grammar() {
360        let schema = r#"{
361            "type": "object",
362            "properties": {
363                "city": { "type": "string" },
364                "unit": { "enum": ["c", "f"] }
365            },
366            "required": ["city"]
367        }"#;
368
369        let grammar =
370            json_schema_to_grammar(schema).expect("string-based schema conversion should succeed");
371
372        assert!(grammar.contains("root ::="));
373    }
374}
375
376/// An error that can occur when converting a token to a string.
377#[derive(Debug, thiserror::Error, Clone)]
378#[non_exhaustive]
379pub enum TokenToStringError {
380    /// the token type was unknown
381    #[error("Unknown Token Type")]
382    UnknownTokenType,
383    /// There was insufficient buffer space to convert the token to a string.
384    #[error("Insufficient Buffer Space {0}")]
385    InsufficientBufferSpace(c_int),
386    /// The token was not valid utf8.
387    #[error("FromUtf8Error {0}")]
388    FromUtf8Error(#[from] FromUtf8Error),
389}
390
391/// Failed to convert a string to a token sequence.
392#[derive(Debug, thiserror::Error)]
393pub enum StringToTokenError {
394    /// the string contained a null byte and thus could not be converted to a c string.
395    #[error("{0}")]
396    NulError(#[from] NulError),
397    #[error("{0}")]
398    /// Failed to convert a provided integer to a [`c_int`].
399    CIntConversionError(#[from] std::num::TryFromIntError),
400}
401
402/// Failed to apply model chat template.
403#[derive(Debug, thiserror::Error)]
404pub enum NewLlamaChatMessageError {
405    /// the string contained a null byte and thus could not be converted to a c string.
406    #[error("{0}")]
407    NulError(#[from] NulError),
408}
409
410/// Failed to apply model chat template.
411#[derive(Debug, thiserror::Error)]
412pub enum ApplyChatTemplateError {
413    /// the string contained a null byte and thus could not be converted to a c string.
414    #[error("{0}")]
415    NulError(#[from] NulError),
416    /// the string could not be converted to utf8.
417    #[error("{0}")]
418    FromUtf8Error(#[from] FromUtf8Error),
419    /// llama.cpp returned a null pointer for the template result.
420    #[error("null result from llama.cpp")]
421    NullResult,
422    /// llama.cpp returned an error code.
423    #[error("ffi error {0}")]
424    FfiError(i32),
425}
426
427/// Failed to accept a token in a sampler.
428#[derive(Debug, thiserror::Error)]
429pub enum SamplerAcceptError {
430    /// llama.cpp returned an error code.
431    #[error("ffi error {0}")]
432    FfiError(i32),
433}
434
435/// Get the time in microseconds according to ggml
436///
437/// ```
438/// # use std::time::Duration;
439/// # use llama_cpp_2::llama_backend::LlamaBackend;
440/// let backend = LlamaBackend::init().unwrap();
441/// use llama_cpp_2::ggml_time_us;
442///
443/// let start = ggml_time_us();
444///
445/// std::thread::sleep(Duration::from_micros(10));
446///
447/// let end = ggml_time_us();
448///
449/// let elapsed = end - start;
450///
451/// assert!(elapsed >= 10)
452#[must_use]
453pub fn ggml_time_us() -> i64 {
454    unsafe { llama_cpp_sys_2::ggml_time_us() }
455}
456
457/// checks if mlock is supported
458///
459/// ```
460/// # use llama_cpp_2::llama_supports_mlock;
461///
462/// if llama_supports_mlock() {
463///   println!("mlock is supported!");
464/// } else {
465///   println!("mlock is not supported!");
466/// }
467/// ```
468#[must_use]
469pub fn llama_supports_mlock() -> bool {
470    unsafe { llama_cpp_sys_2::llama_supports_mlock() }
471}
472
473/// Backend device type
474#[derive(Debug, Clone, Copy, PartialEq, Eq)]
475pub enum LlamaBackendDeviceType {
476    /// CPU device
477    Cpu,
478    /// ACCEL device
479    Accelerator,
480    /// GPU device
481    Gpu,
482    /// iGPU device
483    IntegratedGpu,
484    /// Unknown device type
485    Unknown,
486}
487
488/// A ggml backend device
489///
490/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
491#[derive(Debug, Clone)]
492pub struct LlamaBackendDevice {
493    /// The index of the device
494    ///
495    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
496    pub index: usize,
497    /// The name of the device (e.g. "Vulkan0")
498    pub name: String,
499    /// A description of the device (e.g. "NVIDIA GeForce RTX 3080")
500    pub description: String,
501    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
502    pub backend: String,
503    /// Total memory of the device in bytes
504    pub memory_total: usize,
505    /// Free memory of the device in bytes
506    pub memory_free: usize,
507    /// Device type
508    pub device_type: LlamaBackendDeviceType,
509}
510
511/// List ggml backend devices
512#[must_use]
513pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
514    let mut devices = Vec::new();
515    for i in 0..unsafe { llama_cpp_sys_2::ggml_backend_dev_count() } {
516        fn cstr_to_string(ptr: *const c_char) -> String {
517            if ptr.is_null() {
518                String::new()
519            } else {
520                unsafe { std::ffi::CStr::from_ptr(ptr) }
521                    .to_string_lossy()
522                    .to_string()
523            }
524        }
525        let dev = unsafe { llama_cpp_sys_2::ggml_backend_dev_get(i) };
526        let props = unsafe {
527            let mut props = std::mem::zeroed();
528            llama_cpp_sys_2::ggml_backend_dev_get_props(dev, &raw mut props);
529            props
530        };
531        let name = cstr_to_string(props.name);
532        let description = cstr_to_string(props.description);
533        let backend = unsafe { llama_cpp_sys_2::ggml_backend_dev_backend_reg(dev) };
534        let backend_name = unsafe { llama_cpp_sys_2::ggml_backend_reg_name(backend) };
535        let backend = cstr_to_string(backend_name);
536        let memory_total = props.memory_total;
537        let memory_free = props.memory_free;
538        let device_type = match props.type_ {
539            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
540            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_ACCEL => LlamaBackendDeviceType::Accelerator,
541            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
542            llama_cpp_sys_2::GGML_BACKEND_DEVICE_TYPE_IGPU => LlamaBackendDeviceType::IntegratedGpu,
543            _ => LlamaBackendDeviceType::Unknown,
544        };
545        devices.push(LlamaBackendDevice {
546            index: i,
547            name,
548            description,
549            backend,
550            memory_total,
551            memory_free,
552            device_type,
553        });
554    }
555    devices
556}
557
558/// Options to configure how llama.cpp logs are intercepted.
559#[derive(Default, Debug, Clone)]
560pub struct LogOptions {
561    disabled: bool,
562}
563
564impl LogOptions {
565    /// If enabled, logs are sent to tracing. If disabled, all logs are suppressed. Default is for
566    /// logs to be sent to tracing.
567    #[must_use]
568    pub fn with_logs_enabled(mut self, enabled: bool) -> Self {
569        self.disabled = !enabled;
570        self
571    }
572}
573
574extern "C" fn logs_to_trace(
575    level: llama_cpp_sys_2::ggml_log_level,
576    text: *const ::std::os::raw::c_char,
577    data: *mut ::std::os::raw::c_void,
578) {
579    // In the "fast-path" (i.e. the vast majority of logs) we want to avoid needing to take the log state
580    // lock at all. Similarly, we try to avoid any heap allocations within this function. This is accomplished
581    // by being a dummy pass-through to tracing in the normal case of DEBUG/INFO/WARN/ERROR logs that are
582    // newline terminated and limiting the slow-path of locks and/or heap allocations for other cases.
583    use std::borrow::Borrow;
584
585    let log_state = unsafe { &*(data as *const log::State) };
586
587    if log_state.options.disabled {
588        return;
589    }
590
591    // If the log level is disabled, we can just return early
592    if !log_state.is_enabled_for_level(level) {
593        log_state.update_previous_level_for_disabled_log(level);
594        return;
595    }
596
597    let text = unsafe { std::ffi::CStr::from_ptr(text) };
598    let text = text.to_string_lossy();
599    let text: &str = text.borrow();
600
601    // As best I can tell llama.cpp / ggml require all log format strings at call sites to have the '\n'.
602    // If it's missing, it means that you expect more logs via CONT (or there's a typo in the codebase). To
603    // distinguish typo from intentional support for CONT, we have to buffer until the next message comes in
604    // to know how to flush it.
605
606    if level == llama_cpp_sys_2::GGML_LOG_LEVEL_CONT {
607        log_state.cont_buffered_log(text);
608    } else if text.ends_with('\n') {
609        log_state.emit_non_cont_line(level, text);
610    } else {
611        log_state.buffer_non_cont(level, text);
612    }
613}
614
615/// Redirect llama.cpp logs into tracing.
616pub fn send_logs_to_tracing(options: LogOptions) {
617    // TODO: Reinitialize the state to support calling send_logs_to_tracing multiple times.
618
619    // We set up separate log states for each backend to make sure that CONT
620    // logs between them can't possibly interfere with each other.
621    //
622    // In other words, if llama.cpp emits a log without a trailing newline and
623    // calls a GGML function, the logs won't be weirdly intermixed and instead
624    // we'll llama.cpp logs will CONT previous llama.cpp logs and GGML logs
625    // will CONT previous ggml logs.
626    static LLAMA_STATE: OnceLock<Box<log::State>> = OnceLock::new();
627    #[cfg(feature = "mtmd")]
628    static MTMD_STATE: OnceLock<Box<log::State>> = OnceLock::new();
629    static GGML_STATE: OnceLock<Box<log::State>> = OnceLock::new();
630
631    let llama_heap_state = Box::as_ref(
632        LLAMA_STATE
633            .get_or_init(|| Box::new(log::State::new(log::Module::LlamaCpp, options.clone()))),
634    ) as *const _;
635
636    #[cfg(feature = "mtmd")]
637    let mtmd_heap_state = Box::as_ref(
638        MTMD_STATE.get_or_init(|| Box::new(log::State::new(log::Module::Mtmd, options.clone()))),
639    ) as *const _;
640
641    let ggml_heap_state = Box::as_ref(
642        GGML_STATE.get_or_init(|| Box::new(log::State::new(log::Module::GGML, options))),
643    ) as *const _;
644
645    unsafe {
646        llama_cpp_sys_2::llama_log_set(Some(logs_to_trace), llama_heap_state as *mut _);
647
648        // There is both `mtmd_log_set` and `mtmd_helper_log_set`. We use the
649        // helper, since we don't particularly want to differentiate these
650        // two, and the helper calls the normal `mtmd_log_set` as well.
651        #[cfg(feature = "mtmd")]
652        llama_cpp_sys_2::mtmd_helper_log_set(Some(logs_to_trace), mtmd_heap_state as *mut _);
653
654        // GGML has to be set after llama since setting llama logs sets ggml logs as well.
655        llama_cpp_sys_2::ggml_log_set(Some(logs_to_trace), ggml_heap_state as *mut _);
656    }
657}
llama_cpp_2/lib.rs

llama_cpp_2/
lib.rs
