ik-llama-cpp-2 0.1.3

ik_llama.cpp bindings for Rust (SOTA quants + MTP)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

//! Process-wide ik_llama.cpp backend initialization.

use std::sync::atomic::{AtomicBool, Ordering};

use crate::LlamaError;

static INITIALIZED: AtomicBool = AtomicBool::new(false);

/// RAII proof that the ik_llama.cpp backend has been initialized.
///
/// Initialization is a process-wide singleton (like llama.cpp itself). Hold one
/// `LlamaBackend` for the lifetime of your models/contexts; dropping it calls
/// `llama_backend_free`.
#[derive(Debug)]
pub struct LlamaBackend {
    _private: (),
}

impl LlamaBackend {
    /// Initialize the backend. Returns [`LlamaError::BackendAlreadyInitialized`]
    /// if a `LlamaBackend` is already live in this process.
    pub fn init() -> Result<Self, LlamaError> {
        if INITIALIZED.swap(true, Ordering::SeqCst) {
            return Err(LlamaError::BackendAlreadyInitialized);
        }
        // SAFETY: guarded by the atomic above; called at most once at a time.
        unsafe { ik_llama_cpp_sys::llama_backend_init() };
        Ok(Self { _private: () })
    }
}

impl Drop for LlamaBackend {
    fn drop(&mut self) {
        // SAFETY: matches the single `llama_backend_init` above.
        unsafe { ik_llama_cpp_sys::llama_backend_free() };
        INITIALIZED.store(false, Ordering::SeqCst);
    }
}