llama-cpp-bindings 0.8.0

llama.cpp bindings for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

use std::ffi::c_char;

use crate::llama_backend_device_type::device_type_from_raw;

pub use crate::llama_backend_device_type::LlamaBackendDeviceType;

#[derive(Debug, Clone)]
pub struct LlamaBackendDevice {
    pub index: usize,
    pub name: String,
    pub description: String,
    pub backend: String,
    pub memory_total: usize,
    pub memory_free: usize,
    pub device_type: LlamaBackendDeviceType,
}

fn cstr_to_string(ptr: *const c_char) -> String {
    if ptr.is_null() {
        String::new()
    } else {
        unsafe { std::ffi::CStr::from_ptr(ptr) }
            .to_string_lossy()
            .to_string()
    }
}

#[must_use]
pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
    let mut devices = Vec::new();
    let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() };

    for device_index in 0..device_count {
        let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) };
        let props = unsafe {
            let mut props = std::mem::zeroed();
            llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props);
            props
        };
        let name = cstr_to_string(props.name);
        let description = cstr_to_string(props.description);
        let backend_reg = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_backend_reg(dev) };
        let backend_name = unsafe { llama_cpp_bindings_sys::ggml_backend_reg_name(backend_reg) };
        let backend = cstr_to_string(backend_name);
        let memory_total = props.memory_total;
        let memory_free = props.memory_free;
        let device_type = device_type_from_raw(props.type_);
        devices.push(LlamaBackendDevice {
            index: device_index,
            name,
            description,
            backend,
            memory_total,
            memory_free,
            device_type,
        });
    }

    devices
}

#[cfg(test)]
mod tests {
    use super::{cstr_to_string, list_llama_ggml_backend_devices};

    #[test]
    fn cstr_to_string_with_null_returns_empty() {
        let result = cstr_to_string(std::ptr::null());

        assert_eq!(result, "");
    }

    #[test]
    fn cstr_to_string_with_valid_ptr() {
        let result = cstr_to_string(c"hello".as_ptr());

        assert_eq!(result, "hello");
    }

    #[test]
    fn list_devices_returns_at_least_one() {
        #[cfg(feature = "dynamic-backends")]
        crate::load_backends::load_backends().unwrap();

        let devices = list_llama_ggml_backend_devices();
        assert!(!devices.is_empty());
        assert_eq!(devices[0].index, 0);
        assert!(!devices[0].name.is_empty());
    }
}