llama_cpp_bindings/
llama_backend_device.rs

1use std::ffi::c_char;
2
3/// Backend device type
4#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5pub enum LlamaBackendDeviceType {
6    /// CPU device
7    Cpu,
8    /// ACCEL device
9    Accelerator,
10    /// GPU device
11    Gpu,
12    /// iGPU device
13    IntegratedGpu,
14    /// Unknown device type
15    Unknown,
16}
17
18const fn device_type_from_raw(
19    raw_type: llama_cpp_bindings_sys::ggml_backend_dev_type,
20) -> LlamaBackendDeviceType {
21    match raw_type {
22        llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU => LlamaBackendDeviceType::Cpu,
23        llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL => {
24            LlamaBackendDeviceType::Accelerator
25        }
26        llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU => LlamaBackendDeviceType::Gpu,
27        llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU => {
28            LlamaBackendDeviceType::IntegratedGpu
29        }
30        _ => LlamaBackendDeviceType::Unknown,
31    }
32}
33
34/// A ggml backend device
35///
36/// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
37#[derive(Debug, Clone)]
38pub struct LlamaBackendDevice {
39    /// The index of the device
40    ///
41    /// The index is can be used from `LlamaModelParams::with_devices` to select specific devices.
42    pub index: usize,
43    /// The name of the device (e.g. "Vulkan0")
44    pub name: String,
45    /// A description of the device (e.g. "NVIDIA `GeForce` RTX 3080")
46    pub description: String,
47    /// The backend of the device (e.g. "Vulkan", "CUDA", "CPU")
48    pub backend: String,
49    /// Total memory of the device in bytes
50    pub memory_total: usize,
51    /// Free memory of the device in bytes
52    pub memory_free: usize,
53    /// Device type
54    pub device_type: LlamaBackendDeviceType,
55}
56
57fn cstr_to_string(ptr: *const c_char) -> String {
58    if ptr.is_null() {
59        String::new()
60    } else {
61        unsafe { std::ffi::CStr::from_ptr(ptr) }
62            .to_string_lossy()
63            .to_string()
64    }
65}
66
67/// List ggml backend devices
68#[must_use]
69pub fn list_llama_ggml_backend_devices() -> Vec<LlamaBackendDevice> {
70    let mut devices = Vec::new();
71    let device_count = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_count() };
72
73    for device_index in 0..device_count {
74        let dev = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_get(device_index) };
75        let props = unsafe {
76            let mut props = std::mem::zeroed();
77            llama_cpp_bindings_sys::ggml_backend_dev_get_props(dev, &raw mut props);
78            props
79        };
80        let name = cstr_to_string(props.name);
81        let description = cstr_to_string(props.description);
82        let backend_reg = unsafe { llama_cpp_bindings_sys::ggml_backend_dev_backend_reg(dev) };
83        let backend_name = unsafe { llama_cpp_bindings_sys::ggml_backend_reg_name(backend_reg) };
84        let backend = cstr_to_string(backend_name);
85        let memory_total = props.memory_total;
86        let memory_free = props.memory_free;
87        let device_type = device_type_from_raw(props.type_);
88        devices.push(LlamaBackendDevice {
89            index: device_index,
90            name,
91            description,
92            backend,
93            memory_total,
94            memory_free,
95            device_type,
96        });
97    }
98
99    devices
100}
101
102#[cfg(test)]
103mod tests {
104    use super::{cstr_to_string, list_llama_ggml_backend_devices};
105
106    #[test]
107    fn cstr_to_string_with_null_returns_empty() {
108        let result = cstr_to_string(std::ptr::null());
109
110        assert_eq!(result, "");
111    }
112
113    #[test]
114    fn cstr_to_string_with_valid_ptr() {
115        let result = cstr_to_string(c"hello".as_ptr());
116
117        assert_eq!(result, "hello");
118    }
119
120    #[test]
121    fn list_devices_returns_at_least_one() {
122        #[cfg(feature = "dynamic-backends")]
123        crate::load_backends::load_backends().unwrap();
124
125        let devices = list_llama_ggml_backend_devices();
126        assert!(!devices.is_empty());
127        assert_eq!(devices[0].index, 0);
128        assert!(!devices[0].name.is_empty());
129    }
130
131    #[test]
132    fn device_type_from_raw_all_variants() {
133        use super::LlamaBackendDeviceType;
134        use super::device_type_from_raw;
135
136        assert_eq!(
137            device_type_from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_CPU),
138            LlamaBackendDeviceType::Cpu
139        );
140        assert_eq!(
141            device_type_from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_ACCEL),
142            LlamaBackendDeviceType::Accelerator
143        );
144        assert_eq!(
145            device_type_from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_GPU),
146            LlamaBackendDeviceType::Gpu
147        );
148        assert_eq!(
149            device_type_from_raw(llama_cpp_bindings_sys::GGML_BACKEND_DEVICE_TYPE_IGPU),
150            LlamaBackendDeviceType::IntegratedGpu
151        );
152        assert_eq!(device_type_from_raw(9999), LlamaBackendDeviceType::Unknown);
153    }
154}
llama_cpp_bindings/llama_backend_device.rs

llama_cpp_bindings/
llama_backend_device.rs
