#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HostGpu {
None,
Cuda {
major: u32,
},
Rocm,
Opencl,
Vulkan,
Metal,
}
pub fn detect() -> HostGpu {
if let Some(forced) = llm_library_override() {
return forced;
}
#[cfg(target_os = "macos")]
{
detect_macos()
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
{
detect_gpu_api_isolated()
}
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
{
HostGpu::None
}
}
fn llm_library_override() -> Option<HostGpu> {
parse_llm_library(std::env::var("LLMMAN_LLM_LIBRARY").ok().as_deref())
}
fn parse_llm_library(value: Option<&str>) -> Option<HostGpu> {
let v = value?.trim();
if v.is_empty() {
return None;
}
match v.to_ascii_lowercase().as_str() {
"cpu" | "none" => Some(HostGpu::None),
"cuda" | "cuda12" | "cuda_v12" => Some(HostGpu::Cuda { major: 12 }),
"cuda13" | "cuda_v13" => Some(HostGpu::Cuda { major: 13 }),
"rocm" => Some(HostGpu::Rocm),
"opencl" => Some(HostGpu::Opencl),
"vulkan" => Some(HostGpu::Vulkan),
"metal" => Some(HostGpu::Metal),
_ => None,
}
}
pub fn igpu_enabled() -> bool {
parse_igpu_enabled(std::env::var("LLMMAN_IGPU_ENABLE").ok().as_deref())
}
fn parse_igpu_enabled(value: Option<&str>) -> bool {
match value.map(str::trim) {
Some(v) if !v.is_empty() => {
matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")
}
_ => false,
}
}
pub fn memory_bytes(vram_bytes: u64) -> u64 {
if vram_bytes > 0 {
return vram_bytes;
}
#[cfg(any(target_os = "linux", target_os = "android"))]
{
std::fs::read_to_string("/proc/meminfo")
.ok()
.and_then(|s| parse_meminfo_total(&s))
.unwrap_or(0)
}
#[cfg(target_os = "macos")]
{
apple_unified_memory_bytes().unwrap_or(0)
}
#[cfg(windows)]
{
windows_memory_bytes().unwrap_or(0)
}
#[cfg(not(any(
target_os = "linux",
target_os = "android",
target_os = "macos",
windows
)))]
{
0
}
}
#[cfg(windows)]
fn windows_memory_bytes() -> Option<u64> {
#[repr(C)]
struct MemoryStatusEx {
length: u32,
memory_load: u32,
total_phys: u64,
avail_phys: u64,
total_page_file: u64,
avail_page_file: u64,
total_virtual: u64,
avail_virtual: u64,
avail_extended_virtual: u64,
}
#[link(name = "kernel32")]
extern "system" {
fn GlobalMemoryStatusEx(buffer: *mut MemoryStatusEx) -> i32;
}
let mut status = MemoryStatusEx {
length: std::mem::size_of::<MemoryStatusEx>() as u32,
memory_load: 0,
total_phys: 0,
avail_phys: 0,
total_page_file: 0,
avail_page_file: 0,
total_virtual: 0,
avail_virtual: 0,
avail_extended_virtual: 0,
};
(unsafe { GlobalMemoryStatusEx(&mut status) } != 0).then_some(status.total_phys)
}
#[cfg(any(target_os = "linux", target_os = "android", test))]
fn parse_meminfo_total(meminfo: &str) -> Option<u64> {
let kib = meminfo
.lines()
.find_map(|l| l.strip_prefix("MemTotal:"))?
.trim()
.strip_suffix("kB")?
.trim()
.parse::<u64>()
.ok()?;
Some(kib * 1024)
}
pub const PROBE_SUBPROCESS_ARG: &str = "__hostgpu-probe";
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn detect_gpu_api_isolated() -> HostGpu {
spawn_probe_subprocess().unwrap_or(HostGpu::None)
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn spawn_probe_subprocess() -> Option<HostGpu> {
Some(spawn_probe_subprocess_with_vram()?.0)
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn spawn_probe_subprocess_with_vram() -> Option<(HostGpu, u64)> {
let exe = std::env::current_exe().ok()?;
let output = std::process::Command::new(exe)
.arg(PROBE_SUBPROCESS_ARG)
.output()
.ok()?;
if !output.status.success() {
return None;
}
parse_probe_output_with_vram(&String::from_utf8_lossy(&output.stdout))
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn parse_probe_output_with_vram(stdout: &str) -> Option<(HostGpu, u64)> {
let gpu = parse_probe_output(stdout)?;
let vram = stdout
.lines()
.nth(1)
.and_then(|l| l.trim().parse().ok())
.unwrap_or(0);
Some((gpu, vram))
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn parse_probe_output(stdout: &str) -> Option<HostGpu> {
let line = stdout.lines().next()?.trim();
if let Some(major) = line.strip_prefix("cuda:") {
return Some(HostGpu::Cuda {
major: major.parse().ok()?,
});
}
match line {
"rocm" => Some(HostGpu::Rocm),
"opencl" => Some(HostGpu::Opencl),
"vulkan" => Some(HostGpu::Vulkan),
"none" => Some(HostGpu::None),
_ => None,
}
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
pub fn probe_subprocess_main() -> ! {
let (gpu, vram) = detect_gpu_api_uncontained();
let line = match gpu {
HostGpu::None => "none".to_string(),
HostGpu::Cuda { major } => format!("cuda:{major}"),
HostGpu::Rocm => "rocm".to_string(),
HostGpu::Opencl => "opencl".to_string(),
HostGpu::Vulkan => "vulkan".to_string(),
HostGpu::Metal => "none".to_string(),
};
println!("{line}\n{vram}");
std::process::exit(0);
}
#[cfg(not(any(target_os = "linux", target_os = "windows")))]
pub fn probe_subprocess_main() -> ! {
println!("none");
std::process::exit(0);
}
#[cfg(target_os = "macos")]
fn detect_macos() -> HostGpu {
if std::env::consts::ARCH == "aarch64" {
HostGpu::Metal
} else {
HostGpu::None
}
}
pub fn detect_with_vram() -> (HostGpu, u64) {
#[cfg(target_os = "macos")]
{
(detect_macos(), apple_unified_memory_bytes().unwrap_or(0))
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
{
spawn_probe_subprocess_with_vram().unwrap_or((HostGpu::None, 0))
}
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
{
(HostGpu::None, 0)
}
}
#[cfg(target_os = "macos")]
fn apple_unified_memory_bytes() -> Option<u64> {
let mut bytes: u64 = 0;
let mut size = std::mem::size_of::<u64>();
let name = c"hw.memsize";
let rc = unsafe {
libc::sysctlbyname(
name.as_ptr(),
&mut bytes as *mut u64 as *mut std::ffi::c_void,
&mut size,
std::ptr::null_mut(),
0,
)
};
(rc == 0).then_some(bytes)
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn detect_gpu_api_uncontained() -> (HostGpu, u64) {
if let Some((gpu, vram)) = detect_cuda() {
return (gpu, vram);
}
if let Some(vram) = detect_rocm() {
return (HostGpu::Rocm, vram);
}
if let Some(vram) = detect_opencl() {
return (HostGpu::Opencl, vram);
}
if let Some(vram) = detect_vulkan() {
return (HostGpu::Vulkan, vram);
}
(HostGpu::None, 0)
}
#[cfg(any(target_os = "linux", target_os = "windows"))]
mod vendor;
#[cfg(any(target_os = "linux", target_os = "windows"))]
use vendor::{detect_cuda, detect_opencl, detect_rocm, detect_vulkan};
#[cfg(test)]
mod tests {
use super::*;
const GIB: u64 = 1024 * 1024 * 1024;
#[test]
fn parse_meminfo_total_reads_the_kib_line() {
let meminfo = "MemTotal: 7864320 kB\nMemFree: 1234 kB\n";
assert_eq!(parse_meminfo_total(meminfo), Some(7864320 * 1024));
assert_eq!(parse_meminfo_total("MemFree: 1 kB\n"), None);
assert_eq!(parse_meminfo_total("MemTotal: lots\n"), None);
}
#[test]
fn memory_bytes_prefers_vram() {
assert_eq!(memory_bytes(8 * GIB), 8 * GIB);
}
#[test]
fn parse_llm_library_recognizes_every_documented_name_case_insensitively() {
assert_eq!(parse_llm_library(Some("cpu")), Some(HostGpu::None));
assert_eq!(parse_llm_library(Some("CPU")), Some(HostGpu::None));
assert_eq!(parse_llm_library(Some("none")), Some(HostGpu::None));
assert_eq!(
parse_llm_library(Some("cuda")),
Some(HostGpu::Cuda { major: 12 })
);
assert_eq!(
parse_llm_library(Some("cuda12")),
Some(HostGpu::Cuda { major: 12 })
);
assert_eq!(
parse_llm_library(Some("CUDA13")),
Some(HostGpu::Cuda { major: 13 })
);
assert_eq!(parse_llm_library(Some("rocm")), Some(HostGpu::Rocm));
assert_eq!(parse_llm_library(Some("OpenCL")), Some(HostGpu::Opencl));
assert_eq!(parse_llm_library(Some("Vulkan")), Some(HostGpu::Vulkan));
assert_eq!(parse_llm_library(Some("metal")), Some(HostGpu::Metal));
}
#[test]
fn parse_llm_library_falls_through_to_autodetection_when_unset_blank_or_unknown() {
assert_eq!(parse_llm_library(None), None);
assert_eq!(parse_llm_library(Some("")), None);
assert_eq!(parse_llm_library(Some(" ")), None);
assert_eq!(parse_llm_library(Some("intel-arc")), None);
}
#[test]
fn parse_igpu_enabled_recognizes_truthy_spellings_only() {
assert!(!parse_igpu_enabled(None));
assert!(!parse_igpu_enabled(Some("")));
assert!(!parse_igpu_enabled(Some("0")));
assert!(!parse_igpu_enabled(Some("false")));
assert!(!parse_igpu_enabled(Some("no")));
assert!(!parse_igpu_enabled(Some("off")));
assert!(parse_igpu_enabled(Some("1")));
assert!(parse_igpu_enabled(Some("true")));
assert!(parse_igpu_enabled(Some("YES")));
assert!(parse_igpu_enabled(Some("on")));
}
#[test]
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn parse_probe_output_round_trips_every_variant() {
assert_eq!(parse_probe_output("none\n"), Some(HostGpu::None));
assert_eq!(parse_probe_output("rocm\n"), Some(HostGpu::Rocm));
assert_eq!(parse_probe_output("opencl\n"), Some(HostGpu::Opencl));
assert_eq!(parse_probe_output("vulkan\n"), Some(HostGpu::Vulkan));
assert_eq!(
parse_probe_output("cuda:12\n"),
Some(HostGpu::Cuda { major: 12 })
);
assert_eq!(
parse_probe_output("cuda:13\n"),
Some(HostGpu::Cuda { major: 13 })
);
assert_eq!(parse_probe_output(""), None);
assert_eq!(parse_probe_output("garbage\n"), None);
assert_eq!(parse_probe_output("cuda:not-a-number\n"), None);
}
#[test]
#[cfg(any(target_os = "linux", target_os = "windows"))]
fn parse_probe_output_with_vram_reads_the_second_line() {
assert_eq!(
parse_probe_output_with_vram("cuda:12\n8589934592\n"),
Some((HostGpu::Cuda { major: 12 }, 8589934592))
);
assert_eq!(
parse_probe_output_with_vram("none\n"),
Some((HostGpu::None, 0))
);
assert_eq!(
parse_probe_output_with_vram("rocm\ngarbage\n"),
Some((HostGpu::Rocm, 0))
);
assert_eq!(parse_probe_output_with_vram("garbage\n0\n"), None);
}
#[test]
#[cfg(unix)]
fn a_failing_subprocess_status_is_not_mistaken_for_a_real_result() {
let output = std::process::Command::new("false")
.output()
.expect("run `false`");
assert!(!output.status.success());
}
}