car_inference/

hardware.rs

//! Hardware detection — auto-configure models and context based on system capabilities.

use serde::{Serialize, Deserialize};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HardwareInfo {
    pub os: String,
    pub arch: String,
    pub cpu_cores: usize,
    pub total_ram_mb: u64,
    pub gpu_backend: GpuBackend,
    pub gpu_memory_mb: Option<u64>,
    /// Recommended model based on available resources.
    pub recommended_model: String,
    /// Recommended max context length in tokens.
    pub recommended_context: usize,
    /// Maximum model size in MB that fits in memory (with headroom for KV cache).
    pub max_model_mb: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GpuBackend {
    Metal,
    Cuda,
    Cpu,
}

impl HardwareInfo {
    /// Auto-detect system hardware and compute recommendations.
    pub fn detect() -> Self {
        let os = detect_os();
        let arch = std::env::consts::ARCH.to_string();
        let cpu_cores = std::thread::available_parallelism()
            .map(|n| n.get()).unwrap_or(1);
        let total_ram_mb = detect_ram_mb();
        let gpu_backend = detect_gpu_backend();
        let gpu_memory_mb = detect_gpu_memory_mb(&gpu_backend, total_ram_mb);

        // Compute how much memory is available for models
        // On unified memory (Apple Silicon): GPU shares system RAM
        // Budget: use ~60% of available memory for model weights.
        // Reserve ~700MB for the embedding model (Qwen3-Embedding-0.6B Q8_0).
        // Rest for KV cache, activations, and OS.
        let available_mb = gpu_memory_mb.unwrap_or(total_ram_mb);
        let embedding_model_mb: u64 = 700; // Qwen3-Embedding-0.6B-Q8_0
        let max_model_mb = ((available_mb as f64 * 0.6) as u64).saturating_sub(embedding_model_mb);

        let recommended_model = recommend_model(max_model_mb);
        let recommended_context = recommend_context(available_mb, &recommended_model);

        Self {
            os, arch, cpu_cores, total_ram_mb,
            gpu_backend, gpu_memory_mb,
            recommended_model, recommended_context, max_model_mb,
        }
    }
}

fn detect_os() -> String {
    if cfg!(target_os = "macos") { "macos".into() }
    else if cfg!(target_os = "linux") { "linux".into() }
    else if cfg!(target_os = "windows") { "windows".into() }
    else { std::env::consts::OS.into() }
}

fn detect_ram_mb() -> u64 {
    // macOS: sysctl hw.memsize
    #[cfg(target_os = "macos")]
    {
        if let Ok(output) = std::process::Command::new("sysctl")
            .args(["-n", "hw.memsize"])
            .output()
        {
            if let Ok(s) = String::from_utf8(output.stdout) {
                if let Ok(bytes) = s.trim().parse::<u64>() {
                    return bytes / (1024 * 1024);
                }
            }
        }
    }
    // Linux: /proc/meminfo
    #[cfg(target_os = "linux")]
    {
        if let Ok(content) = std::fs::read_to_string("/proc/meminfo") {
            for line in content.lines() {
                if line.starts_with("MemTotal:") {
                    let parts: Vec<&str> = line.split_whitespace().collect();
                    if parts.len() >= 2 {
                        if let Ok(kb) = parts[1].parse::<u64>() {
                            return kb / 1024;
                        }
                    }
                }
            }
        }
    }
    // Fallback: assume 8GB
    8192
}

fn detect_gpu_backend() -> GpuBackend {
    #[cfg(feature = "metal")]
    { return GpuBackend::Metal; }
    #[cfg(feature = "cuda")]
    { return GpuBackend::Cuda; }
    #[cfg(not(any(feature = "metal", feature = "cuda")))]
    {
        // Check if we're on Apple Silicon (could use Metal even if feature isn't compiled)
        if cfg!(target_os = "macos") && cfg!(target_arch = "aarch64") {
            GpuBackend::Metal
        } else {
            GpuBackend::Cpu
        }
    }
}

fn detect_gpu_memory_mb(backend: &GpuBackend, total_ram_mb: u64) -> Option<u64> {
    match backend {
        GpuBackend::Metal => {
            // Apple Silicon has unified memory — GPU can use most of system RAM.
            // macOS reserves ~2-4GB for OS, so usable = total - 4GB
            Some(total_ram_mb.saturating_sub(4096))
        }
        GpuBackend::Cuda => {
            // Try nvidia-smi
            #[cfg(target_os = "linux")]
            {
                if let Ok(output) = std::process::Command::new("nvidia-smi")
                    .args(["--query-gpu=memory.total", "--format=csv,noheader,nounits"])
                    .output()
                {
                    if let Ok(s) = String::from_utf8(output.stdout) {
                        if let Ok(mb) = s.trim().parse::<u64>() {
                            return Some(mb);
                        }
                    }
                }
            }
            None
        }
        GpuBackend::Cpu => None,
    }
}

/// Recommend the best model that fits in available memory.
fn recommend_model(max_model_mb: u64) -> String {
    if max_model_mb >= 17000 {
        "Qwen3-30B-A3B".into()
    } else if max_model_mb >= 4900 {
        "Qwen3-8B".into()
    } else if max_model_mb >= 2500 {
        "Qwen3-4B".into()
    } else if max_model_mb >= 1800 {
        "Qwen3-1.7B".into()
    } else {
        "Qwen3-0.6B".into()
    }
}

/// Recommend context length based on available memory and model size.
fn recommend_context(available_mb: u64, model_name: &str) -> usize {
    let model_mb = match model_name {
        "Qwen3-0.6B" => 650,
        "Qwen3-1.7B" => 1800,
        "Qwen3-4B" => 2500,
        "Qwen3-8B" => 4900,
        "Qwen3-30B-A3B" => 17000,
        _ => 650,
    };
    let kv_cost_per_1k = match model_name {
        "Qwen3-0.6B" => 0.1,
        "Qwen3-1.7B" => 0.3,
        "Qwen3-4B" => 0.5,
        "Qwen3-8B" => 1.0,
        "Qwen3-30B-A3B" => 1.5,
        _ => 0.1,
    };

    let kv_budget_mb = available_mb.saturating_sub(model_mb).saturating_sub(1024) as f64;
    let max_context = (kv_budget_mb / kv_cost_per_1k * 1000.0) as usize;

    max_context.clamp(2048, 131072)
}