ferrox_cuda/capability.rs
1//! Runtime hardware capability detection: probe once, report a plain
2//! struct, and let every performance-relevant decision (thread pool
3//! width, SIMD kernel selection, GPU residency) derive from the
4//! *detected* machine rather than being hardcoded. Ferrox
5//! today only has a CPU execution path, so `HardwareProfile::detect()`
6//! is honest about that: the CUDA fields are always populated (zero /
7//! false / None) unless built with `--features cuda`, and even then
8//! they report exactly what `ferrox-cuda`'s device probe finds, no
9//! more.
10//!
11//! Everything in this module is real and testable in any environment,
12//! including one with no GPU: CPU core count and SIMD flags are always
13//! detectable, and "zero CUDA devices found" is itself a correct,
14//! verifiable answer on a CPU-only host, not a stand-in for an
15//! untested code path.
16
17/// CPU SIMD instruction-set availability (runtime-detected via
18/// `is_x86_feature_detected!`, not compile-time `#[cfg]`), matching
19/// the fields `ferrox_quant`'s kernel dispatch actually checks.
20#[derive(Debug, Clone, Default, PartialEq, Eq)]
21pub struct SimdCaps {
22 pub avx2: bool,
23 pub avx512f: bool,
24 pub fma: bool,
25 pub neon: bool,
26}
27
28impl SimdCaps {
29 pub fn detect() -> Self {
30 #[cfg(target_arch = "x86_64")]
31 {
32 SimdCaps {
33 avx2: is_x86_feature_detected!("avx2"),
34 avx512f: is_x86_feature_detected!("avx512f"),
35 fma: is_x86_feature_detected!("fma"),
36 neon: false,
37 }
38 }
39 #[cfg(target_arch = "aarch64")]
40 {
41 SimdCaps {
42 avx2: false,
43 avx512f: false,
44 fma: false,
45 neon: std::arch::is_aarch64_feature_detected!("neon"),
46 }
47 }
48 #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
49 {
50 SimdCaps::default()
51 }
52 }
53
54 /// Short human label of the widest available SIMD path ferrox
55 /// actually has a kernel for today (Q8_0/Q4_0 dispatch in
56 /// `ferrox-quant` currently only implements the AVX2+FMA path, so
57 /// `avx512f` is reported here as detected-but-unused -- no
58 /// AVX-512 kernel exists yet).
59 pub fn label(&self) -> &'static str {
60 if self.avx2 && self.fma {
61 "AVX2+FMA (ferrox's fastest implemented CPU kernel)"
62 } else if self.neon {
63 "NEON (ferrox Q4_K/Q6_K/Q8_0/Q4_0 fused dots live)"
64 } else {
65 "scalar (no SIMD kernel available for this host)"
66 }
67 }
68}
69
70/// A snapshot of the host's inference-relevant capabilities. CPU/RAM
71/// fields are always real; CUDA fields are always present but only
72/// ever non-default when built with `--features cuda` on a host that
73/// actually has a CUDA-capable device.
74#[derive(Debug, Clone)]
75pub struct HardwareProfile {
76 pub cpu_logical_cores: usize,
77 pub host_ram_total_bytes: u64,
78 pub simd: SimdCaps,
79 pub cuda_available: bool,
80 pub cuda_device_count: usize,
81 pub cuda_device_name: Option<String>,
82 pub cuda_vram_total_bytes: u64,
83 /// Free device memory at probe time (`cuMemGetInfo`'s first half).
84 /// Zero without `--features cuda` or without a device, exactly like
85 /// the other CUDA fields. A memory budget should be drawn against
86 /// this rather than the total -- see
87 /// `ferrox_models::device_budget`.
88 pub cuda_vram_free_bytes: u64,
89}
90
91impl HardwareProfile {
92 /// Probe the machine. Cheap and side-effect-free on the CPU side;
93 /// the CUDA probe (when built with `--features cuda`) opens a
94 /// driver context to enumerate devices and degrades to "no CUDA"
95 /// cleanly on hosts without one, in the exact form the fields
96 /// below already represent for a CPU-only build.
97 pub fn detect() -> Self {
98 let cpu_logical_cores = std::thread::available_parallelism()
99 .map(|n| n.get())
100 .unwrap_or(1);
101 let host_ram_total_bytes = detect_total_ram_bytes();
102 let simd = SimdCaps::detect();
103
104 #[cfg(feature = "cuda")]
105 let (
106 cuda_available,
107 cuda_device_count,
108 cuda_device_name,
109 cuda_vram_total_bytes,
110 cuda_vram_free_bytes,
111 ) = {
112 match crate::gpu::probe() {
113 Some(info) => (
114 true,
115 info.device_count,
116 info.first_device_name,
117 info.total_vram_bytes,
118 info.free_vram_bytes,
119 ),
120 None => (false, 0, None, 0, 0),
121 }
122 };
123 #[cfg(not(feature = "cuda"))]
124 let (
125 cuda_available,
126 cuda_device_count,
127 cuda_device_name,
128 cuda_vram_total_bytes,
129 cuda_vram_free_bytes,
130 ) = (false, 0, None, 0, 0);
131
132 HardwareProfile {
133 cpu_logical_cores,
134 host_ram_total_bytes,
135 simd,
136 cuda_available,
137 cuda_device_count,
138 cuda_device_name,
139 cuda_vram_total_bytes,
140 cuda_vram_free_bytes,
141 }
142 }
143}
144
145/// Reads total physical RAM from `/proc/meminfo` on Linux (no external
146/// `sysinfo`-style crate dependency, keeping this pure-Rust and
147/// dependency-light). Returns 0 on any parse failure or non-Linux host
148/// rather than panicking -- this is diagnostic information, not
149/// something correctness depends on.
150fn detect_total_ram_bytes() -> u64 {
151 #[cfg(target_os = "linux")]
152 {
153 if let Ok(contents) = std::fs::read_to_string("/proc/meminfo") {
154 for line in contents.lines() {
155 if let Some(rest) = line.strip_prefix("MemTotal:") {
156 let kb: u64 = rest
157 .trim()
158 .trim_end_matches(" kB")
159 .trim()
160 .parse()
161 .unwrap_or(0);
162 return kb * 1024;
163 }
164 }
165 }
166 0
167 }
168 #[cfg(target_os = "macos")]
169 {
170 // `sysctl -n hw.memsize` rather than a new libc/sysinfo-style
171 // dependency, matching this module's existing dependency-light
172 // stance for /proc/meminfo above. Real gap found via actually
173 // running `ferrox inspect-plan` on this dev machine: without
174 // this, host_ram_total_bytes silently stayed 0 on every macOS
175 // host, making --strict always report "DOES NOT FIT" regardless
176 // of real available RAM.
177 std::process::Command::new("sysctl")
178 .args(["-n", "hw.memsize"])
179 .output()
180 .ok()
181 .and_then(|out| String::from_utf8(out.stdout).ok())
182 .and_then(|s| s.trim().parse().ok())
183 .unwrap_or(0)
184 }
185 #[cfg(not(any(target_os = "linux", target_os = "macos")))]
186 {
187 0
188 }
189}
190
191#[cfg(test)]
192mod tests {
193 use super::*;
194
195 #[test]
196 fn detect_never_panics_and_reports_at_least_one_core() {
197 let profile = HardwareProfile::detect();
198 assert!(profile.cpu_logical_cores >= 1);
199 }
200
201 #[test]
202 fn detect_reports_plausible_ram_on_linux_or_macos() {
203 let profile = HardwareProfile::detect();
204 // Any real Linux or macOS host will always have at least, say,
205 // 128 MB, so treat 0 as "couldn't detect" (unsupported OS) and
206 // anything absurdly small as a parsing bug, not assert an
207 // exact value that would break on other hosts.
208 if profile.host_ram_total_bytes > 0 {
209 assert!(profile.host_ram_total_bytes > 128 * 1024 * 1024);
210 }
211 }
212
213 #[test]
214 fn simd_caps_label_is_never_empty() {
215 let caps = SimdCaps::detect();
216 assert!(!caps.label().is_empty());
217 }
218
219 #[test]
220 #[cfg(not(feature = "cuda"))]
221 fn without_cuda_feature_profile_always_reports_no_cuda() {
222 let profile = HardwareProfile::detect();
223 assert!(!profile.cuda_available);
224 assert_eq!(profile.cuda_device_count, 0);
225 assert_eq!(profile.cuda_device_name, None);
226 assert_eq!(profile.cuda_vram_free_bytes, 0);
227 }
228}