cgp/profilers/
system.rs

1//! System health and VRAM collection via nvidia-smi and /proc.
2//! Spec sections 9.8 (VRAM), 9.10 (System Health), 9.11 (Energy).
3
4use crate::metrics::catalog::{EnergyMetrics, SystemHealthMetrics, VramMetrics};
5use std::process::Command;
6
7/// Collect system health metrics from nvidia-smi (NVML) and /proc.
8pub fn collect_system_health() -> Option<SystemHealthMetrics> {
9    let gpu = query_nvidia_smi(&[
10        "temperature.gpu",
11        "power.draw",
12        "clocks.current.sm",
13        "clocks.current.memory",
14        "memory.used",
15        "memory.total",
16    ])?;
17
18    let fields: Vec<&str> = gpu.split(", ").collect();
19    if fields.len() < 6 {
20        return None;
21    }
22
23    let cpu_freq = read_cpu_frequency().unwrap_or(0.0);
24    let cpu_temp = read_cpu_temperature().unwrap_or(0.0);
25
26    // Unified-memory platforms report memory.total as N/A (parses to 0); fall back to system RAM.
27    let mut gpu_mem_total = parse_nvidia_val(fields[5]);
28    if gpu_mem_total <= 0.0 {
29        gpu_mem_total = read_system_memory_total_mb().unwrap_or(0.0);
30    }
31
32    Some(SystemHealthMetrics {
33        gpu_temperature_celsius: parse_nvidia_val(fields[0]),
34        gpu_power_watts: parse_nvidia_val(fields[1]),
35        gpu_clock_mhz: parse_nvidia_val(fields[2]),
36        gpu_memory_clock_mhz: parse_nvidia_val(fields[3]),
37        cpu_frequency_mhz: cpu_freq,
38        cpu_temperature_celsius: cpu_temp,
39        gpu_memory_used_mb: parse_nvidia_val(fields[4]),
40        gpu_memory_total_mb: gpu_mem_total,
41    })
42}
43
44/// Collect VRAM metrics from nvidia-smi.
45pub fn collect_vram() -> Option<VramMetrics> {
46    let gpu = query_nvidia_smi(&["memory.used", "memory.total", "memory.free"])?;
47
48    let fields: Vec<&str> = gpu.split(", ").collect();
49    if fields.len() < 3 {
50        return None;
51    }
52
53    let used = parse_nvidia_val(fields[0]);
54    let mut total = parse_nvidia_val(fields[1]);
55    let free = parse_nvidia_val(fields[2]);
56    // Unified-memory NVIDIA platforms (GB10/GH200/Jetson) report VRAM total as [N/A] via nvidia-smi
57    // because the GPU shares system RAM. Fall back to total system memory so the profiler reports the
58    // (unified) memory budget instead of 0.
59    if total <= 0.0 {
60        total = read_system_memory_total_mb().unwrap_or(0.0);
61    }
62    let utilization = if total > 0.0 {
63        used / total * 100.0
64    } else {
65        0.0
66    };
67
68    Some(VramMetrics {
69        vram_used_mb: used,
70        vram_total_mb: total,
71        vram_free_mb: free,
72        vram_utilization_pct: utilization,
73        vram_peak_mb: used, // snapshot — no tracking history
74        vram_allocation_count: 0,
75        vram_fragmentation_pct: 0.0,
76    })
77}
78
79/// Compute energy efficiency from power and throughput.
80pub fn compute_energy(power_watts: f64, tflops: f64, duration_us: f64) -> Option<EnergyMetrics> {
81    if power_watts <= 0.0 {
82        return None;
83    }
84    let tflops_per_watt = if power_watts > 0.0 {
85        tflops / power_watts
86    } else {
87        0.0
88    };
89    let joules = power_watts * duration_us * 1e-6;
90    Some(EnergyMetrics {
91        tflops_per_watt,
92        joules_per_inference: joules,
93    })
94}
95
96/// Run nvidia-smi --query-gpu and return the CSV row.
97fn query_nvidia_smi(fields: &[&str]) -> Option<String> {
98    let query = fields.join(",");
99    let output = Command::new("nvidia-smi")
100        .args(["--query-gpu", &query, "--format=csv,noheader,nounits"])
101        .output()
102        .ok()?;
103
104    if !output.status.success() {
105        return None;
106    }
107    let stdout = String::from_utf8_lossy(&output.stdout);
108    let line = stdout.trim();
109    if line.is_empty() || line.contains("[N/A]") && line.chars().all(|c| c == ',' || c == ' ') {
110        return None;
111    }
112    Some(line.to_string())
113}
114
115/// Parse a numeric value from nvidia-smi output (handles "123 W", "45 MiB", etc.)
116fn parse_nvidia_val(s: &str) -> f64 {
117    let s = s.trim();
118    if s == "[N/A]" || s == "N/A" {
119        return 0.0;
120    }
121    // Take the first token that looks numeric
122    s.split_whitespace()
123        .next()
124        .and_then(|token| token.parse::<f64>().ok())
125        .unwrap_or(0.0)
126}
127
128/// Total system RAM in MB, read from `/proc/meminfo` (`MemTotal`).
129///
130/// Used as a fallback for GPU memory total on unified-memory NVIDIA platforms (GB10 / Grace-Blackwell,
131/// GH200, Jetson), where `nvidia-smi --query-gpu=memory.total` reports `[N/A]` because the GPU shares
132/// system RAM rather than exposing dedicated VRAM. On dedicated GPUs nvidia-smi returns a real value,
133/// so this fallback is never reached and behavior is unchanged.
134fn read_system_memory_total_mb() -> Option<f64> {
135    let content = std::fs::read_to_string("/proc/meminfo").ok()?;
136    for line in content.lines() {
137        // Format: "MemTotal:       65780480 kB"
138        if let Some(rest) = line.strip_prefix("MemTotal:") {
139            let kb = rest.split_whitespace().next()?.parse::<f64>().ok()?;
140            return Some(kb / 1024.0);
141        }
142    }
143    None
144}
145
146/// Read current CPU frequency from /proc/cpuinfo (MHz).
147fn read_cpu_frequency() -> Option<f64> {
148    let content = std::fs::read_to_string("/proc/cpuinfo").ok()?;
149    // Take average across all cores
150    let mut total = 0.0;
151    let mut count = 0;
152    for line in content.lines() {
153        if line.starts_with("cpu MHz") {
154            if let Some(val) = line.split(':').nth(1) {
155                if let Ok(mhz) = val.trim().parse::<f64>() {
156                    total += mhz;
157                    count += 1;
158                }
159            }
160        }
161    }
162    if count > 0 {
163        Some(total / count as f64)
164    } else {
165        None
166    }
167}
168
169/// Read CPU temperature from /sys thermal zones.
170fn read_cpu_temperature() -> Option<f64> {
171    // Try thermal_zone0 first (usually CPU package)
172    for i in 0..10 {
173        let path = format!("/sys/class/thermal/thermal_zone{i}/temp");
174        if let Ok(content) = std::fs::read_to_string(&path) {
175            if let Ok(millidegrees) = content.trim().parse::<f64>() {
176                return Some(millidegrees / 1000.0);
177            }
178        }
179    }
180    None
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn test_parse_nvidia_val() {
189        assert!((parse_nvidia_val("285.32 W") - 285.32).abs() < 0.01);
190        assert!((parse_nvidia_val("24564 MiB") - 24564.0).abs() < 1.0);
191        assert!((parse_nvidia_val("62") - 62.0).abs() < 0.01);
192        assert!((parse_nvidia_val("[N/A]")).abs() < 0.01);
193        assert!((parse_nvidia_val("N/A")).abs() < 0.01);
194    }
195
196    #[test]
197    fn test_compute_energy() {
198        let e = compute_energy(300.0, 11.6, 23.2).unwrap();
199        assert!((e.tflops_per_watt - 11.6 / 300.0).abs() < 0.001);
200        assert!((e.joules_per_inference - 300.0 * 23.2e-6).abs() < 0.001);
201    }
202
203    #[test]
204    fn test_compute_energy_zero_power() {
205        assert!(compute_energy(0.0, 11.6, 23.2).is_none());
206    }
207
208    /// System health collection should not panic even without nvidia-smi.
209    #[test]
210    fn test_collect_system_health_no_panic() {
211        let _ = collect_system_health();
212    }
213
214    /// VRAM collection should not panic even without nvidia-smi.
215    #[test]
216    fn test_collect_vram_no_panic() {
217        let _ = collect_vram();
218    }
219
220    #[test]
221    fn test_read_cpu_frequency_no_panic() {
222        let _ = read_cpu_frequency();
223    }
224
225    /// On Linux `/proc/meminfo` always reports a positive `MemTotal`. This backs the
226    /// unified-memory VRAM fallback (GB10/GH200/Jetson report memory.total = N/A).
227    #[test]
228    fn test_read_system_memory_total_mb() {
229        let total = read_system_memory_total_mb();
230        assert!(total.is_some(), "/proc/meminfo MemTotal should be readable");
231        assert!(total.unwrap() > 0.0, "system memory total should be > 0 MB");
232    }
233
234    #[test]
235    fn test_read_cpu_temperature_no_panic() {
236        let _ = read_cpu_temperature();
237    }
238
239    /// If nvidia-smi is available, system health must have valid data.
240    #[test]
241    fn test_system_health_with_gpu() {
242        if which::which("nvidia-smi").is_err() {
243            return; // skip on machines without GPU
244        }
245        let health = collect_system_health();
246        assert!(health.is_some(), "nvidia-smi exists but no health data");
247        let h = health.unwrap();
248        assert!(h.gpu_temperature_celsius > 0.0, "GPU temp should be > 0");
249        assert!(
250            h.gpu_memory_total_mb > 0.0,
251            "GPU memory total should be > 0"
252        );
253    }
254
255    /// If nvidia-smi is available, VRAM must have valid data.
256    #[test]
257    fn test_vram_with_gpu() {
258        if which::which("nvidia-smi").is_err() {
259            return;
260        }
261        let vram = collect_vram();
262        assert!(vram.is_some(), "nvidia-smi exists but no VRAM data");
263        let v = vram.unwrap();
264        assert!(v.vram_total_mb > 0.0, "VRAM total should be > 0");
265        assert!(v.vram_utilization_pct >= 0.0 && v.vram_utilization_pct <= 100.0);
266    }
267}
cgp/profilers/system.rs

cgp/profilers/
system.rs
