use std::collections::HashMap;
use std::io::Write;
use crossterm::{queue, style::Color, style::Print};
use crate::device::GpuInfo;
use crate::device::MigGpuInfo;
use crate::device::VgpuHostInfo;
use crate::device::types::{NvLinkRemoteType, ThermalProximity, ThermalProximityConfig};
use crate::ui::renderers::utils::SUB_ITEM_INDENT;
use crate::ui::text::print_colored_text;
use crate::ui::widgets::draw_bar;
#[allow(dead_code)]
pub struct GpuRenderer;
impl Default for GpuRenderer {
fn default() -> Self {
Self::new()
}
}
#[allow(dead_code)]
impl GpuRenderer {
pub fn new() -> Self {
Self
}
}
#[cfg(test)]
pub fn gpu_render_line_count(
gpu: &GpuInfo,
vgpu_info: &[VgpuHostInfo],
mig_info: &[MigGpuInfo],
) -> usize {
let vgpu_lookup = build_vgpu_uuid_lookup(vgpu_info);
let mig_lookup = build_mig_uuid_lookup(mig_info);
gpu_render_line_count_with_lookup(gpu, vgpu_info, mig_info, &vgpu_lookup, &mig_lookup)
}
pub fn gpu_render_line_count_with_lookup(
gpu: &GpuInfo,
vgpu_info: &[VgpuHostInfo],
mig_info: &[MigGpuInfo],
vgpu_lookup: &HashMap<&str, usize>,
mig_lookup: &HashMap<&str, usize>,
) -> usize {
let mut lines: usize = 2;
if gpu_has_thermal_pstate_row(gpu) {
lines += 1;
}
if gpu_has_hardware_details_row(gpu) {
lines += 1;
}
let matched = vgpu_lookup
.get(gpu.uuid.as_str())
.map(|&idx| &vgpu_info[idx])
.or_else(|| {
vgpu_info
.iter()
.find(|v| v.hostname == gpu.hostname && v.gpu_name == gpu.name)
});
if let Some(host) = matched
&& host.is_vgpu_active()
{
lines += 1 + host.vgpus.len();
}
let mig_matched = mig_lookup
.get(gpu.uuid.as_str())
.map(|&idx| &mig_info[idx])
.or_else(|| {
mig_info
.iter()
.find(|m| m.hostname == gpu.hostname && m.gpu_name == gpu.name)
});
if let Some(host) = mig_matched
&& host.is_mig_active()
{
lines += 1 + host.instances.len();
}
lines
}
pub fn build_vgpu_uuid_lookup(vgpu_info: &[VgpuHostInfo]) -> HashMap<&str, usize> {
let mut map = HashMap::with_capacity(vgpu_info.len());
for (i, host) in vgpu_info.iter().enumerate() {
map.entry(host.gpu_uuid.as_str()).or_insert(i);
}
map
}
pub fn build_mig_uuid_lookup(mig_info: &[MigGpuInfo]) -> HashMap<&str, usize> {
let mut map = HashMap::with_capacity(mig_info.len());
for (i, host) in mig_info.iter().enumerate() {
map.entry(host.gpu_uuid.as_str()).or_insert(i);
}
map
}
pub(crate) fn format_hostname_with_scroll(hostname: &str, scroll_offset: usize) -> String {
if hostname.len() > 9 {
let scroll_len = hostname.len() + 3;
let start_pos = scroll_offset % scroll_len;
if hostname.is_ascii() {
let mut result = String::with_capacity(9);
let extended_len = hostname.len() * 2 + 3;
let mut idx = start_pos;
while result.len() < 9 && idx < extended_len {
let effective_idx = idx % extended_len;
let ch = if effective_idx < hostname.len() {
hostname.as_bytes()[effective_idx] as char
} else if effective_idx < hostname.len() + 3 {
' '
} else {
hostname.as_bytes()[effective_idx - hostname.len() - 3] as char
};
result.push(ch);
idx += 1;
}
result
} else {
let extended_hostname = format!("{hostname} {hostname}");
extended_hostname
.chars()
.skip(start_pos)
.take(9)
.collect::<String>()
}
} else {
format!("{hostname:<9}")
}
}
pub fn print_gpu_info<W: Write>(
stdout: &mut W,
_index: usize,
info: &GpuInfo,
width: usize,
device_name_scroll_offset: usize,
hostname_scroll_offset: usize,
show_hostname: bool,
) {
let device_name = if info.name.len() > 15 {
let scroll_len = info.name.len() + 3;
let start_pos = device_name_scroll_offset % scroll_len;
let extended_name = format!("{} {}", info.name, info.name);
extended_name
.chars()
.skip(start_pos)
.take(15)
.collect::<String>()
} else {
format!("{:<15}", info.name)
};
let memory_gb = info.used_memory as f64 / (1024.0 * 1024.0 * 1024.0);
let total_memory_gb = info.total_memory as f64 / (1024.0 * 1024.0 * 1024.0);
let memory_percent = if info.total_memory > 0 {
(info.used_memory as f64 / info.total_memory as f64) * 100.0
} else {
0.0
};
print_colored_text(
stdout,
&format!("{:<5}", info.device_type),
Color::Cyan,
None,
None,
);
print_colored_text(stdout, &device_name, Color::White, None, None);
if show_hostname {
let hostname_display = format_hostname_with_scroll(&info.hostname, hostname_scroll_offset);
print_colored_text(stdout, " @ ", Color::DarkGreen, None, None);
print_colored_text(stdout, &hostname_display, Color::White, None, None);
}
print_colored_text(stdout, " Util:", Color::Yellow, None, None);
let util_display = match info.utilization_reading() {
Some(util) => format!("{util:>5.1}%"),
None => format!("{:>6}", "N/A"),
};
print_colored_text(stdout, &util_display, Color::White, None, None);
print_colored_text(stdout, " VRAM:", Color::Blue, None, None);
let vram_display = if info.detail.get("metrics_available") == Some(&"false".to_string()) {
format!("{:>11}", "N/A")
} else {
let total_fmt = if total_memory_gb < 1.0 {
format!("{total_memory_gb:.1}")
} else {
format!("{total_memory_gb:.0}")
};
format!("{:>11}", format!("{memory_gb:.1}/{total_fmt}GB"))
};
print_colored_text(stdout, &vram_display, Color::White, None, None);
print_colored_text(stdout, " Temp:", Color::Magenta, None, None);
let (temp_display, temp_color) =
if info.detail.get("metrics_available") == Some(&"false".to_string()) {
(format!("{:>7}", "N/A"), Color::White)
} else if info.temperature_reading().is_none() {
(format!("{:>7}", "N/A"), Color::White)
} else {
let colour = match info.thermal_proximity(ThermalProximityConfig::default()) {
ThermalProximity::Shutdown => Color::Red,
ThermalProximity::Slowdown => Color::Yellow,
ThermalProximity::Normal => Color::White,
};
(format!("{:>4}°C", info.temperature), colour)
};
print_colored_text(stdout, &temp_display, temp_color, None, None);
print_colored_text(stdout, " Freq:", Color::Magenta, None, None);
match info.frequency_reading() {
None => print_colored_text(stdout, &format!("{:>7}", "N/A"), Color::White, None, None),
Some(mhz) if mhz >= 1000 => print_colored_text(
stdout,
&format!("{:.2}GHz", mhz as f64 / 1000.0),
Color::White,
None,
None,
),
Some(mhz) => print_colored_text(stdout, &format!("{mhz}MHz"), Color::White, None, None),
}
print_colored_text(stdout, " Pwr:", Color::Red, None, None);
let is_apple_silicon = info.name.contains("Apple") || info.name.contains("Metal");
let power_display = match info.power_consumption_reading() {
None => "N/A".to_string(),
Some(power) if is_apple_silicon => {
format!("{power:5.2}W")
}
Some(power) => match info
.detail
.get("power_limit_max")
.and_then(|s| s.parse::<f64>().ok())
{
Some(power_max) => format!("{power:.0}/{power_max:.0}W"),
None => format!("{power:.0}W"),
},
};
let display_width = power_display.len().max(8);
print_colored_text(
stdout,
&format!("{power_display:>display_width$}"),
Color::White,
None,
None,
);
if info.device_type == "TPU" {
let hlo_queue_size = info
.detail
.get("HLO Queue Size")
.map(|s| s.as_str())
.unwrap_or("0");
print_colored_text(stdout, " HLO Q:", Color::Cyan, None, None);
print_colored_text(
stdout,
&format!("{hlo_queue_size:>3}"),
Color::White,
None,
None,
);
}
if let Some(driver_version) = info.detail.get("Driver Version") {
print_colored_text(stdout, " Drv:", Color::Green, None, None);
print_colored_text(stdout, driver_version, Color::White, None, None);
}
if let Some(lib_name) = info.detail.get("lib_name") {
if let Some(lib_version) = info.detail.get("lib_version") {
print_colored_text(stdout, &format!(" {lib_name}:"), Color::Green, None, None);
print_colored_text(stdout, lib_version, Color::White, None, None);
}
} else {
if let Some(cuda_version) = info.detail.get("CUDA Version") {
print_colored_text(stdout, " CUDA:", Color::Green, None, None);
print_colored_text(stdout, cuda_version, Color::White, None, None);
} else if let Some(rocm_version) = info.detail.get("ROCm Version") {
print_colored_text(stdout, " ROCm:", Color::Green, None, None);
print_colored_text(stdout, rocm_version, Color::White, None, None);
}
}
queue!(stdout, Print("\r\n")).unwrap();
render_thermal_pstate_row(stdout, info);
render_hardware_details_row(stdout, info);
let available_width = width.saturating_sub(10); let is_apple_silicon = info.name.contains("Apple") || info.name.contains("Metal");
let has_tensorcore = info.device_type == "TPU" && info.tensorcore_utilization.is_some();
let num_gauges = if is_apple_silicon || has_tensorcore {
3
} else {
2
}; let gauge_width = (available_width - (num_gauges - 1) * 2) / num_gauges;
let total_gauge_width = gauge_width * num_gauges + (num_gauges - 1) * 2;
let left_padding = 5;
let right_padding = width - left_padding - total_gauge_width;
print_colored_text(stdout, " ", Color::White, None, None);
let (util_fill, util_label) = match info.utilization_reading() {
Some(util) => (util, format!("{util:.1}%")),
None => (0.0, "N/A".to_string()),
};
draw_bar(
stdout,
"Util",
util_fill,
100.0,
gauge_width,
Some(util_label),
);
print_colored_text(stdout, " ", Color::White, None, None);
draw_bar(
stdout,
"Mem",
memory_percent,
100.0,
gauge_width,
Some(format!("{memory_gb:.1}GB")),
);
if is_apple_silicon {
print_colored_text(stdout, " ", Color::White, None, None);
let is_ultra = info.name.contains("Ultra");
let max_ane_power = if is_ultra { 12.0 } else { 6.0 };
let (ane_percent, ane_label) = match info.ane_utilization_reading() {
Some(ane_mw) => {
let ane_power_w = (ane_mw / 1000.0).min(max_ane_power);
(
(ane_power_w / max_ane_power) * 100.0,
format!("{ane_power_w:.1}W"),
)
}
None => (0.0, "N/A".to_string()),
};
draw_bar(
stdout,
"ANE",
ane_percent,
100.0,
gauge_width,
Some(ane_label),
);
}
if has_tensorcore {
print_colored_text(stdout, " ", Color::White, None, None);
let tc_util = info.tensorcore_utilization.unwrap_or(0.0);
draw_bar(
stdout,
"TC",
tc_util,
100.0,
gauge_width,
Some(format!("{tc_util:.1}%")),
);
}
print_colored_text(stdout, &" ".repeat(right_padding), Color::White, None, None); queue!(stdout, Print("\r\n")).unwrap();
}
fn gpu_has_thermal_pstate_row(gpu: &GpuInfo) -> bool {
gpu.temperature_threshold_slowdown.is_some()
|| gpu.temperature_threshold_shutdown.is_some()
|| gpu.temperature_threshold_max_operating.is_some()
|| gpu.temperature_threshold_acoustic.is_some()
|| gpu.fan_speed_rpm.is_some()
|| gpu.performance_state.is_some()
}
fn render_thermal_pstate_row<W: Write>(stdout: &mut W, info: &GpuInfo) {
if !gpu_has_thermal_pstate_row(info) {
return;
}
print_colored_text(stdout, SUB_ITEM_INDENT, Color::White, None, None);
let proximity = info.thermal_proximity(ThermalProximityConfig::default());
let warn_color = match proximity {
ThermalProximity::Shutdown => Some(Color::Red),
ThermalProximity::Slowdown => Some(Color::Yellow),
ThermalProximity::Normal => None,
};
let mut emitted_any = false;
if let Some(slowdown) = info.temperature_threshold_slowdown {
emitted_any = true;
print_colored_text(stdout, "Slowdown:", Color::DarkYellow, None, None);
let color = warn_color.unwrap_or(Color::White);
print_colored_text(stdout, &format!("{slowdown}°C"), color, None, None);
}
if let Some(shutdown) = info.temperature_threshold_shutdown {
if emitted_any {
print_colored_text(stdout, " ", Color::White, None, None);
}
emitted_any = true;
print_colored_text(stdout, "Shutdown:", Color::DarkRed, None, None);
let color = match proximity {
ThermalProximity::Shutdown => Color::Red,
_ => Color::White,
};
print_colored_text(stdout, &format!("{shutdown}°C"), color, None, None);
}
if let Some(gpu_max) = info.temperature_threshold_max_operating {
if emitted_any {
print_colored_text(stdout, " ", Color::White, None, None);
}
emitted_any = true;
print_colored_text(stdout, "MaxOp:", Color::DarkGreen, None, None);
print_colored_text(stdout, &format!("{gpu_max}°C"), Color::White, None, None);
}
if let Some(acoustic) = info.temperature_threshold_acoustic {
if emitted_any {
print_colored_text(stdout, " ", Color::White, None, None);
}
emitted_any = true;
print_colored_text(stdout, "Acoustic:", Color::DarkCyan, None, None);
print_colored_text(stdout, &format!("{acoustic}°C"), Color::White, None, None);
}
if let Some(rpm) = info.fan_speed_rpm {
if emitted_any {
print_colored_text(stdout, " ", Color::White, None, None);
}
emitted_any = true;
print_colored_text(stdout, "Fan:", Color::DarkMagenta, None, None);
print_colored_text(stdout, &format!("{rpm}rpm"), Color::White, None, None);
}
if let Some(pstate) = info.performance_state {
if emitted_any {
print_colored_text(stdout, " ", Color::White, None, None);
}
print_colored_text(stdout, "P-State:", Color::DarkBlue, None, None);
let color = match pstate {
0 => Color::Green,
15 => Color::DarkGrey,
_ => Color::White,
};
print_colored_text(stdout, &format!("P{pstate}"), color, None, None);
}
queue!(stdout, Print("\r\n")).unwrap();
}
fn gpu_has_hardware_details_row(gpu: &GpuInfo) -> bool {
gpu.numa_node_id.is_some()
|| gpu.gsp_firmware_mode.is_some()
|| gpu.gsp_firmware_version.is_some()
|| !gpu.nvlink_remote_devices.is_empty()
}
fn gsp_firmware_mode_label(code: u8) -> &'static str {
match code {
0 => "disabled",
1 => "enabled",
2 => "default",
_ => "unknown",
}
}
fn render_hardware_details_row<W: Write>(stdout: &mut W, info: &GpuInfo) {
if !gpu_has_hardware_details_row(info) {
return;
}
print_colored_text(stdout, SUB_ITEM_INDENT, Color::White, None, None);
print_colored_text(stdout, "HW", Color::DarkMagenta, None, None);
if let Some(numa) = info.numa_node_id {
print_colored_text(stdout, " NUMA:", Color::DarkYellow, None, None);
print_colored_text(stdout, &format!("{numa}"), Color::White, None, None);
}
if let Some(mode_code) = info.gsp_firmware_mode {
print_colored_text(stdout, " GSP:", Color::DarkGreen, None, None);
print_colored_text(
stdout,
gsp_firmware_mode_label(mode_code),
Color::White,
None,
None,
);
}
if let Some(ref version) = info.gsp_firmware_version {
print_colored_text(stdout, " v", Color::DarkGrey, None, None);
print_colored_text(stdout, version, Color::White, None, None);
}
if !info.nvlink_remote_devices.is_empty() {
let total = info.nvlink_remote_devices.len();
let (gpu_count, switch_count, ibmnpu_count, unknown_count) =
count_nvlink_remote_types(&info.nvlink_remote_devices);
print_colored_text(stdout, " NVLink:", Color::DarkCyan, None, None);
let mut parts: Vec<String> = Vec::with_capacity(4);
if gpu_count > 0 {
parts.push(format!("gpu={gpu_count}"));
}
if switch_count > 0 {
parts.push(format!("sw={switch_count}"));
}
if ibmnpu_count > 0 {
parts.push(format!("ibmnpu={ibmnpu_count}"));
}
if unknown_count > 0 {
parts.push(format!("?={unknown_count}"));
}
let summary = if parts.is_empty() {
format!("{total}x")
} else {
format!("{total}x({})", parts.join(","))
};
print_colored_text(stdout, &summary, Color::White, None, None);
}
if let Some(ref gpm) = info.gpm_metrics
&& (gpm.sm_occupancy.is_some() || gpm.memory_bandwidth_utilization.is_some())
{
print_colored_text(stdout, " GPM:", Color::DarkBlue, None, None);
if let Some(sm) = gpm.sm_occupancy {
print_colored_text(stdout, "SM=", Color::DarkGrey, None, None);
print_colored_text(stdout, &format!("{sm:.2}"), Color::White, None, None);
}
if let Some(mem) = gpm.memory_bandwidth_utilization {
if gpm.sm_occupancy.is_some() {
print_colored_text(stdout, " ", Color::White, None, None);
}
print_colored_text(stdout, "MemBW=", Color::DarkGrey, None, None);
print_colored_text(stdout, &format!("{mem:.2}"), Color::White, None, None);
}
}
queue!(stdout, Print("\r\n")).unwrap();
}
fn count_nvlink_remote_types(
links: &[crate::device::NvLinkRemoteDevice],
) -> (usize, usize, usize, usize) {
let mut gpu_count = 0;
let mut switch_count = 0;
let mut ibmnpu_count = 0;
let mut unknown_count = 0;
for link in links {
match link.remote_type {
NvLinkRemoteType::Gpu => gpu_count += 1,
NvLinkRemoteType::Switch => switch_count += 1,
NvLinkRemoteType::IbmNpu => ibmnpu_count += 1,
NvLinkRemoteType::Unknown => unknown_count += 1,
}
}
(gpu_count, switch_count, ibmnpu_count, unknown_count)
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
fn make_gpu(temp: u32) -> GpuInfo {
GpuInfo {
uuid: "gpu-0".to_string(),
time: String::new(),
name: "Test GPU".to_string(),
device_type: "GPU".to_string(),
host_id: "h".to_string(),
hostname: "h".to_string(),
instance: "h".to_string(),
utilization: 0.0,
ane_utilization: 0.0,
dla_utilization: None,
tensorcore_utilization: None,
temperature: temp,
used_memory: 0,
total_memory: 0,
frequency: 0,
power_consumption: 0.0,
gpu_core_count: None,
temperature_threshold_slowdown: Some(93),
temperature_threshold_shutdown: Some(98),
temperature_threshold_max_operating: Some(87),
temperature_threshold_acoustic: None,
performance_state: Some(2),
fan_speed_rpm: None,
numa_node_id: None,
gsp_firmware_mode: None,
gsp_firmware_version: None,
nvlink_remote_devices: Vec::new(),
gpm_metrics: None,
detail: HashMap::new(),
}
}
#[test]
fn test_format_hostname_with_scroll() {
assert_eq!(format_hostname_with_scroll("host", 0), "host ");
assert_eq!(format_hostname_with_scroll("host", 5), "host ");
assert_eq!(format_hostname_with_scroll("localhost", 0), "localhost");
let long_hostname = "very-long-hostname";
assert_eq!(format_hostname_with_scroll(long_hostname, 0).len(), 9);
assert_eq!(format_hostname_with_scroll(long_hostname, 0), "very-long");
assert_eq!(format_hostname_with_scroll(long_hostname, 5), "long-host");
assert_eq!(format_hostname_with_scroll(long_hostname, 10), "hostname ");
let scroll_len = long_hostname.len() + 3;
assert_eq!(
format_hostname_with_scroll(long_hostname, scroll_len),
format_hostname_with_scroll(long_hostname, 0)
);
}
#[test]
fn test_gpu_renderer_new() {
let renderer = GpuRenderer::new();
let _ = renderer;
}
fn render_row(info: &GpuInfo) -> String {
let mut buf: Vec<u8> = Vec::new();
print_gpu_info(&mut buf, 0, info, 120, 0, 0, false);
let raw = String::from_utf8_lossy(&buf).into_owned();
let mut out = String::with_capacity(raw.len());
let mut chars = raw.chars().peekable();
while let Some(c) = chars.next() {
if c == '\u{1b}' {
if chars.peek() == Some(&'[') {
chars.next();
for c in chars.by_ref() {
if ('@'..='~').contains(&c) {
break;
}
}
}
continue;
}
out.push(c);
}
out
}
#[test]
fn absent_readings_render_as_na_not_zero() {
let mut gpu = make_gpu(0);
gpu.name = "Apple M2 Max GPU".to_string();
gpu.utilization = crate::device::types::GPU_METRIC_UNAVAILABLE;
gpu.power_consumption = crate::device::types::GPU_METRIC_UNAVAILABLE;
gpu.ane_utilization = crate::device::types::GPU_METRIC_UNAVAILABLE;
gpu.frequency = 0;
let rendered = render_row(&gpu);
assert!(rendered.contains("Util: N/A"), "{rendered}");
assert!(rendered.contains("Temp: N/A"), "{rendered}");
assert!(rendered.contains("Freq: N/A"), "{rendered}");
assert!(rendered.contains("Pwr: N/A"), "{rendered}");
assert!(
!rendered.contains("0.0%"),
"util gauge shows 0%: {rendered}"
);
assert!(!rendered.contains("0.0W"), "ANE gauge shows 0W: {rendered}");
}
#[test]
fn unsourced_windows_fields_render_as_na_not_zero() {
let mut gpu = make_gpu(45);
gpu.name = "AMD Radeon Graphics".to_string();
gpu.utilization = crate::device::types::GPU_METRIC_UNAVAILABLE;
gpu.power_consumption = crate::device::types::GPU_METRIC_UNAVAILABLE;
gpu.frequency = 338;
let rendered = render_row(&gpu);
assert!(rendered.contains("Util: N/A"), "{rendered}");
assert!(rendered.contains("Pwr: N/A"), "{rendered}");
assert!(!rendered.contains("Util: 0.0%"), "{rendered}");
assert!(!rendered.contains("Pwr: 0W"), "{rendered}");
}
#[test]
fn zero_readings_render_as_zero() {
let mut gpu = make_gpu(45);
gpu.name = "Apple M2 Max GPU".to_string();
gpu.utilization = 0.0;
gpu.power_consumption = 0.0;
gpu.ane_utilization = 0.0;
gpu.frequency = 338;
let rendered = render_row(&gpu);
assert!(rendered.contains("Util: 0.0%"), "{rendered}");
assert!(rendered.contains("45°C"), "{rendered}");
assert!(rendered.contains("338MHz"), "{rendered}");
assert!(rendered.contains("0.00W"), "{rendered}");
assert!(
!rendered.contains("N/A"),
"nothing should be N/A: {rendered}"
);
}
#[test]
fn thermal_proximity_normal_when_far_from_thresholds() {
let gpu = make_gpu(60);
assert_eq!(
gpu.thermal_proximity(ThermalProximityConfig::default()),
ThermalProximity::Normal
);
}
#[test]
fn thermal_proximity_slowdown_within_margin() {
let gpu = make_gpu(89);
assert_eq!(
gpu.thermal_proximity(ThermalProximityConfig::default()),
ThermalProximity::Slowdown
);
}
#[test]
fn thermal_proximity_shutdown_takes_priority_over_slowdown() {
let gpu = make_gpu(97);
assert_eq!(
gpu.thermal_proximity(ThermalProximityConfig::default()),
ThermalProximity::Shutdown
);
}
#[test]
fn thermal_proximity_zero_thresholds_are_ignored() {
let mut gpu = make_gpu(10);
gpu.temperature_threshold_slowdown = Some(0);
gpu.temperature_threshold_shutdown = Some(0);
assert_eq!(
gpu.thermal_proximity(ThermalProximityConfig::default()),
ThermalProximity::Normal
);
}
#[test]
fn thermal_proximity_none_thresholds_are_normal() {
let mut gpu = make_gpu(95);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
assert_eq!(
gpu.thermal_proximity(ThermalProximityConfig::default()),
ThermalProximity::Normal
);
}
#[test]
fn thermal_proximity_respects_custom_margins() {
let gpu = make_gpu(83);
assert_eq!(
gpu.thermal_proximity(ThermalProximityConfig {
slowdown_margin: 10,
shutdown_margin: 2,
}),
ThermalProximity::Slowdown
);
}
#[test]
fn thermal_proximity_saturates_on_extreme_values() {
let mut gpu = make_gpu(u32::MAX);
gpu.temperature_threshold_slowdown = Some(50);
gpu.temperature_threshold_shutdown = Some(50);
let cfg = ThermalProximityConfig {
slowdown_margin: u32::MAX,
shutdown_margin: u32::MAX,
};
assert_eq!(gpu.thermal_proximity(cfg), ThermalProximity::Shutdown);
}
#[test]
fn render_thermal_pstate_row_is_noop_when_no_data() {
let mut gpu = make_gpu(50);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
assert!(
buf.is_empty(),
"expected no output when nothing is reported"
);
}
#[test]
fn render_thermal_pstate_row_emits_labels_when_data_present() {
let gpu = make_gpu(50);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(rendered.contains("Slowdown:"), "{rendered}");
assert!(rendered.contains("Shutdown:"), "{rendered}");
assert!(rendered.contains("MaxOp:"), "{rendered}");
assert!(rendered.contains("P-State:"), "{rendered}");
assert!(rendered.contains("93°C"), "{rendered}");
assert!(rendered.contains("98°C"), "{rendered}");
assert!(rendered.contains("87°C"), "{rendered}");
assert!(rendered.contains("P2"), "{rendered}");
}
#[test]
fn render_thermal_pstate_row_emits_pstate_only_when_only_pstate_present() {
let mut gpu = make_gpu(50);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = Some(8);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(rendered.contains("P-State:"), "{rendered}");
assert!(rendered.contains("P8"), "{rendered}");
assert!(
!rendered.contains("Slowdown:"),
"should not render Slowdown without data: {rendered}"
);
}
#[test]
fn render_pstate_only_has_no_double_leading_space() {
let mut gpu = make_gpu(50);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = Some(3);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let raw = String::from_utf8(buf).expect("valid utf-8");
let plain: String = {
let mut out = String::new();
let mut chars = raw.chars().peekable();
while let Some(c) = chars.next() {
if c == '\x1b' {
for ch in chars.by_ref() {
if ch == 'm' {
break;
}
}
} else {
out.push(c);
}
}
out
};
assert!(plain.contains("P-State:"), "missing P-State: in {plain:?}");
assert!(plain.contains("P3"), "missing P3 in {plain:?}");
let after_indent = plain.trim_start_matches(' ');
assert!(
!after_indent.starts_with(' '),
"double leading space detected in {plain:?}"
);
}
#[test]
fn render_thermal_pstate_row_includes_fan_speed_when_present() {
let mut gpu = make_gpu(50);
gpu.fan_speed_rpm = Some(1450);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(rendered.contains("Fan:"), "{rendered}");
assert!(rendered.contains("1450rpm"), "{rendered}");
}
#[test]
fn render_thermal_pstate_row_omits_fan_speed_when_absent() {
let mut gpu = make_gpu(50);
gpu.fan_speed_rpm = None;
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(!rendered.contains("Fan:"), "{rendered}");
assert!(!rendered.contains("rpm"), "{rendered}");
}
#[test]
fn render_fan_only_row_has_no_double_leading_space() {
let mut gpu = make_gpu(50);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.fan_speed_rpm = Some(1450);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let raw = String::from_utf8(buf).expect("valid utf-8");
let plain = strip_ansi(&raw);
assert!(plain.contains("Fan:1450rpm"), "{plain:?}");
assert!(!plain.contains("P-State:"), "{plain:?}");
let after_indent = plain.trim_start_matches(' ');
assert!(
!after_indent.starts_with(' '),
"double leading space detected in {plain:?}"
);
}
#[test]
fn render_thermal_pstate_row_separates_fan_from_its_neighbours() {
let mut gpu = make_gpu(50);
gpu.temperature_threshold_acoustic = Some(75);
gpu.fan_speed_rpm = Some(1450);
gpu.performance_state = Some(2);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let plain = strip_ansi(&String::from_utf8(buf).expect("valid utf-8"));
assert!(plain.contains("75°C Fan:1450rpm P-State:P2"), "{plain:?}");
}
#[test]
fn render_thermal_pstate_row_includes_acoustic_when_present() {
let mut gpu = make_gpu(50);
gpu.temperature_threshold_acoustic = Some(75);
let mut buf: Vec<u8> = Vec::new();
render_thermal_pstate_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(rendered.contains("Acoustic:"), "{rendered}");
assert!(rendered.contains("75°C"), "{rendered}");
}
fn make_vgpu_host(gpu_uuid: &str, instances: usize) -> VgpuHostInfo {
use crate::device::types::VgpuInfo;
let vgpus = (0..instances)
.map(|i| VgpuInfo {
instance_id: i as u32,
uuid: format!("vgpu-{i}"),
vm_id: String::new(),
vgpu_type_name: "GRID".into(),
fb_used_bytes: 0,
fb_total_bytes: 1 << 30,
gpu_utilization: Some(0),
memory_utilization: Some(0),
is_active: true,
})
.collect();
VgpuHostInfo {
host_id: "h".to_string(),
hostname: "h".to_string(),
instance: "h".to_string(),
gpu_index: 0,
gpu_uuid: gpu_uuid.to_string(),
gpu_name: "Test GPU".to_string(),
host_mode: "Sriov".to_string(),
scheduler_policy: 1,
scheduler_arr_mode: 2,
is_arr_supported: true,
vgpus,
detail: HashMap::new(),
}
}
#[test]
fn line_count_is_two_for_minimal_non_nvidia_gpu() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
assert_eq!(gpu_render_line_count(&gpu, &[], &[]), 2);
}
#[test]
fn line_count_grows_to_three_when_thermal_or_pstate_present() {
for setter in [
|g: &mut GpuInfo| g.temperature_threshold_slowdown = Some(93),
|g: &mut GpuInfo| g.temperature_threshold_shutdown = Some(98),
|g: &mut GpuInfo| g.temperature_threshold_max_operating = Some(87),
|g: &mut GpuInfo| g.temperature_threshold_acoustic = Some(75),
|g: &mut GpuInfo| g.fan_speed_rpm = Some(1450),
|g: &mut GpuInfo| g.performance_state = Some(2),
] {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
setter(&mut gpu);
assert_eq!(
gpu_render_line_count(&gpu, &[], &[]),
3,
"expected 3 lines for {gpu:?}"
);
}
}
#[test]
fn line_count_includes_vgpu_section_when_uuid_matches() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
let host = make_vgpu_host("GPU-A", 3);
assert_eq!(
gpu_render_line_count(&gpu, std::slice::from_ref(&host), &[]),
6
);
}
#[test]
fn line_count_falls_back_to_hostname_and_name_when_uuid_does_not_match() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
gpu.hostname = "h".to_string();
gpu.name = "Test GPU".to_string();
let host = make_vgpu_host("GPU-OTHER", 2);
assert_eq!(
gpu_render_line_count(&gpu, std::slice::from_ref(&host), &[]),
5
);
}
#[test]
fn line_count_ignores_disabled_vgpu_host_with_no_instances() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
let mut host = make_vgpu_host("GPU-A", 0);
host.host_mode = "Disabled".to_string();
assert_eq!(
gpu_render_line_count(&gpu, std::slice::from_ref(&host), &[]),
2
);
}
#[test]
fn line_count_combines_thermal_and_vgpu_contributions() {
let mut gpu = make_gpu(40);
gpu.uuid = "GPU-A".to_string();
let host = make_vgpu_host("GPU-A", 2);
assert_eq!(
gpu_render_line_count(&gpu, std::slice::from_ref(&host), &[]),
6
);
}
fn make_mig_host(gpu_uuid: &str, instances: usize) -> MigGpuInfo {
use crate::device::types::MigInstanceInfo;
let entries = (0..instances)
.map(|i| MigInstanceInfo {
instance_id: i as u32,
gpu_instance_id: Some((i as u32) + 1),
compute_instance_id: Some(0),
uuid: format!("MIG-{i}"),
profile_name: "1g.5gb".into(),
utilization_gpu: Some(0),
utilization_memory: Some(0),
memory_used_bytes: 0,
memory_total_bytes: 5 << 30,
})
.collect();
MigGpuInfo {
host_id: "h".to_string(),
hostname: "h".to_string(),
instance: "h".to_string(),
gpu_index: 0,
gpu_uuid: gpu_uuid.to_string(),
gpu_name: "Test GPU".to_string(),
mig_mode: true,
instances: entries,
}
}
#[test]
fn line_count_includes_mig_section_when_uuid_matches() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
let mig_host = make_mig_host("GPU-A", 4);
assert_eq!(
gpu_render_line_count(&gpu, &[], std::slice::from_ref(&mig_host)),
7
);
}
#[test]
fn line_count_falls_back_to_hostname_for_mig_when_uuid_missing() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
gpu.hostname = "h".to_string();
gpu.name = "Test GPU".to_string();
let mig_host = make_mig_host("GPU-OTHER", 2);
assert_eq!(
gpu_render_line_count(&gpu, &[], std::slice::from_ref(&mig_host)),
5
);
}
#[test]
fn line_count_combines_vgpu_and_mig_contributions() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
let vgpu_host = make_vgpu_host("GPU-A", 2);
let mig_host = make_mig_host("GPU-A", 3);
assert_eq!(
gpu_render_line_count(
&gpu,
std::slice::from_ref(&vgpu_host),
std::slice::from_ref(&mig_host),
),
9
);
}
#[test]
fn line_count_ignores_mig_host_with_no_instances_and_disabled_mode() {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu.uuid = "GPU-A".to_string();
let mut mig_host = make_mig_host("GPU-A", 0);
mig_host.mig_mode = false;
assert_eq!(
gpu_render_line_count(&gpu, &[], std::slice::from_ref(&mig_host)),
2
);
}
fn bare_gpu() -> GpuInfo {
let mut gpu = make_gpu(40);
gpu.temperature_threshold_slowdown = None;
gpu.temperature_threshold_shutdown = None;
gpu.temperature_threshold_max_operating = None;
gpu.temperature_threshold_acoustic = None;
gpu.performance_state = None;
gpu
}
#[test]
fn hw_row_noop_when_no_hardware_details() {
let gpu = bare_gpu();
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
assert!(
buf.is_empty(),
"expected no output when no hw details populated"
);
}
#[test]
fn hw_row_renders_numa_when_populated() {
let mut gpu = bare_gpu();
gpu.numa_node_id = Some(1);
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(rendered.contains("HW"), "{rendered}");
assert!(rendered.contains("NUMA:"), "{rendered}");
assert!(rendered.contains('1'), "{rendered}");
}
#[test]
fn hw_row_renders_gsp_mode_label() {
for (code, label) in [(0u8, "disabled"), (1, "enabled"), (2, "default")] {
let mut gpu = bare_gpu();
gpu.gsp_firmware_mode = Some(code);
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(
rendered.contains(label),
"expected '{label}' in: {rendered}"
);
}
}
fn strip_ansi(raw: &str) -> String {
let mut out = String::with_capacity(raw.len());
let mut chars = raw.chars().peekable();
while let Some(c) = chars.next() {
if c == '\x1b' {
for ch in chars.by_ref() {
if ch.is_ascii_alphabetic() {
break;
}
}
} else {
out.push(c);
}
}
out
}
#[test]
fn hw_row_renders_gsp_version_prefix() {
let mut gpu = bare_gpu();
gpu.gsp_firmware_version = Some("550.54.15".to_string());
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
let rendered = strip_ansi(&String::from_utf8(buf).expect("valid utf-8"));
assert!(rendered.contains("v550.54.15"), "{rendered}");
}
#[test]
fn hw_row_renders_nvlink_summary_with_type_counts() {
use crate::device::{NvLinkRemoteDevice, NvLinkRemoteType};
let mut gpu = bare_gpu();
gpu.nvlink_remote_devices = vec![
NvLinkRemoteDevice {
link_index: 0,
remote_type: NvLinkRemoteType::Gpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 1,
remote_type: NvLinkRemoteType::Gpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 2,
remote_type: NvLinkRemoteType::Gpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 3,
remote_type: NvLinkRemoteType::Gpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 4,
remote_type: NvLinkRemoteType::Gpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 5,
remote_type: NvLinkRemoteType::Switch,
bandwidth_mb_s: None,
},
];
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(rendered.contains("NVLink:"), "{rendered}");
assert!(rendered.contains("6x"), "{rendered}");
assert!(rendered.contains("gpu=5"), "{rendered}");
assert!(rendered.contains("sw=1"), "{rendered}");
}
#[test]
fn hw_row_omits_gpm_when_only_support_probe_populated() {
use crate::device::GpmMetrics;
let mut gpu = bare_gpu();
gpu.numa_node_id = Some(0);
gpu.gpm_metrics = Some(GpmMetrics::default());
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
let rendered = String::from_utf8(buf).expect("valid utf-8");
assert!(
!rendered.contains("GPM:"),
"GPM label should be hidden when no values sampled: {rendered}"
);
}
#[test]
fn hw_row_renders_gpm_values_when_populated() {
use crate::device::GpmMetrics;
let mut gpu = bare_gpu();
gpu.numa_node_id = Some(0);
gpu.gpm_metrics = Some(GpmMetrics {
sm_occupancy: Some(0.67),
memory_bandwidth_utilization: Some(0.42),
});
let mut buf: Vec<u8> = Vec::new();
render_hardware_details_row(&mut buf, &gpu);
let rendered = strip_ansi(&String::from_utf8(buf).expect("valid utf-8"));
assert!(rendered.contains("GPM:"), "{rendered}");
assert!(rendered.contains("SM=0.67"), "{rendered}");
assert!(rendered.contains("MemBW=0.42"), "{rendered}");
}
#[test]
fn hw_row_accounts_for_line_in_gpu_render_line_count() {
let mut gpu = bare_gpu();
gpu.numa_node_id = Some(0);
assert_eq!(gpu_render_line_count(&gpu, &[], &[]), 3);
}
#[test]
fn hw_row_and_thermal_row_both_counted() {
let gpu = make_gpu(50); let mut gpu = gpu;
gpu.numa_node_id = Some(0);
assert_eq!(gpu_render_line_count(&gpu, &[], &[]), 4);
}
#[test]
fn hw_row_gpm_only_does_not_emit_row() {
use crate::device::GpmMetrics;
let mut gpu = bare_gpu();
gpu.gpm_metrics = Some(GpmMetrics::default());
assert_eq!(gpu_render_line_count(&gpu, &[], &[]), 2);
}
#[test]
fn count_nvlink_remote_types_classifies_all_variants() {
use crate::device::{NvLinkRemoteDevice, NvLinkRemoteType};
let links = vec![
NvLinkRemoteDevice {
link_index: 0,
remote_type: NvLinkRemoteType::Gpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 1,
remote_type: NvLinkRemoteType::Switch,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 2,
remote_type: NvLinkRemoteType::IbmNpu,
bandwidth_mb_s: None,
},
NvLinkRemoteDevice {
link_index: 3,
remote_type: NvLinkRemoteType::Unknown,
bandwidth_mb_s: None,
},
];
assert_eq!(count_nvlink_remote_types(&links), (1, 1, 1, 1));
}
}