all-smi 0.26.2

// Copyright 2025 Lablup Inc. and Jeongkyu Shin
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use super::{MetricBuilder, MetricExporter};
use crate::device::GpuInfo;
use crate::device::types::MAX_GPU_FAN_RPM;
use crate::parsing::common::{sanitize_label_name, sanitize_label_value};

/// Legacy `detail` key every reader used before `GpuInfo::fan_speed_rpm`
/// existed, and still writes alongside it. `sanitize_label_name` turns the
/// key into the `fan_speed` label on `all_smi_gpu_info`, which is how a node
/// running an older build puts the reading on the wire.
pub(crate) const FAN_SPEED_DETAIL_KEY: &str = "Fan Speed";

/// Recover an RPM reading from the legacy `Fan Speed` detail string.
///
/// Shared with `network::metrics_parser` so the exporter and the remote
/// parser cannot disagree about the format, the same way
/// `ProcessMetricExporter::parse_start_time_seconds_public` is shared with
/// `ParsedProcessRow::from_local_process`.
///
/// Readers spell the value `"1450 RPM"`, and the Level Zero reader appends a
/// duty cycle when it has one (`"1600 RPM (40%)"`), so the number is parsed
/// from the text before the ` RPM` marker rather than by stripping a suffix.
/// A duty-cycle-only value (`"40%"`) carries no tachometer reading and
/// returns `None`, as does anything non-numeric or negative.
///
/// The result is also bounded by [`MAX_GPU_FAN_RPM`] and rejected when
/// fractional, the same checks `network::metrics_parser` applies to the
/// structured `all_smi_gpu_fan_speed_rpm` metric. Enforcing them here,
/// rather than leaving each caller to repeat them, is what keeps this
/// function's two call sites (the exporter's own detail fallback below and
/// the parser's legacy-node recovery) from disagreeing about what counts as
/// a valid reading: a fractional or out-of-range detail string (a garbled
/// reader value, or a hostile snapshot) can no longer be exported here only
/// for every downstream parser to silently drop it.
pub(crate) fn parse_fan_speed_detail(value: &str) -> Option<f64> {
    let (rpm, _) = value.split_once(" RPM")?;
    let rpm = rpm.trim().parse::<f64>().ok()?;
    (rpm.is_finite() && rpm.fract() == 0.0 && (0.0..=f64::from(MAX_GPU_FAN_RPM)).contains(&rpm))
        .then_some(rpm)
}

pub struct GpuMetricExporter<'a> {
    pub gpu_info: &'a [GpuInfo],
}

impl<'a> GpuMetricExporter<'a> {
    pub fn new(gpu_info: &'a [GpuInfo]) -> Self {
        Self { gpu_info }
    }

    fn export_basic_metrics(&self, builder: &mut MetricBuilder, row: &GpuRow<'a>) {
        let info = row.gpu;
        let base_labels = [
            ("gpu", info.name.as_str()),
            ("instance", info.instance.as_str()),
            ("gpu_uuid", info.uuid.as_str()),
            ("gpu_index", row.index_str.as_str()),
        ];

        // GPU utilization.
        //
        // Omitted when the reader had no source for it, following the same
        // Prometheus "absence means no data" convention this exporter already
        // applies to `all_smi_gpu_performance_state` and the thermal
        // thresholds below. Emitting `0` instead would be indistinguishable
        // from a genuinely idle GPU (issue #325); `all_smi_gpu_info` is still
        // emitted for the device, so a consumer can tell "device present but
        // not reporting" from "device gone" and, on Apple Silicon, read the
        // reason off the `native_metrics` label.
        if let Some(utilization) = info.utilization_reading() {
            builder
                .help(
                    "all_smi_gpu_utilization",
                    "GPU utilization percentage (omitted when the device reports no utilization)",
                )
                .type_("all_smi_gpu_utilization", "gauge")
                .metric("all_smi_gpu_utilization", &base_labels, utilization);
        }

        // Memory metrics
        builder
            .help("all_smi_gpu_memory_used_bytes", "GPU memory used in bytes")
            .type_("all_smi_gpu_memory_used_bytes", "gauge")
            .metric(
                "all_smi_gpu_memory_used_bytes",
                &base_labels,
                info.used_memory,
            );

        builder
            .help(
                "all_smi_gpu_memory_total_bytes",
                "GPU memory total in bytes",
            )
            .type_("all_smi_gpu_memory_total_bytes", "gauge")
            .metric(
                "all_smi_gpu_memory_total_bytes",
                &base_labels,
                info.total_memory,
            );

        // Temperature. Omitted when no sensor answered — a powered die never
        // reads 0 °C, so the old unconditional `0` made a missing SMC/NVML
        // key look like a cryogenic GPU.
        if let Some(temperature) = info.temperature_reading() {
            builder
                .help(
                    "all_smi_gpu_temperature_celsius",
                    "GPU temperature in celsius (omitted when no sensor reports one)",
                )
                .type_("all_smi_gpu_temperature_celsius", "gauge")
                .metric("all_smi_gpu_temperature_celsius", &base_labels, temperature);
        }

        // Power consumption. Omitted when the device exposes no power rail.
        if let Some(power) = info.power_consumption_reading() {
            builder
                .help(
                    "all_smi_gpu_power_consumption_watts",
                    "GPU power consumption in watts (omitted when the device reports no power)",
                )
                .type_("all_smi_gpu_power_consumption_watts", "gauge")
                .metric("all_smi_gpu_power_consumption_watts", &base_labels, power);
        }

        // Frequency. Omitted when the platform has no clock probe. This also
        // covers the readers that have always reported a static `0` to mean
        // "no probe" (Rebellions, Intel Gaudi, AMD via WMI), which the TUI
        // already rendered as N/A; the exposition now agrees with it instead
        // of publishing a flat 0 MHz line.
        if let Some(frequency) = info.frequency_reading() {
            builder
                .help(
                    "all_smi_gpu_frequency_mhz",
                    "GPU frequency in MHz (omitted when the device exposes no clock probe)",
                )
                .type_("all_smi_gpu_frequency_mhz", "gauge")
                .metric("all_smi_gpu_frequency_mhz", &base_labels, frequency);
        }

        // ANE utilization (Apple Silicon). Non-Apple readers set this to a
        // literal 0.0 meaning "not applicable" and keep emitting it, so this
        // gate only fires for an Apple Silicon row with no live sample.
        if let Some(ane) = info.ane_utilization_reading() {
            builder
                .help(
                    "all_smi_ane_utilization",
                    "ANE utilization in mW (omitted when the native metrics source is unavailable)",
                )
                .type_("all_smi_ane_utilization", "gauge")
                .metric("all_smi_ane_utilization", &base_labels, ane);
        }

        // DLA utilization (if available)
        if let Some(dla_util) = info.dla_utilization {
            builder
                .help("all_smi_dla_utilization", "DLA utilization percentage")
                .type_("all_smi_dla_utilization", "gauge")
                .metric("all_smi_dla_utilization", &base_labels, dla_util);
        }
    }

    fn export_apple_silicon_metrics(&self, builder: &mut MetricBuilder, row: &GpuRow<'a>) {
        let info = row.gpu;
        if !info.name.contains("Apple") && !info.name.contains("Metal") {
            return;
        }

        let base_labels = [
            ("gpu", info.name.as_str()),
            ("instance", info.instance.as_str()),
            ("gpu_uuid", info.uuid.as_str()),
            ("gpu_index", row.index_str.as_str()),
        ];

        // ANE power in watts. Same source as `all_smi_ane_utilization`, so
        // the two families appear and disappear together.
        if let Some(ane_mw) = info.ane_utilization_reading() {
            builder
                .help(
                    "all_smi_ane_power_watts",
                    "ANE power consumption in watts (omitted when the native metrics source is unavailable)",
                )
                .type_("all_smi_ane_power_watts", "gauge")
                .metric("all_smi_ane_power_watts", &base_labels, ane_mw / 1000.0);
        }

        // Thermal pressure level
        if let Some(thermal_level) = info.detail.get("thermal_pressure") {
            let thermal_labels = [
                ("gpu", info.name.as_str()),
                ("instance", info.instance.as_str()),
                ("gpu_uuid", info.uuid.as_str()),
                ("gpu_index", row.index_str.as_str()),
                ("level", thermal_level.as_str()),
            ];
            builder
                .help("all_smi_thermal_pressure_info", "Thermal pressure level")
                .type_("all_smi_thermal_pressure_info", "gauge")
                .metric("all_smi_thermal_pressure_info", &thermal_labels, 1);
        }

        // Combined power (CPU + GPU + ANE) for Apple Silicon
        if let Some(combined_power_str) = info.detail.get("combined_power_mw")
            && let Ok(combined_power_mw) = combined_power_str.parse::<f64>()
        {
            let combined_power_watts = combined_power_mw / 1000.0;
            builder
                .help(
                    "all_smi_combined_power_watts",
                    "Combined power consumption (CPU + GPU + ANE) in watts",
                )
                .type_("all_smi_combined_power_watts", "gauge")
                .metric(
                    "all_smi_combined_power_watts",
                    &base_labels,
                    combined_power_watts,
                );
        }
    }

    fn export_device_info(&self, builder: &mut MetricBuilder, row: &GpuRow<'a>) {
        let info = row.gpu;

        // Build label string with all detail fields
        let labels = [
            ("gpu", info.name.as_str()),
            ("instance", info.instance.as_str()),
            ("gpu_uuid", info.uuid.as_str()),
            ("gpu_index", row.index_str.as_str()),
            ("type", info.device_type.as_str()),
        ];

        // Convert detail HashMap to label pairs with sanitized names and values.
        // Values are sanitized to strip control characters and prevent
        // injection of ANSI escape sequences from NVML.
        let detail_labels: Vec<(String, String)> = info
            .detail
            .iter()
            .map(|(k, v)| (sanitize_label_name(k), sanitize_label_value(v)))
            .collect();

        builder
            .help("all_smi_gpu_info", "GPU/NPU device information")
            .type_("all_smi_gpu_info", "gauge");

        // Build dynamic labels by combining base and detail labels
        let mut all_labels = Vec::new();

        // Add base labels
        for (key, value) in labels.iter() {
            all_labels.push((*key, *value));
        }

        // Add detail labels
        for (key, value) in &detail_labels {
            all_labels.push((key.as_str(), value.as_str()));
        }

        // Use the metric method with all labels
        builder.metric("all_smi_gpu_info", &all_labels, 1);
    }

    fn export_cuda_metrics(&self, builder: &mut MetricBuilder, row: &GpuRow<'a>) {
        let info = row.gpu;
        let base_labels = [
            ("gpu", info.name.as_str()),
            ("instance", info.instance.as_str()),
            ("gpu_uuid", info.uuid.as_str()),
            ("gpu_index", row.index_str.as_str()),
        ];

        // PCIe metrics
        if let Some(pcie_gen) = info.detail.get("pcie_gen_current")
            && let Ok(pcie_gen_value) = pcie_gen.parse::<f64>()
        {
            builder
                .help("all_smi_gpu_pcie_gen_current", "Current PCIe generation")
                .type_("all_smi_gpu_pcie_gen_current", "gauge")
                .metric("all_smi_gpu_pcie_gen_current", &base_labels, pcie_gen_value);
        }

        if let Some(pcie_width) = info.detail.get("pcie_width_current")
            && let Ok(width) = pcie_width.parse::<f64>()
        {
            builder
                .help("all_smi_gpu_pcie_width_current", "Current PCIe link width")
                .type_("all_smi_gpu_pcie_width_current", "gauge")
                .metric("all_smi_gpu_pcie_width_current", &base_labels, width);
        }

        // Clock metrics
        if let Some(clock_max) = info.detail.get("clock_graphics_max")
            && let Ok(clock) = clock_max.parse::<f64>()
        {
            builder
                .help(
                    "all_smi_gpu_clock_graphics_max_mhz",
                    "Maximum graphics clock in MHz",
                )
                .type_("all_smi_gpu_clock_graphics_max_mhz", "gauge")
                .metric("all_smi_gpu_clock_graphics_max_mhz", &base_labels, clock);
        }

        if let Some(clock_max) = info.detail.get("clock_memory_max")
            && let Ok(clock) = clock_max.parse::<f64>()
        {
            builder
                .help(
                    "all_smi_gpu_clock_memory_max_mhz",
                    "Maximum memory clock in MHz",
                )
                .type_("all_smi_gpu_clock_memory_max_mhz", "gauge")
                .metric("all_smi_gpu_clock_memory_max_mhz", &base_labels, clock);
        }

        // Power limit metrics
        if let Some(power_limit) = info.detail.get("power_limit_current")
            && let Ok(power) = power_limit.parse::<f64>()
        {
            builder
                .help(
                    "all_smi_gpu_power_limit_current_watts",
                    "Current power limit in watts",
                )
                .type_("all_smi_gpu_power_limit_current_watts", "gauge")
                .metric("all_smi_gpu_power_limit_current_watts", &base_labels, power);
        }

        if let Some(power_limit) = info.detail.get("power_limit_max")
            && let Ok(power) = power_limit.parse::<f64>()
        {
            builder
                .help(
                    "all_smi_gpu_power_limit_max_watts",
                    "Maximum power limit in watts",
                )
                .type_("all_smi_gpu_power_limit_max_watts", "gauge")
                .metric("all_smi_gpu_power_limit_max_watts", &base_labels, power);
        }

        // Performance state — first prefer the structured per-device field
        // populated by the NVIDIA reader, fall back to the legacy
        // `detail.performance_state` string so mock servers and older
        // collectors keep working. When neither is available, the metric
        // is omitted entirely (Prometheus convention for "no data") so
        // dashboards can distinguish "unsupported" from P0 by absence
        // rather than relying on a sentinel value.
        if let Some(pstate) = info.performance_state {
            builder
                .help(
                    "all_smi_gpu_performance_state",
                    "GPU performance state (0=P0 fastest, 15=P15 idlest; metric is omitted when the device does not report a P-state)",
                )
                .type_("all_smi_gpu_performance_state", "gauge")
                .metric("all_smi_gpu_performance_state", &base_labels, pstate as f64);
        } else if let Some(pstate_str) = info.detail.get("performance_state")
            && let Some(state_str) = pstate_str.strip_prefix('P')
            && let Ok(state_num) = state_str.parse::<f64>()
        {
            builder
                .help(
                    "all_smi_gpu_performance_state",
                    "GPU performance state (0=P0 fastest, 15=P15 idlest; metric is omitted when the device does not report a P-state)",
                )
                .type_("all_smi_gpu_performance_state", "gauge")
                .metric("all_smi_gpu_performance_state", &base_labels, state_num);
        }

        // Fan speed, same shape as the P-state block above: prefer the
        // structured `fan_speed_rpm` field the AMD / Intel readers now
        // populate, fall back to the legacy `Fan Speed` detail string. The
        // fallback covers any `GpuInfo` that carries the legacy string
        // without the typed field, such as one deserialized from a snapshot
        // recorded before the field existed. A remote node running an older
        // build is handled upstream in `network::metrics_parser` instead:
        // this exporter only ever runs over locally read `GpuInfo`, and the
        // remote node's reading arrives as the `fan_speed` label on
        // `all_smi_gpu_info`, so it never reaches this branch. Omitted
        // entirely when neither is available, so a passively cooled card is
        // distinguishable from a stalled fan by absence rather than by a 0
        // reading.
        if let Some(rpm) = info.fan_speed_rpm {
            builder
                .help(
                    "all_smi_gpu_fan_speed_rpm",
                    "GPU fan speed in revolutions per minute (metric is omitted when the device reports no tachometer)",
                )
                .type_("all_smi_gpu_fan_speed_rpm", "gauge")
                .metric("all_smi_gpu_fan_speed_rpm", &base_labels, rpm);
        } else if let Some(fan) = info.detail.get(FAN_SPEED_DETAIL_KEY)
            && let Some(rpm) = parse_fan_speed_detail(fan)
        {
            builder
                .help(
                    "all_smi_gpu_fan_speed_rpm",
                    "GPU fan speed in revolutions per minute (metric is omitted when the device reports no tachometer)",
                )
                .type_("all_smi_gpu_fan_speed_rpm", "gauge")
                .metric("all_smi_gpu_fan_speed_rpm", &base_labels, rpm);
        }
    }

    /// Export extended NVML temperature thresholds and the P-state gauge.
    ///
    /// Emitted only when the GPU populated any of the new fields — so
    /// Apple Silicon / AMD / Jetson rows produce no output and the metrics
    /// surface stays unchanged for them.
    ///
    /// Each metric carries the standard GPU label set (`gpu`, `instance`,
    /// `uuid`, `index`) so dashboards can correlate with the existing
    /// `all_smi_gpu_temperature_celsius` series by the same labels.
    fn export_thermal_thresholds(&self, builder: &mut MetricBuilder, row: &GpuRow<'a>) {
        let info = row.gpu;
        let base_labels = [
            ("gpu", info.name.as_str()),
            ("instance", info.instance.as_str()),
            ("gpu_uuid", info.uuid.as_str()),
            ("gpu_index", row.index_str.as_str()),
        ];

        if let Some(slowdown) = info.temperature_threshold_slowdown {
            builder
                .help(
                    "all_smi_gpu_temperature_threshold_slowdown_celsius",
                    "GPU slowdown temperature threshold in Celsius",
                )
                .type_(
                    "all_smi_gpu_temperature_threshold_slowdown_celsius",
                    "gauge",
                )
                .metric(
                    "all_smi_gpu_temperature_threshold_slowdown_celsius",
                    &base_labels,
                    slowdown,
                );
        }

        if let Some(shutdown) = info.temperature_threshold_shutdown {
            builder
                .help(
                    "all_smi_gpu_temperature_threshold_shutdown_celsius",
                    "GPU shutdown temperature threshold in Celsius",
                )
                .type_(
                    "all_smi_gpu_temperature_threshold_shutdown_celsius",
                    "gauge",
                )
                .metric(
                    "all_smi_gpu_temperature_threshold_shutdown_celsius",
                    &base_labels,
                    shutdown,
                );
        }

        if let Some(gpu_max) = info.temperature_threshold_max_operating {
            builder
                .help(
                    "all_smi_gpu_temperature_threshold_max_operating_celsius",
                    "GPU maximum operating temperature threshold in Celsius",
                )
                .type_(
                    "all_smi_gpu_temperature_threshold_max_operating_celsius",
                    "gauge",
                )
                .metric(
                    "all_smi_gpu_temperature_threshold_max_operating_celsius",
                    &base_labels,
                    gpu_max,
                );
        }

        if let Some(acoustic) = info.temperature_threshold_acoustic {
            builder
                .help(
                    "all_smi_gpu_temperature_threshold_acoustic_celsius",
                    "GPU acoustic (noise) temperature threshold in Celsius",
                )
                .type_(
                    "all_smi_gpu_temperature_threshold_acoustic_celsius",
                    "gauge",
                )
                .metric(
                    "all_smi_gpu_temperature_threshold_acoustic_celsius",
                    &base_labels,
                    acoustic,
                );
        }
    }

    /// Pre-compute the stringified `gpu_index` label for each eligible GPU.
    /// Allocates once per scrape regardless of how many metric families are
    /// later emitted, reducing per-family string allocations from 5*N to N.
    fn collect_rows(&self) -> Vec<GpuRow<'a>> {
        self.gpu_info
            .iter()
            .enumerate()
            .filter(|(_, info)| {
                info.device_type == "GPU" || info.device_type == "NPU" || info.device_type == "TPU"
            })
            .map(|(idx, gpu)| GpuRow {
                gpu,
                index_str: idx.to_string(),
            })
            .collect()
    }
}

/// Borrowed view of a single GPU row with its stringified index cached.
/// Exists purely to amortise the `.to_string()` on the index label across
/// the five export methods.
struct GpuRow<'a> {
    gpu: &'a GpuInfo,
    index_str: String,
}

impl<'a> MetricExporter for GpuMetricExporter<'a> {
    fn export_metrics(&self) -> String {
        let rows = self.collect_rows();
        let mut builder = MetricBuilder::new();

        for row in &rows {
            self.export_basic_metrics(&mut builder, row);
            self.export_apple_silicon_metrics(&mut builder, row);
            self.export_device_info(&mut builder, row);
            self.export_cuda_metrics(&mut builder, row);
            self.export_thermal_thresholds(&mut builder, row);
        }

        builder.build()
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashMap;

    fn make_nvidia_gpu() -> GpuInfo {
        GpuInfo {
            uuid: "GPU-ABC".to_string(),
            time: String::new(),
            name: "NVIDIA A100".to_string(),
            device_type: "GPU".to_string(),
            host_id: "node-1".to_string(),
            hostname: "node-1".to_string(),
            instance: "node-1".to_string(),
            utilization: 50.0,
            ane_utilization: 0.0,
            dla_utilization: None,
            tensorcore_utilization: None,
            temperature: 70,
            used_memory: 1024,
            total_memory: 8192,
            frequency: 1500,
            power_consumption: 200.0,
            gpu_core_count: None,
            temperature_threshold_slowdown: Some(90),
            temperature_threshold_shutdown: Some(95),
            temperature_threshold_max_operating: Some(85),
            temperature_threshold_acoustic: Some(77),
            performance_state: Some(2),
            fan_speed_rpm: None,
            numa_node_id: None,
            gsp_firmware_mode: None,
            gsp_firmware_version: None,
            nvlink_remote_devices: Vec::new(),
            gpm_metrics: None,
            detail: HashMap::new(),
        }
    }

    #[test]
    fn exporter_emits_all_new_threshold_metrics() {
        let gpu = make_nvidia_gpu();
        let gpus = vec![gpu];
        let exporter = GpuMetricExporter::new(&gpus);
        let output = exporter.export_metrics();

        assert!(
            output.contains("all_smi_gpu_temperature_threshold_slowdown_celsius{"),
            "slowdown metric missing:\n{output}"
        );
        assert!(
            output.contains("all_smi_gpu_temperature_threshold_shutdown_celsius{"),
            "shutdown metric missing:\n{output}"
        );
        assert!(
            output.contains("all_smi_gpu_temperature_threshold_max_operating_celsius{"),
            "max_operating metric missing:\n{output}"
        );
        assert!(
            output.contains("all_smi_gpu_temperature_threshold_acoustic_celsius{"),
            "acoustic metric missing:\n{output}"
        );
    }

    #[test]
    fn exporter_sanitizes_dynamic_detail_label_names() {
        let mut gpu = make_nvidia_gpu();
        gpu.detail
            .insert("Source: Fan".to_string(), "hwmon".to_string());
        gpu.detail
            .insert("3D Engine".to_string(), "busy".to_string());
        gpu.detail
            .insert("GPU.Temp/Limit".to_string(), "90".to_string());
        let gpus = vec![gpu];
        let output = GpuMetricExporter::new(&gpus).export_metrics();
        let info_line = output
            .lines()
            .find(|line| line.starts_with("all_smi_gpu_info{"))
            .expect("GPU info metric");

        assert!(info_line.contains("source__fan=\"hwmon\""));
        assert!(info_line.contains("_3d_engine=\"busy\""));
        assert!(info_line.contains("gpu_temp_limit=\"90\""));
    }

    #[test]
    fn exporter_emits_pstate_from_structured_field() {
        let gpu = make_nvidia_gpu();
        let gpus = vec![gpu];
        let output = GpuMetricExporter::new(&gpus).export_metrics();
        // Structured field wins over the legacy detail-map path.
        assert!(
            output.contains("all_smi_gpu_performance_state{"),
            "pstate metric missing:\n{output}"
        );
        // Make sure the value is the structured `2`, not a truncated value.
        let pstate_line = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_performance_state{"))
            .expect("pstate line");
        assert!(
            pstate_line.ends_with(" 2"),
            "expected P2, got {pstate_line}"
        );
    }

    #[test]
    fn fan_speed_detail_parser_accepts_every_reader_value_shape() {
        // `amd.rs`, `intel_gpu_linux`, and `amd_adl` all write "<rpm> RPM";
        // the Level Zero reader appends the duty cycle when it has one.
        assert_eq!(parse_fan_speed_detail("1450 RPM"), Some(1450.0));
        assert_eq!(parse_fan_speed_detail("1600 RPM (40%)"), Some(1600.0));
        // Duty cycle only: no tachometer reading to publish.
        assert_eq!(parse_fan_speed_detail("40%"), None);
        // Garbage and out-of-domain values must not reach the exposition.
        assert_eq!(parse_fan_speed_detail("unknown RPM"), None);
        assert_eq!(parse_fan_speed_detail("-1 RPM"), None);
        assert_eq!(parse_fan_speed_detail(""), None);
    }

    #[test]
    fn fan_speed_detail_parser_rejects_fractional_and_out_of_range_values() {
        // A fractional value would be exported as e.g. "1450.5" on the
        // wire, which every all-smi parser then silently drops because it
        // requires an integer. Rejecting it here, at the one function both
        // the exporter and `network::metrics_parser` call, means a garbled
        // detail string can no longer be published only for downstream
        // consumers to lose it. An out-of-range value is the reader-side
        // clamp's failure mode (a corrupted sysfs/PMLog/Sysman read) and
        // must be rejected the same way.
        assert_eq!(parse_fan_speed_detail("1450.5 RPM"), None);
        assert_eq!(parse_fan_speed_detail("1600.25 RPM (40%)"), None);
        assert_eq!(parse_fan_speed_detail("4294967295 RPM"), None);
        assert_eq!(
            parse_fan_speed_detail(&format!("{} RPM", MAX_GPU_FAN_RPM as u64 + 1)),
            None
        );
        // The bound is inclusive.
        assert_eq!(
            parse_fan_speed_detail(&format!("{MAX_GPU_FAN_RPM} RPM")),
            Some(f64::from(MAX_GPU_FAN_RPM))
        );
    }

    #[test]
    fn exporter_emits_fan_speed_from_structured_field() {
        let mut gpu = make_nvidia_gpu();
        gpu.fan_speed_rpm = Some(1450);
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();

        let line = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_fan_speed_rpm{"))
            .unwrap_or_else(|| panic!("fan speed metric missing:\n{output}"));
        assert!(line.ends_with(" 1450"), "expected 1450 RPM, got {line}");
        // Same label set as the other per-device gauges so dashboards can
        // join on it.
        assert!(line.contains("gpu=\"NVIDIA A100\""));
        assert!(line.contains("instance=\"node-1\""));
        assert!(line.contains("gpu_uuid=\"GPU-ABC\""));
        assert!(line.contains("gpu_index=\"0\""));
    }

    #[test]
    fn exporter_omits_fan_speed_when_device_reports_none() {
        // A passively cooled card has no fan at all. Absence, not 0, is how
        // that is expressed on the wire.
        let gpu = make_nvidia_gpu();
        assert!(gpu.fan_speed_rpm.is_none());
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();
        assert!(
            !output.contains("all_smi_gpu_fan_speed_rpm"),
            "fan speed must be omitted without data:\n{output}"
        );
    }

    #[test]
    fn exporter_emits_fan_speed_from_detail_fallback() {
        // Mock servers and remote nodes running a build that predates the
        // typed field only carry the legacy detail string.
        let mut gpu = make_nvidia_gpu();
        gpu.fan_speed_rpm = None;
        gpu.detail
            .insert("Fan Speed".to_string(), "1450 RPM".to_string());
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();

        let line = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_fan_speed_rpm{"))
            .unwrap_or_else(|| panic!("fan speed fallback missing:\n{output}"));
        assert!(line.ends_with(" 1450"), "expected 1450 RPM, got {line}");
    }

    #[test]
    fn exporter_prefers_structured_fan_speed_over_the_detail_string() {
        let mut gpu = make_nvidia_gpu();
        gpu.fan_speed_rpm = Some(1450);
        gpu.detail
            .insert("Fan Speed".to_string(), "9999 RPM".to_string());
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();

        let lines: Vec<&str> = output
            .lines()
            .filter(|l| l.starts_with("all_smi_gpu_fan_speed_rpm{"))
            .collect();
        assert_eq!(lines.len(), 1, "exactly one sample per device: {lines:?}");
        assert!(lines[0].ends_with(" 1450"), "{}", lines[0]);
    }

    #[test]
    fn exporter_omits_fan_speed_for_a_duty_cycle_only_detail() {
        // The Level Zero reader writes a bare percentage when the driver
        // exposes no tachometer. That is not an RPM and must not be
        // published as one.
        let mut gpu = make_nvidia_gpu();
        gpu.fan_speed_rpm = None;
        gpu.detail
            .insert("Fan Speed".to_string(), "40%".to_string());
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();
        assert!(
            !output.contains("all_smi_gpu_fan_speed_rpm"),
            "a duty cycle is not an RPM reading:\n{output}"
        );
    }

    #[test]
    fn exporter_omits_fan_speed_for_a_fractional_detail_value() {
        // A snapshot recorded before the typed field existed, or a
        // hand-edited one, can carry a fractional `Fan Speed` string. Every
        // all-smi parser rejects a fractional
        // `all_smi_gpu_fan_speed_rpm` reading, so the exporter must not
        // publish one from the fallback either. Otherwise the two paths
        // would disagree, with this side happily emitting a value the
        // other side always drops.
        let mut gpu = make_nvidia_gpu();
        gpu.fan_speed_rpm = None;
        gpu.detail
            .insert("Fan Speed".to_string(), "1450.5 RPM".to_string());
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();
        assert!(
            !output.contains("all_smi_gpu_fan_speed_rpm"),
            "a fractional detail value must not reach the exposition:\n{output}"
        );
    }

    #[test]
    fn exporter_omits_fan_speed_for_an_out_of_range_detail_value() {
        // Mirrors the reader-side clamp's failure mode: a corrupted sensor
        // read that still parses as a huge integer must not be published,
        // the same way `network::metrics_parser` rejects it on the way
        // back in.
        let mut gpu = make_nvidia_gpu();
        gpu.fan_speed_rpm = None;
        gpu.detail
            .insert("Fan Speed".to_string(), "4294967295 RPM".to_string());
        let output = GpuMetricExporter::new(&[gpu]).export_metrics();
        assert!(
            !output.contains("all_smi_gpu_fan_speed_rpm"),
            "an out-of-range detail value must not reach the exposition:\n{output}"
        );
    }

    #[test]
    fn exporter_skips_thresholds_when_none_present() {
        let mut gpu = make_nvidia_gpu();
        gpu.temperature_threshold_slowdown = None;
        gpu.temperature_threshold_shutdown = None;
        gpu.temperature_threshold_max_operating = None;
        gpu.temperature_threshold_acoustic = None;
        gpu.performance_state = None;
        let gpus = vec![gpu];
        let output = GpuMetricExporter::new(&gpus).export_metrics();
        assert!(
            !output.contains("all_smi_gpu_temperature_threshold_"),
            "should not emit threshold metrics without data:\n{output}"
        );
        assert!(
            !output.contains("all_smi_gpu_performance_state{"),
            "should not emit pstate metric without data:\n{output}"
        );
    }

    #[test]
    fn exporter_emits_only_available_thresholds() {
        // Older drivers: slowdown + shutdown known, others absent.
        let mut gpu = make_nvidia_gpu();
        gpu.temperature_threshold_max_operating = None;
        gpu.temperature_threshold_acoustic = None;
        let gpus = vec![gpu];
        let output = GpuMetricExporter::new(&gpus).export_metrics();
        assert!(output.contains("all_smi_gpu_temperature_threshold_slowdown_celsius"));
        assert!(output.contains("all_smi_gpu_temperature_threshold_shutdown_celsius"));
        assert!(!output.contains("all_smi_gpu_temperature_threshold_max_operating_celsius"));
        assert!(!output.contains("all_smi_gpu_temperature_threshold_acoustic_celsius"));
    }

    /// Issue #325: an Apple Silicon row whose native metrics manager never
    /// initialized must omit the value series rather than publish zeros.
    #[test]
    fn exporter_omits_series_with_no_reading() {
        use crate::device::types::GPU_METRIC_UNAVAILABLE;

        let mut gpu = make_nvidia_gpu();
        gpu.name = "Apple M2 Max GPU".to_string();
        gpu.utilization = GPU_METRIC_UNAVAILABLE;
        gpu.power_consumption = GPU_METRIC_UNAVAILABLE;
        gpu.ane_utilization = GPU_METRIC_UNAVAILABLE;
        gpu.temperature = 0;
        gpu.frequency = 0;
        gpu.detail
            .insert("native_metrics".to_string(), "unavailable".to_string());

        let output = GpuMetricExporter::new(&[gpu]).export_metrics();

        for family in [
            "all_smi_gpu_utilization{",
            "all_smi_gpu_power_consumption_watts{",
            "all_smi_gpu_temperature_celsius{",
            "all_smi_gpu_frequency_mhz{",
            "all_smi_ane_utilization{",
            "all_smi_ane_power_watts{",
        ] {
            assert!(
                !output.contains(family),
                "{family} must be omitted, not emitted as 0:\n{output}"
            );
        }

        // The sentinel must never appear on the wire in any form.
        assert!(
            !output.contains(" -1"),
            "the in-band unavailable encoding leaked into the exposition:\n{output}"
        );

        // The device must still be discoverable, and must say why.
        let info_line = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_info{"))
            .expect("identity series must survive");
        assert!(
            info_line.contains("native_metrics=\"unavailable\""),
            "{info_line}"
        );
        // Memory comes from sysinfo, not IOReport, so it keeps reporting.
        assert!(output.contains("all_smi_gpu_memory_total_bytes{"));
    }

    /// Issue #378: a Windows WMI baseline must keep an unsourced field
    /// absent through Prometheus instead of turning it into a zero sample.
    #[test]
    fn exporter_omits_unsourced_windows_utilization_and_power() {
        use crate::device::types::GPU_METRIC_UNAVAILABLE;

        let mut gpu = make_nvidia_gpu();
        gpu.name = "AMD Radeon Graphics".to_string();
        gpu.utilization = GPU_METRIC_UNAVAILABLE;
        gpu.power_consumption = GPU_METRIC_UNAVAILABLE;
        gpu.detail
            .insert("Source: Utilization".to_string(), "unavailable".to_string());
        gpu.detail
            .insert("Source: Power".to_string(), "unavailable".to_string());

        let output = GpuMetricExporter::new(&[gpu]).export_metrics();

        assert!(!output.contains("all_smi_gpu_utilization{"), "{output}");
        assert!(
            !output.contains("all_smi_gpu_power_consumption_watts{"),
            "{output}"
        );
        assert!(output.contains("all_smi_gpu_info{"), "{output}");
        assert!(output.contains("source__utilization=\"unavailable\""));
        assert!(output.contains("source__power=\"unavailable\""));
    }

    /// The other half of the contract: a real zero is still published, so
    /// omission unambiguously means "no data".
    #[test]
    fn exporter_emits_genuine_zero_readings() {
        let mut gpu = make_nvidia_gpu();
        gpu.utilization = 0.0;
        gpu.power_consumption = 0.0;
        gpu.temperature = 42;
        gpu.frequency = 300;

        let output = GpuMetricExporter::new(&[gpu]).export_metrics();

        let util = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_utilization{"))
            .expect("idle GPU must still report 0% utilization");
        assert!(util.ends_with(" 0"), "expected a zero reading, got {util}");

        let power = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_power_consumption_watts{"))
            .expect("a 0 W rail is a reading");
        assert!(power.ends_with(" 0"), "{power}");
    }

    /// Non-Apple readers set `ane_utilization` to a literal 0.0 meaning "not
    /// applicable". That has always been published and must keep being
    /// published, so this change does not silently alter NVIDIA scrapes.
    #[test]
    fn exporter_keeps_emitting_zero_ane_for_non_apple_gpus() {
        let output = GpuMetricExporter::new(&[make_nvidia_gpu()]).export_metrics();
        assert!(output.contains("all_smi_ane_utilization{"));
        // `all_smi_ane_power_watts` stays Apple-only, as before.
        assert!(!output.contains("all_smi_ane_power_watts{"));
    }

    #[test]
    fn exporter_preserves_standard_gpu_labels_on_new_metrics() {
        let gpu = make_nvidia_gpu();
        let gpus = vec![gpu];
        let output = GpuMetricExporter::new(&gpus).export_metrics();
        // Sanity-check label set on the new metrics — should match the
        // legacy `all_smi_gpu_temperature_celsius` labels exactly.
        let slowdown_line = output
            .lines()
            .find(|l| l.starts_with("all_smi_gpu_temperature_threshold_slowdown_celsius{"))
            .expect("slowdown line");
        assert!(slowdown_line.contains("gpu=\"NVIDIA A100\""));
        assert!(slowdown_line.contains("instance=\"node-1\""));
        assert!(slowdown_line.contains("gpu_uuid=\"GPU-ABC\""));
        assert!(slowdown_line.contains("gpu_index=\"0\""));
    }
}