use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use metrale_bench::hardware::throttle_monitor::{ThrottleMonitor, ThrottleWindow};
const SAMPLE_EVERY: Duration = Duration::from_secs(2);
#[derive(Debug, Clone, Copy, Default)]
pub struct ThermalSnapshot {
pub window: Option<ThrottleWindow>,
pub transitions: usize,
pub samples: usize,
pub gpu_temp_c: Option<f64>,
pub sm_clock_mhz: Option<f64>,
pub sm_clock_max_mhz: Option<f64>,
pub have_data: bool,
}
impl ThermalSnapshot {
pub fn clock_frac(&self) -> Option<f64> {
match (self.sm_clock_mhz, self.sm_clock_max_mhz) {
(Some(now), Some(max)) if max > 0.0 => Some((now / max).clamp(0.0, 1.0)),
_ => None,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct ThermalProbe {
inner: Arc<Mutex<ThermalSnapshot>>,
}
impl ThermalProbe {
pub fn spawn() -> Self {
let probe = Self::default();
let sink = Arc::clone(&probe.inner);
std::thread::Builder::new()
.name("metrale-thermal".into())
.spawn(move || {
let mut monitor = ThrottleMonitor::new();
loop {
let started = Instant::now();
let state = metrale_bench::hardware::collect::collect();
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let window = monitor.observe(now_ms, state.throttle_counters);
if let Ok(mut slot) = sink.lock() {
if window.is_some() {
slot.window = window;
}
slot.transitions = monitor.transitions();
slot.samples = monitor.samples();
slot.gpu_temp_c = state.gpu_temp_c;
slot.sm_clock_mhz = state.sm_clock_mhz;
slot.sm_clock_max_mhz = state.sm_clock_max_mhz;
slot.have_data = true;
}
std::thread::sleep(SAMPLE_EVERY.saturating_sub(started.elapsed()));
}
})
.ok();
probe
}
pub fn snapshot(&self) -> ThermalSnapshot {
self.inner
.lock()
.map(|s| *s)
.unwrap_or_else(|p| *p.into_inner())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ThermalAlert {
Unknown,
Ok,
Throttling,
Thrashing,
}
pub const THROTTLE_WARN_FRAC: f64 = 0.20;
pub const THRASH_TRANSITIONS: usize = 6;
pub const THRASH_MIN_SAMPLES: usize = 8;
impl ThermalSnapshot {
pub fn alert(&self) -> ThermalAlert {
let Some(w) = self.window else {
return ThermalAlert::Unknown;
};
let Some(frac) = w.thermal_frac else {
return ThermalAlert::Unknown;
};
if self.samples >= THRASH_MIN_SAMPLES && self.transitions >= THRASH_TRANSITIONS {
return ThermalAlert::Thrashing;
}
if frac >= THROTTLE_WARN_FRAC {
return ThermalAlert::Throttling;
}
ThermalAlert::Ok
}
}
#[cfg(test)]
mod tests {
use super::*;
fn snap(frac: Option<f64>, transitions: usize, samples: usize) -> ThermalSnapshot {
ThermalSnapshot {
window: Some(ThrottleWindow {
window_ms: 2_000,
thermal_frac: frac,
power_cap_frac: Some(1.0),
thermal_active: frac.is_some_and(|f| f > 0.0),
}),
transitions,
samples,
have_data: true,
..Default::default()
}
}
#[test]
fn absent_data_is_unknown_not_ok() {
assert_eq!(ThermalSnapshot::default().alert(), ThermalAlert::Unknown);
assert_eq!(snap(None, 0, 20).alert(), ThermalAlert::Unknown);
}
#[test]
fn a_quiet_box_is_ok() {
assert_eq!(snap(Some(0.01), 0, 20).alert(), ThermalAlert::Ok);
}
#[test]
fn sustained_throttling_warns() {
assert_eq!(snap(Some(0.55), 0, 20).alert(), ThermalAlert::Throttling);
}
#[test]
fn flapping_outranks_a_steady_hold() {
let a = snap(Some(0.05), THRASH_TRANSITIONS, 20).alert();
assert_eq!(a, ThermalAlert::Thrashing);
assert!(ThermalAlert::Thrashing > ThermalAlert::Throttling);
}
#[test]
fn thrashing_needs_enough_history_to_be_believable() {
assert_ne!(
snap(Some(0.05), THRASH_TRANSITIONS, THRASH_MIN_SAMPLES - 1).alert(),
ThermalAlert::Thrashing
);
}
#[test]
fn a_pinned_power_cap_alone_is_still_ok() {
assert_eq!(snap(Some(0.0), 0, 20).alert(), ThermalAlert::Ok);
}
}