use std::collections::HashMap;
use std::path::PathBuf;
use std::process::{Child, Command};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use gpuviewer_core::{
all_backends, normalize_pci_id, now_ms, Confidence, DeviceId, DynamicSample, Event,
EventEngine, EventKind, GpuBackend, ProcessSample, Severity, StaticInfo,
};
use gpuviewer_history::{DataSource, HistoryStore, Recorder, SqliteStore, WriteTransition};
use serde::Serialize;
#[derive(Serialize)]
pub struct FrameDevice {
pub id: DeviceId,
pub name: String,
pub mem_total_bytes: Option<u64>,
pub sample: Option<DynamicSample>,
pub processes: Option<Vec<ProcessSample>>,
}
pub struct Frame {
pub ts_ms: u64,
pub devices: Vec<FrameDevice>,
pub events: Vec<Event>,
}
pub fn stall_threshold(interval: Duration) -> Duration {
(interval * 3).max(Duration::from_secs(5))
}
pub fn is_stall(gap: Duration, interval: Duration) -> bool {
gap > stall_threshold(interval)
}
pub const MAX_CONSECUTIVE_PANICS: u32 = 3;
pub const DEVICE_LOST_AFTER_FAILED_PROBES: u32 = 5;
#[derive(Clone, Default)]
struct ProbeHealth {
consecutive_failures: u32,
first_fail_ms: u64,
first_error: String,
last_error: String,
last_good_ms: Option<u64>,
lost_at_ms: Option<u64>,
}
pub enum TickOutcome {
Frame(Frame),
Panicked {
event: Event,
summary: String,
fatal: bool,
},
}
fn panic_summary(payload: &(dyn std::any::Any + Send)) -> String {
if let Some(s) = payload.downcast_ref::<&str>() {
(*s).to_string()
} else if let Some(s) = payload.downcast_ref::<String>() {
s.clone()
} else {
"non-string panic payload".into()
}
}
fn device_lost_event(di: usize, id: &DeviceId, h: &ProbeHealth, at_ms: u64) -> Event {
let wall = Duration::from_millis(at_ms.saturating_sub(h.first_fail_ms));
let last_good = match h.last_good_ms {
Some(ts) => format!("last good data {}", fmt_clock(ts)),
None => "it never answered this session".to_string(),
};
let last_good_ev = match h.last_good_ms {
Some(ts) => format!("last good sample at {ts} ms ({})", fmt_clock(ts)),
None => "no good sample this session".to_string(),
};
let errors = if h.first_error == h.last_error {
format!("error: {}", h.last_error)
} else {
format!(
"first error: {}; last error: {}",
h.first_error, h.last_error
)
};
Event {
ts_ms: at_ms,
device: id.clone(),
kind: EventKind::DeviceLost,
severity: Severity::Critical,
confidence: Confidence::Fact,
title: format!(
"GPU{di} stopped answering — device lost after {} consecutive failed probes; \
{last_good}",
h.consecutive_failures
),
evidence: format!(
"refresh_dynamic for {id} failed {} consecutive ticks over {} of wall time \
(the threshold counts ticks; a failing probe pins full cadence, so only the \
gap into the first failure can be backoff-stretched); {last_good_ev}; \
{errors}; cause not asserted — driver reset, device removal, and library \
death look identical from here",
h.consecutive_failures,
fmt_dur(wall),
),
}
}
fn device_returned_event(
di: usize,
id: &DeviceId,
h: &ProbeHealth,
lost_at_ms: u64,
ts_ms: u64,
) -> Event {
let gap_start = h.last_good_ms.unwrap_or(h.first_fail_ms);
let gap = Duration::from_millis(ts_ms.saturating_sub(gap_start));
Event {
ts_ms,
device: id.clone(),
kind: EventKind::DeviceReturned,
severity: Severity::Info,
confidence: Confidence::Fact,
title: format!(
"GPU{di} answering again — device returned; {} of data are missing",
fmt_dur(gap)
),
evidence: format!(
"refresh_dynamic for {id} succeeded after {} consecutive failures; declared \
lost at {}; nothing was collected between {} and {} — the gap stays blank \
in history, never zero-filled",
h.consecutive_failures,
fmt_clock(lost_at_ms),
fmt_clock(gap_start),
fmt_clock(ts_ms),
),
}
}
fn recording_degraded_event(ts_ms: u64, anchor: DeviceId, t: &WriteTransition) -> Event {
let (severity, title, evidence) = match t {
WriteTransition::Degraded { error } => (
Severity::Critical,
"recording degraded — history writes are failing".to_string(),
format!(
"the live view keeps running, but nothing is reaching the history \
database, so this period will NOT be replayable; first error: {error}"
),
),
WriteTransition::Recovered { lost } => (
Severity::Warning,
format!("recording resumed — {lost} write(s) were lost"),
format!(
"history writes are succeeding again; the {lost} write(s) that failed are \
gone and that span stays blank in replay rather than being backfilled"
),
),
};
Event {
ts_ms,
device: anchor,
kind: EventKind::RecordingDegraded,
severity,
confidence: Confidence::Fact,
title,
evidence,
}
}
fn recording_started_event(
ts_ms: u64,
anchor: DeviceId,
interval: Duration,
backend_list: &str,
n_devices: usize,
db_name: &str,
dangling_start_ms: Option<u64>,
) -> Event {
let title = match dangling_start_ms {
Some(_) => "recording started — previous session ended without a stop mark \
(crash, kill, or power loss)"
.to_string(),
None => format!(
"recording started — gpuviewer {}",
env!("CARGO_PKG_VERSION")
),
};
let devices_word = if n_devices == 1 { "device" } else { "devices" };
let mut evidence = format!(
"gpuviewer {}; interval {}; backends: {backend_list} ({n_devices} {devices_word}); \
db {db_name}",
env!("CARGO_PKG_VERSION"),
fmt_dur(interval),
);
if let Some(prev) = dangling_start_ms {
evidence.push_str(&format!(
"; the previous session's start mark at {} has no matching stop — it died \
without flushing (crash, kill, or power loss), so where its recording \
actually ends is unknown and the gap size is unknowable",
fmt_stamp(prev)
));
}
Event {
ts_ms,
device: anchor,
kind: EventKind::RecordingStarted,
severity: Severity::Info,
confidence: Confidence::Fact,
title,
evidence,
}
}
pub const SUSPEND_MIN_MS: u64 = 30_000;
fn host_suspended_event(
prev_wall_ms: u64,
now_wall_ms: u64,
mono_gap: Duration,
unseen_ms: u64,
anchor: DeviceId,
) -> Event {
let unseen = Duration::from_millis(unseen_ms);
Event {
ts_ms: now_wall_ms,
device: anchor,
kind: EventKind::HostSuspended,
severity: Severity::Info,
confidence: Confidence::Likely,
title: format!(
"host likely asleep for {} ({}–{}) — nothing was recorded; the gap is the \
machine suspended, not the GPU idle",
fmt_dur(unseen),
fmt_clock(prev_wall_ms),
fmt_clock(now_wall_ms),
),
evidence: format!(
"wall clock advanced {} between ticks while the monotonic clock advanced {}; the \
{} difference is time the OS did not run gpuviewer — a system suspend, or a \
forward wall-clock step (manual or NTP), which this check cannot tell apart",
fmt_dur(Duration::from_millis(
now_wall_ms.saturating_sub(prev_wall_ms)
)),
fmt_dur(mono_gap),
fmt_dur(unseen),
),
}
}
fn paused_event(paused_at_ms: u64, now_ms: u64, anchor: DeviceId) -> Event {
let dur = Duration::from_millis(now_ms.saturating_sub(paused_at_ms));
Event {
ts_ms: now_ms,
device: anchor,
kind: EventKind::CollectorStall,
severity: Severity::Info,
confidence: Confidence::Fact,
title: format!(
"collection was paused for {} (since {}) — the recording has a hole there",
fmt_dur(dur),
fmt_clock(paused_at_ms)
),
evidence: format!(
"the user paused collection at {} and resumed at {}; no samples were taken \
in between",
fmt_stamp(paused_at_ms),
fmt_stamp(now_ms)
),
}
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
const XID_REPEAT_WINDOW_MS: u64 = 60_000;
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
const XID_CRITICAL: &[u32] = &[48, 64, 74, 79, 95, 119, 120, 154];
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn xid_event(
di: usize,
id: &DeviceId,
x: &gpuviewer_core::kmsg::XidRecord,
folded: u32,
at_ms: u64,
) -> Event {
let mut pid = None;
let mut pname = None;
let mut rest = x.detail.as_str();
loop {
let r = rest.trim_start();
if let Some(v) = r.strip_prefix("pid=") {
let (val, tail) = v.split_once(',').unwrap_or((v, ""));
pid = Some(val.trim().to_string());
rest = tail;
} else if let Some(v) = r.strip_prefix("name=") {
let (val, tail) = v.split_once(',').unwrap_or((v, ""));
pname = Some(val.trim().to_string());
rest = tail;
} else {
rest = r;
break;
}
}
let who = match (pname, pid) {
(Some(n), Some(p)) => format!(" ({n} pid {p})"),
(None, Some(p)) => format!(" (pid {p})"),
_ => String::new(),
};
let desc = if rest.is_empty() {
String::new()
} else {
format!(": {rest}")
};
Event {
ts_ms: at_ms,
device: id.clone(),
kind: EventKind::XidError,
severity: if XID_CRITICAL.contains(&x.code) {
Severity::Critical
} else {
Severity::Warning
},
confidence: Confidence::Fact,
title: format!("GPU{di} driver reported Xid {}{desc}{who}", x.code),
evidence: format!(
"kernel log: {}{}",
x.raw,
if folded > 0 {
format!(
" — {folded} earlier repeat{} of this Xid in the previous minute folded",
if folded == 1 { "" } else { "s" }
)
} else {
String::new()
}
),
}
}
const SLOW_PROBE: Duration = Duration::from_millis(700);
const SLOW_PROBE_COOLDOWN: Duration = Duration::from_secs(300);
pub struct EngineConfig {
pub force_mock: bool,
pub persist: bool,
pub db_path: Option<PathBuf>,
pub interval: Duration,
pub on_event: Option<String>,
}
impl Default for EngineConfig {
fn default() -> Self {
Self {
force_mock: false,
persist: false,
db_path: None,
interval: Duration::from_millis(1000),
on_event: None,
}
}
}
pub struct Engine {
backends: Vec<Box<dyn GpuBackend>>,
devices: Vec<(usize, DeviceId, StaticInfo)>,
event_engine: EventEngine,
recorder: Option<Recorder>,
db_path: Option<PathBuf>,
interval: Duration,
last_tick_end: Option<Instant>,
last_tick_end_wall_ms: Option<u64>,
cadence: Duration,
paused_since_ms: Option<u64>,
#[cfg(target_os = "linux")]
xid: Option<gpuviewer_core::kmsg::XidWatcher>,
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
xid_last: HashMap<(usize, u32), (u64, u32)>,
last_slow_note: HashMap<DeviceId, Instant>,
pending_reset: Option<Event>,
pending_session_start: Option<Event>,
sink: Option<EventSink>,
consecutive_panics: u32,
health: Vec<ProbeHealth>,
session_start_ms: u64,
frames_folded: u64,
finished: bool,
}
impl Engine {
pub fn new(config: EngineConfig) -> Self {
Self::with_backends(all_backends(config.force_mock), config)
}
pub(crate) fn with_backends(
mut backends: Vec<Box<dyn GpuBackend>>,
config: EngineConfig,
) -> Self {
let mut devices = Vec::new();
let mut event_engine = EventEngine::new();
let mut seen_pci: HashMap<String, &'static str> = HashMap::new();
for (bi, b) in backends.iter_mut().enumerate() {
for id in b.devices() {
let pci_key = normalize_pci_id(&id.0);
if let Some(key) = &pci_key {
if let Some(first) = seen_pci.get(key.as_str()) {
eprintln!(
"gpuviewer: {id} ({}) duplicates a device already registered by {first}; skipping",
b.name()
);
continue;
}
}
match b.static_info(&id) {
Ok(info) => {
if let Some(key) = pci_key {
seen_pci.insert(key, b.name());
}
event_engine.register_device(id.clone(), format!("GPU{}", devices.len()));
event_engine.register_pcie_max(
id.clone(),
info.pcie_max_gen,
info.pcie_max_width,
);
devices.push((bi, id, info));
}
Err(e) => {
eprintln!("gpuviewer: skipping {id} ({}): {e}", b.name());
}
}
}
}
let mock_in_use = backends.iter().any(|b| b.name() == "mock");
let session_start_ms = now_ms();
let mut recorder = None;
let mut db_path = None;
let mut pending_reset = None;
let mut pending_session_start = None;
if config.persist {
let source = if mock_in_use {
DataSource::Mock
} else {
DataSource::Real
};
let opened = match &config.db_path {
Some(p) => SqliteStore::open_recording(p, source),
None => SqliteStore::open_default(mock_in_use),
};
match opened {
Ok((store, was_reset)) => {
db_path = Some(store.path().to_path_buf());
let mut rec = Recorder::new(store);
for (_, id, info) in &devices {
let _ = rec.store_mut().register_device(
id,
&info.name,
info.vendor,
info.mem_total_bytes,
);
}
if was_reset {
let path_note = db_path
.as_ref()
.map(|p| p.display().to_string())
.unwrap_or_default();
let anchor = devices
.first()
.map(|(_, id, _)| id.clone())
.unwrap_or_else(|| DeviceId("history".into()));
pending_reset = Some(Event {
ts_ms: now_ms(),
device: anchor,
kind: EventKind::HistoryReset,
severity: Severity::Warning,
confidence: Confidence::Fact,
title: "history file was corrupt — renamed aside and started fresh"
.into(),
evidence: format!(
"previous file kept at {path_note}.corrupt-<unix_seconds>; \
a fresh database was created at {path_note}"
),
});
}
let prev_start = rec
.store()
.latest_event_ms(Some(EventKind::RecordingStarted))
.ok()
.flatten();
let prev_stop = rec
.store()
.latest_event_ms(Some(EventKind::RecordingStopped))
.ok()
.flatten();
let dangling = prev_start.filter(|s| *s > prev_stop.unwrap_or(0));
let mut backend_names: Vec<&'static str> = Vec::new();
for (bi, _, _) in &devices {
let n = backends[*bi].name();
if !backend_names.contains(&n) {
backend_names.push(n);
}
}
let backend_list = if backend_names.is_empty() {
"none".to_string()
} else {
backend_names.join(", ")
};
let db_name = db_path
.as_ref()
.and_then(|p| p.file_name())
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "history.db".into());
let anchor = devices
.first()
.map(|(_, id, _)| id.clone())
.unwrap_or_else(|| DeviceId("collector".into()));
let start_ev = recording_started_event(
session_start_ms,
anchor,
config.interval,
&backend_list,
devices.len(),
&db_name,
dangling,
);
rec.record_events(std::slice::from_ref(&start_ev));
pending_session_start = Some(start_ev);
recorder = Some(rec);
}
Err(e) => {
eprintln!("gpuviewer: history persistence disabled (store open failed): {e}");
}
}
}
let sink = config.on_event.map(EventSink::new);
let health = vec![ProbeHealth::default(); devices.len()];
#[cfg(target_os = "linux")]
let xid = (!mock_in_use
&& devices
.iter()
.any(|(_, _, i)| i.vendor == gpuviewer_core::Vendor::Nvidia))
.then(gpuviewer_core::kmsg::XidWatcher::open)
.and_then(Result::ok);
Self {
backends,
devices,
event_engine,
recorder,
db_path,
interval: config.interval,
last_tick_end: None,
last_tick_end_wall_ms: None,
cadence: config.interval,
paused_since_ms: None,
#[cfg(target_os = "linux")]
xid,
xid_last: HashMap::new(),
last_slow_note: HashMap::new(),
pending_reset,
pending_session_start,
sink,
consecutive_panics: 0,
health,
session_start_ms,
frames_folded: 0,
finished: false,
}
}
pub fn static_infos(&self) -> Vec<StaticInfo> {
self.devices.iter().map(|(_, _, i)| i.clone()).collect()
}
pub fn mock_in_use(&self) -> bool {
self.backends.iter().any(|b| b.name() == "mock")
}
pub fn db_path(&self) -> Option<PathBuf> {
self.db_path.clone()
}
pub fn tick(&mut self) -> Frame {
let mut frame_devices = Vec::with_capacity(self.devices.len());
let mut events = Vec::new();
let now_instant = Instant::now();
let now_wall = now_ms();
if let Some(paused_at) = self.paused_since_ms.take() {
events.push(paused_event(paused_at, now_wall, self.anchor_device()));
} else if let (Some(prev_end), Some(prev_wall)) =
(self.last_tick_end, self.last_tick_end_wall_ms)
{
let gap = now_instant.duration_since(prev_end);
let wall_gap_ms = now_wall.saturating_sub(prev_wall);
let unseen_ms = wall_gap_ms.saturating_sub(gap.as_millis() as u64);
if unseen_ms >= SUSPEND_MIN_MS {
events.push(host_suspended_event(
prev_wall,
now_wall,
gap,
unseen_ms,
self.anchor_device(),
));
} else if is_stall(gap, self.cadence.max(self.interval)) {
let last_good = fmt_clock(now_wall.saturating_sub(gap.as_millis() as u64));
let anchor = self.anchor_device();
events.push(Event {
ts_ms: now_wall,
device: anchor,
kind: EventKind::CollectorStall,
severity: Severity::Warning,
confidence: Confidence::Fact,
title: format!(
"collection stalled {} — no frames were taken; the data gap is \
recorded, last good frame at {last_good}",
fmt_dur(gap)
),
evidence: format!(
"inter-tick gap {} exceeds the {} stall threshold (cadence {}, \
interval {}); the collector was not scheduled — the process was \
stopped, starved of CPU, or blocked writing its output",
fmt_dur(gap),
fmt_dur(stall_threshold(self.cadence.max(self.interval))),
fmt_dur(self.cadence),
fmt_dur(self.interval),
),
});
}
}
if let Some(reset) = self.pending_reset.take() {
events.push(reset);
}
if let Some(start) = self.pending_session_start.take() {
events.push(start);
}
for (di, (bi, id, info)) in self.devices.iter().enumerate() {
let backend = &mut self.backends[*bi];
let probe_start = Instant::now();
let (sample, probe_err) = match backend.refresh_dynamic(id) {
Ok(s) => (Some(s), None),
Err(e) => (None, Some(e.to_string())),
};
let probe_dur = probe_start.elapsed();
let processes = backend.refresh_processes(id).ok();
let h = &mut self.health[di];
match &sample {
Some(s) => {
if let Some(lost_at) = h.lost_at_ms.take() {
events.push(device_returned_event(di, id, h, lost_at, s.ts_ms));
}
h.consecutive_failures = 0;
h.last_good_ms = Some(s.ts_ms);
}
None => {
let err = probe_err.unwrap_or_else(|| "unknown error".into());
h.consecutive_failures += 1;
if h.consecutive_failures == 1 {
h.first_fail_ms = now_ms();
h.first_error = err.clone();
}
h.last_error = err;
if h.lost_at_ms.is_none()
&& h.consecutive_failures >= DEVICE_LOST_AFTER_FAILED_PROBES
{
let at = now_ms();
h.lost_at_ms = Some(at);
events.push(device_lost_event(di, id, h, at));
}
}
}
if probe_dur > SLOW_PROBE {
let fresh = self
.last_slow_note
.get(id)
.map(|t| now_instant.duration_since(*t) >= SLOW_PROBE_COOLDOWN)
.unwrap_or(true);
if fresh {
self.last_slow_note.insert(id.clone(), now_instant);
events.push(Event {
ts_ms: now_ms(),
device: id.clone(),
kind: EventKind::CollectorStall,
severity: Severity::Info,
confidence: Confidence::Fact,
title: format!(
"{} probe took {} — driver slow to respond",
backend.name(),
fmt_dur(probe_dur)
),
evidence: format!(
"refresh_dynamic for {id} took {} (> {} threshold)",
fmt_dur(probe_dur),
fmt_dur(SLOW_PROBE)
),
});
}
}
if let Some(s) = &sample {
events.extend(self.event_engine.observe(
id,
s,
processes.as_deref(),
info.mem_total_bytes,
info.temp_slowdown_c,
));
if let Some(rec) = &mut self.recorder {
rec.observe(id, s, processes.as_deref().unwrap_or(&[]));
}
}
frame_devices.push(FrameDevice {
id: id.clone(),
name: info.name.clone(),
mem_total_bytes: info.mem_total_bytes,
sample,
processes,
});
}
#[cfg(target_os = "linux")]
{
let records = self.xid.as_mut().map(|w| w.poll()).unwrap_or_default();
let ids: Vec<DeviceId> = self.devices.iter().map(|(_, id, _)| id.clone()).collect();
events.extend(self.xid_events(&ids, records, now_ms()));
}
if let Some(rec) = &mut self.recorder {
if let Some(t) = rec.take_write_transition() {
if let Some((_, anchor, _)) = self.devices.first() {
events.push(recording_degraded_event(now_ms(), anchor.clone(), &t));
}
}
}
if let Some(rec) = &mut self.recorder {
rec.record_events(&events);
}
if let Some(sink) = &mut self.sink {
for e in &events {
sink.fire(e);
}
}
self.last_tick_end = Some(Instant::now());
self.last_tick_end_wall_ms = Some(now_ms());
self.frames_folded += 1;
Frame {
ts_ms: now_ms(),
devices: frame_devices,
events,
}
}
#[cfg(test)]
pub(crate) fn set_last_tick_end(&mut self, t: Instant) {
self.last_tick_end = Some(t);
let gap = Instant::now().saturating_duration_since(t).as_millis() as u64;
self.last_tick_end_wall_ms = Some(now_ms().saturating_sub(gap));
}
#[cfg(test)]
pub(crate) fn set_last_tick_end_wall(&mut self, ms: u64) {
self.last_tick_end_wall_ms = Some(ms);
}
pub fn set_cadence(&mut self, cadence: Duration) {
self.cadence = cadence;
}
pub fn note_paused(&mut self) {
self.paused_since_ms.get_or_insert_with(now_ms);
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn xid_events(
&mut self,
ids: &[DeviceId],
records: Vec<gpuviewer_core::kmsg::XidRecord>,
at_ms: u64,
) -> Vec<Event> {
let mut out = Vec::new();
for x in records {
let Some(di) = ids
.iter()
.position(|id| gpuviewer_core::kmsg::xid_matches_device(&x.pci, &id.0))
else {
continue;
};
let key = (di, x.code);
let folded = match self.xid_last.get_mut(&key) {
Some((last, n)) if at_ms.saturating_sub(*last) < XID_REPEAT_WINDOW_MS => {
*n += 1;
continue;
}
Some((_, n)) => std::mem::take(n),
None => 0,
};
self.xid_last.insert(key, (at_ms, 0));
out.push(xid_event(di, &ids[di], &x, folded, at_ms));
}
out
}
#[cfg(test)]
pub(crate) fn inject_xid(
&mut self,
records: Vec<gpuviewer_core::kmsg::XidRecord>,
at_ms: u64,
) -> Vec<Event> {
let ids: Vec<DeviceId> = self.devices.iter().map(|(_, id, _)| id.clone()).collect();
self.xid_events(&ids, records, at_ms)
}
fn anchor_device(&self) -> DeviceId {
self.devices
.first()
.map(|(_, id, _)| id.clone())
.unwrap_or_else(|| DeviceId("collector".into()))
}
pub fn tick_guarded(&mut self) -> TickOutcome {
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.tick())) {
Ok(frame) => {
self.consecutive_panics = 0;
TickOutcome::Frame(frame)
}
Err(payload) => {
self.consecutive_panics += 1;
let summary = panic_summary(payload.as_ref());
let fatal = self.consecutive_panics >= MAX_CONSECUTIVE_PANICS;
eprintln!(
"gpuviewer: collector tick panicked ({}/{MAX_CONSECUTIVE_PANICS} \
consecutive): {summary} (run with RUST_BACKTRACE=1 for a backtrace)",
self.consecutive_panics
);
let event = self.panic_event(&summary, fatal);
let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
if let Some(rec) = &mut self.recorder {
rec.record_events(std::slice::from_ref(&event));
}
if let Some(sink) = &mut self.sink {
sink.fire(&event);
}
}));
if recorded.is_err() {
eprintln!(
"gpuviewer: recording the panic event panicked too — the event \
reaches the live view only; stderr holds the full trail"
);
}
TickOutcome::Panicked {
event,
summary,
fatal,
}
}
}
}
fn panic_event(&self, summary: &str, fatal: bool) -> Event {
let anchor = self.anchor_device();
let n = self.consecutive_panics;
if fatal {
Event {
ts_ms: now_ms(),
device: anchor,
kind: EventKind::CollectorStall,
severity: Severity::Critical,
confidence: Confidence::Fact,
title: format!("collection STOPPED — collector panicked {n} ticks in a row"),
evidence: format!(
"panic: {summary}; {n} consecutive tick panics spent the \
{MAX_CONSECUTIVE_PANICS}-tick budget — collection will not resume \
and the recording ends here; run with RUST_BACKTRACE=1 for a backtrace"
),
}
} else {
Event {
ts_ms: now_ms(),
device: anchor,
kind: EventKind::CollectorStall,
severity: Severity::Warning,
confidence: Confidence::Fact,
title: format!(
"collector tick panicked ({n}/{MAX_CONSECUTIVE_PANICS}) — frame \
dropped, the recording has a hole"
),
evidence: format!(
"panic: {summary}; consecutive panic {n} of {MAX_CONSECUTIVE_PANICS} \
tolerated, collection retries next tick; run with RUST_BACKTRACE=1 \
for a backtrace"
),
}
}
}
pub fn finish(&mut self) -> Option<Event> {
let stop = if self.recorder.is_some() && !self.finished {
self.finished = true;
let ev = self.stop_event();
if let Some(rec) = &mut self.recorder {
rec.record_events(std::slice::from_ref(&ev));
}
if let Some(sink) = &mut self.sink {
sink.fire(&ev);
}
Some(ev)
} else {
None
};
self.flush();
stop
}
fn stop_event(&self) -> Event {
let now = now_ms();
let dur = Duration::from_millis(now.saturating_sub(self.session_start_ms));
let frames_word = if self.frames_folded == 1 {
"frame"
} else {
"frames"
};
Event {
ts_ms: now,
device: self.anchor_device(),
kind: EventKind::RecordingStopped,
severity: Severity::Info,
confidence: Confidence::Fact,
title: format!("recording stopped — clean shutdown after {}", fmt_dur(dur)),
evidence: format!(
"session started {}; {} {frames_word} folded over {}; the partial rollup \
tail was flushed; time between this mark and the next recording_started \
is gpuviewer not running, not the GPU sitting idle",
fmt_stamp(self.session_start_ms),
self.frames_folded,
fmt_dur(dur),
),
}
}
pub fn flush(&mut self) {
if let Some(rec) = &mut self.recorder {
rec.flush();
}
}
}
impl Drop for Engine {
fn drop(&mut self) {
let _ = self.finish();
}
}
pub struct EventSink {
cmd: String,
children: Vec<(Child, Instant)>,
window: Vec<Instant>,
warned: bool,
}
const SINK_MAX_PER_MIN: usize = 60;
const SINK_MAX_LIVE: usize = 16;
const SINK_HOOK_TIMEOUT: Duration = Duration::from_secs(60);
impl EventSink {
pub fn new(cmd: String) -> Self {
Self {
cmd,
children: Vec::new(),
window: Vec::new(),
warned: false,
}
}
pub fn fire(&mut self, event: &Event) {
self.reap();
let now = Instant::now();
self.window
.retain(|t| now.duration_since(*t) < Duration::from_secs(60));
if self.children.len() >= SINK_MAX_LIVE {
if !self.warned {
self.warned = true;
eprintln!(
"gpuviewer: --on-event has {SINK_MAX_LIVE} hooks still running — dropping \
event hooks until some exit (hooks are killed after {}s)",
SINK_HOOK_TIMEOUT.as_secs()
);
}
return;
}
if self.window.len() >= SINK_MAX_PER_MIN {
if !self.warned {
self.warned = true;
eprintln!(
"gpuviewer: --on-event rate cap ({SINK_MAX_PER_MIN}/min) hit — dropping \
further event hooks this minute"
);
}
return;
}
self.warned = false;
let json = serde_json::to_string(event).unwrap_or_default();
#[cfg(windows)]
let mut command = {
use std::os::windows::process::CommandExt;
let mut c = Command::new("cmd");
c.arg("/C");
c.raw_arg(&self.cmd);
c
};
#[cfg(not(windows))]
let mut command = {
let mut c = Command::new("sh");
c.args(["-c", &self.cmd]);
c
};
command
.env("GPV_EVENT_KIND", env_token(event.kind))
.env("GPV_EVENT_SEVERITY", env_token(event.severity))
.env("GPV_EVENT_CONFIDENCE", env_token(event.confidence))
.env("GPV_EVENT_TITLE", &event.title)
.env("GPV_EVENT_EVIDENCE", &event.evidence)
.env("GPV_EVENT_DEVICE", &event.device.0)
.env("GPV_EVENT_TS_MS", event.ts_ms.to_string())
.env("GPV_EVENT_JSON", json);
match command.spawn() {
Ok(child) => {
self.window.push(now);
self.children.push((child, now));
}
Err(e) => eprintln!("gpuviewer: --on-event spawn failed: {e}"),
}
}
fn reap(&mut self) {
let now = Instant::now();
self.children.retain_mut(|(c, spawned)| {
if matches!(c.try_wait(), Ok(Some(_))) {
return false;
}
if now.duration_since(*spawned) >= SINK_HOOK_TIMEOUT {
let _ = c.kill();
let _ = c.wait();
return false;
}
true
});
}
#[cfg(all(test, unix))]
fn live(&self) -> usize {
self.children.len()
}
#[cfg(test)]
fn wait_all(&mut self) {
for (c, _) in &mut self.children {
let _ = c.wait();
}
self.children.clear();
}
#[cfg(all(test, unix))]
fn kill_all(&mut self) {
for (c, _) in &mut self.children {
let _ = c.kill();
let _ = c.wait();
}
self.children.clear();
}
}
impl Drop for EventSink {
fn drop(&mut self) {
self.reap();
}
}
fn env_token<T: Serialize>(v: T) -> String {
serde_json::to_string(&v)
.ok()
.map(|s| s.trim_matches('"').to_string())
.unwrap_or_default()
}
fn fmt_dur(d: Duration) -> String {
let ms = d.as_millis() as u64;
if ms < 10_000 {
format!("{:.1}s", ms as f64 / 1000.0)
} else {
let s = (ms + 500) / 1000;
if s >= 60 {
format!("{}m {}s", s / 60, s % 60)
} else {
format!("{s}s")
}
}
}
fn fmt_clock(ms: u64) -> String {
use chrono::{Local, TimeZone};
Local
.timestamp_millis_opt(ms as i64)
.single()
.map(|t| t.format("%H:%M:%S").to_string())
.unwrap_or_else(|| "--:--:--".into())
}
fn fmt_stamp(ms: u64) -> String {
use chrono::{Local, TimeZone};
Local
.timestamp_millis_opt(ms as i64)
.single()
.map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "????-??-?? --:--:--".into())
}
pub fn device_is_idle(
prev: Option<&DynamicSample>,
sample: Option<&DynamicSample>,
procs: &[ProcessSample],
) -> bool {
let Some(s) = sample else { return false };
if procs
.iter()
.any(|p| p.util_pct.map(|u| u >= IDLE_UTIL_PCT).unwrap_or(false))
{
return false;
}
match s.util_pct {
Some(util) => util < IDLE_UTIL_PCT,
None => !util_less_activity(prev, s),
}
}
fn util_less_activity(prev: Option<&DynamicSample>, s: &DynamicSample) -> bool {
if s.encoder_pct.map(|u| u >= IDLE_UTIL_PCT).unwrap_or(false)
|| s.decoder_pct.map(|u| u >= IDLE_UTIL_PCT).unwrap_or(false)
{
return true;
}
if s.throttle.is_some_and(|t| t.any()) {
return true;
}
let Some(p) = prev else { return false };
if let (Some(a), Some(b)) = (p.mem_used_bytes, s.mem_used_bytes) {
if a.abs_diff(b) >= IDLE_MEM_DELTA_BYTES {
return true;
}
}
match (p.sm_clock_mhz, s.sm_clock_mhz) {
(Some(a), Some(b)) => a.abs_diff(b) > IDLE_CLOCK_JITTER_MHZ,
(None, Some(_)) => true,
_ => false,
}
}
const IDLE_UTIL_PCT: f32 = 5.0;
const IDLE_MEM_DELTA_BYTES: u64 = 4 * 1024 * 1024;
const IDLE_CLOCK_JITTER_MHZ: u32 = 25;
pub const BACKOFF_AFTER_IDLE_TICKS: u32 = 60;
const BACKOFF_MULTIPLIER: u32 = 5;
const BACKOFF_CAP: Duration = Duration::from_secs(10);
pub fn effective_interval(interval: Duration, idle_streak: u32) -> Duration {
if idle_streak >= BACKOFF_AFTER_IDLE_TICKS {
(interval * BACKOFF_MULTIPLIER).min(BACKOFF_CAP)
} else {
interval
}
}
pub struct Backoff {
enabled: bool,
interval: Duration,
idle_streak: u32,
prev: Vec<Option<DynamicSample>>,
}
impl Backoff {
pub fn new(enabled: bool, interval: Duration, devices: usize) -> Self {
Self {
enabled,
interval,
idle_streak: 0,
prev: vec![None; devices],
}
}
pub fn observe(&mut self, frame: &Frame) -> Duration {
let mut all_idle = true;
for (i, fd) in frame.devices.iter().enumerate() {
let prev = self.prev.get(i).and_then(Option::as_ref);
let procs = fd.processes.as_deref().unwrap_or(&[]);
if !device_is_idle(prev, fd.sample.as_ref(), procs) {
all_idle = false;
}
if let (Some(slot), Some(s)) = (self.prev.get_mut(i), &fd.sample) {
*slot = Some(s.clone());
}
}
if !self.enabled {
return self.interval;
}
if all_idle {
self.idle_streak = self.idle_streak.saturating_add(1);
} else {
self.idle_streak = 0;
}
effective_interval(self.interval, self.idle_streak)
}
pub fn skip(&mut self) -> Duration {
self.idle_streak = 0;
self.interval
}
}
#[derive(Clone)]
pub struct CollectorStop {
pub at_ms: u64,
pub reason: String,
}
pub struct Shared {
pub infos: Vec<StaticInfo>,
pub latest: Vec<Option<DynamicSample>>,
pub processes: Vec<Vec<ProcessSample>>,
pub history: HistoryStore,
pub mock: bool,
pub db_path: Option<PathBuf>,
pub interval_ms: u64,
pub effective_interval_ms: Arc<AtomicU64>,
pub tick_seq: u64,
pub stopped: Option<CollectorStop>,
pub lost: Vec<Option<u64>>,
}
pub struct Collector {
pub shared: Arc<Mutex<Shared>>,
pub paused: Arc<AtomicBool>,
stop: Arc<AtomicBool>,
handle: Option<std::thread::JoinHandle<()>>,
}
const SLEEP_SLICE: Duration = Duration::from_millis(100);
const SHUTDOWN_GRACE: Duration = Duration::from_secs(2);
impl Collector {
pub fn start(mut engine: Engine, interval: Duration, backoff: bool) -> Self {
let infos = engine.static_infos();
let n = infos.len();
let mock = engine.mock_in_use();
let db_path = engine.db_path();
let effective_interval_ms = Arc::new(AtomicU64::new(interval.as_millis() as u64));
let shared = Arc::new(Mutex::new(Shared {
infos,
latest: vec![None; n],
processes: vec![Vec::new(); n],
history: HistoryStore::new(1800, 5000),
mock,
db_path,
interval_ms: interval.as_millis() as u64,
effective_interval_ms: Arc::clone(&effective_interval_ms),
tick_seq: 0,
stopped: None,
lost: vec![None; n],
}));
let paused = Arc::new(AtomicBool::new(false));
let stop = Arc::new(AtomicBool::new(false));
let s = Arc::clone(&shared);
let p = Arc::clone(&paused);
let st = Arc::clone(&stop);
let handle = std::thread::spawn(move || {
let mut cadence = Backoff::new(backoff, interval, n);
loop {
if st.load(Ordering::Relaxed) {
return;
}
let effective = if !p.load(Ordering::Relaxed) {
match engine.tick_guarded() {
TickOutcome::Frame(frame) => {
let effective = cadence.observe(&frame);
engine.set_cadence(effective);
let mut sh = s.lock().unwrap();
for (i, fd) in frame.devices.iter().enumerate() {
if let Some(sample) = &fd.sample {
sh.history.push_sample(&fd.id, sample.clone());
sh.latest[i] = Some(sample.clone());
if let Some(procs) = &fd.processes {
sh.processes[i] = procs.clone();
}
}
}
for e in &frame.events {
let mark = match e.kind {
EventKind::DeviceLost => Some(Some(e.ts_ms)),
EventKind::DeviceReturned => Some(None),
_ => None,
};
if let Some(mark) = mark {
if let Some(i) =
sh.infos.iter().position(|inf| inf.id == e.device)
{
sh.lost[i] = mark;
}
}
}
sh.history.push_events(frame.events);
sh.tick_seq += 1;
effective
}
TickOutcome::Panicked {
event,
summary,
fatal,
} => {
let mut sh = s.lock().unwrap();
sh.history.push_events([event]);
sh.tick_seq += 1;
if fatal {
sh.stopped = Some(CollectorStop {
at_ms: now_ms(),
reason: summary,
});
return;
}
cadence.skip()
}
}
} else {
engine.note_paused();
let skip = cadence.skip();
engine.set_cadence(skip);
skip
};
effective_interval_ms.store(effective.as_millis() as u64, Ordering::Relaxed);
let deadline = Instant::now() + effective;
loop {
if st.load(Ordering::Relaxed) {
return;
}
let now = Instant::now();
if now >= deadline {
break;
}
std::thread::sleep((deadline - now).min(SLEEP_SLICE));
}
}
});
Self {
shared,
paused,
stop,
handle: Some(handle),
}
}
pub fn shutdown(&mut self) {
self.stop.store(true, Ordering::Relaxed);
let Some(handle) = self.handle.take() else {
return;
};
let deadline = Instant::now() + SHUTDOWN_GRACE;
while !handle.is_finished() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
if handle.is_finished() {
let _ = handle.join();
} else {
eprintln!(
"gpuviewer: the collector did not stop within {}s (a backend probe is \
likely blocked); exiting without the recording's stop mark — the next \
session will report it",
SHUTDOWN_GRACE.as_secs()
);
}
}
pub fn stationary(infos: Vec<StaticInfo>, db_path: Option<PathBuf>) -> Self {
let n = infos.len();
Self {
shared: Arc::new(Mutex::new(Shared {
infos,
latest: vec![None; n],
processes: vec![Vec::new(); n],
history: HistoryStore::new(1800, 5000),
mock: false,
db_path,
interval_ms: 1000,
effective_interval_ms: Arc::new(AtomicU64::new(1000)),
tick_seq: 0,
stopped: None,
lost: vec![None; n],
})),
paused: Arc::new(AtomicBool::new(false)),
stop: Arc::new(AtomicBool::new(false)),
handle: None,
}
}
}
#[cfg(test)]
pub(crate) fn test_collector(db_path: Option<PathBuf>) -> Collector {
let engine = Engine::new(EngineConfig {
force_mock: true,
..Default::default()
});
let collector = Collector::stationary(engine.static_infos(), db_path);
collector.shared.lock().unwrap().mock = true;
collector
}
#[cfg(test)]
mod tests {
use super::*;
use gpuviewer_core::{BackendError, ProcessKind, ThrottleReasons, Vendor};
use gpuviewer_history::Tier;
fn sample(util: Option<f32>) -> DynamicSample {
DynamicSample {
ts_ms: 1_000,
util_pct: util,
util_engine: None,
mem_used_bytes: None,
power_mw: None,
temp_c: None,
fan_pct: None,
sm_clock_mhz: None,
mem_clock_mhz: None,
encoder_pct: None,
decoder_pct: None,
throttle: Some(ThrottleReasons::default()),
power_limit_mw: None,
pcie_gen: None,
pcie_width: None,
ecc_uncorrected: None,
}
}
fn proc(util: Option<f32>) -> ProcessSample {
ProcessSample {
pid: 1,
name: "x".into(),
kind: ProcessKind::Compute,
mem_bytes: None,
util_pct: util,
cpu_pct: None,
container: None,
cmdline: None,
}
}
const GIB: u64 = 1024 * 1024 * 1024;
fn frame_of(devs: Vec<(Option<DynamicSample>, Vec<ProcessSample>)>) -> Frame {
Frame {
ts_ms: 0,
devices: devs
.into_iter()
.enumerate()
.map(|(i, (sample, processes))| FrameDevice {
id: DeviceId(format!("test:{i}")),
name: format!("dev{i}"),
mem_total_bytes: None,
sample,
processes: Some(processes),
})
.collect(),
events: Vec::new(),
}
}
fn quiet_igpu() -> (Option<DynamicSample>, Vec<ProcessSample>) {
let mut s = sample(None);
s.sm_clock_mhz = Some(300);
(Some(s), vec![proc(Some(1.0))])
}
#[test]
fn stall_threshold_is_max_of_three_intervals_and_five_seconds() {
assert_eq!(
stall_threshold(Duration::from_millis(100)),
Duration::from_secs(5)
);
assert_eq!(
stall_threshold(Duration::from_secs(3)),
Duration::from_secs(9)
);
assert_eq!(
stall_threshold(Duration::from_secs(2)),
Duration::from_secs(6)
);
}
#[test]
fn is_stall_fires_only_strictly_past_the_threshold() {
let interval = Duration::from_secs(1); assert!(
!is_stall(Duration::from_secs(5), interval),
"at threshold: not yet"
);
assert!(!is_stall(Duration::from_secs(4), interval));
assert!(
is_stall(Duration::from_millis(5_001), interval),
"just over: stall"
);
assert!(is_stall(Duration::from_secs(30), interval));
}
#[test]
fn effective_interval_holds_until_the_idle_streak_threshold() {
let interval = Duration::from_secs(1);
assert_eq!(effective_interval(interval, 0), interval);
assert_eq!(
effective_interval(interval, BACKOFF_AFTER_IDLE_TICKS - 1),
interval
);
assert_eq!(
effective_interval(interval, BACKOFF_AFTER_IDLE_TICKS),
Duration::from_secs(5)
);
assert_eq!(
effective_interval(interval, BACKOFF_AFTER_IDLE_TICKS + 1000),
Duration::from_secs(5)
);
}
#[test]
fn effective_interval_caps_at_ten_seconds() {
assert_eq!(
effective_interval(Duration::from_secs(3), BACKOFF_AFTER_IDLE_TICKS),
Duration::from_secs(10)
);
}
#[test]
fn device_is_idle_requires_low_device_and_process_util() {
assert!(device_is_idle(
None,
Some(&sample(Some(2.0))),
&[proc(Some(1.0))]
));
assert!(!device_is_idle(None, Some(&sample(Some(40.0))), &[]));
assert!(!device_is_idle(
None,
Some(&sample(Some(1.0))),
&[proc(Some(80.0))]
));
assert!(!device_is_idle(None, None, &[]));
}
#[test]
fn util_less_device_with_no_activity_signal_is_backoff_eligible() {
let mut bare = sample(None);
bare.throttle = None;
assert!(device_is_idle(None, Some(&bare), &[]));
assert!(device_is_idle(None, Some(&sample(None)), &[]));
assert!(device_is_idle(
Some(&sample(None)),
Some(&sample(None)),
&[proc(Some(1.0))]
));
assert!(device_is_idle(None, Some(&sample(None)), &[proc(None)]));
}
#[test]
fn util_less_device_with_genuine_activity_is_not_idle() {
assert!(!device_is_idle(
None,
Some(&sample(None)),
&[proc(Some(80.0))]
));
let mut prev = sample(None);
prev.mem_used_bytes = Some(GIB);
let mut cur = sample(None);
cur.mem_used_bytes = Some(GIB + IDLE_MEM_DELTA_BYTES);
assert!(!device_is_idle(Some(&prev), Some(&cur), &[]));
cur.mem_used_bytes = Some(GIB + IDLE_MEM_DELTA_BYTES - 1);
assert!(device_is_idle(Some(&prev), Some(&cur), &[]));
let mut prev = sample(None);
prev.sm_clock_mhz = Some(300);
let mut cur = sample(None);
cur.sm_clock_mhz = Some(900);
assert!(!device_is_idle(Some(&prev), Some(&cur), &[]));
cur.sm_clock_mhz = Some(300 + IDLE_CLOCK_JITTER_MHZ);
assert!(device_is_idle(Some(&prev), Some(&cur), &[]));
prev.sm_clock_mhz = None;
cur.sm_clock_mhz = Some(300);
assert!(!device_is_idle(Some(&prev), Some(&cur), &[]));
let mut enc = sample(None);
enc.encoder_pct = Some(60.0);
assert!(!device_is_idle(None, Some(&enc), &[]));
let mut thr = sample(None);
thr.throttle = Some(ThrottleReasons {
thermal: true,
..Default::default()
});
assert!(!device_is_idle(None, Some(&thr), &[]));
}
#[test]
fn always_none_util_device_does_not_pin_cadence() {
let interval = Duration::from_secs(1);
let mut cadence = Backoff::new(true, interval, 1);
for i in 1..BACKOFF_AFTER_IDLE_TICKS {
assert_eq!(
cadence.observe(&frame_of(vec![quiet_igpu()])),
interval,
"tick {i}: configured cadence until the streak threshold"
);
}
assert_eq!(
cadence.observe(&frame_of(vec![quiet_igpu()])),
Duration::from_secs(5),
"a quiet always-None-util device must reach the low-power cadence"
);
}
#[test]
fn active_device_holds_full_cadence() {
let interval = Duration::from_secs(1);
let mut cadence = Backoff::new(true, interval, 1);
for _ in 0..BACKOFF_AFTER_IDLE_TICKS * 2 {
let f = frame_of(vec![(Some(sample(Some(90.0))), vec![])]);
assert_eq!(
cadence.observe(&f),
interval,
"measured load: never stretch"
);
}
let mut cadence = Backoff::new(true, interval, 1);
let mut clock = 600u32;
for _ in 0..BACKOFF_AFTER_IDLE_TICKS * 2 {
clock = if clock == 600 { 1_100 } else { 600 };
let mut s = sample(None);
s.sm_clock_mhz = Some(clock);
assert_eq!(
cadence.observe(&frame_of(vec![(Some(s), vec![])])),
interval,
"util-less device with moving clocks: never stretch"
);
}
}
#[test]
fn busy_device_next_to_util_less_one_holds_full_cadence() {
let interval = Duration::from_secs(1);
let mut cadence = Backoff::new(true, interval, 2);
for _ in 0..BACKOFF_AFTER_IDLE_TICKS * 2 {
let f = frame_of(vec![(Some(sample(Some(95.0))), vec![]), quiet_igpu()]);
assert_eq!(cadence.observe(&f), interval);
}
}
#[test]
fn wake_on_util_less_device_snaps_cadence_back() {
let interval = Duration::from_secs(1);
let mut cadence = Backoff::new(true, interval, 1);
for _ in 0..BACKOFF_AFTER_IDLE_TICKS + 5 {
cadence.observe(&frame_of(vec![quiet_igpu()]));
}
assert_eq!(
cadence.observe(&frame_of(vec![quiet_igpu()])),
Duration::from_secs(5),
"precondition: well into low-power cadence"
);
let (s, _) = quiet_igpu();
let woke = frame_of(vec![(s, vec![proc(Some(40.0))])]);
assert_eq!(
cadence.observe(&woke),
interval,
"wake snaps back instantly"
);
}
#[test]
fn disabled_backoff_never_stretches() {
let interval = Duration::from_secs(1);
let mut cadence = Backoff::new(false, interval, 1);
for _ in 0..BACKOFF_AFTER_IDLE_TICKS * 2 {
assert_eq!(cadence.observe(&frame_of(vec![quiet_igpu()])), interval);
}
}
#[test]
fn paused_ticks_reset_the_streak() {
let interval = Duration::from_secs(1);
let mut cadence = Backoff::new(true, interval, 1);
for _ in 0..BACKOFF_AFTER_IDLE_TICKS - 1 {
cadence.observe(&frame_of(vec![quiet_igpu()]));
}
assert_eq!(cadence.skip(), interval);
assert_eq!(cadence.observe(&frame_of(vec![quiet_igpu()])), interval);
}
#[test]
fn event_sink_passes_event_through_environment() {
use std::sync::atomic::{AtomicU64, Ordering};
static SEQ: AtomicU64 = AtomicU64::new(0);
let n = SEQ.fetch_add(1, Ordering::Relaxed);
let out = std::env::temp_dir().join(format!(
"gpuviewer-onevent-test-{}-{n}.txt",
std::process::id()
));
let _ = std::fs::remove_file(&out);
#[cfg(windows)]
let cmd = format!("echo %GPV_EVENT_KIND%> \"{}\"", out.display());
#[cfg(not(windows))]
let cmd = format!("printenv GPV_EVENT_KIND > \"{}\"", out.display());
let mut sink = EventSink::new(cmd);
sink.fire(&Event {
ts_ms: 42,
device: DeviceId("0000:01:00.0".into()),
kind: EventKind::ThrottleStart,
severity: Severity::Warning,
confidence: Confidence::Fact,
title: "t".into(),
evidence: "e".into(),
});
let mut got = String::new();
for _ in 0..1000 {
if let Ok(s) = std::fs::read_to_string(&out) {
if !s.trim().is_empty() {
got = s;
break;
}
}
std::thread::sleep(Duration::from_millis(10));
}
let _ = std::fs::remove_file(&out);
assert_eq!(
got.trim(),
"throttle_start",
"the hook must see the event kind as the wire token in GPV_EVENT_KIND"
);
}
struct PanickyBackend<F: Fn(u32) -> bool + Send> {
tick: u32,
panics: F,
}
impl<F: Fn(u32) -> bool + Send> GpuBackend for PanickyBackend<F> {
fn name(&self) -> &'static str {
"panicky"
}
fn devices(&mut self) -> Vec<DeviceId> {
vec![DeviceId("panic:0".into())]
}
fn static_info(&mut self, dev: &DeviceId) -> Result<StaticInfo, BackendError> {
Ok(StaticInfo {
id: dev.clone(),
vendor: Vendor::Unknown,
name: "Panicky GPU".into(),
backend: "panicky".into(),
mem_total_bytes: Some(8 << 30),
power_limit_mw: None,
max_sm_clock_mhz: None,
temp_slowdown_c: None,
driver_version: None,
process_hint: None,
source_caveat: None,
pcie_max_gen: None,
pcie_max_width: None,
})
}
fn refresh_dynamic(&mut self, _dev: &DeviceId) -> Result<DynamicSample, BackendError> {
let t = self.tick;
self.tick += 1;
if (self.panics)(t) {
panic!("decoder choked on tick {t}");
}
Ok(sample(Some(50.0)))
}
fn refresh_processes(
&mut self,
_dev: &DeviceId,
) -> Result<Vec<ProcessSample>, BackendError> {
Ok(Vec::new())
}
}
fn panicky_engine(
db: Option<PathBuf>,
panics: impl Fn(u32) -> bool + Send + 'static,
) -> Engine {
Engine::with_backends(
vec![Box::new(PanickyBackend { tick: 0, panics })],
EngineConfig {
persist: db.is_some(),
db_path: db,
..Default::default()
},
)
}
fn scratch_db() -> PathBuf {
use std::sync::atomic::{AtomicU64, Ordering};
static SEQ: AtomicU64 = AtomicU64::new(0);
let n = SEQ.fetch_add(1, Ordering::Relaxed);
std::env::temp_dir().join(format!(
"gpuviewer-panic-test-{}-{n}.db",
std::process::id()
))
}
fn cleanup_db(path: &PathBuf) {
let _ = std::fs::remove_file(path);
for ext in ["-wal", "-shm", ".lock"] {
let mut p = path.as_os_str().to_os_string();
p.push(ext);
let _ = std::fs::remove_file(p);
}
}
#[test]
fn transient_tick_panic_narrates_and_collection_resumes() {
let mut engine = panicky_engine(None, |t| t == 0);
let TickOutcome::Panicked {
event,
summary,
fatal,
} = engine.tick_guarded()
else {
panic!("the scripted first tick must panic");
};
assert!(!fatal, "one panic is weather, not death");
assert_eq!(event.kind, EventKind::CollectorStall);
assert_eq!(event.confidence, Confidence::Fact);
assert_eq!(event.severity, Severity::Warning);
assert!(
summary.contains("decoder choked on tick 0"),
"the panic payload must survive into the summary: {summary}"
);
assert!(
event.evidence.contains("decoder choked on tick 0"),
"the payload must reach the event evidence: {}",
event.evidence
);
assert!(
event.evidence.contains("RUST_BACKTRACE"),
"the evidence must carry the backtrace hint: {}",
event.evidence
);
let TickOutcome::Frame(frame) = engine.tick_guarded() else {
panic!("the next tick is clean and must produce a frame");
};
assert!(
frame.devices[0].sample.is_some(),
"collection resumed with real data"
);
}
#[test]
fn panic_budget_resets_on_a_clean_tick() {
let mut engine = panicky_engine(None, |t| t != 1);
let fatals: Vec<bool> = (0..5)
.map(|_| match engine.tick_guarded() {
TickOutcome::Frame(_) => false,
TickOutcome::Panicked { fatal, .. } => fatal,
})
.collect();
assert_eq!(
fatals,
vec![false, false, false, false, true],
"fatal exactly when {MAX_CONSECUTIVE_PANICS} panics run consecutively"
);
}
#[test]
fn deterministic_panic_turns_fatal_with_a_critical_fact() {
let mut engine = panicky_engine(None, |_| true);
for i in 1..MAX_CONSECUTIVE_PANICS {
match engine.tick_guarded() {
TickOutcome::Panicked { fatal, .. } => {
assert!(!fatal, "panic {i} is within the budget")
}
TickOutcome::Frame(_) => panic!("scripted to panic"),
}
}
let TickOutcome::Panicked { event, fatal, .. } = engine.tick_guarded() else {
panic!("scripted to panic");
};
assert!(
fatal,
"the {MAX_CONSECUTIVE_PANICS}th consecutive panic is fatal"
);
assert_eq!(event.kind, EventKind::CollectorStall);
assert_eq!(event.severity, Severity::Critical);
assert_eq!(event.confidence, Confidence::Fact);
assert!(
event.title.contains("STOPPED"),
"the stop fact must say collection stopped: {}",
event.title
);
}
#[test]
fn panic_events_are_recorded_to_the_store() {
let path = scratch_db();
let mut engine = panicky_engine(Some(path.clone()), |_| true);
for _ in 0..MAX_CONSECUTIVE_PANICS {
assert!(matches!(
engine.tick_guarded(),
TickOutcome::Panicked { .. }
));
}
drop(engine);
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
let stalls: Vec<_> = events
.iter()
.filter(|e| e.kind == EventKind::CollectorStall)
.collect();
assert_eq!(
stalls.len(),
MAX_CONSECUTIVE_PANICS as usize,
"every panicked tick must persist its narration"
);
assert!(
stalls.iter().any(|e| e.severity == Severity::Critical),
"the stop itself is a Critical fact in the recording"
);
cleanup_db(&path);
}
#[test]
fn collector_thread_survives_panics_and_marks_shared_stopped() {
let engine = panicky_engine(None, |_| true);
let collector = Collector::start(engine, Duration::from_millis(5), false);
let mut stopped = None;
for _ in 0..400 {
if let Some(stop) = collector.shared.lock().unwrap().stopped.clone() {
stopped = Some(stop);
break;
}
std::thread::sleep(Duration::from_millis(10));
}
let stop = stopped.expect("the collector must declare itself dead, never vanish silently");
assert!(
stop.reason.contains("decoder choked"),
"the stop reason carries the panic summary: {}",
stop.reason
);
assert!(stop.at_ms > 0);
let sh = collector.shared.lock().unwrap();
let stalls: Vec<_> = sh
.history
.events()
.iter()
.filter(|e| e.kind == EventKind::CollectorStall)
.collect();
assert_eq!(
stalls.len(),
MAX_CONSECUTIVE_PANICS as usize,
"every panicked tick narrates into the story feed"
);
assert_eq!(
stalls.last().unwrap().severity,
Severity::Critical,
"the final narration is the Critical stop fact"
);
assert!(
sh.latest[0].is_none(),
"no fabricated sample may appear for a device that never answered"
);
assert!(sh.tick_seq >= MAX_CONSECUTIVE_PANICS as u64);
}
struct FlakyBackend<F: Fn(u32) -> bool + Send> {
tick: u32,
fails: F,
}
impl<F: Fn(u32) -> bool + Send> GpuBackend for FlakyBackend<F> {
fn name(&self) -> &'static str {
"flaky"
}
fn devices(&mut self) -> Vec<DeviceId> {
vec![DeviceId("flaky:0".into())]
}
fn static_info(&mut self, dev: &DeviceId) -> Result<StaticInfo, BackendError> {
Ok(StaticInfo {
id: dev.clone(),
vendor: Vendor::Unknown,
name: "Flaky GPU".into(),
backend: "flaky".into(),
mem_total_bytes: Some(8 << 30),
power_limit_mw: None,
max_sm_clock_mhz: None,
temp_slowdown_c: None,
driver_version: None,
process_hint: None,
source_caveat: None,
pcie_max_gen: None,
pcie_max_width: None,
})
}
fn refresh_dynamic(&mut self, _dev: &DeviceId) -> Result<DynamicSample, BackendError> {
let t = self.tick;
self.tick += 1;
if (self.fails)(t) {
Err(BackendError::Unavailable("probe timed out".into()))
} else {
let mut s = sample(Some(50.0));
s.ts_ms = 10_000 * (t as u64 + 1);
Ok(s)
}
}
fn refresh_processes(
&mut self,
_dev: &DeviceId,
) -> Result<Vec<ProcessSample>, BackendError> {
if self.tick > 0 && (self.fails)(self.tick - 1) {
Err(BackendError::Unavailable("probe timed out".into()))
} else {
Ok(vec![proc(Some(10.0))])
}
}
}
fn flaky_engine(db: Option<PathBuf>, fails: impl Fn(u32) -> bool + Send + 'static) -> Engine {
Engine::with_backends(
vec![Box::new(FlakyBackend { tick: 0, fails })],
EngineConfig {
persist: db.is_some(),
db_path: db,
..Default::default()
},
)
}
fn flaky_engine_persisting(path: &std::path::Path) -> Engine {
for _ in 0..500 {
let engine = flaky_engine(Some(path.to_path_buf()), |_| false);
if engine.db_path().is_some() {
return engine;
}
drop(engine);
std::thread::sleep(Duration::from_millis(10));
}
panic!(
"instance lock on {} still held after 5s — a real lock leak, not the \
fork-inheritance window",
path.display()
);
}
#[test]
fn device_lifecycle_kinds_use_documented_wire_tokens() {
assert_eq!(env_token(EventKind::DeviceLost), "device_lost");
assert_eq!(env_token(EventKind::DeviceReturned), "device_returned");
}
#[test]
fn device_lost_fires_exactly_at_the_threshold_with_evidence() {
const N: u32 = DEVICE_LOST_AFTER_FAILED_PROBES;
let mut engine = flaky_engine(None, |t| t >= 2);
let lost_tick = 2 + N - 1;
for t in 0..(lost_tick + 4) {
let TickOutcome::Frame(frame) = engine.tick_guarded() else {
panic!("flaky ticks never panic");
};
assert_eq!(frame.devices.len(), 1, "a lost device must stay listed");
assert_eq!(frame.devices[0].sample.is_some(), t < 2);
let lost: Vec<&Event> = frame
.events
.iter()
.filter(|e| e.kind == EventKind::DeviceLost)
.collect();
if t == lost_tick {
assert_eq!(lost.len(), 1, "the {N}th consecutive failure declares it");
let e = lost[0];
assert_eq!(e.severity, Severity::Critical);
assert_eq!(e.confidence, Confidence::Fact, "the silence is observed");
assert_eq!(e.device.0, "flaky:0");
assert!(
e.title.contains("GPU0") && e.title.contains("device lost"),
"title must name the device and the loss: {}",
e.title
);
assert!(
!e.title.contains("likely"),
"a fact must not read hedged: {}",
e.title
);
assert!(
e.evidence.contains(&format!("{N} consecutive")),
"evidence must carry the tick count: {}",
e.evidence
);
assert!(
e.evidence.contains("20000 ms"),
"evidence must carry the last-good timestamp: {}",
e.evidence
);
assert!(
e.evidence.contains("probe timed out"),
"evidence must carry the driver's error string: {}",
e.evidence
);
assert!(
e.evidence.contains("cause not asserted"),
"the cause must be explicitly unclaimed: {}",
e.evidence
);
} else {
assert!(
lost.is_empty(),
"tick {t}: device_lost only at the threshold tick"
);
}
assert!(
frame
.events
.iter()
.all(|e| e.kind != EventKind::DeviceReturned),
"the device never recovers in this script"
);
}
}
#[test]
fn transient_probe_failures_below_threshold_stay_silent() {
const N: u32 = DEVICE_LOST_AFTER_FAILED_PROBES;
let mut engine = flaky_engine(None, move |t| (2..2 + N - 1).contains(&t));
for _ in 0..12 {
let TickOutcome::Frame(frame) = engine.tick_guarded() else {
panic!("flaky ticks never panic");
};
assert!(
frame.events.iter().all(|e| {
e.kind != EventKind::DeviceLost && e.kind != EventKind::DeviceReturned
}),
"a sub-threshold blip must not narrate device loss"
);
}
}
#[test]
fn device_returned_closes_the_story_and_a_second_outage_reopens_it() {
const N: u32 = DEVICE_LOST_AFTER_FAILED_PROBES;
assert_eq!(
N, 5,
"the scripted outage windows below assume the threshold"
);
let mut engine = flaky_engine(None, |t| matches!(t, 2..=6 | 9..=13));
let mut lost = Vec::new();
let mut returned = Vec::new();
for _ in 0..16 {
let TickOutcome::Frame(frame) = engine.tick_guarded() else {
panic!("flaky ticks never panic");
};
for e in frame.events {
match e.kind {
EventKind::DeviceLost => lost.push(e),
EventKind::DeviceReturned => returned.push(e),
_ => {}
}
}
}
assert_eq!(lost.len(), 2, "each outage narrates its own loss");
assert_eq!(returned.len(), 2, "each recovery narrates its own return");
let r = &returned[0];
assert_eq!(
r.severity,
Severity::Info,
"recovery is good news, not an alarm"
);
assert_eq!(r.confidence, Confidence::Fact);
assert!(
r.title.contains("GPU0") && r.title.contains("returned"),
"return title must name the device: {}",
r.title
);
assert!(
r.title.contains("data are missing"),
"the return must size the hole it closes: {}",
r.title
);
assert!(
r.evidence.contains("never zero-filled"),
"the return must state the gap stays blank: {}",
r.evidence
);
}
#[test]
fn lost_device_marks_shared_and_recovery_clears_it() {
let engine = flaky_engine(None, |t| (3..100).contains(&t));
let collector = Collector::start(engine, Duration::from_millis(5), false);
let mut lost_seen = false;
for _ in 0..1000 {
let sh = collector.shared.lock().unwrap();
if sh.lost[0].is_some() {
lost_seen = true;
assert!(sh.stopped.is_none(), "device loss is not a collector stop");
assert!(
!sh.processes[0].is_empty(),
"the last good process list stays frozen, never flashes empty"
);
assert!(sh.latest[0].is_some(), "the last good sample is kept");
break;
}
drop(sh);
std::thread::sleep(Duration::from_millis(2));
}
assert!(lost_seen, "the device loss must reach Shared");
let mut cleared = false;
for _ in 0..2000 {
let sh = collector.shared.lock().unwrap();
if sh.lost[0].is_none() {
cleared = true;
break;
}
drop(sh);
std::thread::sleep(Duration::from_millis(5));
}
assert!(cleared, "recovery must clear the per-device marker");
let sh = collector.shared.lock().unwrap();
let kinds: Vec<EventKind> = sh.history.events().iter().map(|e| e.kind).collect();
assert!(kinds.contains(&EventKind::DeviceLost));
assert!(kinds.contains(&EventKind::DeviceReturned));
}
#[test]
fn device_lifecycle_events_persist_and_the_gap_has_no_rollups() {
const N: u32 = DEVICE_LOST_AFTER_FAILED_PROBES;
let path = scratch_db();
let mut engine = flaky_engine(Some(path.clone()), move |t| (1..=N).contains(&t));
for _ in 0..(N + 2) {
assert!(matches!(engine.tick_guarded(), TickOutcome::Frame(_)));
}
drop(engine);
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
assert_eq!(
events
.iter()
.filter(|e| e.kind == EventKind::DeviceLost)
.count(),
1,
"the loss is recorded exactly once"
);
assert_eq!(
events
.iter()
.filter(|e| e.kind == EventKind::DeviceReturned)
.count(),
1,
"the return is recorded exactly once"
);
let buckets: Vec<u64> = store
.samples_between(&DeviceId("flaky:0".into()), 0, 1_000_000, Tier::TenSec)
.unwrap()
.iter()
.map(|r| r.bucket_ms)
.collect();
assert_eq!(
buckets,
vec![10_000, 10_000 * (N as u64 + 2)],
"only the good ticks produced rollups — the lost stretch is a blank gap"
);
cleanup_db(&path);
}
struct SluggishBackend<F: Fn(u32) -> bool + Send> {
tick: u32,
slow: F,
}
impl<F: Fn(u32) -> bool + Send> GpuBackend for SluggishBackend<F> {
fn name(&self) -> &'static str {
"sluggish"
}
fn devices(&mut self) -> Vec<DeviceId> {
vec![DeviceId("sluggish:0".into())]
}
fn static_info(&mut self, dev: &DeviceId) -> Result<StaticInfo, BackendError> {
Ok(StaticInfo {
id: dev.clone(),
vendor: Vendor::Unknown,
name: "Sluggish GPU".into(),
backend: "sluggish".into(),
mem_total_bytes: Some(8 << 30),
power_limit_mw: None,
max_sm_clock_mhz: None,
temp_slowdown_c: None,
driver_version: None,
process_hint: None,
source_caveat: None,
pcie_max_gen: None,
pcie_max_width: None,
})
}
fn refresh_dynamic(&mut self, _dev: &DeviceId) -> Result<DynamicSample, BackendError> {
let t = self.tick;
self.tick += 1;
if (self.slow)(t) {
std::thread::sleep(SLOW_PROBE + Duration::from_millis(50));
}
Ok(sample(Some(50.0)))
}
fn refresh_processes(
&mut self,
_dev: &DeviceId,
) -> Result<Vec<ProcessSample>, BackendError> {
Ok(Vec::new())
}
}
fn sluggish_engine(slow: impl Fn(u32) -> bool + Send + 'static) -> Engine {
Engine::with_backends(
vec![Box::new(SluggishBackend { tick: 0, slow })],
EngineConfig::default(),
)
}
#[test]
fn inter_tick_gap_past_threshold_narrates_a_collector_stall() {
let mut engine = flaky_engine(None, |_| false);
let TickOutcome::Frame(first) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
first
.events
.iter()
.all(|e| e.kind != EventKind::CollectorStall),
"no stall may be narrated before a gap was ever measurable"
);
let past = Instant::now()
.checked_sub(Duration::from_secs(6))
.expect("system uptime exceeds 6s");
engine.set_last_tick_end(past);
let TickOutcome::Frame(stalled) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
let stalls: Vec<&Event> = stalled
.events
.iter()
.filter(|e| e.kind == EventKind::CollectorStall)
.collect();
assert_eq!(stalls.len(), 1, "the gap must be narrated exactly once");
let e = stalls[0];
assert_eq!(
e.severity,
Severity::Warning,
"a recording hole is a warning"
);
assert_eq!(
e.confidence,
Confidence::Fact,
"the gap was measured, not inferred"
);
assert_eq!(
e.device.0, "flaky:0",
"anchored to the first device like every collector-scoped fact"
);
assert!(
e.title.contains("collection stalled"),
"title must say collection stalled: {}",
e.title
);
assert!(
e.title.contains("last good frame at"),
"title must place the last good frame in time: {}",
e.title
);
assert!(
e.evidence.contains("inter-tick gap") && e.evidence.contains("stall threshold"),
"evidence must carry the measured gap and the threshold it crossed: {}",
e.evidence
);
assert!(
stalled.devices[0].sample.is_some(),
"the narrating tick itself collected normally — the hole is behind it"
);
let TickOutcome::Frame(after) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
after
.events
.iter()
.all(|e| e.kind != EventKind::CollectorStall),
"a normal gap after a narrated stall must not re-narrate"
);
}
#[test]
fn a_backed_off_sleep_is_not_a_stall() {
let mut engine = flaky_engine(None, |_| false);
let _ = engine.tick_guarded();
engine.set_cadence(Duration::from_secs(5));
engine.set_last_tick_end(
Instant::now()
.checked_sub(Duration::from_millis(5_050))
.expect("uptime exceeds 6s"),
);
let TickOutcome::Frame(f) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
f.events.iter().all(|e| e.kind != EventKind::CollectorStall),
"a gap that matches the backoff cadence is the schedule, not a stall: {:?}",
f.events.iter().map(|e| &e.title).collect::<Vec<_>>()
);
engine.set_last_tick_end(
Instant::now()
.checked_sub(Duration::from_secs(16))
.expect("uptime exceeds 16s"),
);
let TickOutcome::Frame(f) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
let stall = f
.events
.iter()
.find(|e| e.kind == EventKind::CollectorStall)
.expect("a real hole past the cadence threshold must still be narrated");
assert!(
!stall.title.contains("probe blocked"),
"an inter-tick gap cannot be a blocked probe (probes run inside a tick): {}",
stall.title
);
assert!(
stall.evidence.contains("cadence 5.0s"),
"{}",
stall.evidence
);
}
#[test]
fn wall_clock_jump_without_monotonic_advance_narrates_a_suspend() {
let mut engine = flaky_engine(None, |_| false);
let _ = engine.tick_guarded();
engine.set_last_tick_end(
Instant::now()
.checked_sub(Duration::from_secs(1))
.expect("uptime exceeds 1s"),
);
engine.set_last_tick_end_wall(now_ms() - 3_600_000);
let TickOutcome::Frame(f) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
let sus: Vec<&Event> = f
.events
.iter()
.filter(|e| e.kind == EventKind::HostSuspended)
.collect();
assert_eq!(sus.len(), 1, "exactly one suspend narration");
let e = sus[0];
assert_eq!(
e.confidence,
Confidence::Likely,
"a clock step looks the same"
);
assert_eq!(e.severity, Severity::Info, "sleeping is normal life");
assert!(
e.title.contains("likely"),
"inference titles hedge: {}",
e.title
);
assert!(
e.evidence.contains("wall clock") && e.evidence.contains("monotonic"),
"{}",
e.evidence
);
assert!(
f.events.iter().all(|e| e.kind != EventKind::CollectorStall),
"a suspend is not a collector stall"
);
engine.set_last_tick_end(
Instant::now()
.checked_sub(Duration::from_secs(1))
.expect("uptime exceeds 1s"),
);
engine.set_last_tick_end_wall(now_ms() - 3_000);
let TickOutcome::Frame(f) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(f.events.iter().all(|e| e.kind != EventKind::HostSuspended));
}
#[test]
fn a_user_pause_is_narrated_as_a_pause_not_a_stall() {
let mut engine = flaky_engine(None, |_| false);
let _ = engine.tick_guarded();
engine.note_paused();
engine.set_last_tick_end(
Instant::now()
.checked_sub(Duration::from_secs(60))
.expect("uptime exceeds 60s"),
);
let TickOutcome::Frame(f) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
let holes: Vec<&Event> = f
.events
.iter()
.filter(|e| e.kind == EventKind::CollectorStall)
.collect();
assert_eq!(holes.len(), 1);
assert_eq!(holes[0].severity, Severity::Info);
assert!(holes[0].title.contains("paused"), "{}", holes[0].title);
let TickOutcome::Frame(f) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
f.events.iter().all(|e| e.kind != EventKind::CollectorStall),
"the pause narrates once"
);
}
#[test]
fn slow_probe_notes_once_per_cooldown_and_recovers_silently() {
let mut engine = sluggish_engine(|t| t <= 1);
let TickOutcome::Frame(first) = engine.tick_guarded() else {
panic!("sluggish ticks never panic");
};
let notes: Vec<&Event> = first
.events
.iter()
.filter(|e| e.kind == EventKind::CollectorStall)
.collect();
assert_eq!(notes.len(), 1, "a slow probe must be noted");
let e = notes[0];
assert_eq!(
e.severity,
Severity::Info,
"a slow probe is a foreshadowing note, not an alarm"
);
assert_eq!(
e.confidence,
Confidence::Fact,
"the probe duration was measured"
);
assert_eq!(
e.device.0, "sluggish:0",
"the note is anchored to the slow device itself"
);
assert!(
e.title.contains("probe took") && e.title.contains("driver slow to respond"),
"title must name the slowness and its shape: {}",
e.title
);
assert!(
e.title.contains("sluggish"),
"title must name which backend was slow: {}",
e.title
);
assert!(
e.evidence.contains("refresh_dynamic") && e.evidence.contains("threshold"),
"evidence must carry the probe and the threshold it crossed: {}",
e.evidence
);
assert!(
first.devices[0].sample.is_some(),
"the slow tick still collected — slow is not lost"
);
let TickOutcome::Frame(second) = engine.tick_guarded() else {
panic!("sluggish ticks never panic");
};
assert!(
second
.events
.iter()
.all(|e| e.kind != EventKind::CollectorStall),
"a persistently-slow driver must not flood the story feed"
);
assert!(second.devices[0].sample.is_some());
let TickOutcome::Frame(third) = engine.tick_guarded() else {
panic!("sluggish ticks never panic");
};
assert!(
third
.events
.iter()
.all(|e| e.kind != EventKind::CollectorStall),
"a recovered (fast) probe must stay silent"
);
assert!(third.devices[0].sample.is_some());
}
#[test]
fn corrupt_history_file_narrates_history_reset_on_the_first_tick() {
const GARBAGE: &[u8] = b"definitely not a sqlite database";
let path = scratch_db();
std::fs::write(&path, GARBAGE).unwrap();
let mut engine = flaky_engine(Some(path.clone()), |_| false);
let TickOutcome::Frame(first) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
let resets: Vec<usize> = first
.events
.iter()
.enumerate()
.filter(|(_, e)| e.kind == EventKind::HistoryReset)
.map(|(i, _)| i)
.collect();
assert_eq!(
resets.len(),
1,
"the quarantine must be narrated exactly once"
);
let e = &first.events[resets[0]];
assert_eq!(
e.severity,
Severity::Warning,
"lost history is a warning, not routine lifecycle"
);
assert_eq!(
e.confidence,
Confidence::Fact,
"the quarantine happened — observed, not inferred"
);
assert_eq!(
e.device.0, "flaky:0",
"anchored to the first device like every collector-scoped fact"
);
assert!(
e.title.contains("corrupt") && e.title.contains("started fresh"),
"title must state the loss and the recovery: {}",
e.title
);
assert!(
e.evidence.contains(".corrupt-"),
"evidence must point at the preserved quarantine file: {}",
e.evidence
);
assert!(
e.evidence.contains("fresh database"),
"evidence must state a fresh database was created: {}",
e.evidence
);
let start_pos = first
.events
.iter()
.position(|ev| ev.kind == EventKind::RecordingStarted)
.expect("the session's start mark rides the first frame too");
assert!(
resets[0] < start_pos,
"the reset narrates before this session's start mark — the amnesia \
predates the session"
);
let TickOutcome::Frame(second) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
second
.events
.iter()
.all(|e| e.kind != EventKind::HistoryReset),
"the reset is narrated once, on the first tick only"
);
drop(engine);
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
assert_eq!(
events
.iter()
.filter(|e| e.kind == EventKind::HistoryReset)
.count(),
1,
"the reset fact survives into replay and report"
);
assert!(
events.iter().any(|e| e.kind == EventKind::RecordingStarted),
"the fresh store carries this session's start mark"
);
assert!(
!store.devices().unwrap().is_empty(),
"the fresh store registered the devices"
);
drop(store);
let dir = path.parent().unwrap();
let prefix = format!("{}.corrupt-", path.file_name().unwrap().to_string_lossy());
let quarantined: Vec<PathBuf> = std::fs::read_dir(dir)
.unwrap()
.filter_map(|d| d.ok())
.map(|d| d.path())
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with(&prefix))
})
.collect();
assert!(
!quarantined.is_empty(),
"the corrupt file must be preserved for manual recovery, not deleted"
);
assert_eq!(
std::fs::read(&quarantined[0]).unwrap(),
GARBAGE,
"preservation means the original bytes survive untouched"
);
for q in quarantined {
let _ = std::fs::remove_file(q);
}
cleanup_db(&path);
}
#[test]
fn session_boundary_kinds_use_documented_wire_tokens() {
assert_eq!(env_token(EventKind::RecordingStarted), "recording_started");
assert_eq!(env_token(EventKind::RecordingStopped), "recording_stopped");
}
#[test]
fn session_start_and_stop_marks_land_in_the_store() {
let path = scratch_db();
let mut engine = flaky_engine(Some(path.clone()), |_| false);
for _ in 0..3 {
assert!(matches!(engine.tick_guarded(), TickOutcome::Frame(_)));
}
assert!(
engine.finish().is_some(),
"finish must return the stop mark it wrote"
);
drop(engine);
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
let starts: Vec<&Event> = events
.iter()
.filter(|e| e.kind == EventKind::RecordingStarted)
.collect();
let stops: Vec<&Event> = events
.iter()
.filter(|e| e.kind == EventKind::RecordingStopped)
.collect();
assert_eq!(
starts.len(),
1,
"exactly one start mark (the first tick's re-insert is deduped)"
);
assert_eq!(
stops.len(),
1,
"exactly one stop mark (finish is idempotent across Drop)"
);
let start = starts[0];
assert_eq!(start.severity, Severity::Info);
assert_eq!(start.confidence, Confidence::Fact);
assert_eq!(start.device.0, "flaky:0", "anchored like CollectorStall");
assert!(
start.title.contains("recording started"),
"title: {}",
start.title
);
assert!(
!start.title.contains("without a stop mark"),
"a fresh database has no predecessor to indict: {}",
start.title
);
assert!(
start.evidence.contains(env!("CARGO_PKG_VERSION")),
"evidence must carry the binary version: {}",
start.evidence
);
assert!(
start.evidence.contains("interval 1.0s"),
"evidence must carry the tick interval: {}",
start.evidence
);
assert!(
start.evidence.contains("flaky (1 device)"),
"evidence must carry backend names and device count: {}",
start.evidence
);
let db_name = path.file_name().unwrap().to_str().unwrap();
assert!(
start.evidence.contains(db_name),
"evidence must carry the db basename: {}",
start.evidence
);
let stop = stops[0];
assert_eq!(stop.severity, Severity::Info);
assert_eq!(stop.confidence, Confidence::Fact);
assert_eq!(stop.device.0, "flaky:0");
assert!(
stop.title.contains("recording stopped"),
"title: {}",
stop.title
);
assert!(
stop.evidence.contains("3 frames folded"),
"evidence must count the session's frames: {}",
stop.evidence
);
assert!(stop.ts_ms >= start.ts_ms, "the stop mark follows the start");
drop(store);
std::thread::sleep(Duration::from_millis(2));
drop(flaky_engine_persisting(&path));
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
let second_start = events
.iter()
.filter(|e| e.kind == EventKind::RecordingStarted)
.max_by_key(|e| e.ts_ms)
.unwrap();
assert!(
!second_start.title.contains("without a stop mark"),
"a cleanly-stopped predecessor must not be narrated as a crash: {}",
second_start.title
);
assert_eq!(
events
.iter()
.filter(|e| e.kind == EventKind::RecordingStopped)
.count(),
2,
"both sessions wrote their stop marks"
);
cleanup_db(&path);
}
#[test]
fn unclean_shutdown_is_narrated_by_the_next_start() {
let path = scratch_db();
{
let (mut store, _) = SqliteStore::open(&path).unwrap();
store
.insert_events(&[Event {
ts_ms: 1_000,
device: DeviceId("flaky:0".into()),
kind: EventKind::RecordingStarted,
severity: Severity::Info,
confidence: Confidence::Fact,
title: "recording started — gpuviewer (previous session)".into(),
evidence: "test fixture".into(),
}])
.unwrap();
}
drop(flaky_engine_persisting(&path));
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
let new_start = events
.iter()
.filter(|e| e.kind == EventKind::RecordingStarted)
.max_by_key(|e| e.ts_ms)
.expect("the new session must write its own start mark");
assert!(new_start.ts_ms > 1_000, "must be the NEW session's mark");
assert_eq!(
new_start.confidence,
Confidence::Fact,
"the missing stop mark is observed, not inferred"
);
assert!(
new_start
.title
.contains("previous session ended without a stop mark"),
"the title must narrate the unclean predecessor: {}",
new_start.title
);
assert!(
new_start.title.contains("crash, kill, or power loss"),
"the title names the possible causes without asserting one: {}",
new_start.title
);
assert!(
new_start.evidence.contains("gap size is unknowable"),
"the evidence must state the gap cannot be sized: {}",
new_start.evidence
);
cleanup_db(&path);
}
#[test]
fn start_mark_rides_the_first_frame_only() {
let path = scratch_db();
let mut engine = flaky_engine(Some(path.clone()), |_| false);
let TickOutcome::Frame(first) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert_eq!(
first
.events
.iter()
.filter(|e| e.kind == EventKind::RecordingStarted)
.count(),
1,
"the start mark must ride the first frame"
);
let TickOutcome::Frame(second) = engine.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
second
.events
.iter()
.all(|e| e.kind != EventKind::RecordingStarted),
"the start mark must not repeat"
);
drop(engine);
cleanup_db(&path);
let mut live_only = flaky_engine(None, |_| false);
let TickOutcome::Frame(frame) = live_only.tick_guarded() else {
panic!("scripted to never fail");
};
assert!(
frame.events.iter().all(|e| {
e.kind != EventKind::RecordingStarted && e.kind != EventKind::RecordingStopped
}),
"a live-only session records nothing, so it must not claim a recording began"
);
assert!(
live_only.finish().is_none(),
"no recorder, no stop mark to write"
);
}
#[test]
fn collector_shutdown_writes_the_stop_mark() {
let path = scratch_db();
let engine = flaky_engine(Some(path.clone()), |_| false);
let mut collector = Collector::start(engine, Duration::from_millis(5), false);
let mut ticked = false;
for _ in 0..1000 {
if collector.shared.lock().unwrap().tick_seq > 0 {
ticked = true;
break;
}
std::thread::sleep(Duration::from_millis(2));
}
assert!(ticked, "the collector must tick before the shutdown");
collector.shutdown();
let mut found = false;
for _ in 0..1000 {
let store = SqliteStore::open_readonly(&path).unwrap();
let events = store.events_between(0, now_ms() + 60_000).unwrap();
if events.iter().any(|e| e.kind == EventKind::RecordingStopped) {
found = true;
break;
}
drop(store);
std::thread::sleep(Duration::from_millis(5));
}
assert!(
found,
"a TUI-style quit must write the recording_stopped mark"
);
cleanup_db(&path);
}
#[test]
fn mock_fallback_engine_refuses_a_real_stamped_db_with_zero_writes() {
let path = scratch_db();
drop(SqliteStore::open_recording(&path, DataSource::Real).unwrap());
let engine = Engine::with_backends(
vec![Box::new(gpuviewer_core::mock::MockBackend::new())],
EngineConfig {
persist: true,
db_path: Some(path.clone()),
..Default::default()
},
);
assert!(engine.mock_in_use(), "precondition: the fallback shape");
assert!(
engine.db_path().is_none(),
"the engine must not hold a recorder on a mismatched database"
);
drop(engine);
let store = SqliteStore::open_readonly(&path).unwrap();
assert_eq!(
store.data_source().unwrap(),
Some(DataSource::Real),
"the stamp survives untouched"
);
assert!(
store.devices().unwrap().is_empty(),
"ZERO writes — not even device-identity rows may leak"
);
assert!(
store
.events_between(0, now_ms() + 60_000)
.unwrap()
.is_empty(),
"no events either — including session marks"
);
cleanup_db(&path);
}
#[test]
fn engine_stamps_a_fresh_db_with_the_backends_actual_source() {
let path = scratch_db();
let engine = Engine::with_backends(
vec![Box::new(gpuviewer_core::mock::MockBackend::new())],
EngineConfig {
persist: true,
db_path: Some(path.clone()),
..Default::default()
},
);
assert!(engine.db_path().is_some());
drop(engine);
let store = SqliteStore::open_readonly(&path).unwrap();
assert_eq!(store.data_source().unwrap(), Some(DataSource::Mock));
drop(store);
cleanup_db(&path);
let path = scratch_db();
drop(flaky_engine(Some(path.clone()), |_| false));
let store = SqliteStore::open_readonly(&path).unwrap();
assert_eq!(
store.data_source().unwrap(),
Some(DataSource::Real),
"a non-mock backend set claims real"
);
drop(store);
cleanup_db(&path);
}
#[test]
fn event_sink_rate_caps_spawns_per_minute() {
let mut sink = EventSink::new("true".into());
let ev = Event {
ts_ms: 1,
device: DeviceId("d".into()),
kind: EventKind::ProcessExited,
severity: Severity::Info,
confidence: Confidence::Fact,
title: "t".into(),
evidence: "e".into(),
};
for _ in 0..(SINK_MAX_PER_MIN + 25) {
sink.fire(&ev);
sink.wait_all();
}
assert_eq!(
sink.window.len(),
SINK_MAX_PER_MIN,
"spawns within the minute must be capped"
);
assert!(
sink.warned,
"exceeding the cap must set the one-time warning"
);
}
#[cfg(unix)]
#[test]
fn event_sink_caps_live_hooks_that_never_exit() {
let mut sink = EventSink::new("sleep 30".into());
let ev = Event {
ts_ms: 1,
device: DeviceId("d".into()),
kind: EventKind::ProcessExited,
severity: Severity::Info,
confidence: Confidence::Fact,
title: "t".into(),
evidence: "e".into(),
};
for _ in 0..(SINK_MAX_LIVE + 10) {
sink.fire(&ev);
}
assert_eq!(sink.live(), SINK_MAX_LIVE, "survivors are capped");
assert!(sink.warned, "hitting the live cap warns");
sink.kill_all();
}
#[test]
fn xid_records_become_attributed_facts_with_repeats_folded() {
use gpuviewer_core::kmsg::parse_xid;
let mut engine = flaky_engine(None, |_| false);
engine.devices[0].1 = DeviceId("0000:01:00.0".into());
let x79 = parse_xid(
"NVRM: Xid (PCI:0000:01:00): 79, pid=4521, name=python, GPU has fallen off the bus.",
)
.unwrap();
let other = parse_xid("NVRM: Xid (PCI:0000:02:00): 13, Graphics Exception").unwrap();
let ev = engine.inject_xid(vec![x79.clone(), other], 1_000);
assert_eq!(ev.len(), 1, "an Xid for an unmonitored device is dropped");
let e = &ev[0];
assert_eq!(e.kind, EventKind::XidError);
assert_eq!(e.confidence, Confidence::Fact);
assert_eq!(e.severity, Severity::Critical, "Xid 79 is fatal");
assert_eq!(
e.title,
"GPU0 driver reported Xid 79: GPU has fallen off the bus (python pid 4521)"
);
assert!(
e.evidence.contains("NVRM: Xid (PCI:0000:01:00): 79"),
"{}",
e.evidence
);
assert!(engine
.inject_xid(vec![x79.clone(), x79.clone()], 2_000)
.is_empty());
let ev = engine.inject_xid(vec![x79], 70_000);
assert_eq!(ev.len(), 1);
assert!(
ev[0].evidence.contains("2 earlier repeats"),
"{}",
ev[0].evidence
);
}
}