1use std::collections::{HashMap, VecDeque};
12
13use gpuviewer_core::{DeviceId, DynamicSample, Event, ProcessKind, ProcessSample};
14
15pub mod store;
16
17pub use store::{
18 DataSource, DeviceRow, ExportCounts, ProcessRollup, SampleRollup, SqliteStore, StoreError,
19 Tier, RETAIN_10S_MS, RETAIN_1M_MS, RETAIN_EVENTS_MS, SCHEMA_VERSION,
20};
21
22pub struct DeviceHistory {
24 samples: VecDeque<DynamicSample>,
25 cap: usize,
26}
27
28impl DeviceHistory {
29 pub fn new(cap: usize) -> Self {
30 Self {
31 samples: VecDeque::with_capacity(cap),
32 cap,
33 }
34 }
35
36 pub fn push(&mut self, s: DynamicSample) {
37 if self.samples.len() == self.cap {
38 self.samples.pop_front();
39 }
40 self.samples.push_back(s);
41 }
42
43 pub fn latest(&self) -> Option<&DynamicSample> {
44 self.samples.back()
45 }
46
47 pub fn iter(&self) -> impl Iterator<Item = &DynamicSample> {
48 self.samples.iter()
49 }
50
51 pub fn len(&self) -> usize {
52 self.samples.len()
53 }
54
55 pub fn is_empty(&self) -> bool {
56 self.samples.is_empty()
57 }
58}
59
60pub struct HistoryStore {
62 per_device: HashMap<DeviceId, DeviceHistory>,
63 events: Vec<Event>,
64 sample_cap: usize,
65 event_cap: usize,
66}
67
68impl HistoryStore {
69 pub fn new(sample_cap: usize, event_cap: usize) -> Self {
70 Self {
71 per_device: HashMap::new(),
72 events: Vec::new(),
73 sample_cap,
74 event_cap,
75 }
76 }
77
78 pub fn push_sample(&mut self, dev: &DeviceId, s: DynamicSample) {
79 self.per_device
80 .entry(dev.clone())
81 .or_insert_with(|| DeviceHistory::new(self.sample_cap))
82 .push(s);
83 }
84
85 pub fn push_events(&mut self, evs: impl IntoIterator<Item = Event>) {
86 self.events.extend(evs);
87 if self.events.len() > self.event_cap {
88 let drop = self.events.len() - self.event_cap;
89 self.events.drain(..drop);
90 }
91 }
92
93 pub fn device(&self, dev: &DeviceId) -> Option<&DeviceHistory> {
94 self.per_device.get(dev)
95 }
96
97 pub fn events(&self) -> &[Event] {
98 &self.events
99 }
100}
101
102const PRUNE_EVERY_FLUSHES: u64 = 360;
109
110#[derive(Clone, Copy, Default)]
114struct Agg {
115 min: Option<f64>,
116 max: Option<f64>,
117 sum: f64,
118 n: u32,
120}
121
122impl Agg {
123 fn push(&mut self, v: Option<f64>) {
124 let Some(v) = v else { return };
125 self.min = Some(self.min.map_or(v, |m| m.min(v)));
126 self.max = Some(self.max.map_or(v, |m| m.max(v)));
127 self.sum += v;
128 self.n += 1;
129 }
130
131 fn avg(&self) -> Option<f64> {
132 (self.n > 0).then(|| self.sum / self.n as f64)
133 }
134}
135
136#[derive(Default)]
140struct SampleAccum {
141 bucket_ms: Option<u64>,
143 n: u32,
145 util: Agg,
146 mem: Agg,
147 power: Agg,
148 temp: Agg,
149 fan: Agg,
150 sm_clock: Agg,
151 throttle_n: u32,
152 throttle_thermal_n: u32,
153 throttle_power_n: u32,
154 throttle_hw_n: u32,
155 throttle_observed_n: u32,
156}
157
158impl SampleAccum {
159 fn fold(&mut self, s: &DynamicSample) {
160 self.n += 1;
161 self.util.push(s.util_pct.map(|v| v as f64));
162 self.mem.push(s.mem_used_bytes.map(|v| v as f64));
163 self.power.push(s.power_mw.map(|v| v as f64));
164 self.temp.push(s.temp_c.map(|v| v as f64));
165 self.fan.push(s.fan_pct.map(|v| v as f64));
166 self.sm_clock.push(s.sm_clock_mhz.map(|v| v as f64));
167 self.throttle_observed_n += s.throttle.is_some() as u32;
174 if let Some(t) = s.throttle {
175 let (any, thermal, power_cap, hw) = store::throttle_flags(&t);
176 self.throttle_n += any as u32;
177 self.throttle_thermal_n += thermal as u32;
178 self.throttle_power_n += power_cap as u32;
179 self.throttle_hw_n += hw as u32;
180 }
181 }
182
183 fn to_rollup(&self, device_id: &DeviceId) -> Option<SampleRollup> {
184 let bucket_ms = self.bucket_ms?;
185 Some(SampleRollup {
186 device_id: device_id.clone(),
187 bucket_ms,
188 n: self.n,
189 util_min: self.util.min.map(|v| v as f32),
190 util_avg: self.util.avg().map(|v| v as f32),
191 util_max: self.util.max.map(|v| v as f32),
192 mem_avg: self.mem.avg().map(|v| v as u64),
193 mem_max: self.mem.max.map(|v| v as u64),
194 power_avg_mw: self.power.avg().map(|v| v as u32),
195 power_max_mw: self.power.max.map(|v| v as u32),
196 temp_avg_c: self.temp.avg().map(|v| v as f32),
197 temp_max_c: self.temp.max.map(|v| v as f32),
198 fan_max_pct: self.fan.max.map(|v| v as f32),
199 sm_clock_min: self.sm_clock.min.map(|v| v as u32),
200 sm_clock_avg: self.sm_clock.avg().map(|v| v as u32),
201 sm_clock_max: self.sm_clock.max.map(|v| v as u32),
202 throttle_n: self.throttle_n,
203 throttle_thermal_n: self.throttle_thermal_n,
204 throttle_power_n: self.throttle_power_n,
205 throttle_hw_n: self.throttle_hw_n,
206 throttle_observed_n: self.throttle_observed_n,
207 })
208 }
209
210 fn reset(&mut self, bucket_ms: u64) {
211 *self = SampleAccum::default();
212 self.bucket_ms = Some(bucket_ms);
213 }
214}
215
216struct ProcAccum {
221 name: String,
222 kind: ProcessKind,
223 mem_max: Option<u64>,
224 util: Agg,
225 cpu: Agg,
226 container: Option<String>,
227 cmdline: Option<String>,
230}
231
232impl ProcAccum {
233 fn new(p: &ProcessSample) -> Self {
234 let mut a = Self {
235 name: p.name.clone(),
236 kind: p.kind,
237 mem_max: None,
238 util: Agg::default(),
239 cpu: Agg::default(),
240 container: None,
241 cmdline: None,
242 };
243 a.fold(p);
244 a
245 }
246
247 fn fold(&mut self, p: &ProcessSample) {
248 self.name = p.name.clone();
249 self.kind = p.kind;
250 if let Some(m) = p.mem_bytes {
251 self.mem_max = Some(self.mem_max.map_or(m, |cur| cur.max(m)));
252 }
253 self.util.push(p.util_pct.map(|v| v as f64));
254 self.cpu.push(p.cpu_pct.map(|v| v as f64));
255 self.container = p.container.clone();
256 if p.cmdline.is_some() {
257 self.cmdline = p.cmdline.clone();
258 }
259 }
260
261 fn to_rollup(&self, device_id: &DeviceId, bucket_ms: u64, pid: u32) -> ProcessRollup {
262 ProcessRollup {
263 device_id: device_id.clone(),
264 bucket_ms,
265 pid,
266 name: self.name.clone(),
267 kind: self.kind,
268 mem_max: self.mem_max,
269 util_avg: self.util.avg().map(|v| v as f32),
270 cpu_avg: self.cpu.avg().map(|v| v as f32),
271 container: self.container.clone(),
272 cmdline: self.cmdline.clone(),
273 }
274 }
275}
276
277#[derive(Default)]
279struct ProcBucket {
280 bucket_ms: Option<u64>,
281 procs: HashMap<u32, ProcAccum>,
282}
283
284#[derive(Default)]
287struct DevRecord {
288 s10: SampleAccum,
289 s1m: SampleAccum,
290 p10: ProcBucket,
291}
292
293pub struct Recorder {
302 store: SqliteStore,
303 devices: HashMap<DeviceId, DevRecord>,
304 flushes_since_prune: u64,
307 newest_ts_ms: u64,
308 failing: bool,
313 fails_in_streak: u64,
314 streak_error: Option<String>,
315 pending: Option<WriteTransition>,
316}
317
318#[derive(Clone, Debug, PartialEq, Eq)]
322pub enum WriteTransition {
323 Degraded { error: String },
325 Recovered { lost: u64 },
327}
328
329impl Recorder {
330 pub fn new(store: SqliteStore) -> Self {
331 Self {
332 store,
333 devices: HashMap::new(),
334 flushes_since_prune: 0,
335 newest_ts_ms: 0,
336 failing: false,
337 fails_in_streak: 0,
338 streak_error: None,
339 pending: None,
340 }
341 }
342
343 fn note_write<T>(&mut self, r: Result<T, StoreError>) {
345 match r {
346 Ok(_) => {
347 if self.failing {
348 self.failing = false;
349 self.pending = Some(WriteTransition::Recovered {
350 lost: self.fails_in_streak,
351 });
352 self.fails_in_streak = 0;
353 self.streak_error = None;
354 }
355 }
356 Err(e) => {
357 self.fails_in_streak += 1;
358 if !self.failing {
359 self.failing = true;
360 let msg = e.to_string();
361 self.streak_error = Some(msg.clone());
362 self.pending = Some(WriteTransition::Degraded { error: msg });
363 }
364 }
365 }
366 }
367
368 pub fn take_write_transition(&mut self) -> Option<WriteTransition> {
371 self.pending.take()
372 }
373
374 pub fn is_degraded(&self) -> bool {
377 self.failing
378 }
379
380 pub fn store(&self) -> &SqliteStore {
382 &self.store
383 }
384
385 pub fn store_mut(&mut self) -> &mut SqliteStore {
387 &mut self.store
388 }
389
390 pub fn observe(&mut self, dev: &DeviceId, s: &DynamicSample, procs: &[ProcessSample]) {
394 self.newest_ts_ms = self.newest_ts_ms.max(s.ts_ms);
395 let b10 = s.ts_ms - s.ts_ms % Tier::TenSec.width_ms();
396 let b1m = s.ts_ms - s.ts_ms % Tier::OneMin.width_ms();
397
398 let mut sample_flushes: Vec<(Tier, SampleRollup)> = Vec::new();
400 let mut proc_flush: Option<Vec<ProcessRollup>> = None;
401 let mut crossed_10s = false;
402 {
403 let rec = self.devices.entry(dev.clone()).or_default();
404
405 if rec.s10.bucket_ms.is_some_and(|b| b10 > b) {
406 if let Some(r) = rec.s10.to_rollup(dev) {
407 sample_flushes.push((Tier::TenSec, r));
408 }
409 rec.s10.reset(b10);
410 crossed_10s = true;
411 } else if rec.s10.bucket_ms.is_none() {
412 rec.s10.bucket_ms = Some(b10);
413 }
414 rec.s10.fold(s);
415
416 if rec.s1m.bucket_ms.is_some_and(|b| b1m > b) {
417 if let Some(r) = rec.s1m.to_rollup(dev) {
418 sample_flushes.push((Tier::OneMin, r));
419 }
420 rec.s1m.reset(b1m);
421 } else if rec.s1m.bucket_ms.is_none() {
422 rec.s1m.bucket_ms = Some(b1m);
423 }
424 rec.s1m.fold(s);
425
426 if rec.p10.bucket_ms.is_some_and(|b| b10 > b) {
428 let bucket = rec.p10.bucket_ms.unwrap();
429 let rows: Vec<ProcessRollup> = rec
430 .p10
431 .procs
432 .iter()
433 .map(|(pid, pa)| pa.to_rollup(dev, bucket, *pid))
434 .collect();
435 proc_flush = Some(rows);
436 rec.p10 = ProcBucket {
437 bucket_ms: Some(b10),
438 procs: HashMap::new(),
439 };
440 } else if rec.p10.bucket_ms.is_none() {
441 rec.p10.bucket_ms = Some(b10);
442 }
443 for p in procs {
444 match rec.p10.procs.get_mut(&p.pid) {
445 Some(acc) => acc.fold(p),
446 None => {
447 rec.p10.procs.insert(p.pid, ProcAccum::new(p));
448 }
449 }
450 }
451 }
452
453 for (tier, rollup) in sample_flushes {
454 let r = self.store.insert_sample_rollups(tier, &[rollup]);
459 self.note_write(r);
460 }
461 if let Some(rows) = proc_flush {
462 let r = self.store.insert_process_rollups(&rows);
463 self.note_write(r);
464 }
465
466 if crossed_10s {
467 self.flushes_since_prune += 1;
468 if self.flushes_since_prune >= PRUNE_EVERY_FLUSHES {
469 self.flushes_since_prune = 0;
470 let r = self.store.prune(self.newest_ts_ms);
471 self.note_write(r);
472 }
473 }
474 }
475
476 pub fn record_events(&mut self, events: &[Event]) {
478 let r = self.store.insert_events(events);
479 self.note_write(r);
480 }
481
482 pub fn flush(&mut self) {
486 let mut samples: Vec<(Tier, SampleRollup)> = Vec::new();
487 let mut procs: Vec<ProcessRollup> = Vec::new();
488 for (dev, rec) in &mut self.devices {
489 if let Some(r) = rec.s10.to_rollup(dev) {
490 if r.n > 0 {
491 samples.push((Tier::TenSec, r));
492 }
493 }
494 if let Some(r) = rec.s1m.to_rollup(dev) {
495 if r.n > 0 {
496 samples.push((Tier::OneMin, r));
497 }
498 }
499 if let Some(bucket) = rec.p10.bucket_ms {
500 for (pid, pa) in &rec.p10.procs {
501 procs.push(pa.to_rollup(dev, bucket, *pid));
502 }
503 }
504 rec.s10 = SampleAccum::default();
505 rec.s1m = SampleAccum::default();
506 rec.p10 = ProcBucket::default();
507 }
508 let tens: Vec<SampleRollup> = samples
509 .iter()
510 .filter(|(t, _)| *t == Tier::TenSec)
511 .map(|(_, r)| r.clone())
512 .collect();
513 let mins: Vec<SampleRollup> = samples
514 .iter()
515 .filter(|(t, _)| *t == Tier::OneMin)
516 .map(|(_, r)| r.clone())
517 .collect();
518 let r = self.store.insert_sample_rollups(Tier::TenSec, &tens);
519 self.note_write(r);
520 let r = self.store.insert_sample_rollups(Tier::OneMin, &mins);
521 self.note_write(r);
522 let r = self.store.insert_process_rollups(&procs);
523 self.note_write(r);
524 }
525}
526
527#[cfg(test)]
528mod tests {
529 use super::*;
530 use gpuviewer_core::ThrottleReasons;
531
532 fn sample(ts: u64) -> DynamicSample {
533 DynamicSample {
534 ts_ms: ts,
535 util_pct: Some(1.0),
536 util_engine: None,
537 mem_used_bytes: None,
538 power_mw: None,
539 temp_c: None,
540 fan_pct: None,
541 sm_clock_mhz: None,
542 mem_clock_mhz: None,
543 encoder_pct: None,
544 decoder_pct: None,
545 throttle: Some(ThrottleReasons::default()),
546 power_limit_mw: None,
547 pcie_gen: None,
548 pcie_width: None,
549 ecc_uncorrected: None,
550 }
551 }
552
553 #[test]
554 fn ring_caps_at_capacity() {
555 let mut h = DeviceHistory::new(3);
556 for i in 0..10 {
557 h.push(sample(i));
558 }
559 assert_eq!(h.len(), 3);
560 assert_eq!(h.latest().unwrap().ts_ms, 9);
561 assert_eq!(h.iter().next().unwrap().ts_ms, 7);
562 }
563
564 use gpuviewer_core::{
570 Confidence, Event, EventKind, ProcessKind, ProcessSample, Severity, Vendor,
571 };
572 use std::sync::atomic::{AtomicU64, Ordering};
573
574 static SEQ: AtomicU64 = AtomicU64::new(0);
575
576 struct Scratch {
579 path: std::path::PathBuf,
580 }
581
582 impl Scratch {
583 fn new() -> Self {
584 let n = SEQ.fetch_add(1, Ordering::Relaxed);
585 let name = format!("gpuviewer-hist-test-{}-{}.db", std::process::id(), n);
586 Scratch {
587 path: std::env::temp_dir().join(name),
588 }
589 }
590 }
591
592 impl Drop for Scratch {
593 fn drop(&mut self) {
594 let _ = std::fs::remove_file(&self.path);
595 for ext in ["-wal", "-shm", ".lock"] {
596 let mut p = self.path.as_os_str().to_os_string();
597 p.push(ext);
598 let _ = std::fs::remove_file(p);
599 }
600 if let Some(dir) = self.path.parent() {
602 if let Some(stem) = self.path.file_name().and_then(|s| s.to_str()) {
603 if let Ok(rd) = std::fs::read_dir(dir) {
604 for e in rd.flatten() {
605 if let Some(n) = e.file_name().to_str() {
606 if n.starts_with(&format!("{stem}.corrupt-")) {
607 let _ = std::fs::remove_file(e.path());
608 }
609 }
610 }
611 }
612 }
613 }
614 }
615 }
616
617 fn dev(id: &str) -> DeviceId {
618 DeviceId(id.into())
619 }
620
621 fn full_sample(
624 ts_ms: u64,
625 util: f32,
626 mem: u64,
627 throttle: Option<ThrottleReasons>,
628 ) -> DynamicSample {
629 DynamicSample {
630 ts_ms,
631 util_pct: Some(util),
632 util_engine: None,
633 mem_used_bytes: Some(mem),
634 power_mw: Some(100_000),
635 temp_c: Some(60.0),
636 fan_pct: Some(40.0),
637 sm_clock_mhz: Some(1500),
638 mem_clock_mhz: Some(7000),
639 encoder_pct: None,
640 decoder_pct: None,
641 throttle,
642 power_limit_mw: None,
643 pcie_gen: None,
644 pcie_width: None,
645 ecc_uncorrected: None,
646 }
647 }
648
649 #[test]
654 fn rollup_min_avg_max_and_bucket_boundary() {
655 let scratch = Scratch::new();
656 let (store, was_reset) = SqliteStore::open(&scratch.path).unwrap();
657 assert!(!was_reset);
658 let mut rec = Recorder::new(store);
659 let d = dev("0000:01:00.0");
660
661 rec.observe(
663 &d,
664 &full_sample(1_000, 10.0, 1 << 30, Default::default()),
665 &[],
666 );
667 rec.observe(
668 &d,
669 &full_sample(2_000, 20.0, 2 << 30, Default::default()),
670 &[],
671 );
672 rec.observe(
673 &d,
674 &full_sample(9_000, 60.0, 3 << 30, Default::default()),
675 &[],
676 );
677 rec.observe(
679 &d,
680 &full_sample(11_000, 5.0, 1 << 30, Default::default()),
681 &[],
682 );
683
684 let rows = rec
685 .store()
686 .samples_between(&d, 0, 10_000, Tier::TenSec)
687 .unwrap();
688 assert_eq!(rows.len(), 1, "exactly one completed 10s bucket");
689 let r = &rows[0];
690 assert_eq!(r.bucket_ms, 0);
691 assert_eq!(r.n, 3);
692 assert_eq!(r.util_min, Some(10.0));
693 assert_eq!(r.util_max, Some(60.0));
694 assert_eq!(r.util_avg, Some(30.0));
695 assert_eq!(r.mem_max, Some(3 << 30));
696 assert_eq!(r.mem_avg, Some(2 << 30)); assert_eq!(r.power_max_mw, Some(100_000));
698 assert_eq!(r.sm_clock_min, Some(1500));
699 }
700
701 #[test]
704 fn absent_metric_is_null_not_zero() {
705 let scratch = Scratch::new();
706 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
707 let mut rec = Recorder::new(store);
708 let d = dev("dev-null");
709
710 let mut s = full_sample(1_000, 50.0, 1 << 30, Default::default());
712 s.power_mw = None;
713 s.temp_c = None;
714 rec.observe(&d, &s, &[]);
715 let mut s2 = full_sample(2_000, 70.0, 2 << 30, Default::default());
716 s2.power_mw = None;
717 s2.temp_c = None;
718 rec.observe(&d, &s2, &[]);
719 rec.observe(
720 &d,
721 &full_sample(11_000, 1.0, 1 << 30, Default::default()),
722 &[],
723 );
724
725 let r = &rec
726 .store()
727 .samples_between(&d, 0, 10_000, Tier::TenSec)
728 .unwrap()[0];
729 assert_eq!(r.power_avg_mw, None, "all-absent power must be NULL");
730 assert_eq!(r.power_max_mw, None);
731 assert_eq!(r.temp_avg_c, None);
732 assert_eq!(r.util_avg, Some(60.0));
734 }
735
736 #[test]
739 fn partial_present_metric_averages_over_present_only() {
740 let scratch = Scratch::new();
741 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
742 let mut rec = Recorder::new(store);
743 let d = dev("partial");
744
745 let mut a = full_sample(1_000, 0.0, 1 << 30, Default::default());
746 a.temp_c = Some(80.0); let mut b = full_sample(2_000, 0.0, 1 << 30, Default::default());
748 b.temp_c = None; rec.observe(&d, &a, &[]);
750 rec.observe(&d, &b, &[]);
751 rec.observe(
752 &d,
753 &full_sample(11_000, 0.0, 1 << 30, Default::default()),
754 &[],
755 );
756
757 let r = &rec
758 .store()
759 .samples_between(&d, 0, 10_000, Tier::TenSec)
760 .unwrap()[0];
761 assert_eq!(r.temp_avg_c, Some(80.0));
763 assert_eq!(r.temp_max_c, Some(80.0));
764 }
765
766 #[test]
772 fn throttle_counters_tally_per_reason() {
773 let scratch = Scratch::new();
774 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
775 let mut rec = Recorder::new(store);
776 let d = dev("throttle");
777
778 let thermal = ThrottleReasons {
779 thermal: true,
780 ..Default::default()
781 };
782 let hw = ThrottleReasons {
783 hw_slowdown: true,
784 ..Default::default()
785 };
786 rec.observe(&d, &full_sample(1_000, 90.0, 1 << 30, Some(thermal)), &[]);
787 rec.observe(&d, &full_sample(2_000, 90.0, 1 << 30, Some(thermal)), &[]);
788 rec.observe(&d, &full_sample(3_000, 90.0, 1 << 30, Some(hw)), &[]);
789 rec.observe(&d, &full_sample(4_000, 90.0, 1 << 30, None), &[]);
792 rec.observe(
793 &d,
794 &full_sample(5_000, 90.0, 1 << 30, Some(ThrottleReasons::default())),
795 &[],
796 );
797 rec.observe(
798 &d,
799 &full_sample(11_000, 1.0, 1 << 30, Default::default()),
800 &[],
801 );
802
803 let r = &rec
804 .store()
805 .samples_between(&d, 0, 10_000, Tier::TenSec)
806 .unwrap()[0];
807 assert_eq!(r.throttle_n, 3, "3 frames had any throttle reason");
808 assert_eq!(r.throttle_thermal_n, 2);
809 assert_eq!(r.throttle_hw_n, 1);
810 assert_eq!(r.throttle_power_n, 0);
811 }
812
813 #[test]
815 fn flush_persists_partial_buckets() {
816 let scratch = Scratch::new();
817 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
818 let mut rec = Recorder::new(store);
819 let d = dev("flush");
820
821 rec.observe(
822 &d,
823 &full_sample(1_000, 50.0, 1 << 30, Default::default()),
824 &[],
825 );
826 assert!(rec
828 .store()
829 .samples_between(&d, 0, 10_000, Tier::TenSec)
830 .unwrap()
831 .is_empty());
832 rec.flush();
833 let rows = rec
834 .store()
835 .samples_between(&d, 0, 10_000, Tier::TenSec)
836 .unwrap();
837 assert_eq!(rows.len(), 1);
838 assert_eq!(rows[0].n, 1);
839 assert_eq!(rows[0].util_avg, Some(50.0));
840 }
841
842 #[test]
844 fn one_min_tier_aggregates_across_ten_sec_buckets() {
845 let scratch = Scratch::new();
846 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
847 let mut rec = Recorder::new(store);
848 let d = dev("minute");
849
850 for (i, ts) in (5_000..60_000).step_by(10_000).enumerate() {
852 rec.observe(
853 &d,
854 &full_sample(ts, (i as f32) * 10.0, 1 << 30, Default::default()),
855 &[],
856 );
857 }
858 rec.observe(
860 &d,
861 &full_sample(61_000, 0.0, 1 << 30, Default::default()),
862 &[],
863 );
864
865 let rows = rec
866 .store()
867 .samples_between(&d, 0, 60_000, Tier::OneMin)
868 .unwrap();
869 assert_eq!(rows.len(), 1);
870 assert_eq!(rows[0].bucket_ms, 0);
871 assert_eq!(rows[0].n, 6, "six frames in the minute");
872 assert_eq!(rows[0].util_min, Some(0.0));
873 assert_eq!(rows[0].util_max, Some(50.0));
874 assert_eq!(rows[0].util_avg, Some(25.0)); }
876
877 #[test]
880 fn event_round_trip_preserves_all_fields() {
881 let scratch = Scratch::new();
882 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
883 let evs = vec![
884 Event {
885 ts_ms: 5_000,
886 device: dev("gpu0"),
887 kind: EventKind::ThrottleStart,
888 severity: Severity::Critical,
889 confidence: Confidence::Fact,
890 title: "GPU0 began throttling (hw slowdown)".into(),
891 evidence: "throttle bits: [hw slowdown]; 95°C".into(),
892 },
893 Event {
894 ts_ms: 6_000,
895 device: dev("gpu0"),
896 kind: EventKind::IdleGap,
897 severity: Severity::Info,
898 confidence: Confidence::Likely,
899 title: "GPU0 sat idle 14s — likely a dataloader stall".into(),
900 evidence: "util 92% -> mean 2% over 14s".into(),
901 },
902 ];
903 store.insert_events(&evs).unwrap();
904
905 let got = store.events_between(0, 10_000).unwrap();
906 assert_eq!(got.len(), 2);
907 for (a, b) in evs.iter().zip(&got) {
908 assert_eq!(a.ts_ms, b.ts_ms);
909 assert_eq!(a.device, b.device);
910 assert_eq!(a.kind, b.kind, "kind must round-trip");
911 assert_eq!(a.severity, b.severity, "severity must round-trip");
912 assert_eq!(
913 a.confidence, b.confidence,
914 "confidence must round-trip (honesty contract)"
915 );
916 assert_eq!(a.title, b.title);
917 assert_eq!(a.evidence, b.evidence);
918 }
919 }
920
921 #[test]
923 fn recorder_record_events_passthrough() {
924 let scratch = Scratch::new();
925 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
926 let mut rec = Recorder::new(store);
927 rec.record_events(&[Event {
928 ts_ms: 1_000,
929 device: dev("g"),
930 kind: EventKind::ProcessExited,
931 severity: Severity::Info,
932 confidence: Confidence::Fact,
933 title: "left".into(),
934 evidence: "gone".into(),
935 }]);
936 assert_eq!(rec.store().events_between(0, 2_000).unwrap().len(), 1);
937 }
938
939 #[test]
942 fn corruption_recovery_quarantines_and_reopens() {
943 let scratch = Scratch::new();
944 std::fs::write(
945 &scratch.path,
946 b"this is not a sqlite database, just junk bytes",
947 )
948 .unwrap();
949
950 let (mut store, was_reset) = SqliteStore::open(&scratch.path).unwrap();
951 assert!(was_reset, "a garbage file must trigger a reset");
952
953 let dir = scratch.path.parent().unwrap();
955 let stem = scratch.path.file_name().unwrap().to_str().unwrap();
956 let preserved = std::fs::read_dir(dir).unwrap().flatten().any(|e| {
957 e.file_name()
958 .to_str()
959 .map(|n| n.starts_with(&format!("{stem}.corrupt-")))
960 .unwrap_or(false)
961 });
962 assert!(
963 preserved,
964 "corrupt file must be renamed to *.corrupt-*, not deleted"
965 );
966
967 store
969 .insert_events(&[Event {
970 ts_ms: 1,
971 device: dev("d"),
972 kind: EventKind::HistoryReset,
973 severity: Severity::Info,
974 confidence: Confidence::Fact,
975 title: "reset".into(),
976 evidence: "fresh".into(),
977 }])
978 .unwrap();
979 assert_eq!(store.events_between(0, 10).unwrap().len(), 1);
980 }
981
982 #[test]
984 fn healthy_reopen_is_not_a_reset() {
985 let scratch = Scratch::new();
986 {
987 let (store, was_reset) = SqliteStore::open(&scratch.path).unwrap();
988 assert!(!was_reset);
989 store
990 .register_device(&dev("d"), "GPU", Vendor::Nvidia, Some(1 << 30))
991 .unwrap();
992 }
993 let (store, was_reset) = SqliteStore::open(&scratch.path).unwrap();
994 assert!(!was_reset, "reopening a healthy db must not reset it");
995 assert_eq!(store.devices().unwrap().len(), 1);
997 }
998
999 #[test]
1002 fn prune_drops_only_expired_rows() {
1003 let scratch = Scratch::new();
1004 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1005 let d = dev("retain");
1006 let now: u64 = 100 * RETAIN_1M_MS; let old_10s = now - RETAIN_10S_MS - 60_000; let new_10s = now - 60_000; let old_1m = now - RETAIN_1M_MS - 60_000; let new_1m = now - RETAIN_10S_MS - 120_000; let mk = |bucket: u64| SampleRollup {
1014 device_id: d.clone(),
1015 bucket_ms: bucket,
1016 n: 1,
1017 util_min: Some(1.0),
1018 util_avg: Some(1.0),
1019 util_max: Some(1.0),
1020 mem_avg: None,
1021 mem_max: None,
1022 power_avg_mw: None,
1023 power_max_mw: None,
1024 temp_avg_c: None,
1025 temp_max_c: None,
1026 fan_max_pct: None,
1027 sm_clock_min: None,
1028 sm_clock_avg: None,
1029 sm_clock_max: None,
1030 throttle_n: 0,
1031 throttle_observed_n: 1,
1032 throttle_thermal_n: 0,
1033 throttle_power_n: 0,
1034 throttle_hw_n: 0,
1035 };
1036 store
1037 .insert_sample_rollups(Tier::TenSec, &[mk(old_10s), mk(new_10s)])
1038 .unwrap();
1039 store
1040 .insert_sample_rollups(Tier::OneMin, &[mk(old_1m), mk(new_1m)])
1041 .unwrap();
1042 store
1043 .insert_process_rollups(&[ProcessRollup {
1044 device_id: d.clone(),
1045 bucket_ms: old_10s,
1046 pid: 1,
1047 name: "old".into(),
1048 kind: ProcessKind::Compute,
1049 mem_max: None,
1050 util_avg: None,
1051 cpu_avg: None,
1052 container: None,
1053 cmdline: None,
1054 }])
1055 .unwrap();
1056 store
1057 .insert_events(&[
1058 Event {
1059 ts_ms: old_1m,
1060 device: d.clone(),
1061 kind: EventKind::ThrottleEnd,
1062 severity: Severity::Info,
1063 confidence: Confidence::Fact,
1064 title: "old".into(),
1065 evidence: "old".into(),
1066 },
1067 Event {
1068 ts_ms: new_1m,
1069 device: d.clone(),
1070 kind: EventKind::ThrottleEnd,
1071 severity: Severity::Info,
1072 confidence: Confidence::Fact,
1073 title: "new".into(),
1074 evidence: "new".into(),
1075 },
1076 ])
1077 .unwrap();
1078
1079 store.prune(now).unwrap();
1080
1081 let tens = store.samples_between(&d, 0, now, Tier::TenSec).unwrap();
1082 assert_eq!(tens.len(), 1, "only the within-48h 10s row remains");
1083 assert_eq!(tens[0].bucket_ms, new_10s);
1084
1085 let mins = store.samples_between(&d, 0, now, Tier::OneMin).unwrap();
1086 assert_eq!(mins.len(), 1, "only the within-30d 1m row remains");
1087 assert_eq!(mins[0].bucket_ms, new_1m);
1088
1089 assert!(store.processes_at(&d, old_10s).unwrap().is_empty());
1091
1092 let evs = store.events_between(0, now).unwrap();
1093 assert_eq!(evs.len(), 1, "only the within-30d event remains");
1094 assert_eq!(evs[0].title, "new");
1095 }
1096
1097 #[test]
1100 fn processes_at_picks_right_bucket_with_fallback() {
1101 let scratch = Scratch::new();
1102 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1103 let d = dev("procs");
1104
1105 let mk = |bucket: u64, pid: u32, name: &str, mem: u64| ProcessRollup {
1106 device_id: d.clone(),
1107 bucket_ms: bucket,
1108 pid,
1109 name: name.into(),
1110 kind: ProcessKind::Compute,
1111 mem_max: Some(mem),
1112 util_avg: Some(10.0),
1113 cpu_avg: None,
1114 container: None,
1115 cmdline: None,
1116 };
1117 store
1119 .insert_process_rollups(&[mk(100_000, 1, "a", 1 << 30), mk(110_000, 2, "b", 2 << 30)])
1120 .unwrap();
1121
1122 let at = store.processes_at(&d, 115_000).unwrap();
1124 assert_eq!(at.len(), 1);
1125 assert_eq!(at[0].pid, 2);
1126 assert_eq!(at[0].bucket_ms, 110_000);
1127
1128 let fb = store.processes_at(&d, 125_000).unwrap();
1130 assert_eq!(fb.len(), 1);
1131 assert_eq!(fb[0].pid, 2, "falls back to nearest earlier bucket");
1132 assert_eq!(fb[0].bucket_ms, 110_000);
1133
1134 let none = store.processes_at(&d, 300_000).unwrap();
1136 assert!(
1137 none.is_empty(),
1138 "no bucket within 60s -> empty, not stale data"
1139 );
1140 }
1141
1142 #[test]
1144 fn recorder_process_rollup_aggregates() {
1145 let scratch = Scratch::new();
1146 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
1147 let mut rec = Recorder::new(store);
1148 let d = dev("procagg");
1149
1150 let proc = |mem: u64, util: f32, cpu: f32| ProcessSample {
1151 pid: 42,
1152 name: "python".into(),
1153 kind: ProcessKind::Compute,
1154 mem_bytes: Some(mem),
1155 util_pct: Some(util),
1156 cpu_pct: Some(cpu),
1157 container: None,
1158 cmdline: None,
1159 };
1160 rec.observe(
1161 &d,
1162 &full_sample(1_000, 50.0, 1 << 30, Default::default()),
1163 &[proc(1 << 30, 10.0, 100.0)],
1164 );
1165 rec.observe(
1166 &d,
1167 &full_sample(2_000, 50.0, 1 << 30, Default::default()),
1168 &[proc(3 << 30, 30.0, 200.0)],
1169 );
1170 rec.observe(
1172 &d,
1173 &full_sample(11_000, 1.0, 1 << 30, Default::default()),
1174 &[],
1175 );
1176
1177 let rows = rec.store().processes_at(&d, 5_000).unwrap();
1178 assert_eq!(rows.len(), 1);
1179 let r = &rows[0];
1180 assert_eq!(r.pid, 42);
1181 assert_eq!(r.mem_max, Some(3 << 30), "peak memory across the bucket");
1182 assert_eq!(r.util_avg, Some(20.0)); assert_eq!(r.cpu_avg, Some(150.0)); }
1185
1186 const DATA_DIR_VARS: [&str; 4] = ["XDG_DATA_HOME", "HOME", "LOCALAPPDATA", "USERPROFILE"];
1196
1197 static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
1198
1199 struct DataDirEnv {
1202 _lock: std::sync::MutexGuard<'static, ()>,
1203 saved: Vec<(&'static str, Option<std::ffi::OsString>)>,
1204 }
1205
1206 impl DataDirEnv {
1207 fn new() -> Self {
1208 let lock = ENV_LOCK
1211 .lock()
1212 .unwrap_or_else(std::sync::PoisonError::into_inner);
1213 let saved = DATA_DIR_VARS
1214 .iter()
1215 .map(|v| (*v, std::env::var_os(v)))
1216 .collect();
1217 Self { _lock: lock, saved }
1218 }
1219
1220 fn set(&self, var: &str, value: impl AsRef<std::ffi::OsStr>) {
1221 unsafe { std::env::set_var(var, value) };
1224 }
1225
1226 fn clear(&self, var: &str) {
1227 unsafe { std::env::remove_var(var) };
1229 }
1230
1231 fn redirect_all(&self, dir: &std::path::Path) {
1235 for var in DATA_DIR_VARS {
1236 self.set(var, dir);
1237 }
1238 }
1239 }
1240
1241 impl Drop for DataDirEnv {
1242 fn drop(&mut self) {
1243 for (var, value) in &self.saved {
1244 unsafe {
1246 match value {
1247 Some(v) => std::env::set_var(var, v),
1248 None => std::env::remove_var(var),
1249 }
1250 }
1251 }
1252 }
1253 }
1254
1255 #[cfg(not(any(target_os = "windows", target_os = "macos")))]
1259 #[test]
1260 fn data_dir_linux_prefers_xdg_then_home_then_errors() {
1261 let env = DataDirEnv::new();
1262 env.set("XDG_DATA_HOME", "/scratch/xdg");
1263 env.set("HOME", "/scratch/home"); assert_eq!(
1265 store::default_data_dir().unwrap(),
1266 std::path::Path::new("/scratch/xdg/gpuviewer")
1267 );
1268 env.set("XDG_DATA_HOME", ""); assert_eq!(
1270 store::default_data_dir().unwrap(),
1271 std::path::Path::new("/scratch/home/.local/share/gpuviewer")
1272 );
1273 env.clear("XDG_DATA_HOME");
1274 env.set("HOME", "");
1275 assert!(
1276 matches!(store::default_data_dir(), Err(StoreError::NoDataDir)),
1277 "with nothing to resolve from, the only honest answer is a refusal"
1278 );
1279 }
1280
1281 #[cfg(target_os = "windows")]
1285 #[test]
1286 fn data_dir_windows_prefers_localappdata_then_userprofile_then_errors() {
1287 let env = DataDirEnv::new();
1288 env.set("XDG_DATA_HOME", "C:\\decoy-xdg"); env.set("LOCALAPPDATA", "C:\\scratch\\Local");
1290 env.set("USERPROFILE", "C:\\scratch\\profile"); assert_eq!(
1292 store::default_data_dir().unwrap(),
1293 std::path::Path::new("C:\\scratch\\Local\\gpuviewer")
1294 );
1295 env.set("LOCALAPPDATA", ""); assert_eq!(
1297 store::default_data_dir().unwrap(),
1298 std::path::Path::new("C:\\scratch\\profile\\AppData\\Local\\gpuviewer")
1299 );
1300 env.clear("LOCALAPPDATA"); assert_eq!(
1302 store::default_data_dir().unwrap(),
1303 std::path::Path::new("C:\\scratch\\profile\\AppData\\Local\\gpuviewer")
1304 );
1305 env.set("USERPROFILE", "");
1306 assert!(matches!(
1307 store::default_data_dir(),
1308 Err(StoreError::NoDataDir)
1309 ));
1310 }
1311
1312 #[cfg(target_os = "macos")]
1315 #[test]
1316 fn data_dir_macos_uses_home_library_application_support() {
1317 let env = DataDirEnv::new();
1318 env.set("XDG_DATA_HOME", "/decoy-xdg"); env.set("HOME", "/scratch/home");
1320 assert_eq!(
1321 store::default_data_dir().unwrap(),
1322 std::path::Path::new("/scratch/home/Library/Application Support/gpuviewer")
1323 );
1324 env.set("HOME", ""); assert!(matches!(
1326 store::default_data_dir(),
1327 Err(StoreError::NoDataDir)
1328 ));
1329 env.clear("HOME"); assert!(matches!(
1331 store::default_data_dir(),
1332 Err(StoreError::NoDataDir)
1333 ));
1334 }
1335
1336 #[test]
1339 fn mock_default_path_is_separate_from_real() {
1340 let scratch_dir = std::env::temp_dir().join(format!(
1343 "gpuviewer-xdg-{}-{}",
1344 std::process::id(),
1345 SEQ.fetch_add(1, Ordering::Relaxed)
1346 ));
1347 let env = DataDirEnv::new();
1348 env.redirect_all(&scratch_dir);
1349
1350 let (mock_store, _) = SqliteStore::open_default(true).unwrap();
1351 let (real_store, _) = SqliteStore::open_default(false).unwrap();
1352 assert_eq!(mock_store.data_source().unwrap(), Some(DataSource::Mock));
1355 assert_eq!(real_store.data_source().unwrap(), Some(DataSource::Real));
1356 let mock_name = mock_store.path().file_name().unwrap().to_str().unwrap();
1357 let real_name = real_store.path().file_name().unwrap().to_str().unwrap();
1358 assert!(
1359 mock_name.contains("-mock"),
1360 "mock db filename must contain -mock, got {mock_name}"
1361 );
1362 assert!(
1363 !real_name.contains("-mock"),
1364 "real db filename must not contain -mock, got {real_name}"
1365 );
1366 assert_ne!(
1367 mock_name, real_name,
1368 "mock and real must be different files"
1369 );
1370 assert!(mock_store.path().starts_with(&scratch_dir));
1373
1374 drop(mock_store);
1375 drop(real_store);
1376 drop(env); let _ = std::fs::remove_dir_all(&scratch_dir);
1378 }
1379
1380 #[test]
1387 fn fresh_db_is_stamped_with_session_source() {
1388 for source in [DataSource::Real, DataSource::Mock] {
1389 let scratch = Scratch::new();
1390 let (store, was_reset) = SqliteStore::open_recording(&scratch.path, source).unwrap();
1391 assert!(!was_reset);
1392 assert_eq!(store.data_source().unwrap(), Some(source));
1393 drop(store);
1394 let (store, _) = SqliteStore::open_recording(&scratch.path, source).unwrap();
1395 assert_eq!(
1396 store.data_source().unwrap(),
1397 Some(source),
1398 "a same-source reopen must keep the stamp"
1399 );
1400 }
1401 }
1402
1403 #[test]
1407 fn mock_recording_into_real_db_is_refused() {
1408 let scratch = Scratch::new();
1409 drop(SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap());
1410
1411 let err = SqliteStore::open_recording(&scratch.path, DataSource::Mock).unwrap_err();
1412 assert!(
1413 matches!(err, StoreError::DataSourceMismatch { .. }),
1414 "must refuse with DataSourceMismatch, got: {err}"
1415 );
1416 assert_eq!(
1417 err.to_string(),
1418 format!(
1419 "refusing to record mock data into {} — it contains real history \
1420 (use --db elsewhere or --no-persist)",
1421 scratch.path.display()
1422 ),
1423 );
1424 let reader = SqliteStore::open_readonly(&scratch.path).unwrap();
1426 assert_eq!(reader.data_source().unwrap(), Some(DataSource::Real));
1427 }
1428
1429 #[test]
1432 fn real_recording_into_mock_db_is_refused() {
1433 let scratch = Scratch::new();
1434 drop(SqliteStore::open_recording(&scratch.path, DataSource::Mock).unwrap());
1435
1436 let err = SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap_err();
1437 let msg = err.to_string();
1438 assert!(
1439 msg.contains("refusing to record real data into"),
1440 "wrong direction in message: {msg}"
1441 );
1442 assert!(
1443 msg.contains("it contains mock history"),
1444 "must name the db's flavor: {msg}"
1445 );
1446 assert!(
1447 msg.contains(scratch.path.to_str().unwrap()),
1448 "must name the file: {msg}"
1449 );
1450 }
1451
1452 #[test]
1456 fn readonly_open_ignores_the_data_source_stamp() {
1457 let scratch = Scratch::new();
1458 let (mut store, _) = SqliteStore::open_recording(&scratch.path, DataSource::Mock).unwrap();
1459 store
1460 .insert_events(&[event_at(
1461 &dev("sim"),
1462 5,
1463 EventKind::ThrottleStart,
1464 "likely simulated",
1465 )])
1466 .unwrap();
1467 drop(store);
1468
1469 let reader = SqliteStore::open_readonly(&scratch.path).unwrap();
1471 assert_eq!(reader.events_between(0, 10).unwrap().len(), 1);
1472 assert_eq!(reader.data_source().unwrap(), Some(DataSource::Mock));
1473 }
1474
1475 #[test]
1479 fn legacy_unstamped_db_adopts_next_session_source() {
1480 let scratch = Scratch::new();
1481 {
1482 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1484 assert_eq!(store.data_source().unwrap(), None, "legacy db has no stamp");
1485 store
1486 .insert_events(&[event_at(
1487 &dev("old"),
1488 3,
1489 EventKind::ThrottleEnd,
1490 "likely legacy",
1491 )])
1492 .unwrap();
1493 }
1494
1495 let (store, _) = SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap();
1496 assert_eq!(store.data_source().unwrap(), Some(DataSource::Real));
1497 assert_eq!(
1498 store.events_between(0, 10).unwrap().len(),
1499 1,
1500 "adoption must not disturb the legacy contents"
1501 );
1502 drop(store);
1503
1504 let err = SqliteStore::open_recording(&scratch.path, DataSource::Mock).unwrap_err();
1506 assert!(matches!(err, StoreError::DataSourceMismatch { .. }));
1507 }
1508
1509 #[test]
1511 fn readonly_reader_sees_committed_writes() {
1512 let scratch = Scratch::new();
1513 let (mut writer, _) = SqliteStore::open(&scratch.path).unwrap();
1514 writer
1515 .insert_events(&[Event {
1516 ts_ms: 7,
1517 device: dev("d"),
1518 kind: EventKind::VramPressure,
1519 severity: Severity::Warning,
1520 confidence: Confidence::Likely,
1521 title: "likely full soon".into(),
1522 evidence: "slope".into(),
1523 }])
1524 .unwrap();
1525
1526 let reader = SqliteStore::open_readonly(&scratch.path).unwrap();
1527 assert_eq!(reader.events_between(0, 10).unwrap().len(), 1);
1528 }
1529
1530 #[test]
1532 fn bucket_extremes_span_both_tiers() {
1533 let scratch = Scratch::new();
1534 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1535 assert_eq!(store.earliest_bucket_ms().unwrap(), None);
1536 let d = dev("span");
1537 let mk = |bucket: u64| SampleRollup {
1538 device_id: d.clone(),
1539 bucket_ms: bucket,
1540 n: 1,
1541 util_min: Some(1.0),
1542 util_avg: Some(1.0),
1543 util_max: Some(1.0),
1544 mem_avg: None,
1545 mem_max: None,
1546 power_avg_mw: None,
1547 power_max_mw: None,
1548 temp_avg_c: None,
1549 temp_max_c: None,
1550 fan_max_pct: None,
1551 sm_clock_min: None,
1552 sm_clock_avg: None,
1553 sm_clock_max: None,
1554 throttle_n: 0,
1555 throttle_observed_n: 1,
1556 throttle_thermal_n: 0,
1557 throttle_power_n: 0,
1558 throttle_hw_n: 0,
1559 };
1560 store
1561 .insert_sample_rollups(Tier::TenSec, &[mk(50_000)])
1562 .unwrap();
1563 store
1564 .insert_sample_rollups(Tier::OneMin, &[mk(600_000)])
1565 .unwrap();
1566 assert_eq!(store.earliest_bucket_ms().unwrap(), Some(50_000));
1567 assert_eq!(store.latest_bucket_ms().unwrap(), Some(600_000));
1568 }
1569
1570 fn rollup_at(d: &DeviceId, bucket: u64) -> SampleRollup {
1577 SampleRollup {
1578 device_id: d.clone(),
1579 bucket_ms: bucket,
1580 n: 1,
1581 util_min: Some(1.0),
1582 util_avg: Some(1.0),
1583 util_max: Some(1.0),
1584 mem_avg: None,
1585 mem_max: None,
1586 power_avg_mw: None,
1587 power_max_mw: None,
1588 temp_avg_c: None,
1589 temp_max_c: None,
1590 fan_max_pct: None,
1591 sm_clock_min: None,
1592 sm_clock_avg: None,
1593 sm_clock_max: None,
1594 throttle_n: 0,
1595 throttle_observed_n: 1,
1596 throttle_thermal_n: 0,
1597 throttle_power_n: 0,
1598 throttle_hw_n: 0,
1599 }
1600 }
1601
1602 fn event_at(d: &DeviceId, ts_ms: u64, kind: EventKind, title: &str) -> Event {
1603 Event {
1604 ts_ms,
1605 device: d.clone(),
1606 kind,
1607 severity: Severity::Warning,
1608 confidence: Confidence::Likely,
1609 title: title.into(),
1610 evidence: "likely evidence".into(),
1611 }
1612 }
1613
1614 #[test]
1619 fn export_copies_only_the_window_into_a_standalone_file() {
1620 let scratch = Scratch::new();
1621 let out = Scratch::new(); let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1623 let d = dev("0000:01:00.0");
1624 store
1625 .register_device(&d, "Exported GPU", Vendor::Amd, Some(16 << 30))
1626 .unwrap();
1627
1628 store
1630 .insert_sample_rollups(
1631 Tier::TenSec,
1632 &[
1633 rollup_at(&d, 50_000), rollup_at(&d, 60_000), rollup_at(&d, 170_000), rollup_at(&d, 180_000), ],
1638 )
1639 .unwrap();
1640 store
1641 .insert_sample_rollups(
1642 Tier::OneMin,
1643 &[
1644 rollup_at(&d, 0), rollup_at(&d, 60_000), rollup_at(&d, 120_000), rollup_at(&d, 180_000), ],
1649 )
1650 .unwrap();
1651 let proc_at = |bucket: u64, name: &str| ProcessRollup {
1652 device_id: d.clone(),
1653 bucket_ms: bucket,
1654 pid: 9,
1655 name: name.into(),
1656 kind: ProcessKind::Compute,
1657 mem_max: Some(1 << 30),
1658 util_avg: Some(50.0),
1659 cpu_avg: Some(120.0),
1660 container: Some("docker:abc123".into()),
1661 cmdline: None,
1662 };
1663 store
1664 .insert_process_rollups(&[proc_at(50_000, "decoy-proc"), proc_at(70_000, "kept-proc")])
1665 .unwrap();
1666 store
1667 .insert_events(&[
1668 event_at(&d, 59_999, EventKind::ThrottleStart, "decoy just before"),
1669 event_at(&d, 60_000, EventKind::ThrottleStart, "likely edge start"),
1670 event_at(&d, 179_999, EventKind::VramPressure, "likely edge end"),
1671 event_at(&d, 180_000, EventKind::ThrottleEnd, "decoy just after"),
1672 ])
1673 .unwrap();
1674
1675 let counts = store.export_to(&out.path, 60_000, 179_999, true).unwrap();
1676 assert_eq!(counts.devices, 1);
1677 assert_eq!(counts.samples_10s, 2, "only the in-window 10s buckets copy");
1678 assert_eq!(counts.samples_1m, 2, "only the in-window 1m buckets copy");
1679 assert_eq!(counts.processes_10s, 1, "the decoy process bucket stays");
1680 assert_eq!(counts.events, 2, "edge rows in, decoys out");
1681
1682 drop(store);
1684 std::fs::remove_file(&scratch.path).unwrap();
1685 let exported = SqliteStore::open_readonly(&out.path).unwrap();
1686 let devs = exported.devices().unwrap();
1687 assert_eq!(devs.len(), 1);
1688 assert_eq!(devs[0].name, "Exported GPU");
1689 assert_eq!(devs[0].mem_total_bytes, Some(16 << 30));
1690 let tens = exported
1691 .samples_between(&d, 0, 1_000_000, Tier::TenSec)
1692 .unwrap();
1693 assert_eq!(
1694 tens.iter().map(|r| r.bucket_ms).collect::<Vec<_>>(),
1695 vec![60_000, 170_000]
1696 );
1697 let procs = exported.processes_at(&d, 75_000).unwrap();
1698 assert_eq!(procs.len(), 1);
1699 assert_eq!(procs[0].name, "kept-proc");
1700 assert_eq!(procs[0].container.as_deref(), Some("docker:abc123"));
1701 let evs = exported.events_between(0, 1_000_000).unwrap();
1702 assert_eq!(evs.len(), 2);
1703 assert!(
1704 evs.iter().all(|e| !e.title.contains("decoy")),
1705 "no decoy may cross the window: {evs:?}"
1706 );
1707 assert!(evs.iter().all(|e| e.confidence == Confidence::Likely));
1709 }
1710
1711 #[test]
1714 fn export_refuses_to_overwrite_an_existing_file() {
1715 let scratch = Scratch::new();
1716 let out = Scratch::new();
1717 let (store, _) = SqliteStore::open(&scratch.path).unwrap();
1718 std::fs::write(&out.path, b"someone's shared incident file").unwrap();
1719
1720 let err = store.export_to(&out.path, 0, 1_000, true).unwrap_err();
1721 assert!(
1722 matches!(err, StoreError::OutputExists(_)),
1723 "must refuse with OutputExists, got: {err}"
1724 );
1725 assert_eq!(
1726 std::fs::read(&out.path).unwrap(),
1727 b"someone's shared incident file",
1728 "the existing file must be untouched"
1729 );
1730 }
1731
1732 #[test]
1742 fn second_recording_open_is_refused_while_lock_is_held() {
1743 let scratch = Scratch::new();
1744 let _held = SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap();
1745
1746 let err = SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap_err();
1747 assert!(
1748 matches!(err, StoreError::Locked { .. }),
1749 "a second recording open must fail with Locked, got: {err}"
1750 );
1751 let msg = err.to_string();
1752 assert!(
1753 msg.contains("another gpuviewer instance is already recording"),
1754 "the refusal must say who has it: {msg}"
1755 );
1756 assert!(
1757 msg.contains(scratch.path.to_str().unwrap()),
1758 "the refusal must name the file: {msg}"
1759 );
1760 assert!(
1761 msg.contains("--no-persist"),
1762 "the refusal must offer the live-only escape hatch: {msg}"
1763 );
1764
1765 let err = SqliteStore::open(&scratch.path).unwrap_err();
1766 assert!(
1767 matches!(err, StoreError::Locked { .. }),
1768 "the plain write open must be refused identically, got: {err}"
1769 );
1770 }
1771
1772 #[test]
1775 fn losing_the_lock_race_never_quarantines_the_database() {
1776 let scratch = Scratch::new();
1777 let (mut held, _) = SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap();
1778 held.insert_events(&[event_at(&dev("g"), 5, EventKind::ThrottleStart, "kept")])
1779 .unwrap();
1780
1781 assert!(SqliteStore::open(&scratch.path).is_err());
1782
1783 let dir = scratch.path.parent().unwrap();
1786 let stem = scratch.path.file_name().unwrap().to_str().unwrap();
1787 let quarantined = std::fs::read_dir(dir).unwrap().flatten().any(|e| {
1788 e.file_name()
1789 .to_str()
1790 .is_some_and(|n| n.starts_with(&format!("{stem}.corrupt-")))
1791 });
1792 assert!(
1793 !quarantined,
1794 "a busy database must never be quarantined as corrupt"
1795 );
1796
1797 held.insert_events(&[event_at(
1799 &dev("g"),
1800 6,
1801 EventKind::ThrottleEnd,
1802 "still writing",
1803 )])
1804 .unwrap();
1805 assert_eq!(held.events_between(0, 10).unwrap().len(), 2);
1806 }
1807
1808 #[test]
1811 fn readonly_open_succeeds_alongside_a_held_lock() {
1812 let scratch = Scratch::new();
1813 let (mut held, _) = SqliteStore::open_recording(&scratch.path, DataSource::Mock).unwrap();
1814 held.insert_events(&[event_at(
1815 &dev("sim"),
1816 7,
1817 EventKind::VramPressure,
1818 "likely visible to readers",
1819 )])
1820 .unwrap();
1821
1822 let reader = SqliteStore::open_readonly(&scratch.path).unwrap();
1823 assert_eq!(
1824 reader.events_between(0, 10).unwrap().len(),
1825 1,
1826 "a reader must work while the recording lock is held"
1827 );
1828 let reader2 = SqliteStore::open_readonly(&scratch.path).unwrap();
1830 assert_eq!(reader2.events_between(0, 10).unwrap().len(), 1);
1831 }
1832
1833 #[test]
1837 fn lock_released_on_drop_lets_the_next_instance_record() {
1838 let scratch = Scratch::new();
1839 drop(SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap());
1840
1841 let mut lock_file = scratch.path.as_os_str().to_os_string();
1844 lock_file.push(".lock");
1845 assert!(
1846 std::path::Path::new(&lock_file).exists(),
1847 "the sidecar persists; only the kernel lock is released"
1848 );
1849
1850 let (mut store, was_reset) =
1851 SqliteStore::open_recording(&scratch.path, DataSource::Real).unwrap();
1852 assert!(!was_reset, "relocking must not look like a reset");
1853 store
1854 .insert_events(&[event_at(&dev("g"), 1, EventKind::ProcessExited, "next run")])
1855 .unwrap();
1856 }
1857
1858 #[test]
1868 fn same_event_inserted_twice_lands_once() {
1869 let scratch = Scratch::new();
1870 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1871 let e = event_at(
1872 &dev("gpu0"),
1873 5_000,
1874 EventKind::ThrottleStart,
1875 "began throttling",
1876 );
1877
1878 store.insert_events(std::slice::from_ref(&e)).unwrap();
1879 store.insert_events(std::slice::from_ref(&e)).unwrap();
1881 store.insert_events(&[e.clone(), e.clone()]).unwrap();
1882 assert_eq!(
1883 store.events_between(0, 10_000).unwrap().len(),
1884 1,
1885 "one narration, no matter how many times it is fed"
1886 );
1887
1888 store
1890 .insert_events(&[
1891 event_at(
1892 &dev("gpu1"),
1893 5_000,
1894 EventKind::ThrottleStart,
1895 "began throttling",
1896 ),
1897 event_at(
1898 &dev("gpu0"),
1899 5_000,
1900 EventKind::ThrottleStart,
1901 "another story",
1902 ),
1903 ])
1904 .unwrap();
1905 assert_eq!(
1906 store.events_between(0, 10_000).unwrap().len(),
1907 3,
1908 "the dedupe key must not over-collapse distinct events"
1909 );
1910 }
1911
1912 #[test]
1917 fn migration_collapses_preexisting_duplicate_events() {
1918 let scratch = Scratch::new();
1919 {
1920 let conn = rusqlite::Connection::open(&scratch.path).unwrap();
1922 conn.execute_batch(
1923 "CREATE TABLE events (
1924 id INTEGER PRIMARY KEY AUTOINCREMENT,
1925 ts_ms INTEGER NOT NULL,
1926 device_id TEXT NOT NULL,
1927 kind TEXT NOT NULL,
1928 severity TEXT NOT NULL,
1929 confidence TEXT NOT NULL,
1930 title TEXT NOT NULL,
1931 evidence TEXT NOT NULL
1932 );
1933 CREATE INDEX idx_events_ts ON events (ts_ms);
1934 PRAGMA user_version = 1;
1935 -- Three copies of one narration (double-recorded, then some), with
1936 -- evidence differing so the keep-MIN(id) rule is observable; one
1937 -- distinct event that must survive untouched.
1938 INSERT INTO events (ts_ms, device_id, kind, severity, confidence, title, evidence)
1939 VALUES
1940 (5000, 'gpu0', 'throttle_start', 'warning', 'fact', 'GPU0 began throttling', 'original'),
1941 (5000, 'gpu0', 'throttle_start', 'warning', 'fact', 'GPU0 began throttling', 'echo one'),
1942 (5000, 'gpu0', 'throttle_start', 'warning', 'fact', 'GPU0 began throttling', 'echo two'),
1943 (6000, 'gpu0', 'throttle_end', 'info', 'fact', 'GPU0 stopped throttling', 'distinct');",
1944 )
1945 .unwrap();
1946 }
1947
1948 let (mut store, was_reset) = SqliteStore::open(&scratch.path).unwrap();
1949 assert!(
1950 !was_reset,
1951 "a healthy pre-v2 database migrates in place, never resets"
1952 );
1953
1954 let evs = store.events_between(0, 10_000).unwrap();
1955 assert_eq!(evs.len(), 2, "3 copies -> 1, plus the distinct event");
1956 assert_eq!(
1957 evs[0].evidence, "original",
1958 "MIN(id) — the FIRST insertion — must be the survivor"
1959 );
1960 assert_eq!(evs[1].title, "GPU0 stopped throttling");
1961
1962 store
1964 .insert_events(&[Event {
1965 ts_ms: 5_000,
1966 device: dev("gpu0"),
1967 kind: EventKind::ThrottleStart,
1968 severity: Severity::Warning,
1969 confidence: Confidence::Fact,
1970 title: "GPU0 began throttling".into(),
1971 evidence: "post-migration echo".into(),
1972 }])
1973 .unwrap();
1974 assert_eq!(store.events_between(0, 10_000).unwrap().len(), 2);
1975 }
1976
1977 #[test]
1980 fn migration_is_idempotent_across_reopens() {
1981 let scratch = Scratch::new();
1982 let e = event_at(&dev("g"), 1_000, EventKind::IdleGap, "likely a stall");
1983 {
1984 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1985 store.insert_events(std::slice::from_ref(&e)).unwrap();
1986 }
1987 let (mut store, was_reset) = SqliteStore::open(&scratch.path).unwrap();
1988 assert!(!was_reset);
1989 store.insert_events(std::slice::from_ref(&e)).unwrap();
1990 assert_eq!(store.events_between(0, 2_000).unwrap().len(), 1);
1991 }
1992
1993 #[test]
1996 fn latest_event_ms_overall_and_by_kind() {
1997 let scratch = Scratch::new();
1998 let (mut store, _) = SqliteStore::open(&scratch.path).unwrap();
1999 let d = dev("anchors");
2000 assert_eq!(store.latest_event_ms(None).unwrap(), None);
2001
2002 store
2003 .insert_events(&[
2004 event_at(&d, 5_000, EventKind::ThrottleStart, "likely a"),
2005 event_at(&d, 9_000, EventKind::ProcessExited, "likely b"),
2006 event_at(&d, 7_000, EventKind::ThrottleStart, "likely c"),
2007 ])
2008 .unwrap();
2009 assert_eq!(store.latest_event_ms(None).unwrap(), Some(9_000));
2010 assert_eq!(
2011 store
2012 .latest_event_ms(Some(EventKind::ThrottleStart))
2013 .unwrap(),
2014 Some(7_000),
2015 "per-kind max must ignore newer events of other kinds"
2016 );
2017 assert_eq!(
2018 store
2019 .latest_event_ms(Some(EventKind::VramPressure))
2020 .unwrap(),
2021 None
2022 );
2023 }
2024}