omp_tui/
watchdog.rs

1//! Render-loop stall detection with an optional background probe.
2
3use std::{
4	sync::Arc,
5	thread::{self, JoinHandle},
6	time::{Duration, Instant},
7};
8
9use omp_core::Str;
10use parking_lot::{Condvar, Mutex};
11
12const STALL_THRESHOLD: Duration = Duration::from_millis(250);
13const SYSTEM_SLEEP_THRESHOLD: Duration = Duration::from_secs(60);
14
15/// A render-loop stall returned by [`LoopWatchdogCore::check`].
16#[derive(Clone, Debug, Eq, PartialEq)]
17pub struct StallReport {
18	/// Time elapsed since the render loop's most recent tick.
19	pub elapsed: Duration,
20	/// Phase label active when the stall was detected.
21	pub phase:   Str,
22}
23
24/// Deterministic state machine underlying [`LoopWatchdog`].
25///
26/// Times are monotonic durations from any caller-chosen epoch. A continuous
27/// stall is reported once, and gaps over 60 seconds are treated as system
28/// sleep.
29#[derive(Debug)]
30pub struct LoopWatchdogCore {
31	last_tick: Duration,
32	phase:     Str,
33	reported:  bool,
34}
35
36impl LoopWatchdogCore {
37	/// Create a watchdog core whose last successful render-loop tick was `now`.
38	#[must_use]
39	pub fn new(now: Duration) -> Self {
40		Self { last_tick: now, phase: "unknown".into(), reported: false }
41	}
42
43	/// Record progress by the render loop at monotonic time `now`.
44	pub const fn tick(&mut self, now: Duration) {
45		self.last_tick = now;
46		self.reported = false;
47	}
48
49	/// Set the label attached to a subsequently detected stall.
50	pub fn set_phase(&mut self, phase: impl Into<Str>) {
51		self.phase = phase.into();
52	}
53
54	/// Check for a newly detected stall at monotonic time `now`.
55	///
56	/// Returns one report after 250 ms without a tick. Further checks stay
57	/// silent until [`Self::tick`] records progress. Gaps over 60 seconds are
58	/// suppressed.
59	#[must_use]
60	pub fn check(&mut self, now: Duration) -> Option<StallReport> {
61		let elapsed = now.saturating_sub(self.last_tick);
62		if elapsed > SYSTEM_SLEEP_THRESHOLD {
63			self.reported = false;
64			return None;
65		}
66		if elapsed <= STALL_THRESHOLD {
67			self.reported = false;
68			return None;
69		}
70		if self.reported {
71			return None;
72		}
73		self.reported = true;
74		Some(StallReport { elapsed, phase: self.phase.clone() })
75	}
76}
77
78struct Shared {
79	core:     LoopWatchdogCore,
80	stopping: bool,
81}
82
83/// Background render-loop watchdog.
84///
85/// Call [`Self::tick`] after each successful render-loop iteration and update
86/// [`Self::set_phase`] when entering a diagnostic phase. A background probe
87/// invokes the supplied callback once per continuous stall longer than 250 ms.
88/// Gaps longer than 60 seconds are ignored as probable system sleep.
89pub struct LoopWatchdog {
90	origin: Instant,
91	shared: Arc<(Mutex<Shared>, Condvar)>,
92	worker: Option<JoinHandle<()>>,
93}
94
95impl LoopWatchdog {
96	/// Start a watchdog and send detected stalls to `report`.
97	#[must_use]
98	pub fn new(report: impl Fn(Duration, &str) + Send + 'static) -> Self {
99		let origin = Instant::now();
100		let shared = Arc::new((
101			Mutex::new(Shared { core: LoopWatchdogCore::new(Duration::ZERO), stopping: false }),
102			Condvar::new(),
103		));
104		let worker_shared = Arc::clone(&shared);
105		let worker = thread::spawn(move || {
106			loop {
107				let (lock, wake) = &*worker_shared;
108				let mut guard = lock.lock();
109				wake.wait_for(&mut guard, STALL_THRESHOLD);
110				if guard.stopping {
111					break;
112				}
113				let stall = guard.core.check(origin.elapsed());
114				drop(guard);
115				if let Some(stall) = stall {
116					report(stall.elapsed, &stall.phase);
117				}
118			}
119		});
120
121		Self { origin, shared, worker: Some(worker) }
122	}
123
124	/// Record progress by the render loop.
125	pub fn tick(&self) {
126		let (lock, wake) = &*self.shared;
127		let mut shared = lock.lock();
128		shared.core.tick(self.origin.elapsed());
129		drop(shared);
130		wake.notify_one();
131	}
132
133	/// Set the phase label attached to a subsequently detected stall.
134	pub fn set_phase(&self, phase: impl Into<Str>) {
135		let (lock, _) = &*self.shared;
136		let mut shared = lock.lock();
137		shared.core.set_phase(phase);
138	}
139
140	/// Stop the background probe and wait for it to exit.
141	pub fn stop(&mut self) {
142		let Some(worker) = self.worker.take() else {
143			return;
144		};
145		let (lock, wake) = &*self.shared;
146		let mut shared = lock.lock();
147		shared.stopping = true;
148		drop(shared);
149		wake.notify_one();
150		let _ = worker.join();
151	}
152}
153
154impl Drop for LoopWatchdog {
155	fn drop(&mut self) {
156		self.stop();
157	}
158}
159
160#[cfg(test)]
161mod tests {
162	use std::time::Duration;
163
164	use super::LoopWatchdogCore;
165
166	#[test]
167	fn reports_a_300ms_stall_with_phase() {
168		let mut watchdog = LoopWatchdogCore::new(Duration::ZERO);
169		watchdog.set_phase("render");
170		let report = watchdog
171			.check(Duration::from_millis(300))
172			.expect("300 ms should exceed the stall threshold");
173		assert_eq!(report.elapsed, Duration::from_millis(300));
174		assert_eq!(report.phase, "render");
175		assert_eq!(watchdog.check(Duration::from_millis(400)), None);
176	}
177
178	#[test]
179	fn ignores_a_90s_system_sleep_gap() {
180		let mut watchdog = LoopWatchdogCore::new(Duration::ZERO);
181		watchdog.set_phase("render");
182		assert_eq!(watchdog.check(Duration::from_secs(90)), None);
183	}
184
185	#[test]
186	fn stays_silent_under_normal_ticks() {
187		let mut watchdog = LoopWatchdogCore::new(Duration::ZERO);
188		for millis in [200, 400, 600, 800] {
189			let now = Duration::from_millis(millis);
190			assert_eq!(watchdog.check(now), None);
191			watchdog.tick(now);
192		}
193	}
194}
omp_tui/watchdog.rs

omp_tui/
watchdog.rs
