Skip to main content

gam_gpu/
engagement.rs

1//! One-shot engagement reports for device routes.
2//!
3//! The #1551 "GPU 0%" class: an `Auto` run that silently declines the device
4//! and falls back to the CPU otherwise leaves no trace of WHY. Every router
5//! reports the first engagement and the first decline it sees, once per
6//! process per route — the routes are per-minibatch or per-iterate, so an
7//! unconditional line would flood the fit log with thousands of identical
8//! entries. Routed through `log::warn!`, the repo's sanctioned diagnostics
9//! path, so an initialised `log` backend lands it in the job logs.
10
11use std::sync::Mutex;
12
13/// Routes that have already reported, by `(route, engaged)`.
14static REPORTED: Mutex<Vec<(&'static str, bool)>> = Mutex::new(Vec::new());
15
16/// Report `route`'s first engagement (`engaged == true`) or first decline
17/// (`engaged == false`, with `fallback` naming what runs instead — "falling
18/// back to CPU", "CPU reference"); later calls with the same `(route,
19/// engaged)` are silent.
20pub fn note_route_engagement(
21    route: &'static str,
22    fallback: &'static str,
23    engaged: bool,
24    detail: &str,
25) {
26    let first = match REPORTED.lock() {
27        Ok(mut reported) => {
28            if reported.contains(&(route, engaged)) {
29                false
30            } else {
31                reported.push((route, engaged));
32                true
33            }
34        }
35        // A poisoned registry only means a reporter panicked mid-push; losing
36        // the once-only guarantee is preferable to losing the report.
37        Err(_) => true,
38    };
39    if !first {
40        return;
41    }
42    if engaged {
43        log::warn!("[{route}] device ENGAGED: {detail}");
44    } else {
45        log::warn!("[{route}] device DECLINED - {fallback}: {detail}");
46    }
47}