self_hosted_node/
metrics.rs

1use std::time::{Duration, Instant};
2
3use metrics::{counter, gauge, histogram};
4use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
5use tracing::{info, warn};
6
7const ROOMS_HOSTED: &str = "manabrew_node_rooms_hosted";
8const GAMES_ACTIVE: &str = "manabrew_node_games_active";
9const GAME_DURATION_SECONDS: &str = "manabrew_node_game_duration_seconds";
10const FORGE_DECISION_STAGE_SECONDS: &str = "manabrew_node_forge_decision_stage_seconds";
11const FORGE_DECISION_SECONDS: &str = "manabrew_node_forge_decision_seconds";
12const ENGINE_ERRORS: &str = "manabrew_node_engine_errors_total";
13const RELAY_RECONNECTS: &str = "manabrew_node_relay_reconnects_total";
14const BUILD_INFO: &str = "manabrew_node_build_info";
15const RELAY_SEND_SECONDS: &str = "manabrew_node_relay_send_seconds";
16const JVM_GC_PAUSE_SECONDS: &str = "manabrew_node_jvm_gc_pause_seconds";
17const JVM_GC_TOTAL: &str = "manabrew_node_jvm_gc_total";
18const JVM_HEAP_AFTER_GC_BYTES: &str = "manabrew_node_jvm_heap_after_gc_bytes";
19const ENGINE_GC_COLLECTIONS: &str = "manabrew_node_engine_gc_collections_total";
20const ENGINE_GC_PAUSE_MILLIS: &str = "manabrew_node_engine_gc_pause_millis_total";
21const ENGINE_HEAP_USED_BYTES: &str = "manabrew_node_engine_heap_used_bytes";
22const ENGINE_HEAP_MAX_BYTES: &str = "manabrew_node_engine_heap_max_bytes";
23const ENGINE_STALL_MILLIS: &str = "manabrew_node_engine_stall_millis_total";
24const ENGINE_LONG_STALLS: &str = "manabrew_node_engine_long_stalls_total";
25const ENGINE_STALL_MAX_MILLIS: &str = "manabrew_node_engine_stall_max_millis";
26
27const LABEL_POOL: &str = "pool";
28const LABEL_KIND: &str = "kind";
29const LABEL_CLEAN: &str = "clean";
30const LABEL_PLAYERS: &str = "players";
31const LABEL_SIGNATURE: &str = "signature";
32const LABEL_STAGE: &str = "stage";
33const LABEL_COLLECTOR: &str = "collector";
34const LABEL_VERSION: &str = "version";
35const LABEL_SEATS: &str = "seats";
36
37const ENV_PUSH_URL: &str = "SELF_HOSTED_NODE_METRICS_PUSH_URL";
38const ENV_PUSH_USERNAME: &str = "SELF_HOSTED_NODE_METRICS_PUSH_USERNAME";
39const ENV_PUSH_PASSWORD: &str = "SELF_HOSTED_NODE_METRICS_PUSH_PASSWORD";
40
41const PUSH_INTERVAL: Duration = Duration::from_secs(15);
42
43// Boundaries are close together through the range decisions actually land in.
44// `histogram_quantile` interpolates linearly inside a bucket, so a wide one
45// flatters the quantile upward: with 2.0 and 5.0 adjacent, a window holding a
46// single decision somewhere between them reported 4.58s.
47const DECISION_BUCKETS: &[f64] = &[
48    0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
49];
50
51#[derive(Clone, Copy)]
52pub enum PoolKind {
53    Solo,
54    Pod,
55}
56
57impl PoolKind {
58    fn as_str(self) -> &'static str {
59        match self {
60            Self::Solo => "solo",
61            Self::Pod => "pod",
62        }
63    }
64}
65
66#[derive(Clone, Copy)]
67enum ErrorSignature {
68    IndexOob,
69    UnsupportedKind,
70    Comparator,
71    UnwrapNone,
72    Trigger,
73    Other,
74}
75
76impl ErrorSignature {
77    fn bucket(message: &str) -> Self {
78        let lower = message.to_lowercase();
79        if lower.contains("index out of bounds") || lower.contains("indexoutofbounds") {
80            Self::IndexOob
81        } else if lower.contains("unsupported") || lower.contains("unimplemented") {
82            Self::UnsupportedKind
83        } else if lower.contains("comparator") {
84            Self::Comparator
85        } else if lower.contains("unwrap") || lower.contains("nullpointer") {
86            Self::UnwrapNone
87        } else if lower.contains("trigger") {
88            Self::Trigger
89        } else {
90            Self::Other
91        }
92    }
93
94    fn as_str(self) -> &'static str {
95        match self {
96            Self::IndexOob => "index_oob",
97            Self::UnsupportedKind => "unsupported_kind",
98            Self::Comparator => "comparator",
99            Self::UnwrapNone => "unwrap_none",
100            Self::Trigger => "trigger",
101            Self::Other => "other",
102        }
103    }
104}
105
106pub fn init_from_env() {
107    let Some(url) = std::env::var(ENV_PUSH_URL).ok().filter(|v| !v.is_empty()) else {
108        return;
109    };
110    let username = std::env::var(ENV_PUSH_USERNAME)
111        .ok()
112        .filter(|v| !v.is_empty());
113    let password = std::env::var(ENV_PUSH_PASSWORD)
114        .ok()
115        .filter(|v| !v.is_empty());
116    let _ = rustls::crypto::ring::default_provider().install_default();
117    // Only this metric gets explicit buckets. Everything else stays a summary,
118    // whose quantiles are per process and cannot be aggregated across the fleet
119    // — which is why a fleet-wide p99 was never really a fleet-wide p99.
120    let builder = match PrometheusBuilder::new()
121        .set_buckets_for_metric(
122            Matcher::Full(FORGE_DECISION_SECONDS.to_string()),
123            DECISION_BUCKETS,
124        )
125        .expect("decision buckets are a non-empty literal")
126        .with_push_gateway(&url, PUSH_INTERVAL, username, password, false)
127    {
128        Ok(builder) => builder,
129        Err(error) => {
130            warn!(%error, url, "invalid metrics push gateway config");
131            return;
132        }
133    };
134    match builder.install() {
135        Ok(()) => {
136            gauge!(BUILD_INFO, LABEL_VERSION => env!("CARGO_PKG_VERSION")).set(1.0);
137            info!(url, "metrics push exporter installed");
138        }
139        Err(error) => warn!(%error, url, "failed to install metrics push exporter"),
140    }
141}
142
143pub struct RoomHostedGuard {
144    pool: PoolKind,
145}
146
147impl RoomHostedGuard {
148    pub fn new(pool: PoolKind) -> Self {
149        gauge!(ROOMS_HOSTED, LABEL_POOL => pool.as_str()).increment(1.0);
150        RoomHostedGuard { pool }
151    }
152}
153
154impl Drop for RoomHostedGuard {
155    fn drop(&mut self) {
156        gauge!(ROOMS_HOSTED, LABEL_POOL => self.pool.as_str()).decrement(1.0);
157    }
158}
159
160pub fn record_relay_send(elapsed: Duration) {
161    histogram!(RELAY_SEND_SECONDS).record(elapsed.as_secs_f64());
162}
163
164/// Pause and retained heap reported by the engine JVM's own GC log. The fleet
165/// ships no logs, so without this a stalled JVM is invisible from off-box.
166pub fn record_jvm_gc(kind: &'static str, pause: Duration, heap_after_mb: Option<u64>) {
167    histogram!(JVM_GC_PAUSE_SECONDS, LABEL_KIND => kind).record(pause.as_secs_f64());
168    counter!(JVM_GC_TOTAL, LABEL_KIND => kind).increment(1);
169    if let Some(megabytes) = heap_after_mb {
170        gauge!(JVM_HEAP_AFTER_GC_BYTES, LABEL_KIND => kind).set((megabytes * 1024 * 1024) as f64);
171    }
172}
173
174/// Isolate-wide GC and heap, polled from the engine rather than parsed from a
175/// log the graal fleet does not write. Cumulative, so
176/// `rate(engine_gc_pause_millis_total) / 1000` is the fraction of wall clock
177/// stopped, which is the figure that identified #684. Every room on the node
178/// shares one isolate, so a collection here stops all of them at once.
179/// The collection counters carry a `collector` label and are emitted by
180/// [`record_engine_gc_collector`]; emitting an unlabelled copy of the same name
181/// here would double every `sum`.
182pub fn record_engine_gc(_collections: i64, _pause_millis: i64, heap_used: u64, heap_max: u64) {
183    gauge!(ENGINE_HEAP_USED_BYTES).set(heap_used as f64);
184    gauge!(ENGINE_HEAP_MAX_BYTES).set(heap_max as f64);
185}
186
187/// Split by collector. Incremental collections are a few milliseconds and
188/// harmless; a complete collection traces the whole live set, and this live set
189/// is a permanently reachable card database, so it costs about 1.3ms per MB
190/// whether it reclaims anything or not. Only the complete series is worth
191/// alerting on.
192pub fn record_engine_gc_collector(collector: &str, collections: u64, pause_millis: u64) {
193    let name = collector.to_string();
194    counter!(ENGINE_GC_COLLECTIONS, LABEL_COLLECTOR => name.clone()).absolute(collections);
195    counter!(ENGINE_GC_PAUSE_MILLIS, LABEL_COLLECTOR => name).absolute(pause_millis);
196}
197
198/// Wall clock the engine was stopped, from a probe inside the isolate rather
199/// than from the collector. [`record_engine_gc_collector`] is the better signal
200/// where it exists, but Substrate registers collector beans for its Serial GC
201/// only, so on the G1 image it reports nothing at all and this is the only
202/// series that shows a stall. Cumulative, so
203/// `rate(engine_stall_millis_total) / 1000` is the fraction of wall clock lost.
204pub fn record_engine_stall(stalled_millis: u64, max_stall_millis: u64, long_stalls: u64) {
205    counter!(ENGINE_STALL_MILLIS).absolute(stalled_millis);
206    counter!(ENGINE_LONG_STALLS).absolute(long_stalls);
207    gauge!(ENGINE_STALL_MAX_MILLIS).set(max_stall_millis as f64);
208}
209
210pub fn record_relay_reconnect() {
211    counter!(RELAY_RECONNECTS).increment(1);
212}
213
214pub fn record_forge_decision_stage(stage: &'static str, elapsed: Duration) {
215    histogram!(FORGE_DECISION_STAGE_SECONDS, LABEL_STAGE => stage).record(elapsed.as_secs_f64());
216}
217
218/// The rules work between a seat answering and the next prompt appearing, split
219/// by seat count. Measured over two days of captures, four seats run this at a
220/// p99 of about 2.9s against 0.35s for two, and put 2-5% of decisions over two
221/// seconds where two seats put none. A fleet-wide quantile averages the many
222/// clean rooms against the few bad ones and shows neither.
223///
224/// There is no bot count. It was recorded from the engine's own AI seat list,
225/// which is empty for every hosted game because bots join as ordinary relay
226/// clients, so the label read zero on every series it ever produced. Only the
227/// relay knows which seats are bots, and the node is not told.
228pub fn record_forge_decision(seats: usize, elapsed: Duration) {
229    histogram!(FORGE_DECISION_SECONDS, LABEL_SEATS => seats.to_string())
230        .record(elapsed.as_secs_f64());
231}
232
233pub fn record_engine_session_started() {
234    gauge!(GAMES_ACTIVE).increment(1.0);
235}
236
237pub fn record_engine_session_finished(players: usize, started: Instant, fatal: Option<&str>) {
238    gauge!(GAMES_ACTIVE).decrement(1.0);
239    let clean = if fatal.is_none() { "true" } else { "false" };
240    histogram!(
241        GAME_DURATION_SECONDS,
242        LABEL_PLAYERS => players.to_string(),
243        LABEL_CLEAN => clean
244    )
245    .record(started.elapsed().as_secs_f64());
246    if let Some(message) = fatal {
247        counter!(ENGINE_ERRORS, LABEL_SIGNATURE => ErrorSignature::bucket(message).as_str())
248            .increment(1);
249    }
250}
self_hosted_node/metrics.rs

self_hosted_node/
metrics.rs
