self_hosted_node/
metrics.rs

1use std::time::{Duration, Instant};
2
3use metrics::{counter, gauge, histogram};
4use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
5use tracing::{info, warn};
6
7const ROOMS_HOSTED: &str = "manabrew_node_rooms_hosted";
8const GAMES_ACTIVE: &str = "manabrew_node_games_active";
9const GAME_DURATION_SECONDS: &str = "manabrew_node_game_duration_seconds";
10const FORGE_DECISION_STAGE_SECONDS: &str = "manabrew_node_forge_decision_stage_seconds";
11const FORGE_CHECKPOINT_SECONDS: &str = "manabrew_node_forge_checkpoint_seconds";
12const FORGE_CHECKPOINT_DECISION_SECONDS: &str = "manabrew_node_forge_checkpoint_decision_seconds";
13const FORGE_DECISION_SECONDS: &str = "manabrew_node_forge_decision_seconds";
14const ENGINE_ERRORS: &str = "manabrew_node_engine_errors_total";
15const RELAY_RECONNECTS: &str = "manabrew_node_relay_reconnects_total";
16const BUILD_INFO: &str = "manabrew_node_build_info";
17const RELAY_SEND_SECONDS: &str = "manabrew_node_relay_send_seconds";
18const JVM_GC_PAUSE_SECONDS: &str = "manabrew_node_jvm_gc_pause_seconds";
19const JVM_GC_TOTAL: &str = "manabrew_node_jvm_gc_total";
20const JVM_HEAP_AFTER_GC_BYTES: &str = "manabrew_node_jvm_heap_after_gc_bytes";
21const ENGINE_GC_COLLECTIONS: &str = "manabrew_node_engine_gc_collections_total";
22const ENGINE_GC_PAUSE_MILLIS: &str = "manabrew_node_engine_gc_pause_millis_total";
23const ENGINE_HEAP_USED_BYTES: &str = "manabrew_node_engine_heap_used_bytes";
24const ENGINE_HEAP_MAX_BYTES: &str = "manabrew_node_engine_heap_max_bytes";
25const ENGINE_STALL_MILLIS: &str = "manabrew_node_engine_stall_millis_total";
26const ENGINE_LONG_STALLS: &str = "manabrew_node_engine_long_stalls_total";
27const ENGINE_STALL_MAX_MILLIS: &str = "manabrew_node_engine_stall_max_millis";
28
29const LABEL_POOL: &str = "pool";
30const LABEL_KIND: &str = "kind";
31const LABEL_CLEAN: &str = "clean";
32const LABEL_PLAYERS: &str = "players";
33const LABEL_SIGNATURE: &str = "signature";
34const LABEL_STAGE: &str = "stage";
35const LABEL_COLLECTOR: &str = "collector";
36const LABEL_VERSION: &str = "version";
37const LABEL_SEATS: &str = "seats";
38
39const ENV_PUSH_URL: &str = "SELF_HOSTED_NODE_METRICS_PUSH_URL";
40const ENV_PUSH_USERNAME: &str = "SELF_HOSTED_NODE_METRICS_PUSH_USERNAME";
41const ENV_PUSH_PASSWORD: &str = "SELF_HOSTED_NODE_METRICS_PUSH_PASSWORD";
42
43const PUSH_INTERVAL: Duration = Duration::from_secs(15);
44
45// Boundaries are close together through the range decisions actually land in.
46// `histogram_quantile` interpolates linearly inside a bucket, so a wide one
47// flatters the quantile upward: with 2.0 and 5.0 adjacent, a window holding a
48// single decision somewhere between them reported 4.58s.
49const DECISION_BUCKETS: &[f64] = &[
50    0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
51];
52
53#[derive(Clone, Copy)]
54pub enum PoolKind {
55    Solo,
56    Pod,
57}
58
59impl PoolKind {
60    fn as_str(self) -> &'static str {
61        match self {
62            Self::Solo => "solo",
63            Self::Pod => "pod",
64        }
65    }
66}
67
68#[derive(Clone, Copy)]
69enum ErrorSignature {
70    IndexOob,
71    UnsupportedKind,
72    Comparator,
73    UnwrapNone,
74    Trigger,
75    Other,
76}
77
78impl ErrorSignature {
79    fn bucket(message: &str) -> Self {
80        let lower = message.to_lowercase();
81        if lower.contains("index out of bounds") || lower.contains("indexoutofbounds") {
82            Self::IndexOob
83        } else if lower.contains("unsupported") || lower.contains("unimplemented") {
84            Self::UnsupportedKind
85        } else if lower.contains("comparator") {
86            Self::Comparator
87        } else if lower.contains("unwrap") || lower.contains("nullpointer") {
88            Self::UnwrapNone
89        } else if lower.contains("trigger") {
90            Self::Trigger
91        } else {
92            Self::Other
93        }
94    }
95
96    fn as_str(self) -> &'static str {
97        match self {
98            Self::IndexOob => "index_oob",
99            Self::UnsupportedKind => "unsupported_kind",
100            Self::Comparator => "comparator",
101            Self::UnwrapNone => "unwrap_none",
102            Self::Trigger => "trigger",
103            Self::Other => "other",
104        }
105    }
106}
107
108pub fn init_from_env() {
109    let Some(url) = std::env::var(ENV_PUSH_URL).ok().filter(|v| !v.is_empty()) else {
110        return;
111    };
112    let username = std::env::var(ENV_PUSH_USERNAME)
113        .ok()
114        .filter(|v| !v.is_empty());
115    let password = std::env::var(ENV_PUSH_PASSWORD)
116        .ok()
117        .filter(|v| !v.is_empty());
118    let _ = rustls::crypto::ring::default_provider().install_default();
119    let builder = match PrometheusBuilder::new()
120        .set_buckets_for_metric(
121            Matcher::Full(FORGE_DECISION_SECONDS.to_string()),
122            DECISION_BUCKETS,
123        )
124        .expect("decision buckets are a non-empty literal")
125        .set_buckets_for_metric(
126            Matcher::Prefix("manabrew_node_forge_checkpoint".to_string()),
127            &[
128                0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1,
129                0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
130            ],
131        )
132        .expect("checkpoint buckets are a non-empty literal")
133        .with_push_gateway(&url, PUSH_INTERVAL, username, password, false)
134    {
135        Ok(builder) => builder,
136        Err(error) => {
137            warn!(%error, url, "invalid metrics push gateway config");
138            return;
139        }
140    };
141    match builder.install() {
142        Ok(()) => {
143            gauge!(BUILD_INFO, LABEL_VERSION => env!("CARGO_PKG_VERSION")).set(1.0);
144            info!(url, "metrics push exporter installed");
145        }
146        Err(error) => warn!(%error, url, "failed to install metrics push exporter"),
147    }
148}
149
150pub struct RoomHostedGuard {
151    pool: PoolKind,
152}
153
154impl RoomHostedGuard {
155    pub fn new(pool: PoolKind) -> Self {
156        gauge!(ROOMS_HOSTED, LABEL_POOL => pool.as_str()).increment(1.0);
157        RoomHostedGuard { pool }
158    }
159}
160
161impl Drop for RoomHostedGuard {
162    fn drop(&mut self) {
163        gauge!(ROOMS_HOSTED, LABEL_POOL => self.pool.as_str()).decrement(1.0);
164    }
165}
166
167pub fn record_relay_send(elapsed: Duration) {
168    histogram!(RELAY_SEND_SECONDS).record(elapsed.as_secs_f64());
169}
170
171/// Pause and retained heap reported by the engine JVM's own GC log. The fleet
172/// ships no logs, so without this a stalled JVM is invisible from off-box.
173pub fn record_jvm_gc(kind: &'static str, pause: Duration, heap_after_mb: Option<u64>) {
174    histogram!(JVM_GC_PAUSE_SECONDS, LABEL_KIND => kind).record(pause.as_secs_f64());
175    counter!(JVM_GC_TOTAL, LABEL_KIND => kind).increment(1);
176    if let Some(megabytes) = heap_after_mb {
177        gauge!(JVM_HEAP_AFTER_GC_BYTES, LABEL_KIND => kind).set((megabytes * 1024 * 1024) as f64);
178    }
179}
180
181/// Isolate-wide GC and heap, polled from the engine rather than parsed from a
182/// log the graal fleet does not write. Cumulative, so
183/// `rate(engine_gc_pause_millis_total) / 1000` is the fraction of wall clock
184/// stopped, which is the figure that identified #684. Every room on the node
185/// shares one isolate, so a collection here stops all of them at once.
186/// The collection counters carry a `collector` label and are emitted by
187/// [`record_engine_gc_collector`]; emitting an unlabelled copy of the same name
188/// here would double every `sum`.
189pub fn record_engine_gc(_collections: i64, _pause_millis: i64, heap_used: u64, heap_max: u64) {
190    gauge!(ENGINE_HEAP_USED_BYTES).set(heap_used as f64);
191    gauge!(ENGINE_HEAP_MAX_BYTES).set(heap_max as f64);
192}
193
194/// Split by collector. Incremental collections are a few milliseconds and
195/// harmless; a complete collection traces the whole live set, and this live set
196/// is a permanently reachable card database, so it costs about 1.3ms per MB
197/// whether it reclaims anything or not. Only the complete series is worth
198/// alerting on.
199pub fn record_engine_gc_collector(collector: &str, collections: u64, pause_millis: u64) {
200    let name = collector.to_string();
201    counter!(ENGINE_GC_COLLECTIONS, LABEL_COLLECTOR => name.clone()).absolute(collections);
202    counter!(ENGINE_GC_PAUSE_MILLIS, LABEL_COLLECTOR => name).absolute(pause_millis);
203}
204
205/// Wall clock the engine was stopped, from a probe inside the isolate rather
206/// than from the collector. [`record_engine_gc_collector`] is the better signal
207/// where it exists, but Substrate registers collector beans for its Serial GC
208/// only, so on the G1 image it reports nothing at all and this is the only
209/// series that shows a stall. Cumulative, so
210/// `rate(engine_stall_millis_total) / 1000` is the fraction of wall clock lost.
211pub fn record_engine_stall(stalled_millis: u64, max_stall_millis: u64, long_stalls: u64) {
212    counter!(ENGINE_STALL_MILLIS).absolute(stalled_millis);
213    counter!(ENGINE_LONG_STALLS).absolute(long_stalls);
214    gauge!(ENGINE_STALL_MAX_MILLIS).set(max_stall_millis as f64);
215}
216
217pub fn record_relay_reconnect() {
218    counter!(RELAY_RECONNECTS).increment(1);
219}
220
221pub fn record_forge_checkpoint(seats: usize, copy: Duration, bookkeeping: Duration) {
222    for (stage, elapsed) in [
223        ("copy", copy),
224        ("bookkeeping", bookkeeping),
225        ("total", copy + bookkeeping),
226    ] {
227        histogram!(FORGE_CHECKPOINT_SECONDS, LABEL_STAGE => stage, LABEL_SEATS => seats.to_string())
228            .record(elapsed.as_secs_f64());
229    }
230}
231
232pub fn record_forge_checkpoint_decision(seats: usize, elapsed: Duration) {
233    histogram!(FORGE_CHECKPOINT_DECISION_SECONDS, LABEL_SEATS => seats.to_string())
234        .record(elapsed.as_secs_f64());
235}
236
237pub fn record_forge_decision_stage(stage: &'static str, elapsed: Duration) {
238    histogram!(FORGE_DECISION_STAGE_SECONDS, LABEL_STAGE => stage).record(elapsed.as_secs_f64());
239}
240
241/// The rules work between a seat answering and the next prompt appearing, split
242/// by seat count. Measured over two days of captures, four seats run this at a
243/// p99 of about 2.9s against 0.35s for two, and put 2-5% of decisions over two
244/// seconds where two seats put none. A fleet-wide quantile averages the many
245/// clean rooms against the few bad ones and shows neither.
246///
247/// There is no bot count. It was recorded from the engine's own AI seat list,
248/// which is empty for every hosted game because bots join as ordinary relay
249/// clients, so the label read zero on every series it ever produced. Only the
250/// relay knows which seats are bots, and the node is not told.
251pub fn record_forge_decision(seats: usize, elapsed: Duration) {
252    histogram!(FORGE_DECISION_SECONDS, LABEL_SEATS => seats.to_string())
253        .record(elapsed.as_secs_f64());
254}
255
256pub fn record_engine_session_started() {
257    gauge!(GAMES_ACTIVE).increment(1.0);
258}
259
260pub fn record_engine_session_finished(players: usize, started: Instant, fatal: Option<&str>) {
261    gauge!(GAMES_ACTIVE).decrement(1.0);
262    let clean = if fatal.is_none() { "true" } else { "false" };
263    histogram!(
264        GAME_DURATION_SECONDS,
265        LABEL_PLAYERS => players.to_string(),
266        LABEL_CLEAN => clean
267    )
268    .record(started.elapsed().as_secs_f64());
269    if let Some(message) = fatal {
270        counter!(ENGINE_ERRORS, LABEL_SIGNATURE => ErrorSignature::bucket(message).as_str())
271            .increment(1);
272    }
273}
self_hosted_node/metrics.rs

self_hosted_node/
metrics.rs
