self_hosted_node/
metrics.rs

1use std::time::{Duration, Instant};
2
3use metrics::{counter, gauge, histogram};
4use metrics_exporter_prometheus::PrometheusBuilder;
5use tracing::{info, warn};
6
7const ROOMS_HOSTED: &str = "manabrew_node_rooms_hosted";
8const GAMES_ACTIVE: &str = "manabrew_node_games_active";
9const GAME_DURATION_SECONDS: &str = "manabrew_node_game_duration_seconds";
10const FORGE_DECISION_STAGE_SECONDS: &str = "manabrew_node_forge_decision_stage_seconds";
11const ENGINE_ERRORS: &str = "manabrew_node_engine_errors_total";
12const RELAY_RECONNECTS: &str = "manabrew_node_relay_reconnects_total";
13const BUILD_INFO: &str = "manabrew_node_build_info";
14const RELAY_SEND_SECONDS: &str = "manabrew_node_relay_send_seconds";
15const JVM_GC_PAUSE_SECONDS: &str = "manabrew_node_jvm_gc_pause_seconds";
16const JVM_GC_TOTAL: &str = "manabrew_node_jvm_gc_total";
17const JVM_HEAP_AFTER_GC_BYTES: &str = "manabrew_node_jvm_heap_after_gc_bytes";
18
19const LABEL_POOL: &str = "pool";
20const LABEL_KIND: &str = "kind";
21const LABEL_CLEAN: &str = "clean";
22const LABEL_PLAYERS: &str = "players";
23const LABEL_SIGNATURE: &str = "signature";
24const LABEL_STAGE: &str = "stage";
25const LABEL_VERSION: &str = "version";
26
27const ENV_PUSH_URL: &str = "SELF_HOSTED_NODE_METRICS_PUSH_URL";
28const ENV_PUSH_USERNAME: &str = "SELF_HOSTED_NODE_METRICS_PUSH_USERNAME";
29const ENV_PUSH_PASSWORD: &str = "SELF_HOSTED_NODE_METRICS_PUSH_PASSWORD";
30
31const PUSH_INTERVAL: Duration = Duration::from_secs(15);
32
33#[derive(Clone, Copy)]
34pub enum PoolKind {
35    Solo,
36    Pod,
37}
38
39impl PoolKind {
40    fn as_str(self) -> &'static str {
41        match self {
42            Self::Solo => "solo",
43            Self::Pod => "pod",
44        }
45    }
46}
47
48#[derive(Clone, Copy)]
49enum ErrorSignature {
50    IndexOob,
51    UnsupportedKind,
52    Comparator,
53    UnwrapNone,
54    Trigger,
55    Other,
56}
57
58impl ErrorSignature {
59    fn bucket(message: &str) -> Self {
60        let lower = message.to_lowercase();
61        if lower.contains("index out of bounds") || lower.contains("indexoutofbounds") {
62            Self::IndexOob
63        } else if lower.contains("unsupported") || lower.contains("unimplemented") {
64            Self::UnsupportedKind
65        } else if lower.contains("comparator") {
66            Self::Comparator
67        } else if lower.contains("unwrap") || lower.contains("nullpointer") {
68            Self::UnwrapNone
69        } else if lower.contains("trigger") {
70            Self::Trigger
71        } else {
72            Self::Other
73        }
74    }
75
76    fn as_str(self) -> &'static str {
77        match self {
78            Self::IndexOob => "index_oob",
79            Self::UnsupportedKind => "unsupported_kind",
80            Self::Comparator => "comparator",
81            Self::UnwrapNone => "unwrap_none",
82            Self::Trigger => "trigger",
83            Self::Other => "other",
84        }
85    }
86}
87
88pub fn init_from_env() {
89    let Some(url) = std::env::var(ENV_PUSH_URL).ok().filter(|v| !v.is_empty()) else {
90        return;
91    };
92    let username = std::env::var(ENV_PUSH_USERNAME)
93        .ok()
94        .filter(|v| !v.is_empty());
95    let password = std::env::var(ENV_PUSH_PASSWORD)
96        .ok()
97        .filter(|v| !v.is_empty());
98    let _ = rustls::crypto::ring::default_provider().install_default();
99    let builder = match PrometheusBuilder::new().with_push_gateway(
100        &url,
101        PUSH_INTERVAL,
102        username,
103        password,
104        false,
105    ) {
106        Ok(builder) => builder,
107        Err(error) => {
108            warn!(%error, url, "invalid metrics push gateway config");
109            return;
110        }
111    };
112    match builder.install() {
113        Ok(()) => {
114            gauge!(BUILD_INFO, LABEL_VERSION => env!("CARGO_PKG_VERSION")).set(1.0);
115            info!(url, "metrics push exporter installed");
116        }
117        Err(error) => warn!(%error, url, "failed to install metrics push exporter"),
118    }
119}
120
121pub struct RoomHostedGuard {
122    pool: PoolKind,
123}
124
125impl RoomHostedGuard {
126    pub fn new(pool: PoolKind) -> Self {
127        gauge!(ROOMS_HOSTED, LABEL_POOL => pool.as_str()).increment(1.0);
128        RoomHostedGuard { pool }
129    }
130}
131
132impl Drop for RoomHostedGuard {
133    fn drop(&mut self) {
134        gauge!(ROOMS_HOSTED, LABEL_POOL => self.pool.as_str()).decrement(1.0);
135    }
136}
137
138pub fn record_relay_send(elapsed: Duration) {
139    histogram!(RELAY_SEND_SECONDS).record(elapsed.as_secs_f64());
140}
141
142/// Pause and retained heap reported by the engine JVM's own GC log. The fleet
143/// ships no logs, so without this a stalled JVM is invisible from off-box.
144pub fn record_jvm_gc(kind: &'static str, pause: Duration, heap_after_mb: Option<u64>) {
145    histogram!(JVM_GC_PAUSE_SECONDS, LABEL_KIND => kind).record(pause.as_secs_f64());
146    counter!(JVM_GC_TOTAL, LABEL_KIND => kind).increment(1);
147    if let Some(megabytes) = heap_after_mb {
148        gauge!(JVM_HEAP_AFTER_GC_BYTES, LABEL_KIND => kind).set((megabytes * 1024 * 1024) as f64);
149    }
150}
151
152pub fn record_relay_reconnect() {
153    counter!(RELAY_RECONNECTS).increment(1);
154}
155
156pub fn record_forge_decision_stage(stage: &'static str, elapsed: Duration) {
157    histogram!(FORGE_DECISION_STAGE_SECONDS, LABEL_STAGE => stage).record(elapsed.as_secs_f64());
158}
159
160pub fn record_engine_session_started() {
161    gauge!(GAMES_ACTIVE).increment(1.0);
162}
163
164pub fn record_engine_session_finished(players: usize, started: Instant, fatal: Option<&str>) {
165    gauge!(GAMES_ACTIVE).decrement(1.0);
166    let clean = if fatal.is_none() { "true" } else { "false" };
167    histogram!(
168        GAME_DURATION_SECONDS,
169        LABEL_PLAYERS => players.to_string(),
170        LABEL_CLEAN => clean
171    )
172    .record(started.elapsed().as_secs_f64());
173    if let Some(message) = fatal {
174        counter!(ENGINE_ERRORS, LABEL_SIGNATURE => ErrorSignature::bucket(message).as_str())
175            .increment(1);
176    }
177}
self_hosted_node/metrics.rs

self_hosted_node/
metrics.rs
