mentedb_server/
metrics.rs

1//! Prometheus metrics exposed at `GET /metrics` in the standard text format.
2//!
3//! Aggregate only, with no per-account or per-agent labels, so it is safe to
4//! scrape without auth. It covers process health (CPU and memory on Linux) plus
5//! MenteDB gauges (memories stored, uptime, live cluster nodes) and HTTP request
6//! rate and latency, which together answer the "is it healthy and how hard is it
7//! working" question a dashboard needs.
8
9use std::sync::Arc;
10use std::sync::OnceLock;
11use std::time::Instant;
12
13use axum::{
14    body::Body,
15    extract::{Request, State},
16    http::{StatusCode, header},
17    middleware::Next,
18    response::{IntoResponse, Response},
19};
20use prometheus::{
21    Encoder, HistogramVec, IntCounterVec, IntGauge, Registry, TextEncoder, histogram_opts, opts,
22};
23
24use crate::state::AppState;
25
26/// The metric set, held in a process-global registry.
27pub struct Metrics {
28    registry: Registry,
29    uptime: IntGauge,
30    memory_count: IntGauge,
31    live_nodes: IntGauge,
32    http_requests: IntCounterVec,
33    http_duration: HistogramVec,
34    // Engine metrics, refreshed from `db.metrics()` on each scrape. Cumulative
35    // totals are exposed as gauges set to the running total (rate() still works).
36    stores: IntGauge,
37    recalls: IntGauge,
38    bp_hits: IntGauge,
39    bp_misses: IntGauge,
40    bp_evictions: IntGauge,
41    bp_pages: IntGauge,
42    storage_bytes: IntGauge,
43    pages: IntGauge,
44    vector_index_size: IntGauge,
45    graph_nodes: IntGauge,
46    standing_rules: IntGauge,
47}
48
49static METRICS: OnceLock<Metrics> = OnceLock::new();
50
51/// The process-global metric set, initialized on first use.
52pub fn metrics() -> &'static Metrics {
53    METRICS.get_or_init(Metrics::new)
54}
55
56impl Metrics {
57    fn new() -> Self {
58        let registry = Registry::new();
59
60        // Process CPU and memory, populated on Linux via /proc.
61        #[cfg(target_os = "linux")]
62        {
63            let collector = prometheus::process_collector::ProcessCollector::for_self();
64            let _ = registry.register(Box::new(collector));
65        }
66
67        // `mentedb_up` is a constant 1; register it and let the registry own it.
68        let up = IntGauge::new("mentedb_up", "1 if the server is serving").unwrap();
69        up.set(1);
70        registry.register(Box::new(up)).ok();
71
72        let uptime = IntGauge::new("mentedb_uptime_seconds", "Seconds since start").unwrap();
73        let memory_count =
74            IntGauge::new("mentedb_memory_count", "Memories stored on this node").unwrap();
75        let live_nodes = IntGauge::new(
76            "mentedb_cluster_live_nodes",
77            "Live nodes in the gossip cluster (1 when sharding is off)",
78        )
79        .unwrap();
80        let http_requests = IntCounterVec::new(
81            opts!(
82                "mentedb_http_requests_total",
83                "HTTP requests by method and status"
84            ),
85            &["method", "status"],
86        )
87        .unwrap();
88        let http_duration = HistogramVec::new(
89            histogram_opts!(
90                "mentedb_http_request_duration_seconds",
91                "HTTP request latency in seconds"
92            ),
93            &["method"],
94        )
95        .unwrap();
96
97        // Engine gauges: create, register, and return in one step.
98        let gauge = |name: &str, help: &str| {
99            let g = IntGauge::new(name, help).unwrap();
100            registry.register(Box::new(g.clone())).ok();
101            g
102        };
103        let stores = gauge("mentedb_stores_total", "Memories written (writes)");
104        let recalls = gauge("mentedb_recalls_total", "Recall/query operations (reads)");
105        let bp_hits = gauge("mentedb_buffer_pool_hits_total", "Page cache hits");
106        let bp_misses = gauge("mentedb_buffer_pool_misses_total", "Page cache misses");
107        let bp_evictions = gauge(
108            "mentedb_buffer_pool_evictions_total",
109            "Page cache evictions",
110        );
111        let bp_pages = gauge("mentedb_buffer_pool_pages", "Frames holding a page");
112        let storage_bytes = gauge("mentedb_storage_bytes", "On-disk data size in bytes");
113        let pages = gauge("mentedb_pages_total", "Pages in the store");
114        let vector_index_size = gauge("mentedb_vector_index_size", "Vectors in the HNSW index");
115        let graph_nodes = gauge("mentedb_graph_nodes", "Nodes in the memory graph");
116        let standing_rules = gauge(
117            "mentedb_standing_rules",
118            "Pinned standing rules (scope:always)",
119        );
120
121        for g in [&uptime, &memory_count, &live_nodes] {
122            registry.register(Box::new(g.clone())).ok();
123        }
124        registry.register(Box::new(http_requests.clone())).ok();
125        registry.register(Box::new(http_duration.clone())).ok();
126
127        Self {
128            registry,
129            uptime,
130            memory_count,
131            live_nodes,
132            http_requests,
133            http_duration,
134            stores,
135            recalls,
136            bp_hits,
137            bp_misses,
138            bp_evictions,
139            bp_pages,
140            storage_bytes,
141            pages,
142            vector_index_size,
143            graph_nodes,
144            standing_rules,
145        }
146    }
147}
148
149/// Middleware: time every request and count it by method and response status.
150pub async fn track(req: Request, next: Next) -> Response {
151    let method = req.method().as_str().to_string();
152    let start = Instant::now();
153    let resp = next.run(req).await;
154    let m = metrics();
155    let status = resp.status().as_u16().to_string();
156    m.http_requests.with_label_values(&[&method, &status]).inc();
157    m.http_duration
158        .with_label_values(&[&method])
159        .observe(start.elapsed().as_secs_f64());
160    resp
161}
162
163/// `GET /metrics`: refresh the point-in-time gauges, then encode the registry.
164pub async fn handler(State(state): State<Arc<AppState>>) -> impl IntoResponse {
165    let m = metrics();
166    m.uptime.set(state.start_time.elapsed().as_secs() as i64);
167    let dm = state.db.metrics();
168    m.memory_count.set(dm.memory_count as i64);
169    m.stores.set(dm.stores as i64);
170    m.recalls.set(dm.recalls as i64);
171    m.bp_hits.set(dm.buffer_pool_hits as i64);
172    m.bp_misses.set(dm.buffer_pool_misses as i64);
173    m.bp_evictions.set(dm.buffer_pool_evictions as i64);
174    m.bp_pages.set(dm.buffer_pool_pages as i64);
175    m.storage_bytes.set(dm.storage_bytes as i64);
176    m.pages.set(dm.page_count as i64);
177    m.vector_index_size.set(dm.vector_index_size as i64);
178    m.graph_nodes.set(dm.graph_nodes as i64);
179    m.standing_rules.set(dm.standing_rules as i64);
180    let live = match &state.cluster {
181        Some(c) => c.live_node_count().await as i64,
182        None => 1,
183    };
184    m.live_nodes.set(live);
185
186    let mut buf = Vec::new();
187    if TextEncoder::new()
188        .encode(&m.registry.gather(), &mut buf)
189        .is_err()
190    {
191        return (StatusCode::INTERNAL_SERVER_ERROR, "metric encode error").into_response();
192    }
193    (
194        [(header::CONTENT_TYPE, "text/plain; version=0.0.4")],
195        Body::from(buf),
196    )
197        .into_response()
198}
mentedb_server/metrics.rs

mentedb_server/
metrics.rs
