mentedb_server/
metrics.rs

1//! Prometheus metrics exposed at `GET /metrics` in the standard text format.
2//!
3//! Aggregate only, with no per-account or per-agent labels, so it is safe to
4//! scrape without auth. It covers process health (CPU and memory on Linux) plus
5//! MenteDB gauges (memories stored, uptime, live cluster nodes) and HTTP request
6//! rate and latency, which together answer the "is it healthy and how hard is it
7//! working" question a dashboard needs.
8
9use std::sync::Arc;
10use std::sync::OnceLock;
11use std::time::Instant;
12
13use axum::{
14    body::Body,
15    extract::{Request, State},
16    http::{StatusCode, header},
17    middleware::Next,
18    response::{IntoResponse, Response},
19};
20use prometheus::{
21    Encoder, HistogramVec, IntCounterVec, IntGauge, Registry, TextEncoder, histogram_opts, opts,
22};
23
24use crate::state::AppState;
25
26/// The metric set, held in a process-global registry.
27pub struct Metrics {
28    registry: Registry,
29    uptime: IntGauge,
30    memory_count: IntGauge,
31    live_nodes: IntGauge,
32    http_requests: IntCounterVec,
33    http_duration: HistogramVec,
34    // Engine metrics, refreshed from `db.metrics()` on each scrape. Cumulative
35    // totals are exposed as gauges set to the running total (rate() still works).
36    stores: IntGauge,
37    recalls: IntGauge,
38    bp_hits: IntGauge,
39    bp_misses: IntGauge,
40    bp_evictions: IntGauge,
41    bp_pages: IntGauge,
42    storage_bytes: IntGauge,
43    pages: IntGauge,
44    vector_index_size: IntGauge,
45    graph_nodes: IntGauge,
46    standing_rules: IntGauge,
47    store_latency_us: IntGauge,
48    search_latency_us: IntGauge,
49}
50
51static METRICS: OnceLock<Metrics> = OnceLock::new();
52
53/// The process-global metric set, initialized on first use.
54pub fn metrics() -> &'static Metrics {
55    METRICS.get_or_init(Metrics::new)
56}
57
58impl Metrics {
59    fn new() -> Self {
60        let registry = Registry::new();
61
62        // Process CPU and memory, populated on Linux via /proc.
63        #[cfg(target_os = "linux")]
64        {
65            let collector = prometheus::process_collector::ProcessCollector::for_self();
66            let _ = registry.register(Box::new(collector));
67        }
68
69        // `mentedb_up` is a constant 1; register it and let the registry own it.
70        let up = IntGauge::new("mentedb_up", "1 if the server is serving").unwrap();
71        up.set(1);
72        registry.register(Box::new(up)).ok();
73
74        let uptime = IntGauge::new("mentedb_uptime_seconds", "Seconds since start").unwrap();
75        let memory_count =
76            IntGauge::new("mentedb_memory_count", "Memories stored on this node").unwrap();
77        let live_nodes = IntGauge::new(
78            "mentedb_cluster_live_nodes",
79            "Live nodes in the gossip cluster (1 when sharding is off)",
80        )
81        .unwrap();
82        let http_requests = IntCounterVec::new(
83            opts!(
84                "mentedb_http_requests_total",
85                "HTTP requests by method and status"
86            ),
87            &["method", "status"],
88        )
89        .unwrap();
90        let http_duration = HistogramVec::new(
91            histogram_opts!(
92                "mentedb_http_request_duration_seconds",
93                "HTTP request latency in seconds"
94            ),
95            &["method"],
96        )
97        .unwrap();
98
99        // Engine gauges: create, register, and return in one step.
100        let gauge = |name: &str, help: &str| {
101            let g = IntGauge::new(name, help).unwrap();
102            registry.register(Box::new(g.clone())).ok();
103            g
104        };
105        let stores = gauge("mentedb_stores_total", "Memories written (writes)");
106        let recalls = gauge("mentedb_recalls_total", "Recall/query operations (reads)");
107        let bp_hits = gauge("mentedb_buffer_pool_hits_total", "Page cache hits");
108        let bp_misses = gauge("mentedb_buffer_pool_misses_total", "Page cache misses");
109        let bp_evictions = gauge(
110            "mentedb_buffer_pool_evictions_total",
111            "Page cache evictions",
112        );
113        let bp_pages = gauge("mentedb_buffer_pool_pages", "Frames holding a page");
114        let storage_bytes = gauge("mentedb_storage_bytes", "On-disk data size in bytes");
115        let pages = gauge("mentedb_pages_total", "Pages in the store");
116        let vector_index_size = gauge("mentedb_vector_index_size", "Vectors in the HNSW index");
117        let graph_nodes = gauge("mentedb_graph_nodes", "Nodes in the memory graph");
118        let standing_rules = gauge(
119            "mentedb_standing_rules",
120            "Pinned standing rules (scope:always)",
121        );
122        let store_latency_us = gauge(
123            "mentedb_store_latency_microseconds",
124            "EMA of store op latency in microseconds",
125        );
126        let search_latency_us = gauge(
127            "mentedb_search_latency_microseconds",
128            "EMA of hybrid-search op latency in microseconds",
129        );
130
131        for g in [&uptime, &memory_count, &live_nodes] {
132            registry.register(Box::new(g.clone())).ok();
133        }
134        registry.register(Box::new(http_requests.clone())).ok();
135        registry.register(Box::new(http_duration.clone())).ok();
136
137        Self {
138            registry,
139            uptime,
140            memory_count,
141            live_nodes,
142            http_requests,
143            http_duration,
144            stores,
145            recalls,
146            bp_hits,
147            bp_misses,
148            bp_evictions,
149            bp_pages,
150            storage_bytes,
151            pages,
152            vector_index_size,
153            graph_nodes,
154            standing_rules,
155            store_latency_us,
156            search_latency_us,
157        }
158    }
159}
160
161/// Middleware: time every request and count it by method and response status.
162pub async fn track(req: Request, next: Next) -> Response {
163    let method = req.method().as_str().to_string();
164    let start = Instant::now();
165    let resp = next.run(req).await;
166    let m = metrics();
167    let status = resp.status().as_u16().to_string();
168    m.http_requests.with_label_values(&[&method, &status]).inc();
169    m.http_duration
170        .with_label_values(&[&method])
171        .observe(start.elapsed().as_secs_f64());
172    resp
173}
174
175/// `GET /metrics`: refresh the point-in-time gauges, then encode the registry.
176pub async fn handler(State(state): State<Arc<AppState>>) -> impl IntoResponse {
177    let m = metrics();
178    m.uptime.set(state.start_time.elapsed().as_secs() as i64);
179    let dm = state.db.metrics();
180    m.memory_count.set(dm.memory_count as i64);
181    m.stores.set(dm.stores as i64);
182    m.recalls.set(dm.recalls as i64);
183    m.bp_hits.set(dm.buffer_pool_hits as i64);
184    m.bp_misses.set(dm.buffer_pool_misses as i64);
185    m.bp_evictions.set(dm.buffer_pool_evictions as i64);
186    m.bp_pages.set(dm.buffer_pool_pages as i64);
187    m.storage_bytes.set(dm.storage_bytes as i64);
188    m.pages.set(dm.page_count as i64);
189    m.vector_index_size.set(dm.vector_index_size as i64);
190    m.graph_nodes.set(dm.graph_nodes as i64);
191    m.standing_rules.set(dm.standing_rules as i64);
192    m.store_latency_us.set(dm.avg_store_latency_us as i64);
193    m.search_latency_us.set(dm.avg_search_latency_us as i64);
194    let live = match &state.cluster {
195        Some(c) => c.live_node_count().await as i64,
196        None => 1,
197    };
198    m.live_nodes.set(live);
199
200    let mut buf = Vec::new();
201    if TextEncoder::new()
202        .encode(&m.registry.gather(), &mut buf)
203        .is_err()
204    {
205        return (StatusCode::INTERNAL_SERVER_ERROR, "metric encode error").into_response();
206    }
207    (
208        [(header::CONTENT_TYPE, "text/plain; version=0.0.4")],
209        Body::from(buf),
210    )
211        .into_response()
212}
mentedb_server/metrics.rs

mentedb_server/
metrics.rs
