Skip to main content

rws_server/metrics/
mod.rs

1//! Server-wide and per-route Prometheus metrics.
2//!
3//! **Server-wide counters** (`REQUESTS_TOTAL`, `ERRORS_TOTAL`,
4//! `ACTIVE_CONNECTIONS`) are updated by the server core automatically.
5//!
6//! **Per-route metrics** are opt-in: wrap your application with
7//! [`MetricsLayer`] and each request will be attributed to its
8//! `(method, path)` pair, emitting:
9//! - `rws_route_requests_total{method,path,status}` — request counts
10//! - `rws_route_duration_seconds{method,path}` — latency histogram
11//!
12//! **HTTP server duration** — `http_server_request_duration_seconds`, the
13//! OpenTelemetry `http.server.request.duration` metric, recorded by the server
14//! core for every request ([`http`]); native-histogram buckets are served in
15//! the protobuf format ([`exposition`]) to a scraper that asks for it.
16//!
17//! **Circuit breaker state** — `rws_circuit_breaker_state{backend}` (gauge,
18//! `0`=closed, `1`=half_open, `2`=open) is emitted automatically for every
19//! backend known to [`crate::circuit_breaker::global`], with no opt-in layer
20//! needed — see [`crate::circuit_breaker`] and
21//! `proxy::ReverseProxy::with_circuit_breaker`.
22//!
23//! # Example
24//!
25//! ```rust,no_run
26//! # extern crate rws_server as rust_web_server;
27//! use rust_web_server::app::App;
28//! use rust_web_server::core::New;
29//! use rust_web_server::metrics::MetricsLayer;
30//!
31//! let app = App::new().wrap(MetricsLayer);
32//! ```
33
34use std::collections::HashMap;
35use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
36use std::sync::{Mutex, OnceLock};
37use std::time::Instant;
38
39use crate::application::Application;
40use crate::middleware::Middleware;
41use crate::request::Request;
42use crate::response::Response;
43use crate::server::ConnectionInfo;
44
45pub mod exposition;
46pub mod http;
47#[cfg(not(target_arch = "wasm32"))]
48pub mod llm;
49pub mod tls;
50
51// ── server-wide atomics ───────────────────────────────────────────────────────
52
53/// Set to `true` after [`crate::server::Server::setup`] completes.
54/// The `/readyz` controller returns `503` until this is `true`.
55/// Set back to `false` when a shutdown signal is received so that
56/// Kubernetes stops routing traffic before the pod exits.
57pub static SERVER_READY: AtomicBool = AtomicBool::new(false);
58
59/// Total HTTP requests handled across all connections and protocols.
60pub static REQUESTS_TOTAL: AtomicU64 = AtomicU64::new(0);
61
62/// Requests that caused an application-level error (app.execute returned Err).
63pub static ERRORS_TOTAL: AtomicU64 = AtomicU64::new(0);
64
65/// Number of currently open TCP/QUIC connections.
66pub static ACTIVE_CONNECTIONS: AtomicI64 = AtomicI64::new(0);
67
68/// Requests whose handler is running (18.35.0): what a graceful shutdown
69/// waits for (`server::drain_requests`).
70pub static REQUESTS_IN_FLIGHT: AtomicI64 = AtomicI64::new(0);
71
72/// Counts one request in [`REQUESTS_IN_FLIGHT`] until dropped.
73pub struct InFlight(());
74
75impl InFlight {
76    pub fn start() -> Self {
77        REQUESTS_IN_FLIGHT.fetch_add(1, Ordering::SeqCst);
78        InFlight(())
79    }
80}
81
82impl Drop for InFlight {
83    fn drop(&mut self) {
84        REQUESTS_IN_FLIGHT.fetch_sub(1, Ordering::SeqCst);
85    }
86}
87
88/// Jobs queued in the thread pool waiting for a free worker.
89pub static THREAD_POOL_QUEUED: AtomicI64 = AtomicI64::new(0);
90
91thread_local! {
92    static UNCOUNTED: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
93}
94
95/// Requests served on this thread are left out of the counters and the
96/// access log while `true`: the admin listener sets it on its connection
97/// threads, so the admin UI's own polling doesn't show up as traffic
98/// (18.37.0; `[admin] count_own_requests = true` keeps counting them).
99#[doc(hidden)]
100pub fn set_thread_uncounted(uncounted: bool) {
101    UNCOUNTED.with(|u| u.set(uncounted));
102}
103
104/// See [`set_thread_uncounted`].
105#[doc(hidden)]
106pub fn thread_uncounted() -> bool {
107    UNCOUNTED.with(|u| u.get())
108}
109
110pub fn record_request() {
111    if thread_uncounted() {
112        return;
113    }
114    REQUESTS_TOTAL.fetch_add(1, Ordering::Relaxed);
115}
116
117pub fn record_error() {
118    if thread_uncounted() {
119        return;
120    }
121    ERRORS_TOTAL.fetch_add(1, Ordering::Relaxed);
122}
123
124pub fn connection_open() {
125    ACTIVE_CONNECTIONS.fetch_add(1, Ordering::Relaxed);
126}
127
128pub fn connection_close() {
129    ACTIVE_CONNECTIONS.fetch_sub(1, Ordering::Relaxed);
130}
131
132// ── per-route store ───────────────────────────────────────────────────────────
133
134/// Histogram bucket upper bounds (seconds). Matches the default Prometheus
135/// buckets used by `prometheus_client` and most Go instrumentation libraries.
136const BUCKET_BOUNDS: [f64; 11] = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0];
137
138struct HistogramEntry {
139    /// `buckets[i]` = cumulative count of observations where duration ≤ `BUCKET_BOUNDS[i]`.
140    buckets: [u64; 11],
141    sum: f64,
142    count: u64,
143}
144
145impl HistogramEntry {
146    fn new() -> Self {
147        HistogramEntry { buckets: [0; 11], sum: 0.0, count: 0 }
148    }
149
150    fn observe(&mut self, secs: f64) {
151        for (i, &upper) in BUCKET_BOUNDS.iter().enumerate() {
152            if secs <= upper {
153                self.buckets[i] += 1;
154            }
155        }
156        self.sum += secs;
157        self.count += 1;
158    }
159}
160
161struct RouteEntry {
162    counts: HashMap<i16, u64>,
163    latency: HistogramEntry,
164}
165
166impl RouteEntry {
167    fn new() -> Self {
168        RouteEntry { counts: HashMap::new(), latency: HistogramEntry::new() }
169    }
170}
171
172struct RouteStore {
173    /// Key: `(method, path)` — path has query string stripped.
174    entries: HashMap<(String, String), RouteEntry>,
175}
176
177static ROUTE_STORE: OnceLock<Mutex<RouteStore>> = OnceLock::new();
178
179fn route_store() -> &'static Mutex<RouteStore> {
180    ROUTE_STORE.get_or_init(|| Mutex::new(RouteStore { entries: HashMap::new() }))
181}
182
183/// Record a completed request in the per-route store.
184///
185/// `path` must have the query string already stripped. Called automatically by
186/// [`MetricsLayer`]; exposed publicly for testing and custom instrumentation.
187pub fn record_route(method: &str, path: &str, status: i16, elapsed_secs: f64) {
188    let key = (method.to_string(), path.to_string());
189    let mut guard = route_store().lock().unwrap();
190    let entry = guard.entries.entry(key).or_insert_with(RouteEntry::new);
191    *entry.counts.entry(status).or_insert(0) += 1;
192    entry.latency.observe(elapsed_secs);
193}
194
195/// Strip query string from a URI so `/users?page=2` → `/users`.
196fn strip_query(uri: &str) -> &str {
197    match uri.find('?') {
198        Some(i) => &uri[..i],
199        None => uri,
200    }
201}
202
203// ── MetricsLayer middleware ───────────────────────────────────────────────────
204
205/// Middleware that records per-route request counts and latency histograms.
206///
207/// Wrap any application with this layer once at startup; the data is collected
208/// into a global store and emitted via `GET /metrics`.
209///
210/// ```rust,no_run
211/// # extern crate rws_server as rust_web_server;
212/// use rust_web_server::app::App;
213/// use rust_web_server::core::New;
214/// use rust_web_server::metrics::MetricsLayer;
215///
216/// let app = App::new().wrap(MetricsLayer);
217/// ```
218pub struct MetricsLayer;
219
220impl Middleware for MetricsLayer {
221    /// Headers only: a streamed body passes through untouched.
222    fn accepts_streamed_body(&self, _request: &Request, _connection: &ConnectionInfo) -> bool {
223        true
224    }
225
226
227    fn handle(
228        &self,
229        request: &Request,
230        connection: &ConnectionInfo,
231        next: &dyn Application,
232    ) -> Result<Response, String> {
233        let start = Instant::now();
234        let result = next.execute(request, connection);
235        let elapsed = start.elapsed().as_secs_f64();
236
237        let path = strip_query(&request.request_uri).to_string();
238        let status = match &result {
239            Ok(r) => r.status_code,
240            Err(_) => 500,
241        };
242        record_route(&request.method, &path, status, elapsed);
243
244        result
245    }
246
247    /// In-memory only: safe on the event loop.
248    fn runs_inline(&self, _request: &Request, _connection: &ConnectionInfo) -> bool {
249        true
250    }
251}
252
253// ── prometheus output ─────────────────────────────────────────────────────────
254
255#[cfg(test)]
256mod tests;
257
258/// Returns a Prometheus text-format snapshot of all server-wide and per-route metrics.
259pub fn prometheus_text() -> String {
260    let mut out = prometheus_text_without_http();
261    let http_text = http::prometheus_text();
262    if !http_text.is_empty() {
263        out.push('\n');
264        out.push_str(&http_text);
265    }
266    out
267}
268
269/// Everything but `http_server_request_duration_seconds` (which the protobuf
270/// format builds itself, with its native buckets).
271fn prometheus_text_without_http() -> String {
272    let requests = REQUESTS_TOTAL.load(Ordering::Relaxed);
273    let errors   = ERRORS_TOTAL.load(Ordering::Relaxed);
274    let active   = ACTIVE_CONNECTIONS.load(Ordering::Relaxed);
275
276    let mut out = format!(
277        "# HELP rws_requests_total Total HTTP requests handled\n\
278         # TYPE rws_requests_total counter\n\
279         rws_requests_total {}\n\n\
280         # HELP rws_errors_total HTTP requests that returned an application error\n\
281         # TYPE rws_errors_total counter\n\
282         rws_errors_total {}\n\n\
283         # HELP rws_active_connections Currently open connections\n\
284         # TYPE rws_active_connections gauge\n\
285         rws_active_connections {}\n",
286        requests, errors, active
287    );
288
289    // Sections in a fixed order: this layer's own, and those subsystems
290    // above it registered (`register_section`) when first used.
291    let mut sections: Vec<(u16, fn() -> String)> = vec![
292        (section::ROUTES, route_prometheus_text),
293        (section::DISK, crate::disk_guard::prometheus_text),
294        (section::EARLY_HINTS, crate::early_hints::prometheus_text),
295        (section::TLS, tls::prometheus_text),
296        (section::CIRCUIT_BREAKER, circuit_breaker_prometheus_text),
297    ];
298    #[cfg(not(target_arch = "wasm32"))]
299    sections.push((section::LLM, llm::prometheus_text));
300    sections.extend(registered_sections());
301    sections.sort_by_key(|(order, _)| *order);
302    for (_, text) in sections {
303        let text = text();
304        if !text.is_empty() {
305            out.push('\n');
306            out.push_str(&text);
307        }
308    }
309
310    out
311}
312
313/// Where each section of [`prometheus_text`] goes; subsystems above this
314/// layer pass theirs to [`register_section`]. The order is the output order.
315#[doc(hidden)]
316pub mod section {
317    pub const ROUTES: u16 = 10;
318    pub const UPSTREAMS: u16 = 20;
319    pub const WAF: u16 = 30;
320    pub const MIRROR: u16 = 40;
321    pub const DISK: u16 = 50;
322    pub const EARLY_HINTS: u16 = 60;
323    pub const WASM_PLUGINS: u16 = 70;
324    pub const LLM: u16 = 80;
325    pub const TLS: u16 = 90;
326    pub const CIRCUIT_BREAKER: u16 = 100;
327 /// Kubernetes controllers (`k8s::controller`, 18.35.0).
328 pub const CONTROLLERS: u16 = 110;
329}
330
331static SECTIONS: std::sync::Mutex<Vec<(u16, fn() -> String)>> = std::sync::Mutex::new(Vec::new());
332
333/// Add a subsystem's series to [`prometheus_text`] at `order` (a
334/// [`section`] constant). Called when the subsystem is first used — its
335/// section is empty before that anyway; registering twice is a no-op.
336#[doc(hidden)]
337pub fn register_section(order: u16, text: fn() -> String) {
338    let mut sections = SECTIONS.lock().unwrap_or_else(|e| e.into_inner());
339    if !sections.iter().any(|(o, _)| *o == order) {
340        sections.push((order, text));
341    }
342}
343
344fn registered_sections() -> Vec<(u16, fn() -> String)> {
345    SECTIONS.lock().unwrap_or_else(|e| e.into_inner()).clone()
346}
347
348/// `rws_circuit_breaker_state{backend}` — the state of every backend known to
349/// the process-wide [`crate::circuit_breaker::global`] breaker, encoded as
350/// `0` (Closed/healthy), `1` (HalfOpen/probing), or `2` (Open/unhealthy).
351///
352/// Only the in-memory global breaker is covered — `RedisCircuitBreaker`
353/// state can't be enumerated here (no `SCAN`/`KEYS` support in the minimal
354/// hand-rolled RESP client; see [`crate::circuit_breaker::CircuitBreaker::all_states`]
355/// docs). A backend never seen by the global breaker (e.g. one whose
356/// `ReverseProxy` was wired to a different, non-global breaker instance)
357/// emits no line rather than a synthetic `0` — there's nothing to report.
358// `circuit_breaker` is native-only (backed by `redis_protocol`'s `TcpStream`)
359// — see spec/WASM_SHIM.md. No breaker state exists to report under wasm32.
360#[cfg(target_arch = "wasm32")]
361fn circuit_breaker_prometheus_text() -> String {
362    String::new()
363}
364
365#[cfg(not(target_arch = "wasm32"))]
366fn circuit_breaker_prometheus_text() -> String {
367    let mut states = crate::circuit_breaker::global().lock().unwrap().all_states();
368    if states.is_empty() {
369        return String::new();
370    }
371    states.sort_by(|a, b| a.0.cmp(&b.0));
372
373    let mut out = String::new();
374    out.push_str("# HELP rws_circuit_breaker_state Circuit breaker state per backend (0=closed, 1=half_open, 2=open)\n");
375    out.push_str("# TYPE rws_circuit_breaker_state gauge\n");
376    for (backend, state) in states {
377        let value = match state {
378            crate::circuit_breaker::BreakerState::Closed => 0,
379            crate::circuit_breaker::BreakerState::HalfOpen => 1,
380            crate::circuit_breaker::BreakerState::Open => 2,
381        };
382        out.push_str(&format!("rws_circuit_breaker_state{{backend=\"{}\"}} {}\n", backend, value));
383    }
384    out
385}
386
387fn route_prometheus_text() -> String {
388    let guard = route_store().lock().unwrap();
389    if guard.entries.is_empty() {
390        return String::new();
391    }
392
393    // Sort for deterministic output.
394    let mut keys: Vec<&(String, String)> = guard.entries.keys().collect();
395    keys.sort();
396
397    let mut out = String::new();
398
399    // ── rws_route_requests_total ──────────────────────────────────────────────
400    out.push_str("# HELP rws_route_requests_total Total requests handled per route\n");
401    out.push_str("# TYPE rws_route_requests_total counter\n");
402    for key in &keys {
403        let entry = &guard.entries[*key];
404        let (method, path) = key;
405        let mut statuses: Vec<i16> = entry.counts.keys().cloned().collect();
406        statuses.sort();
407        for status in statuses {
408            let count = entry.counts[&status];
409            out.push_str(&format!(
410                "rws_route_requests_total{{method=\"{}\",path=\"{}\",status=\"{}\"}} {}\n",
411                method, path, status, count,
412            ));
413        }
414    }
415
416    // ── rws_route_duration_seconds histogram ──────────────────────────────────
417    out.push('\n');
418    out.push_str("# HELP rws_route_duration_seconds Request duration in seconds per route\n");
419    out.push_str("# TYPE rws_route_duration_seconds histogram\n");
420    for key in &keys {
421        let entry = &guard.entries[*key];
422        let (method, path) = key;
423        let lat = &entry.latency;
424
425        for (i, &upper) in BUCKET_BOUNDS.iter().enumerate() {
426            out.push_str(&format!(
427                "rws_route_duration_seconds_bucket{{method=\"{}\",path=\"{}\",le=\"{}\"}} {}\n",
428                method, path, upper, lat.buckets[i],
429            ));
430        }
431        out.push_str(&format!(
432            "rws_route_duration_seconds_bucket{{method=\"{}\",path=\"{}\",le=\"+Inf\"}} {}\n",
433            method, path, lat.count,
434        ));
435        out.push_str(&format!(
436            "rws_route_duration_seconds_sum{{method=\"{}\",path=\"{}\"}} {:.9}\n",
437            method, path, lat.sum,
438        ));
439        out.push_str(&format!(
440            "rws_route_duration_seconds_count{{method=\"{}\",path=\"{}\"}} {}\n",
441            method, path, lat.count,
442        ));
443    }
444
445    out
446}