dynamo_runtime/metrics/
frontend_perf.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026-2027 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Frontend pipeline stage and finer-grained perf metrics.
5//! Used by both runtime (route, transport_roundtrip) and llm (preprocess, postprocess, tokenize, template, detokenize).
6
7use once_cell::sync::{Lazy, OnceCell};
8use prometheus::{
9    Counter, Histogram, HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry,
10};
11
12use super::prometheus_names::{frontend_perf, labels, name_prefix};
13use crate::MetricsRegistry;
14
15pub use super::prometheus_names::frontend_perf::{STAGE_DISPATCH, STAGE_PREPROCESS, STAGE_ROUTE};
16
17fn frontend_metric_name(suffix: &str) -> String {
18    format!("{}_{}", name_prefix::FRONTEND, suffix)
19}
20
21/// Per-stage inflight request count: preprocess, route, dispatch.
22/// Labels: stage (pipeline stage), phase (prefill/decode/aggregated or empty for preprocess).
23pub static STAGE_REQUESTS: Lazy<IntGaugeVec> = Lazy::new(|| {
24    IntGaugeVec::new(
25        Opts::new(
26            frontend_metric_name(frontend_perf::STAGE_REQUESTS),
27            "Number of requests currently in the given pipeline stage",
28        ),
29        &["stage", "phase"],
30    )
31    .expect("failed to create dynamo_frontend_stage_requests gauge")
32});
33
34/// RAII guard that increments a per-stage gauge on creation and decrements on drop.
35///
36/// Used to track how many requests are in each frontend pipeline stage at any given time.
37/// Create with [`StageGuard::new`] at stage entry; the gauge decrements automatically when
38/// the guard is dropped (end of scope, explicit drop, or stream completion).
39pub struct StageGuard {
40    gauge: prometheus::IntGauge,
41}
42
43impl StageGuard {
44    /// Increment the stage gauge and return a guard that decrements on drop.
45    ///
46    /// * `stage` — pipeline stage name; use `frontend_perf::STAGE_{PREPROCESS,ROUTE,DISPATCH}`
47    ///   constants from [`crate::metrics::prometheus_names`].
48    /// * `phase` — request phase; use [`RequestPhase::to_string`] output
49    ///   (`"prefill"|"decode"|"aggregated"`), or `""` for stages without a phase.
50    pub fn new(stage: &str, phase: &str) -> Self {
51        let gauge = STAGE_REQUESTS.with_label_values(&[stage, phase]);
52        gauge.inc();
53        Self { gauge }
54    }
55}
56
57impl Drop for StageGuard {
58    fn drop(&mut self) {
59        self.gauge.dec();
60    }
61}
62
63/// Per-stage latency: preprocess, route, transport_roundtrip, postprocess.
64pub static STAGE_DURATION_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
65    HistogramVec::new(
66        HistogramOpts::new(
67            frontend_metric_name(frontend_perf::STAGE_DURATION_SECONDS),
68            "Pipeline stage duration (seconds)",
69        )
70        .buckets(vec![
71            0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0,
72        ]),
73        &["stage"],
74    )
75    .expect("stage_duration_seconds histogram vec")
76});
77
78/// Tokenization time in preprocessor (gather_tokens).
79pub static TOKENIZE_SECONDS: Lazy<Histogram> = Lazy::new(|| {
80    Histogram::with_opts(
81        HistogramOpts::new(
82            frontend_metric_name(frontend_perf::TOKENIZE_SECONDS),
83            "Tokenization time in preprocessor (seconds)",
84        )
85        .buckets(vec![
86            0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0,
87        ]),
88    )
89    .expect("tokenize_seconds histogram")
90});
91
92/// Template application time in preprocessor (apply_template).
93pub static TEMPLATE_SECONDS: Lazy<Histogram> = Lazy::new(|| {
94    Histogram::with_opts(
95        HistogramOpts::new(
96            frontend_metric_name(frontend_perf::TEMPLATE_SECONDS),
97            "Template application time in preprocessor (seconds)",
98        )
99        .buckets(vec![
100            0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05,
101        ]),
102    )
103    .expect("template_seconds histogram")
104});
105
106/// Cumulative detokenization time across all tokens (microseconds).
107/// Use `rate(total) / rate(count)` in Prometheus to derive per-token average.
108pub static DETOKENIZE_TOTAL_US: Lazy<Counter> = Lazy::new(|| {
109    Counter::with_opts(Opts::new(
110        frontend_metric_name(frontend_perf::DETOKENIZE_TOTAL_US),
111        "Cumulative detokenization time (microseconds)",
112    ))
113    .expect("detokenize_total_us counter")
114});
115
116/// Total number of tokens detokenized.
117pub static DETOKENIZE_TOKEN_COUNT: Lazy<Counter> = Lazy::new(|| {
118    Counter::with_opts(Opts::new(
119        frontend_metric_name(frontend_perf::DETOKENIZE_TOKEN_COUNT),
120        "Total tokens detokenized",
121    ))
122    .expect("detokenize_token_count counter")
123});
124
125/// Cumulative L1 tokenizer cache hits. The cache is enabled unless `DYN_TOKENIZER_CACHE=0`.
126pub static TOKENIZER_CACHE_HITS_TOTAL: Lazy<Counter> = Lazy::new(|| {
127    Counter::with_opts(Opts::new(
128        frontend_metric_name(frontend_perf::TOKENIZER_CACHE_HITS_TOTAL),
129        "Cumulative L1 tokenizer prefix-cache hits",
130    ))
131    .expect("tokenizer_cache_hits_total counter")
132});
133
134/// Cumulative L1 tokenizer cache misses. The cache is enabled unless `DYN_TOKENIZER_CACHE=0`.
135pub static TOKENIZER_CACHE_MISSES_TOTAL: Lazy<Counter> = Lazy::new(|| {
136    Counter::with_opts(Opts::new(
137        frontend_metric_name(frontend_perf::TOKENIZER_CACHE_MISSES_TOTAL),
138        "Cumulative L1 tokenizer prefix-cache misses",
139    ))
140    .expect("tokenizer_cache_misses_total counter")
141});
142
143/// Tokens returned from the L1 tokenizer prefix cache, labeled by served model name.
144pub static TOKENIZER_CACHE_CACHED_TOKENS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
145    IntCounterVec::new(
146        Opts::new(
147            frontend_metric_name(frontend_perf::TOKENIZER_CACHE_CACHED_TOKENS_TOTAL),
148            "Total tokens returned from the L1 tokenizer prefix cache",
149        ),
150        &[labels::MODEL],
151    )
152    .expect("tokenizer_cache_cached_tokens_total counter vec")
153});
154
155/// Tokens freshly encoded after an L1 tokenizer prefix-cache lookup, labeled by served model name.
156pub static TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
157    IntCounterVec::new(
158        Opts::new(
159            frontend_metric_name(frontend_perf::TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL),
160            "Total tokens freshly encoded after an L1 tokenizer prefix-cache lookup",
161        ),
162        &[labels::MODEL],
163    )
164    .expect("tokenizer_cache_uncached_tokens_total counter vec")
165});
166
167/// Guards idempotency for the `MetricsRegistry` registration path.
168static REGISTERED: OnceCell<()> = OnceCell::new();
169
170/// Guards idempotency for the raw `prometheus::Registry` registration path.
171/// Kept separate from `REGISTERED` so that calling `ensure_frontend_perf_metrics_registered`
172/// first does not silently prevent the metrics from being registered in the prometheus registry.
173static PROMETHEUS_REGISTERED: OnceCell<()> = OnceCell::new();
174
175fn register_frontend_perf_metrics(registry: &MetricsRegistry) {
176    registry.add_metric(Box::new(STAGE_REQUESTS.clone())).ok();
177    registry
178        .add_metric(Box::new(STAGE_DURATION_SECONDS.clone()))
179        .ok();
180    registry.add_metric(Box::new(TOKENIZE_SECONDS.clone())).ok();
181    registry.add_metric(Box::new(TEMPLATE_SECONDS.clone())).ok();
182    registry
183        .add_metric(Box::new(DETOKENIZE_TOTAL_US.clone()))
184        .ok();
185    registry
186        .add_metric(Box::new(DETOKENIZE_TOKEN_COUNT.clone()))
187        .ok();
188    registry
189        .add_metric(Box::new(TOKENIZER_CACHE_HITS_TOTAL.clone()))
190        .ok();
191    registry
192        .add_metric(Box::new(TOKENIZER_CACHE_MISSES_TOTAL.clone()))
193        .ok();
194    registry
195        .add_metric(Box::new(TOKENIZER_CACHE_CACHED_TOKENS_TOTAL.clone()))
196        .ok();
197    registry
198        .add_metric(Box::new(TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL.clone()))
199        .ok();
200}
201
202fn register_frontend_perf_metrics_prometheus(registry: &Registry) -> Result<(), prometheus::Error> {
203    registry.register(Box::new(STAGE_REQUESTS.clone()))?;
204    registry.register(Box::new(STAGE_DURATION_SECONDS.clone()))?;
205    registry.register(Box::new(TOKENIZE_SECONDS.clone()))?;
206    registry.register(Box::new(TEMPLATE_SECONDS.clone()))?;
207    registry.register(Box::new(DETOKENIZE_TOTAL_US.clone()))?;
208    registry.register(Box::new(DETOKENIZE_TOKEN_COUNT.clone()))?;
209    registry.register(Box::new(TOKENIZER_CACHE_HITS_TOTAL.clone()))?;
210    registry.register(Box::new(TOKENIZER_CACHE_MISSES_TOTAL.clone()))?;
211    registry.register(Box::new(TOKENIZER_CACHE_CACHED_TOKENS_TOTAL.clone()))?;
212    registry.register(Box::new(TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL.clone()))?;
213    Ok(())
214}
215
216/// Register frontend perf metrics with the given registry. Idempotent.
217pub fn ensure_frontend_perf_metrics_registered(registry: &MetricsRegistry) {
218    let _ = REGISTERED.get_or_init(|| register_frontend_perf_metrics(registry));
219}
220
221/// Register frontend perf metrics with a raw Prometheus registry (e.g. for LLM HTTP service /metrics).
222/// Idempotent. Call this when the service exposes /metrics from its own registry.
223pub fn ensure_frontend_perf_metrics_registered_prometheus(
224    registry: &Registry,
225) -> Result<(), prometheus::Error> {
226    if PROMETHEUS_REGISTERED.get().is_some() {
227        return Ok(());
228    }
229    register_frontend_perf_metrics_prometheus(registry)?;
230    let _ = PROMETHEUS_REGISTERED.set(());
231    Ok(())
232}
233
234#[cfg(test)]
235mod tests {
236    use super::*;
237
238    fn assert_tokenizer_cache_token_metrics_registered(
239        families: &[prometheus::proto::MetricFamily],
240        model: &str,
241    ) {
242        for name in [
243            "dynamo_frontend_tokenizer_cache_cached_tokens_total",
244            "dynamo_frontend_tokenizer_cache_uncached_tokens_total",
245        ] {
246            let family = families
247                .iter()
248                .find(|family| family.name() == name)
249                .unwrap_or_else(|| panic!("missing metric family {name}"));
250            assert!(family.get_metric().iter().any(|metric| {
251                metric
252                    .get_label()
253                    .iter()
254                    .any(|label| label.name() == labels::MODEL && label.value() == model)
255            }));
256        }
257    }
258
259    #[test]
260    fn test_tokenizer_cache_token_metrics_registered_with_model_label() {
261        let model = "frontend-perf-registration-test-model";
262        let _ = TOKENIZER_CACHE_CACHED_TOKENS_TOTAL.with_label_values(&[model]);
263        let _ = TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL.with_label_values(&[model]);
264
265        let metrics_registry = MetricsRegistry::new();
266        register_frontend_perf_metrics(&metrics_registry);
267        assert_tokenizer_cache_token_metrics_registered(
268            &metrics_registry.get_prometheus_registry().gather(),
269            model,
270        );
271
272        let prometheus_registry = Registry::new();
273        register_frontend_perf_metrics_prometheus(&prometheus_registry).unwrap();
274        assert_tokenizer_cache_token_metrics_registered(&prometheus_registry.gather(), model);
275    }
276
277    #[test]
278    fn test_stage_guard_inc_dec() {
279        let gauge = STAGE_REQUESTS.with_label_values(&["test_stage", "test_phase"]);
280        assert_eq!(gauge.get(), 0);
281
282        {
283            let _guard = StageGuard::new("test_stage", "test_phase");
284            assert_eq!(gauge.get(), 1);
285
286            {
287                let _guard2 = StageGuard::new("test_stage", "test_phase");
288                assert_eq!(gauge.get(), 2);
289            }
290            // guard2 dropped
291            assert_eq!(gauge.get(), 1);
292        }
293        // guard dropped
294        assert_eq!(gauge.get(), 0);
295    }
296
297    #[test]
298    fn test_stage_guard_different_labels() {
299        let preprocess = STAGE_REQUESTS.with_label_values(&["preprocess_t", ""]);
300        let route_prefill = STAGE_REQUESTS.with_label_values(&["route_t", "prefill"]);
301        let route_decode = STAGE_REQUESTS.with_label_values(&["route_t", "decode"]);
302
303        let _g1 = StageGuard::new("preprocess_t", "");
304        let _g2 = StageGuard::new("route_t", "prefill");
305        let _g3 = StageGuard::new("route_t", "decode");
306
307        assert_eq!(preprocess.get(), 1);
308        assert_eq!(route_prefill.get(), 1);
309        assert_eq!(route_decode.get(), 1);
310
311        drop(_g2);
312        assert_eq!(preprocess.get(), 1);
313        assert_eq!(route_prefill.get(), 0);
314        assert_eq!(route_decode.get(), 1);
315    }
316}
dynamo_runtime/metrics/frontend_perf.rs

dynamo_runtime/metrics/
frontend_perf.rs
