dynamo_runtime/metrics/
frontend_perf.rs

1// SPDX-FileCopyrightText: Copyright (c) 2026-2027 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Frontend pipeline stage and finer-grained perf metrics.
5//! Used by both runtime (route, transport_roundtrip) and llm (preprocess, postprocess, tokenize, template, detokenize).
6
7use once_cell::sync::{Lazy, OnceCell};
8use prometheus::{
9    Counter, Histogram, HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry,
10};
11
12use super::prometheus_names::{frontend_perf, labels, name_prefix};
13
14pub use super::prometheus_names::frontend_perf::{STAGE_DISPATCH, STAGE_PREPROCESS, STAGE_ROUTE};
15
16fn frontend_metric_name(suffix: &str) -> String {
17    format!("{}_{}", name_prefix::FRONTEND, suffix)
18}
19
20/// Per-stage inflight request count: preprocess, route, dispatch.
21/// Labels: stage (pipeline stage), phase (prefill/decode/aggregated or empty for preprocess).
22pub static STAGE_REQUESTS: Lazy<IntGaugeVec> = Lazy::new(|| {
23    IntGaugeVec::new(
24        Opts::new(
25            frontend_metric_name(frontend_perf::STAGE_REQUESTS),
26            "Number of requests currently in the given pipeline stage",
27        ),
28        &["stage", "phase"],
29    )
30    .expect("failed to create dynamo_frontend_stage_requests gauge")
31});
32
33/// RAII guard that increments a per-stage gauge on creation and decrements on drop.
34///
35/// Used to track how many requests are in each frontend pipeline stage at any given time.
36/// Create with [`StageGuard::new`] at stage entry; the gauge decrements automatically when
37/// the guard is dropped (end of scope, explicit drop, or stream completion).
38pub struct StageGuard {
39    gauge: prometheus::IntGauge,
40}
41
42impl StageGuard {
43    /// Increment the stage gauge and return a guard that decrements on drop.
44    ///
45    /// * `stage` — pipeline stage name; use `frontend_perf::STAGE_{PREPROCESS,ROUTE,DISPATCH}`
46    ///   constants from [`crate::metrics::prometheus_names`].
47    /// * `phase` — request phase; use `RequestPhase::to_string` output
48    ///   (`"prefill"|"decode"|"aggregated"`), or `""` for stages without a phase.
49    pub fn new(stage: &str, phase: &str) -> Self {
50        let gauge = STAGE_REQUESTS.with_label_values(&[stage, phase]);
51        gauge.inc();
52        Self { gauge }
53    }
54}
55
56impl Drop for StageGuard {
57    fn drop(&mut self) {
58        self.gauge.dec();
59    }
60}
61
62/// Per-stage latency: preprocess, route, transport_roundtrip, postprocess.
63pub static STAGE_DURATION_SECONDS: Lazy<HistogramVec> = Lazy::new(|| {
64    HistogramVec::new(
65        HistogramOpts::new(
66            frontend_metric_name(frontend_perf::STAGE_DURATION_SECONDS),
67            "Pipeline stage duration (seconds)",
68        )
69        .buckets(vec![
70            0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0,
71        ]),
72        &["stage"],
73    )
74    .expect("stage_duration_seconds histogram vec")
75});
76
77/// Tokenization time in preprocessor (gather_tokens).
78pub static TOKENIZE_SECONDS: Lazy<Histogram> = Lazy::new(|| {
79    Histogram::with_opts(
80        HistogramOpts::new(
81            frontend_metric_name(frontend_perf::TOKENIZE_SECONDS),
82            "Tokenization time in preprocessor (seconds)",
83        )
84        .buckets(vec![
85            0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0,
86        ]),
87    )
88    .expect("tokenize_seconds histogram")
89});
90
91/// Template application time in preprocessor (apply_template).
92pub static TEMPLATE_SECONDS: Lazy<Histogram> = Lazy::new(|| {
93    Histogram::with_opts(
94        HistogramOpts::new(
95            frontend_metric_name(frontend_perf::TEMPLATE_SECONDS),
96            "Template application time in preprocessor (seconds)",
97        )
98        .buckets(vec![
99            0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05,
100        ]),
101    )
102    .expect("template_seconds histogram")
103});
104
105/// Cumulative detokenization time across all tokens (microseconds).
106/// Use `rate(total) / rate(count)` in Prometheus to derive per-token average.
107pub static DETOKENIZE_TOTAL_US: Lazy<Counter> = Lazy::new(|| {
108    Counter::with_opts(Opts::new(
109        frontend_metric_name(frontend_perf::DETOKENIZE_TOTAL_US),
110        "Cumulative detokenization time (microseconds)",
111    ))
112    .expect("detokenize_total_us counter")
113});
114
115/// Total number of tokens detokenized.
116pub static DETOKENIZE_TOKEN_COUNT: Lazy<Counter> = Lazy::new(|| {
117    Counter::with_opts(Opts::new(
118        frontend_metric_name(frontend_perf::DETOKENIZE_TOKEN_COUNT),
119        "Total tokens detokenized",
120    ))
121    .expect("detokenize_token_count counter")
122});
123
124/// Cumulative L1 tokenizer cache hits. The cache is enabled unless `DYN_TOKENIZER_CACHE=0`.
125pub static TOKENIZER_CACHE_HITS_TOTAL: Lazy<Counter> = Lazy::new(|| {
126    Counter::with_opts(Opts::new(
127        frontend_metric_name(frontend_perf::TOKENIZER_CACHE_HITS_TOTAL),
128        "Cumulative L1 tokenizer prefix-cache hits",
129    ))
130    .expect("tokenizer_cache_hits_total counter")
131});
132
133/// Cumulative L1 tokenizer cache misses. The cache is enabled unless `DYN_TOKENIZER_CACHE=0`.
134pub static TOKENIZER_CACHE_MISSES_TOTAL: Lazy<Counter> = Lazy::new(|| {
135    Counter::with_opts(Opts::new(
136        frontend_metric_name(frontend_perf::TOKENIZER_CACHE_MISSES_TOTAL),
137        "Cumulative L1 tokenizer prefix-cache misses",
138    ))
139    .expect("tokenizer_cache_misses_total counter")
140});
141
142/// Tokens returned from the L1 tokenizer prefix cache, labeled by served model name.
143pub static TOKENIZER_CACHE_CACHED_TOKENS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
144    IntCounterVec::new(
145        Opts::new(
146            frontend_metric_name(frontend_perf::TOKENIZER_CACHE_CACHED_TOKENS_TOTAL),
147            "Total tokens returned from the L1 tokenizer prefix cache",
148        ),
149        &[labels::MODEL],
150    )
151    .expect("tokenizer_cache_cached_tokens_total counter vec")
152});
153
154/// Tokens freshly encoded after an L1 tokenizer prefix-cache lookup, labeled by served model name.
155pub static TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL: Lazy<IntCounterVec> = Lazy::new(|| {
156    IntCounterVec::new(
157        Opts::new(
158            frontend_metric_name(frontend_perf::TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL),
159            "Total tokens freshly encoded after an L1 tokenizer prefix-cache lookup",
160        ),
161        &[labels::MODEL],
162    )
163    .expect("tokenizer_cache_uncached_tokens_total counter vec")
164});
165
166/// Guards idempotency for the raw `prometheus::Registry` registration path.
167static PROMETHEUS_REGISTERED: OnceCell<()> = OnceCell::new();
168
169fn register_frontend_perf_metrics_prometheus(registry: &Registry) -> Result<(), prometheus::Error> {
170    registry.register(Box::new(STAGE_REQUESTS.clone()))?;
171    registry.register(Box::new(STAGE_DURATION_SECONDS.clone()))?;
172    registry.register(Box::new(TOKENIZE_SECONDS.clone()))?;
173    registry.register(Box::new(TEMPLATE_SECONDS.clone()))?;
174    registry.register(Box::new(DETOKENIZE_TOTAL_US.clone()))?;
175    registry.register(Box::new(DETOKENIZE_TOKEN_COUNT.clone()))?;
176    registry.register(Box::new(TOKENIZER_CACHE_HITS_TOTAL.clone()))?;
177    registry.register(Box::new(TOKENIZER_CACHE_MISSES_TOTAL.clone()))?;
178    registry.register(Box::new(TOKENIZER_CACHE_CACHED_TOKENS_TOTAL.clone()))?;
179    registry.register(Box::new(TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL.clone()))?;
180    Ok(())
181}
182
183/// Register frontend perf metrics with a raw Prometheus registry (e.g. for LLM HTTP service /metrics).
184/// Idempotent. Call this when the service exposes /metrics from its own registry.
185pub fn ensure_frontend_perf_metrics_registered_prometheus(
186    registry: &Registry,
187) -> Result<(), prometheus::Error> {
188    if PROMETHEUS_REGISTERED.get().is_some() {
189        return Ok(());
190    }
191    register_frontend_perf_metrics_prometheus(registry)?;
192    let _ = PROMETHEUS_REGISTERED.set(());
193    Ok(())
194}
195
196#[cfg(test)]
197mod tests {
198    use super::*;
199
200    fn assert_tokenizer_cache_token_metrics_registered(
201        families: &[prometheus::proto::MetricFamily],
202        model: &str,
203    ) {
204        for name in [
205            "dynamo_frontend_tokenizer_cache_cached_tokens_total",
206            "dynamo_frontend_tokenizer_cache_uncached_tokens_total",
207        ] {
208            let family = families
209                .iter()
210                .find(|family| family.name() == name)
211                .unwrap_or_else(|| panic!("missing metric family {name}"));
212            assert!(family.get_metric().iter().any(|metric| {
213                metric
214                    .get_label()
215                    .iter()
216                    .any(|label| label.name() == labels::MODEL && label.value() == model)
217            }));
218        }
219    }
220
221    #[test]
222    fn test_tokenizer_cache_token_metrics_registered_with_model_label() {
223        let model = "frontend-perf-registration-test-model";
224        let _ = TOKENIZER_CACHE_CACHED_TOKENS_TOTAL.with_label_values(&[model]);
225        let _ = TOKENIZER_CACHE_UNCACHED_TOKENS_TOTAL.with_label_values(&[model]);
226
227        let prometheus_registry = Registry::new();
228        register_frontend_perf_metrics_prometheus(&prometheus_registry).unwrap();
229        assert_tokenizer_cache_token_metrics_registered(&prometheus_registry.gather(), model);
230    }
231
232    #[test]
233    fn test_stage_guard_inc_dec() {
234        let gauge = STAGE_REQUESTS.with_label_values(&["test_stage", "test_phase"]);
235        assert_eq!(gauge.get(), 0);
236
237        {
238            let _guard = StageGuard::new("test_stage", "test_phase");
239            assert_eq!(gauge.get(), 1);
240
241            {
242                let _guard2 = StageGuard::new("test_stage", "test_phase");
243                assert_eq!(gauge.get(), 2);
244            }
245            // guard2 dropped
246            assert_eq!(gauge.get(), 1);
247        }
248        // guard dropped
249        assert_eq!(gauge.get(), 0);
250    }
251
252    #[test]
253    fn test_stage_guard_different_labels() {
254        let preprocess = STAGE_REQUESTS.with_label_values(&["preprocess_t", ""]);
255        let route_prefill = STAGE_REQUESTS.with_label_values(&["route_t", "prefill"]);
256        let route_decode = STAGE_REQUESTS.with_label_values(&["route_t", "decode"]);
257
258        let _g1 = StageGuard::new("preprocess_t", "");
259        let _g2 = StageGuard::new("route_t", "prefill");
260        let _g3 = StageGuard::new("route_t", "decode");
261
262        assert_eq!(preprocess.get(), 1);
263        assert_eq!(route_prefill.get(), 1);
264        assert_eq!(route_decode.get(), 1);
265
266        drop(_g2);
267        assert_eq!(preprocess.get(), 1);
268        assert_eq!(route_prefill.get(), 0);
269        assert_eq!(route_decode.get(), 1);
270    }
271}
dynamo_runtime/metrics/frontend_perf.rs

dynamo_runtime/metrics/
frontend_perf.rs
