dynamo_runtime/metrics/
request_plane.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Request-plane metrics for AddressedPushRouter.
5//! Used to pinpoint serialization vs transport roundtrip latency.
6
7use once_cell::sync::{Lazy, OnceCell};
8use prometheus::{Gauge, Histogram, HistogramOpts};
9
10use super::prometheus_names::{name_prefix, request_plane};
11
12fn request_plane_metric_name(suffix: &str) -> String {
13    format!("{}_{}", name_prefix::REQUEST_PLANE, suffix)
14}
15
16/// Time from generate() entry to send_request() (serialization + encoding + control message).
17pub static REQUEST_PLANE_QUEUE_SECONDS: Lazy<Histogram> = Lazy::new(|| {
18    Histogram::with_opts(
19        HistogramOpts::new(
20            request_plane_metric_name(request_plane::QUEUE_SECONDS),
21            "Time from generate() entry to send_request() (seconds)",
22        )
23        .buckets(vec![
24            0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0,
25        ]),
26    )
27    .expect("request_plane_queue_seconds histogram")
28});
29
30/// Time for send_request() to complete (frontend view: network + queue + ack).
31pub static REQUEST_PLANE_SEND_SECONDS: Lazy<Histogram> = Lazy::new(|| {
32    Histogram::with_opts(
33        HistogramOpts::new(
34            request_plane_metric_name(request_plane::SEND_SECONDS),
35            "Time for send_request() to complete (seconds)",
36        )
37        .buckets(vec![
38            0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0,
39        ]),
40    )
41    .expect("request_plane_send_seconds histogram")
42});
43
44/// Time from send_request() to first response item (transport roundtrip TTFT).
45pub static REQUEST_PLANE_ROUNDTRIP_TTFT_SECONDS: Lazy<Histogram> = Lazy::new(|| {
46    Histogram::with_opts(
47        HistogramOpts::new(
48            request_plane_metric_name(request_plane::ROUNDTRIP_TTFT_SECONDS),
49            "Time from send_request() to first response item (seconds)",
50        )
51        .buckets(vec![
52            0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0,
53        ]),
54    )
55    .expect("request_plane_roundtrip_ttft_seconds histogram")
56});
57
58/// Currently in-flight requests (incremented at generate() entry, decremented on stream complete).
59pub static REQUEST_PLANE_INFLIGHT: Lazy<Gauge> = Lazy::new(|| {
60    Gauge::new(
61        request_plane_metric_name(request_plane::INFLIGHT_REQUESTS),
62        "Currently in-flight requests at AddressedPushRouter",
63    )
64    .expect("request_plane_inflight gauge")
65});
66
67/// Guards idempotency for the raw `prometheus::Registry` registration path.
68static PROMETHEUS_REGISTERED: OnceCell<Result<(), String>> = OnceCell::new();
69
70/// Register request-plane metrics with a raw Prometheus registry (e.g. for LLM HTTP service /metrics).
71/// Idempotent; only the first call registers. Call this when the service exposes /metrics from its own registry.
72pub fn ensure_request_plane_metrics_registered_prometheus(
73    registry: &prometheus::Registry,
74) -> Result<(), prometheus::Error> {
75    PROMETHEUS_REGISTERED
76        .get_or_init(|| {
77            (|| -> Result<(), prometheus::Error> {
78                registry.register(Box::new(REQUEST_PLANE_QUEUE_SECONDS.clone()))?;
79                registry.register(Box::new(REQUEST_PLANE_SEND_SECONDS.clone()))?;
80                registry.register(Box::new(REQUEST_PLANE_ROUNDTRIP_TTFT_SECONDS.clone()))?;
81                registry.register(Box::new(REQUEST_PLANE_INFLIGHT.clone()))?;
82                Ok(())
83            })()
84            .map_err(|e| e.to_string())
85        })
86        .as_ref()
87        .map(|_| ())
88        .map_err(|e| prometheus::Error::Msg(e.clone()))
89}
dynamo_runtime/metrics/request_plane.rs

dynamo_runtime/metrics/
request_plane.rs
