Skip to main content

shard_telemetry/
prometheus_api.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::num::NonZeroU16;
3use std::sync::Arc;
4
5use axum::Router;
6use axum::body::Bytes;
7use axum::extract::{DefaultBodyLimit, Form, Path, Query, State};
8use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
9use axum::response::{IntoResponse, Response};
10use axum::routing::{get, post};
11use prost::Message;
12use serde::Deserialize;
13use serde_json::{Value, json};
14
15use crate::prometheus_protocol::v1 as prometheus_v1;
16use crate::prometheus_xor::encode_xor_chunk;
17use crate::{
18    DurableTelemetryStore, ExponentialHistogramBuckets, HistogramCount, MetricKind, MetricValue,
19    NumberValue, ProductionRuntime, PromqlEngine, PromqlLimits, PromqlValue, RemoteWriteDecoder,
20    RemoteWriteStats, RemoteWriteVersion, ServiceState, TelemetryError, TelemetryResult,
21    TelemetryRouter, TelemetryValue,
22};
23
24/// Single-tenant Prometheus compatibility limits.
25#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct PrometheusApiConfig {
27    /// Authenticated tenant assigned to every series.
28    pub tenant: Arc<str>,
29    /// Maximum compressed and Snappy-decompressed request bytes.
30    pub max_request_bytes: usize,
31    /// Stable logical metric partitions.
32    pub logical_partitions: NonZeroU16,
33}
34
35impl Default for PrometheusApiConfig {
36    fn default() -> Self {
37        Self {
38            tenant: Arc::from("default"),
39            max_request_bytes: 64 * 1024 * 1024,
40            logical_partitions: NonZeroU16::new(256).expect("constant is nonzero"),
41        }
42    }
43}
44
45impl PrometheusApiConfig {
46    fn validate(&self) -> TelemetryResult<()> {
47        if self.tenant.is_empty() {
48            return Err(TelemetryError::InvalidConfiguration(
49                "Prometheus tenant must not be empty".into(),
50            ));
51        }
52        if self.max_request_bytes == 0 || self.max_request_bytes > 64 * 1024 * 1024 {
53            return Err(TelemetryError::InvalidConfiguration(
54                "Prometheus request limit must be in 1..=64 MiB".into(),
55            ));
56        }
57        Ok(())
58    }
59}
60
61/// Shared Prometheus protocol service backed by signal-native metric stripes.
62#[derive(Clone)]
63pub struct PrometheusService {
64    store: Arc<DurableTelemetryStore>,
65    config: PrometheusApiConfig,
66    router: TelemetryRouter,
67    production: Option<Arc<ProductionRuntime>>,
68}
69
70impl std::fmt::Debug for PrometheusService {
71    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        formatter
73            .debug_struct("PrometheusService")
74            .field("config", &self.config)
75            .finish_non_exhaustive()
76    }
77}
78
79impl PrometheusService {
80    /// Creates a Prometheus API service.
81    pub fn new(
82        store: Arc<DurableTelemetryStore>,
83        config: PrometheusApiConfig,
84    ) -> TelemetryResult<Self> {
85        config.validate()?;
86        Ok(Self {
87            store,
88            router: TelemetryRouter::new(config.logical_partitions),
89            config,
90            production: None,
91        })
92    }
93
94    /// Attaches shared fail-closed production admission.
95    #[must_use]
96    pub fn with_production(mut self, production: Option<Arc<ProductionRuntime>>) -> Self {
97        self.production = production;
98        self
99    }
100
101    #[allow(clippy::result_large_err)]
102    fn authorize(
103        &self,
104        headers: &HeaderMap,
105    ) -> Result<Option<tokio::sync::OwnedSemaphorePermit>, Response> {
106        if headers
107            .get("x-scope-orgid")
108            .and_then(|value| value.to_str().ok())
109            .is_some_and(|tenant| tenant != self.config.tenant.as_ref())
110        {
111            return Err(write_error(
112                StatusCode::FORBIDDEN,
113                "tenant header does not match the configured tenant",
114            ));
115        }
116        let Some(runtime) = &self.production else {
117            return Ok(None);
118        };
119        let credential = headers
120            .get(header::AUTHORIZATION)
121            .and_then(|value| value.to_str().ok())
122            .and_then(|value| value.strip_prefix("Bearer "));
123        if !credential.is_some_and(|value| runtime.authenticates(value)) {
124            if credential.is_none() {
125                runtime.record_authentication_failure();
126            }
127            return Err(write_error(
128                StatusCode::UNAUTHORIZED,
129                "valid production bearer token is required",
130            ));
131        }
132        runtime.try_http().map(Some).ok_or_else(|| {
133            write_error(
134                StatusCode::TOO_MANY_REQUESTS,
135                "HTTP concurrency limit exceeded",
136            )
137        })
138    }
139}
140
141/// Builds Prometheus Remote Write and query compatibility routes.
142pub fn prometheus_router(service: PrometheusService) -> Router {
143    let max_request_bytes = service.config.max_request_bytes;
144    Router::new()
145        .route("/api/v1/write", post(remote_write))
146        .route("/api/v1/read", post(remote_read))
147        .route("/api/v1/query", get(query_get).post(query_post))
148        .route(
149            "/api/v1/query_range",
150            get(query_range_get).post(query_range_post),
151        )
152        .route("/api/v1/series", get(series_get).post(series_post))
153        .route("/api/v1/labels", get(labels_get).post(labels_post))
154        .route(
155            "/api/v1/label/{name}/values",
156            get(label_values_get).post(label_values_post),
157        )
158        .route("/api/v1/metadata", get(metadata_get))
159        .route(
160            "/api/v1/query_exemplars",
161            get(exemplars_get).post(exemplars_post),
162        )
163        .layer(DefaultBodyLimit::max(max_request_bytes))
164        .with_state(service)
165}
166
167#[derive(Debug, Clone, Deserialize)]
168struct InstantQueryParameters {
169    query: String,
170    time: Option<String>,
171}
172
173#[derive(Debug, Clone, Deserialize)]
174struct RangeQueryParameters {
175    query: String,
176    start: String,
177    end: String,
178    step: String,
179}
180
181#[derive(Debug, Clone, Default, Deserialize)]
182struct DiscoveryParameters {
183    #[serde(default, rename = "match[]")]
184    selectors: Vec<String>,
185    start: Option<String>,
186    end: Option<String>,
187}
188
189#[derive(Debug, Clone, Default, Deserialize)]
190struct MetadataParameters {
191    metric: Option<String>,
192    limit: Option<usize>,
193}
194
195#[derive(Debug, Clone, Deserialize)]
196struct ExemplarParameters {
197    query: String,
198    start: String,
199    end: String,
200}
201
202async fn query_get(
203    State(service): State<PrometheusService>,
204    headers: HeaderMap,
205    Query(parameters): Query<InstantQueryParameters>,
206) -> Response {
207    execute_instant_query(service, headers, parameters).await
208}
209
210async fn query_post(
211    State(service): State<PrometheusService>,
212    headers: HeaderMap,
213    Form(parameters): Form<InstantQueryParameters>,
214) -> Response {
215    execute_instant_query(service, headers, parameters).await
216}
217
218async fn execute_instant_query(
219    service: PrometheusService,
220    headers: HeaderMap,
221    parameters: InstantQueryParameters,
222) -> Response {
223    let _permit = match service.authorize(&headers) {
224        Ok(permit) => permit,
225        Err(response) => return response,
226    };
227    let time_ms = match parameters
228        .time
229        .as_deref()
230        .map(parse_prometheus_time)
231        .transpose()
232    {
233        Ok(Some(value)) => value,
234        Ok(None) => current_time_ms(),
235        Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
236    };
237    let engine = PromqlEngine::new(
238        Arc::clone(&service.store),
239        Arc::clone(&service.config.tenant),
240        PromqlLimits::default(),
241    );
242    let expression = parameters.query;
243    match tokio::task::spawn_blocking(move || engine.query(&expression, time_ms)).await {
244        Ok(Ok(value)) => query_success(value),
245        Ok(Err(error)) => query_error(
246            StatusCode::UNPROCESSABLE_ENTITY,
247            "execution",
248            &error.to_string(),
249        ),
250        Err(error) => query_error(
251            StatusCode::INTERNAL_SERVER_ERROR,
252            "internal",
253            &format!("PromQL worker failed: {error}"),
254        ),
255    }
256}
257
258async fn query_range_get(
259    State(service): State<PrometheusService>,
260    headers: HeaderMap,
261    Query(parameters): Query<RangeQueryParameters>,
262) -> Response {
263    execute_range_query(service, headers, parameters).await
264}
265
266async fn query_range_post(
267    State(service): State<PrometheusService>,
268    headers: HeaderMap,
269    Form(parameters): Form<RangeQueryParameters>,
270) -> Response {
271    execute_range_query(service, headers, parameters).await
272}
273
274async fn execute_range_query(
275    service: PrometheusService,
276    headers: HeaderMap,
277    parameters: RangeQueryParameters,
278) -> Response {
279    let _permit = match service.authorize(&headers) {
280        Ok(permit) => permit,
281        Err(response) => return response,
282    };
283    let start_ms = match parse_prometheus_time(&parameters.start) {
284        Ok(value) => value,
285        Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
286    };
287    let end_ms = match parse_prometheus_time(&parameters.end) {
288        Ok(value) => value,
289        Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
290    };
291    let step_ms = match parse_prometheus_duration(&parameters.step) {
292        Ok(value) => value,
293        Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
294    };
295    let engine = PromqlEngine::new(
296        Arc::clone(&service.store),
297        Arc::clone(&service.config.tenant),
298        PromqlLimits::default(),
299    );
300    let expression = parameters.query;
301    match tokio::task::spawn_blocking(move || {
302        engine.query_range(&expression, start_ms, end_ms, step_ms)
303    })
304    .await
305    {
306        Ok(Ok(value)) => query_success(value),
307        Ok(Err(error)) => query_error(
308            StatusCode::UNPROCESSABLE_ENTITY,
309            "execution",
310            &error.to_string(),
311        ),
312        Err(error) => query_error(
313            StatusCode::INTERNAL_SERVER_ERROR,
314            "internal",
315            &format!("PromQL worker failed: {error}"),
316        ),
317    }
318}
319
320async fn series_get(
321    State(service): State<PrometheusService>,
322    headers: HeaderMap,
323    Query(parameters): Query<DiscoveryParameters>,
324) -> Response {
325    execute_series(service, headers, parameters).await
326}
327
328async fn series_post(
329    State(service): State<PrometheusService>,
330    headers: HeaderMap,
331    Form(parameters): Form<DiscoveryParameters>,
332) -> Response {
333    execute_series(service, headers, parameters).await
334}
335
336async fn execute_series(
337    service: PrometheusService,
338    headers: HeaderMap,
339    parameters: DiscoveryParameters,
340) -> Response {
341    let points = match discovery_points(&service, &headers, &parameters).await {
342        Ok(points) => points,
343        Err(response) => return response,
344    };
345    let series = points
346        .iter()
347        .map(metric_labels)
348        .collect::<BTreeSet<_>>()
349        .into_iter()
350        .collect::<Vec<_>>();
351    api_success(json!(series))
352}
353
354async fn labels_get(
355    State(service): State<PrometheusService>,
356    headers: HeaderMap,
357    Query(parameters): Query<DiscoveryParameters>,
358) -> Response {
359    execute_labels(service, headers, parameters).await
360}
361
362async fn labels_post(
363    State(service): State<PrometheusService>,
364    headers: HeaderMap,
365    Form(parameters): Form<DiscoveryParameters>,
366) -> Response {
367    execute_labels(service, headers, parameters).await
368}
369
370async fn execute_labels(
371    service: PrometheusService,
372    headers: HeaderMap,
373    parameters: DiscoveryParameters,
374) -> Response {
375    let points = match discovery_points(&service, &headers, &parameters).await {
376        Ok(points) => points,
377        Err(response) => return response,
378    };
379    let mut labels = BTreeSet::new();
380    for point in &points {
381        labels.extend(metric_labels(point).into_keys());
382    }
383    api_success(json!(labels))
384}
385
386async fn label_values_get(
387    State(service): State<PrometheusService>,
388    headers: HeaderMap,
389    Path(name): Path<String>,
390    Query(parameters): Query<DiscoveryParameters>,
391) -> Response {
392    execute_label_values(service, headers, name, parameters).await
393}
394
395async fn label_values_post(
396    State(service): State<PrometheusService>,
397    headers: HeaderMap,
398    Path(name): Path<String>,
399    Form(parameters): Form<DiscoveryParameters>,
400) -> Response {
401    execute_label_values(service, headers, name, parameters).await
402}
403
404async fn execute_label_values(
405    service: PrometheusService,
406    headers: HeaderMap,
407    name: String,
408    parameters: DiscoveryParameters,
409) -> Response {
410    if name.is_empty() {
411        return query_error(StatusCode::BAD_REQUEST, "bad_data", "label name is empty");
412    }
413    let points = match discovery_points(&service, &headers, &parameters).await {
414        Ok(points) => points,
415        Err(response) => return response,
416    };
417    let values = points
418        .iter()
419        .filter_map(|point| metric_labels(point).remove(&name))
420        .collect::<BTreeSet<_>>();
421    api_success(json!(values))
422}
423
424async fn metadata_get(
425    State(service): State<PrometheusService>,
426    headers: HeaderMap,
427    Query(parameters): Query<MetadataParameters>,
428) -> Response {
429    let _permit = match service.authorize(&headers) {
430        Ok(permit) => permit,
431        Err(response) => return response,
432    };
433    let engine = PromqlEngine::new(
434        Arc::clone(&service.store),
435        Arc::clone(&service.config.tenant),
436        PromqlLimits::default(),
437    );
438    let selectors = parameters
439        .metric
440        .as_ref()
441        .map(|metric| vec![metric.clone()])
442        .unwrap_or_default();
443    let points = match tokio::task::spawn_blocking(move || {
444        engine.raw_points(&selectors, 0, current_time_ms())
445    })
446    .await
447    {
448        Ok(Ok(points)) => points,
449        Ok(Err(error)) => {
450            return query_error(
451                StatusCode::UNPROCESSABLE_ENTITY,
452                "execution",
453                &error.to_string(),
454            );
455        }
456        Err(error) => {
457            return query_error(
458                StatusCode::INTERNAL_SERVER_ERROR,
459                "internal",
460                &format!("metadata worker failed: {error}"),
461            );
462        }
463    };
464    let limit = parameters.limit.unwrap_or(usize::MAX);
465    let mut metadata = BTreeMap::<String, BTreeSet<(String, String, String)>>::new();
466    for point in points {
467        if metadata.len() >= limit && !metadata.contains_key(point.identity.name.as_ref()) {
468            continue;
469        }
470        metadata
471            .entry(point.identity.name.to_string())
472            .or_default()
473            .insert((
474                prometheus_metric_type(&point.identity.kind).into(),
475                point.description.to_string(),
476                point.identity.unit.to_string(),
477            ));
478    }
479    let data = metadata
480        .into_iter()
481        .map(|(name, entries)| {
482            let entries = entries
483                .into_iter()
484                .map(|(metric_type, help, unit)| {
485                    json!({"type": metric_type, "help": help, "unit": unit})
486                })
487                .collect::<Vec<_>>();
488            (name, Value::Array(entries))
489        })
490        .collect::<serde_json::Map<_, _>>();
491    api_success(Value::Object(data))
492}
493
494async fn exemplars_get(
495    State(service): State<PrometheusService>,
496    headers: HeaderMap,
497    Query(parameters): Query<ExemplarParameters>,
498) -> Response {
499    execute_exemplars(service, headers, parameters).await
500}
501
502async fn exemplars_post(
503    State(service): State<PrometheusService>,
504    headers: HeaderMap,
505    Form(parameters): Form<ExemplarParameters>,
506) -> Response {
507    execute_exemplars(service, headers, parameters).await
508}
509
510async fn execute_exemplars(
511    service: PrometheusService,
512    headers: HeaderMap,
513    parameters: ExemplarParameters,
514) -> Response {
515    let start_ms = match parse_prometheus_time(&parameters.start) {
516        Ok(value) => value,
517        Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
518    };
519    let end_ms = match parse_prometheus_time(&parameters.end) {
520        Ok(value) => value,
521        Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
522    };
523    let discovery = DiscoveryParameters {
524        selectors: vec![parameters.query],
525        start: Some((start_ms as f64 / 1_000.0).to_string()),
526        end: Some((end_ms as f64 / 1_000.0).to_string()),
527    };
528    let points = match discovery_points(&service, &headers, &discovery).await {
529        Ok(points) => points,
530        Err(response) => return response,
531    };
532    let mut series = BTreeMap::<BTreeMap<String, String>, Vec<Value>>::new();
533    for point in points {
534        let labels = metric_labels(&point);
535        for exemplar in point.exemplars.iter() {
536            let exemplar_labels = exemplar
537                .filtered_attributes
538                .iter()
539                .filter_map(|attribute| {
540                    attribute
541                        .value
542                        .as_ref()
543                        .map(|value| (attribute.key.to_string(), render_telemetry_value(value)))
544                })
545                .collect::<BTreeMap<_, _>>();
546            series.entry(labels.clone()).or_default().push(json!({
547                "labels": exemplar_labels,
548                "value": format_number(exemplar.value),
549                "timestamp": timestamp_seconds(
550                    i64::try_from(exemplar.timestamp_unix_nanos / 1_000_000).unwrap_or(i64::MAX)
551                ),
552                "traceID": exemplar.trace_id.map(|id| id.to_string()).unwrap_or_default(),
553                "spanID": exemplar.span_id.map(|id| id.to_string()).unwrap_or_default()
554            }));
555        }
556    }
557    let data = series
558        .into_iter()
559        .map(|(series_labels, exemplars)| {
560            json!({"seriesLabels": series_labels, "exemplars": exemplars})
561        })
562        .collect::<Vec<_>>();
563    api_success(json!(data))
564}
565
566#[allow(clippy::result_large_err)]
567async fn discovery_points(
568    service: &PrometheusService,
569    headers: &HeaderMap,
570    parameters: &DiscoveryParameters,
571) -> Result<Vec<crate::DurableMetricPoint>, Response> {
572    let _permit = service.authorize(headers)?;
573    let start_ms = parameters
574        .start
575        .as_deref()
576        .map(parse_prometheus_time)
577        .transpose()
578        .map_err(|error| query_error(StatusCode::BAD_REQUEST, "bad_data", &error))?
579        .unwrap_or(0);
580    let end_ms = parameters
581        .end
582        .as_deref()
583        .map(parse_prometheus_time)
584        .transpose()
585        .map_err(|error| query_error(StatusCode::BAD_REQUEST, "bad_data", &error))?
586        .unwrap_or_else(current_time_ms);
587    let engine = PromqlEngine::new(
588        Arc::clone(&service.store),
589        Arc::clone(&service.config.tenant),
590        PromqlLimits::default(),
591    );
592    let selectors = parameters.selectors.clone();
593    match tokio::task::spawn_blocking(move || engine.raw_points(&selectors, start_ms, end_ms)).await
594    {
595        Ok(Ok(points)) => Ok(points),
596        Ok(Err(error)) => Err(query_error(
597            StatusCode::UNPROCESSABLE_ENTITY,
598            "execution",
599            &error.to_string(),
600        )),
601        Err(error) => Err(query_error(
602            StatusCode::INTERNAL_SERVER_ERROR,
603            "internal",
604            &format!("discovery worker failed: {error}"),
605        )),
606    }
607}
608
609async fn remote_write(
610    State(service): State<PrometheusService>,
611    headers: HeaderMap,
612    body: Bytes,
613) -> Response {
614    let _http_permit = match service.authorize(&headers) {
615        Ok(permit) => permit,
616        Err(response) => return response,
617    };
618    if body.len() > service.config.max_request_bytes {
619        return write_error(
620            StatusCode::PAYLOAD_TOO_LARGE,
621            "Remote Write body is too large",
622        );
623    }
624    if !headers
625        .get(header::CONTENT_ENCODING)
626        .and_then(|value| value.to_str().ok())
627        .is_some_and(|value| value.eq_ignore_ascii_case("snappy"))
628    {
629        return write_error(
630            StatusCode::UNSUPPORTED_MEDIA_TYPE,
631            "Remote Write requires Content-Encoding: snappy",
632        );
633    }
634    let content_type = match headers
635        .get(header::CONTENT_TYPE)
636        .and_then(|value| value.to_str().ok())
637    {
638        Some(value) => value,
639        None => {
640            return write_error(
641                StatusCode::UNSUPPORTED_MEDIA_TYPE,
642                "Remote Write requires a protobuf Content-Type",
643            );
644        }
645    };
646    let version = match RemoteWriteDecoder::version_from_content_type(content_type) {
647        Ok(version) => version,
648        Err(error) => return write_error(StatusCode::UNSUPPORTED_MEDIA_TYPE, &error.to_string()),
649    };
650    if let Err(response) = validate_version_header(&headers, version) {
651        return response;
652    }
653    let decompressed_len = match snap::raw::decompress_len(&body) {
654        Ok(length) if length <= service.config.max_request_bytes => length,
655        Ok(_) => {
656            return write_error(
657                StatusCode::PAYLOAD_TOO_LARGE,
658                "Remote Write Snappy payload expands beyond 64 MiB",
659            );
660        }
661        Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
662    };
663    let mut protobuf = vec![0_u8; decompressed_len];
664    if let Err(error) = snap::raw::Decoder::new().decompress(&body, &mut protobuf) {
665        return write_error(StatusCode::BAD_REQUEST, &error.to_string());
666    }
667    let decoded = match RemoteWriteDecoder.decode(&service.config.tenant, version, &protobuf) {
668        Ok(decoded) => decoded,
669        Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
670    };
671    let stats = decoded.stats;
672    let batch = match decoded.into_native_batch(&service.router) {
673        Ok(batch) => batch,
674        Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
675    };
676    if batch.partitions.is_empty() {
677        return write_success(stats);
678    }
679    let _ingest_permit = match &service.production {
680        Some(runtime) => match runtime.try_ingest(protobuf.len()) {
681            Some(permit) => Some(permit),
682            None if runtime.lifecycle().state() == ServiceState::Ready => {
683                return write_error(
684                    StatusCode::TOO_MANY_REQUESTS,
685                    "Remote Write concurrency or rate limit exceeded",
686                );
687            }
688            None => {
689                return write_error(
690                    StatusCode::SERVICE_UNAVAILABLE,
691                    "Remote Write ingestion is unavailable",
692                );
693            }
694        },
695        None => None,
696    };
697    let store = Arc::clone(&service.store);
698    let result = tokio::task::spawn_blocking(move || store.append_remote_write_batch(&batch)).await;
699    match result {
700        Ok(Ok(_)) => {
701            if let Some(runtime) = &service.production {
702                let records = stats.samples.saturating_add(stats.histograms);
703                runtime.record_ingest(protobuf.len(), records as usize);
704            }
705            write_success(stats)
706        }
707        Ok(Err(error)) => write_error(error.status(), &error.to_string()),
708        Err(error) => write_error(
709            StatusCode::INTERNAL_SERVER_ERROR,
710            &format!("Remote Write worker failed: {error}"),
711        ),
712    }
713}
714
715async fn remote_read(
716    State(service): State<PrometheusService>,
717    headers: HeaderMap,
718    body: Bytes,
719) -> Response {
720    let _permit = match service.authorize(&headers) {
721        Ok(permit) => permit,
722        Err(response) => return response,
723    };
724    if body.len() > service.config.max_request_bytes {
725        return write_error(
726            StatusCode::PAYLOAD_TOO_LARGE,
727            "Remote Read body is too large",
728        );
729    }
730    if !headers
731        .get(header::CONTENT_ENCODING)
732        .and_then(|value| value.to_str().ok())
733        .is_some_and(|value| value.eq_ignore_ascii_case("snappy"))
734    {
735        return write_error(
736            StatusCode::UNSUPPORTED_MEDIA_TYPE,
737            "Remote Read requires Content-Encoding: snappy",
738        );
739    }
740    let decompressed_len = match snap::raw::decompress_len(&body) {
741        Ok(length) if length <= service.config.max_request_bytes => length,
742        Ok(_) => {
743            return write_error(
744                StatusCode::PAYLOAD_TOO_LARGE,
745                "Remote Read Snappy payload expands beyond 64 MiB",
746            );
747        }
748        Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
749    };
750    let mut protobuf = vec![0_u8; decompressed_len];
751    if let Err(error) = snap::raw::Decoder::new().decompress(&body, &mut protobuf) {
752        return write_error(StatusCode::BAD_REQUEST, &error.to_string());
753    }
754    let request = match prometheus_v1::ReadRequest::decode(protobuf.as_slice()) {
755        Ok(request) => request,
756        Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
757    };
758    let response_type = if request.accepted_response_types.is_empty() {
759        Some(prometheus_v1::ReadRequestResponseType::Samples)
760    } else {
761        request
762            .accepted_response_types
763            .iter()
764            .find_map(|value| prometheus_v1::ReadRequestResponseType::try_from(*value).ok())
765    };
766    let Some(response_type) = response_type else {
767        return write_error(
768            StatusCode::UNSUPPORTED_MEDIA_TYPE,
769            "Remote Read requested no supported response type",
770        );
771    };
772    if response_type == prometheus_v1::ReadRequestResponseType::StreamedXorChunks {
773        return remote_read_streamed(service, request).await;
774    }
775    let engine = PromqlEngine::new(
776        Arc::clone(&service.store),
777        Arc::clone(&service.config.tenant),
778        PromqlLimits::default(),
779    );
780    let result = tokio::task::spawn_blocking(move || {
781        let mut results = Vec::with_capacity(request.queries.len());
782        for query in request.queries {
783            let selector = remote_read_selector(&query.matchers)?;
784            let points = engine.raw_points(
785                &[selector],
786                query.start_timestamp_ms,
787                query.end_timestamp_ms,
788            )?;
789            let mut series = BTreeMap::<
790                BTreeMap<String, String>,
791                BTreeMap<i64, (shard_stream_core::LogicalOffset, RemoteReadValue)>,
792            >::new();
793            for point in points {
794                let timestamp =
795                    i64::try_from(point.timestamp_unix_nanos / 1_000_000).unwrap_or(i64::MAX);
796                let Some(value) = remote_read_value(&point, timestamp) else {
797                    continue;
798                };
799                let values = series.entry(metric_labels(&point)).or_default();
800                if values
801                    .get(&timestamp)
802                    .is_none_or(|(offset, _)| *offset < point.record_ref.offset)
803                {
804                    values.insert(timestamp, (point.record_ref.offset, value));
805                }
806            }
807            let timeseries = series
808                .into_iter()
809                .map(|(labels, values)| {
810                    let mut samples = Vec::new();
811                    let mut histograms = Vec::new();
812                    for (_, (_, value)) in values {
813                        match value {
814                            RemoteReadValue::Sample(sample) => samples.push(sample),
815                            RemoteReadValue::Histogram(histogram) => histograms.push(*histogram),
816                        }
817                    }
818                    prometheus_v1::TimeSeries {
819                        labels: labels
820                            .into_iter()
821                            .map(|(name, value)| prometheus_v1::Label { name, value })
822                            .collect(),
823                        samples,
824                        exemplars: Vec::new(),
825                        histograms,
826                    }
827                })
828                .collect();
829            results.push(prometheus_v1::QueryResult { timeseries });
830        }
831        Ok::<_, crate::PromqlError>(prometheus_v1::ReadResponse { results })
832    })
833    .await;
834    let response = match result {
835        Ok(Ok(response)) => response,
836        Ok(Err(error)) => {
837            return query_error(
838                StatusCode::UNPROCESSABLE_ENTITY,
839                "execution",
840                &error.to_string(),
841            );
842        }
843        Err(error) => {
844            return query_error(
845                StatusCode::INTERNAL_SERVER_ERROR,
846                "internal",
847                &format!("Remote Read worker failed: {error}"),
848            );
849        }
850    };
851    let protobuf = response.encode_to_vec();
852    let compressed = match snap::raw::Encoder::new().compress_vec(&protobuf) {
853        Ok(compressed) => compressed,
854        Err(error) => return write_error(StatusCode::INTERNAL_SERVER_ERROR, &error.to_string()),
855    };
856    (
857        StatusCode::OK,
858        [
859            (
860                header::CONTENT_TYPE,
861                HeaderValue::from_static("application/x-protobuf"),
862            ),
863            (header::CONTENT_ENCODING, HeaderValue::from_static("snappy")),
864        ],
865        compressed,
866    )
867        .into_response()
868}
869
870async fn remote_read_streamed(
871    service: PrometheusService,
872    request: prometheus_v1::ReadRequest,
873) -> Response {
874    const XOR_SAMPLES_PER_CHUNK: usize = 120;
875    const MAX_FRAME_BYTES: usize = 1024 * 1024;
876
877    let engine = PromqlEngine::new(
878        Arc::clone(&service.store),
879        Arc::clone(&service.config.tenant),
880        PromqlLimits::default(),
881    );
882    let result = tokio::task::spawn_blocking(move || {
883        let mut stream = Vec::new();
884        for (query_index, query) in request.queries.into_iter().enumerate() {
885            let selector = remote_read_selector(&query.matchers)?;
886            let points = engine.raw_points(
887                &[selector],
888                query.start_timestamp_ms,
889                query.end_timestamp_ms,
890            )?;
891            let mut series = BTreeMap::<
892                BTreeMap<String, String>,
893                BTreeMap<i64, (shard_stream_core::LogicalOffset, f64)>,
894            >::new();
895            for point in points {
896                let Some(value) = remote_read_float(&point.value) else {
897                    continue;
898                };
899                let timestamp =
900                    i64::try_from(point.timestamp_unix_nanos / 1_000_000).unwrap_or(i64::MAX);
901                let samples = series.entry(metric_labels(&point)).or_default();
902                if samples
903                    .get(&timestamp)
904                    .is_none_or(|(offset, _)| *offset < point.record_ref.offset)
905                {
906                    samples.insert(timestamp, (point.record_ref.offset, value));
907                }
908            }
909            for (labels, samples) in series {
910                let labels = labels
911                    .into_iter()
912                    .map(|(name, value)| prometheus_v1::Label { name, value })
913                    .collect::<Vec<_>>();
914                let samples = samples
915                    .into_iter()
916                    .map(|(timestamp, (_, value))| (timestamp, value))
917                    .collect::<Vec<_>>();
918                let mut chunks = Vec::new();
919                for samples in samples.chunks(XOR_SAMPLES_PER_CHUNK) {
920                    chunks.push(prometheus_v1::Chunk {
921                        min_time_ms: samples.first().expect("chunk is nonempty").0,
922                        max_time_ms: samples.last().expect("chunk is nonempty").0,
923                        r#type: prometheus_v1::ChunkEncoding::Xor as i32,
924                        data: encode_xor_chunk(samples)
925                            .map_err(|error| crate::PromqlError::new(error.to_string()))?,
926                    });
927                }
928                append_chunked_series_frames(
929                    &mut stream,
930                    i64::try_from(query_index).unwrap_or(i64::MAX),
931                    &labels,
932                    chunks,
933                    MAX_FRAME_BYTES,
934                )?;
935            }
936        }
937        Ok::<_, crate::PromqlError>(stream)
938    })
939    .await;
940    match result {
941        Ok(Ok(stream)) => (
942            StatusCode::OK,
943            [(
944                header::CONTENT_TYPE,
945                HeaderValue::from_static(
946                    "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse",
947                ),
948            )],
949            stream,
950        )
951            .into_response(),
952        Ok(Err(error)) => query_error(
953            StatusCode::UNPROCESSABLE_ENTITY,
954            "execution",
955            &error.to_string(),
956        ),
957        Err(error) => query_error(
958            StatusCode::INTERNAL_SERVER_ERROR,
959            "internal",
960            &format!("streamed Remote Read worker failed: {error}"),
961        ),
962    }
963}
964
965fn append_chunked_series_frames(
966    stream: &mut Vec<u8>,
967    query_index: i64,
968    labels: &[prometheus_v1::Label],
969    chunks: Vec<prometheus_v1::Chunk>,
970    max_frame_bytes: usize,
971) -> Result<(), crate::PromqlError> {
972    const CHUNKS_PER_FRAME: usize = 256;
973    for frame_chunks in chunks.chunks(CHUNKS_PER_FRAME) {
974        let protobuf = prometheus_v1::ChunkedReadResponse {
975            chunked_series: vec![prometheus_v1::ChunkedSeries {
976                labels: labels.to_vec(),
977                chunks: frame_chunks.to_vec(),
978            }],
979            query_index,
980        }
981        .encode_to_vec();
982        if protobuf.len() > max_frame_bytes {
983            return Err(crate::PromqlError::new(
984                "one streamed Remote Read series frame exceeds 1 MiB",
985            ));
986        }
987        append_stream_frame(stream, &protobuf)?;
988    }
989    Ok(())
990}
991
992fn append_stream_frame(stream: &mut Vec<u8>, protobuf: &[u8]) -> Result<(), crate::PromqlError> {
993    if protobuf.len() > 1024 * 1024 {
994        return Err(crate::PromqlError::new(
995            "one streamed Remote Read frame exceeds 1 MiB",
996        ));
997    }
998    append_uvarint(
999        stream,
1000        u64::try_from(protobuf.len())
1001            .map_err(|_| crate::PromqlError::new("Remote Read frame is too large"))?,
1002    );
1003    stream.extend_from_slice(&crc32c::crc32c(protobuf).to_be_bytes());
1004    stream.extend_from_slice(protobuf);
1005    Ok(())
1006}
1007
1008fn append_uvarint(output: &mut Vec<u8>, mut value: u64) {
1009    while value >= 0x80 {
1010        output.push((value as u8) | 0x80);
1011        value >>= 7;
1012    }
1013    output.push(value as u8);
1014}
1015
1016#[allow(clippy::result_large_err)]
1017fn validate_version_header(
1018    headers: &HeaderMap,
1019    version: RemoteWriteVersion,
1020) -> Result<(), Response> {
1021    let Some(observed) = headers
1022        .get("x-prometheus-remote-write-version")
1023        .and_then(|value| value.to_str().ok())
1024    else {
1025        return Err(write_error(
1026            StatusCode::BAD_REQUEST,
1027            "missing X-Prometheus-Remote-Write-Version",
1028        ));
1029    };
1030    let valid = match version {
1031        RemoteWriteVersion::V1 => observed == "0.1.0" || observed.starts_with("1."),
1032        RemoteWriteVersion::V2 => observed.starts_with("2."),
1033    };
1034    if valid {
1035        Ok(())
1036    } else {
1037        Err(write_error(
1038            StatusCode::BAD_REQUEST,
1039            "Remote Write version header conflicts with Content-Type",
1040        ))
1041    }
1042}
1043
1044fn write_success(stats: RemoteWriteStats) -> Response {
1045    let mut response = StatusCode::NO_CONTENT.into_response();
1046    for (name, value) in [
1047        ("x-prometheus-remote-write-samples-written", stats.samples),
1048        (
1049            "x-prometheus-remote-write-histograms-written",
1050            stats.histograms,
1051        ),
1052        (
1053            "x-prometheus-remote-write-exemplars-written",
1054            stats.exemplars,
1055        ),
1056    ] {
1057        response.headers_mut().insert(
1058            name,
1059            HeaderValue::from_str(&value.to_string()).expect("u64 is a valid header value"),
1060        );
1061    }
1062    response
1063}
1064
1065fn write_error(status: StatusCode, message: &str) -> Response {
1066    (
1067        status,
1068        [(header::CONTENT_TYPE, HeaderValue::from_static("text/plain"))],
1069        message.to_owned(),
1070    )
1071        .into_response()
1072}
1073
1074fn query_success(value: PromqlValue) -> Response {
1075    let (result_type, result) = match value {
1076        PromqlValue::Scalar {
1077            timestamp_ms,
1078            value,
1079        } => (
1080            "scalar",
1081            json!([timestamp_seconds(timestamp_ms), format_sample(value)]),
1082        ),
1083        PromqlValue::String {
1084            timestamp_ms,
1085            value,
1086        } => ("string", json!([timestamp_seconds(timestamp_ms), value])),
1087        PromqlValue::Vector(samples) => (
1088            "vector",
1089            Value::Array(
1090                samples
1091                    .into_iter()
1092                    .map(|sample| {
1093                        json!({
1094                            "metric": sample.labels,
1095                            "value": [
1096                                timestamp_seconds(sample.timestamp_ms),
1097                                format_sample(sample.value)
1098                            ]
1099                        })
1100                    })
1101                    .collect(),
1102            ),
1103        ),
1104        PromqlValue::Matrix(series) => (
1105            "matrix",
1106            Value::Array(
1107                series
1108                    .into_iter()
1109                    .map(|series| {
1110                        let values = series
1111                            .samples
1112                            .into_iter()
1113                            .map(|(timestamp, value)| {
1114                                json!([timestamp_seconds(timestamp), format_sample(value)])
1115                            })
1116                            .collect::<Vec<_>>();
1117                        json!({"metric": series.labels, "values": values})
1118                    })
1119                    .collect(),
1120            ),
1121        ),
1122    };
1123    (
1124        StatusCode::OK,
1125        axum::Json(json!({
1126            "status": "success",
1127            "data": {"resultType": result_type, "result": result}
1128        })),
1129    )
1130        .into_response()
1131}
1132
1133fn api_success(data: Value) -> Response {
1134    (
1135        StatusCode::OK,
1136        axum::Json(json!({"status": "success", "data": data})),
1137    )
1138        .into_response()
1139}
1140
1141fn metric_labels(point: &crate::DurableMetricPoint) -> BTreeMap<String, String> {
1142    let mut labels = crate::prometheus_string_labels(&point.identity)
1143        .into_iter()
1144        .map(|(name, value)| (name.to_string(), value.to_string()))
1145        .collect::<BTreeMap<_, _>>();
1146    labels.insert("__name__".into(), point.identity.name.to_string());
1147    labels
1148}
1149
1150fn remote_read_selector(
1151    matchers: &[prometheus_v1::LabelMatcher],
1152) -> Result<String, crate::PromqlError> {
1153    if matchers.is_empty() {
1154        return Ok("{__name__=~\".+\"}".into());
1155    }
1156    let mut selector = String::from("{");
1157    for (index, matcher) in matchers.iter().enumerate() {
1158        if matcher.name.is_empty() {
1159            return Err(crate::PromqlError::new(
1160                "Remote Read matcher has an empty label name",
1161            ));
1162        }
1163        if index != 0 {
1164            selector.push(',');
1165        }
1166        selector.push_str(&matcher.name);
1167        selector.push_str(
1168            match prometheus_v1::LabelMatcherType::try_from(matcher.r#type) {
1169                Ok(prometheus_v1::LabelMatcherType::Equal) => "=",
1170                Ok(prometheus_v1::LabelMatcherType::NotEqual) => "!=",
1171                Ok(prometheus_v1::LabelMatcherType::RegexMatch) => "=~",
1172                Ok(prometheus_v1::LabelMatcherType::RegexNoMatch) => "!~",
1173                Err(_) => {
1174                    return Err(crate::PromqlError::new(
1175                        "Remote Read matcher has an unknown type",
1176                    ));
1177                }
1178            },
1179        );
1180        selector.push_str(
1181            &serde_json::to_string(&matcher.value)
1182                .map_err(|error| crate::PromqlError::new(error.to_string()))?,
1183        );
1184    }
1185    selector.push('}');
1186    Ok(selector)
1187}
1188
1189fn remote_read_float(value: &MetricValue) -> Option<f64> {
1190    match value {
1191        MetricValue::Gauge(value) | MetricValue::Sum(value) => Some(match value {
1192            NumberValue::Integer(value) => *value as f64,
1193            NumberValue::DoubleBits(bits) => f64::from_bits(*bits),
1194        }),
1195        MetricValue::ExplicitHistogram(_)
1196        | MetricValue::ExponentialHistogram(_)
1197        | MetricValue::Summary(_) => None,
1198    }
1199}
1200
1201enum RemoteReadValue {
1202    Sample(prometheus_v1::Sample),
1203    Histogram(Box<prometheus_v1::Histogram>),
1204}
1205
1206fn remote_read_value(
1207    point: &crate::DurableMetricPoint,
1208    timestamp_ms: i64,
1209) -> Option<RemoteReadValue> {
1210    if let Some(value) = remote_read_float(&point.value) {
1211        return Some(RemoteReadValue::Sample(prometheus_v1::Sample {
1212            value,
1213            timestamp: timestamp_ms,
1214        }));
1215    }
1216    let start_timestamp = i64::try_from(point.start_time_unix_nanos / 1_000_000).ok()?;
1217    let histogram = match &point.value {
1218        MetricValue::ExplicitHistogram(value) => {
1219            let (positive_deltas, positive_counts) = histogram_count_lanes(&value.bucket_counts)?;
1220            let positive_spans = if value.bucket_counts.is_empty() {
1221                Vec::new()
1222            } else {
1223                vec![prometheus_v1::BucketSpan {
1224                    offset: 0,
1225                    length: u32::try_from(value.bucket_counts.len()).ok()?,
1226                }]
1227            };
1228            prometheus_v1::Histogram {
1229                count: Some(histogram_count(value.count)),
1230                sum: value.sum_bits.map_or(0.0, f64::from_bits),
1231                schema: -53,
1232                zero_threshold: 0.0,
1233                zero_count: Some(prometheus_v1::histogram::ZeroCount::Int(0)),
1234                negative_spans: Vec::new(),
1235                negative_deltas: Vec::new(),
1236                negative_counts: Vec::new(),
1237                positive_spans,
1238                positive_deltas,
1239                positive_counts,
1240                reset_hint: value.reset_hint,
1241                timestamp: timestamp_ms,
1242                custom_values: value
1243                    .explicit_bounds_bits
1244                    .iter()
1245                    .copied()
1246                    .map(f64::from_bits)
1247                    .collect(),
1248                start_timestamp,
1249            }
1250        }
1251        MetricValue::ExponentialHistogram(value) => {
1252            let (negative_spans, negative_deltas, negative_counts) =
1253                histogram_buckets(value.negative.as_ref())?;
1254            let (positive_spans, positive_deltas, positive_counts) =
1255                histogram_buckets(value.positive.as_ref())?;
1256            prometheus_v1::Histogram {
1257                count: Some(histogram_count(value.count)),
1258                sum: value.sum_bits.map_or(0.0, f64::from_bits),
1259                schema: value.scale,
1260                zero_threshold: f64::from_bits(value.zero_threshold_bits),
1261                zero_count: Some(histogram_zero_count(value.zero_count)),
1262                negative_spans,
1263                negative_deltas,
1264                negative_counts,
1265                positive_spans,
1266                positive_deltas,
1267                positive_counts,
1268                reset_hint: value.reset_hint,
1269                timestamp: timestamp_ms,
1270                custom_values: Vec::new(),
1271                start_timestamp,
1272            }
1273        }
1274        MetricValue::Gauge(_) | MetricValue::Sum(_) | MetricValue::Summary(_) => return None,
1275    };
1276    Some(RemoteReadValue::Histogram(Box::new(histogram)))
1277}
1278
1279fn histogram_count(value: HistogramCount) -> prometheus_v1::histogram::Count {
1280    match value {
1281        HistogramCount::Integer(value) => prometheus_v1::histogram::Count::Int(value),
1282        HistogramCount::DoubleBits(bits) => {
1283            prometheus_v1::histogram::Count::Float(f64::from_bits(bits))
1284        }
1285    }
1286}
1287
1288fn histogram_zero_count(value: HistogramCount) -> prometheus_v1::histogram::ZeroCount {
1289    match value {
1290        HistogramCount::Integer(value) => prometheus_v1::histogram::ZeroCount::Int(value),
1291        HistogramCount::DoubleBits(bits) => {
1292            prometheus_v1::histogram::ZeroCount::Float(f64::from_bits(bits))
1293        }
1294    }
1295}
1296
1297fn histogram_buckets(
1298    buckets: Option<&ExponentialHistogramBuckets>,
1299) -> Option<(Vec<prometheus_v1::BucketSpan>, Vec<i64>, Vec<f64>)> {
1300    let Some(buckets) = buckets else {
1301        return Some((Vec::new(), Vec::new(), Vec::new()));
1302    };
1303    let spans = buckets
1304        .spans
1305        .iter()
1306        .map(|span| prometheus_v1::BucketSpan {
1307            offset: span.offset,
1308            length: span.length,
1309        })
1310        .collect();
1311    let (deltas, counts) = histogram_count_lanes(&buckets.bucket_counts)?;
1312    Some((spans, deltas, counts))
1313}
1314
1315fn histogram_count_lanes(counts: &[HistogramCount]) -> Option<(Vec<i64>, Vec<f64>)> {
1316    if counts
1317        .iter()
1318        .all(|count| matches!(count, HistogramCount::Integer(_)))
1319    {
1320        let mut previous = 0_i64;
1321        let mut deltas = Vec::with_capacity(counts.len());
1322        for count in counts {
1323            let HistogramCount::Integer(count) = count else {
1324                unreachable!("integer lane was checked")
1325            };
1326            let count = i64::try_from(*count).ok()?;
1327            deltas.push(count.checked_sub(previous)?);
1328            previous = count;
1329        }
1330        Some((deltas, Vec::new()))
1331    } else if counts
1332        .iter()
1333        .all(|count| matches!(count, HistogramCount::DoubleBits(_)))
1334    {
1335        Some((
1336            Vec::new(),
1337            counts
1338                .iter()
1339                .map(|count| match count {
1340                    HistogramCount::DoubleBits(bits) => f64::from_bits(*bits),
1341                    HistogramCount::Integer(_) => unreachable!("float lane was checked"),
1342                })
1343                .collect(),
1344        ))
1345    } else {
1346        None
1347    }
1348}
1349
1350fn prometheus_metric_type(kind: &MetricKind) -> &'static str {
1351    match kind {
1352        MetricKind::Gauge => "gauge",
1353        MetricKind::Sum {
1354            monotonic: true, ..
1355        } => "counter",
1356        MetricKind::Sum { .. } => "gauge",
1357        MetricKind::ExplicitHistogram { .. } | MetricKind::ExponentialHistogram { .. } => {
1358            "histogram"
1359        }
1360        MetricKind::Summary => "summary",
1361    }
1362}
1363
1364fn format_number(value: NumberValue) -> String {
1365    match value {
1366        NumberValue::Integer(value) => value.to_string(),
1367        NumberValue::DoubleBits(bits) => format_sample(f64::from_bits(bits)),
1368    }
1369}
1370
1371fn render_telemetry_value(value: &TelemetryValue) -> String {
1372    match value {
1373        TelemetryValue::Empty => String::new(),
1374        TelemetryValue::String(value) => value.to_string(),
1375        TelemetryValue::Boolean(value) => value.to_string(),
1376        TelemetryValue::Integer(value) => value.to_string(),
1377        TelemetryValue::DoubleBits(bits) => format_sample(f64::from_bits(*bits)),
1378        TelemetryValue::Bytes(value) => value.iter().map(|byte| format!("{byte:02x}")).collect(),
1379        TelemetryValue::Array(_) | TelemetryValue::Map(_) => {
1380            serde_json::to_string(value).unwrap_or_default()
1381        }
1382        TelemetryValue::StringTableIndex(value) => value.to_string(),
1383    }
1384}
1385
1386fn query_error(status: StatusCode, error_type: &str, message: &str) -> Response {
1387    (
1388        status,
1389        axum::Json(json!({
1390            "status": "error",
1391            "errorType": error_type,
1392            "error": message
1393        })),
1394    )
1395        .into_response()
1396}
1397
1398fn format_sample(value: f64) -> String {
1399    if value.is_nan() {
1400        "NaN".into()
1401    } else if value == f64::INFINITY {
1402        "+Inf".into()
1403    } else if value == f64::NEG_INFINITY {
1404        "-Inf".into()
1405    } else {
1406        value.to_string()
1407    }
1408}
1409
1410fn timestamp_seconds(timestamp_ms: i64) -> f64 {
1411    timestamp_ms as f64 / 1_000.0
1412}
1413
1414fn current_time_ms() -> i64 {
1415    std::time::SystemTime::now()
1416        .duration_since(std::time::UNIX_EPOCH)
1417        .ok()
1418        .and_then(|duration| i64::try_from(duration.as_millis()).ok())
1419        .unwrap_or(i64::MAX)
1420}
1421
1422fn parse_prometheus_time(value: &str) -> Result<i64, String> {
1423    let seconds = value
1424        .parse::<f64>()
1425        .map_err(|_| format!("invalid Prometheus timestamp {value:?}"))?;
1426    if !seconds.is_finite() {
1427        return Err(format!("invalid Prometheus timestamp {value:?}"));
1428    }
1429    let milliseconds = seconds * 1_000.0;
1430    if milliseconds < i64::MIN as f64 || milliseconds > i64::MAX as f64 {
1431        return Err("Prometheus timestamp is out of range".into());
1432    }
1433    Ok(milliseconds.round() as i64)
1434}
1435
1436fn parse_prometheus_duration(value: &str) -> Result<i64, String> {
1437    if let Ok(seconds) = value.parse::<f64>()
1438        && seconds.is_finite()
1439        && seconds > 0.0
1440    {
1441        return Ok((seconds * 1_000.0).round() as i64);
1442    }
1443    let (number, multiplier) = [
1444        ("ms", 1_i64),
1445        ("s", 1_000),
1446        ("m", 60_000),
1447        ("h", 3_600_000),
1448        ("d", 86_400_000),
1449        ("w", 604_800_000),
1450        ("y", 31_536_000_000),
1451    ]
1452    .into_iter()
1453    .find_map(|(suffix, multiplier)| {
1454        value
1455            .strip_suffix(suffix)
1456            .map(|number| (number, multiplier))
1457    })
1458    .ok_or_else(|| format!("invalid Prometheus duration {value:?}"))?;
1459    let number = number
1460        .parse::<f64>()
1461        .map_err(|_| format!("invalid Prometheus duration {value:?}"))?;
1462    let milliseconds = number * multiplier as f64;
1463    if !milliseconds.is_finite() || milliseconds <= 0.0 || milliseconds > i64::MAX as f64 {
1464        return Err(format!("invalid Prometheus duration {value:?}"));
1465    }
1466    Ok(milliseconds.round() as i64)
1467}
1468
1469#[cfg(test)]
1470mod tests {
1471    use std::fs;
1472    use std::sync::atomic::{AtomicU64, Ordering};
1473    use std::time::{Duration, SystemTime, UNIX_EPOCH};
1474
1475    use axum::body::{Body, to_bytes};
1476    use axum::http::Request;
1477    use tower::ServiceExt;
1478
1479    use super::*;
1480    use crate::{DurableTelemetryConfig, StripeConfig};
1481
1482    static TEST_DIRECTORY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
1483
1484    fn test_service() -> (PrometheusService, std::path::PathBuf) {
1485        let nonce = SystemTime::now()
1486            .duration_since(UNIX_EPOCH)
1487            .expect("clock")
1488            .as_nanos();
1489        let directory = std::env::temp_dir().join(format!(
1490            "shard-telemetry-prometheus-api-{}-{nonce}-{}",
1491            std::process::id(),
1492            TEST_DIRECTORY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
1493        ));
1494        let store = Arc::new(
1495            DurableTelemetryStore::open(DurableTelemetryConfig {
1496                data_directory: directory.clone(),
1497                object_store_directory: None,
1498                s3_object_store: None,
1499                recovery_journal: false,
1500                retention: None,
1501                shard_count: 2,
1502                tenant_partitions: 8,
1503                append_linger: Duration::from_micros(250),
1504                stripe: StripeConfig::default(),
1505                indexed_ack_timeout: Duration::from_secs(30),
1506            })
1507            .expect("store opens"),
1508        );
1509        let service = PrometheusService::new(
1510            store,
1511            PrometheusApiConfig {
1512                tenant: Arc::from("tenant-a"),
1513                logical_partitions: NonZeroU16::new(8).unwrap(),
1514                ..PrometheusApiConfig::default()
1515            },
1516        )
1517        .expect("service");
1518        (service, directory)
1519    }
1520
1521    #[test]
1522    fn version_header_must_match_negotiated_schema() {
1523        let mut headers = HeaderMap::new();
1524        headers.insert(
1525            "x-prometheus-remote-write-version",
1526            HeaderValue::from_static("2.0.0"),
1527        );
1528        assert!(validate_version_header(&headers, RemoteWriteVersion::V2).is_ok());
1529        assert!(validate_version_header(&headers, RemoteWriteVersion::V1).is_err());
1530    }
1531
1532    #[test]
1533    fn success_reports_all_required_written_headers() {
1534        let response = write_success(RemoteWriteStats {
1535            samples: 3,
1536            histograms: 2,
1537            exemplars: 1,
1538        });
1539        assert_eq!(response.status(), StatusCode::NO_CONTENT);
1540        assert_eq!(
1541            response.headers()["x-prometheus-remote-write-samples-written"],
1542            "3"
1543        );
1544        assert_eq!(
1545            response.headers()["x-prometheus-remote-write-histograms-written"],
1546            "2"
1547        );
1548        assert_eq!(
1549            response.headers()["x-prometheus-remote-write-exemplars-written"],
1550            "1"
1551        );
1552    }
1553
1554    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1555    async fn stable_prometheus_route_surface_has_no_missing_or_wrong_method_routes() {
1556        let (service, directory) = test_service();
1557        let app = prometheus_router(service.clone());
1558        let routes = [
1559            (axum::http::Method::POST, "/api/v1/write"),
1560            (axum::http::Method::POST, "/api/v1/read"),
1561            (axum::http::Method::GET, "/api/v1/query?query=1"),
1562            (axum::http::Method::POST, "/api/v1/query"),
1563            (
1564                axum::http::Method::GET,
1565                "/api/v1/query_range?query=1&start=0&end=1&step=1",
1566            ),
1567            (axum::http::Method::POST, "/api/v1/query_range"),
1568            (axum::http::Method::GET, "/api/v1/series?start=0&end=1"),
1569            (axum::http::Method::POST, "/api/v1/series"),
1570            (axum::http::Method::GET, "/api/v1/labels?start=0&end=1"),
1571            (axum::http::Method::POST, "/api/v1/labels"),
1572            (
1573                axum::http::Method::GET,
1574                "/api/v1/label/job/values?start=0&end=1",
1575            ),
1576            (axum::http::Method::POST, "/api/v1/label/job/values"),
1577            (axum::http::Method::GET, "/api/v1/metadata"),
1578            (
1579                axum::http::Method::GET,
1580                "/api/v1/query_exemplars?query=%7B__name__%3D%22x%22%7D&start=0&end=1",
1581            ),
1582            (axum::http::Method::POST, "/api/v1/query_exemplars"),
1583        ];
1584        for (method, path) in routes {
1585            let response = app
1586                .clone()
1587                .oneshot(
1588                    Request::builder()
1589                        .method(method.clone())
1590                        .uri(path)
1591                        .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
1592                        .body(Body::empty())
1593                        .expect("request"),
1594                )
1595                .await
1596                .expect("Prometheus response");
1597            assert_ne!(response.status(), StatusCode::NOT_FOUND, "{method} {path}");
1598            assert_ne!(
1599                response.status(),
1600                StatusCode::METHOD_NOT_ALLOWED,
1601                "{method} {path}"
1602            );
1603        }
1604
1605        drop(app);
1606        drop(service);
1607        fs::remove_dir_all(directory).expect("cleanup");
1608    }
1609
1610    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1611    async fn query_discovery_and_remote_read_routes_use_prometheus_envelopes() {
1612        let (service, directory) = test_service();
1613        let app = prometheus_router(service.clone());
1614        let response = app
1615            .clone()
1616            .oneshot(
1617                Request::builder()
1618                    .uri("/api/v1/query?query=1%2B2&time=10")
1619                    .body(Body::empty())
1620                    .unwrap(),
1621            )
1622            .await
1623            .expect("query response");
1624        assert_eq!(response.status(), StatusCode::OK);
1625        let body = to_bytes(response.into_body(), 1_024 * 1_024)
1626            .await
1627            .expect("query body");
1628        let body: Value = serde_json::from_slice(&body).expect("query JSON");
1629        assert_eq!(body["status"], "success");
1630        assert_eq!(body["data"]["resultType"], "scalar");
1631        assert_eq!(body["data"]["result"][1], "3");
1632
1633        let response = app
1634            .clone()
1635            .oneshot(
1636                Request::builder()
1637                    .uri("/api/v1/series?start=0&end=10")
1638                    .body(Body::empty())
1639                    .unwrap(),
1640            )
1641            .await
1642            .expect("series response");
1643        assert_eq!(response.status(), StatusCode::OK);
1644
1645        let request = prometheus_v1::ReadRequest {
1646            queries: Vec::new(),
1647            accepted_response_types: vec![prometheus_v1::ReadRequestResponseType::Samples as i32],
1648        };
1649        let compressed = snap::raw::Encoder::new()
1650            .compress_vec(&request.encode_to_vec())
1651            .expect("Snappy request");
1652        let response = app
1653            .oneshot(
1654                Request::builder()
1655                    .method("POST")
1656                    .uri("/api/v1/read")
1657                    .header(header::CONTENT_ENCODING, "snappy")
1658                    .body(Body::from(compressed))
1659                    .unwrap(),
1660            )
1661            .await
1662            .expect("read response");
1663        assert_eq!(response.status(), StatusCode::OK);
1664        let body = to_bytes(response.into_body(), 1_024 * 1_024)
1665            .await
1666            .expect("read body");
1667        let protobuf = snap::raw::Decoder::new()
1668            .decompress_vec(&body)
1669            .expect("Snappy response");
1670        assert!(
1671            prometheus_v1::ReadResponse::decode(protobuf.as_slice())
1672                .expect("read response protobuf")
1673                .results
1674                .is_empty()
1675        );
1676
1677        drop(service);
1678        fs::remove_dir_all(directory).expect("remove test store");
1679    }
1680
1681    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1682    async fn streamed_remote_read_returns_crc_framed_prometheus_xor_chunks() {
1683        let (service, directory) = test_service();
1684        let app = prometheus_router(service.clone());
1685        let write = prometheus_v1::WriteRequest {
1686            timeseries: vec![prometheus_v1::TimeSeries {
1687                labels: vec![
1688                    prometheus_v1::Label {
1689                        name: "job".into(),
1690                        value: "api".into(),
1691                    },
1692                    prometheus_v1::Label {
1693                        name: "__name__".into(),
1694                        value: "requests_total".into(),
1695                    },
1696                ],
1697                samples: vec![
1698                    prometheus_v1::Sample {
1699                        value: 1.0,
1700                        timestamp: 1_000,
1701                    },
1702                    prometheus_v1::Sample {
1703                        value: 2.0,
1704                        timestamp: 2_000,
1705                    },
1706                ],
1707                exemplars: Vec::new(),
1708                histograms: Vec::new(),
1709            }],
1710            metadata: Vec::new(),
1711        };
1712        let compressed = snap::raw::Encoder::new()
1713            .compress_vec(&write.encode_to_vec())
1714            .expect("Snappy write request");
1715        let response = app
1716            .clone()
1717            .oneshot(
1718                Request::builder()
1719                    .method("POST")
1720                    .uri("/api/v1/write")
1721                    .header(header::CONTENT_TYPE, "application/x-protobuf")
1722                    .header(header::CONTENT_ENCODING, "snappy")
1723                    .header("x-prometheus-remote-write-version", "1.0.0")
1724                    .body(Body::from(compressed))
1725                    .unwrap(),
1726            )
1727            .await
1728            .expect("write response");
1729        assert_eq!(response.status(), StatusCode::NO_CONTENT);
1730
1731        let request = prometheus_v1::ReadRequest {
1732            queries: vec![prometheus_v1::Query {
1733                start_timestamp_ms: 0,
1734                end_timestamp_ms: 3_000,
1735                matchers: vec![prometheus_v1::LabelMatcher {
1736                    r#type: prometheus_v1::LabelMatcherType::Equal as i32,
1737                    name: "__name__".into(),
1738                    value: "requests_total".into(),
1739                }],
1740                hints: None,
1741            }],
1742            accepted_response_types: vec![
1743                prometheus_v1::ReadRequestResponseType::StreamedXorChunks as i32,
1744                prometheus_v1::ReadRequestResponseType::Samples as i32,
1745            ],
1746        };
1747        let compressed = snap::raw::Encoder::new()
1748            .compress_vec(&request.encode_to_vec())
1749            .expect("Snappy read request");
1750        let response = app
1751            .oneshot(
1752                Request::builder()
1753                    .method("POST")
1754                    .uri("/api/v1/read")
1755                    .header(header::CONTENT_ENCODING, "snappy")
1756                    .body(Body::from(compressed))
1757                    .unwrap(),
1758            )
1759            .await
1760            .expect("streamed read response");
1761        assert_eq!(response.status(), StatusCode::OK);
1762        assert_eq!(
1763            response.headers()[header::CONTENT_TYPE],
1764            "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse"
1765        );
1766        assert!(!response.headers().contains_key(header::CONTENT_ENCODING));
1767        let body = to_bytes(response.into_body(), 2 * 1024 * 1024)
1768            .await
1769            .expect("stream body");
1770        let (frame_bytes, delimiter_bytes) = decode_test_uvarint(&body);
1771        let checksum_start = delimiter_bytes;
1772        let protobuf_start = checksum_start + 4;
1773        let protobuf_end = protobuf_start + usize::try_from(frame_bytes).unwrap();
1774        assert_eq!(protobuf_end, body.len());
1775        assert_eq!(
1776            u32::from_be_bytes(body[checksum_start..protobuf_start].try_into().unwrap()),
1777            crc32c::crc32c(&body[protobuf_start..protobuf_end])
1778        );
1779        let frame = prometheus_v1::ChunkedReadResponse::decode(&body[protobuf_start..protobuf_end])
1780            .expect("chunked response protobuf");
1781        assert_eq!(frame.query_index, 0);
1782        assert_eq!(frame.chunked_series.len(), 1);
1783        assert_eq!(
1784            frame.chunked_series[0]
1785                .labels
1786                .iter()
1787                .map(|label| label.name.as_str())
1788                .collect::<Vec<_>>(),
1789            vec!["__name__", "job"]
1790        );
1791        assert_eq!(frame.chunked_series[0].chunks.len(), 1);
1792        let chunk = &frame.chunked_series[0].chunks[0];
1793        assert_eq!(chunk.min_time_ms, 1_000);
1794        assert_eq!(chunk.max_time_ms, 2_000);
1795        assert_eq!(chunk.r#type, prometheus_v1::ChunkEncoding::Xor as i32);
1796        assert_eq!(u16::from_be_bytes(chunk.data[..2].try_into().unwrap()), 2);
1797
1798        drop(service);
1799        fs::remove_dir_all(directory).expect("remove test store");
1800    }
1801
1802    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1803    async fn sample_remote_read_round_trips_native_histograms() {
1804        let (service, directory) = test_service();
1805        let app = prometheus_router(service.clone());
1806        let write = prometheus_v1::WriteRequest {
1807            timeseries: vec![prometheus_v1::TimeSeries {
1808                labels: vec![prometheus_v1::Label {
1809                    name: "__name__".into(),
1810                    value: "request_duration".into(),
1811                }],
1812                samples: Vec::new(),
1813                exemplars: Vec::new(),
1814                histograms: vec![prometheus_v1::Histogram {
1815                    count: Some(prometheus_v1::histogram::Count::Int(3)),
1816                    sum: 6.0,
1817                    schema: 0,
1818                    zero_threshold: 0.001,
1819                    zero_count: Some(prometheus_v1::histogram::ZeroCount::Int(0)),
1820                    negative_spans: Vec::new(),
1821                    negative_deltas: Vec::new(),
1822                    negative_counts: Vec::new(),
1823                    positive_spans: vec![prometheus_v1::BucketSpan {
1824                        offset: 0,
1825                        length: 2,
1826                    }],
1827                    positive_deltas: vec![1, 1],
1828                    positive_counts: Vec::new(),
1829                    reset_hint: prometheus_v1::histogram::ResetHint::No as i32,
1830                    timestamp: 2_000,
1831                    custom_values: Vec::new(),
1832                    start_timestamp: 1_000,
1833                }],
1834            }],
1835            metadata: Vec::new(),
1836        };
1837        let compressed = snap::raw::Encoder::new()
1838            .compress_vec(&write.encode_to_vec())
1839            .unwrap();
1840        let response = app
1841            .clone()
1842            .oneshot(
1843                Request::builder()
1844                    .method("POST")
1845                    .uri("/api/v1/write")
1846                    .header(header::CONTENT_TYPE, "application/x-protobuf")
1847                    .header(header::CONTENT_ENCODING, "snappy")
1848                    .header("x-prometheus-remote-write-version", "1.0.0")
1849                    .body(Body::from(compressed))
1850                    .unwrap(),
1851            )
1852            .await
1853            .unwrap();
1854        assert_eq!(response.status(), StatusCode::NO_CONTENT);
1855
1856        let read = prometheus_v1::ReadRequest {
1857            queries: vec![prometheus_v1::Query {
1858                start_timestamp_ms: 0,
1859                end_timestamp_ms: 3_000,
1860                matchers: vec![prometheus_v1::LabelMatcher {
1861                    r#type: prometheus_v1::LabelMatcherType::Equal as i32,
1862                    name: "__name__".into(),
1863                    value: "request_duration".into(),
1864                }],
1865                hints: None,
1866            }],
1867            accepted_response_types: vec![prometheus_v1::ReadRequestResponseType::Samples as i32],
1868        };
1869        let compressed = snap::raw::Encoder::new()
1870            .compress_vec(&read.encode_to_vec())
1871            .unwrap();
1872        let response = app
1873            .oneshot(
1874                Request::builder()
1875                    .method("POST")
1876                    .uri("/api/v1/read")
1877                    .header(header::CONTENT_ENCODING, "snappy")
1878                    .body(Body::from(compressed))
1879                    .unwrap(),
1880            )
1881            .await
1882            .unwrap();
1883        assert_eq!(response.status(), StatusCode::OK);
1884        let body = to_bytes(response.into_body(), 1_024 * 1_024).await.unwrap();
1885        let protobuf = snap::raw::Decoder::new().decompress_vec(&body).unwrap();
1886        let response = prometheus_v1::ReadResponse::decode(protobuf.as_slice()).unwrap();
1887        let histogram = &response.results[0].timeseries[0].histograms[0];
1888        assert_eq!(histogram.schema, 0);
1889        assert_eq!(histogram.positive_deltas, vec![1, 1]);
1890        assert_eq!(histogram.sum.to_bits(), 6.0_f64.to_bits());
1891        assert_eq!(histogram.start_timestamp, 1_000);
1892
1893        drop(service);
1894        fs::remove_dir_all(directory).expect("remove test store");
1895    }
1896
1897    fn decode_test_uvarint(bytes: &[u8]) -> (u64, usize) {
1898        let mut value = 0_u64;
1899        for (index, byte) in bytes.iter().copied().enumerate().take(10) {
1900            value |= u64::from(byte & 0x7f) << (index * 7);
1901            if byte & 0x80 == 0 {
1902                return (value, index + 1);
1903            }
1904        }
1905        panic!("invalid test frame delimiter")
1906    }
1907}