1use std::collections::{BTreeMap, BTreeSet};
2use std::num::NonZeroU16;
3use std::sync::Arc;
4
5use axum::Router;
6use axum::body::Bytes;
7use axum::extract::{DefaultBodyLimit, Form, Path, Query, State};
8use axum::http::{HeaderMap, HeaderValue, StatusCode, header};
9use axum::response::{IntoResponse, Response};
10use axum::routing::{get, post};
11use prost::Message;
12use serde::Deserialize;
13use serde_json::{Value, json};
14
15use crate::prometheus_protocol::v1 as prometheus_v1;
16use crate::prometheus_xor::encode_xor_chunk;
17use crate::{
18 DurableTelemetryStore, ExponentialHistogramBuckets, HistogramCount, MetricKind, MetricValue,
19 NumberValue, ProductionRuntime, PromqlEngine, PromqlLimits, PromqlValue, RemoteWriteDecoder,
20 RemoteWriteStats, RemoteWriteVersion, ServiceState, TelemetryError, TelemetryResult,
21 TelemetryRouter, TelemetryValue,
22};
23
24#[derive(Debug, Clone, PartialEq, Eq)]
26pub struct PrometheusApiConfig {
27 pub tenant: Arc<str>,
29 pub max_request_bytes: usize,
31 pub logical_partitions: NonZeroU16,
33}
34
35impl Default for PrometheusApiConfig {
36 fn default() -> Self {
37 Self {
38 tenant: Arc::from("default"),
39 max_request_bytes: 64 * 1024 * 1024,
40 logical_partitions: NonZeroU16::new(256).expect("constant is nonzero"),
41 }
42 }
43}
44
45impl PrometheusApiConfig {
46 fn validate(&self) -> TelemetryResult<()> {
47 if self.tenant.is_empty() {
48 return Err(TelemetryError::InvalidConfiguration(
49 "Prometheus tenant must not be empty".into(),
50 ));
51 }
52 if self.max_request_bytes == 0 || self.max_request_bytes > 64 * 1024 * 1024 {
53 return Err(TelemetryError::InvalidConfiguration(
54 "Prometheus request limit must be in 1..=64 MiB".into(),
55 ));
56 }
57 Ok(())
58 }
59}
60
61#[derive(Clone)]
63pub struct PrometheusService {
64 store: Arc<DurableTelemetryStore>,
65 config: PrometheusApiConfig,
66 router: TelemetryRouter,
67 production: Option<Arc<ProductionRuntime>>,
68}
69
70impl std::fmt::Debug for PrometheusService {
71 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72 formatter
73 .debug_struct("PrometheusService")
74 .field("config", &self.config)
75 .finish_non_exhaustive()
76 }
77}
78
79impl PrometheusService {
80 pub fn new(
82 store: Arc<DurableTelemetryStore>,
83 config: PrometheusApiConfig,
84 ) -> TelemetryResult<Self> {
85 config.validate()?;
86 Ok(Self {
87 store,
88 router: TelemetryRouter::new(config.logical_partitions),
89 config,
90 production: None,
91 })
92 }
93
94 #[must_use]
96 pub fn with_production(mut self, production: Option<Arc<ProductionRuntime>>) -> Self {
97 self.production = production;
98 self
99 }
100
101 #[allow(clippy::result_large_err)]
102 fn authorize(
103 &self,
104 headers: &HeaderMap,
105 ) -> Result<Option<tokio::sync::OwnedSemaphorePermit>, Response> {
106 if headers
107 .get("x-scope-orgid")
108 .and_then(|value| value.to_str().ok())
109 .is_some_and(|tenant| tenant != self.config.tenant.as_ref())
110 {
111 return Err(write_error(
112 StatusCode::FORBIDDEN,
113 "tenant header does not match the configured tenant",
114 ));
115 }
116 let Some(runtime) = &self.production else {
117 return Ok(None);
118 };
119 let credential = headers
120 .get(header::AUTHORIZATION)
121 .and_then(|value| value.to_str().ok())
122 .and_then(|value| value.strip_prefix("Bearer "));
123 if !credential.is_some_and(|value| runtime.authenticates(value)) {
124 if credential.is_none() {
125 runtime.record_authentication_failure();
126 }
127 return Err(write_error(
128 StatusCode::UNAUTHORIZED,
129 "valid production bearer token is required",
130 ));
131 }
132 runtime.try_http().map(Some).ok_or_else(|| {
133 write_error(
134 StatusCode::TOO_MANY_REQUESTS,
135 "HTTP concurrency limit exceeded",
136 )
137 })
138 }
139}
140
141pub fn prometheus_router(service: PrometheusService) -> Router {
143 let max_request_bytes = service.config.max_request_bytes;
144 Router::new()
145 .route("/api/v1/write", post(remote_write))
146 .route("/api/v1/read", post(remote_read))
147 .route("/api/v1/query", get(query_get).post(query_post))
148 .route(
149 "/api/v1/query_range",
150 get(query_range_get).post(query_range_post),
151 )
152 .route("/api/v1/series", get(series_get).post(series_post))
153 .route("/api/v1/labels", get(labels_get).post(labels_post))
154 .route(
155 "/api/v1/label/{name}/values",
156 get(label_values_get).post(label_values_post),
157 )
158 .route("/api/v1/metadata", get(metadata_get))
159 .route(
160 "/api/v1/query_exemplars",
161 get(exemplars_get).post(exemplars_post),
162 )
163 .layer(DefaultBodyLimit::max(max_request_bytes))
164 .with_state(service)
165}
166
167#[derive(Debug, Clone, Deserialize)]
168struct InstantQueryParameters {
169 query: String,
170 time: Option<String>,
171}
172
173#[derive(Debug, Clone, Deserialize)]
174struct RangeQueryParameters {
175 query: String,
176 start: String,
177 end: String,
178 step: String,
179}
180
181#[derive(Debug, Clone, Default, Deserialize)]
182struct DiscoveryParameters {
183 #[serde(default, rename = "match[]")]
184 selectors: Vec<String>,
185 start: Option<String>,
186 end: Option<String>,
187}
188
189#[derive(Debug, Clone, Default, Deserialize)]
190struct MetadataParameters {
191 metric: Option<String>,
192 limit: Option<usize>,
193}
194
195#[derive(Debug, Clone, Deserialize)]
196struct ExemplarParameters {
197 query: String,
198 start: String,
199 end: String,
200}
201
202async fn query_get(
203 State(service): State<PrometheusService>,
204 headers: HeaderMap,
205 Query(parameters): Query<InstantQueryParameters>,
206) -> Response {
207 execute_instant_query(service, headers, parameters).await
208}
209
210async fn query_post(
211 State(service): State<PrometheusService>,
212 headers: HeaderMap,
213 Form(parameters): Form<InstantQueryParameters>,
214) -> Response {
215 execute_instant_query(service, headers, parameters).await
216}
217
218async fn execute_instant_query(
219 service: PrometheusService,
220 headers: HeaderMap,
221 parameters: InstantQueryParameters,
222) -> Response {
223 let _permit = match service.authorize(&headers) {
224 Ok(permit) => permit,
225 Err(response) => return response,
226 };
227 let time_ms = match parameters
228 .time
229 .as_deref()
230 .map(parse_prometheus_time)
231 .transpose()
232 {
233 Ok(Some(value)) => value,
234 Ok(None) => current_time_ms(),
235 Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
236 };
237 let engine = PromqlEngine::new(
238 Arc::clone(&service.store),
239 Arc::clone(&service.config.tenant),
240 PromqlLimits::default(),
241 );
242 let expression = parameters.query;
243 match tokio::task::spawn_blocking(move || engine.query(&expression, time_ms)).await {
244 Ok(Ok(value)) => query_success(value),
245 Ok(Err(error)) => query_error(
246 StatusCode::UNPROCESSABLE_ENTITY,
247 "execution",
248 &error.to_string(),
249 ),
250 Err(error) => query_error(
251 StatusCode::INTERNAL_SERVER_ERROR,
252 "internal",
253 &format!("PromQL worker failed: {error}"),
254 ),
255 }
256}
257
258async fn query_range_get(
259 State(service): State<PrometheusService>,
260 headers: HeaderMap,
261 Query(parameters): Query<RangeQueryParameters>,
262) -> Response {
263 execute_range_query(service, headers, parameters).await
264}
265
266async fn query_range_post(
267 State(service): State<PrometheusService>,
268 headers: HeaderMap,
269 Form(parameters): Form<RangeQueryParameters>,
270) -> Response {
271 execute_range_query(service, headers, parameters).await
272}
273
274async fn execute_range_query(
275 service: PrometheusService,
276 headers: HeaderMap,
277 parameters: RangeQueryParameters,
278) -> Response {
279 let _permit = match service.authorize(&headers) {
280 Ok(permit) => permit,
281 Err(response) => return response,
282 };
283 let start_ms = match parse_prometheus_time(¶meters.start) {
284 Ok(value) => value,
285 Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
286 };
287 let end_ms = match parse_prometheus_time(¶meters.end) {
288 Ok(value) => value,
289 Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
290 };
291 let step_ms = match parse_prometheus_duration(¶meters.step) {
292 Ok(value) => value,
293 Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
294 };
295 let engine = PromqlEngine::new(
296 Arc::clone(&service.store),
297 Arc::clone(&service.config.tenant),
298 PromqlLimits::default(),
299 );
300 let expression = parameters.query;
301 match tokio::task::spawn_blocking(move || {
302 engine.query_range(&expression, start_ms, end_ms, step_ms)
303 })
304 .await
305 {
306 Ok(Ok(value)) => query_success(value),
307 Ok(Err(error)) => query_error(
308 StatusCode::UNPROCESSABLE_ENTITY,
309 "execution",
310 &error.to_string(),
311 ),
312 Err(error) => query_error(
313 StatusCode::INTERNAL_SERVER_ERROR,
314 "internal",
315 &format!("PromQL worker failed: {error}"),
316 ),
317 }
318}
319
320async fn series_get(
321 State(service): State<PrometheusService>,
322 headers: HeaderMap,
323 Query(parameters): Query<DiscoveryParameters>,
324) -> Response {
325 execute_series(service, headers, parameters).await
326}
327
328async fn series_post(
329 State(service): State<PrometheusService>,
330 headers: HeaderMap,
331 Form(parameters): Form<DiscoveryParameters>,
332) -> Response {
333 execute_series(service, headers, parameters).await
334}
335
336async fn execute_series(
337 service: PrometheusService,
338 headers: HeaderMap,
339 parameters: DiscoveryParameters,
340) -> Response {
341 let points = match discovery_points(&service, &headers, ¶meters).await {
342 Ok(points) => points,
343 Err(response) => return response,
344 };
345 let series = points
346 .iter()
347 .map(metric_labels)
348 .collect::<BTreeSet<_>>()
349 .into_iter()
350 .collect::<Vec<_>>();
351 api_success(json!(series))
352}
353
354async fn labels_get(
355 State(service): State<PrometheusService>,
356 headers: HeaderMap,
357 Query(parameters): Query<DiscoveryParameters>,
358) -> Response {
359 execute_labels(service, headers, parameters).await
360}
361
362async fn labels_post(
363 State(service): State<PrometheusService>,
364 headers: HeaderMap,
365 Form(parameters): Form<DiscoveryParameters>,
366) -> Response {
367 execute_labels(service, headers, parameters).await
368}
369
370async fn execute_labels(
371 service: PrometheusService,
372 headers: HeaderMap,
373 parameters: DiscoveryParameters,
374) -> Response {
375 let points = match discovery_points(&service, &headers, ¶meters).await {
376 Ok(points) => points,
377 Err(response) => return response,
378 };
379 let mut labels = BTreeSet::new();
380 for point in &points {
381 labels.extend(metric_labels(point).into_keys());
382 }
383 api_success(json!(labels))
384}
385
386async fn label_values_get(
387 State(service): State<PrometheusService>,
388 headers: HeaderMap,
389 Path(name): Path<String>,
390 Query(parameters): Query<DiscoveryParameters>,
391) -> Response {
392 execute_label_values(service, headers, name, parameters).await
393}
394
395async fn label_values_post(
396 State(service): State<PrometheusService>,
397 headers: HeaderMap,
398 Path(name): Path<String>,
399 Form(parameters): Form<DiscoveryParameters>,
400) -> Response {
401 execute_label_values(service, headers, name, parameters).await
402}
403
404async fn execute_label_values(
405 service: PrometheusService,
406 headers: HeaderMap,
407 name: String,
408 parameters: DiscoveryParameters,
409) -> Response {
410 if name.is_empty() {
411 return query_error(StatusCode::BAD_REQUEST, "bad_data", "label name is empty");
412 }
413 let points = match discovery_points(&service, &headers, ¶meters).await {
414 Ok(points) => points,
415 Err(response) => return response,
416 };
417 let values = points
418 .iter()
419 .filter_map(|point| metric_labels(point).remove(&name))
420 .collect::<BTreeSet<_>>();
421 api_success(json!(values))
422}
423
424async fn metadata_get(
425 State(service): State<PrometheusService>,
426 headers: HeaderMap,
427 Query(parameters): Query<MetadataParameters>,
428) -> Response {
429 let _permit = match service.authorize(&headers) {
430 Ok(permit) => permit,
431 Err(response) => return response,
432 };
433 let engine = PromqlEngine::new(
434 Arc::clone(&service.store),
435 Arc::clone(&service.config.tenant),
436 PromqlLimits::default(),
437 );
438 let selectors = parameters
439 .metric
440 .as_ref()
441 .map(|metric| vec![metric.clone()])
442 .unwrap_or_default();
443 let points = match tokio::task::spawn_blocking(move || {
444 engine.raw_points(&selectors, 0, current_time_ms())
445 })
446 .await
447 {
448 Ok(Ok(points)) => points,
449 Ok(Err(error)) => {
450 return query_error(
451 StatusCode::UNPROCESSABLE_ENTITY,
452 "execution",
453 &error.to_string(),
454 );
455 }
456 Err(error) => {
457 return query_error(
458 StatusCode::INTERNAL_SERVER_ERROR,
459 "internal",
460 &format!("metadata worker failed: {error}"),
461 );
462 }
463 };
464 let limit = parameters.limit.unwrap_or(usize::MAX);
465 let mut metadata = BTreeMap::<String, BTreeSet<(String, String, String)>>::new();
466 for point in points {
467 if metadata.len() >= limit && !metadata.contains_key(point.identity.name.as_ref()) {
468 continue;
469 }
470 metadata
471 .entry(point.identity.name.to_string())
472 .or_default()
473 .insert((
474 prometheus_metric_type(&point.identity.kind).into(),
475 point.description.to_string(),
476 point.identity.unit.to_string(),
477 ));
478 }
479 let data = metadata
480 .into_iter()
481 .map(|(name, entries)| {
482 let entries = entries
483 .into_iter()
484 .map(|(metric_type, help, unit)| {
485 json!({"type": metric_type, "help": help, "unit": unit})
486 })
487 .collect::<Vec<_>>();
488 (name, Value::Array(entries))
489 })
490 .collect::<serde_json::Map<_, _>>();
491 api_success(Value::Object(data))
492}
493
494async fn exemplars_get(
495 State(service): State<PrometheusService>,
496 headers: HeaderMap,
497 Query(parameters): Query<ExemplarParameters>,
498) -> Response {
499 execute_exemplars(service, headers, parameters).await
500}
501
502async fn exemplars_post(
503 State(service): State<PrometheusService>,
504 headers: HeaderMap,
505 Form(parameters): Form<ExemplarParameters>,
506) -> Response {
507 execute_exemplars(service, headers, parameters).await
508}
509
510async fn execute_exemplars(
511 service: PrometheusService,
512 headers: HeaderMap,
513 parameters: ExemplarParameters,
514) -> Response {
515 let start_ms = match parse_prometheus_time(¶meters.start) {
516 Ok(value) => value,
517 Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
518 };
519 let end_ms = match parse_prometheus_time(¶meters.end) {
520 Ok(value) => value,
521 Err(error) => return query_error(StatusCode::BAD_REQUEST, "bad_data", &error),
522 };
523 let discovery = DiscoveryParameters {
524 selectors: vec![parameters.query],
525 start: Some((start_ms as f64 / 1_000.0).to_string()),
526 end: Some((end_ms as f64 / 1_000.0).to_string()),
527 };
528 let points = match discovery_points(&service, &headers, &discovery).await {
529 Ok(points) => points,
530 Err(response) => return response,
531 };
532 let mut series = BTreeMap::<BTreeMap<String, String>, Vec<Value>>::new();
533 for point in points {
534 let labels = metric_labels(&point);
535 for exemplar in point.exemplars.iter() {
536 let exemplar_labels = exemplar
537 .filtered_attributes
538 .iter()
539 .filter_map(|attribute| {
540 attribute
541 .value
542 .as_ref()
543 .map(|value| (attribute.key.to_string(), render_telemetry_value(value)))
544 })
545 .collect::<BTreeMap<_, _>>();
546 series.entry(labels.clone()).or_default().push(json!({
547 "labels": exemplar_labels,
548 "value": format_number(exemplar.value),
549 "timestamp": timestamp_seconds(
550 i64::try_from(exemplar.timestamp_unix_nanos / 1_000_000).unwrap_or(i64::MAX)
551 ),
552 "traceID": exemplar.trace_id.map(|id| id.to_string()).unwrap_or_default(),
553 "spanID": exemplar.span_id.map(|id| id.to_string()).unwrap_or_default()
554 }));
555 }
556 }
557 let data = series
558 .into_iter()
559 .map(|(series_labels, exemplars)| {
560 json!({"seriesLabels": series_labels, "exemplars": exemplars})
561 })
562 .collect::<Vec<_>>();
563 api_success(json!(data))
564}
565
566#[allow(clippy::result_large_err)]
567async fn discovery_points(
568 service: &PrometheusService,
569 headers: &HeaderMap,
570 parameters: &DiscoveryParameters,
571) -> Result<Vec<crate::DurableMetricPoint>, Response> {
572 let _permit = service.authorize(headers)?;
573 let start_ms = parameters
574 .start
575 .as_deref()
576 .map(parse_prometheus_time)
577 .transpose()
578 .map_err(|error| query_error(StatusCode::BAD_REQUEST, "bad_data", &error))?
579 .unwrap_or(0);
580 let end_ms = parameters
581 .end
582 .as_deref()
583 .map(parse_prometheus_time)
584 .transpose()
585 .map_err(|error| query_error(StatusCode::BAD_REQUEST, "bad_data", &error))?
586 .unwrap_or_else(current_time_ms);
587 let engine = PromqlEngine::new(
588 Arc::clone(&service.store),
589 Arc::clone(&service.config.tenant),
590 PromqlLimits::default(),
591 );
592 let selectors = parameters.selectors.clone();
593 match tokio::task::spawn_blocking(move || engine.raw_points(&selectors, start_ms, end_ms)).await
594 {
595 Ok(Ok(points)) => Ok(points),
596 Ok(Err(error)) => Err(query_error(
597 StatusCode::UNPROCESSABLE_ENTITY,
598 "execution",
599 &error.to_string(),
600 )),
601 Err(error) => Err(query_error(
602 StatusCode::INTERNAL_SERVER_ERROR,
603 "internal",
604 &format!("discovery worker failed: {error}"),
605 )),
606 }
607}
608
609async fn remote_write(
610 State(service): State<PrometheusService>,
611 headers: HeaderMap,
612 body: Bytes,
613) -> Response {
614 let _http_permit = match service.authorize(&headers) {
615 Ok(permit) => permit,
616 Err(response) => return response,
617 };
618 if body.len() > service.config.max_request_bytes {
619 return write_error(
620 StatusCode::PAYLOAD_TOO_LARGE,
621 "Remote Write body is too large",
622 );
623 }
624 if !headers
625 .get(header::CONTENT_ENCODING)
626 .and_then(|value| value.to_str().ok())
627 .is_some_and(|value| value.eq_ignore_ascii_case("snappy"))
628 {
629 return write_error(
630 StatusCode::UNSUPPORTED_MEDIA_TYPE,
631 "Remote Write requires Content-Encoding: snappy",
632 );
633 }
634 let content_type = match headers
635 .get(header::CONTENT_TYPE)
636 .and_then(|value| value.to_str().ok())
637 {
638 Some(value) => value,
639 None => {
640 return write_error(
641 StatusCode::UNSUPPORTED_MEDIA_TYPE,
642 "Remote Write requires a protobuf Content-Type",
643 );
644 }
645 };
646 let version = match RemoteWriteDecoder::version_from_content_type(content_type) {
647 Ok(version) => version,
648 Err(error) => return write_error(StatusCode::UNSUPPORTED_MEDIA_TYPE, &error.to_string()),
649 };
650 if let Err(response) = validate_version_header(&headers, version) {
651 return response;
652 }
653 let decompressed_len = match snap::raw::decompress_len(&body) {
654 Ok(length) if length <= service.config.max_request_bytes => length,
655 Ok(_) => {
656 return write_error(
657 StatusCode::PAYLOAD_TOO_LARGE,
658 "Remote Write Snappy payload expands beyond 64 MiB",
659 );
660 }
661 Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
662 };
663 let mut protobuf = vec![0_u8; decompressed_len];
664 if let Err(error) = snap::raw::Decoder::new().decompress(&body, &mut protobuf) {
665 return write_error(StatusCode::BAD_REQUEST, &error.to_string());
666 }
667 let decoded = match RemoteWriteDecoder.decode(&service.config.tenant, version, &protobuf) {
668 Ok(decoded) => decoded,
669 Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
670 };
671 let stats = decoded.stats;
672 let batch = match decoded.into_native_batch(&service.router) {
673 Ok(batch) => batch,
674 Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
675 };
676 if batch.partitions.is_empty() {
677 return write_success(stats);
678 }
679 let _ingest_permit = match &service.production {
680 Some(runtime) => match runtime.try_ingest(protobuf.len()) {
681 Some(permit) => Some(permit),
682 None if runtime.lifecycle().state() == ServiceState::Ready => {
683 return write_error(
684 StatusCode::TOO_MANY_REQUESTS,
685 "Remote Write concurrency or rate limit exceeded",
686 );
687 }
688 None => {
689 return write_error(
690 StatusCode::SERVICE_UNAVAILABLE,
691 "Remote Write ingestion is unavailable",
692 );
693 }
694 },
695 None => None,
696 };
697 let store = Arc::clone(&service.store);
698 let result = tokio::task::spawn_blocking(move || store.append_remote_write_batch(&batch)).await;
699 match result {
700 Ok(Ok(_)) => {
701 if let Some(runtime) = &service.production {
702 let records = stats.samples.saturating_add(stats.histograms);
703 runtime.record_ingest(protobuf.len(), records as usize);
704 }
705 write_success(stats)
706 }
707 Ok(Err(error)) => write_error(error.status(), &error.to_string()),
708 Err(error) => write_error(
709 StatusCode::INTERNAL_SERVER_ERROR,
710 &format!("Remote Write worker failed: {error}"),
711 ),
712 }
713}
714
715async fn remote_read(
716 State(service): State<PrometheusService>,
717 headers: HeaderMap,
718 body: Bytes,
719) -> Response {
720 let _permit = match service.authorize(&headers) {
721 Ok(permit) => permit,
722 Err(response) => return response,
723 };
724 if body.len() > service.config.max_request_bytes {
725 return write_error(
726 StatusCode::PAYLOAD_TOO_LARGE,
727 "Remote Read body is too large",
728 );
729 }
730 if !headers
731 .get(header::CONTENT_ENCODING)
732 .and_then(|value| value.to_str().ok())
733 .is_some_and(|value| value.eq_ignore_ascii_case("snappy"))
734 {
735 return write_error(
736 StatusCode::UNSUPPORTED_MEDIA_TYPE,
737 "Remote Read requires Content-Encoding: snappy",
738 );
739 }
740 let decompressed_len = match snap::raw::decompress_len(&body) {
741 Ok(length) if length <= service.config.max_request_bytes => length,
742 Ok(_) => {
743 return write_error(
744 StatusCode::PAYLOAD_TOO_LARGE,
745 "Remote Read Snappy payload expands beyond 64 MiB",
746 );
747 }
748 Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
749 };
750 let mut protobuf = vec![0_u8; decompressed_len];
751 if let Err(error) = snap::raw::Decoder::new().decompress(&body, &mut protobuf) {
752 return write_error(StatusCode::BAD_REQUEST, &error.to_string());
753 }
754 let request = match prometheus_v1::ReadRequest::decode(protobuf.as_slice()) {
755 Ok(request) => request,
756 Err(error) => return write_error(StatusCode::BAD_REQUEST, &error.to_string()),
757 };
758 let response_type = if request.accepted_response_types.is_empty() {
759 Some(prometheus_v1::ReadRequestResponseType::Samples)
760 } else {
761 request
762 .accepted_response_types
763 .iter()
764 .find_map(|value| prometheus_v1::ReadRequestResponseType::try_from(*value).ok())
765 };
766 let Some(response_type) = response_type else {
767 return write_error(
768 StatusCode::UNSUPPORTED_MEDIA_TYPE,
769 "Remote Read requested no supported response type",
770 );
771 };
772 if response_type == prometheus_v1::ReadRequestResponseType::StreamedXorChunks {
773 return remote_read_streamed(service, request).await;
774 }
775 let engine = PromqlEngine::new(
776 Arc::clone(&service.store),
777 Arc::clone(&service.config.tenant),
778 PromqlLimits::default(),
779 );
780 let result = tokio::task::spawn_blocking(move || {
781 let mut results = Vec::with_capacity(request.queries.len());
782 for query in request.queries {
783 let selector = remote_read_selector(&query.matchers)?;
784 let points = engine.raw_points(
785 &[selector],
786 query.start_timestamp_ms,
787 query.end_timestamp_ms,
788 )?;
789 let mut series = BTreeMap::<
790 BTreeMap<String, String>,
791 BTreeMap<i64, (shard_stream_core::LogicalOffset, RemoteReadValue)>,
792 >::new();
793 for point in points {
794 let timestamp =
795 i64::try_from(point.timestamp_unix_nanos / 1_000_000).unwrap_or(i64::MAX);
796 let Some(value) = remote_read_value(&point, timestamp) else {
797 continue;
798 };
799 let values = series.entry(metric_labels(&point)).or_default();
800 if values
801 .get(×tamp)
802 .is_none_or(|(offset, _)| *offset < point.record_ref.offset)
803 {
804 values.insert(timestamp, (point.record_ref.offset, value));
805 }
806 }
807 let timeseries = series
808 .into_iter()
809 .map(|(labels, values)| {
810 let mut samples = Vec::new();
811 let mut histograms = Vec::new();
812 for (_, (_, value)) in values {
813 match value {
814 RemoteReadValue::Sample(sample) => samples.push(sample),
815 RemoteReadValue::Histogram(histogram) => histograms.push(*histogram),
816 }
817 }
818 prometheus_v1::TimeSeries {
819 labels: labels
820 .into_iter()
821 .map(|(name, value)| prometheus_v1::Label { name, value })
822 .collect(),
823 samples,
824 exemplars: Vec::new(),
825 histograms,
826 }
827 })
828 .collect();
829 results.push(prometheus_v1::QueryResult { timeseries });
830 }
831 Ok::<_, crate::PromqlError>(prometheus_v1::ReadResponse { results })
832 })
833 .await;
834 let response = match result {
835 Ok(Ok(response)) => response,
836 Ok(Err(error)) => {
837 return query_error(
838 StatusCode::UNPROCESSABLE_ENTITY,
839 "execution",
840 &error.to_string(),
841 );
842 }
843 Err(error) => {
844 return query_error(
845 StatusCode::INTERNAL_SERVER_ERROR,
846 "internal",
847 &format!("Remote Read worker failed: {error}"),
848 );
849 }
850 };
851 let protobuf = response.encode_to_vec();
852 let compressed = match snap::raw::Encoder::new().compress_vec(&protobuf) {
853 Ok(compressed) => compressed,
854 Err(error) => return write_error(StatusCode::INTERNAL_SERVER_ERROR, &error.to_string()),
855 };
856 (
857 StatusCode::OK,
858 [
859 (
860 header::CONTENT_TYPE,
861 HeaderValue::from_static("application/x-protobuf"),
862 ),
863 (header::CONTENT_ENCODING, HeaderValue::from_static("snappy")),
864 ],
865 compressed,
866 )
867 .into_response()
868}
869
870async fn remote_read_streamed(
871 service: PrometheusService,
872 request: prometheus_v1::ReadRequest,
873) -> Response {
874 const XOR_SAMPLES_PER_CHUNK: usize = 120;
875 const MAX_FRAME_BYTES: usize = 1024 * 1024;
876
877 let engine = PromqlEngine::new(
878 Arc::clone(&service.store),
879 Arc::clone(&service.config.tenant),
880 PromqlLimits::default(),
881 );
882 let result = tokio::task::spawn_blocking(move || {
883 let mut stream = Vec::new();
884 for (query_index, query) in request.queries.into_iter().enumerate() {
885 let selector = remote_read_selector(&query.matchers)?;
886 let points = engine.raw_points(
887 &[selector],
888 query.start_timestamp_ms,
889 query.end_timestamp_ms,
890 )?;
891 let mut series = BTreeMap::<
892 BTreeMap<String, String>,
893 BTreeMap<i64, (shard_stream_core::LogicalOffset, f64)>,
894 >::new();
895 for point in points {
896 let Some(value) = remote_read_float(&point.value) else {
897 continue;
898 };
899 let timestamp =
900 i64::try_from(point.timestamp_unix_nanos / 1_000_000).unwrap_or(i64::MAX);
901 let samples = series.entry(metric_labels(&point)).or_default();
902 if samples
903 .get(×tamp)
904 .is_none_or(|(offset, _)| *offset < point.record_ref.offset)
905 {
906 samples.insert(timestamp, (point.record_ref.offset, value));
907 }
908 }
909 for (labels, samples) in series {
910 let labels = labels
911 .into_iter()
912 .map(|(name, value)| prometheus_v1::Label { name, value })
913 .collect::<Vec<_>>();
914 let samples = samples
915 .into_iter()
916 .map(|(timestamp, (_, value))| (timestamp, value))
917 .collect::<Vec<_>>();
918 let mut chunks = Vec::new();
919 for samples in samples.chunks(XOR_SAMPLES_PER_CHUNK) {
920 chunks.push(prometheus_v1::Chunk {
921 min_time_ms: samples.first().expect("chunk is nonempty").0,
922 max_time_ms: samples.last().expect("chunk is nonempty").0,
923 r#type: prometheus_v1::ChunkEncoding::Xor as i32,
924 data: encode_xor_chunk(samples)
925 .map_err(|error| crate::PromqlError::new(error.to_string()))?,
926 });
927 }
928 append_chunked_series_frames(
929 &mut stream,
930 i64::try_from(query_index).unwrap_or(i64::MAX),
931 &labels,
932 chunks,
933 MAX_FRAME_BYTES,
934 )?;
935 }
936 }
937 Ok::<_, crate::PromqlError>(stream)
938 })
939 .await;
940 match result {
941 Ok(Ok(stream)) => (
942 StatusCode::OK,
943 [(
944 header::CONTENT_TYPE,
945 HeaderValue::from_static(
946 "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse",
947 ),
948 )],
949 stream,
950 )
951 .into_response(),
952 Ok(Err(error)) => query_error(
953 StatusCode::UNPROCESSABLE_ENTITY,
954 "execution",
955 &error.to_string(),
956 ),
957 Err(error) => query_error(
958 StatusCode::INTERNAL_SERVER_ERROR,
959 "internal",
960 &format!("streamed Remote Read worker failed: {error}"),
961 ),
962 }
963}
964
965fn append_chunked_series_frames(
966 stream: &mut Vec<u8>,
967 query_index: i64,
968 labels: &[prometheus_v1::Label],
969 chunks: Vec<prometheus_v1::Chunk>,
970 max_frame_bytes: usize,
971) -> Result<(), crate::PromqlError> {
972 const CHUNKS_PER_FRAME: usize = 256;
973 for frame_chunks in chunks.chunks(CHUNKS_PER_FRAME) {
974 let protobuf = prometheus_v1::ChunkedReadResponse {
975 chunked_series: vec![prometheus_v1::ChunkedSeries {
976 labels: labels.to_vec(),
977 chunks: frame_chunks.to_vec(),
978 }],
979 query_index,
980 }
981 .encode_to_vec();
982 if protobuf.len() > max_frame_bytes {
983 return Err(crate::PromqlError::new(
984 "one streamed Remote Read series frame exceeds 1 MiB",
985 ));
986 }
987 append_stream_frame(stream, &protobuf)?;
988 }
989 Ok(())
990}
991
992fn append_stream_frame(stream: &mut Vec<u8>, protobuf: &[u8]) -> Result<(), crate::PromqlError> {
993 if protobuf.len() > 1024 * 1024 {
994 return Err(crate::PromqlError::new(
995 "one streamed Remote Read frame exceeds 1 MiB",
996 ));
997 }
998 append_uvarint(
999 stream,
1000 u64::try_from(protobuf.len())
1001 .map_err(|_| crate::PromqlError::new("Remote Read frame is too large"))?,
1002 );
1003 stream.extend_from_slice(&crc32c::crc32c(protobuf).to_be_bytes());
1004 stream.extend_from_slice(protobuf);
1005 Ok(())
1006}
1007
1008fn append_uvarint(output: &mut Vec<u8>, mut value: u64) {
1009 while value >= 0x80 {
1010 output.push((value as u8) | 0x80);
1011 value >>= 7;
1012 }
1013 output.push(value as u8);
1014}
1015
1016#[allow(clippy::result_large_err)]
1017fn validate_version_header(
1018 headers: &HeaderMap,
1019 version: RemoteWriteVersion,
1020) -> Result<(), Response> {
1021 let Some(observed) = headers
1022 .get("x-prometheus-remote-write-version")
1023 .and_then(|value| value.to_str().ok())
1024 else {
1025 return Err(write_error(
1026 StatusCode::BAD_REQUEST,
1027 "missing X-Prometheus-Remote-Write-Version",
1028 ));
1029 };
1030 let valid = match version {
1031 RemoteWriteVersion::V1 => observed == "0.1.0" || observed.starts_with("1."),
1032 RemoteWriteVersion::V2 => observed.starts_with("2."),
1033 };
1034 if valid {
1035 Ok(())
1036 } else {
1037 Err(write_error(
1038 StatusCode::BAD_REQUEST,
1039 "Remote Write version header conflicts with Content-Type",
1040 ))
1041 }
1042}
1043
1044fn write_success(stats: RemoteWriteStats) -> Response {
1045 let mut response = StatusCode::NO_CONTENT.into_response();
1046 for (name, value) in [
1047 ("x-prometheus-remote-write-samples-written", stats.samples),
1048 (
1049 "x-prometheus-remote-write-histograms-written",
1050 stats.histograms,
1051 ),
1052 (
1053 "x-prometheus-remote-write-exemplars-written",
1054 stats.exemplars,
1055 ),
1056 ] {
1057 response.headers_mut().insert(
1058 name,
1059 HeaderValue::from_str(&value.to_string()).expect("u64 is a valid header value"),
1060 );
1061 }
1062 response
1063}
1064
1065fn write_error(status: StatusCode, message: &str) -> Response {
1066 (
1067 status,
1068 [(header::CONTENT_TYPE, HeaderValue::from_static("text/plain"))],
1069 message.to_owned(),
1070 )
1071 .into_response()
1072}
1073
1074fn query_success(value: PromqlValue) -> Response {
1075 let (result_type, result) = match value {
1076 PromqlValue::Scalar {
1077 timestamp_ms,
1078 value,
1079 } => (
1080 "scalar",
1081 json!([timestamp_seconds(timestamp_ms), format_sample(value)]),
1082 ),
1083 PromqlValue::String {
1084 timestamp_ms,
1085 value,
1086 } => ("string", json!([timestamp_seconds(timestamp_ms), value])),
1087 PromqlValue::Vector(samples) => (
1088 "vector",
1089 Value::Array(
1090 samples
1091 .into_iter()
1092 .map(|sample| {
1093 json!({
1094 "metric": sample.labels,
1095 "value": [
1096 timestamp_seconds(sample.timestamp_ms),
1097 format_sample(sample.value)
1098 ]
1099 })
1100 })
1101 .collect(),
1102 ),
1103 ),
1104 PromqlValue::Matrix(series) => (
1105 "matrix",
1106 Value::Array(
1107 series
1108 .into_iter()
1109 .map(|series| {
1110 let values = series
1111 .samples
1112 .into_iter()
1113 .map(|(timestamp, value)| {
1114 json!([timestamp_seconds(timestamp), format_sample(value)])
1115 })
1116 .collect::<Vec<_>>();
1117 json!({"metric": series.labels, "values": values})
1118 })
1119 .collect(),
1120 ),
1121 ),
1122 };
1123 (
1124 StatusCode::OK,
1125 axum::Json(json!({
1126 "status": "success",
1127 "data": {"resultType": result_type, "result": result}
1128 })),
1129 )
1130 .into_response()
1131}
1132
1133fn api_success(data: Value) -> Response {
1134 (
1135 StatusCode::OK,
1136 axum::Json(json!({"status": "success", "data": data})),
1137 )
1138 .into_response()
1139}
1140
1141fn metric_labels(point: &crate::DurableMetricPoint) -> BTreeMap<String, String> {
1142 let mut labels = crate::prometheus_string_labels(&point.identity)
1143 .into_iter()
1144 .map(|(name, value)| (name.to_string(), value.to_string()))
1145 .collect::<BTreeMap<_, _>>();
1146 labels.insert("__name__".into(), point.identity.name.to_string());
1147 labels
1148}
1149
1150fn remote_read_selector(
1151 matchers: &[prometheus_v1::LabelMatcher],
1152) -> Result<String, crate::PromqlError> {
1153 if matchers.is_empty() {
1154 return Ok("{__name__=~\".+\"}".into());
1155 }
1156 let mut selector = String::from("{");
1157 for (index, matcher) in matchers.iter().enumerate() {
1158 if matcher.name.is_empty() {
1159 return Err(crate::PromqlError::new(
1160 "Remote Read matcher has an empty label name",
1161 ));
1162 }
1163 if index != 0 {
1164 selector.push(',');
1165 }
1166 selector.push_str(&matcher.name);
1167 selector.push_str(
1168 match prometheus_v1::LabelMatcherType::try_from(matcher.r#type) {
1169 Ok(prometheus_v1::LabelMatcherType::Equal) => "=",
1170 Ok(prometheus_v1::LabelMatcherType::NotEqual) => "!=",
1171 Ok(prometheus_v1::LabelMatcherType::RegexMatch) => "=~",
1172 Ok(prometheus_v1::LabelMatcherType::RegexNoMatch) => "!~",
1173 Err(_) => {
1174 return Err(crate::PromqlError::new(
1175 "Remote Read matcher has an unknown type",
1176 ));
1177 }
1178 },
1179 );
1180 selector.push_str(
1181 &serde_json::to_string(&matcher.value)
1182 .map_err(|error| crate::PromqlError::new(error.to_string()))?,
1183 );
1184 }
1185 selector.push('}');
1186 Ok(selector)
1187}
1188
1189fn remote_read_float(value: &MetricValue) -> Option<f64> {
1190 match value {
1191 MetricValue::Gauge(value) | MetricValue::Sum(value) => Some(match value {
1192 NumberValue::Integer(value) => *value as f64,
1193 NumberValue::DoubleBits(bits) => f64::from_bits(*bits),
1194 }),
1195 MetricValue::ExplicitHistogram(_)
1196 | MetricValue::ExponentialHistogram(_)
1197 | MetricValue::Summary(_) => None,
1198 }
1199}
1200
1201enum RemoteReadValue {
1202 Sample(prometheus_v1::Sample),
1203 Histogram(Box<prometheus_v1::Histogram>),
1204}
1205
1206fn remote_read_value(
1207 point: &crate::DurableMetricPoint,
1208 timestamp_ms: i64,
1209) -> Option<RemoteReadValue> {
1210 if let Some(value) = remote_read_float(&point.value) {
1211 return Some(RemoteReadValue::Sample(prometheus_v1::Sample {
1212 value,
1213 timestamp: timestamp_ms,
1214 }));
1215 }
1216 let start_timestamp = i64::try_from(point.start_time_unix_nanos / 1_000_000).ok()?;
1217 let histogram = match &point.value {
1218 MetricValue::ExplicitHistogram(value) => {
1219 let (positive_deltas, positive_counts) = histogram_count_lanes(&value.bucket_counts)?;
1220 let positive_spans = if value.bucket_counts.is_empty() {
1221 Vec::new()
1222 } else {
1223 vec![prometheus_v1::BucketSpan {
1224 offset: 0,
1225 length: u32::try_from(value.bucket_counts.len()).ok()?,
1226 }]
1227 };
1228 prometheus_v1::Histogram {
1229 count: Some(histogram_count(value.count)),
1230 sum: value.sum_bits.map_or(0.0, f64::from_bits),
1231 schema: -53,
1232 zero_threshold: 0.0,
1233 zero_count: Some(prometheus_v1::histogram::ZeroCount::Int(0)),
1234 negative_spans: Vec::new(),
1235 negative_deltas: Vec::new(),
1236 negative_counts: Vec::new(),
1237 positive_spans,
1238 positive_deltas,
1239 positive_counts,
1240 reset_hint: value.reset_hint,
1241 timestamp: timestamp_ms,
1242 custom_values: value
1243 .explicit_bounds_bits
1244 .iter()
1245 .copied()
1246 .map(f64::from_bits)
1247 .collect(),
1248 start_timestamp,
1249 }
1250 }
1251 MetricValue::ExponentialHistogram(value) => {
1252 let (negative_spans, negative_deltas, negative_counts) =
1253 histogram_buckets(value.negative.as_ref())?;
1254 let (positive_spans, positive_deltas, positive_counts) =
1255 histogram_buckets(value.positive.as_ref())?;
1256 prometheus_v1::Histogram {
1257 count: Some(histogram_count(value.count)),
1258 sum: value.sum_bits.map_or(0.0, f64::from_bits),
1259 schema: value.scale,
1260 zero_threshold: f64::from_bits(value.zero_threshold_bits),
1261 zero_count: Some(histogram_zero_count(value.zero_count)),
1262 negative_spans,
1263 negative_deltas,
1264 negative_counts,
1265 positive_spans,
1266 positive_deltas,
1267 positive_counts,
1268 reset_hint: value.reset_hint,
1269 timestamp: timestamp_ms,
1270 custom_values: Vec::new(),
1271 start_timestamp,
1272 }
1273 }
1274 MetricValue::Gauge(_) | MetricValue::Sum(_) | MetricValue::Summary(_) => return None,
1275 };
1276 Some(RemoteReadValue::Histogram(Box::new(histogram)))
1277}
1278
1279fn histogram_count(value: HistogramCount) -> prometheus_v1::histogram::Count {
1280 match value {
1281 HistogramCount::Integer(value) => prometheus_v1::histogram::Count::Int(value),
1282 HistogramCount::DoubleBits(bits) => {
1283 prometheus_v1::histogram::Count::Float(f64::from_bits(bits))
1284 }
1285 }
1286}
1287
1288fn histogram_zero_count(value: HistogramCount) -> prometheus_v1::histogram::ZeroCount {
1289 match value {
1290 HistogramCount::Integer(value) => prometheus_v1::histogram::ZeroCount::Int(value),
1291 HistogramCount::DoubleBits(bits) => {
1292 prometheus_v1::histogram::ZeroCount::Float(f64::from_bits(bits))
1293 }
1294 }
1295}
1296
1297fn histogram_buckets(
1298 buckets: Option<&ExponentialHistogramBuckets>,
1299) -> Option<(Vec<prometheus_v1::BucketSpan>, Vec<i64>, Vec<f64>)> {
1300 let Some(buckets) = buckets else {
1301 return Some((Vec::new(), Vec::new(), Vec::new()));
1302 };
1303 let spans = buckets
1304 .spans
1305 .iter()
1306 .map(|span| prometheus_v1::BucketSpan {
1307 offset: span.offset,
1308 length: span.length,
1309 })
1310 .collect();
1311 let (deltas, counts) = histogram_count_lanes(&buckets.bucket_counts)?;
1312 Some((spans, deltas, counts))
1313}
1314
1315fn histogram_count_lanes(counts: &[HistogramCount]) -> Option<(Vec<i64>, Vec<f64>)> {
1316 if counts
1317 .iter()
1318 .all(|count| matches!(count, HistogramCount::Integer(_)))
1319 {
1320 let mut previous = 0_i64;
1321 let mut deltas = Vec::with_capacity(counts.len());
1322 for count in counts {
1323 let HistogramCount::Integer(count) = count else {
1324 unreachable!("integer lane was checked")
1325 };
1326 let count = i64::try_from(*count).ok()?;
1327 deltas.push(count.checked_sub(previous)?);
1328 previous = count;
1329 }
1330 Some((deltas, Vec::new()))
1331 } else if counts
1332 .iter()
1333 .all(|count| matches!(count, HistogramCount::DoubleBits(_)))
1334 {
1335 Some((
1336 Vec::new(),
1337 counts
1338 .iter()
1339 .map(|count| match count {
1340 HistogramCount::DoubleBits(bits) => f64::from_bits(*bits),
1341 HistogramCount::Integer(_) => unreachable!("float lane was checked"),
1342 })
1343 .collect(),
1344 ))
1345 } else {
1346 None
1347 }
1348}
1349
1350fn prometheus_metric_type(kind: &MetricKind) -> &'static str {
1351 match kind {
1352 MetricKind::Gauge => "gauge",
1353 MetricKind::Sum {
1354 monotonic: true, ..
1355 } => "counter",
1356 MetricKind::Sum { .. } => "gauge",
1357 MetricKind::ExplicitHistogram { .. } | MetricKind::ExponentialHistogram { .. } => {
1358 "histogram"
1359 }
1360 MetricKind::Summary => "summary",
1361 }
1362}
1363
1364fn format_number(value: NumberValue) -> String {
1365 match value {
1366 NumberValue::Integer(value) => value.to_string(),
1367 NumberValue::DoubleBits(bits) => format_sample(f64::from_bits(bits)),
1368 }
1369}
1370
1371fn render_telemetry_value(value: &TelemetryValue) -> String {
1372 match value {
1373 TelemetryValue::Empty => String::new(),
1374 TelemetryValue::String(value) => value.to_string(),
1375 TelemetryValue::Boolean(value) => value.to_string(),
1376 TelemetryValue::Integer(value) => value.to_string(),
1377 TelemetryValue::DoubleBits(bits) => format_sample(f64::from_bits(*bits)),
1378 TelemetryValue::Bytes(value) => value.iter().map(|byte| format!("{byte:02x}")).collect(),
1379 TelemetryValue::Array(_) | TelemetryValue::Map(_) => {
1380 serde_json::to_string(value).unwrap_or_default()
1381 }
1382 TelemetryValue::StringTableIndex(value) => value.to_string(),
1383 }
1384}
1385
1386fn query_error(status: StatusCode, error_type: &str, message: &str) -> Response {
1387 (
1388 status,
1389 axum::Json(json!({
1390 "status": "error",
1391 "errorType": error_type,
1392 "error": message
1393 })),
1394 )
1395 .into_response()
1396}
1397
1398fn format_sample(value: f64) -> String {
1399 if value.is_nan() {
1400 "NaN".into()
1401 } else if value == f64::INFINITY {
1402 "+Inf".into()
1403 } else if value == f64::NEG_INFINITY {
1404 "-Inf".into()
1405 } else {
1406 value.to_string()
1407 }
1408}
1409
1410fn timestamp_seconds(timestamp_ms: i64) -> f64 {
1411 timestamp_ms as f64 / 1_000.0
1412}
1413
1414fn current_time_ms() -> i64 {
1415 std::time::SystemTime::now()
1416 .duration_since(std::time::UNIX_EPOCH)
1417 .ok()
1418 .and_then(|duration| i64::try_from(duration.as_millis()).ok())
1419 .unwrap_or(i64::MAX)
1420}
1421
1422fn parse_prometheus_time(value: &str) -> Result<i64, String> {
1423 let seconds = value
1424 .parse::<f64>()
1425 .map_err(|_| format!("invalid Prometheus timestamp {value:?}"))?;
1426 if !seconds.is_finite() {
1427 return Err(format!("invalid Prometheus timestamp {value:?}"));
1428 }
1429 let milliseconds = seconds * 1_000.0;
1430 if milliseconds < i64::MIN as f64 || milliseconds > i64::MAX as f64 {
1431 return Err("Prometheus timestamp is out of range".into());
1432 }
1433 Ok(milliseconds.round() as i64)
1434}
1435
1436fn parse_prometheus_duration(value: &str) -> Result<i64, String> {
1437 if let Ok(seconds) = value.parse::<f64>()
1438 && seconds.is_finite()
1439 && seconds > 0.0
1440 {
1441 return Ok((seconds * 1_000.0).round() as i64);
1442 }
1443 let (number, multiplier) = [
1444 ("ms", 1_i64),
1445 ("s", 1_000),
1446 ("m", 60_000),
1447 ("h", 3_600_000),
1448 ("d", 86_400_000),
1449 ("w", 604_800_000),
1450 ("y", 31_536_000_000),
1451 ]
1452 .into_iter()
1453 .find_map(|(suffix, multiplier)| {
1454 value
1455 .strip_suffix(suffix)
1456 .map(|number| (number, multiplier))
1457 })
1458 .ok_or_else(|| format!("invalid Prometheus duration {value:?}"))?;
1459 let number = number
1460 .parse::<f64>()
1461 .map_err(|_| format!("invalid Prometheus duration {value:?}"))?;
1462 let milliseconds = number * multiplier as f64;
1463 if !milliseconds.is_finite() || milliseconds <= 0.0 || milliseconds > i64::MAX as f64 {
1464 return Err(format!("invalid Prometheus duration {value:?}"));
1465 }
1466 Ok(milliseconds.round() as i64)
1467}
1468
1469#[cfg(test)]
1470mod tests {
1471 use std::fs;
1472 use std::sync::atomic::{AtomicU64, Ordering};
1473 use std::time::{Duration, SystemTime, UNIX_EPOCH};
1474
1475 use axum::body::{Body, to_bytes};
1476 use axum::http::Request;
1477 use tower::ServiceExt;
1478
1479 use super::*;
1480 use crate::{DurableTelemetryConfig, StripeConfig};
1481
1482 static TEST_DIRECTORY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
1483
1484 fn test_service() -> (PrometheusService, std::path::PathBuf) {
1485 let nonce = SystemTime::now()
1486 .duration_since(UNIX_EPOCH)
1487 .expect("clock")
1488 .as_nanos();
1489 let directory = std::env::temp_dir().join(format!(
1490 "shard-telemetry-prometheus-api-{}-{nonce}-{}",
1491 std::process::id(),
1492 TEST_DIRECTORY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
1493 ));
1494 let store = Arc::new(
1495 DurableTelemetryStore::open(DurableTelemetryConfig {
1496 data_directory: directory.clone(),
1497 object_store_directory: None,
1498 s3_object_store: None,
1499 recovery_journal: false,
1500 retention: None,
1501 shard_count: 2,
1502 tenant_partitions: 8,
1503 append_linger: Duration::from_micros(250),
1504 stripe: StripeConfig::default(),
1505 indexed_ack_timeout: Duration::from_secs(30),
1506 })
1507 .expect("store opens"),
1508 );
1509 let service = PrometheusService::new(
1510 store,
1511 PrometheusApiConfig {
1512 tenant: Arc::from("tenant-a"),
1513 logical_partitions: NonZeroU16::new(8).unwrap(),
1514 ..PrometheusApiConfig::default()
1515 },
1516 )
1517 .expect("service");
1518 (service, directory)
1519 }
1520
1521 #[test]
1522 fn version_header_must_match_negotiated_schema() {
1523 let mut headers = HeaderMap::new();
1524 headers.insert(
1525 "x-prometheus-remote-write-version",
1526 HeaderValue::from_static("2.0.0"),
1527 );
1528 assert!(validate_version_header(&headers, RemoteWriteVersion::V2).is_ok());
1529 assert!(validate_version_header(&headers, RemoteWriteVersion::V1).is_err());
1530 }
1531
1532 #[test]
1533 fn success_reports_all_required_written_headers() {
1534 let response = write_success(RemoteWriteStats {
1535 samples: 3,
1536 histograms: 2,
1537 exemplars: 1,
1538 });
1539 assert_eq!(response.status(), StatusCode::NO_CONTENT);
1540 assert_eq!(
1541 response.headers()["x-prometheus-remote-write-samples-written"],
1542 "3"
1543 );
1544 assert_eq!(
1545 response.headers()["x-prometheus-remote-write-histograms-written"],
1546 "2"
1547 );
1548 assert_eq!(
1549 response.headers()["x-prometheus-remote-write-exemplars-written"],
1550 "1"
1551 );
1552 }
1553
1554 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1555 async fn stable_prometheus_route_surface_has_no_missing_or_wrong_method_routes() {
1556 let (service, directory) = test_service();
1557 let app = prometheus_router(service.clone());
1558 let routes = [
1559 (axum::http::Method::POST, "/api/v1/write"),
1560 (axum::http::Method::POST, "/api/v1/read"),
1561 (axum::http::Method::GET, "/api/v1/query?query=1"),
1562 (axum::http::Method::POST, "/api/v1/query"),
1563 (
1564 axum::http::Method::GET,
1565 "/api/v1/query_range?query=1&start=0&end=1&step=1",
1566 ),
1567 (axum::http::Method::POST, "/api/v1/query_range"),
1568 (axum::http::Method::GET, "/api/v1/series?start=0&end=1"),
1569 (axum::http::Method::POST, "/api/v1/series"),
1570 (axum::http::Method::GET, "/api/v1/labels?start=0&end=1"),
1571 (axum::http::Method::POST, "/api/v1/labels"),
1572 (
1573 axum::http::Method::GET,
1574 "/api/v1/label/job/values?start=0&end=1",
1575 ),
1576 (axum::http::Method::POST, "/api/v1/label/job/values"),
1577 (axum::http::Method::GET, "/api/v1/metadata"),
1578 (
1579 axum::http::Method::GET,
1580 "/api/v1/query_exemplars?query=%7B__name__%3D%22x%22%7D&start=0&end=1",
1581 ),
1582 (axum::http::Method::POST, "/api/v1/query_exemplars"),
1583 ];
1584 for (method, path) in routes {
1585 let response = app
1586 .clone()
1587 .oneshot(
1588 Request::builder()
1589 .method(method.clone())
1590 .uri(path)
1591 .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
1592 .body(Body::empty())
1593 .expect("request"),
1594 )
1595 .await
1596 .expect("Prometheus response");
1597 assert_ne!(response.status(), StatusCode::NOT_FOUND, "{method} {path}");
1598 assert_ne!(
1599 response.status(),
1600 StatusCode::METHOD_NOT_ALLOWED,
1601 "{method} {path}"
1602 );
1603 }
1604
1605 drop(app);
1606 drop(service);
1607 fs::remove_dir_all(directory).expect("cleanup");
1608 }
1609
1610 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1611 async fn query_discovery_and_remote_read_routes_use_prometheus_envelopes() {
1612 let (service, directory) = test_service();
1613 let app = prometheus_router(service.clone());
1614 let response = app
1615 .clone()
1616 .oneshot(
1617 Request::builder()
1618 .uri("/api/v1/query?query=1%2B2&time=10")
1619 .body(Body::empty())
1620 .unwrap(),
1621 )
1622 .await
1623 .expect("query response");
1624 assert_eq!(response.status(), StatusCode::OK);
1625 let body = to_bytes(response.into_body(), 1_024 * 1_024)
1626 .await
1627 .expect("query body");
1628 let body: Value = serde_json::from_slice(&body).expect("query JSON");
1629 assert_eq!(body["status"], "success");
1630 assert_eq!(body["data"]["resultType"], "scalar");
1631 assert_eq!(body["data"]["result"][1], "3");
1632
1633 let response = app
1634 .clone()
1635 .oneshot(
1636 Request::builder()
1637 .uri("/api/v1/series?start=0&end=10")
1638 .body(Body::empty())
1639 .unwrap(),
1640 )
1641 .await
1642 .expect("series response");
1643 assert_eq!(response.status(), StatusCode::OK);
1644
1645 let request = prometheus_v1::ReadRequest {
1646 queries: Vec::new(),
1647 accepted_response_types: vec![prometheus_v1::ReadRequestResponseType::Samples as i32],
1648 };
1649 let compressed = snap::raw::Encoder::new()
1650 .compress_vec(&request.encode_to_vec())
1651 .expect("Snappy request");
1652 let response = app
1653 .oneshot(
1654 Request::builder()
1655 .method("POST")
1656 .uri("/api/v1/read")
1657 .header(header::CONTENT_ENCODING, "snappy")
1658 .body(Body::from(compressed))
1659 .unwrap(),
1660 )
1661 .await
1662 .expect("read response");
1663 assert_eq!(response.status(), StatusCode::OK);
1664 let body = to_bytes(response.into_body(), 1_024 * 1_024)
1665 .await
1666 .expect("read body");
1667 let protobuf = snap::raw::Decoder::new()
1668 .decompress_vec(&body)
1669 .expect("Snappy response");
1670 assert!(
1671 prometheus_v1::ReadResponse::decode(protobuf.as_slice())
1672 .expect("read response protobuf")
1673 .results
1674 .is_empty()
1675 );
1676
1677 drop(service);
1678 fs::remove_dir_all(directory).expect("remove test store");
1679 }
1680
1681 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1682 async fn streamed_remote_read_returns_crc_framed_prometheus_xor_chunks() {
1683 let (service, directory) = test_service();
1684 let app = prometheus_router(service.clone());
1685 let write = prometheus_v1::WriteRequest {
1686 timeseries: vec![prometheus_v1::TimeSeries {
1687 labels: vec![
1688 prometheus_v1::Label {
1689 name: "job".into(),
1690 value: "api".into(),
1691 },
1692 prometheus_v1::Label {
1693 name: "__name__".into(),
1694 value: "requests_total".into(),
1695 },
1696 ],
1697 samples: vec![
1698 prometheus_v1::Sample {
1699 value: 1.0,
1700 timestamp: 1_000,
1701 },
1702 prometheus_v1::Sample {
1703 value: 2.0,
1704 timestamp: 2_000,
1705 },
1706 ],
1707 exemplars: Vec::new(),
1708 histograms: Vec::new(),
1709 }],
1710 metadata: Vec::new(),
1711 };
1712 let compressed = snap::raw::Encoder::new()
1713 .compress_vec(&write.encode_to_vec())
1714 .expect("Snappy write request");
1715 let response = app
1716 .clone()
1717 .oneshot(
1718 Request::builder()
1719 .method("POST")
1720 .uri("/api/v1/write")
1721 .header(header::CONTENT_TYPE, "application/x-protobuf")
1722 .header(header::CONTENT_ENCODING, "snappy")
1723 .header("x-prometheus-remote-write-version", "1.0.0")
1724 .body(Body::from(compressed))
1725 .unwrap(),
1726 )
1727 .await
1728 .expect("write response");
1729 assert_eq!(response.status(), StatusCode::NO_CONTENT);
1730
1731 let request = prometheus_v1::ReadRequest {
1732 queries: vec![prometheus_v1::Query {
1733 start_timestamp_ms: 0,
1734 end_timestamp_ms: 3_000,
1735 matchers: vec![prometheus_v1::LabelMatcher {
1736 r#type: prometheus_v1::LabelMatcherType::Equal as i32,
1737 name: "__name__".into(),
1738 value: "requests_total".into(),
1739 }],
1740 hints: None,
1741 }],
1742 accepted_response_types: vec![
1743 prometheus_v1::ReadRequestResponseType::StreamedXorChunks as i32,
1744 prometheus_v1::ReadRequestResponseType::Samples as i32,
1745 ],
1746 };
1747 let compressed = snap::raw::Encoder::new()
1748 .compress_vec(&request.encode_to_vec())
1749 .expect("Snappy read request");
1750 let response = app
1751 .oneshot(
1752 Request::builder()
1753 .method("POST")
1754 .uri("/api/v1/read")
1755 .header(header::CONTENT_ENCODING, "snappy")
1756 .body(Body::from(compressed))
1757 .unwrap(),
1758 )
1759 .await
1760 .expect("streamed read response");
1761 assert_eq!(response.status(), StatusCode::OK);
1762 assert_eq!(
1763 response.headers()[header::CONTENT_TYPE],
1764 "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse"
1765 );
1766 assert!(!response.headers().contains_key(header::CONTENT_ENCODING));
1767 let body = to_bytes(response.into_body(), 2 * 1024 * 1024)
1768 .await
1769 .expect("stream body");
1770 let (frame_bytes, delimiter_bytes) = decode_test_uvarint(&body);
1771 let checksum_start = delimiter_bytes;
1772 let protobuf_start = checksum_start + 4;
1773 let protobuf_end = protobuf_start + usize::try_from(frame_bytes).unwrap();
1774 assert_eq!(protobuf_end, body.len());
1775 assert_eq!(
1776 u32::from_be_bytes(body[checksum_start..protobuf_start].try_into().unwrap()),
1777 crc32c::crc32c(&body[protobuf_start..protobuf_end])
1778 );
1779 let frame = prometheus_v1::ChunkedReadResponse::decode(&body[protobuf_start..protobuf_end])
1780 .expect("chunked response protobuf");
1781 assert_eq!(frame.query_index, 0);
1782 assert_eq!(frame.chunked_series.len(), 1);
1783 assert_eq!(
1784 frame.chunked_series[0]
1785 .labels
1786 .iter()
1787 .map(|label| label.name.as_str())
1788 .collect::<Vec<_>>(),
1789 vec!["__name__", "job"]
1790 );
1791 assert_eq!(frame.chunked_series[0].chunks.len(), 1);
1792 let chunk = &frame.chunked_series[0].chunks[0];
1793 assert_eq!(chunk.min_time_ms, 1_000);
1794 assert_eq!(chunk.max_time_ms, 2_000);
1795 assert_eq!(chunk.r#type, prometheus_v1::ChunkEncoding::Xor as i32);
1796 assert_eq!(u16::from_be_bytes(chunk.data[..2].try_into().unwrap()), 2);
1797
1798 drop(service);
1799 fs::remove_dir_all(directory).expect("remove test store");
1800 }
1801
1802 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1803 async fn sample_remote_read_round_trips_native_histograms() {
1804 let (service, directory) = test_service();
1805 let app = prometheus_router(service.clone());
1806 let write = prometheus_v1::WriteRequest {
1807 timeseries: vec![prometheus_v1::TimeSeries {
1808 labels: vec![prometheus_v1::Label {
1809 name: "__name__".into(),
1810 value: "request_duration".into(),
1811 }],
1812 samples: Vec::new(),
1813 exemplars: Vec::new(),
1814 histograms: vec![prometheus_v1::Histogram {
1815 count: Some(prometheus_v1::histogram::Count::Int(3)),
1816 sum: 6.0,
1817 schema: 0,
1818 zero_threshold: 0.001,
1819 zero_count: Some(prometheus_v1::histogram::ZeroCount::Int(0)),
1820 negative_spans: Vec::new(),
1821 negative_deltas: Vec::new(),
1822 negative_counts: Vec::new(),
1823 positive_spans: vec![prometheus_v1::BucketSpan {
1824 offset: 0,
1825 length: 2,
1826 }],
1827 positive_deltas: vec![1, 1],
1828 positive_counts: Vec::new(),
1829 reset_hint: prometheus_v1::histogram::ResetHint::No as i32,
1830 timestamp: 2_000,
1831 custom_values: Vec::new(),
1832 start_timestamp: 1_000,
1833 }],
1834 }],
1835 metadata: Vec::new(),
1836 };
1837 let compressed = snap::raw::Encoder::new()
1838 .compress_vec(&write.encode_to_vec())
1839 .unwrap();
1840 let response = app
1841 .clone()
1842 .oneshot(
1843 Request::builder()
1844 .method("POST")
1845 .uri("/api/v1/write")
1846 .header(header::CONTENT_TYPE, "application/x-protobuf")
1847 .header(header::CONTENT_ENCODING, "snappy")
1848 .header("x-prometheus-remote-write-version", "1.0.0")
1849 .body(Body::from(compressed))
1850 .unwrap(),
1851 )
1852 .await
1853 .unwrap();
1854 assert_eq!(response.status(), StatusCode::NO_CONTENT);
1855
1856 let read = prometheus_v1::ReadRequest {
1857 queries: vec![prometheus_v1::Query {
1858 start_timestamp_ms: 0,
1859 end_timestamp_ms: 3_000,
1860 matchers: vec![prometheus_v1::LabelMatcher {
1861 r#type: prometheus_v1::LabelMatcherType::Equal as i32,
1862 name: "__name__".into(),
1863 value: "request_duration".into(),
1864 }],
1865 hints: None,
1866 }],
1867 accepted_response_types: vec![prometheus_v1::ReadRequestResponseType::Samples as i32],
1868 };
1869 let compressed = snap::raw::Encoder::new()
1870 .compress_vec(&read.encode_to_vec())
1871 .unwrap();
1872 let response = app
1873 .oneshot(
1874 Request::builder()
1875 .method("POST")
1876 .uri("/api/v1/read")
1877 .header(header::CONTENT_ENCODING, "snappy")
1878 .body(Body::from(compressed))
1879 .unwrap(),
1880 )
1881 .await
1882 .unwrap();
1883 assert_eq!(response.status(), StatusCode::OK);
1884 let body = to_bytes(response.into_body(), 1_024 * 1_024).await.unwrap();
1885 let protobuf = snap::raw::Decoder::new().decompress_vec(&body).unwrap();
1886 let response = prometheus_v1::ReadResponse::decode(protobuf.as_slice()).unwrap();
1887 let histogram = &response.results[0].timeseries[0].histograms[0];
1888 assert_eq!(histogram.schema, 0);
1889 assert_eq!(histogram.positive_deltas, vec![1, 1]);
1890 assert_eq!(histogram.sum.to_bits(), 6.0_f64.to_bits());
1891 assert_eq!(histogram.start_timestamp, 1_000);
1892
1893 drop(service);
1894 fs::remove_dir_all(directory).expect("remove test store");
1895 }
1896
1897 fn decode_test_uvarint(bytes: &[u8]) -> (u64, usize) {
1898 let mut value = 0_u64;
1899 for (index, byte) in bytes.iter().copied().enumerate().take(10) {
1900 value |= u64::from(byte & 0x7f) << (index * 7);
1901 if byte & 0x80 == 0 {
1902 return (value, index + 1);
1903 }
1904 }
1905 panic!("invalid test frame delimiter")
1906 }
1907}