1use std::collections::{BTreeMap, HashMap, HashSet};
2use std::sync::Arc;
3
4use prost::Message;
5use shard_stream_core::{LogicalOffset, LogicalPartitionId, ShardId, TopicPartition};
6
7use crate::prometheus_protocol::{v1, v2};
8use crate::{
9 DurableMetricPoint, ExplicitHistogramValue, ExponentialHistogramBuckets,
10 ExponentialHistogramValue, HistogramBucketSpan, HistogramCount, METRICS_TOPIC_ID,
11 MetricExemplar, MetricIdentity, MetricKind, MetricValue, NativePartitionAppend,
12 NativeTelemetryBatch, NumberValue, OtlpMetricEvent, ResourceContext, ScopeContext, SpanId,
13 TelemetryAttribute, TelemetryError, TelemetryRecordRef, TelemetryResult, TelemetryRouter,
14 TelemetrySignal, TelemetryValue, TraceId, prepare_metric_envelope_with_protocol,
15};
16
17pub const PROMETHEUS_STALE_NAN_BITS: u64 = 0x7ff0_0000_0000_0002;
19pub const METRIC_FLAG_STALE: u32 = 1;
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum RemoteWriteVersion {
25 V1,
27 V2,
29}
30
31#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
33pub struct RemoteWriteStats {
34 pub samples: u64,
36 pub histograms: u64,
38 pub exemplars: u64,
40}
41
42#[derive(Debug)]
44pub struct DecodedRemoteWrite {
45 pub version: RemoteWriteVersion,
47 pub events: Vec<OtlpMetricEvent>,
49 pub stats: RemoteWriteStats,
51}
52
53impl DecodedRemoteWrite {
54 pub fn into_native_batch(
56 self,
57 router: &TelemetryRouter,
58 ) -> TelemetryResult<NativeTelemetryBatch> {
59 let mut partitions = BTreeMap::<TopicPartition, Vec<OtlpMetricEvent>>::new();
60 for event in self.events {
61 let partition = router.metric(event.tenant(), event.series_fingerprint());
62 partitions.entry(partition).or_default().push(event);
63 }
64 let mut appends = Vec::with_capacity(partitions.len());
65 for (topic_partition, events) in partitions {
66 appends.push(NativePartitionAppend {
67 topic_partition,
68 envelope: prepare_metric_envelope_with_protocol(
69 topic_partition,
70 events,
71 crate::MetricIngestProtocol::RemoteWrite,
72 )?,
73 transient_context: None,
74 });
75 }
76 Ok(NativeTelemetryBatch {
77 partitions: appends,
78 })
79 }
80}
81
82#[derive(Debug, Default, Clone, Copy)]
84pub struct RemoteWriteDecoder;
85
86impl RemoteWriteDecoder {
87 pub fn version_from_content_type(content_type: &str) -> TelemetryResult<RemoteWriteVersion> {
89 let mut parts = content_type.split(';');
90 let media_type = parts.next().unwrap_or_default().trim();
91 if !media_type.eq_ignore_ascii_case("application/x-protobuf") {
92 return Err(TelemetryError::InvalidMetricSample(
93 "Remote Write Content-Type must be application/x-protobuf".into(),
94 ));
95 }
96 let mut proto = None;
97 for parameter in parts {
98 let Some((name, value)) = parameter.trim().split_once('=') else {
99 return Err(TelemetryError::InvalidMetricSample(
100 "malformed Remote Write Content-Type parameter".into(),
101 ));
102 };
103 if name.trim().eq_ignore_ascii_case("proto") {
104 proto = Some(value.trim().trim_matches('"'));
105 }
106 }
107 match proto {
108 None | Some("prometheus.WriteRequest") => Ok(RemoteWriteVersion::V1),
109 Some("io.prometheus.write.v2.Request") => Ok(RemoteWriteVersion::V2),
110 Some(_) => Err(TelemetryError::InvalidMetricSample(
111 "unsupported Remote Write protobuf message".into(),
112 )),
113 }
114 }
115
116 pub fn decode(
118 &self,
119 tenant: &str,
120 version: RemoteWriteVersion,
121 protobuf: &[u8],
122 ) -> TelemetryResult<DecodedRemoteWrite> {
123 if tenant.is_empty() {
124 return Err(TelemetryError::InvalidMetricSample(
125 "Remote Write tenant must not be empty".into(),
126 ));
127 }
128 match version {
129 RemoteWriteVersion::V1 => self.decode_v1(tenant, protobuf),
130 RemoteWriteVersion::V2 => self.decode_v2(tenant, protobuf),
131 }
132 }
133
134 fn decode_v1(&self, tenant: &str, protobuf: &[u8]) -> TelemetryResult<DecodedRemoteWrite> {
135 let request = v1::WriteRequest::decode(protobuf)
136 .map_err(|error| TelemetryError::InvalidMetricSample(error.to_string()))?;
137 let metadata = request
138 .metadata
139 .into_iter()
140 .map(|metadata| (metadata.metric_family_name.clone(), metadata))
141 .collect::<HashMap<_, _>>();
142 let mut events = Vec::new();
143 let mut stats = RemoteWriteStats::default();
144 for series in request.timeseries {
145 let labels = validate_v1_labels(series.labels)?;
146 let name = metric_name(&labels)?.to_owned();
147 let metadata = metadata_for_name(&metadata, &name);
148 let metric_type = metadata.map_or(v1::MetricType::Unknown as i32, |value| value.r#type);
149 let unit = metadata.map_or_else(String::new, |value| value.unit.clone());
150 let description = metadata.map_or_else(String::new, |value| value.help.clone());
151 let exemplars = series
152 .exemplars
153 .into_iter()
154 .map(decode_v1_exemplar)
155 .collect::<TelemetryResult<Vec<_>>>()?;
156 stats.exemplars = stats.exemplars.saturating_add(exemplars.len() as u64);
157 let mut raw_points = Vec::with_capacity(series.samples.len() + series.histograms.len());
158 for sample in series.samples {
159 raw_points.push(RawPoint {
160 timestamp_unix_nanos: millis_to_nanos(sample.timestamp)?,
161 start_time_unix_nanos: 0,
162 value: scalar_value(metric_type, sample.value.to_bits()),
163 });
164 stats.samples = stats.samples.saturating_add(1);
165 }
166 for histogram in series.histograms {
167 raw_points.push(decode_v1_histogram(histogram)?);
168 stats.histograms = stats.histograms.saturating_add(1);
169 }
170 append_series_events(
171 tenant,
172 &name,
173 &unit,
174 &description,
175 metric_type,
176 labels,
177 raw_points,
178 exemplars,
179 &mut events,
180 )?;
181 }
182 Ok(DecodedRemoteWrite {
183 version: RemoteWriteVersion::V1,
184 events,
185 stats,
186 })
187 }
188
189 fn decode_v2(&self, tenant: &str, protobuf: &[u8]) -> TelemetryResult<DecodedRemoteWrite> {
190 let request = v2::Request::decode(protobuf)
191 .map_err(|error| TelemetryError::InvalidMetricSample(error.to_string()))?;
192 validate_symbols(&request.symbols)?;
193 let mut events = Vec::new();
194 let mut stats = RemoteWriteStats::default();
195 for series in request.timeseries {
196 let labels = decode_symbol_labels(&request.symbols, &series.labels_refs, true)?;
197 let name = metric_name(&labels)?.to_owned();
198 let metric_type = series
199 .metadata
200 .as_ref()
201 .map_or(v2::MetricType::Unspecified as i32, |metadata| {
202 metadata.r#type
203 });
204 let unit = symbol(
205 &request.symbols,
206 series
207 .metadata
208 .as_ref()
209 .map_or(0, |metadata| metadata.unit_ref),
210 )?
211 .to_owned();
212 let description = symbol(
213 &request.symbols,
214 series
215 .metadata
216 .as_ref()
217 .map_or(0, |metadata| metadata.help_ref),
218 )?
219 .to_owned();
220 let exemplars = series
221 .exemplars
222 .into_iter()
223 .map(|exemplar| decode_v2_exemplar(&request.symbols, exemplar))
224 .collect::<TelemetryResult<Vec<_>>>()?;
225 stats.exemplars = stats.exemplars.saturating_add(exemplars.len() as u64);
226 let mut raw_points = Vec::with_capacity(series.samples.len() + series.histograms.len());
227 for sample in series.samples {
228 raw_points.push(RawPoint {
229 timestamp_unix_nanos: millis_to_nanos(sample.timestamp)?,
230 start_time_unix_nanos: optional_millis_to_nanos(sample.start_timestamp)?,
231 value: scalar_value(metric_type, sample.value.to_bits()),
232 });
233 stats.samples = stats.samples.saturating_add(1);
234 }
235 for histogram in series.histograms {
236 raw_points.push(decode_v2_histogram(histogram)?);
237 stats.histograms = stats.histograms.saturating_add(1);
238 }
239 append_series_events(
240 tenant,
241 &name,
242 &unit,
243 &description,
244 metric_type,
245 labels,
246 raw_points,
247 exemplars,
248 &mut events,
249 )?;
250 }
251 Ok(DecodedRemoteWrite {
252 version: RemoteWriteVersion::V2,
253 events,
254 stats,
255 })
256 }
257}
258
259#[derive(Debug)]
260struct RawPoint {
261 timestamp_unix_nanos: u64,
262 start_time_unix_nanos: u64,
263 value: MetricValue,
264}
265
266#[allow(clippy::too_many_arguments)]
267fn append_series_events(
268 tenant: &str,
269 name: &str,
270 unit: &str,
271 description: &str,
272 metric_type: i32,
273 labels: Vec<(String, String)>,
274 mut raw_points: Vec<RawPoint>,
275 exemplars: Vec<MetricExemplar>,
276 events: &mut Vec<OtlpMetricEvent>,
277) -> TelemetryResult<()> {
278 if raw_points.is_empty() {
279 if exemplars.is_empty() {
280 return Ok(());
281 }
282 return Err(TelemetryError::InvalidMetricSample(
283 "Remote Write exemplar-only series cannot be attached to a metric point".into(),
284 ));
285 }
286 raw_points.sort_unstable_by_key(|point| point.timestamp_unix_nanos);
287 let mut assigned = vec![Vec::new(); raw_points.len()];
288 for exemplar in exemplars {
289 let index = raw_points
290 .binary_search_by_key(&exemplar.timestamp_unix_nanos, |point| {
291 point.timestamp_unix_nanos
292 })
293 .unwrap_or_else(|index| index.min(raw_points.len() - 1));
294 assigned[index].push(exemplar);
295 }
296 let point_attributes = Arc::new(
297 labels
298 .into_iter()
299 .filter(|(key, _)| key != "__name__")
300 .map(|(key, value)| TelemetryAttribute::new(key, TelemetryValue::String(value.into())))
301 .collect(),
302 );
303 let kind = metric_kind(metric_type, &raw_points);
304 let identity = Arc::new(MetricIdentity {
305 tenant: Arc::from(tenant),
306 resource: Arc::new(ResourceContext::default()),
307 scope: Arc::new(ScopeContext::default()),
308 name: Arc::from(name),
309 unit: Arc::from(unit),
310 kind,
311 point_attributes,
312 });
313 for (ordinal, (raw, exemplars)) in raw_points.into_iter().zip(assigned).enumerate() {
314 let flags = match raw.value {
315 MetricValue::Gauge(NumberValue::DoubleBits(PROMETHEUS_STALE_NAN_BITS))
316 | MetricValue::Sum(NumberValue::DoubleBits(PROMETHEUS_STALE_NAN_BITS)) => {
317 METRIC_FLAG_STALE
318 }
319 _ => 0,
320 };
321 let point = DurableMetricPoint {
322 stream_shard_id: ShardId::new(0),
323 record_ref: TelemetryRecordRef::for_signal(
324 TelemetrySignal::Metrics,
325 TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(0)),
326 LogicalOffset::new(ordinal as u64),
327 ),
328 identity: Arc::clone(&identity),
329 description: Arc::from(description),
330 metadata: Arc::new(Vec::new()),
331 start_time_unix_nanos: raw.start_time_unix_nanos,
332 timestamp_unix_nanos: raw.timestamp_unix_nanos,
333 flags,
334 value: raw.value,
335 exemplars: Arc::new(exemplars),
336 };
337 events.push(OtlpMetricEvent::from_durable(point)?);
338 }
339 Ok(())
340}
341
342fn metric_kind(metric_type: i32, points: &[RawPoint]) -> MetricKind {
343 if points
344 .iter()
345 .any(|point| matches!(point.value, MetricValue::ExplicitHistogram(_)))
346 {
347 return MetricKind::ExplicitHistogram { temporality: 2 };
348 }
349 if points
350 .iter()
351 .any(|point| matches!(point.value, MetricValue::ExponentialHistogram(_)))
352 {
353 return MetricKind::ExponentialHistogram { temporality: 2 };
354 }
355 if metric_type == v1::MetricType::Counter as i32 {
356 MetricKind::Sum {
357 temporality: 2,
358 monotonic: true,
359 }
360 } else {
361 MetricKind::Gauge
362 }
363}
364
365fn scalar_value(metric_type: i32, bits: u64) -> MetricValue {
366 if metric_type == v1::MetricType::Counter as i32 {
367 MetricValue::Sum(NumberValue::DoubleBits(bits))
368 } else {
369 MetricValue::Gauge(NumberValue::DoubleBits(bits))
370 }
371}
372
373fn validate_v1_labels(labels: Vec<v1::Label>) -> TelemetryResult<Vec<(String, String)>> {
374 let mut seen = HashSet::with_capacity(labels.len());
375 let mut decoded = Vec::with_capacity(labels.len());
376 for label in labels {
377 if label.name.is_empty() || !seen.insert(label.name.clone()) {
378 return Err(TelemetryError::InvalidMetricSample(
379 "Remote Write v1 labels require unique non-empty names".into(),
380 ));
381 }
382 decoded.push((label.name, label.value));
383 }
384 decoded.sort_unstable();
385 Ok(decoded)
386}
387
388fn validate_symbols(symbols: &[String]) -> TelemetryResult<()> {
389 if symbols.first().is_none_or(|symbol| !symbol.is_empty()) {
390 return Err(TelemetryError::InvalidMetricSample(
391 "Remote Write v2 symbol zero must be the empty string".into(),
392 ));
393 }
394 let mut seen = HashSet::with_capacity(symbols.len());
395 if symbols.iter().any(|symbol| !seen.insert(symbol)) {
396 return Err(TelemetryError::InvalidMetricSample(
397 "Remote Write v2 symbols must be deduplicated".into(),
398 ));
399 }
400 Ok(())
401}
402
403fn decode_symbol_labels(
404 symbols: &[String],
405 refs: &[u32],
406 require_sorted: bool,
407) -> TelemetryResult<Vec<(String, String)>> {
408 if !refs.len().is_multiple_of(2) {
409 return Err(TelemetryError::InvalidMetricSample(
410 "Remote Write v2 label references must be name/value pairs".into(),
411 ));
412 }
413 let mut labels = Vec::with_capacity(refs.len() / 2);
414 let mut seen = HashSet::with_capacity(refs.len() / 2);
415 for pair in refs.chunks_exact(2) {
416 let name = symbol(symbols, pair[0])?;
417 let value = symbol(symbols, pair[1])?;
418 if name.is_empty() || value.is_empty() || !seen.insert(name) {
419 return Err(TelemetryError::InvalidMetricSample(
420 "Remote Write v2 labels require unique non-empty names and values".into(),
421 ));
422 }
423 labels.push((name.to_owned(), value.to_owned()));
424 }
425 if require_sorted && labels.windows(2).any(|pair| pair[0].0 >= pair[1].0) {
426 return Err(TelemetryError::InvalidMetricSample(
427 "Remote Write v2 labels must be lexicographically sorted".into(),
428 ));
429 }
430 Ok(labels)
431}
432
433fn symbol(symbols: &[String], reference: u32) -> TelemetryResult<&str> {
434 symbols
435 .get(reference as usize)
436 .map(String::as_str)
437 .ok_or_else(|| {
438 TelemetryError::InvalidMetricSample(
439 "Remote Write v2 symbol reference is out of bounds".into(),
440 )
441 })
442}
443
444fn metric_name(labels: &[(String, String)]) -> TelemetryResult<&str> {
445 labels
446 .iter()
447 .find_map(|(name, value)| (name == "__name__").then_some(value.as_str()))
448 .filter(|name| !name.is_empty())
449 .ok_or_else(|| {
450 TelemetryError::InvalidMetricSample(
451 "Remote Write series requires a non-empty __name__ label".into(),
452 )
453 })
454}
455
456fn metadata_for_name<'a>(
457 metadata: &'a HashMap<String, v1::MetricMetadata>,
458 name: &str,
459) -> Option<&'a v1::MetricMetadata> {
460 metadata.get(name).or_else(|| {
461 name.strip_suffix("_total")
462 .and_then(|family| metadata.get(family))
463 })
464}
465
466fn decode_v1_exemplar(exemplar: v1::Exemplar) -> TelemetryResult<MetricExemplar> {
467 let labels = validate_v1_labels(exemplar.labels)?;
468 decode_exemplar(labels, exemplar.value.to_bits(), exemplar.timestamp)
469}
470
471fn decode_v2_exemplar(
472 symbols: &[String],
473 exemplar: v2::Exemplar,
474) -> TelemetryResult<MetricExemplar> {
475 let labels = decode_symbol_labels(symbols, &exemplar.labels_refs, true)?;
476 decode_exemplar(labels, exemplar.value.to_bits(), exemplar.timestamp)
477}
478
479fn decode_exemplar(
480 labels: Vec<(String, String)>,
481 value_bits: u64,
482 timestamp_ms: i64,
483) -> TelemetryResult<MetricExemplar> {
484 let trace_id = labels
485 .iter()
486 .find_map(|(key, value)| (key == "trace_id").then_some(value))
487 .map(|value| decode_hex::<16>(value).and_then(TraceId::from_bytes))
488 .transpose()?;
489 let span_id = labels
490 .iter()
491 .find_map(|(key, value)| (key == "span_id").then_some(value))
492 .map(|value| decode_hex::<8>(value).and_then(SpanId::from_bytes))
493 .transpose()?;
494 Ok(MetricExemplar {
495 filtered_attributes: Arc::new(
496 labels
497 .into_iter()
498 .map(|(key, value)| {
499 TelemetryAttribute::new(key, TelemetryValue::String(value.into()))
500 })
501 .collect(),
502 ),
503 timestamp_unix_nanos: millis_to_nanos(timestamp_ms)?,
504 value: NumberValue::DoubleBits(value_bits),
505 span_id,
506 trace_id,
507 })
508}
509
510fn decode_hex<const N: usize>(value: &str) -> TelemetryResult<[u8; N]> {
511 if value.len() != N * 2 {
512 return Err(TelemetryError::InvalidMetricSample(
513 "Prometheus exemplar trace/span ID has invalid length".into(),
514 ));
515 }
516 let mut decoded = [0_u8; N];
517 for (index, byte) in decoded.iter_mut().enumerate() {
518 let pair = &value.as_bytes()[index * 2..index * 2 + 2];
519 *byte = (hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?;
520 }
521 Ok(decoded)
522}
523
524fn hex_nibble(value: u8) -> TelemetryResult<u8> {
525 match value {
526 b'0'..=b'9' => Ok(value - b'0'),
527 b'a'..=b'f' => Ok(value - b'a' + 10),
528 b'A'..=b'F' => Ok(value - b'A' + 10),
529 _ => Err(TelemetryError::InvalidMetricSample(
530 "Prometheus exemplar trace/span ID is not hexadecimal".into(),
531 )),
532 }
533}
534
535fn decode_v1_histogram(histogram: v1::Histogram) -> TelemetryResult<RawPoint> {
536 let count = histogram.count.ok_or_else(|| {
537 TelemetryError::InvalidMetricSample("Prometheus histogram is missing count".into())
538 })?;
539 let zero_count = histogram.zero_count.ok_or_else(|| {
540 TelemetryError::InvalidMetricSample("Prometheus histogram is missing zero count".into())
541 })?;
542 decode_histogram(
543 histogram.schema,
544 count_v1(count),
545 histogram.sum.to_bits(),
546 histogram.zero_threshold.to_bits(),
547 zero_count_v1(zero_count),
548 spans_v1(histogram.negative_spans),
549 histogram.negative_deltas,
550 histogram.negative_counts,
551 spans_v1(histogram.positive_spans),
552 histogram.positive_deltas,
553 histogram.positive_counts,
554 histogram.reset_hint,
555 histogram.timestamp,
556 histogram.start_timestamp,
557 histogram.custom_values,
558 )
559}
560
561fn decode_v2_histogram(histogram: v2::Histogram) -> TelemetryResult<RawPoint> {
562 let count = histogram.count.ok_or_else(|| {
563 TelemetryError::InvalidMetricSample("Prometheus histogram is missing count".into())
564 })?;
565 let zero_count = histogram.zero_count.ok_or_else(|| {
566 TelemetryError::InvalidMetricSample("Prometheus histogram is missing zero count".into())
567 })?;
568 decode_histogram(
569 histogram.schema,
570 count_v2(count),
571 histogram.sum.to_bits(),
572 histogram.zero_threshold.to_bits(),
573 zero_count_v2(zero_count),
574 spans_v1(histogram.negative_spans),
575 histogram.negative_deltas,
576 histogram.negative_counts,
577 spans_v1(histogram.positive_spans),
578 histogram.positive_deltas,
579 histogram.positive_counts,
580 histogram.reset_hint,
581 histogram.timestamp,
582 histogram.start_timestamp,
583 histogram.custom_values,
584 )
585}
586
587#[allow(clippy::too_many_arguments)]
588fn decode_histogram(
589 schema: i32,
590 count: HistogramCount,
591 sum_bits: u64,
592 zero_threshold_bits: u64,
593 zero_count: HistogramCount,
594 negative_spans: Vec<HistogramBucketSpan>,
595 negative_deltas: Vec<i64>,
596 negative_counts: Vec<f64>,
597 positive_spans: Vec<HistogramBucketSpan>,
598 positive_deltas: Vec<i64>,
599 positive_counts: Vec<f64>,
600 reset_hint: i32,
601 timestamp_ms: i64,
602 start_timestamp_ms: i64,
603 custom_values: Vec<f64>,
604) -> TelemetryResult<RawPoint> {
605 let negative = decode_buckets(negative_spans, negative_deltas, negative_counts)?;
606 let positive = decode_buckets(positive_spans, positive_deltas, positive_counts)?;
607 let value = if schema == -53 {
608 if negative.is_some() || zero_count != HistogramCount::Integer(0) {
609 return Err(TelemetryError::InvalidMetricSample(
610 "custom Prometheus histogram cannot use negative or zero buckets".into(),
611 ));
612 }
613 let bounds = custom_values.iter().map(|value| value.to_bits()).collect();
614 let bucket_counts = expand_custom_buckets(positive, custom_values.len() + 1)?;
615 MetricValue::ExplicitHistogram(ExplicitHistogramValue {
616 count,
617 sum_bits: Some(sum_bits),
618 bucket_counts: Arc::new(bucket_counts),
619 explicit_bounds_bits: Arc::new(bounds),
620 min_bits: None,
621 max_bits: None,
622 reset_hint,
623 })
624 } else {
625 if !(-4..=8).contains(&schema) {
626 return Err(TelemetryError::InvalidMetricSample(format!(
627 "unsupported Prometheus native histogram schema {schema}"
628 )));
629 }
630 MetricValue::ExponentialHistogram(ExponentialHistogramValue {
631 count,
632 sum_bits: Some(sum_bits),
633 scale: schema,
634 zero_count,
635 positive,
636 negative,
637 min_bits: None,
638 max_bits: None,
639 zero_threshold_bits,
640 reset_hint,
641 })
642 };
643 Ok(RawPoint {
644 timestamp_unix_nanos: millis_to_nanos(timestamp_ms)?,
645 start_time_unix_nanos: optional_millis_to_nanos(start_timestamp_ms)?,
646 value,
647 })
648}
649
650fn decode_buckets(
651 spans: Vec<HistogramBucketSpan>,
652 deltas: Vec<i64>,
653 counts: Vec<f64>,
654) -> TelemetryResult<Option<ExponentialHistogramBuckets>> {
655 let expected = spans.iter().try_fold(0_usize, |total, span| {
656 total.checked_add(span.length as usize).ok_or_else(|| {
657 TelemetryError::InvalidMetricSample("histogram span length overflow".into())
658 })
659 })?;
660 if expected == 0 {
661 if !deltas.is_empty() || !counts.is_empty() {
662 return Err(TelemetryError::InvalidMetricSample(
663 "histogram bucket values have no spans".into(),
664 ));
665 }
666 return Ok(None);
667 }
668 if !deltas.is_empty() && !counts.is_empty() {
669 return Err(TelemetryError::InvalidMetricSample(
670 "histogram cannot mix integer deltas and float counts".into(),
671 ));
672 }
673 let bucket_counts = if !deltas.is_empty() {
674 if deltas.len() != expected {
675 return Err(TelemetryError::InvalidMetricSample(
676 "histogram integer bucket count does not match spans".into(),
677 ));
678 }
679 let mut previous = 0_i128;
680 let mut decoded = Vec::with_capacity(expected);
681 for delta in deltas {
682 previous = previous.checked_add(i128::from(delta)).ok_or_else(|| {
683 TelemetryError::InvalidMetricSample("histogram bucket delta overflow".into())
684 })?;
685 let count = u64::try_from(previous).map_err(|_| {
686 TelemetryError::InvalidMetricSample("histogram bucket count is negative".into())
687 })?;
688 decoded.push(HistogramCount::Integer(count));
689 }
690 decoded
691 } else {
692 if counts.len() != expected {
693 return Err(TelemetryError::InvalidMetricSample(
694 "histogram float bucket count does not match spans".into(),
695 ));
696 }
697 counts
698 .into_iter()
699 .map(|value| HistogramCount::DoubleBits(value.to_bits()))
700 .collect()
701 };
702 Ok(Some(ExponentialHistogramBuckets {
703 spans: Arc::new(spans),
704 bucket_counts: Arc::new(bucket_counts),
705 }))
706}
707
708fn expand_custom_buckets(
709 buckets: Option<ExponentialHistogramBuckets>,
710 output_len: usize,
711) -> TelemetryResult<Vec<HistogramCount>> {
712 let Some(buckets) = buckets else {
713 return Ok(vec![HistogramCount::Integer(0); output_len]);
714 };
715 let fill = if buckets
716 .bucket_counts
717 .iter()
718 .any(|count| matches!(count, HistogramCount::DoubleBits(_)))
719 {
720 HistogramCount::DoubleBits(0.0_f64.to_bits())
721 } else {
722 HistogramCount::Integer(0)
723 };
724 let mut output = vec![fill; output_len];
725 let mut bucket_index = 0_usize;
726 let mut prior_end = 0_i64;
727 for (span_index, span) in buckets.spans.iter().enumerate() {
728 let start = if span_index == 0 {
729 i64::from(span.offset)
730 } else {
731 prior_end.saturating_add(i64::from(span.offset))
732 };
733 if start < 0 {
734 return Err(TelemetryError::InvalidMetricSample(
735 "custom histogram span starts before bucket zero".into(),
736 ));
737 }
738 let end = start.checked_add(i64::from(span.length)).ok_or_else(|| {
739 TelemetryError::InvalidMetricSample("custom histogram span overflow".into())
740 })?;
741 if end as usize > output.len() {
742 return Err(TelemetryError::InvalidMetricSample(
743 "custom histogram span exceeds custom bounds".into(),
744 ));
745 }
746 for destination in &mut output[start as usize..end as usize] {
747 *destination = buckets.bucket_counts[bucket_index];
748 bucket_index += 1;
749 }
750 prior_end = end;
751 }
752 Ok(output)
753}
754
755fn spans_v1(spans: Vec<v1::BucketSpan>) -> Vec<HistogramBucketSpan> {
756 spans
757 .into_iter()
758 .map(|span| HistogramBucketSpan {
759 offset: span.offset,
760 length: span.length,
761 })
762 .collect()
763}
764
765fn count_v1(count: v1::histogram::Count) -> HistogramCount {
766 match count {
767 v1::histogram::Count::Int(value) => HistogramCount::Integer(value),
768 v1::histogram::Count::Float(value) => HistogramCount::DoubleBits(value.to_bits()),
769 }
770}
771
772fn zero_count_v1(count: v1::histogram::ZeroCount) -> HistogramCount {
773 match count {
774 v1::histogram::ZeroCount::Int(value) => HistogramCount::Integer(value),
775 v1::histogram::ZeroCount::Float(value) => HistogramCount::DoubleBits(value.to_bits()),
776 }
777}
778
779fn count_v2(count: v2::histogram::Count) -> HistogramCount {
780 match count {
781 v2::histogram::Count::Int(value) => HistogramCount::Integer(value),
782 v2::histogram::Count::Float(value) => HistogramCount::DoubleBits(value.to_bits()),
783 }
784}
785
786fn zero_count_v2(count: v2::histogram::ZeroCount) -> HistogramCount {
787 match count {
788 v2::histogram::ZeroCount::Int(value) => HistogramCount::Integer(value),
789 v2::histogram::ZeroCount::Float(value) => HistogramCount::DoubleBits(value.to_bits()),
790 }
791}
792
793fn millis_to_nanos(timestamp_ms: i64) -> TelemetryResult<u64> {
794 let timestamp_ms = u64::try_from(timestamp_ms).map_err(|_| {
795 TelemetryError::InvalidMetricSample(
796 "negative Prometheus timestamps are outside the storage epoch".into(),
797 )
798 })?;
799 timestamp_ms.checked_mul(1_000_000).ok_or_else(|| {
800 TelemetryError::InvalidMetricSample("Prometheus timestamp overflows nanoseconds".into())
801 })
802}
803
804fn optional_millis_to_nanos(timestamp_ms: i64) -> TelemetryResult<u64> {
805 if timestamp_ms == 0 {
806 Ok(0)
807 } else {
808 millis_to_nanos(timestamp_ms)
809 }
810}
811
812#[cfg(test)]
813mod tests {
814 use super::*;
815
816 #[test]
817 fn content_negotiation_is_deterministic() {
818 assert_eq!(
819 RemoteWriteDecoder::version_from_content_type("application/x-protobuf").unwrap(),
820 RemoteWriteVersion::V1
821 );
822 assert_eq!(
823 RemoteWriteDecoder::version_from_content_type(
824 "application/x-protobuf;proto=io.prometheus.write.v2.Request"
825 )
826 .unwrap(),
827 RemoteWriteVersion::V2
828 );
829 assert!(
830 RemoteWriteDecoder::version_from_content_type(
831 "application/x-protobuf;proto=unknown.Request"
832 )
833 .is_err()
834 );
835 }
836
837 #[test]
838 fn v1_preserves_stale_nan_and_exemplar_ids() {
839 let request = v1::WriteRequest {
840 timeseries: vec![v1::TimeSeries {
841 labels: vec![
842 v1::Label {
843 name: "__name__".into(),
844 value: "requests_total".into(),
845 },
846 v1::Label {
847 name: "service".into(),
848 value: "api".into(),
849 },
850 ],
851 samples: vec![v1::Sample {
852 value: f64::from_bits(PROMETHEUS_STALE_NAN_BITS),
853 timestamp: 10,
854 }],
855 exemplars: vec![v1::Exemplar {
856 labels: vec![
857 v1::Label {
858 name: "trace_id".into(),
859 value: "01010101010101010101010101010101".into(),
860 },
861 v1::Label {
862 name: "span_id".into(),
863 value: "0202020202020202".into(),
864 },
865 ],
866 value: 3.0,
867 timestamp: 10,
868 }],
869 histograms: Vec::new(),
870 }],
871 metadata: vec![v1::MetricMetadata {
872 r#type: v1::MetricType::Counter as i32,
873 metric_family_name: "requests".into(),
874 help: "requests".into(),
875 unit: String::new(),
876 }],
877 };
878 let decoded = RemoteWriteDecoder
879 .decode("tenant-a", RemoteWriteVersion::V1, &request.encode_to_vec())
880 .unwrap();
881 assert_eq!(decoded.stats.samples, 1);
882 assert_eq!(decoded.stats.exemplars, 1);
883 let point = decoded.events[0].clone().into_durable(
884 ShardId::new(0),
885 TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(1)),
886 LogicalOffset::new(1),
887 );
888 assert_eq!(point.flags & METRIC_FLAG_STALE, METRIC_FLAG_STALE);
889 assert_eq!(
890 point.exemplars[0].trace_id,
891 Some(TraceId::from_bytes([1; 16]).unwrap())
892 );
893 assert_eq!(
894 point.exemplars[0].span_id,
895 Some(SpanId::from_bytes([2; 8]).unwrap())
896 );
897 }
898
899 #[test]
900 fn v2_float_histogram_counts_round_trip_without_integer_coercion() {
901 let request = v2::Request {
902 symbols: vec!["".into(), "__name__".into(), "latency".into()],
903 timeseries: vec![v2::TimeSeries {
904 labels_refs: vec![1, 2],
905 samples: Vec::new(),
906 histograms: vec![v2::Histogram {
907 count: Some(v2::histogram::Count::Float(3.5)),
908 sum: 9.0,
909 schema: 1,
910 zero_threshold: 0.001,
911 zero_count: Some(v2::histogram::ZeroCount::Float(0.5)),
912 negative_spans: Vec::new(),
913 negative_deltas: Vec::new(),
914 negative_counts: Vec::new(),
915 positive_spans: vec![v1::BucketSpan {
916 offset: 0,
917 length: 2,
918 }],
919 positive_deltas: Vec::new(),
920 positive_counts: vec![1.25, 1.75],
921 reset_hint: v2::histogram::ResetHint::Gauge as i32,
922 timestamp: 100,
923 custom_values: Vec::new(),
924 start_timestamp: 50,
925 }],
926 exemplars: Vec::new(),
927 metadata: Some(v2::Metadata {
928 r#type: v2::MetricType::GaugeHistogram as i32,
929 help_ref: 0,
930 unit_ref: 0,
931 }),
932 }],
933 };
934 let decoded = RemoteWriteDecoder
935 .decode("tenant-a", RemoteWriteVersion::V2, &request.encode_to_vec())
936 .unwrap();
937 let point = decoded.events[0].clone().into_durable(
938 ShardId::new(0),
939 TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(1)),
940 LogicalOffset::new(1),
941 );
942 let MetricValue::ExponentialHistogram(histogram) = point.value else {
943 panic!("expected native histogram");
944 };
945 assert_eq!(
946 histogram.count,
947 HistogramCount::DoubleBits(3.5f64.to_bits())
948 );
949 assert_eq!(
950 histogram.positive.unwrap().bucket_counts.as_slice(),
951 &[
952 HistogramCount::DoubleBits(1.25f64.to_bits()),
953 HistogramCount::DoubleBits(1.75f64.to_bits())
954 ]
955 );
956 }
957}