1use std::sync::Arc;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::time::{Duration, SystemTime, UNIX_EPOCH};
11
12use ::fast_telemetry::{
13 DistributionSnapshot, HistogramSnapshot, MetricLabels, MetricMeta, MetricVisitor, Runtime,
14};
15use shard_stream_core::{LogicalOffset, LogicalPartitionId, ShardId, TopicPartition};
16
17use crate::{
18 DurableMetricPoint, EmbeddedTelemetryRuntime, ExplicitHistogramValue,
19 ExponentialHistogramBuckets, ExponentialHistogramValue, HistogramBucketSpan, HistogramCount,
20 LokiApiError, METRICS_TOPIC_ID, MetricIdentity, MetricKind, MetricValue,
21 NativeTelemetryAppendAck, NumberValue, ResourceContext, ScopeContext, TelemetryAttribute,
22 TelemetryRecordRef, TelemetrySignal, TelemetryValue,
23};
24
25const CUMULATIVE_TEMPORALITY: i32 = 2;
26const DEFAULT_EXPORT_INTERVAL: Duration = Duration::from_secs(5);
27const DEFAULT_MAX_POINTS_PER_EXPORT: usize = 65_536;
28
29#[derive(Debug, Clone, PartialEq, Eq)]
31pub struct FastTelemetryConfig {
32 pub tenant: Arc<str>,
34 pub resource_attributes: Arc<Vec<TelemetryAttribute>>,
36 pub scope_version: Arc<str>,
38 pub interval: Duration,
40 pub max_points_per_export: usize,
42 pub skip_empty_cumulative: bool,
44 pub wait_for_index: bool,
47}
48
49impl FastTelemetryConfig {
50 #[must_use]
53 pub fn new(tenant: impl Into<Arc<str>>, service_name: impl Into<Arc<str>>) -> Self {
54 Self {
55 tenant: tenant.into(),
56 resource_attributes: Arc::new(vec![TelemetryAttribute::new(
57 "service.name",
58 TelemetryValue::String(service_name.into()),
59 )]),
60 scope_version: Arc::from("fast-telemetry"),
61 interval: DEFAULT_EXPORT_INTERVAL,
62 max_points_per_export: DEFAULT_MAX_POINTS_PER_EXPORT,
63 skip_empty_cumulative: true,
64 wait_for_index: true,
65 }
66 }
67
68 #[must_use]
70 pub fn with_resource_attribute(mut self, attribute: TelemetryAttribute) -> Self {
71 Arc::make_mut(&mut self.resource_attributes).push(attribute);
72 self
73 }
74
75 #[must_use]
77 pub const fn with_interval(mut self, interval: Duration) -> Self {
78 self.interval = interval;
79 self
80 }
81
82 #[must_use]
84 pub const fn with_max_points_per_export(mut self, max_points: usize) -> Self {
85 self.max_points_per_export = max_points;
86 self
87 }
88
89 #[must_use]
91 pub const fn with_skip_empty_cumulative(mut self, skip: bool) -> Self {
92 self.skip_empty_cumulative = skip;
93 self
94 }
95
96 #[must_use]
98 pub const fn with_wait_for_index(mut self, wait_for_index: bool) -> Self {
99 self.wait_for_index = wait_for_index;
100 self
101 }
102
103 fn validate(&self) -> Result<(), LokiApiError> {
104 if self.tenant.is_empty() {
105 return Err(LokiApiError::configuration(
106 "fast-telemetry tenant must not be empty",
107 ));
108 }
109 if self.interval.is_zero() {
110 return Err(LokiApiError::configuration(
111 "fast-telemetry export interval must be nonzero",
112 ));
113 }
114 if self.max_points_per_export == 0 {
115 return Err(LokiApiError::configuration(
116 "fast-telemetry export point limit must be nonzero",
117 ));
118 }
119 Ok(())
120 }
121}
122
123#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
125pub struct FastTelemetryExportReport {
126 pub observed_points: usize,
128 pub dropped_points: usize,
130 pub appended_points: usize,
132 pub append_batches: usize,
134 pub skipped_busy: bool,
136}
137
138pub struct FastTelemetryExporter {
141 runtime: Arc<Runtime>,
142 embedded: Arc<EmbeddedTelemetryRuntime>,
143 config: FastTelemetryConfig,
144 exporting: AtomicBool,
145}
146
147impl FastTelemetryExporter {
148 pub fn new(
154 runtime: Arc<Runtime>,
155 embedded: Arc<EmbeddedTelemetryRuntime>,
156 config: FastTelemetryConfig,
157 ) -> Result<Self, LokiApiError> {
158 config.validate()?;
159 Ok(Self {
160 runtime,
161 embedded,
162 config,
163 exporting: AtomicBool::new(false),
164 })
165 }
166
167 #[must_use]
169 pub const fn interval(&self) -> Duration {
170 self.config.interval
171 }
172
173 pub fn export_once(&self) -> Result<FastTelemetryExportReport, LokiApiError> {
175 self.export_once_at(unix_nanos_now())
176 }
177
178 pub fn export_once_at(
180 &self,
181 timestamp_unix_nanos: u64,
182 ) -> Result<FastTelemetryExportReport, LokiApiError> {
183 if self
184 .exporting
185 .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
186 .is_err()
187 {
188 return Ok(FastTelemetryExportReport {
189 skipped_busy: true,
190 ..FastTelemetryExportReport::default()
191 });
192 }
193 let _guard = ExportGuard {
194 exporting: &self.exporting,
195 };
196 let mut export = FastMetricExport::new(&self.embedded, &self.config, timestamp_unix_nanos);
197
198 let mut scopes = self.runtime.scopes();
202 scopes.sort_unstable();
203 scopes.dedup();
204 for scope in scopes {
205 export.set_scope(scope.name());
206 self.runtime.visit_metrics_for_scope(&scope, &mut export);
207 }
208
209 let observed_points = export.observed_points;
210 let dropped_points = export.dropped_points;
211 let (appended_points, acknowledgement) = export.finish()?;
212 Ok(FastTelemetryExportReport {
213 observed_points,
214 dropped_points,
215 appended_points,
216 append_batches: acknowledgement.partitions.len(),
217 skipped_busy: false,
218 })
219 }
220}
221
222struct ExportGuard<'a> {
223 exporting: &'a AtomicBool,
224}
225
226impl Drop for ExportGuard<'_> {
227 fn drop(&mut self) {
228 self.exporting.store(false, Ordering::Release);
229 }
230}
231
232struct FastMetricExport<'a> {
233 embedded: &'a EmbeddedTelemetryRuntime,
234 tenant: Arc<str>,
235 resource: Arc<ResourceContext>,
236 scope_version: Arc<str>,
237 scope: Arc<ScopeContext>,
238 timestamp_unix_nanos: u64,
239 max_points_per_export: usize,
240 skip_empty_cumulative: bool,
241 wait_for_index: bool,
242 points: Vec<DurableMetricPoint>,
243 tail: Option<DurableMetricPoint>,
244 observed_points: usize,
245 dropped_points: usize,
246}
247
248impl<'a> FastMetricExport<'a> {
249 fn new(
250 embedded: &'a EmbeddedTelemetryRuntime,
251 config: &FastTelemetryConfig,
252 timestamp_unix_nanos: u64,
253 ) -> Self {
254 Self {
255 embedded,
256 tenant: Arc::clone(&config.tenant),
257 resource: Arc::new(ResourceContext {
258 attributes: Arc::clone(&config.resource_attributes),
259 ..ResourceContext::default()
260 }),
261 scope_version: Arc::clone(&config.scope_version),
262 scope: Arc::new(ScopeContext::default()),
263 timestamp_unix_nanos,
264 max_points_per_export: config.max_points_per_export,
265 skip_empty_cumulative: config.skip_empty_cumulative,
266 wait_for_index: config.wait_for_index,
267 points: Vec::new(),
268 tail: None,
269 observed_points: 0,
270 dropped_points: 0,
271 }
272 }
273
274 fn set_scope(&mut self, scope_name: &str) {
275 self.scope = Arc::new(ScopeContext {
276 name: Arc::from(scope_name),
277 version: Arc::clone(&self.scope_version),
278 ..ScopeContext::default()
279 });
280 }
281
282 fn finish(mut self) -> Result<(usize, NativeTelemetryAppendAck), LokiApiError> {
283 let Some(tail) = self.tail.take() else {
284 return Ok((
285 0,
286 NativeTelemetryAppendAck {
287 partitions: Vec::new(),
288 },
289 ));
290 };
291 let point_count = self.points.len().saturating_add(1);
292 let acknowledgement = if self.points.is_empty() {
293 self.embedded
294 .append_metric_point(tail, self.wait_for_index)?
295 } else {
296 self.points.push(tail);
297 self.embedded
298 .append_metric_points(self.points, self.wait_for_index)?
299 };
300 Ok((point_count, acknowledgement))
301 }
302
303 fn push(
304 &mut self,
305 meta: MetricMeta<'_>,
306 labels: MetricLabels<'_>,
307 kind: MetricKind,
308 value: MetricValue,
309 ) {
310 self.observed_points = self.observed_points.saturating_add(1);
311 if self
312 .points
313 .len()
314 .saturating_add(usize::from(self.tail.is_some()))
315 >= self.max_points_per_export
316 {
317 self.dropped_points = self.dropped_points.saturating_add(1);
318 return;
319 }
320 let point_attributes = labels
321 .iter()
322 .map(|label| {
323 TelemetryAttribute::new(
324 Arc::<str>::from(label.name),
325 TelemetryValue::String(Arc::from(label.value)),
326 )
327 })
328 .collect();
329 let point = DurableMetricPoint {
330 stream_shard_id: ShardId::new(0),
331 record_ref: TelemetryRecordRef::for_signal(
332 TelemetrySignal::Metrics,
333 TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(0)),
334 LogicalOffset::new(0),
335 ),
336 identity: Arc::new(MetricIdentity {
337 tenant: Arc::clone(&self.tenant),
338 resource: Arc::clone(&self.resource),
339 scope: Arc::clone(&self.scope),
340 name: Arc::from(meta.name),
341 unit: Arc::from(meta.unit.unwrap_or("")),
342 kind,
343 point_attributes: Arc::new(point_attributes),
344 }),
345 description: Arc::from(meta.help),
346 metadata: Arc::new(Vec::new()),
347 start_time_unix_nanos: 0,
348 timestamp_unix_nanos: self.timestamp_unix_nanos,
349 flags: 0,
350 value,
351 exemplars: Arc::new(Vec::new()),
352 };
353 if let Some(previous) = self.tail.replace(point) {
354 self.points.push(previous);
355 }
356 }
357}
358
359impl MetricVisitor for FastMetricExport<'_> {
360 fn counter(&mut self, meta: MetricMeta<'_>, labels: MetricLabels<'_>, value: i64) {
361 if self.skip_empty_cumulative && value == 0 {
362 return;
363 }
364 self.push(
365 meta,
366 labels,
367 MetricKind::Sum {
368 temporality: CUMULATIVE_TEMPORALITY,
369 monotonic: true,
370 },
371 MetricValue::Sum(NumberValue::Integer(value)),
372 );
373 }
374
375 fn gauge_i64(&mut self, meta: MetricMeta<'_>, labels: MetricLabels<'_>, value: i64) {
376 self.push(
377 meta,
378 labels,
379 MetricKind::Gauge,
380 MetricValue::Gauge(NumberValue::Integer(value)),
381 );
382 }
383
384 fn gauge_f64(&mut self, meta: MetricMeta<'_>, labels: MetricLabels<'_>, value: f64) {
385 self.push(
386 meta,
387 labels,
388 MetricKind::Gauge,
389 MetricValue::Gauge(NumberValue::from_f64(value)),
390 );
391 }
392
393 fn histogram(
394 &mut self,
395 meta: MetricMeta<'_>,
396 labels: MetricLabels<'_>,
397 histogram: &dyn HistogramSnapshot,
398 ) {
399 if self.skip_empty_cumulative && histogram.count() == 0 {
400 return;
401 }
402 let mut explicit_bounds_bits = Vec::new();
403 let mut bucket_counts = Vec::new();
404 let mut previous = 0_u64;
405 histogram.visit_buckets(&mut |bound, cumulative_count| {
406 explicit_bounds_bits.push((bound as f64).to_bits());
407 bucket_counts.push(HistogramCount::Integer(
408 cumulative_count.saturating_sub(previous),
409 ));
410 previous = cumulative_count;
411 });
412 bucket_counts.push(HistogramCount::Integer(
413 histogram.count().saturating_sub(previous),
414 ));
415 self.push(
416 meta,
417 labels,
418 MetricKind::ExplicitHistogram {
419 temporality: CUMULATIVE_TEMPORALITY,
420 },
421 MetricValue::ExplicitHistogram(ExplicitHistogramValue {
422 count: HistogramCount::Integer(histogram.count()),
423 sum_bits: Some((histogram.sum() as f64).to_bits()),
424 bucket_counts: Arc::new(bucket_counts),
425 explicit_bounds_bits: Arc::new(explicit_bounds_bits),
426 min_bits: None,
427 max_bits: None,
428 reset_hint: 0,
429 }),
430 );
431 }
432
433 fn distribution(
434 &mut self,
435 meta: MetricMeta<'_>,
436 labels: MetricLabels<'_>,
437 distribution: &dyn DistributionSnapshot,
438 ) {
439 if self.skip_empty_cumulative && distribution.count() == 0 {
440 return;
441 }
442 let mut sparse = Vec::new();
443 distribution.visit_positive_buckets(&mut |index, count| sparse.push((index, count)));
444 sparse.sort_unstable_by_key(|(index, _)| *index);
445 let positive = (!sparse.is_empty()).then(|| {
446 let mut spans = Vec::<HistogramBucketSpan>::new();
447 let mut counts = Vec::with_capacity(sparse.len());
448 let mut prior_end = 0_i64;
449 for (index, count) in sparse {
450 let index = i64::from(index);
451 if let Some(span) = spans.last_mut()
452 && index == prior_end
453 {
454 span.length = span.length.saturating_add(1);
455 } else {
456 let offset = if spans.is_empty() {
457 index
458 } else {
459 index.saturating_sub(prior_end)
460 };
461 spans.push(HistogramBucketSpan {
462 offset: i32::try_from(offset).unwrap_or(i32::MAX),
463 length: 1,
464 });
465 }
466 counts.push(HistogramCount::Integer(count));
467 prior_end = index.saturating_add(1);
468 }
469 ExponentialHistogramBuckets {
470 spans: Arc::new(spans),
471 bucket_counts: Arc::new(counts),
472 }
473 });
474 self.push(
475 meta,
476 labels,
477 MetricKind::ExponentialHistogram {
478 temporality: CUMULATIVE_TEMPORALITY,
479 },
480 MetricValue::ExponentialHistogram(ExponentialHistogramValue {
481 count: HistogramCount::Integer(distribution.count()),
482 sum_bits: Some((distribution.sum() as f64).to_bits()),
483 scale: 0,
484 zero_count: HistogramCount::Integer(distribution.zero_count()),
485 positive,
486 negative: None,
487 min_bits: distribution.min().map(|value| (value as f64).to_bits()),
488 max_bits: distribution.max().map(|value| (value as f64).to_bits()),
489 zero_threshold_bits: 0.0_f64.to_bits(),
490 reset_hint: 0,
491 }),
492 );
493 }
494}
495
496fn unix_nanos_now() -> u64 {
497 SystemTime::now()
498 .duration_since(UNIX_EPOCH)
499 .unwrap_or(Duration::ZERO)
500 .as_nanos()
501 .try_into()
502 .unwrap_or(u64::MAX)
503}
504
505#[cfg(test)]
506mod tests {
507 use std::time::{SystemTime, UNIX_EPOCH};
508
509 use ::fast_telemetry::{
510 Counter, Distribution, ExportMetrics, Gauge, Histogram, MetricKind as FastMetricKind,
511 MetricLabel, MetricLabels, MetricMeta, MetricScope, RuntimeConfig,
512 };
513
514 use super::*;
515 use crate::{EmbeddedTelemetryConfig, MetricQuery};
516
517 struct TestMetrics {
518 requests: Counter,
519 in_flight: Gauge,
520 latency: Histogram,
521 payload: Distribution,
522 }
523
524 impl TestMetrics {
525 fn new() -> Self {
526 Self {
527 requests: Counter::new(4),
528 in_flight: Gauge::new(),
529 latency: Histogram::new(&[10, 100], 4),
530 payload: Distribution::new(4),
531 }
532 }
533 }
534
535 impl ExportMetrics for TestMetrics {
536 fn visit_metrics<V: MetricVisitor + ?Sized>(&self, visitor: &mut V) {
537 let route = [MetricLabel {
538 name: "route",
539 value: "/health",
540 }];
541 visitor.counter(
542 MetricMeta {
543 name: "requests_total",
544 help: "Total requests.",
545 kind: FastMetricKind::Counter,
546 unit: Some("1"),
547 },
548 MetricLabels::slice(&route),
549 self.requests.sum() as i64,
550 );
551 visitor.gauge_i64(
552 MetricMeta {
553 name: "in_flight",
554 help: "In-flight requests.",
555 kind: FastMetricKind::Gauge,
556 unit: Some("1"),
557 },
558 MetricLabels::none(),
559 self.in_flight.get(),
560 );
561 visitor.histogram(
562 MetricMeta {
563 name: "latency_us",
564 help: "Request latency.",
565 kind: FastMetricKind::Histogram,
566 unit: Some("us"),
567 },
568 MetricLabels::none(),
569 &self.latency,
570 );
571 visitor.distribution(
572 MetricMeta {
573 name: "payload_bytes",
574 help: "Payload size.",
575 kind: FastMetricKind::Distribution,
576 unit: Some("By"),
577 },
578 MetricLabels::none(),
579 &self.payload,
580 );
581 }
582 }
583
584 #[test]
585 fn runtime_snapshot_is_queryable_from_the_embedded_store() {
586 let nonce = SystemTime::now()
587 .duration_since(UNIX_EPOCH)
588 .expect("clock")
589 .as_nanos();
590 let directory = std::env::temp_dir().join(format!(
591 "shard-telemetry-fast-runtime-{}-{nonce}",
592 std::process::id()
593 ));
594 let embedded = Arc::new(
595 EmbeddedTelemetryRuntime::open(EmbeddedTelemetryConfig::bounded_local(
596 directory.clone(),
597 Duration::from_secs(60),
598 ))
599 .expect("embedded runtime"),
600 );
601 let runtime = Runtime::new(RuntimeConfig::default());
602 let registered = runtime.register_metrics(MetricScope::new("api"), TestMetrics::new());
603 registered.requests.add(3);
604 registered.in_flight.set(7);
605 registered.latency.record(5);
606 registered.latency.record(50);
607 registered.payload.record(0);
608 registered.payload.record(16);
609
610 let exporter = embedded
611 .attach(|| {
612 FastTelemetryExporter::new(
613 runtime,
614 Arc::clone(&embedded),
615 FastTelemetryConfig::new("tenant-a", "embedded-test"),
616 )
617 })
618 .expect("exporter attachment");
619 embedded.mark_ready().expect("ready");
620 let report = exporter.export_once().expect("snapshot export");
621 assert_eq!(report.observed_points, 4);
622 assert_eq!(report.dropped_points, 0);
623 assert_eq!(report.appended_points, 4);
624 assert!(!report.skipped_busy);
625
626 let points = embedded
627 .query_metrics(&MetricQuery {
628 tenant: Arc::from("tenant-a"),
629 limit: 10,
630 ..MetricQuery::default()
631 })
632 .expect("local metric query");
633 assert_eq!(points.len(), 4);
634 let requests = points
635 .iter()
636 .find(|point| point.identity.name.as_ref() == "requests_total")
637 .expect("counter");
638 assert_eq!(requests.identity.scope.name.as_ref(), "api");
639 assert_eq!(requests.identity.resource.attributes.len(), 1);
640 assert_eq!(requests.identity.point_attributes.len(), 1);
641 assert!(matches!(
642 requests.value,
643 MetricValue::Sum(NumberValue::Integer(3))
644 ));
645 let distribution = points
646 .iter()
647 .find(|point| point.identity.name.as_ref() == "payload_bytes")
648 .expect("distribution");
649 assert!(matches!(
650 distribution.value,
651 MetricValue::ExponentialHistogram(ExponentialHistogramValue {
652 scale: 0,
653 zero_count: HistogramCount::Integer(1),
654 ..
655 })
656 ));
657
658 registered.requests.add(2);
659 registered.payload.record(4_096);
660 let second = exporter.export_once().expect("second snapshot export");
661 assert_eq!(second.appended_points, 4);
662 let rollups = embedded
663 .query_lifetime_metric_rollups("tenant-a", None)
664 .expect("lifetime outcomes");
665 assert_eq!(rollups.len(), 4);
666 let requests = rollups
667 .iter()
668 .find(|rollup| rollup.identity.name.as_ref() == "requests_total")
669 .expect("request lifetime outcome");
670 assert_eq!(requests.observed_points, 2);
671 assert!(matches!(
672 requests.outcome,
673 MetricValue::Sum(NumberValue::Integer(5))
674 ));
675
676 embedded.compact_retention().expect("retention pass");
677 embedded.drain().expect("drain");
678 drop(exporter);
679 drop(embedded);
680 std::fs::remove_dir_all(directory).expect("cleanup");
681 }
682}