Skip to main content

shard_telemetry/
fast_telemetry.rs

1//! Direct metric snapshots from `fast-telemetry` into an embedded store.
2//!
3//! Metric recording remains entirely inside `fast-telemetry`. A host-owned
4//! background task calls [`FastTelemetryExporter::export_once`] at a bounded
5//! interval; only that snapshot path allocates storage-native points or waits
6//! for the local durable WAL.
7
8use std::sync::Arc;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::time::{Duration, SystemTime, UNIX_EPOCH};
11
12use ::fast_telemetry::{
13    DistributionSnapshot, HistogramSnapshot, MetricLabels, MetricMeta, MetricVisitor, Runtime,
14};
15use shard_stream_core::{LogicalOffset, LogicalPartitionId, ShardId, TopicPartition};
16
17use crate::{
18    DurableMetricPoint, EmbeddedTelemetryRuntime, ExplicitHistogramValue,
19    ExponentialHistogramBuckets, ExponentialHistogramValue, HistogramBucketSpan, HistogramCount,
20    LokiApiError, METRICS_TOPIC_ID, MetricIdentity, MetricKind, MetricValue,
21    NativeTelemetryAppendAck, NumberValue, ResourceContext, ScopeContext, TelemetryAttribute,
22    TelemetryRecordRef, TelemetrySignal, TelemetryValue,
23};
24
25const CUMULATIVE_TEMPORALITY: i32 = 2;
26const DEFAULT_EXPORT_INTERVAL: Duration = Duration::from_secs(5);
27const DEFAULT_MAX_POINTS_PER_EXPORT: usize = 65_536;
28
29/// Bounded configuration for a periodic `fast-telemetry` snapshot.
30#[derive(Debug, Clone, PartialEq, Eq)]
31pub struct FastTelemetryConfig {
32    /// Tenant attached to every exported metric series.
33    pub tenant: Arc<str>,
34    /// Resource attributes attached to every exported metric series.
35    pub resource_attributes: Arc<Vec<TelemetryAttribute>>,
36    /// Version attached to each registered `fast-telemetry` scope.
37    pub scope_version: Arc<str>,
38    /// Recommended interval for the host-owned export timer.
39    pub interval: Duration,
40    /// Maximum points emitted by one runtime snapshot.
41    pub max_points_per_export: usize,
42    /// Whether zero counters and empty histograms/distributions are skipped.
43    pub skip_empty_cumulative: bool,
44    /// Whether an export waits until the local query index has applied its WAL
45    /// append.
46    pub wait_for_index: bool,
47}
48
49impl FastTelemetryConfig {
50    /// Creates a conventional configuration with one `service.name` resource
51    /// attribute.
52    #[must_use]
53    pub fn new(tenant: impl Into<Arc<str>>, service_name: impl Into<Arc<str>>) -> Self {
54        Self {
55            tenant: tenant.into(),
56            resource_attributes: Arc::new(vec![TelemetryAttribute::new(
57                "service.name",
58                TelemetryValue::String(service_name.into()),
59            )]),
60            scope_version: Arc::from("fast-telemetry"),
61            interval: DEFAULT_EXPORT_INTERVAL,
62            max_points_per_export: DEFAULT_MAX_POINTS_PER_EXPORT,
63            skip_empty_cumulative: true,
64            wait_for_index: true,
65        }
66    }
67
68    /// Adds one typed resource attribute to every exported series.
69    #[must_use]
70    pub fn with_resource_attribute(mut self, attribute: TelemetryAttribute) -> Self {
71        Arc::make_mut(&mut self.resource_attributes).push(attribute);
72        self
73    }
74
75    /// Sets the recommended host-owned export interval.
76    #[must_use]
77    pub const fn with_interval(mut self, interval: Duration) -> Self {
78        self.interval = interval;
79        self
80    }
81
82    /// Sets the maximum points accepted from one snapshot.
83    #[must_use]
84    pub const fn with_max_points_per_export(mut self, max_points: usize) -> Self {
85        self.max_points_per_export = max_points;
86        self
87    }
88
89    /// Sets whether empty cumulative instruments are omitted.
90    #[must_use]
91    pub const fn with_skip_empty_cumulative(mut self, skip: bool) -> Self {
92        self.skip_empty_cumulative = skip;
93        self
94    }
95
96    /// Chooses between query-visible and WAL-only local acknowledgement.
97    #[must_use]
98    pub const fn with_wait_for_index(mut self, wait_for_index: bool) -> Self {
99        self.wait_for_index = wait_for_index;
100        self
101    }
102
103    fn validate(&self) -> Result<(), LokiApiError> {
104        if self.tenant.is_empty() {
105            return Err(LokiApiError::configuration(
106                "fast-telemetry tenant must not be empty",
107            ));
108        }
109        if self.interval.is_zero() {
110            return Err(LokiApiError::configuration(
111                "fast-telemetry export interval must be nonzero",
112            ));
113        }
114        if self.max_points_per_export == 0 {
115            return Err(LokiApiError::configuration(
116                "fast-telemetry export point limit must be nonzero",
117            ));
118        }
119        Ok(())
120    }
121}
122
123/// Counters returned by one embedded snapshot export.
124#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
125pub struct FastTelemetryExportReport {
126    /// Points observed after empty-cumulative filtering and before the cap.
127    pub observed_points: usize,
128    /// Points omitted because the configured snapshot cap was reached.
129    pub dropped_points: usize,
130    /// Points acknowledged by the embedded store.
131    pub appended_points: usize,
132    /// Per-series durable envelopes appended by the store.
133    pub append_batches: usize,
134    /// Whether another export was already active.
135    pub skipped_busy: bool,
136}
137
138/// Background-driven bridge from a `fast-telemetry` runtime to local durable
139/// storage.
140pub struct FastTelemetryExporter {
141    runtime: Arc<Runtime>,
142    embedded: Arc<EmbeddedTelemetryRuntime>,
143    config: FastTelemetryConfig,
144    exporting: AtomicBool,
145}
146
147impl FastTelemetryExporter {
148    /// Creates an exporter for one runtime and embedded store.
149    ///
150    /// Construct this through [`EmbeddedTelemetryRuntime::attach`] before
151    /// marking the embedded runtime ready. Call [`Self::export_once`] only
152    /// from a background worker, never from an application metric update.
153    pub fn new(
154        runtime: Arc<Runtime>,
155        embedded: Arc<EmbeddedTelemetryRuntime>,
156        config: FastTelemetryConfig,
157    ) -> Result<Self, LokiApiError> {
158        config.validate()?;
159        Ok(Self {
160            runtime,
161            embedded,
162            config,
163            exporting: AtomicBool::new(false),
164        })
165    }
166
167    /// Returns the recommended interval for a host-owned periodic task.
168    #[must_use]
169    pub const fn interval(&self) -> Duration {
170        self.config.interval
171    }
172
173    /// Snapshots registered metric groups at the current wall-clock time.
174    pub fn export_once(&self) -> Result<FastTelemetryExportReport, LokiApiError> {
175        self.export_once_at(unix_nanos_now())
176    }
177
178    /// Snapshots registered metric groups at an explicit timestamp.
179    pub fn export_once_at(
180        &self,
181        timestamp_unix_nanos: u64,
182    ) -> Result<FastTelemetryExportReport, LokiApiError> {
183        if self
184            .exporting
185            .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
186            .is_err()
187        {
188            return Ok(FastTelemetryExportReport {
189                skipped_busy: true,
190                ..FastTelemetryExportReport::default()
191            });
192        }
193        let _guard = ExportGuard {
194            exporting: &self.exporting,
195        };
196        let mut export = FastMetricExport::new(&self.embedded, &self.config, timestamp_unix_nanos);
197
198        // Runtime registration is a construction-time operation. Deduplicating
199        // scopes avoids revisiting all groups when several groups intentionally
200        // share one logical scope.
201        let mut scopes = self.runtime.scopes();
202        scopes.sort_unstable();
203        scopes.dedup();
204        for scope in scopes {
205            export.set_scope(scope.name());
206            self.runtime.visit_metrics_for_scope(&scope, &mut export);
207        }
208
209        let observed_points = export.observed_points;
210        let dropped_points = export.dropped_points;
211        let (appended_points, acknowledgement) = export.finish()?;
212        Ok(FastTelemetryExportReport {
213            observed_points,
214            dropped_points,
215            appended_points,
216            append_batches: acknowledgement.partitions.len(),
217            skipped_busy: false,
218        })
219    }
220}
221
222struct ExportGuard<'a> {
223    exporting: &'a AtomicBool,
224}
225
226impl Drop for ExportGuard<'_> {
227    fn drop(&mut self) {
228        self.exporting.store(false, Ordering::Release);
229    }
230}
231
232struct FastMetricExport<'a> {
233    embedded: &'a EmbeddedTelemetryRuntime,
234    tenant: Arc<str>,
235    resource: Arc<ResourceContext>,
236    scope_version: Arc<str>,
237    scope: Arc<ScopeContext>,
238    timestamp_unix_nanos: u64,
239    max_points_per_export: usize,
240    skip_empty_cumulative: bool,
241    wait_for_index: bool,
242    points: Vec<DurableMetricPoint>,
243    tail: Option<DurableMetricPoint>,
244    observed_points: usize,
245    dropped_points: usize,
246}
247
248impl<'a> FastMetricExport<'a> {
249    fn new(
250        embedded: &'a EmbeddedTelemetryRuntime,
251        config: &FastTelemetryConfig,
252        timestamp_unix_nanos: u64,
253    ) -> Self {
254        Self {
255            embedded,
256            tenant: Arc::clone(&config.tenant),
257            resource: Arc::new(ResourceContext {
258                attributes: Arc::clone(&config.resource_attributes),
259                ..ResourceContext::default()
260            }),
261            scope_version: Arc::clone(&config.scope_version),
262            scope: Arc::new(ScopeContext::default()),
263            timestamp_unix_nanos,
264            max_points_per_export: config.max_points_per_export,
265            skip_empty_cumulative: config.skip_empty_cumulative,
266            wait_for_index: config.wait_for_index,
267            points: Vec::new(),
268            tail: None,
269            observed_points: 0,
270            dropped_points: 0,
271        }
272    }
273
274    fn set_scope(&mut self, scope_name: &str) {
275        self.scope = Arc::new(ScopeContext {
276            name: Arc::from(scope_name),
277            version: Arc::clone(&self.scope_version),
278            ..ScopeContext::default()
279        });
280    }
281
282    fn finish(mut self) -> Result<(usize, NativeTelemetryAppendAck), LokiApiError> {
283        let Some(tail) = self.tail.take() else {
284            return Ok((
285                0,
286                NativeTelemetryAppendAck {
287                    partitions: Vec::new(),
288                },
289            ));
290        };
291        let point_count = self.points.len().saturating_add(1);
292        let acknowledgement = if self.points.is_empty() {
293            self.embedded
294                .append_metric_point(tail, self.wait_for_index)?
295        } else {
296            self.points.push(tail);
297            self.embedded
298                .append_metric_points(self.points, self.wait_for_index)?
299        };
300        Ok((point_count, acknowledgement))
301    }
302
303    fn push(
304        &mut self,
305        meta: MetricMeta<'_>,
306        labels: MetricLabels<'_>,
307        kind: MetricKind,
308        value: MetricValue,
309    ) {
310        self.observed_points = self.observed_points.saturating_add(1);
311        if self
312            .points
313            .len()
314            .saturating_add(usize::from(self.tail.is_some()))
315            >= self.max_points_per_export
316        {
317            self.dropped_points = self.dropped_points.saturating_add(1);
318            return;
319        }
320        let point_attributes = labels
321            .iter()
322            .map(|label| {
323                TelemetryAttribute::new(
324                    Arc::<str>::from(label.name),
325                    TelemetryValue::String(Arc::from(label.value)),
326                )
327            })
328            .collect();
329        let point = DurableMetricPoint {
330            stream_shard_id: ShardId::new(0),
331            record_ref: TelemetryRecordRef::for_signal(
332                TelemetrySignal::Metrics,
333                TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(0)),
334                LogicalOffset::new(0),
335            ),
336            identity: Arc::new(MetricIdentity {
337                tenant: Arc::clone(&self.tenant),
338                resource: Arc::clone(&self.resource),
339                scope: Arc::clone(&self.scope),
340                name: Arc::from(meta.name),
341                unit: Arc::from(meta.unit.unwrap_or("")),
342                kind,
343                point_attributes: Arc::new(point_attributes),
344            }),
345            description: Arc::from(meta.help),
346            metadata: Arc::new(Vec::new()),
347            start_time_unix_nanos: 0,
348            timestamp_unix_nanos: self.timestamp_unix_nanos,
349            flags: 0,
350            value,
351            exemplars: Arc::new(Vec::new()),
352        };
353        if let Some(previous) = self.tail.replace(point) {
354            self.points.push(previous);
355        }
356    }
357}
358
359impl MetricVisitor for FastMetricExport<'_> {
360    fn counter(&mut self, meta: MetricMeta<'_>, labels: MetricLabels<'_>, value: i64) {
361        if self.skip_empty_cumulative && value == 0 {
362            return;
363        }
364        self.push(
365            meta,
366            labels,
367            MetricKind::Sum {
368                temporality: CUMULATIVE_TEMPORALITY,
369                monotonic: true,
370            },
371            MetricValue::Sum(NumberValue::Integer(value)),
372        );
373    }
374
375    fn gauge_i64(&mut self, meta: MetricMeta<'_>, labels: MetricLabels<'_>, value: i64) {
376        self.push(
377            meta,
378            labels,
379            MetricKind::Gauge,
380            MetricValue::Gauge(NumberValue::Integer(value)),
381        );
382    }
383
384    fn gauge_f64(&mut self, meta: MetricMeta<'_>, labels: MetricLabels<'_>, value: f64) {
385        self.push(
386            meta,
387            labels,
388            MetricKind::Gauge,
389            MetricValue::Gauge(NumberValue::from_f64(value)),
390        );
391    }
392
393    fn histogram(
394        &mut self,
395        meta: MetricMeta<'_>,
396        labels: MetricLabels<'_>,
397        histogram: &dyn HistogramSnapshot,
398    ) {
399        if self.skip_empty_cumulative && histogram.count() == 0 {
400            return;
401        }
402        let mut explicit_bounds_bits = Vec::new();
403        let mut bucket_counts = Vec::new();
404        let mut previous = 0_u64;
405        histogram.visit_buckets(&mut |bound, cumulative_count| {
406            explicit_bounds_bits.push((bound as f64).to_bits());
407            bucket_counts.push(HistogramCount::Integer(
408                cumulative_count.saturating_sub(previous),
409            ));
410            previous = cumulative_count;
411        });
412        bucket_counts.push(HistogramCount::Integer(
413            histogram.count().saturating_sub(previous),
414        ));
415        self.push(
416            meta,
417            labels,
418            MetricKind::ExplicitHistogram {
419                temporality: CUMULATIVE_TEMPORALITY,
420            },
421            MetricValue::ExplicitHistogram(ExplicitHistogramValue {
422                count: HistogramCount::Integer(histogram.count()),
423                sum_bits: Some((histogram.sum() as f64).to_bits()),
424                bucket_counts: Arc::new(bucket_counts),
425                explicit_bounds_bits: Arc::new(explicit_bounds_bits),
426                min_bits: None,
427                max_bits: None,
428                reset_hint: 0,
429            }),
430        );
431    }
432
433    fn distribution(
434        &mut self,
435        meta: MetricMeta<'_>,
436        labels: MetricLabels<'_>,
437        distribution: &dyn DistributionSnapshot,
438    ) {
439        if self.skip_empty_cumulative && distribution.count() == 0 {
440            return;
441        }
442        let mut sparse = Vec::new();
443        distribution.visit_positive_buckets(&mut |index, count| sparse.push((index, count)));
444        sparse.sort_unstable_by_key(|(index, _)| *index);
445        let positive = (!sparse.is_empty()).then(|| {
446            let mut spans = Vec::<HistogramBucketSpan>::new();
447            let mut counts = Vec::with_capacity(sparse.len());
448            let mut prior_end = 0_i64;
449            for (index, count) in sparse {
450                let index = i64::from(index);
451                if let Some(span) = spans.last_mut()
452                    && index == prior_end
453                {
454                    span.length = span.length.saturating_add(1);
455                } else {
456                    let offset = if spans.is_empty() {
457                        index
458                    } else {
459                        index.saturating_sub(prior_end)
460                    };
461                    spans.push(HistogramBucketSpan {
462                        offset: i32::try_from(offset).unwrap_or(i32::MAX),
463                        length: 1,
464                    });
465                }
466                counts.push(HistogramCount::Integer(count));
467                prior_end = index.saturating_add(1);
468            }
469            ExponentialHistogramBuckets {
470                spans: Arc::new(spans),
471                bucket_counts: Arc::new(counts),
472            }
473        });
474        self.push(
475            meta,
476            labels,
477            MetricKind::ExponentialHistogram {
478                temporality: CUMULATIVE_TEMPORALITY,
479            },
480            MetricValue::ExponentialHistogram(ExponentialHistogramValue {
481                count: HistogramCount::Integer(distribution.count()),
482                sum_bits: Some((distribution.sum() as f64).to_bits()),
483                scale: 0,
484                zero_count: HistogramCount::Integer(distribution.zero_count()),
485                positive,
486                negative: None,
487                min_bits: distribution.min().map(|value| (value as f64).to_bits()),
488                max_bits: distribution.max().map(|value| (value as f64).to_bits()),
489                zero_threshold_bits: 0.0_f64.to_bits(),
490                reset_hint: 0,
491            }),
492        );
493    }
494}
495
496fn unix_nanos_now() -> u64 {
497    SystemTime::now()
498        .duration_since(UNIX_EPOCH)
499        .unwrap_or(Duration::ZERO)
500        .as_nanos()
501        .try_into()
502        .unwrap_or(u64::MAX)
503}
504
505#[cfg(test)]
506mod tests {
507    use std::time::{SystemTime, UNIX_EPOCH};
508
509    use ::fast_telemetry::{
510        Counter, Distribution, ExportMetrics, Gauge, Histogram, MetricKind as FastMetricKind,
511        MetricLabel, MetricLabels, MetricMeta, MetricScope, RuntimeConfig,
512    };
513
514    use super::*;
515    use crate::{EmbeddedTelemetryConfig, MetricQuery};
516
517    struct TestMetrics {
518        requests: Counter,
519        in_flight: Gauge,
520        latency: Histogram,
521        payload: Distribution,
522    }
523
524    impl TestMetrics {
525        fn new() -> Self {
526            Self {
527                requests: Counter::new(4),
528                in_flight: Gauge::new(),
529                latency: Histogram::new(&[10, 100], 4),
530                payload: Distribution::new(4),
531            }
532        }
533    }
534
535    impl ExportMetrics for TestMetrics {
536        fn visit_metrics<V: MetricVisitor + ?Sized>(&self, visitor: &mut V) {
537            let route = [MetricLabel {
538                name: "route",
539                value: "/health",
540            }];
541            visitor.counter(
542                MetricMeta {
543                    name: "requests_total",
544                    help: "Total requests.",
545                    kind: FastMetricKind::Counter,
546                    unit: Some("1"),
547                },
548                MetricLabels::slice(&route),
549                self.requests.sum() as i64,
550            );
551            visitor.gauge_i64(
552                MetricMeta {
553                    name: "in_flight",
554                    help: "In-flight requests.",
555                    kind: FastMetricKind::Gauge,
556                    unit: Some("1"),
557                },
558                MetricLabels::none(),
559                self.in_flight.get(),
560            );
561            visitor.histogram(
562                MetricMeta {
563                    name: "latency_us",
564                    help: "Request latency.",
565                    kind: FastMetricKind::Histogram,
566                    unit: Some("us"),
567                },
568                MetricLabels::none(),
569                &self.latency,
570            );
571            visitor.distribution(
572                MetricMeta {
573                    name: "payload_bytes",
574                    help: "Payload size.",
575                    kind: FastMetricKind::Distribution,
576                    unit: Some("By"),
577                },
578                MetricLabels::none(),
579                &self.payload,
580            );
581        }
582    }
583
584    #[test]
585    fn runtime_snapshot_is_queryable_from_the_embedded_store() {
586        let nonce = SystemTime::now()
587            .duration_since(UNIX_EPOCH)
588            .expect("clock")
589            .as_nanos();
590        let directory = std::env::temp_dir().join(format!(
591            "shard-telemetry-fast-runtime-{}-{nonce}",
592            std::process::id()
593        ));
594        let embedded = Arc::new(
595            EmbeddedTelemetryRuntime::open(EmbeddedTelemetryConfig::bounded_local(
596                directory.clone(),
597                Duration::from_secs(60),
598            ))
599            .expect("embedded runtime"),
600        );
601        let runtime = Runtime::new(RuntimeConfig::default());
602        let registered = runtime.register_metrics(MetricScope::new("api"), TestMetrics::new());
603        registered.requests.add(3);
604        registered.in_flight.set(7);
605        registered.latency.record(5);
606        registered.latency.record(50);
607        registered.payload.record(0);
608        registered.payload.record(16);
609
610        let exporter = embedded
611            .attach(|| {
612                FastTelemetryExporter::new(
613                    runtime,
614                    Arc::clone(&embedded),
615                    FastTelemetryConfig::new("tenant-a", "embedded-test"),
616                )
617            })
618            .expect("exporter attachment");
619        embedded.mark_ready().expect("ready");
620        let report = exporter.export_once().expect("snapshot export");
621        assert_eq!(report.observed_points, 4);
622        assert_eq!(report.dropped_points, 0);
623        assert_eq!(report.appended_points, 4);
624        assert!(!report.skipped_busy);
625
626        let points = embedded
627            .query_metrics(&MetricQuery {
628                tenant: Arc::from("tenant-a"),
629                limit: 10,
630                ..MetricQuery::default()
631            })
632            .expect("local metric query");
633        assert_eq!(points.len(), 4);
634        let requests = points
635            .iter()
636            .find(|point| point.identity.name.as_ref() == "requests_total")
637            .expect("counter");
638        assert_eq!(requests.identity.scope.name.as_ref(), "api");
639        assert_eq!(requests.identity.resource.attributes.len(), 1);
640        assert_eq!(requests.identity.point_attributes.len(), 1);
641        assert!(matches!(
642            requests.value,
643            MetricValue::Sum(NumberValue::Integer(3))
644        ));
645        let distribution = points
646            .iter()
647            .find(|point| point.identity.name.as_ref() == "payload_bytes")
648            .expect("distribution");
649        assert!(matches!(
650            distribution.value,
651            MetricValue::ExponentialHistogram(ExponentialHistogramValue {
652                scale: 0,
653                zero_count: HistogramCount::Integer(1),
654                ..
655            })
656        ));
657
658        registered.requests.add(2);
659        registered.payload.record(4_096);
660        let second = exporter.export_once().expect("second snapshot export");
661        assert_eq!(second.appended_points, 4);
662        let rollups = embedded
663            .query_lifetime_metric_rollups("tenant-a", None)
664            .expect("lifetime outcomes");
665        assert_eq!(rollups.len(), 4);
666        let requests = rollups
667            .iter()
668            .find(|rollup| rollup.identity.name.as_ref() == "requests_total")
669            .expect("request lifetime outcome");
670        assert_eq!(requests.observed_points, 2);
671        assert!(matches!(
672            requests.outcome,
673            MetricValue::Sum(NumberValue::Integer(5))
674        ));
675
676        embedded.compact_retention().expect("retention pass");
677        embedded.drain().expect("drain");
678        drop(exporter);
679        drop(embedded);
680        std::fs::remove_dir_all(directory).expect("cleanup");
681    }
682}