Skip to main content

shard_telemetry/
lib.rs

1//! Signal-native logs, traces, and metrics storage for shard-stream.
2//!
3//! A [`ShardTelemetry`] owns one single-writer [`LogStripe`] per shard-stream
4//! physical shard. Call [`ShardTelemetry::apply_durable`] from the owning
5//! shard-stream worker only after the associated append is durable. This keeps
6//! the append log authoritative while allowing term and metadata indexes to be
7//! queried through a per-partition indexed watermark.
8
9#![warn(missing_docs)]
10
11mod analytics;
12mod block;
13mod correlation;
14mod deletion;
15mod dictionary;
16mod embedded;
17mod envelope;
18mod error;
19#[cfg(feature = "fast-telemetry")]
20mod fast_telemetry;
21mod ingest_pack;
22mod locality;
23mod loki_api;
24mod metric;
25mod native_client;
26mod native_protocol;
27mod native_server;
28mod offload;
29mod otlp;
30mod otlp_server;
31mod otlp_signal;
32mod production;
33mod prometheus_api;
34mod prometheus_protocol;
35mod prometheus_xor;
36mod promql;
37mod query;
38mod query_index;
39mod realtime_dictionary;
40mod remote_write;
41mod rollup;
42mod s3_object_store;
43mod signal_ingest;
44mod sink;
45mod sink_journal;
46mod storage_format;
47mod stripe;
48mod structural;
49mod telemetry;
50mod telemetry_store;
51mod tempo_api;
52mod tempo_protocol;
53mod tier;
54mod tier_ingest;
55mod trace;
56mod traceql;
57mod types;
58
59pub use analytics::{
60    ANALYTICS_SCHEMA_VERSION, AnalyticsColumn, AnalyticsRelation, AnalyticsRow, AnalyticsScanOrder,
61    AnalyticsScanRequest, AnalyticsWireFormat, CLICKHOUSE_COMPATIBILITY_TARGET,
62};
63pub use block::{BlockCatalog, BlockDescriptor, BlockId, CompressionCodec};
64pub use correlation::{
65    CorrelationBlockFilter, CorrelationConfig, CorrelationIndex, CorrelationQuery, CorrelationStats,
66};
67pub use deletion::DeleteRequest;
68pub use dictionary::{
69    CompressionCohortId, DictionaryCache, DictionaryCatalog, DictionaryCatalogSnapshot,
70    DictionaryId, DictionaryInsert, DictionaryPublication,
71};
72pub use embedded::{
73    EmbeddedEvictionPolicy, EmbeddedTelemetryConfig, EmbeddedTelemetryRuntime,
74    EmbeddedTelemetryState,
75};
76pub use envelope::{MAX_TELEMETRY_ENVELOPE_BYTES, TelemetryEnvelope};
77pub use error::{TelemetryError, TelemetryResult};
78#[cfg(feature = "fast-telemetry")]
79pub use fast_telemetry::{FastTelemetryConfig, FastTelemetryExportReport, FastTelemetryExporter};
80pub use locality::{
81    CompressionBlockAssignment, CompressionBlockCollator, CompressionBlockScore,
82    CompressionLocalityConfig, CompressionLocalityRecord, CompressionLocalityStats,
83    CompressionPlacement, CompressionPlacementId, CompressionShardProfile, CompressionTemperature,
84    LocalityGranularity, MessageFingerprint, analyze_message, fingerprint_message,
85    scan_message_terms,
86};
87pub use loki_api::{
88    LokiApiConfig, LokiApiError, LokiApiStore, LokiEntry, LokiStore, StoreHealth, StoreMetrics,
89    loki_router, loki_router_with_clickhouse, single_tenant_loki_api_router,
90    single_tenant_loki_router,
91};
92pub use metric::{
93    DurableMetricPoint, ExplicitHistogramValue, ExponentialHistogramBuckets,
94    ExponentialHistogramValue, HistogramBucketSpan, HistogramCount, MetricApplyOutcome,
95    MetricExemplar, MetricIdentity, MetricIngestProtocol, MetricKind, MetricQuery, MetricStripe,
96    MetricValue, NumberValue, SeriesAccumulatorCheckpoint, SummaryQuantileValue, SummaryValue,
97    decode_metric_chunk, encode_metric_chunk, prometheus_string_labels,
98};
99pub use native_client::{NativeClientConfig, NativeClientError, ShardTelemetryClient};
100pub use native_protocol::{
101    MAX_NATIVE_FRAME_BYTES, NATIVE_FRAME_HEADER_BYTES, NativeCapabilities, NativeFrame,
102    NativeFrameHeader, NativeLogQueryResult, NativeOpcode, NativePartitionAck,
103    NativePartitionAppend, NativeProtocolError, NativeQuery, NativeQueryDirection, NativeStatus,
104    NativeTelemetryAppendAck, NativeTelemetryBatch, decode_native_capabilities,
105    decode_native_log_query_result, decode_native_metric_query, decode_native_metric_query_result,
106    decode_native_query, decode_native_trace_query, decode_native_trace_query_result,
107    encode_native_capabilities, encode_native_log_query_result, encode_native_metric_query,
108    encode_native_metric_query_result, encode_native_query, encode_native_trace_query,
109    encode_native_trace_query_result, is_native_telemetry_batch,
110};
111pub use native_server::{NativeRequestGate, NativeServerConfig, serve_native};
112pub use offload::{
113    OffloadCheckpoint, OffloadError, OffloadLoopReport, OffloadReport, UpstreamOffloadConfig,
114    UpstreamOffloadLoopConfig, UpstreamOffloader,
115};
116pub use otlp::{OtlpLogDecoder, OtlpLogEvent};
117pub use otlp_server::{OtlpIngestService, OtlpReceiverConfig, otlp_http_router, serve_otlp_grpc};
118pub use otlp_signal::{OtlpMetricEvent, OtlpSpanEvent, OtlpTelemetryDecoder};
119pub use production::{
120    ProductionMetricsSnapshot, ProductionRuntime, ServiceLifecycle, ServiceState,
121    SingleTenantConfig,
122};
123pub use prometheus_api::{PrometheusApiConfig, PrometheusService, prometheus_router};
124pub use promql::{
125    PromqlEngine, PromqlError, PromqlLimits, PromqlSample, PromqlSeries, PromqlValue,
126};
127pub use query_index::{BlockQueryIndex, PersistentQueryIndex, QueryBlockMetadata, QueryHit};
128pub use realtime_dictionary::{
129    RealtimeDictionaryConfig, RealtimeDictionaryObserver, RealtimeDictionaryStats,
130    RealtimeDictionaryTrainer,
131};
132pub use remote_write::{
133    DecodedRemoteWrite, METRIC_FLAG_STALE, PROMETHEUS_STALE_NAN_BITS, RemoteWriteDecoder,
134    RemoteWriteStats, RemoteWriteVersion,
135};
136pub use rollup::LifetimeMetricRollup;
137pub use s3_object_store::{S3ObjectStore, S3ObjectStoreConfig};
138pub use signal_ingest::{
139    DockerLogRecord, DockerLogStream, decode_log_envelope, prepare_docker_log_envelope,
140    prepare_docker_log_envelope_with_context, prepare_log_envelope,
141    prepare_log_envelope_with_context, prepare_loki_log_envelope, prepare_metric_envelope,
142    prepare_metric_envelope_with_protocol, prepare_trace_envelope,
143};
144pub use sink::{OtlpSinkConfig, SinkObjectTierConfig, TelemetryService, TelemetrySinkFactory};
145pub use stripe::{IndexReceipt, LogStripe, ShardStreamDurableSink, ShardTelemetry, StripeConfig};
146pub use structural::{
147    DecodedStructuralRecord, EmbeddedFrameIndex, IndexedStructuralBlock, StructuralLogMetadataRef,
148    StructuralRecordView, decode_embedded_frame_index, decode_structural_block,
149    decode_structural_records, encode_indexed_structural_records, encode_structural_block,
150    encode_structural_records, message_pattern,
151};
152pub use telemetry::{
153    AttributeFingerprint, LOGS_TOPIC_ID, METRICS_TOPIC_ID, ResourceContext, ResourceContextId,
154    ScopeContext, ScopeContextId, SeriesFingerprint, ShardTelemetryConfig, SignalConfig, SpanId,
155    TRACES_TOPIC_ID, TelemetryAttribute, TelemetryEntityRef, TelemetryRouter, TelemetrySignal,
156    TelemetryValue, TraceId,
157};
158pub use telemetry_store::{
159    DurableTelemetryConfig, DurableTelemetryLimits, DurableTelemetryStore, FetchedTelemetryBatch,
160    LifetimeRollupReport, RetentionReport, TelemetryAppendDurability, TelemetryAppendGate,
161    TelemetryHostAttachment,
162};
163pub use tempo_api::{TempoApiConfig, TempoService, tempo_router};
164pub use tier::{
165    CachedObjectRange, CatalogGroupEntry, CatalogLease, CatalogPage, CatalogPageRef,
166    CatalogPointer, CatalogRoot, LocalObjectStore, ObjectMetadata, ObjectStoreStats,
167    ObjectTierConfig, RetiredObject, SharedTelemetryObjectStore, SignalTierPayload, SsdCacheConfig,
168    SsdCacheStats, SsdObjectCache, TelemetryObjectStore, TelemetryObjectTier, TierArtifact,
169    TierArtifactKind, TierArtifactSource, TierBlockEntry, TierCheckpoint, TierGroupManifest,
170    TierGroupSource, TierQueryRange, TierRetentionReport, decode_signal_recovery_state,
171    mark_group_offloaded, stage_signal_group, write_staged_payload_pack,
172};
173pub use trace::{
174    DurableSpan, SpanEvent, SpanLink, SpanStatus, TraceApplyOutcome, TraceDirectory, TraceQuery,
175    TraceStripe, TraceSummary, decode_trace_block, encode_trace_block,
176};
177pub use traceql::{
178    TraceqlEngine, TraceqlError, TraceqlLimits, TraceqlMetricExemplar, TraceqlMetricSample,
179    TraceqlMetricSeries, TraceqlTrace,
180};
181pub use types::{
182    CaseSensitivity, DurableLog, LogMatch, LogPredicate, LogQuery, LogRegex, MetadataField,
183    NumericComparison, QueryCursor, QueryOrder, QuerySort, TelemetryRecordRef, TextMatchKind,
184    TextMatcher,
185};