Skip to main content

shard_telemetry/
embedded.rs

1use std::path::PathBuf;
2use std::sync::atomic::{AtomicU8, Ordering};
3use std::sync::{Arc, Mutex, RwLock};
4use std::time::Duration;
5
6use crate::{
7    DurableMetricPoint, DurableTelemetryConfig, DurableTelemetryLimits, DurableTelemetryStore,
8    LokiApiError, LokiStore, NativeTelemetryAppendAck, ObjectTierConfig, OtlpLogEvent,
9    OtlpSpanEvent, RetentionReport, S3ObjectStoreConfig, SsdCacheConfig, StripeConfig,
10};
11
12fn usize_from_u64(value: u64) -> usize {
13    usize::try_from(value).unwrap_or(usize::MAX)
14}
15
16fn configure_cache_ssd(config: &mut SsdCacheConfig, max_bytes: u64) {
17    config.max_bytes = max_bytes.max(1);
18    config.chunk_bytes = config.chunk_bytes.min(config.max_bytes).max(1);
19    config.max_read_bytes = config.max_read_bytes.max(config.chunk_bytes);
20}
21
22/// Fate of raw records that leave an embedded node's recent local window.
23#[derive(Debug, Clone, PartialEq, Eq)]
24pub enum EmbeddedEvictionPolicy {
25    /// Permanently delete expired or over-budget immutable groups.
26    Delete,
27    /// Commit immutable groups to S3, then keep only an LRU-bounded local cache.
28    OffloadToS3(S3ObjectStoreConfig),
29}
30
31/// Lifecycle configuration for a one-process embedded ShardTelemetry runtime.
32#[derive(Debug, Clone)]
33pub struct EmbeddedTelemetryConfig {
34    /// Durable local WAL, index, and object-tier configuration.
35    pub store: DurableTelemetryConfig,
36    /// Bounded hot-memory, journal, and SSD-cache limits.
37    pub local_limits: DurableTelemetryLimits,
38    /// Immutable group sizing used for local deletion or S3 publication.
39    pub object_tier: ObjectTierConfig,
40    /// Optional bound for engine-controlled resident telemetry state.
41    ///
42    /// Transient request decoding and query result allocations are outside this
43    /// storage-state budget.
44    pub max_ram_bytes: Option<u64>,
45    /// Optional bound for local immutable payload and SSD-cache bytes.
46    ///
47    /// Filesystem metadata and one in-flight WAL/spool group may temporarily
48    /// exceed this steady-state data budget.
49    pub max_ssd_bytes: Option<u64>,
50    /// Upper bound applied when draining the embedded store during shutdown.
51    pub shutdown_flush_timeout: Duration,
52}
53
54impl EmbeddedTelemetryConfig {
55    /// Creates an embedded runtime configuration around one durable store.
56    #[must_use]
57    pub fn new(store: DurableTelemetryConfig) -> Self {
58        let local_limits = DurableTelemetryLimits {
59            max_lifetime_rollup_series: Some(100_000),
60            ..DurableTelemetryLimits::default()
61        };
62        Self {
63            store,
64            local_limits,
65            object_tier: ObjectTierConfig::default(),
66            max_ram_bytes: None,
67            max_ssd_bytes: None,
68            shutdown_flush_timeout: Duration::from_secs(30),
69        }
70    }
71
72    /// Creates a one-shard, SSD-backed embedded configuration with time-based
73    /// retention.
74    ///
75    /// The local immutable object directory lives below `data_directory`, so
76    /// completed blocks can replace raw WAL packs while remaining queryable.
77    /// Call [`EmbeddedTelemetryRuntime::compact_retention`] periodically to
78    /// reclaim complete groups older than `retention`.
79    #[must_use]
80    pub fn bounded_local(data_directory: impl Into<PathBuf>, retention: Duration) -> Self {
81        Self::bounded(data_directory, retention, EmbeddedEvictionPolicy::Delete)
82    }
83
84    /// Creates a one-shard embedded store with a recent local window and an
85    /// explicit eviction destination.
86    #[must_use]
87    pub fn bounded(
88        data_directory: impl Into<PathBuf>,
89        retention: Duration,
90        eviction: EmbeddedEvictionPolicy,
91    ) -> Self {
92        let data_directory = data_directory.into();
93        let (object_store_directory, s3_object_store) = match eviction {
94            EmbeddedEvictionPolicy::Delete => (Some(data_directory.join("objects")), None),
95            EmbeddedEvictionPolicy::OffloadToS3(s3) => (None, Some(s3)),
96        };
97        let mut config = Self::new(DurableTelemetryConfig {
98            object_store_directory,
99            data_directory,
100            s3_object_store,
101            recovery_journal: false,
102            retention: Some(retention),
103            shard_count: 1,
104            tenant_partitions: 1,
105            append_linger: Duration::from_micros(250),
106            stripe: StripeConfig::default(),
107            indexed_ack_timeout: Duration::from_secs(30),
108        });
109        if config.store.s3_object_store.is_none() {
110            // Embedded mode has one process and the tier already protects
111            // active readers with generation leases, so no cross-process grace
112            // window is required before deleting expired local objects.
113            config.object_tier.retirement_grace = Duration::ZERO;
114        }
115        config
116    }
117
118    /// Sets a total bound for storage-engine telemetry state in RAM.
119    ///
120    /// Two thirds is divided across log, trace, and metric heads on every
121    /// physical stripe, one twelfth is reserved for compression dictionaries,
122    /// and the remainder is divided across verified payload and control caches.
123    #[must_use]
124    pub fn with_max_ram_bytes(mut self, max_bytes: u64) -> Self {
125        self.max_ram_bytes = Some(max_bytes);
126        let stripes = u64::from(
127            self.store
128                .shard_count
129                .min(self.store.tenant_partitions)
130                .max(1),
131        );
132        let head_total = max_bytes.saturating_mul(2) / 3;
133        let head_per_stripe = head_total / stripes;
134        let logs = (head_per_stripe / 8).max(1);
135        let traces = (head_per_stripe / 4).max(1);
136        let metrics = head_per_stripe
137            .saturating_sub(logs)
138            .saturating_sub(traces)
139            .max(1);
140        self.local_limits.signals.logs.head_memory_bytes_per_stripe = usize_from_u64(logs);
141        self.local_limits
142            .signals
143            .traces
144            .head_memory_bytes_per_stripe = usize_from_u64(traces);
145        self.local_limits
146            .signals
147            .metrics
148            .head_memory_bytes_per_stripe = usize_from_u64(metrics);
149
150        let dictionary_per_stripe = (max_bytes / 12 / stripes).max(1);
151        self.store.stripe.dictionary_cache_bytes = usize_from_u64(dictionary_per_stripe);
152        let charged_heads =
153            stripes.saturating_mul(logs.saturating_add(traces).saturating_add(metrics));
154        let charged_dictionaries = stripes.saturating_mul(dictionary_per_stripe);
155        let cache_total = max_bytes
156            .saturating_sub(charged_heads)
157            .saturating_sub(charged_dictionaries);
158        let control_total = cache_total / 4;
159        self.local_limits.control_cache.memory_bytes = control_total.saturating_mul(3) / 4;
160        self.local_limits.control_cache.parsed_memory_bytes = control_total / 4;
161        self.local_limits.payload_cache.memory_bytes = cache_total.saturating_sub(control_total);
162        self.local_limits.payload_cache.parsed_memory_bytes = 0;
163        self
164    }
165
166    /// Sets the steady-state local data budget and derives cache/group limits.
167    ///
168    /// Local-delete mode reserves three quarters for immutable payloads, one
169    /// eighth for caches, one sixteenth for lifetime rollups, and the remainder
170    /// for catalog/filesystem overhead. S3 mode assigns the remaining managed
171    /// budget to local caches because authoritative immutable objects are remote.
172    #[must_use]
173    pub fn with_max_ssd_bytes(mut self, max_bytes: u64) -> Self {
174        self.max_ssd_bytes = Some(max_bytes);
175        let archive = self.store.s3_object_store.is_some();
176        let rollup_bytes = (max_bytes / 16).max(1);
177        self.local_limits.max_lifetime_rollup_bytes = rollup_bytes;
178        let cache_and_objects = max_bytes.saturating_sub(rollup_bytes);
179        let (control_bytes, payload_cache_bytes, object_bytes) = if archive {
180            let control = (cache_and_objects / 8).max(1);
181            (control, cache_and_objects.saturating_sub(control).max(1), 0)
182        } else {
183            let cache = (max_bytes / 16).max(1);
184            (cache, cache, max_bytes.saturating_mul(3) / 4)
185        };
186        configure_cache_ssd(&mut self.local_limits.control_cache, control_bytes);
187        configure_cache_ssd(&mut self.local_limits.payload_cache, payload_cache_bytes);
188        if archive {
189            self.local_limits.max_object_payload_bytes_per_partition = None;
190        } else {
191            let catalogs = u64::from(self.store.tenant_partitions).saturating_mul(3);
192            let per_partition = (object_bytes / catalogs.max(1)).max(1);
193            self.local_limits.max_object_payload_bytes_per_partition = Some(per_partition);
194            self.object_tier.max_group_payload_bytes =
195                self.object_tier.max_group_payload_bytes.min(per_partition);
196            self.object_tier.target_group_payload_bytes = self
197                .object_tier
198                .target_group_payload_bytes
199                .min((self.object_tier.max_group_payload_bytes / 2).max(1));
200        }
201        self
202    }
203
204    /// Sets both embedded storage-state budgets.
205    #[must_use]
206    pub fn with_storage_budgets(self, max_ram_bytes: u64, max_ssd_bytes: u64) -> Self {
207        self.with_max_ram_bytes(max_ram_bytes)
208            .with_max_ssd_bytes(max_ssd_bytes)
209    }
210
211    /// Replaces the complete bounded local-storage policy.
212    #[must_use]
213    pub fn with_local_limits(mut self, limits: DurableTelemetryLimits) -> Self {
214        self.local_limits = limits;
215        self
216    }
217
218    /// Sets the hot in-memory head limit for logs, traces, and metrics on each
219    /// physical stripe.
220    #[must_use]
221    pub fn with_head_memory_bytes_per_stripe(
222        mut self,
223        logs: usize,
224        traces: usize,
225        metrics: usize,
226    ) -> Self {
227        self.local_limits.signals.logs.head_memory_bytes_per_stripe = logs;
228        self.local_limits
229            .signals
230            .traces
231            .head_memory_bytes_per_stripe = traces;
232        self.local_limits
233            .signals
234            .metrics
235            .head_memory_bytes_per_stripe = metrics;
236        self
237    }
238
239    /// Sets independent SSD-cache policies for metadata/index objects and
240    /// compressed payload ranges.
241    #[must_use]
242    pub fn with_ssd_caches(
243        mut self,
244        control_cache: SsdCacheConfig,
245        payload_cache: SsdCacheConfig,
246    ) -> Self {
247        self.local_limits.control_cache = control_cache;
248        self.local_limits.payload_cache = payload_cache;
249        self
250    }
251
252    /// Sets the bounded shutdown flush timeout.
253    #[must_use]
254    pub const fn with_shutdown_flush_timeout(mut self, timeout: Duration) -> Self {
255        self.shutdown_flush_timeout = timeout;
256        self
257    }
258
259    fn validate(&self) -> Result<(), LokiApiError> {
260        if self.shutdown_flush_timeout.is_zero() {
261            return Err(LokiApiError::configuration(
262                "embedded shutdown_flush_timeout must be nonzero",
263            ));
264        }
265        if self.max_ram_bytes == Some(0) || self.max_ssd_bytes == Some(0) {
266            return Err(LokiApiError::configuration(
267                "embedded RAM and SSD budgets must be nonzero",
268            ));
269        }
270        if let Some(max_bytes) = self.max_ram_bytes {
271            let stripes = u64::from(
272                self.store
273                    .shard_count
274                    .min(self.store.tenant_partitions)
275                    .max(1),
276            );
277            let heads = [
278                self.local_limits.signals.logs.head_memory_bytes_per_stripe,
279                self.local_limits
280                    .signals
281                    .traces
282                    .head_memory_bytes_per_stripe,
283                self.local_limits
284                    .signals
285                    .metrics
286                    .head_memory_bytes_per_stripe,
287            ]
288            .into_iter()
289            .map(|bytes| u64::try_from(bytes).unwrap_or(u64::MAX))
290            .fold(0_u64, u64::saturating_add)
291            .saturating_mul(stripes);
292            let caches = self
293                .local_limits
294                .control_cache
295                .memory_bytes
296                .saturating_add(self.local_limits.control_cache.parsed_memory_bytes)
297                .saturating_add(self.local_limits.payload_cache.memory_bytes)
298                .saturating_add(self.local_limits.payload_cache.parsed_memory_bytes);
299            let dictionaries = u64::try_from(self.store.stripe.dictionary_cache_bytes)
300                .unwrap_or(u64::MAX)
301                .saturating_mul(stripes);
302            if heads.saturating_add(dictionaries).saturating_add(caches) > max_bytes {
303                return Err(LokiApiError::configuration(
304                    "embedded component RAM limits exceed max_ram_bytes",
305                ));
306            }
307        }
308        if let Some(max_bytes) = self.max_ssd_bytes {
309            let caches = self
310                .local_limits
311                .control_cache
312                .max_bytes
313                .saturating_add(self.local_limits.payload_cache.max_bytes);
314            let objects = self
315                .local_limits
316                .max_object_payload_bytes_per_partition
317                .unwrap_or(0)
318                .saturating_mul(u64::from(self.store.tenant_partitions).saturating_mul(3));
319            let rollup = self
320                .local_limits
321                .max_lifetime_rollup_series
322                .map_or(0, |_| self.local_limits.max_lifetime_rollup_bytes);
323            if caches.saturating_add(objects).saturating_add(rollup) > max_bytes {
324                return Err(LokiApiError::configuration(
325                    "embedded component SSD limits exceed max_ssd_bytes",
326                ));
327            }
328        }
329        Ok(())
330    }
331}
332
333/// Observable lifecycle state of [`EmbeddedTelemetryRuntime`].
334#[derive(Debug, Clone, Copy, PartialEq, Eq)]
335#[repr(u8)]
336pub enum EmbeddedTelemetryState {
337    /// The durable store is recovering local state.
338    Recovering = 0,
339    /// Recovery succeeded; exporters may now be attached before traffic starts.
340    AwaitingAttachment = 1,
341    /// Ingestion and query consumers may use the runtime.
342    Ready = 2,
343    /// The runtime is draining and does not accept new attachment work.
344    Draining = 3,
345    /// The durable store has been flushed and closed by its owner.
346    Stopped = 4,
347}
348
349impl EmbeddedTelemetryState {
350    fn from_u8(value: u8) -> Self {
351        match value {
352            1 => Self::AwaitingAttachment,
353            2 => Self::Ready,
354            3 => Self::Draining,
355            4 => Self::Stopped,
356            _ => Self::Recovering,
357        }
358    }
359}
360
361/// One-owner embedded runtime suitable for a local node telemetry deployment.
362///
363/// Opening the runtime performs all WAL, index, catalog, and metric-accumulator
364/// recovery synchronously. Call [`Self::mark_ready`] only after the host has
365/// attached its exporters, then call [`Self::drain`] before dropping the last
366/// runtime reference during shutdown.
367#[derive(Debug)]
368pub struct EmbeddedTelemetryRuntime {
369    store: Arc<DurableTelemetryStore>,
370    shutdown_flush_timeout: Duration,
371    state: AtomicU8,
372    /// Serializes exporter attachment, readiness, and drain transitions. The
373    /// atomic state remains cheap for steady-state ingestion checks, while this
374    /// lock makes lifecycle transitions linearizable.
375    lifecycle_gate: Mutex<()>,
376    /// Coordinates direct in-process producers with shutdown. A producer holds
377    /// a shared lease across the complete durable append; shutdown first marks
378    /// the runtime draining, then takes the exclusive lease before flushing.
379    ingestion_gate: RwLock<()>,
380}
381
382impl EmbeddedTelemetryRuntime {
383    /// Opens and recovers the local durable store. The data directory remains
384    /// exclusively owned until this runtime and all store clones are dropped.
385    pub fn open(config: EmbeddedTelemetryConfig) -> Result<Self, LokiApiError> {
386        config.validate()?;
387        let runtime = Self {
388            store: Arc::new(
389                DurableTelemetryStore::open_with_object_tier_config_and_local_limits(
390                    config.store,
391                    config.object_tier,
392                    config.local_limits,
393                )?,
394            ),
395            shutdown_flush_timeout: config.shutdown_flush_timeout,
396            state: AtomicU8::new(EmbeddedTelemetryState::Recovering as u8),
397            lifecycle_gate: Mutex::new(()),
398            ingestion_gate: RwLock::new(()),
399        };
400        runtime.state.store(
401            EmbeddedTelemetryState::AwaitingAttachment as u8,
402            Ordering::Release,
403        );
404        Ok(runtime)
405    }
406
407    /// Runs an exporter-attachment operation while the runtime is not yet ready.
408    ///
409    /// A failed operation leaves the runtime unavailable, so callers cannot
410    /// accidentally start a partially attached embedded node.
411    pub fn attach<T>(
412        &self,
413        attach: impl FnOnce() -> Result<T, LokiApiError>,
414    ) -> Result<T, LokiApiError> {
415        let _transition = self
416            .lifecycle_gate
417            .lock()
418            .map_err(|_| LokiApiError::internal("embedded telemetry lifecycle gate poisoned"))?;
419        if self.state() != EmbeddedTelemetryState::AwaitingAttachment {
420            return Err(LokiApiError::configuration(
421                "embedded telemetry accepts exporter attachment only before readiness",
422            ));
423        }
424        attach()
425    }
426
427    /// Marks the recovered, fully attached runtime ready for host traffic.
428    pub fn mark_ready(&self) -> Result<(), LokiApiError> {
429        let _transition = self
430            .lifecycle_gate
431            .lock()
432            .map_err(|_| LokiApiError::internal("embedded telemetry lifecycle gate poisoned"))?;
433        self.state
434            .compare_exchange(
435                EmbeddedTelemetryState::AwaitingAttachment as u8,
436                EmbeddedTelemetryState::Ready as u8,
437                Ordering::AcqRel,
438                Ordering::Acquire,
439            )
440            .map_err(|_| {
441                LokiApiError::configuration("embedded telemetry is not awaiting attachment")
442            })?;
443        Ok(())
444    }
445
446    /// Returns the current lifecycle state.
447    #[must_use]
448    pub fn state(&self) -> EmbeddedTelemetryState {
449        EmbeddedTelemetryState::from_u8(self.state.load(Ordering::Acquire))
450    }
451
452    /// Queries the recovered embedded log index without exposing a writable
453    /// store handle that could bypass this runtime's drain gate.
454    pub fn query_native(
455        &self,
456        query: &crate::NativeQuery,
457    ) -> Result<Vec<crate::LokiEntry>, LokiApiError> {
458        self.store.query_native(query)
459    }
460
461    /// Queries the recovered embedded trace index without exposing writable
462    /// ingestion APIs outside the lifecycle gate.
463    pub fn query_traces(
464        &self,
465        query: &crate::TraceQuery,
466    ) -> Result<Vec<crate::DurableSpan>, LokiApiError> {
467        self.store.query_traces(query)
468    }
469
470    /// Queries the recovered embedded metric index without exposing writable
471    /// ingestion APIs outside the lifecycle gate.
472    pub fn query_metrics(
473        &self,
474        query: &crate::MetricQuery,
475    ) -> Result<Vec<DurableMetricPoint>, LokiApiError> {
476        self.store.query_metrics(query)
477    }
478
479    /// Returns durable lifetime metric outcomes without reading evicted raw
480    /// records or contacting the configured object tier.
481    pub fn query_lifetime_metric_rollups(
482        &self,
483        tenant: &str,
484        name: Option<&str>,
485    ) -> Result<Vec<crate::LifetimeMetricRollup>, LokiApiError> {
486        self.store.query_lifetime_metric_rollups(tenant, name)
487    }
488
489    /// Directly appends normalized logs to this process's embedded store.
490    ///
491    /// This is an in-process fast path: it performs signal routing and the
492    /// required WAL-envelope encoding, but does not create a native protocol
493    /// frame, serialize a native batch, open a socket, or decode the same data
494    /// again. Call it from a bounded exporter worker, not from an application
495    /// logging call site.
496    pub fn append_log_events(
497        &self,
498        tenant: &str,
499        events: Vec<OtlpLogEvent>,
500        wait_for_index: bool,
501    ) -> Result<NativeTelemetryAppendAck, LokiApiError> {
502        let _lease = self.begin_ingest()?;
503        self.store.append_log_events(tenant, events, wait_for_index)
504    }
505
506    /// Directly appends normalized spans to this process's embedded store.
507    ///
508    /// The spans have already passed OTLP validation. This path routes them
509    /// by trace identity and writes their durable envelopes without a local
510    /// protocol round trip. Invoke it from a bounded exporter worker rather
511    /// than an application tracing call site.
512    pub fn append_trace_events(
513        &self,
514        events: Vec<OtlpSpanEvent>,
515        wait_for_index: bool,
516    ) -> Result<NativeTelemetryAppendAck, LokiApiError> {
517        let _lease = self.begin_ingest()?;
518        self.store.append_trace_events(events, wait_for_index)
519    }
520
521    /// Directly appends normalized metrics to this process's embedded store.
522    ///
523    /// The metric points retain their native labels, histograms, resource, and
524    /// scope identity without an OTLP or native-protocol round trip. It is the
525    /// intended target for periodic `fast-telemetry` snapshot export; counter
526    /// and histogram updates themselves must remain independent of this call.
527    pub fn append_metric_point(
528        &self,
529        point: DurableMetricPoint,
530        wait_for_index: bool,
531    ) -> Result<NativeTelemetryAppendAck, LokiApiError> {
532        let _lease = self.begin_ingest()?;
533        self.store.append_metric_point(point, wait_for_index)
534    }
535
536    /// Directly appends normalized metrics to this process's embedded store.
537    ///
538    /// The metric points retain their native labels, histograms, resource, and
539    /// scope identity without an OTLP or native-protocol round trip. It is the
540    /// intended target for periodic `fast-telemetry` snapshot export; counter
541    /// and histogram updates themselves must remain independent of this call.
542    pub fn append_metric_points(
543        &self,
544        points: Vec<DurableMetricPoint>,
545        wait_for_index: bool,
546    ) -> Result<NativeTelemetryAppendAck, LokiApiError> {
547        let _lease = self.begin_ingest()?;
548        self.store.append_metric_points(points, wait_for_index)
549    }
550
551    /// Flushes pending local blocks and reclaims complete WAL/object groups
552    /// older than the configured retention window.
553    ///
554    /// Logical queries enforce the cutoff immediately. Hosts should call this
555    /// periodically (for example, once per minute for a 5–60 minute window)
556    /// so physical SSD usage tracks that logical window.
557    pub fn compact_retention(&self) -> Result<RetentionReport, LokiApiError> {
558        let _lease = self.begin_ingest()?;
559        self.store.compact_retention()
560    }
561
562    /// Stops new host work and flushes every current durable boundary.
563    ///
564    /// The caller remains responsible for stopping its producers before this
565    /// method, because producer ownership is intentionally outside this generic
566    /// runtime.
567    pub fn drain(&self) -> Result<(), LokiApiError> {
568        let _transition = self
569            .lifecycle_gate
570            .lock()
571            .map_err(|_| LokiApiError::internal("embedded telemetry lifecycle gate poisoned"))?;
572        let previous = self
573            .state
574            .swap(EmbeddedTelemetryState::Draining as u8, Ordering::AcqRel);
575        if EmbeddedTelemetryState::from_u8(previous) == EmbeddedTelemetryState::Stopped {
576            return Ok(());
577        }
578        let _ingestion = self
579            .ingestion_gate
580            .write()
581            .map_err(|_| LokiApiError::internal("embedded telemetry ingestion gate poisoned"))?;
582        LokiStore::flush(self.store.as_ref(), self.shutdown_flush_timeout)?;
583        self.state
584            .store(EmbeddedTelemetryState::Stopped as u8, Ordering::Release);
585        Ok(())
586    }
587
588    fn begin_ingest(&self) -> Result<std::sync::RwLockReadGuard<'_, ()>, LokiApiError> {
589        let lease = self
590            .ingestion_gate
591            .read()
592            .map_err(|_| LokiApiError::internal("embedded telemetry ingestion gate poisoned"))?;
593        if self.state() != EmbeddedTelemetryState::Ready {
594            return Err(LokiApiError::configuration(
595                "embedded telemetry is not ready to accept direct ingestion",
596            ));
597        }
598        Ok(lease)
599    }
600}
601
602#[cfg(test)]
603mod tests {
604    use std::collections::BTreeMap;
605    use std::time::{SystemTime, UNIX_EPOCH};
606
607    use opentelemetry_proto::tonic::{
608        collector::trace::v1::ExportTraceServiceRequest,
609        trace::v1::{ResourceSpans, ScopeSpans, Span},
610    };
611    use prost::Message;
612
613    use super::*;
614    use crate::{
615        NativeQuery, NativeQueryDirection, OtlpTelemetryDecoder, ResourceContext, StripeConfig,
616        TelemetryValue, TraceId, TraceQuery,
617    };
618
619    #[test]
620    fn embedded_budget_builder_bounds_local_and_s3_managed_storage() {
621        let local = EmbeddedTelemetryConfig::bounded_local(
622            std::env::temp_dir().join("shard-telemetry-unused-budget-config"),
623            Duration::from_secs(5 * 60),
624        )
625        .with_storage_budgets(256 * 1024 * 1024, 1024 * 1024 * 1024);
626        local.validate().expect("local budgets validate");
627        assert!(
628            local
629                .local_limits
630                .max_object_payload_bytes_per_partition
631                .is_some()
632        );
633        assert_eq!(local.object_tier.retirement_grace, Duration::ZERO);
634
635        let archived = EmbeddedTelemetryConfig::bounded(
636            std::env::temp_dir().join("shard-telemetry-unused-s3-budget-config"),
637            Duration::from_secs(60 * 60),
638            EmbeddedEvictionPolicy::OffloadToS3(S3ObjectStoreConfig {
639                bucket: "telemetry-archive".into(),
640                prefix: "device-a".into(),
641                region: Some("us-east-1".into()),
642                endpoint: None,
643                allow_http: false,
644                virtual_hosted_style: false,
645            }),
646        )
647        .with_storage_budgets(256 * 1024 * 1024, 1024 * 1024 * 1024);
648        archived.validate().expect("S3 budgets validate");
649        assert!(archived.store.object_store_directory.is_none());
650        assert!(archived.store.s3_object_store.is_some());
651        assert!(
652            archived
653                .local_limits
654                .max_object_payload_bytes_per_partition
655                .is_none()
656        );
657        assert_eq!(
658            archived.local_limits.control_cache.max_bytes
659                + archived.local_limits.payload_cache.max_bytes
660                + archived.local_limits.max_lifetime_rollup_bytes,
661            1024 * 1024 * 1024
662        );
663    }
664
665    #[test]
666    fn direct_embedded_log_path_is_lifecycle_checked_and_avoids_the_native_listener() {
667        let nonce = SystemTime::now()
668            .duration_since(UNIX_EPOCH)
669            .expect("clock")
670            .as_nanos();
671        let directory = std::env::temp_dir().join(format!(
672            "shard-telemetry-embedded-direct-{}-{nonce}",
673            std::process::id()
674        ));
675        let runtime =
676            EmbeddedTelemetryRuntime::open(EmbeddedTelemetryConfig::new(DurableTelemetryConfig {
677                data_directory: directory.clone(),
678                object_store_directory: None,
679                s3_object_store: None,
680                recovery_journal: false,
681                retention: None,
682                shard_count: 1,
683                tenant_partitions: 1,
684                append_linger: Duration::from_micros(250),
685                stripe: StripeConfig::default(),
686                indexed_ack_timeout: Duration::from_secs(30),
687            }))
688            .expect("runtime opens");
689        let event = OtlpLogEvent {
690            timestamp_unix_nanos: 42,
691            message: Arc::from("embedded fast path"),
692            body: Some(TelemetryValue::String(Arc::from("embedded fast path"))),
693            resource: Arc::new(ResourceContext::default()),
694            ..OtlpLogEvent::default()
695        };
696
697        assert!(
698            runtime
699                .append_log_events("tenant-a", vec![event.clone()], true)
700                .is_err()
701        );
702        runtime.mark_ready().expect("runtime ready");
703        let acknowledgement = runtime
704            .append_log_events("tenant-a", vec![event.clone()], true)
705            .expect("direct append");
706        assert_eq!(acknowledgement.partitions.len(), 1);
707        let logs = runtime
708            .query_native(&NativeQuery {
709                tenant: "tenant-a".to_owned(),
710                labels: BTreeMap::new(),
711                terms: vec!["fast".to_owned()],
712                start_timestamp_unix_nanos: None,
713                end_timestamp_unix_nanos: None,
714                limit: 10,
715                direction: NativeQueryDirection::OldestFirst,
716            })
717            .expect("query");
718        assert_eq!(logs.len(), 1);
719        assert_eq!(logs[0].line, "embedded fast path");
720
721        let trace_id = TraceId::from_bytes([7; 16]).expect("trace ID");
722        let trace_request = ExportTraceServiceRequest {
723            resource_spans: vec![ResourceSpans {
724                scope_spans: vec![ScopeSpans {
725                    spans: vec![Span {
726                        trace_id: trace_id.as_bytes().to_vec(),
727                        span_id: vec![8; 8],
728                        name: "embedded direct span".into(),
729                        start_time_unix_nano: 50,
730                        end_time_unix_nano: 60,
731                        ..Span::default()
732                    }],
733                    ..ScopeSpans::default()
734                }],
735                ..ResourceSpans::default()
736            }],
737        };
738        let spans = OtlpTelemetryDecoder
739            .decode_traces("tenant-a", &trace_request.encode_to_vec())
740            .expect("spans decode");
741        let acknowledgement = runtime
742            .append_trace_events(spans, true)
743            .expect("direct trace append");
744        assert_eq!(acknowledgement.partitions.len(), 1);
745        let traces = runtime
746            .query_traces(&TraceQuery {
747                tenant: Arc::from("tenant-a"),
748                trace_id: Some(trace_id),
749                limit: 10,
750                ..TraceQuery::default()
751            })
752            .expect("trace query");
753        assert_eq!(traces.len(), 1);
754        assert_eq!(traces[0].name.as_ref(), "embedded direct span");
755
756        runtime.drain().expect("drain");
757        assert!(
758            runtime
759                .append_log_events("tenant-a", vec![event], true)
760                .is_err()
761        );
762        drop(runtime);
763        std::fs::remove_dir_all(directory).expect("cleanup");
764    }
765}