Skip to main content

shard_telemetry/
offload.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::fmt;
3use std::fs::{self, File, OpenOptions};
4use std::future::Future;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7use std::sync::{Arc, Mutex};
8use std::time::Duration;
9
10use fs2::FileExt;
11use futures_util::stream::{FuturesUnordered, StreamExt};
12use serde::{Deserialize, Serialize};
13use shard_stream_core::{LogicalOffset, LogicalPartitionId, TopicId, TopicPartition};
14use tokio::sync::Mutex as AsyncMutex;
15
16use crate::{
17    DurableTelemetryStore, NativePartitionAppend, NativeTelemetryBatch, ShardTelemetryClient,
18    TelemetryEnvelope, TelemetrySignal, decode_metric_chunk,
19};
20
21const OFFLOAD_CHECKPOINT_VERSION: u8 = 3;
22const DEFAULT_MAX_FETCH_BYTES: u32 = 16 * 1024 * 1024;
23const DEFAULT_MAX_IN_FLIGHT_PARTITIONS: usize = 1;
24const DEFAULT_IDLE_INTERVAL: Duration = Duration::from_secs(1);
25const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(5);
26const DEFAULT_MAX_CONSECUTIVE_FAILURES: usize = 8;
27
28/// Durable state for one upstream-offloaded local WAL partition.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub struct OffloadCheckpoint {
31    /// Local source signal partition.
32    pub topic_partition: TopicPartition,
33    /// First source offset not yet acknowledged by the upstream server.
34    pub next_offset: LogicalOffset,
35}
36
37/// Configuration for a generic background upstream offloader.
38#[derive(Debug, Clone)]
39pub struct UpstreamOffloadConfig {
40    /// Crash-safe local progress journal. It should be placed below the local
41    /// telemetry data directory, not on an ephemeral filesystem.
42    pub checkpoint_path: PathBuf,
43    /// Stable identity of the node-local WAL source.
44    ///
45    /// This value scopes deterministic native retry IDs, so it must remain
46    /// unchanged across restarts while `checkpoint_path` is retained. Use a
47    /// node UUID or another deployment identity that is unique among all
48    /// nodes forwarding to the same central ShardTelemetry service.
49    pub source_id: Arc<str>,
50    /// Maximum source WAL bytes fetched in one offload round per partition.
51    pub max_fetch_bytes: u32,
52    /// Maximum independent source partitions advancing concurrently.
53    ///
54    /// Order is always preserved within one physical source partition. This
55    /// bound only permits separate partitions to fetch and append in parallel.
56    pub max_in_flight_partitions: usize,
57    /// Signals eligible for forwarding to the upstream service.
58    ///
59    /// Local retention remains independent of this selection. For example, an
60    /// embedded node can retain logs and traces locally while forwarding only
61    /// metrics to its regional ShardTelemetry deployment.
62    pub signals: BTreeSet<TelemetrySignal>,
63    /// Optional allow-list of metric names eligible for upstream forwarding.
64    ///
65    /// Metric envelopes are forwarded unchanged, so every metric point in an
66    /// eligible envelope must match this allow-list. The direct embedded path
67    /// writes one canonical series per envelope; a mixed envelope fails closed
68    /// rather than silently forwarding an unselected metric.
69    pub metric_names: Option<BTreeSet<Arc<str>>>,
70}
71
72impl UpstreamOffloadConfig {
73    /// Creates a bounded offload configuration.
74    #[must_use]
75    pub fn new(checkpoint_path: PathBuf, source_id: impl Into<Arc<str>>) -> Self {
76        Self {
77            checkpoint_path,
78            source_id: source_id.into(),
79            max_fetch_bytes: DEFAULT_MAX_FETCH_BYTES,
80            max_in_flight_partitions: DEFAULT_MAX_IN_FLIGHT_PARTITIONS,
81            signals: [
82                TelemetrySignal::Logs,
83                TelemetrySignal::Traces,
84                TelemetrySignal::Metrics,
85            ]
86            .into_iter()
87            .collect(),
88            metric_names: None,
89        }
90    }
91
92    /// Changes the maximum bytes read from one source partition in a round.
93    #[must_use]
94    pub const fn with_max_fetch_bytes(mut self, max_fetch_bytes: u32) -> Self {
95        self.max_fetch_bytes = max_fetch_bytes;
96        self
97    }
98
99    /// Changes the maximum number of independent source partitions processed
100    /// in parallel. Set this no higher than the upstream client's configured
101    /// connection pool for network parallelism.
102    #[must_use]
103    pub const fn with_max_in_flight_partitions(mut self, max_in_flight_partitions: usize) -> Self {
104        self.max_in_flight_partitions = max_in_flight_partitions;
105        self
106    }
107
108    /// Replaces the set of signal types eligible for upstream forwarding.
109    ///
110    /// The selected set must not be empty. Use separate embedded stores when
111    /// labeled-series routing policies require independent failure domains or
112    /// retention windows.
113    #[must_use]
114    pub fn with_signals(mut self, signals: impl IntoIterator<Item = TelemetrySignal>) -> Self {
115        self.signals = signals.into_iter().collect();
116        self
117    }
118
119    /// Forwards only metric envelopes whose metric name is in `metric_names`.
120    ///
121    /// The filter applies only when metrics are selected by [`Self::with_signals`].
122    /// It does not alter local ingestion, query visibility, retention, or
123    /// object-tier offload. Use separate local stores for policies that need to
124    /// route one labeled series differently from another series of the same
125    /// metric name.
126    #[must_use]
127    pub fn with_metric_names<I, S>(mut self, metric_names: I) -> Self
128    where
129        I: IntoIterator<Item = S>,
130        S: Into<Arc<str>>,
131    {
132        self.metric_names = Some(metric_names.into_iter().map(Into::into).collect());
133        self
134    }
135
136    fn validate(&self) -> Result<(), OffloadError> {
137        if self.checkpoint_path.as_os_str().is_empty() {
138            return Err(OffloadError::new(
139                "upstream offload checkpoint path must not be empty",
140            ));
141        }
142        if self.source_id.is_empty() {
143            return Err(OffloadError::new(
144                "upstream offload source ID must not be empty",
145            ));
146        }
147        if self.max_fetch_bytes == 0 {
148            return Err(OffloadError::new(
149                "upstream offload max_fetch_bytes must be nonzero",
150            ));
151        }
152        if self.max_in_flight_partitions == 0 {
153            return Err(OffloadError::new(
154                "upstream offload max_in_flight_partitions must be nonzero",
155            ));
156        }
157        if self.signals.is_empty() {
158            return Err(OffloadError::new(
159                "upstream offload must select at least one telemetry signal",
160            ));
161        }
162        if let Some(metric_names) = &self.metric_names {
163            if metric_names.is_empty() {
164                return Err(OffloadError::new(
165                    "upstream metric offload name filter must not be empty",
166                ));
167            }
168            if !self.signals.contains(&TelemetrySignal::Metrics) {
169                return Err(OffloadError::new(
170                    "upstream metric offload name filter requires metrics to be selected",
171                ));
172            }
173            if metric_names.iter().any(|name| name.is_empty()) {
174                return Err(OffloadError::new(
175                    "upstream metric offload names must not be empty",
176                ));
177            }
178        }
179        Ok(())
180    }
181}
182
183/// Scheduling and retry bounds for [`UpstreamOffloader::run_until`].
184#[derive(Debug, Clone, Copy)]
185pub struct UpstreamOffloadLoopConfig {
186    /// Wait after a successful round which found no local work.
187    pub idle_interval: Duration,
188    /// Wait after a retryable local fetch or upstream append failure.
189    pub retry_interval: Duration,
190    /// Consecutive failed rounds allowed before the worker returns an error.
191    pub max_consecutive_failures: usize,
192}
193
194impl UpstreamOffloadLoopConfig {
195    /// Creates a bounded background-loop configuration.
196    #[must_use]
197    pub const fn new() -> Self {
198        Self {
199            idle_interval: DEFAULT_IDLE_INTERVAL,
200            retry_interval: DEFAULT_RETRY_INTERVAL,
201            max_consecutive_failures: DEFAULT_MAX_CONSECUTIVE_FAILURES,
202        }
203    }
204
205    /// Changes the wait used when the local WAL has no eligible data.
206    #[must_use]
207    pub const fn with_idle_interval(mut self, idle_interval: Duration) -> Self {
208        self.idle_interval = idle_interval;
209        self
210    }
211
212    /// Changes the delay before a failed offload round is retried.
213    #[must_use]
214    pub const fn with_retry_interval(mut self, retry_interval: Duration) -> Self {
215        self.retry_interval = retry_interval;
216        self
217    }
218
219    /// Changes the maximum number of consecutive retryable failures.
220    #[must_use]
221    pub const fn with_max_consecutive_failures(mut self, max_consecutive_failures: usize) -> Self {
222        self.max_consecutive_failures = max_consecutive_failures;
223        self
224    }
225
226    fn validate(self) -> Result<(), OffloadError> {
227        if self.idle_interval.is_zero() || self.retry_interval.is_zero() {
228            return Err(OffloadError::new(
229                "upstream offload loop intervals must be nonzero",
230            ));
231        }
232        if self.max_consecutive_failures == 0 {
233            return Err(OffloadError::new(
234                "upstream offload max_consecutive_failures must be nonzero",
235            ));
236        }
237        Ok(())
238    }
239}
240
241impl Default for UpstreamOffloadLoopConfig {
242    fn default() -> Self {
243        Self::new()
244    }
245}
246
247/// Per-round store-and-forward outcome.
248#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
249pub struct OffloadReport {
250    /// Source partitions inspected during the round.
251    pub scanned_partitions: usize,
252    /// Local durable WAL batches acknowledged by the upstream service.
253    pub offloaded_batches: usize,
254    /// Local durable records covered by the acknowledged batches.
255    pub offloaded_records: u64,
256    /// Source offsets advanced in the persisted checkpoint journal.
257    pub advanced_offsets: u64,
258    /// Durable checkpoint-journal writes completed during the round.
259    pub checkpoint_writes: usize,
260    /// Batches deliberately retained locally by the configured metric filter.
261    pub skipped_batches: usize,
262    /// Records deliberately retained locally by the configured metric filter.
263    pub skipped_records: u64,
264}
265
266/// Cumulative outcome returned when a background offload loop is stopped.
267#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
268pub struct OffloadLoopReport {
269    /// Successful calls to [`UpstreamOffloader::offload_once`].
270    pub completed_rounds: u64,
271    /// Failed offload rounds retried before shutdown.
272    pub failed_rounds: u64,
273    /// Source partitions inspected across successful rounds.
274    pub scanned_partitions: u64,
275    /// Local durable WAL batches acknowledged by the upstream service.
276    pub offloaded_batches: u64,
277    /// Local durable records covered by acknowledged batches.
278    pub offloaded_records: u64,
279    /// Source offsets advanced in the persisted checkpoint journal.
280    pub advanced_offsets: u64,
281    /// Durable checkpoint-journal writes completed across successful rounds.
282    pub checkpoint_writes: u64,
283    /// Batches deliberately retained locally by the configured metric filter.
284    pub skipped_batches: u64,
285    /// Records deliberately retained locally by the configured metric filter.
286    pub skipped_records: u64,
287}
288
289impl OffloadLoopReport {
290    fn record(&mut self, report: OffloadReport) {
291        self.completed_rounds = self.completed_rounds.saturating_add(1);
292        self.scanned_partitions = self
293            .scanned_partitions
294            .saturating_add(u64::try_from(report.scanned_partitions).unwrap_or(u64::MAX));
295        self.offloaded_batches = self
296            .offloaded_batches
297            .saturating_add(u64::try_from(report.offloaded_batches).unwrap_or(u64::MAX));
298        self.offloaded_records = self
299            .offloaded_records
300            .saturating_add(report.offloaded_records);
301        self.advanced_offsets = self
302            .advanced_offsets
303            .saturating_add(report.advanced_offsets);
304        self.checkpoint_writes = self
305            .checkpoint_writes
306            .saturating_add(u64::try_from(report.checkpoint_writes).unwrap_or(u64::MAX));
307        self.skipped_batches = self
308            .skipped_batches
309            .saturating_add(u64::try_from(report.skipped_batches).unwrap_or(u64::MAX));
310        self.skipped_records = self.skipped_records.saturating_add(report.skipped_records);
311    }
312}
313
314/// Error returned by the background store-and-forward pipeline.
315#[derive(Debug, Clone, PartialEq, Eq)]
316pub struct OffloadError {
317    message: Arc<str>,
318}
319
320impl OffloadError {
321    fn new(message: impl Into<Arc<str>>) -> Self {
322        Self {
323            message: message.into(),
324        }
325    }
326}
327
328impl fmt::Display for OffloadError {
329    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
330        formatter.write_str(&self.message)
331    }
332}
333
334impl std::error::Error for OffloadError {}
335
336#[derive(Debug, Clone, Serialize, Deserialize)]
337struct PersistedCheckpoint {
338    topic_id: u128,
339    partition_id: u32,
340    next_offset: u64,
341}
342
343#[derive(Debug, Clone, Serialize, Deserialize)]
344struct PersistedCheckpoints {
345    version: u8,
346    #[serde(default)]
347    source_id: Option<String>,
348    #[serde(default)]
349    retry_namespace: Option<String>,
350    checkpoints: Vec<PersistedCheckpoint>,
351}
352
353#[derive(Debug, Clone)]
354enum RetryNamespace {
355    LegacyV1,
356    Source(Arc<str>),
357}
358
359#[derive(Debug)]
360struct CheckpointJournal {
361    path: PathBuf,
362    source_id: Arc<str>,
363    retry_namespace: RetryNamespace,
364    offsets: BTreeMap<TopicPartition, LogicalOffset>,
365    // Held for the lifetime of the offloader so one checkpoint path has one
366    // owner across processes and across independently constructed workers.
367    _exclusive_lock: File,
368}
369
370impl CheckpointJournal {
371    fn open(path: PathBuf, source_id: Arc<str>) -> Result<Self, OffloadError> {
372        if let Some(parent) = path.parent() {
373            fs::create_dir_all(parent).map_err(journal_io_error)?;
374        }
375        let lock = OpenOptions::new()
376            .create(true)
377            .read(true)
378            .write(true)
379            .truncate(false)
380            .open(path.with_extension("lock"))
381            .map_err(journal_io_error)?;
382        lock.try_lock_exclusive().map_err(|error| {
383            OffloadError::new(format!(
384                "upstream offload checkpoint is already owned by another worker: {error}"
385            ))
386        })?;
387        let persisted = match fs::read(&path) {
388            Ok(encoded) => {
389                serde_json::from_slice::<PersistedCheckpoints>(&encoded).map_err(|error| {
390                    OffloadError::new(format!(
391                        "invalid upstream offload checkpoint journal: {error}"
392                    ))
393                })?
394            }
395            Err(error) if error.kind() == std::io::ErrorKind::NotFound => PersistedCheckpoints {
396                version: OFFLOAD_CHECKPOINT_VERSION,
397                source_id: Some(source_id.to_string()),
398                retry_namespace: Some(source_retry_namespace(&source_id)),
399                checkpoints: Vec::new(),
400            },
401            Err(error) => return Err(journal_io_error(error)),
402        };
403        let retry_namespace = match persisted.version {
404            1 => RetryNamespace::LegacyV1,
405            2 => {
406                validate_persisted_source(persisted.source_id.as_deref(), &source_id)?;
407                RetryNamespace::Source(Arc::clone(&source_id))
408            }
409            OFFLOAD_CHECKPOINT_VERSION => {
410                validate_persisted_source(persisted.source_id.as_deref(), &source_id)?;
411                match persisted.retry_namespace.as_deref() {
412                    Some("legacy-v1") => RetryNamespace::LegacyV1,
413                    Some(namespace) if namespace == source_retry_namespace(&source_id) => {
414                        RetryNamespace::Source(Arc::clone(&source_id))
415                    }
416                    _ => {
417                        return Err(OffloadError::new(
418                            "upstream offload checkpoint journal retry namespace is invalid",
419                        ));
420                    }
421                }
422            }
423            version => {
424                return Err(OffloadError::new(format!(
425                    "unsupported upstream offload checkpoint journal version {version}",
426                )));
427            }
428        };
429        let mut offsets = BTreeMap::new();
430        for checkpoint in persisted.checkpoints {
431            let topic_partition = TopicPartition::new(
432                TopicId::new(checkpoint.topic_id),
433                LogicalPartitionId::new(checkpoint.partition_id),
434            );
435            if offsets
436                .insert(topic_partition, LogicalOffset::new(checkpoint.next_offset))
437                .is_some()
438            {
439                return Err(OffloadError::new(
440                    "upstream offload checkpoint journal contains duplicate partitions",
441                ));
442            }
443        }
444        Ok(Self {
445            path,
446            source_id,
447            retry_namespace,
448            offsets,
449            _exclusive_lock: lock,
450        })
451    }
452
453    fn next(&self, partition: TopicPartition) -> Option<LogicalOffset> {
454        self.offsets.get(&partition).copied()
455    }
456
457    fn advance(
458        &mut self,
459        partition: TopicPartition,
460        next_offset: LogicalOffset,
461    ) -> Result<(), OffloadError> {
462        let previous = self.offsets.insert(partition, next_offset);
463        if previous.is_some_and(|previous| previous > next_offset) {
464            return Err(OffloadError::new(
465                "upstream offload checkpoint attempted to move backward",
466            ));
467        }
468        self.persist()
469    }
470
471    fn snapshot(&self) -> Vec<OffloadCheckpoint> {
472        self.offsets
473            .iter()
474            .map(|(topic_partition, next_offset)| OffloadCheckpoint {
475                topic_partition: *topic_partition,
476                next_offset: *next_offset,
477            })
478            .collect()
479    }
480
481    fn retry_namespace(&self) -> RetryNamespace {
482        self.retry_namespace.clone()
483    }
484
485    fn persist(&self) -> Result<(), OffloadError> {
486        let encoded = serde_json::to_vec(&PersistedCheckpoints {
487            version: OFFLOAD_CHECKPOINT_VERSION,
488            source_id: Some(self.source_id.to_string()),
489            retry_namespace: Some(match &self.retry_namespace {
490                RetryNamespace::LegacyV1 => "legacy-v1".to_owned(),
491                RetryNamespace::Source(source_id) => source_retry_namespace(source_id),
492            }),
493            checkpoints: self
494                .offsets
495                .iter()
496                .map(|(topic_partition, next_offset)| PersistedCheckpoint {
497                    topic_id: topic_partition.topic_id.get(),
498                    partition_id: topic_partition.partition_id.get(),
499                    next_offset: next_offset.get(),
500                })
501                .collect(),
502        })
503        .map_err(|error| {
504            OffloadError::new(format!(
505                "upstream offload checkpoint serialization failed: {error}"
506            ))
507        })?;
508        let temporary = temporary_path(&self.path);
509        let mut file = OpenOptions::new()
510            .create(true)
511            .truncate(true)
512            .write(true)
513            .open(&temporary)
514            .map_err(journal_io_error)?;
515        file.write_all(&encoded)
516            .and_then(|()| file.sync_all())
517            .map_err(journal_io_error)?;
518        fs::rename(&temporary, &self.path).map_err(journal_io_error)?;
519        File::open(self.path.parent().unwrap_or_else(|| Path::new(".")))
520            .and_then(|directory| directory.sync_all())
521            .map_err(journal_io_error)
522    }
523}
524
525fn validate_persisted_source(
526    persisted_source: Option<&str>,
527    configured_source: &str,
528) -> Result<(), OffloadError> {
529    match persisted_source {
530        Some(source) if source == configured_source => Ok(()),
531        Some(_) => Err(OffloadError::new(
532            "upstream offload checkpoint journal source ID does not match configuration",
533        )),
534        None => Err(OffloadError::new(
535            "upstream offload checkpoint journal is missing source ID",
536        )),
537    }
538}
539
540fn source_retry_namespace(source_id: &str) -> String {
541    format!("source:{source_id}")
542}
543
544/// Generic background store-and-forward uploader for an embedded local store.
545///
546/// Call [`Self::run_until`] from a dedicated background task, or use
547/// [`Self::offload_once`] when the host owns its own scheduler. The local WAL
548/// stays authoritative: a checkpoint moves only after the upstream native
549/// server has acknowledged the corresponding envelope. Source-byte reclamation
550/// remains controlled by the local store's retention policy, so an offload
551/// outage cannot delete acknowledged local telemetry.
552#[derive(Debug)]
553pub struct UpstreamOffloader {
554    store: Arc<DurableTelemetryStore>,
555    client: Arc<ShardTelemetryClient>,
556    max_fetch_bytes: u32,
557    max_in_flight_partitions: usize,
558    signals: BTreeSet<TelemetrySignal>,
559    metric_names: Option<BTreeSet<Arc<str>>>,
560    checkpoints: Mutex<CheckpointJournal>,
561    run_gate: AsyncMutex<()>,
562}
563
564impl UpstreamOffloader {
565    /// Opens or recovers the progress journal for one local store and upstream client.
566    pub fn open(
567        store: Arc<DurableTelemetryStore>,
568        client: Arc<ShardTelemetryClient>,
569        config: UpstreamOffloadConfig,
570    ) -> Result<Self, OffloadError> {
571        config.validate()?;
572        let UpstreamOffloadConfig {
573            checkpoint_path,
574            source_id,
575            max_fetch_bytes,
576            max_in_flight_partitions,
577            signals,
578            metric_names,
579        } = config;
580        Ok(Self {
581            store,
582            client,
583            max_fetch_bytes,
584            max_in_flight_partitions,
585            signals,
586            metric_names,
587            checkpoints: Mutex::new(CheckpointJournal::open(checkpoint_path, source_id)?),
588            run_gate: AsyncMutex::new(()),
589        })
590    }
591
592    /// Returns the recovered durable progress checkpoints.
593    pub fn checkpoints(&self) -> Result<Vec<OffloadCheckpoint>, OffloadError> {
594        self.checkpoints
595            .lock()
596            .map(|journal| journal.snapshot())
597            .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))
598    }
599
600    /// Offloads at most one bounded WAL fetch from every local signal partition.
601    ///
602    /// One successful fetch advances its source checkpoint with one durable
603    /// journal write. If an append later in that fetch fails, the journal stays
604    /// at its prior boundary and the next run safely reuses deterministic retry
605    /// IDs for every previously acknowledged append in that fetch.
606    pub async fn offload_once(&self) -> Result<OffloadReport, OffloadError> {
607        let _round = self.run_gate.lock().await;
608        let mut report = OffloadReport::default();
609        let mut in_flight = FuturesUnordered::new();
610        let mut first_error = None;
611        for partition in self.store.telemetry_partitions() {
612            if !self
613                .signals
614                .iter()
615                .any(|signal| signal.topic_id() == partition.topic_id)
616            {
617                continue;
618            }
619            in_flight.push(self.offload_partition(partition));
620            if in_flight.len() >= self.max_in_flight_partitions
621                && let Some(result) = in_flight.next().await
622            {
623                record_partition_result(&mut report, &mut first_error, result);
624                if first_error.is_some() {
625                    break;
626                }
627            }
628        }
629        while let Some(result) = in_flight.next().await {
630            record_partition_result(&mut report, &mut first_error, result);
631        }
632        if let Some(error) = first_error {
633            return Err(error);
634        }
635        Ok(report)
636    }
637
638    async fn offload_partition(
639        &self,
640        partition: TopicPartition,
641    ) -> Result<OffloadReport, OffloadError> {
642        let mut report = OffloadReport {
643            scanned_partitions: 1,
644            ..OffloadReport::default()
645        };
646        let start_offset = self.start_offset(partition)?;
647        let store = Arc::clone(&self.store);
648        let max_fetch_bytes = self.max_fetch_bytes;
649        let batches = tokio::task::spawn_blocking(move || {
650            store.fetch_telemetry_batches(partition, start_offset, max_fetch_bytes)
651        })
652        .await
653        .map_err(|error| {
654            OffloadError::new(format!("upstream offload fetch worker failed: {error}"))
655        })?
656        .map_err(|error| OffloadError::new(error.to_string()))?;
657        let mut checkpoint = None;
658        for batch in batches {
659            if batch.topic_partition != partition {
660                return Err(OffloadError::new(
661                    "upstream offload fetch returned a batch from another partition",
662                ));
663            }
664            let record_count = batch
665                .last_offset
666                .get()
667                .saturating_sub(batch.first_offset.get())
668                .saturating_add(1);
669            let should_forward = self.should_forward_envelope(&batch.envelope)?;
670            if should_forward {
671                let native_batch = NativeTelemetryBatch {
672                    partitions: vec![NativePartitionAppend {
673                        topic_partition: batch.topic_partition,
674                        envelope: batch.envelope,
675                        transient_context: None,
676                    }],
677                };
678                let retry_id = retry_id(
679                    self.retry_namespace()?,
680                    batch.topic_partition,
681                    batch.first_offset,
682                    batch.last_offset,
683                    &native_batch,
684                )?;
685                self.client
686                    .append_with_request_id(&native_batch, retry_id)
687                    .await
688                    .map_err(|error| OffloadError::new(error.to_string()))?;
689                report.offloaded_batches = report.offloaded_batches.saturating_add(1);
690                report.offloaded_records = report.offloaded_records.saturating_add(record_count);
691            } else {
692                report.skipped_batches = report.skipped_batches.saturating_add(1);
693                report.skipped_records = report.skipped_records.saturating_add(record_count);
694            }
695            let next = batch
696                .last_offset
697                .get()
698                .checked_add(1)
699                .ok_or_else(|| OffloadError::new("upstream offload source offset exhausted"))?;
700            checkpoint = Some(LogicalOffset::new(next));
701            report.advanced_offsets = report.advanced_offsets.saturating_add(record_count);
702        }
703        if let Some(checkpoint) = checkpoint {
704            self.advance(partition, checkpoint)?;
705            report.checkpoint_writes = report.checkpoint_writes.saturating_add(1);
706        }
707        Ok(report)
708    }
709
710    /// Repeatedly offloads local WAL data until `shutdown` resolves.
711    ///
712    /// Successful rounds with work continue immediately to drain a bounded
713    /// backlog. Idle rounds wait for `idle_interval`; retryable local or
714    /// upstream errors wait for `retry_interval` and stop after the configured
715    /// consecutive-failure limit. Stopping cancels the next round or wait; a
716    /// possibly in-flight native append remains safe because its retry ID is
717    /// deterministic and the checkpoint advances only after acknowledgement.
718    pub async fn run_until<F>(
719        &self,
720        config: UpstreamOffloadLoopConfig,
721        shutdown: F,
722    ) -> Result<OffloadLoopReport, OffloadError>
723    where
724        F: Future<Output = ()> + Send,
725    {
726        config.validate()?;
727        tokio::pin!(shutdown);
728        let mut report = OffloadLoopReport::default();
729        let mut consecutive_failures = 0_usize;
730        loop {
731            let delay = tokio::select! {
732                biased;
733                () = &mut shutdown => return Ok(report),
734                round = self.offload_once() => match round {
735                    Ok(round) => {
736                        let has_backlog = round.advanced_offsets != 0;
737                        report.record(round);
738                        consecutive_failures = 0;
739                        if has_backlog {
740                            None
741                        } else {
742                            Some(config.idle_interval)
743                        }
744                    }
745                    Err(error) => {
746                        report.failed_rounds = report.failed_rounds.saturating_add(1);
747                        consecutive_failures = consecutive_failures.saturating_add(1);
748                        if consecutive_failures >= config.max_consecutive_failures {
749                            return Err(error);
750                        }
751                        Some(config.retry_interval)
752                    }
753                },
754            };
755            let Some(delay) = delay else {
756                continue;
757            };
758            tokio::select! {
759                biased;
760                () = &mut shutdown => return Ok(report),
761                () = tokio::time::sleep(delay) => {}
762            }
763        }
764    }
765
766    fn start_offset(&self, partition: TopicPartition) -> Result<LogicalOffset, OffloadError> {
767        let local_start = self
768            .store
769            .telemetry_partition_start_offset(partition)
770            .map_err(|error| OffloadError::new(error.to_string()))?;
771        let checkpoint = self
772            .checkpoints
773            .lock()
774            .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))?
775            .next(partition);
776        Ok(checkpoint.map_or(local_start, |checkpoint| checkpoint.max(local_start)))
777    }
778
779    fn advance(
780        &self,
781        partition: TopicPartition,
782        next_offset: LogicalOffset,
783    ) -> Result<(), OffloadError> {
784        self.checkpoints
785            .lock()
786            .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))?
787            .advance(partition, next_offset)
788    }
789
790    fn retry_namespace(&self) -> Result<RetryNamespace, OffloadError> {
791        self.checkpoints
792            .lock()
793            .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))
794            .map(|journal| journal.retry_namespace())
795    }
796
797    fn should_forward_envelope(&self, envelope: &TelemetryEnvelope) -> Result<bool, OffloadError> {
798        let Some(metric_names) = &self.metric_names else {
799            return Ok(true);
800        };
801        if envelope.signal != TelemetrySignal::Metrics {
802            return Ok(true);
803        }
804        let points = decode_metric_chunk(&envelope.payload)
805            .map_err(|error| OffloadError::new(error.to_string()))?;
806        if points.len() != envelope.item_count as usize {
807            return Err(OffloadError::new(
808                "metric offload filter decoded a count different from its envelope",
809            ));
810        }
811        let mut selected = false;
812        let mut unselected = false;
813        for point in points {
814            if metric_names.contains(&point.identity.name) {
815                selected = true;
816            } else {
817                unselected = true;
818            }
819        }
820        if selected && unselected {
821            return Err(OffloadError::new(
822                "metric offload filter requires every envelope to have one selection outcome",
823            ));
824        }
825        Ok(selected)
826    }
827}
828
829fn record_partition_result(
830    report: &mut OffloadReport,
831    first_error: &mut Option<OffloadError>,
832    result: Result<OffloadReport, OffloadError>,
833) {
834    match result {
835        Ok(partition) => {
836            report.scanned_partitions = report
837                .scanned_partitions
838                .saturating_add(partition.scanned_partitions);
839            report.offloaded_batches = report
840                .offloaded_batches
841                .saturating_add(partition.offloaded_batches);
842            report.offloaded_records = report
843                .offloaded_records
844                .saturating_add(partition.offloaded_records);
845            report.advanced_offsets = report
846                .advanced_offsets
847                .saturating_add(partition.advanced_offsets);
848            report.checkpoint_writes = report
849                .checkpoint_writes
850                .saturating_add(partition.checkpoint_writes);
851            report.skipped_batches = report
852                .skipped_batches
853                .saturating_add(partition.skipped_batches);
854            report.skipped_records = report
855                .skipped_records
856                .saturating_add(partition.skipped_records);
857        }
858        Err(error) if first_error.is_none() => *first_error = Some(error),
859        Err(_) => {}
860    }
861}
862
863fn retry_id(
864    namespace: RetryNamespace,
865    partition: TopicPartition,
866    first_offset: LogicalOffset,
867    last_offset: LogicalOffset,
868    batch: &NativeTelemetryBatch,
869) -> Result<u128, OffloadError> {
870    let payload = batch
871        .encode_native_append()
872        .map_err(|error| OffloadError::new(error.to_string()))?;
873    let mut hasher = blake3::Hasher::new();
874    hasher.update(b"shard-telemetry-upstream-offload-v1\0");
875    if let RetryNamespace::Source(source_id) = namespace {
876        hasher.update(source_id.as_bytes());
877        hasher.update(&[0]);
878    }
879    hasher.update(&partition.topic_id.get().to_le_bytes());
880    hasher.update(&partition.partition_id.get().to_le_bytes());
881    hasher.update(&first_offset.get().to_le_bytes());
882    hasher.update(&last_offset.get().to_le_bytes());
883    hasher.update(&payload);
884    let digest = hasher.finalize();
885    Ok(u128::from_le_bytes(
886        digest.as_bytes()[..16]
887            .try_into()
888            .expect("BLAKE3 digest contains sixteen bytes"),
889    ))
890}
891
892fn temporary_path(path: &Path) -> PathBuf {
893    let mut name = path.file_name().unwrap_or_default().to_os_string();
894    name.push(".tmp");
895    path.with_file_name(name)
896}
897
898fn journal_io_error(error: std::io::Error) -> OffloadError {
899    OffloadError::new(format!(
900        "upstream offload checkpoint journal I/O failed: {error}"
901    ))
902}
903
904#[cfg(test)]
905mod tests {
906    use std::collections::BTreeMap;
907    use std::time::{Duration, SystemTime, UNIX_EPOCH};
908
909    use opentelemetry_proto::tonic::{
910        collector::metrics::v1::ExportMetricsServiceRequest,
911        metrics::v1::{
912            Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric,
913            number_data_point,
914        },
915    };
916    use prost::Message;
917    use shard_stream_core::{LogicalOffset, LogicalPartitionId, ShardId, TopicId, TopicPartition};
918    use tokio::net::TcpListener;
919
920    use super::*;
921    use crate::{
922        DurableTelemetryConfig, LokiEntry, LokiStore, METRICS_TOPIC_ID, MetricQuery,
923        NativeClientConfig, NativeServerConfig, OtlpTelemetryDecoder, StripeConfig, serve_native,
924    };
925
926    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
927    async fn upstream_offload_persists_progress_only_after_native_acknowledgement() {
928        let nonce = SystemTime::now()
929            .duration_since(UNIX_EPOCH)
930            .expect("clock")
931            .as_nanos();
932        let root = std::env::temp_dir().join(format!(
933            "shard-telemetry-upstream-offload-{}-{nonce}",
934            std::process::id()
935        ));
936        let source_directory = root.join("source");
937        let destination_directory = root.join("destination");
938        let config = |data_directory| DurableTelemetryConfig {
939            data_directory,
940            object_store_directory: None,
941            s3_object_store: None,
942            recovery_journal: false,
943            retention: None,
944            shard_count: 1,
945            tenant_partitions: 1,
946            append_linger: Duration::from_micros(250),
947            stripe: StripeConfig::default(),
948            indexed_ack_timeout: Duration::from_secs(30),
949        };
950        let source = Arc::new(
951            DurableTelemetryStore::open(config(source_directory.clone())).expect("source"),
952        );
953        let destination = Arc::new(
954            DurableTelemetryStore::open(config(destination_directory)).expect("destination"),
955        );
956        let expected = LokiEntry {
957            timestamp_unix_nanos: 100,
958            labels: BTreeMap::from([("node".to_owned(), "node-a".to_owned())]),
959            line: "store and forward".to_owned(),
960            structured_metadata: BTreeMap::new(),
961        };
962        LokiStore::push(source.as_ref(), "tenant-a", vec![expected.clone()])
963            .expect("source append");
964        let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
965        let address = listener.local_addr().expect("address");
966        let (stop, stopped) = tokio::sync::oneshot::channel();
967        let server_destination = Arc::clone(&destination);
968        let server = tokio::spawn(async move {
969            serve_native(
970                listener,
971                server_destination,
972                NativeServerConfig::default(),
973                async {
974                    let _ = stopped.await;
975                },
976            )
977            .await
978        });
979        let client =
980            Arc::new(ShardTelemetryClient::new(NativeClientConfig::new(address)).expect("client"));
981        let metrics_only = UpstreamOffloader::open(
982            Arc::clone(&source),
983            Arc::clone(&client),
984            UpstreamOffloadConfig::new(
985                source_directory.join("metrics-only-offload-v1.json"),
986                "test-node-a",
987            )
988            .with_signals([TelemetrySignal::Metrics]),
989        )
990        .expect("metrics-only offloader");
991        let skipped = metrics_only.offload_once().await.expect("filtered offload");
992        assert_eq!(skipped.scanned_partitions, 1);
993        assert_eq!(skipped.offloaded_batches, 0);
994        assert!(
995            metrics_only
996                .checkpoints()
997                .expect("filtered checkpoints")
998                .is_empty()
999        );
1000        assert!(
1001            LokiStore::entries(destination.as_ref(), "tenant-a")
1002                .expect("destination remains empty")
1003                .is_empty()
1004        );
1005
1006        let metric_request = ExportMetricsServiceRequest {
1007            resource_metrics: vec![ResourceMetrics {
1008                scope_metrics: vec![ScopeMetrics {
1009                    metrics: vec![
1010                        Metric {
1011                            name: "node_cpu_seconds_total".into(),
1012                            data: Some(metric::Data::Gauge(Gauge {
1013                                data_points: vec![NumberDataPoint {
1014                                    time_unix_nano: 200,
1015                                    value: Some(number_data_point::Value::AsInt(7)),
1016                                    ..NumberDataPoint::default()
1017                                }],
1018                            })),
1019                            ..Metric::default()
1020                        },
1021                        Metric {
1022                            name: "eden_local_only".into(),
1023                            data: Some(metric::Data::Gauge(Gauge {
1024                                data_points: vec![NumberDataPoint {
1025                                    time_unix_nano: 201,
1026                                    value: Some(number_data_point::Value::AsInt(3)),
1027                                    ..NumberDataPoint::default()
1028                                }],
1029                            })),
1030                            ..Metric::default()
1031                        },
1032                    ],
1033                    ..ScopeMetrics::default()
1034                }],
1035                ..ResourceMetrics::default()
1036            }],
1037        };
1038        let metric_points = OtlpTelemetryDecoder
1039            .decode_metrics("tenant-a", &metric_request.encode_to_vec())
1040            .expect("metric decode")
1041            .into_iter()
1042            .map(|event| {
1043                event.into_durable(
1044                    ShardId::new(0),
1045                    TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(0)),
1046                    LogicalOffset::new(0),
1047                )
1048            })
1049            .collect::<Vec<_>>();
1050        source
1051            .append_metric_points(metric_points, true)
1052            .expect("source metric append");
1053
1054        let selected_metrics = UpstreamOffloader::open(
1055            Arc::clone(&source),
1056            Arc::clone(&client),
1057            UpstreamOffloadConfig::new(
1058                source_directory.join("selected-metrics-offload-v1.json"),
1059                "test-node-a",
1060            )
1061            .with_signals([TelemetrySignal::Metrics])
1062            .with_metric_names(["node_cpu_seconds_total"]),
1063        )
1064        .expect("selected-metrics offloader");
1065        let selected = selected_metrics
1066            .offload_once()
1067            .await
1068            .expect("selected metric offload");
1069        assert_eq!(selected.scanned_partitions, 1);
1070        assert_eq!(selected.offloaded_batches, 1);
1071        assert_eq!(selected.offloaded_records, 1);
1072        assert_eq!(selected.skipped_batches, 1);
1073        assert_eq!(selected.skipped_records, 1);
1074        assert_eq!(selected.advanced_offsets, 2);
1075        assert_eq!(selected.checkpoint_writes, 1);
1076        assert_eq!(
1077            selected_metrics
1078                .checkpoints()
1079                .expect("metric checkpoints")
1080                .len(),
1081            1
1082        );
1083        assert_eq!(
1084            destination
1085                .query_metrics(&MetricQuery {
1086                    tenant: Arc::from("tenant-a"),
1087                    name: Some(Arc::from("node_cpu_seconds_total")),
1088                    limit: 10,
1089                    ..MetricQuery::default()
1090                })
1091                .expect("selected metric query")
1092                .len(),
1093            1
1094        );
1095        assert!(
1096            destination
1097                .query_metrics(&MetricQuery {
1098                    tenant: Arc::from("tenant-a"),
1099                    name: Some(Arc::from("eden_local_only")),
1100                    limit: 10,
1101                    ..MetricQuery::default()
1102                })
1103                .expect("unselected metric query")
1104                .is_empty()
1105        );
1106        drop(selected_metrics);
1107        let recovered_selected_metrics = UpstreamOffloader::open(
1108            Arc::clone(&source),
1109            Arc::clone(&client),
1110            UpstreamOffloadConfig::new(
1111                source_directory.join("selected-metrics-offload-v1.json"),
1112                "test-node-a",
1113            )
1114            .with_signals([TelemetrySignal::Metrics])
1115            .with_metric_names(["node_cpu_seconds_total"]),
1116        )
1117        .expect("recovered selected-metrics offloader");
1118        let recovered_selected = recovered_selected_metrics
1119            .offload_once()
1120            .await
1121            .expect("recovered selected metric offload");
1122        assert_eq!(recovered_selected.advanced_offsets, 0);
1123        assert_eq!(recovered_selected.offloaded_batches, 0);
1124        assert_eq!(recovered_selected.skipped_batches, 0);
1125        assert_eq!(recovered_selected.checkpoint_writes, 0);
1126
1127        let offloader = Arc::new(
1128            UpstreamOffloader::open(
1129                Arc::clone(&source),
1130                Arc::clone(&client),
1131                UpstreamOffloadConfig::new(
1132                    source_directory.join("upstream-offload-v1.json"),
1133                    "test-node-a",
1134                ),
1135            )
1136            .expect("offloader"),
1137        );
1138        let (offload_stop, offload_stopped) = tokio::sync::oneshot::channel();
1139        let loop_offloader = Arc::clone(&offloader);
1140        let offload_loop = tokio::spawn(async move {
1141            loop_offloader
1142                .run_until(
1143                    UpstreamOffloadLoopConfig::new()
1144                        .with_idle_interval(Duration::from_millis(5))
1145                        .with_retry_interval(Duration::from_millis(5)),
1146                    async {
1147                        let _ = offload_stopped.await;
1148                    },
1149                )
1150                .await
1151        });
1152        tokio::time::timeout(Duration::from_secs(5), async {
1153            loop {
1154                if LokiStore::entries(destination.as_ref(), "tenant-a").expect("destination query")
1155                    == vec![expected.clone()]
1156                    && offloader.checkpoints().expect("checkpoints").len() == 1
1157                {
1158                    break;
1159                }
1160                tokio::time::sleep(Duration::from_millis(5)).await;
1161            }
1162        })
1163        .await
1164        .expect("background offload completes");
1165        offload_stop.send(()).expect("stop offload loop");
1166        let report = offload_loop
1167            .await
1168            .expect("offload loop joins")
1169            .expect("offload loop succeeds");
1170        assert_eq!(report.failed_rounds, 0);
1171        assert_eq!(
1172            LokiStore::entries(destination.as_ref(), "tenant-a").expect("destination query"),
1173            vec![expected.clone()]
1174        );
1175        assert_eq!(offloader.checkpoints().expect("checkpoints").len(), 1);
1176
1177        // Both sources start at the same local WAL offset and emit the exact
1178        // same envelope. Their distinct stable source IDs must prevent the
1179        // central native receipt catalog from treating the second node as a
1180        // retry of the first node's append.
1181        let second_source_directory = root.join("second-source");
1182        let second_source = Arc::new(
1183            DurableTelemetryStore::open(config(second_source_directory.clone()))
1184                .expect("second source"),
1185        );
1186        LokiStore::push(second_source.as_ref(), "tenant-a", vec![expected.clone()])
1187            .expect("second source append");
1188        let second_offloader = UpstreamOffloader::open(
1189            Arc::clone(&second_source),
1190            client,
1191            UpstreamOffloadConfig::new(
1192                second_source_directory.join("upstream-offload-v2.json"),
1193                "test-node-b",
1194            ),
1195        )
1196        .expect("second offloader");
1197        let second_report = second_offloader
1198            .offload_once()
1199            .await
1200            .expect("second offload");
1201        assert_eq!(second_report.offloaded_batches, 1);
1202        assert_eq!(second_report.offloaded_records, 1);
1203        assert_eq!(
1204            LokiStore::entries(destination.as_ref(), "tenant-a")
1205                .expect("both sources reach destination")
1206                .len(),
1207            2
1208        );
1209
1210        stop.send(()).expect("stop");
1211        server
1212            .await
1213            .expect("server joins")
1214            .expect("server succeeds");
1215        drop(source);
1216        drop(second_source);
1217        drop(destination);
1218        std::fs::remove_dir_all(root).expect("cleanup");
1219    }
1220
1221    #[test]
1222    fn metric_name_filter_is_explicit_and_requires_metrics() {
1223        assert!(
1224            UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "")
1225                .validate()
1226                .is_err()
1227        );
1228        assert!(
1229            UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1230                .with_metric_names(std::iter::empty::<&str>())
1231                .validate()
1232                .is_err()
1233        );
1234        assert!(
1235            UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1236                .with_signals([TelemetrySignal::Logs])
1237                .with_metric_names(["node_cpu_seconds_total"])
1238                .validate()
1239                .is_err()
1240        );
1241        assert!(
1242            UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1243                .with_max_in_flight_partitions(0)
1244                .validate()
1245                .is_err()
1246        );
1247        assert!(
1248            UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1249                .with_signals([TelemetrySignal::Metrics])
1250                .with_metric_names(["node_cpu_seconds_total"])
1251                .validate()
1252                .is_ok()
1253        );
1254    }
1255
1256    #[test]
1257    fn checkpoint_journal_binds_the_stable_source_identity() {
1258        let nonce = SystemTime::now()
1259            .duration_since(UNIX_EPOCH)
1260            .expect("clock")
1261            .as_nanos();
1262        let root = std::env::temp_dir().join(format!(
1263            "shard-telemetry-offload-source-id-{}-{nonce}",
1264            std::process::id()
1265        ));
1266        let path = root.join("offload-v2.json");
1267        let partition = TopicPartition::new(TopicId::new(1), LogicalPartitionId::new(0));
1268        let mut journal =
1269            CheckpointJournal::open(path.clone(), Arc::from("node-a")).expect("open journal");
1270        journal
1271            .advance(partition, LogicalOffset::new(1))
1272            .expect("persist checkpoint");
1273        assert!(CheckpointJournal::open(path.clone(), Arc::from("node-a")).is_err());
1274        drop(journal);
1275        let reopened = CheckpointJournal::open(path.clone(), Arc::from("node-a"))
1276            .expect("same source reopens after prior owner exits");
1277        drop(reopened);
1278        assert!(CheckpointJournal::open(path, Arc::from("node-b")).is_err());
1279
1280        std::fs::remove_dir_all(root).expect("cleanup");
1281    }
1282
1283    #[test]
1284    fn v1_checkpoint_journal_migrates_with_its_legacy_retry_namespace() {
1285        let nonce = SystemTime::now()
1286            .duration_since(UNIX_EPOCH)
1287            .expect("clock")
1288            .as_nanos();
1289        let root = std::env::temp_dir().join(format!(
1290            "shard-telemetry-offload-v1-migration-{}-{nonce}",
1291            std::process::id()
1292        ));
1293        let path = root.join("offload.json");
1294        std::fs::create_dir_all(&root).expect("directory");
1295        std::fs::write(
1296            &path,
1297            serde_json::to_vec(&PersistedCheckpoints {
1298                version: 1,
1299                source_id: None,
1300                retry_namespace: None,
1301                checkpoints: vec![PersistedCheckpoint {
1302                    topic_id: 9,
1303                    partition_id: 2,
1304                    next_offset: 11,
1305                }],
1306            })
1307            .expect("legacy journal"),
1308        )
1309        .expect("write legacy journal");
1310        let partition = TopicPartition::new(TopicId::new(9), LogicalPartitionId::new(2));
1311        let mut journal =
1312            CheckpointJournal::open(path.clone(), Arc::from("node-a")).expect("migrate v1 journal");
1313        assert!(matches!(
1314            journal.retry_namespace(),
1315            RetryNamespace::LegacyV1
1316        ));
1317        assert_eq!(journal.next(partition), Some(LogicalOffset::new(11)));
1318        journal
1319            .advance(partition, LogicalOffset::new(12))
1320            .expect("persist migration");
1321        drop(journal);
1322        let persisted: PersistedCheckpoints =
1323            serde_json::from_slice(&std::fs::read(&path).expect("read migrated journal"))
1324                .expect("decode migrated journal");
1325        assert_eq!(persisted.version, OFFLOAD_CHECKPOINT_VERSION);
1326        assert_eq!(persisted.source_id.as_deref(), Some("node-a"));
1327        assert_eq!(persisted.retry_namespace.as_deref(), Some("legacy-v1"));
1328        std::fs::remove_dir_all(root).expect("cleanup");
1329    }
1330}