1use std::collections::{BTreeMap, BTreeSet};
2use std::fmt;
3use std::fs::{self, File, OpenOptions};
4use std::future::Future;
5use std::io::Write;
6use std::path::{Path, PathBuf};
7use std::sync::{Arc, Mutex};
8use std::time::Duration;
9
10use fs2::FileExt;
11use futures_util::stream::{FuturesUnordered, StreamExt};
12use serde::{Deserialize, Serialize};
13use shard_stream_core::{LogicalOffset, LogicalPartitionId, TopicId, TopicPartition};
14use tokio::sync::Mutex as AsyncMutex;
15
16use crate::{
17 DurableTelemetryStore, NativePartitionAppend, NativeTelemetryBatch, ShardTelemetryClient,
18 TelemetryEnvelope, TelemetrySignal, decode_metric_chunk,
19};
20
21const OFFLOAD_CHECKPOINT_VERSION: u8 = 3;
22const DEFAULT_MAX_FETCH_BYTES: u32 = 16 * 1024 * 1024;
23const DEFAULT_MAX_IN_FLIGHT_PARTITIONS: usize = 1;
24const DEFAULT_IDLE_INTERVAL: Duration = Duration::from_secs(1);
25const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(5);
26const DEFAULT_MAX_CONSECUTIVE_FAILURES: usize = 8;
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub struct OffloadCheckpoint {
31 pub topic_partition: TopicPartition,
33 pub next_offset: LogicalOffset,
35}
36
37#[derive(Debug, Clone)]
39pub struct UpstreamOffloadConfig {
40 pub checkpoint_path: PathBuf,
43 pub source_id: Arc<str>,
50 pub max_fetch_bytes: u32,
52 pub max_in_flight_partitions: usize,
57 pub signals: BTreeSet<TelemetrySignal>,
63 pub metric_names: Option<BTreeSet<Arc<str>>>,
70}
71
72impl UpstreamOffloadConfig {
73 #[must_use]
75 pub fn new(checkpoint_path: PathBuf, source_id: impl Into<Arc<str>>) -> Self {
76 Self {
77 checkpoint_path,
78 source_id: source_id.into(),
79 max_fetch_bytes: DEFAULT_MAX_FETCH_BYTES,
80 max_in_flight_partitions: DEFAULT_MAX_IN_FLIGHT_PARTITIONS,
81 signals: [
82 TelemetrySignal::Logs,
83 TelemetrySignal::Traces,
84 TelemetrySignal::Metrics,
85 ]
86 .into_iter()
87 .collect(),
88 metric_names: None,
89 }
90 }
91
92 #[must_use]
94 pub const fn with_max_fetch_bytes(mut self, max_fetch_bytes: u32) -> Self {
95 self.max_fetch_bytes = max_fetch_bytes;
96 self
97 }
98
99 #[must_use]
103 pub const fn with_max_in_flight_partitions(mut self, max_in_flight_partitions: usize) -> Self {
104 self.max_in_flight_partitions = max_in_flight_partitions;
105 self
106 }
107
108 #[must_use]
114 pub fn with_signals(mut self, signals: impl IntoIterator<Item = TelemetrySignal>) -> Self {
115 self.signals = signals.into_iter().collect();
116 self
117 }
118
119 #[must_use]
127 pub fn with_metric_names<I, S>(mut self, metric_names: I) -> Self
128 where
129 I: IntoIterator<Item = S>,
130 S: Into<Arc<str>>,
131 {
132 self.metric_names = Some(metric_names.into_iter().map(Into::into).collect());
133 self
134 }
135
136 fn validate(&self) -> Result<(), OffloadError> {
137 if self.checkpoint_path.as_os_str().is_empty() {
138 return Err(OffloadError::new(
139 "upstream offload checkpoint path must not be empty",
140 ));
141 }
142 if self.source_id.is_empty() {
143 return Err(OffloadError::new(
144 "upstream offload source ID must not be empty",
145 ));
146 }
147 if self.max_fetch_bytes == 0 {
148 return Err(OffloadError::new(
149 "upstream offload max_fetch_bytes must be nonzero",
150 ));
151 }
152 if self.max_in_flight_partitions == 0 {
153 return Err(OffloadError::new(
154 "upstream offload max_in_flight_partitions must be nonzero",
155 ));
156 }
157 if self.signals.is_empty() {
158 return Err(OffloadError::new(
159 "upstream offload must select at least one telemetry signal",
160 ));
161 }
162 if let Some(metric_names) = &self.metric_names {
163 if metric_names.is_empty() {
164 return Err(OffloadError::new(
165 "upstream metric offload name filter must not be empty",
166 ));
167 }
168 if !self.signals.contains(&TelemetrySignal::Metrics) {
169 return Err(OffloadError::new(
170 "upstream metric offload name filter requires metrics to be selected",
171 ));
172 }
173 if metric_names.iter().any(|name| name.is_empty()) {
174 return Err(OffloadError::new(
175 "upstream metric offload names must not be empty",
176 ));
177 }
178 }
179 Ok(())
180 }
181}
182
183#[derive(Debug, Clone, Copy)]
185pub struct UpstreamOffloadLoopConfig {
186 pub idle_interval: Duration,
188 pub retry_interval: Duration,
190 pub max_consecutive_failures: usize,
192}
193
194impl UpstreamOffloadLoopConfig {
195 #[must_use]
197 pub const fn new() -> Self {
198 Self {
199 idle_interval: DEFAULT_IDLE_INTERVAL,
200 retry_interval: DEFAULT_RETRY_INTERVAL,
201 max_consecutive_failures: DEFAULT_MAX_CONSECUTIVE_FAILURES,
202 }
203 }
204
205 #[must_use]
207 pub const fn with_idle_interval(mut self, idle_interval: Duration) -> Self {
208 self.idle_interval = idle_interval;
209 self
210 }
211
212 #[must_use]
214 pub const fn with_retry_interval(mut self, retry_interval: Duration) -> Self {
215 self.retry_interval = retry_interval;
216 self
217 }
218
219 #[must_use]
221 pub const fn with_max_consecutive_failures(mut self, max_consecutive_failures: usize) -> Self {
222 self.max_consecutive_failures = max_consecutive_failures;
223 self
224 }
225
226 fn validate(self) -> Result<(), OffloadError> {
227 if self.idle_interval.is_zero() || self.retry_interval.is_zero() {
228 return Err(OffloadError::new(
229 "upstream offload loop intervals must be nonzero",
230 ));
231 }
232 if self.max_consecutive_failures == 0 {
233 return Err(OffloadError::new(
234 "upstream offload max_consecutive_failures must be nonzero",
235 ));
236 }
237 Ok(())
238 }
239}
240
241impl Default for UpstreamOffloadLoopConfig {
242 fn default() -> Self {
243 Self::new()
244 }
245}
246
247#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
249pub struct OffloadReport {
250 pub scanned_partitions: usize,
252 pub offloaded_batches: usize,
254 pub offloaded_records: u64,
256 pub advanced_offsets: u64,
258 pub checkpoint_writes: usize,
260 pub skipped_batches: usize,
262 pub skipped_records: u64,
264}
265
266#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
268pub struct OffloadLoopReport {
269 pub completed_rounds: u64,
271 pub failed_rounds: u64,
273 pub scanned_partitions: u64,
275 pub offloaded_batches: u64,
277 pub offloaded_records: u64,
279 pub advanced_offsets: u64,
281 pub checkpoint_writes: u64,
283 pub skipped_batches: u64,
285 pub skipped_records: u64,
287}
288
289impl OffloadLoopReport {
290 fn record(&mut self, report: OffloadReport) {
291 self.completed_rounds = self.completed_rounds.saturating_add(1);
292 self.scanned_partitions = self
293 .scanned_partitions
294 .saturating_add(u64::try_from(report.scanned_partitions).unwrap_or(u64::MAX));
295 self.offloaded_batches = self
296 .offloaded_batches
297 .saturating_add(u64::try_from(report.offloaded_batches).unwrap_or(u64::MAX));
298 self.offloaded_records = self
299 .offloaded_records
300 .saturating_add(report.offloaded_records);
301 self.advanced_offsets = self
302 .advanced_offsets
303 .saturating_add(report.advanced_offsets);
304 self.checkpoint_writes = self
305 .checkpoint_writes
306 .saturating_add(u64::try_from(report.checkpoint_writes).unwrap_or(u64::MAX));
307 self.skipped_batches = self
308 .skipped_batches
309 .saturating_add(u64::try_from(report.skipped_batches).unwrap_or(u64::MAX));
310 self.skipped_records = self.skipped_records.saturating_add(report.skipped_records);
311 }
312}
313
314#[derive(Debug, Clone, PartialEq, Eq)]
316pub struct OffloadError {
317 message: Arc<str>,
318}
319
320impl OffloadError {
321 fn new(message: impl Into<Arc<str>>) -> Self {
322 Self {
323 message: message.into(),
324 }
325 }
326}
327
328impl fmt::Display for OffloadError {
329 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
330 formatter.write_str(&self.message)
331 }
332}
333
334impl std::error::Error for OffloadError {}
335
336#[derive(Debug, Clone, Serialize, Deserialize)]
337struct PersistedCheckpoint {
338 topic_id: u128,
339 partition_id: u32,
340 next_offset: u64,
341}
342
343#[derive(Debug, Clone, Serialize, Deserialize)]
344struct PersistedCheckpoints {
345 version: u8,
346 #[serde(default)]
347 source_id: Option<String>,
348 #[serde(default)]
349 retry_namespace: Option<String>,
350 checkpoints: Vec<PersistedCheckpoint>,
351}
352
353#[derive(Debug, Clone)]
354enum RetryNamespace {
355 LegacyV1,
356 Source(Arc<str>),
357}
358
359#[derive(Debug)]
360struct CheckpointJournal {
361 path: PathBuf,
362 source_id: Arc<str>,
363 retry_namespace: RetryNamespace,
364 offsets: BTreeMap<TopicPartition, LogicalOffset>,
365 _exclusive_lock: File,
368}
369
370impl CheckpointJournal {
371 fn open(path: PathBuf, source_id: Arc<str>) -> Result<Self, OffloadError> {
372 if let Some(parent) = path.parent() {
373 fs::create_dir_all(parent).map_err(journal_io_error)?;
374 }
375 let lock = OpenOptions::new()
376 .create(true)
377 .read(true)
378 .write(true)
379 .truncate(false)
380 .open(path.with_extension("lock"))
381 .map_err(journal_io_error)?;
382 lock.try_lock_exclusive().map_err(|error| {
383 OffloadError::new(format!(
384 "upstream offload checkpoint is already owned by another worker: {error}"
385 ))
386 })?;
387 let persisted = match fs::read(&path) {
388 Ok(encoded) => {
389 serde_json::from_slice::<PersistedCheckpoints>(&encoded).map_err(|error| {
390 OffloadError::new(format!(
391 "invalid upstream offload checkpoint journal: {error}"
392 ))
393 })?
394 }
395 Err(error) if error.kind() == std::io::ErrorKind::NotFound => PersistedCheckpoints {
396 version: OFFLOAD_CHECKPOINT_VERSION,
397 source_id: Some(source_id.to_string()),
398 retry_namespace: Some(source_retry_namespace(&source_id)),
399 checkpoints: Vec::new(),
400 },
401 Err(error) => return Err(journal_io_error(error)),
402 };
403 let retry_namespace = match persisted.version {
404 1 => RetryNamespace::LegacyV1,
405 2 => {
406 validate_persisted_source(persisted.source_id.as_deref(), &source_id)?;
407 RetryNamespace::Source(Arc::clone(&source_id))
408 }
409 OFFLOAD_CHECKPOINT_VERSION => {
410 validate_persisted_source(persisted.source_id.as_deref(), &source_id)?;
411 match persisted.retry_namespace.as_deref() {
412 Some("legacy-v1") => RetryNamespace::LegacyV1,
413 Some(namespace) if namespace == source_retry_namespace(&source_id) => {
414 RetryNamespace::Source(Arc::clone(&source_id))
415 }
416 _ => {
417 return Err(OffloadError::new(
418 "upstream offload checkpoint journal retry namespace is invalid",
419 ));
420 }
421 }
422 }
423 version => {
424 return Err(OffloadError::new(format!(
425 "unsupported upstream offload checkpoint journal version {version}",
426 )));
427 }
428 };
429 let mut offsets = BTreeMap::new();
430 for checkpoint in persisted.checkpoints {
431 let topic_partition = TopicPartition::new(
432 TopicId::new(checkpoint.topic_id),
433 LogicalPartitionId::new(checkpoint.partition_id),
434 );
435 if offsets
436 .insert(topic_partition, LogicalOffset::new(checkpoint.next_offset))
437 .is_some()
438 {
439 return Err(OffloadError::new(
440 "upstream offload checkpoint journal contains duplicate partitions",
441 ));
442 }
443 }
444 Ok(Self {
445 path,
446 source_id,
447 retry_namespace,
448 offsets,
449 _exclusive_lock: lock,
450 })
451 }
452
453 fn next(&self, partition: TopicPartition) -> Option<LogicalOffset> {
454 self.offsets.get(&partition).copied()
455 }
456
457 fn advance(
458 &mut self,
459 partition: TopicPartition,
460 next_offset: LogicalOffset,
461 ) -> Result<(), OffloadError> {
462 let previous = self.offsets.insert(partition, next_offset);
463 if previous.is_some_and(|previous| previous > next_offset) {
464 return Err(OffloadError::new(
465 "upstream offload checkpoint attempted to move backward",
466 ));
467 }
468 self.persist()
469 }
470
471 fn snapshot(&self) -> Vec<OffloadCheckpoint> {
472 self.offsets
473 .iter()
474 .map(|(topic_partition, next_offset)| OffloadCheckpoint {
475 topic_partition: *topic_partition,
476 next_offset: *next_offset,
477 })
478 .collect()
479 }
480
481 fn retry_namespace(&self) -> RetryNamespace {
482 self.retry_namespace.clone()
483 }
484
485 fn persist(&self) -> Result<(), OffloadError> {
486 let encoded = serde_json::to_vec(&PersistedCheckpoints {
487 version: OFFLOAD_CHECKPOINT_VERSION,
488 source_id: Some(self.source_id.to_string()),
489 retry_namespace: Some(match &self.retry_namespace {
490 RetryNamespace::LegacyV1 => "legacy-v1".to_owned(),
491 RetryNamespace::Source(source_id) => source_retry_namespace(source_id),
492 }),
493 checkpoints: self
494 .offsets
495 .iter()
496 .map(|(topic_partition, next_offset)| PersistedCheckpoint {
497 topic_id: topic_partition.topic_id.get(),
498 partition_id: topic_partition.partition_id.get(),
499 next_offset: next_offset.get(),
500 })
501 .collect(),
502 })
503 .map_err(|error| {
504 OffloadError::new(format!(
505 "upstream offload checkpoint serialization failed: {error}"
506 ))
507 })?;
508 let temporary = temporary_path(&self.path);
509 let mut file = OpenOptions::new()
510 .create(true)
511 .truncate(true)
512 .write(true)
513 .open(&temporary)
514 .map_err(journal_io_error)?;
515 file.write_all(&encoded)
516 .and_then(|()| file.sync_all())
517 .map_err(journal_io_error)?;
518 fs::rename(&temporary, &self.path).map_err(journal_io_error)?;
519 File::open(self.path.parent().unwrap_or_else(|| Path::new(".")))
520 .and_then(|directory| directory.sync_all())
521 .map_err(journal_io_error)
522 }
523}
524
525fn validate_persisted_source(
526 persisted_source: Option<&str>,
527 configured_source: &str,
528) -> Result<(), OffloadError> {
529 match persisted_source {
530 Some(source) if source == configured_source => Ok(()),
531 Some(_) => Err(OffloadError::new(
532 "upstream offload checkpoint journal source ID does not match configuration",
533 )),
534 None => Err(OffloadError::new(
535 "upstream offload checkpoint journal is missing source ID",
536 )),
537 }
538}
539
540fn source_retry_namespace(source_id: &str) -> String {
541 format!("source:{source_id}")
542}
543
544#[derive(Debug)]
553pub struct UpstreamOffloader {
554 store: Arc<DurableTelemetryStore>,
555 client: Arc<ShardTelemetryClient>,
556 max_fetch_bytes: u32,
557 max_in_flight_partitions: usize,
558 signals: BTreeSet<TelemetrySignal>,
559 metric_names: Option<BTreeSet<Arc<str>>>,
560 checkpoints: Mutex<CheckpointJournal>,
561 run_gate: AsyncMutex<()>,
562}
563
564impl UpstreamOffloader {
565 pub fn open(
567 store: Arc<DurableTelemetryStore>,
568 client: Arc<ShardTelemetryClient>,
569 config: UpstreamOffloadConfig,
570 ) -> Result<Self, OffloadError> {
571 config.validate()?;
572 let UpstreamOffloadConfig {
573 checkpoint_path,
574 source_id,
575 max_fetch_bytes,
576 max_in_flight_partitions,
577 signals,
578 metric_names,
579 } = config;
580 Ok(Self {
581 store,
582 client,
583 max_fetch_bytes,
584 max_in_flight_partitions,
585 signals,
586 metric_names,
587 checkpoints: Mutex::new(CheckpointJournal::open(checkpoint_path, source_id)?),
588 run_gate: AsyncMutex::new(()),
589 })
590 }
591
592 pub fn checkpoints(&self) -> Result<Vec<OffloadCheckpoint>, OffloadError> {
594 self.checkpoints
595 .lock()
596 .map(|journal| journal.snapshot())
597 .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))
598 }
599
600 pub async fn offload_once(&self) -> Result<OffloadReport, OffloadError> {
607 let _round = self.run_gate.lock().await;
608 let mut report = OffloadReport::default();
609 let mut in_flight = FuturesUnordered::new();
610 let mut first_error = None;
611 for partition in self.store.telemetry_partitions() {
612 if !self
613 .signals
614 .iter()
615 .any(|signal| signal.topic_id() == partition.topic_id)
616 {
617 continue;
618 }
619 in_flight.push(self.offload_partition(partition));
620 if in_flight.len() >= self.max_in_flight_partitions
621 && let Some(result) = in_flight.next().await
622 {
623 record_partition_result(&mut report, &mut first_error, result);
624 if first_error.is_some() {
625 break;
626 }
627 }
628 }
629 while let Some(result) = in_flight.next().await {
630 record_partition_result(&mut report, &mut first_error, result);
631 }
632 if let Some(error) = first_error {
633 return Err(error);
634 }
635 Ok(report)
636 }
637
638 async fn offload_partition(
639 &self,
640 partition: TopicPartition,
641 ) -> Result<OffloadReport, OffloadError> {
642 let mut report = OffloadReport {
643 scanned_partitions: 1,
644 ..OffloadReport::default()
645 };
646 let start_offset = self.start_offset(partition)?;
647 let store = Arc::clone(&self.store);
648 let max_fetch_bytes = self.max_fetch_bytes;
649 let batches = tokio::task::spawn_blocking(move || {
650 store.fetch_telemetry_batches(partition, start_offset, max_fetch_bytes)
651 })
652 .await
653 .map_err(|error| {
654 OffloadError::new(format!("upstream offload fetch worker failed: {error}"))
655 })?
656 .map_err(|error| OffloadError::new(error.to_string()))?;
657 let mut checkpoint = None;
658 for batch in batches {
659 if batch.topic_partition != partition {
660 return Err(OffloadError::new(
661 "upstream offload fetch returned a batch from another partition",
662 ));
663 }
664 let record_count = batch
665 .last_offset
666 .get()
667 .saturating_sub(batch.first_offset.get())
668 .saturating_add(1);
669 let should_forward = self.should_forward_envelope(&batch.envelope)?;
670 if should_forward {
671 let native_batch = NativeTelemetryBatch {
672 partitions: vec![NativePartitionAppend {
673 topic_partition: batch.topic_partition,
674 envelope: batch.envelope,
675 transient_context: None,
676 }],
677 };
678 let retry_id = retry_id(
679 self.retry_namespace()?,
680 batch.topic_partition,
681 batch.first_offset,
682 batch.last_offset,
683 &native_batch,
684 )?;
685 self.client
686 .append_with_request_id(&native_batch, retry_id)
687 .await
688 .map_err(|error| OffloadError::new(error.to_string()))?;
689 report.offloaded_batches = report.offloaded_batches.saturating_add(1);
690 report.offloaded_records = report.offloaded_records.saturating_add(record_count);
691 } else {
692 report.skipped_batches = report.skipped_batches.saturating_add(1);
693 report.skipped_records = report.skipped_records.saturating_add(record_count);
694 }
695 let next = batch
696 .last_offset
697 .get()
698 .checked_add(1)
699 .ok_or_else(|| OffloadError::new("upstream offload source offset exhausted"))?;
700 checkpoint = Some(LogicalOffset::new(next));
701 report.advanced_offsets = report.advanced_offsets.saturating_add(record_count);
702 }
703 if let Some(checkpoint) = checkpoint {
704 self.advance(partition, checkpoint)?;
705 report.checkpoint_writes = report.checkpoint_writes.saturating_add(1);
706 }
707 Ok(report)
708 }
709
710 pub async fn run_until<F>(
719 &self,
720 config: UpstreamOffloadLoopConfig,
721 shutdown: F,
722 ) -> Result<OffloadLoopReport, OffloadError>
723 where
724 F: Future<Output = ()> + Send,
725 {
726 config.validate()?;
727 tokio::pin!(shutdown);
728 let mut report = OffloadLoopReport::default();
729 let mut consecutive_failures = 0_usize;
730 loop {
731 let delay = tokio::select! {
732 biased;
733 () = &mut shutdown => return Ok(report),
734 round = self.offload_once() => match round {
735 Ok(round) => {
736 let has_backlog = round.advanced_offsets != 0;
737 report.record(round);
738 consecutive_failures = 0;
739 if has_backlog {
740 None
741 } else {
742 Some(config.idle_interval)
743 }
744 }
745 Err(error) => {
746 report.failed_rounds = report.failed_rounds.saturating_add(1);
747 consecutive_failures = consecutive_failures.saturating_add(1);
748 if consecutive_failures >= config.max_consecutive_failures {
749 return Err(error);
750 }
751 Some(config.retry_interval)
752 }
753 },
754 };
755 let Some(delay) = delay else {
756 continue;
757 };
758 tokio::select! {
759 biased;
760 () = &mut shutdown => return Ok(report),
761 () = tokio::time::sleep(delay) => {}
762 }
763 }
764 }
765
766 fn start_offset(&self, partition: TopicPartition) -> Result<LogicalOffset, OffloadError> {
767 let local_start = self
768 .store
769 .telemetry_partition_start_offset(partition)
770 .map_err(|error| OffloadError::new(error.to_string()))?;
771 let checkpoint = self
772 .checkpoints
773 .lock()
774 .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))?
775 .next(partition);
776 Ok(checkpoint.map_or(local_start, |checkpoint| checkpoint.max(local_start)))
777 }
778
779 fn advance(
780 &self,
781 partition: TopicPartition,
782 next_offset: LogicalOffset,
783 ) -> Result<(), OffloadError> {
784 self.checkpoints
785 .lock()
786 .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))?
787 .advance(partition, next_offset)
788 }
789
790 fn retry_namespace(&self) -> Result<RetryNamespace, OffloadError> {
791 self.checkpoints
792 .lock()
793 .map_err(|_| OffloadError::new("upstream offload checkpoint lock poisoned"))
794 .map(|journal| journal.retry_namespace())
795 }
796
797 fn should_forward_envelope(&self, envelope: &TelemetryEnvelope) -> Result<bool, OffloadError> {
798 let Some(metric_names) = &self.metric_names else {
799 return Ok(true);
800 };
801 if envelope.signal != TelemetrySignal::Metrics {
802 return Ok(true);
803 }
804 let points = decode_metric_chunk(&envelope.payload)
805 .map_err(|error| OffloadError::new(error.to_string()))?;
806 if points.len() != envelope.item_count as usize {
807 return Err(OffloadError::new(
808 "metric offload filter decoded a count different from its envelope",
809 ));
810 }
811 let mut selected = false;
812 let mut unselected = false;
813 for point in points {
814 if metric_names.contains(&point.identity.name) {
815 selected = true;
816 } else {
817 unselected = true;
818 }
819 }
820 if selected && unselected {
821 return Err(OffloadError::new(
822 "metric offload filter requires every envelope to have one selection outcome",
823 ));
824 }
825 Ok(selected)
826 }
827}
828
829fn record_partition_result(
830 report: &mut OffloadReport,
831 first_error: &mut Option<OffloadError>,
832 result: Result<OffloadReport, OffloadError>,
833) {
834 match result {
835 Ok(partition) => {
836 report.scanned_partitions = report
837 .scanned_partitions
838 .saturating_add(partition.scanned_partitions);
839 report.offloaded_batches = report
840 .offloaded_batches
841 .saturating_add(partition.offloaded_batches);
842 report.offloaded_records = report
843 .offloaded_records
844 .saturating_add(partition.offloaded_records);
845 report.advanced_offsets = report
846 .advanced_offsets
847 .saturating_add(partition.advanced_offsets);
848 report.checkpoint_writes = report
849 .checkpoint_writes
850 .saturating_add(partition.checkpoint_writes);
851 report.skipped_batches = report
852 .skipped_batches
853 .saturating_add(partition.skipped_batches);
854 report.skipped_records = report
855 .skipped_records
856 .saturating_add(partition.skipped_records);
857 }
858 Err(error) if first_error.is_none() => *first_error = Some(error),
859 Err(_) => {}
860 }
861}
862
863fn retry_id(
864 namespace: RetryNamespace,
865 partition: TopicPartition,
866 first_offset: LogicalOffset,
867 last_offset: LogicalOffset,
868 batch: &NativeTelemetryBatch,
869) -> Result<u128, OffloadError> {
870 let payload = batch
871 .encode_native_append()
872 .map_err(|error| OffloadError::new(error.to_string()))?;
873 let mut hasher = blake3::Hasher::new();
874 hasher.update(b"shard-telemetry-upstream-offload-v1\0");
875 if let RetryNamespace::Source(source_id) = namespace {
876 hasher.update(source_id.as_bytes());
877 hasher.update(&[0]);
878 }
879 hasher.update(&partition.topic_id.get().to_le_bytes());
880 hasher.update(&partition.partition_id.get().to_le_bytes());
881 hasher.update(&first_offset.get().to_le_bytes());
882 hasher.update(&last_offset.get().to_le_bytes());
883 hasher.update(&payload);
884 let digest = hasher.finalize();
885 Ok(u128::from_le_bytes(
886 digest.as_bytes()[..16]
887 .try_into()
888 .expect("BLAKE3 digest contains sixteen bytes"),
889 ))
890}
891
892fn temporary_path(path: &Path) -> PathBuf {
893 let mut name = path.file_name().unwrap_or_default().to_os_string();
894 name.push(".tmp");
895 path.with_file_name(name)
896}
897
898fn journal_io_error(error: std::io::Error) -> OffloadError {
899 OffloadError::new(format!(
900 "upstream offload checkpoint journal I/O failed: {error}"
901 ))
902}
903
904#[cfg(test)]
905mod tests {
906 use std::collections::BTreeMap;
907 use std::time::{Duration, SystemTime, UNIX_EPOCH};
908
909 use opentelemetry_proto::tonic::{
910 collector::metrics::v1::ExportMetricsServiceRequest,
911 metrics::v1::{
912 Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric,
913 number_data_point,
914 },
915 };
916 use prost::Message;
917 use shard_stream_core::{LogicalOffset, LogicalPartitionId, ShardId, TopicId, TopicPartition};
918 use tokio::net::TcpListener;
919
920 use super::*;
921 use crate::{
922 DurableTelemetryConfig, LokiEntry, LokiStore, METRICS_TOPIC_ID, MetricQuery,
923 NativeClientConfig, NativeServerConfig, OtlpTelemetryDecoder, StripeConfig, serve_native,
924 };
925
926 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
927 async fn upstream_offload_persists_progress_only_after_native_acknowledgement() {
928 let nonce = SystemTime::now()
929 .duration_since(UNIX_EPOCH)
930 .expect("clock")
931 .as_nanos();
932 let root = std::env::temp_dir().join(format!(
933 "shard-telemetry-upstream-offload-{}-{nonce}",
934 std::process::id()
935 ));
936 let source_directory = root.join("source");
937 let destination_directory = root.join("destination");
938 let config = |data_directory| DurableTelemetryConfig {
939 data_directory,
940 object_store_directory: None,
941 s3_object_store: None,
942 recovery_journal: false,
943 retention: None,
944 shard_count: 1,
945 tenant_partitions: 1,
946 append_linger: Duration::from_micros(250),
947 stripe: StripeConfig::default(),
948 indexed_ack_timeout: Duration::from_secs(30),
949 };
950 let source = Arc::new(
951 DurableTelemetryStore::open(config(source_directory.clone())).expect("source"),
952 );
953 let destination = Arc::new(
954 DurableTelemetryStore::open(config(destination_directory)).expect("destination"),
955 );
956 let expected = LokiEntry {
957 timestamp_unix_nanos: 100,
958 labels: BTreeMap::from([("node".to_owned(), "node-a".to_owned())]),
959 line: "store and forward".to_owned(),
960 structured_metadata: BTreeMap::new(),
961 };
962 LokiStore::push(source.as_ref(), "tenant-a", vec![expected.clone()])
963 .expect("source append");
964 let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
965 let address = listener.local_addr().expect("address");
966 let (stop, stopped) = tokio::sync::oneshot::channel();
967 let server_destination = Arc::clone(&destination);
968 let server = tokio::spawn(async move {
969 serve_native(
970 listener,
971 server_destination,
972 NativeServerConfig::default(),
973 async {
974 let _ = stopped.await;
975 },
976 )
977 .await
978 });
979 let client =
980 Arc::new(ShardTelemetryClient::new(NativeClientConfig::new(address)).expect("client"));
981 let metrics_only = UpstreamOffloader::open(
982 Arc::clone(&source),
983 Arc::clone(&client),
984 UpstreamOffloadConfig::new(
985 source_directory.join("metrics-only-offload-v1.json"),
986 "test-node-a",
987 )
988 .with_signals([TelemetrySignal::Metrics]),
989 )
990 .expect("metrics-only offloader");
991 let skipped = metrics_only.offload_once().await.expect("filtered offload");
992 assert_eq!(skipped.scanned_partitions, 1);
993 assert_eq!(skipped.offloaded_batches, 0);
994 assert!(
995 metrics_only
996 .checkpoints()
997 .expect("filtered checkpoints")
998 .is_empty()
999 );
1000 assert!(
1001 LokiStore::entries(destination.as_ref(), "tenant-a")
1002 .expect("destination remains empty")
1003 .is_empty()
1004 );
1005
1006 let metric_request = ExportMetricsServiceRequest {
1007 resource_metrics: vec![ResourceMetrics {
1008 scope_metrics: vec![ScopeMetrics {
1009 metrics: vec![
1010 Metric {
1011 name: "node_cpu_seconds_total".into(),
1012 data: Some(metric::Data::Gauge(Gauge {
1013 data_points: vec![NumberDataPoint {
1014 time_unix_nano: 200,
1015 value: Some(number_data_point::Value::AsInt(7)),
1016 ..NumberDataPoint::default()
1017 }],
1018 })),
1019 ..Metric::default()
1020 },
1021 Metric {
1022 name: "eden_local_only".into(),
1023 data: Some(metric::Data::Gauge(Gauge {
1024 data_points: vec![NumberDataPoint {
1025 time_unix_nano: 201,
1026 value: Some(number_data_point::Value::AsInt(3)),
1027 ..NumberDataPoint::default()
1028 }],
1029 })),
1030 ..Metric::default()
1031 },
1032 ],
1033 ..ScopeMetrics::default()
1034 }],
1035 ..ResourceMetrics::default()
1036 }],
1037 };
1038 let metric_points = OtlpTelemetryDecoder
1039 .decode_metrics("tenant-a", &metric_request.encode_to_vec())
1040 .expect("metric decode")
1041 .into_iter()
1042 .map(|event| {
1043 event.into_durable(
1044 ShardId::new(0),
1045 TopicPartition::new(METRICS_TOPIC_ID, LogicalPartitionId::new(0)),
1046 LogicalOffset::new(0),
1047 )
1048 })
1049 .collect::<Vec<_>>();
1050 source
1051 .append_metric_points(metric_points, true)
1052 .expect("source metric append");
1053
1054 let selected_metrics = UpstreamOffloader::open(
1055 Arc::clone(&source),
1056 Arc::clone(&client),
1057 UpstreamOffloadConfig::new(
1058 source_directory.join("selected-metrics-offload-v1.json"),
1059 "test-node-a",
1060 )
1061 .with_signals([TelemetrySignal::Metrics])
1062 .with_metric_names(["node_cpu_seconds_total"]),
1063 )
1064 .expect("selected-metrics offloader");
1065 let selected = selected_metrics
1066 .offload_once()
1067 .await
1068 .expect("selected metric offload");
1069 assert_eq!(selected.scanned_partitions, 1);
1070 assert_eq!(selected.offloaded_batches, 1);
1071 assert_eq!(selected.offloaded_records, 1);
1072 assert_eq!(selected.skipped_batches, 1);
1073 assert_eq!(selected.skipped_records, 1);
1074 assert_eq!(selected.advanced_offsets, 2);
1075 assert_eq!(selected.checkpoint_writes, 1);
1076 assert_eq!(
1077 selected_metrics
1078 .checkpoints()
1079 .expect("metric checkpoints")
1080 .len(),
1081 1
1082 );
1083 assert_eq!(
1084 destination
1085 .query_metrics(&MetricQuery {
1086 tenant: Arc::from("tenant-a"),
1087 name: Some(Arc::from("node_cpu_seconds_total")),
1088 limit: 10,
1089 ..MetricQuery::default()
1090 })
1091 .expect("selected metric query")
1092 .len(),
1093 1
1094 );
1095 assert!(
1096 destination
1097 .query_metrics(&MetricQuery {
1098 tenant: Arc::from("tenant-a"),
1099 name: Some(Arc::from("eden_local_only")),
1100 limit: 10,
1101 ..MetricQuery::default()
1102 })
1103 .expect("unselected metric query")
1104 .is_empty()
1105 );
1106 drop(selected_metrics);
1107 let recovered_selected_metrics = UpstreamOffloader::open(
1108 Arc::clone(&source),
1109 Arc::clone(&client),
1110 UpstreamOffloadConfig::new(
1111 source_directory.join("selected-metrics-offload-v1.json"),
1112 "test-node-a",
1113 )
1114 .with_signals([TelemetrySignal::Metrics])
1115 .with_metric_names(["node_cpu_seconds_total"]),
1116 )
1117 .expect("recovered selected-metrics offloader");
1118 let recovered_selected = recovered_selected_metrics
1119 .offload_once()
1120 .await
1121 .expect("recovered selected metric offload");
1122 assert_eq!(recovered_selected.advanced_offsets, 0);
1123 assert_eq!(recovered_selected.offloaded_batches, 0);
1124 assert_eq!(recovered_selected.skipped_batches, 0);
1125 assert_eq!(recovered_selected.checkpoint_writes, 0);
1126
1127 let offloader = Arc::new(
1128 UpstreamOffloader::open(
1129 Arc::clone(&source),
1130 Arc::clone(&client),
1131 UpstreamOffloadConfig::new(
1132 source_directory.join("upstream-offload-v1.json"),
1133 "test-node-a",
1134 ),
1135 )
1136 .expect("offloader"),
1137 );
1138 let (offload_stop, offload_stopped) = tokio::sync::oneshot::channel();
1139 let loop_offloader = Arc::clone(&offloader);
1140 let offload_loop = tokio::spawn(async move {
1141 loop_offloader
1142 .run_until(
1143 UpstreamOffloadLoopConfig::new()
1144 .with_idle_interval(Duration::from_millis(5))
1145 .with_retry_interval(Duration::from_millis(5)),
1146 async {
1147 let _ = offload_stopped.await;
1148 },
1149 )
1150 .await
1151 });
1152 tokio::time::timeout(Duration::from_secs(5), async {
1153 loop {
1154 if LokiStore::entries(destination.as_ref(), "tenant-a").expect("destination query")
1155 == vec![expected.clone()]
1156 && offloader.checkpoints().expect("checkpoints").len() == 1
1157 {
1158 break;
1159 }
1160 tokio::time::sleep(Duration::from_millis(5)).await;
1161 }
1162 })
1163 .await
1164 .expect("background offload completes");
1165 offload_stop.send(()).expect("stop offload loop");
1166 let report = offload_loop
1167 .await
1168 .expect("offload loop joins")
1169 .expect("offload loop succeeds");
1170 assert_eq!(report.failed_rounds, 0);
1171 assert_eq!(
1172 LokiStore::entries(destination.as_ref(), "tenant-a").expect("destination query"),
1173 vec![expected.clone()]
1174 );
1175 assert_eq!(offloader.checkpoints().expect("checkpoints").len(), 1);
1176
1177 let second_source_directory = root.join("second-source");
1182 let second_source = Arc::new(
1183 DurableTelemetryStore::open(config(second_source_directory.clone()))
1184 .expect("second source"),
1185 );
1186 LokiStore::push(second_source.as_ref(), "tenant-a", vec![expected.clone()])
1187 .expect("second source append");
1188 let second_offloader = UpstreamOffloader::open(
1189 Arc::clone(&second_source),
1190 client,
1191 UpstreamOffloadConfig::new(
1192 second_source_directory.join("upstream-offload-v2.json"),
1193 "test-node-b",
1194 ),
1195 )
1196 .expect("second offloader");
1197 let second_report = second_offloader
1198 .offload_once()
1199 .await
1200 .expect("second offload");
1201 assert_eq!(second_report.offloaded_batches, 1);
1202 assert_eq!(second_report.offloaded_records, 1);
1203 assert_eq!(
1204 LokiStore::entries(destination.as_ref(), "tenant-a")
1205 .expect("both sources reach destination")
1206 .len(),
1207 2
1208 );
1209
1210 stop.send(()).expect("stop");
1211 server
1212 .await
1213 .expect("server joins")
1214 .expect("server succeeds");
1215 drop(source);
1216 drop(second_source);
1217 drop(destination);
1218 std::fs::remove_dir_all(root).expect("cleanup");
1219 }
1220
1221 #[test]
1222 fn metric_name_filter_is_explicit_and_requires_metrics() {
1223 assert!(
1224 UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "")
1225 .validate()
1226 .is_err()
1227 );
1228 assert!(
1229 UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1230 .with_metric_names(std::iter::empty::<&str>())
1231 .validate()
1232 .is_err()
1233 );
1234 assert!(
1235 UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1236 .with_signals([TelemetrySignal::Logs])
1237 .with_metric_names(["node_cpu_seconds_total"])
1238 .validate()
1239 .is_err()
1240 );
1241 assert!(
1242 UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1243 .with_max_in_flight_partitions(0)
1244 .validate()
1245 .is_err()
1246 );
1247 assert!(
1248 UpstreamOffloadConfig::new(PathBuf::from("checkpoint"), "test-node")
1249 .with_signals([TelemetrySignal::Metrics])
1250 .with_metric_names(["node_cpu_seconds_total"])
1251 .validate()
1252 .is_ok()
1253 );
1254 }
1255
1256 #[test]
1257 fn checkpoint_journal_binds_the_stable_source_identity() {
1258 let nonce = SystemTime::now()
1259 .duration_since(UNIX_EPOCH)
1260 .expect("clock")
1261 .as_nanos();
1262 let root = std::env::temp_dir().join(format!(
1263 "shard-telemetry-offload-source-id-{}-{nonce}",
1264 std::process::id()
1265 ));
1266 let path = root.join("offload-v2.json");
1267 let partition = TopicPartition::new(TopicId::new(1), LogicalPartitionId::new(0));
1268 let mut journal =
1269 CheckpointJournal::open(path.clone(), Arc::from("node-a")).expect("open journal");
1270 journal
1271 .advance(partition, LogicalOffset::new(1))
1272 .expect("persist checkpoint");
1273 assert!(CheckpointJournal::open(path.clone(), Arc::from("node-a")).is_err());
1274 drop(journal);
1275 let reopened = CheckpointJournal::open(path.clone(), Arc::from("node-a"))
1276 .expect("same source reopens after prior owner exits");
1277 drop(reopened);
1278 assert!(CheckpointJournal::open(path, Arc::from("node-b")).is_err());
1279
1280 std::fs::remove_dir_all(root).expect("cleanup");
1281 }
1282
1283 #[test]
1284 fn v1_checkpoint_journal_migrates_with_its_legacy_retry_namespace() {
1285 let nonce = SystemTime::now()
1286 .duration_since(UNIX_EPOCH)
1287 .expect("clock")
1288 .as_nanos();
1289 let root = std::env::temp_dir().join(format!(
1290 "shard-telemetry-offload-v1-migration-{}-{nonce}",
1291 std::process::id()
1292 ));
1293 let path = root.join("offload.json");
1294 std::fs::create_dir_all(&root).expect("directory");
1295 std::fs::write(
1296 &path,
1297 serde_json::to_vec(&PersistedCheckpoints {
1298 version: 1,
1299 source_id: None,
1300 retry_namespace: None,
1301 checkpoints: vec![PersistedCheckpoint {
1302 topic_id: 9,
1303 partition_id: 2,
1304 next_offset: 11,
1305 }],
1306 })
1307 .expect("legacy journal"),
1308 )
1309 .expect("write legacy journal");
1310 let partition = TopicPartition::new(TopicId::new(9), LogicalPartitionId::new(2));
1311 let mut journal =
1312 CheckpointJournal::open(path.clone(), Arc::from("node-a")).expect("migrate v1 journal");
1313 assert!(matches!(
1314 journal.retry_namespace(),
1315 RetryNamespace::LegacyV1
1316 ));
1317 assert_eq!(journal.next(partition), Some(LogicalOffset::new(11)));
1318 journal
1319 .advance(partition, LogicalOffset::new(12))
1320 .expect("persist migration");
1321 drop(journal);
1322 let persisted: PersistedCheckpoints =
1323 serde_json::from_slice(&std::fs::read(&path).expect("read migrated journal"))
1324 .expect("decode migrated journal");
1325 assert_eq!(persisted.version, OFFLOAD_CHECKPOINT_VERSION);
1326 assert_eq!(persisted.source_id.as_deref(), Some("node-a"));
1327 assert_eq!(persisted.retry_namespace.as_deref(), Some("legacy-v1"));
1328 std::fs::remove_dir_all(root).expect("cleanup");
1329 }
1330}