Skip to main content

shard_telemetry/
block.rs

1use std::collections::BTreeMap;
2use std::sync::Arc;
3
4use shard_stream_core::{LogicalOffset, ShardId, TopicPartition};
5
6use crate::{
7    CompressionCohortId, CompressionPlacementId, DictionaryId, TelemetryError, TelemetryResult,
8};
9
10/// Compression format used for one sealed log block.
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
12pub enum CompressionCodec {
13    /// A standalone Zstandard frame, optionally using the descriptor dictionary.
14    Zstd,
15}
16
17/// Monotonically assigned identifier for a sealed log data block.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
19pub struct BlockId(u64);
20
21impl BlockId {
22    /// Creates a block identifier.
23    #[must_use]
24    pub const fn new(value: u64) -> Self {
25        Self(value)
26    }
27
28    /// Returns the raw block identifier.
29    #[must_use]
30    pub const fn get(self) -> u64 {
31        self.0
32    }
33}
34
35/// Immutable metadata describing a sealed compression block.
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub struct BlockDescriptor {
38    /// Stable local block identifier.
39    pub block_id: BlockId,
40    /// Owning physical shard-stream shard.
41    pub stream_shard_id: ShardId,
42    /// Logical partition represented by this block.
43    pub topic_partition: TopicPartition,
44    /// Producer-derived OTLP service/scope cohort retained for reconstruction.
45    pub source_compression_cohort: CompressionCohortId,
46    /// Final locality placement used to select this block and its dictionary.
47    pub placement_id: CompressionPlacementId,
48    /// Immutable compression dictionary required to decode the block, if any.
49    pub dictionary_id: Option<DictionaryId>,
50    /// Compression format used for the staged block payload.
51    pub compression_codec: CompressionCodec,
52    /// Encoder level used for the block.
53    pub compression_level: i32,
54    /// Lowest offset present in the block.
55    pub first_offset: LogicalOffset,
56    /// Highest offset present in the block. Interleaved cohorts may leave gaps.
57    pub last_offset: LogicalOffset,
58    /// Number of log records in this block.
59    pub record_count: u32,
60    /// Uncompressed message and metadata bytes represented by the block.
61    pub source_bytes: u64,
62    /// Bytes in the structural payload before the block's byte codec runs.
63    pub structural_bytes: u64,
64    /// Bytes in the compressed staged payload.
65    pub stored_bytes: u64,
66    /// Lowest event timestamp present in the block.
67    pub min_timestamp_unix_nanos: u64,
68    /// Highest event timestamp present in the block.
69    pub max_timestamp_unix_nanos: u64,
70    /// Byte-weighted majority-bit temperature for this complete block.
71    pub compression_temperature: u16,
72    /// Representative exact template-shape hash for this complete block.
73    pub compression_shape_hash: u64,
74    /// Byte-weighted mean squared locality deviation, Q8.
75    pub compression_temperature_variance_q8: u16,
76    /// Largest record-to-block locality distance.
77    pub max_compression_temperature_deviation: u8,
78    /// Object-store key after the block is offloaded.
79    pub object_key: Option<Arc<str>>,
80    /// Byte offset of this block inside its immutable object payload.
81    pub object_offset: Option<u64>,
82}
83
84/// In-memory directory of sealed blocks owned by one log stripe.
85#[derive(Debug, Default)]
86pub struct BlockCatalog {
87    next_block_id: u64,
88    blocks: BTreeMap<BlockId, BlockDescriptor>,
89    staged_payloads: BTreeMap<BlockId, Arc<[u8]>>,
90}
91
92impl BlockCatalog {
93    /// Returns the number of sealed blocks.
94    #[must_use]
95    pub fn len(&self) -> usize {
96        self.blocks.len()
97    }
98
99    /// Returns whether no blocks have been sealed.
100    #[must_use]
101    pub fn is_empty(&self) -> bool {
102        self.blocks.is_empty()
103    }
104
105    /// Looks up a sealed block descriptor.
106    #[must_use]
107    pub fn get(&self, block_id: BlockId) -> Option<&BlockDescriptor> {
108        self.blocks.get(&block_id)
109    }
110
111    /// Iterates sealed block descriptors in identifier order.
112    pub fn iter(&self) -> impl Iterator<Item = &BlockDescriptor> {
113        self.blocks.values()
114    }
115
116    /// Returns the compressed payload staged locally until an object-tier writer
117    /// durably offloads the block.
118    #[must_use]
119    pub fn staged_payload(&self, block_id: BlockId) -> Option<Arc<[u8]>> {
120        self.staged_payloads.get(&block_id).cloned()
121    }
122
123    pub(crate) fn seal(
124        &mut self,
125        mut block: BlockDescriptor,
126        payload: Arc<[u8]>,
127    ) -> BlockDescriptor {
128        let block_id = BlockId::new(self.next_block_id);
129        self.next_block_id = self.next_block_id.wrapping_add(1);
130        block.block_id = block_id;
131        self.blocks.insert(block_id, block.clone());
132        self.staged_payloads.insert(block_id, payload);
133        block
134    }
135
136    /// Associates a durable object-store key with an already sealed block.
137    pub fn mark_offloaded(
138        &mut self,
139        block_id: BlockId,
140        object_key: impl Into<Arc<str>>,
141    ) -> TelemetryResult<()> {
142        self.mark_offloaded_range(block_id, object_key, 0)
143    }
144
145    /// Associates a durable object-store key and byte offset with a sealed block.
146    pub fn mark_offloaded_range(
147        &mut self,
148        block_id: BlockId,
149        object_key: impl Into<Arc<str>>,
150        object_offset: u64,
151    ) -> TelemetryResult<()> {
152        let block = self
153            .blocks
154            .get_mut(&block_id)
155            .ok_or_else(|| TelemetryError::UnknownBlock(block_id.get()))?;
156        block.object_key = Some(object_key.into());
157        block.object_offset = Some(object_offset);
158        self.staged_payloads.remove(&block_id);
159        Ok(())
160    }
161}