zeppelin_embed/segment/
mod.rs

1//! Immutable, checksummed, memory-mapped segment files.
2
3pub mod layout;
4pub mod reader;
5pub mod writer;
6
7use std::path::PathBuf;
8
9use crate::format::frame::FormatError;
10
11/// Sortable 128-bit segment identifier with a 48-bit millisecond prefix.
12#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
13#[repr(transparent)]
14pub struct SegmentId([u8; 16]);
15
16impl SegmentId {
17    /// Constructs an identifier from a 48-bit millisecond timestamp and entropy.
18    #[must_use]
19    pub fn new(timestamp_millis: u64, entropy: [u8; 10]) -> Self {
20        let timestamp = timestamp_millis.to_be_bytes();
21        let mut bytes = [0_u8; 16];
22        if let (Some(destination), Some(source)) = (bytes.get_mut(..6), timestamp.get(2..)) {
23            destination.copy_from_slice(source);
24        }
25        if let Some(destination) = bytes.get_mut(6..) {
26            destination.copy_from_slice(&entropy);
27        }
28        Self(bytes)
29    }
30
31    /// Restores an identifier from its permanent 16-byte representation.
32    #[must_use]
33    pub const fn from_bytes(bytes: [u8; 16]) -> Self {
34        Self(bytes)
35    }
36
37    /// Returns the permanent 16-byte representation.
38    #[must_use]
39    pub const fn as_bytes(&self) -> &[u8; 16] {
40        &self.0
41    }
42
43    /// Returns the canonical segment filename.
44    #[must_use]
45    pub fn file_name(self) -> String {
46        let mut name = String::with_capacity(8 + 32 + 5);
47        name.push_str("segment-");
48        for byte in self.0 {
49            use std::fmt::Write as _;
50            let _ = write!(name, "{byte:02x}");
51        }
52        name.push_str(".zseg");
53        name
54    }
55}
56
57impl std::fmt::Display for SegmentId {
58    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
59        for byte in self.0 {
60            write!(formatter, "{byte:02x}")?;
61        }
62        Ok(())
63    }
64}
65
66/// Manifest-visible immutable segment metadata.
67#[derive(Clone, Debug, Eq, PartialEq)]
68pub struct SegmentMeta {
69    /// Sortable segment identity.
70    pub id: SegmentId,
71    /// Dense segment-local row count.
72    pub row_count: u32,
73    /// Permanent per-segment quantization scheme id.
74    pub scheme: u16,
75    /// Logical vector dimension.
76    pub dims: u32,
77    /// Exact immutable file length.
78    pub file_size: u64,
79    /// Manifest-owned embedding epoch tag, absent only for an unstamped store.
80    pub epoch_id: Option<crate::epoch::EpochId>,
81    /// Manifest-stamped range of live values in the canonical `ts` column.
82    pub clustering_key_range: ClusteringKeyRange,
83}
84
85impl SegmentMeta {
86    pub(crate) fn same_segment_file(&self, other: &Self) -> bool {
87        self.id == other.id
88            && self.row_count == other.row_count
89            && self.scheme == other.scheme
90            && self.dims == other.dims
91            && self.file_size == other.file_size
92    }
93}
94
95/// Manifest-stamped range of live canonical `ts` clustering keys.
96#[derive(Clone, Copy, Debug, Eq, PartialEq)]
97pub enum ClusteringKeyRange {
98    /// The segment predates clustering-key assignment and cannot be range-pruned.
99    Unstamped,
100    /// The segment contains no live rows, so its clustering-key set is empty.
101    Empty,
102    /// Inclusive minimum and maximum live timestamp values.
103    Bounded {
104        /// Smallest live `ts` value.
105        min_ts: i64,
106        /// Largest live `ts` value.
107        max_ts: i64,
108    },
109}
110
111/// Typed segment encode/open/read failure.
112#[derive(Debug)]
113pub enum SegmentError {
114    /// Filesystem operation failed for a named path.
115    Io {
116        /// Path involved in the operation.
117        path: PathBuf,
118        /// Underlying platform error.
119        source: std::io::Error,
120    },
121    /// A framed or checksummed byte contract failed.
122    Format(FormatError),
123    /// The opened file was a different immutable object.
124    WrongObject {
125        /// Artifact path supplied by the caller.
126        artifact: String,
127        /// Expected manifest identity.
128        expected: SegmentId,
129        /// Identity stored in the file.
130        actual: SegmentId,
131    },
132    /// A required region was absent from the directory.
133    MissingRegion(layout::RegionKind),
134    /// Region geometry or a cross-region shape was invalid.
135    Geometry(String),
136    /// Metadata-column bytes were invalid.
137    Columns(String),
138    /// Alive-set bytes were invalid.
139    Alive(String),
140    /// A catalogued metadata semantic guard rejected bytes with exact provenance.
141    MetadataSemantic {
142        /// Human-readable decoder refusal.
143        detail: String,
144        /// Exact persisted field and values observed by the decoder.
145        provenance: MetadataDecodeProvenance,
146    },
147    /// Fixed-stride graph node-block bytes were invalid.
148    Graph(crate::graph::block::GraphNodeError),
149    /// Whole-segment lexical bytes were invalid.
150    Postings(crate::fts::sealed::SealedSegmentError),
151}
152
153impl SegmentError {
154    pub(crate) fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
155        Self::Io {
156            path: path.into(),
157            source,
158        }
159    }
160}
161
162impl std::fmt::Display for SegmentError {
163    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
164        match self {
165            Self::Io { path, source } => {
166                write!(formatter, "segment I/O {}: {source}", path.display())
167            }
168            Self::Format(error) => error.fmt(formatter),
169            Self::WrongObject {
170                artifact,
171                expected,
172                actual,
173            } => {
174                write!(
175                    formatter,
176                    "artifact {artifact} failed object identity: expected {expected}, got {actual}"
177                )
178            }
179            Self::MissingRegion(kind) => write!(formatter, "segment is missing region {kind:?}"),
180            Self::Geometry(detail) => write!(formatter, "segment geometry is invalid: {detail}"),
181            Self::Columns(detail) => write!(formatter, "segment columns are invalid: {detail}"),
182            Self::Alive(detail) => write!(formatter, "segment alive set is invalid: {detail}"),
183            Self::MetadataSemantic { detail, .. } => {
184                write!(formatter, "segment metadata is invalid: {detail}")
185            }
186            Self::Graph(error) => write!(formatter, "segment graph region is invalid: {error}"),
187            Self::Postings(error) => {
188                write!(formatter, "segment postings region is invalid: {error}")
189            }
190        }
191    }
192}
193
194/// Exact persisted metadata facts captured at a semantic decoder refusal.
195#[derive(Clone, Debug, Eq, PartialEq)]
196pub enum MetadataDecodeProvenance {
197    /// Unused bits in the last nullable-presence byte were non-zero.
198    ColumnsPresenceTail {
199        /// Column whose presence map was rejected.
200        column_id: u32,
201        /// Declared segment row count.
202        row_count: u32,
203        /// Offset of the rejected byte within the Columns region.
204        byte_offset: u64,
205        /// Byte read from persistence.
206        observed_byte: u8,
207        /// Mask of bits allowed for the final partial byte.
208        allowed_mask: u8,
209    },
210    /// A present dictionary row named a code outside the persisted dictionary.
211    ColumnsDictionaryCode {
212        /// Column whose code was rejected.
213        column_id: u32,
214        /// Row whose code was rejected.
215        row: u32,
216        /// Offset of the code within the Columns region.
217        byte_offset: u64,
218        /// Code read from persistence.
219        code: u32,
220        /// Number of persisted dictionary entries.
221        dictionary_cardinality: u32,
222    },
223    /// A raw-string length exceeded the remaining Columns-region bytes.
224    ColumnsRawStringLength {
225        /// Column whose value was rejected.
226        column_id: u32,
227        /// Row whose value was rejected.
228        row: u32,
229        /// Offset of the length field within the Columns region.
230        byte_offset: u64,
231        /// Payload length read from persistence.
232        declared_bytes: u32,
233        /// Bytes available after the length field.
234        available_bytes: u64,
235    },
236    /// The Alive bitmap ended before its declared byte length.
237    AliveBitmapTruncation {
238        /// Declared segment row count.
239        row_count: u32,
240        /// Offset where bitmap bytes begin within the Alive region.
241        byte_offset: u64,
242        /// Persisted bitmap byte length.
243        declared_bytes: u32,
244        /// Bytes actually available after the length field.
245        observed_bytes: u32,
246    },
247}
248
249impl std::error::Error for SegmentError {
250    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
251        match self {
252            Self::Io { source, .. } => Some(source),
253            Self::Format(error) => Some(error),
254            Self::Graph(error) => Some(error),
255            Self::Postings(error) => Some(error),
256            Self::WrongObject { .. }
257            | Self::MissingRegion(_)
258            | Self::Geometry(_)
259            | Self::Columns(_)
260            | Self::Alive(_)
261            | Self::MetadataSemantic { .. } => None,
262        }
263    }
264}
265
266impl From<FormatError> for SegmentError {
267    fn from(error: FormatError) -> Self {
268        Self::Format(error)
269    }
270}
271
272impl From<crate::graph::block::GraphNodeError> for SegmentError {
273    fn from(error: crate::graph::block::GraphNodeError) -> Self {
274        Self::Graph(error)
275    }
276}
277
278impl From<crate::fts::sealed::SealedSegmentError> for SegmentError {
279    fn from(error: crate::fts::sealed::SealedSegmentError) -> Self {
280        Self::Postings(error)
281    }
282}
zeppelin_embed/segment/mod.rs

zeppelin_embed/segment/
mod.rs
