zeppelin_embed/format/
mod.rs

1//! Append-only persisted-format registry.
2
3pub mod frame;
4pub mod golden;
5
6/// A persisted artifact family identifier.
7#[derive(Clone, Copy, Debug, Eq, PartialEq)]
8#[repr(u16)]
9pub enum FormatFamily {
10    /// Generic length-prefixed framing.
11    Frame = 1,
12    /// Immutable segment files.
13    Segment = 2,
14    /// Packed vector-code regions.
15    VectorCodes = 3,
16    /// Quantizer factor-record regions.
17    VectorFactors = 4,
18    /// Exact-rescore vector regions.
19    VectorRescore = 5,
20    /// Typed metadata-column regions.
21    Columns = 6,
22    /// Alive/tombstone regions.
23    Alive = 7,
24    /// Reserved full-text postings regions.
25    Postings = 8,
26    /// Per-chunk xxh3 checksum tables.
27    ChecksumTable = 9,
28    /// Store manifests.
29    Manifest = 10,
30    /// Append-only write-ahead-log records.
31    Wal = 11,
32    /// Fixed-stride graph node-block regions.
33    GraphNodeBlocks = 12,
34    /// Dense sealed-row document identifiers and revisions.
35    DocumentVersions = 13,
36    /// Dense length-delimited opaque stored metadata rows.
37    StoredMetadata = 14,
38    /// Durable request proving an unfinished physical purge must resume.
39    PurgeIntent = 15,
40    /// Dense optional UTF-8 source text aligned to sealed rows.
41    StoredText = 16,
42}
43
44impl FormatFamily {
45    /// Returns the permanent numeric family identifier.
46    #[must_use]
47    pub const fn id(self) -> u16 {
48        self as u16
49    }
50
51    pub(crate) const fn current_version(self) -> u16 {
52        match self {
53            Self::Manifest => 2,
54            Self::Frame
55            | Self::Segment
56            | Self::VectorCodes
57            | Self::VectorFactors
58            | Self::VectorRescore
59            | Self::Columns
60            | Self::Alive
61            | Self::Postings
62            | Self::ChecksumTable
63            | Self::Wal
64            | Self::GraphNodeBlocks
65            | Self::DocumentVersions
66            | Self::StoredMetadata
67            | Self::PurgeIntent
68            | Self::StoredText => 1,
69        }
70    }
71}
72
73/// One append-only version declaration.
74#[derive(Clone, Copy, Debug, Eq, PartialEq)]
75pub struct FamilySpec {
76    /// The persisted family.
77    pub family: FormatFamily,
78    /// The version emitted by writers.
79    pub current_version: u16,
80    /// Lowest version accepted by readers.
81    pub minimum_accepted_version: u16,
82    /// Highest version accepted by readers.
83    pub maximum_accepted_version: u16,
84}
85
86/// Registry validation failure.
87#[derive(Clone, Copy, Debug, Eq, PartialEq)]
88pub enum RegistryError {
89    /// No persisted family owns this identifier.
90    UnknownFamily(u16),
91    /// The declared version is outside the reader's accepted interval.
92    UnsupportedVersion {
93        /// Artifact family identifier.
94        family: u16,
95        /// Version found in the artifact.
96        version: u16,
97        /// Lowest accepted version.
98        minimum: u16,
99        /// Highest accepted version.
100        maximum: u16,
101    },
102    /// A retired quantization scheme identifier was requested.
103    RetiredScheme(u16),
104    /// No quantization scheme owns this identifier.
105    UnknownScheme(u16),
106}
107
108impl std::fmt::Display for RegistryError {
109    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
110        match self {
111            Self::UnknownFamily(family) => write!(formatter, "unknown format family {family}"),
112            Self::UnsupportedVersion {
113                family,
114                version,
115                minimum,
116                maximum,
117            } => write!(
118                formatter,
119                "format family {family} version {version} is outside accepted range {minimum}..={maximum}"
120            ),
121            Self::RetiredScheme(scheme) => {
122                write!(
123                    formatter,
124                    "quantization scheme id {scheme} is permanently retired"
125                )
126            }
127            Self::UnknownScheme(scheme) => {
128                write!(formatter, "unknown quantization scheme {scheme}")
129            }
130        }
131    }
132}
133
134impl std::error::Error for RegistryError {}
135
136/// Static registry for every persisted family and quantization identifier.
137pub struct FormatRegistry;
138
139const FAMILIES: [FamilySpec; 16] = [
140    FamilySpec {
141        family: FormatFamily::Frame,
142        current_version: 1,
143        minimum_accepted_version: 1,
144        maximum_accepted_version: 1,
145    },
146    FamilySpec {
147        family: FormatFamily::Segment,
148        current_version: 1,
149        minimum_accepted_version: 1,
150        maximum_accepted_version: 1,
151    },
152    FamilySpec {
153        family: FormatFamily::VectorCodes,
154        current_version: 1,
155        minimum_accepted_version: 1,
156        maximum_accepted_version: 1,
157    },
158    FamilySpec {
159        family: FormatFamily::VectorFactors,
160        current_version: 1,
161        minimum_accepted_version: 1,
162        maximum_accepted_version: 1,
163    },
164    FamilySpec {
165        family: FormatFamily::VectorRescore,
166        current_version: 1,
167        minimum_accepted_version: 1,
168        maximum_accepted_version: 1,
169    },
170    FamilySpec {
171        family: FormatFamily::Columns,
172        current_version: 1,
173        minimum_accepted_version: 1,
174        maximum_accepted_version: 1,
175    },
176    FamilySpec {
177        family: FormatFamily::Alive,
178        current_version: 1,
179        minimum_accepted_version: 1,
180        maximum_accepted_version: 1,
181    },
182    FamilySpec {
183        family: FormatFamily::Postings,
184        current_version: 1,
185        minimum_accepted_version: 1,
186        maximum_accepted_version: 1,
187    },
188    FamilySpec {
189        family: FormatFamily::ChecksumTable,
190        current_version: 1,
191        minimum_accepted_version: 1,
192        maximum_accepted_version: 1,
193    },
194    FamilySpec {
195        family: FormatFamily::Manifest,
196        current_version: 2,
197        minimum_accepted_version: 2,
198        maximum_accepted_version: 2,
199    },
200    FamilySpec {
201        family: FormatFamily::Wal,
202        current_version: 1,
203        minimum_accepted_version: 1,
204        maximum_accepted_version: 1,
205    },
206    FamilySpec {
207        family: FormatFamily::GraphNodeBlocks,
208        current_version: 1,
209        minimum_accepted_version: 1,
210        maximum_accepted_version: 1,
211    },
212    FamilySpec {
213        family: FormatFamily::DocumentVersions,
214        current_version: 1,
215        minimum_accepted_version: 1,
216        maximum_accepted_version: 1,
217    },
218    FamilySpec {
219        family: FormatFamily::StoredMetadata,
220        current_version: 1,
221        minimum_accepted_version: 1,
222        maximum_accepted_version: 1,
223    },
224    FamilySpec {
225        family: FormatFamily::PurgeIntent,
226        current_version: 1,
227        minimum_accepted_version: 1,
228        maximum_accepted_version: 1,
229    },
230    FamilySpec {
231        family: FormatFamily::StoredText,
232        current_version: 1,
233        minimum_accepted_version: 1,
234        maximum_accepted_version: 1,
235    },
236];
237
238impl FormatRegistry {
239    /// Returns every registered family in permanent identifier order.
240    #[must_use]
241    pub const fn families() -> &'static [FamilySpec] {
242        &FAMILIES
243    }
244
245    /// Resolves and validates one family version.
246    pub fn require(family: u16, version: u16) -> Result<&'static FamilySpec, RegistryError> {
247        let spec = FAMILIES
248            .iter()
249            .find(|candidate| candidate.family.id() == family)
250            .ok_or(RegistryError::UnknownFamily(family))?;
251        if version < spec.minimum_accepted_version || version > spec.maximum_accepted_version {
252            return Err(RegistryError::UnsupportedVersion {
253                family,
254                version,
255                minimum: spec.minimum_accepted_version,
256                maximum: spec.maximum_accepted_version,
257            });
258        }
259        Ok(spec)
260    }
261
262    /// Rejects retired ids 3 and 5 and validates every live v1 scheme id.
263    pub const fn require_scheme(scheme: u16) -> Result<(), RegistryError> {
264        match scheme {
265            0 | 1 | 2 | 4 => Ok(()),
266            3 | 5 => Err(RegistryError::RetiredScheme(scheme)),
267            _ => Err(RegistryError::UnknownScheme(scheme)),
268        }
269    }
270}
zeppelin_embed/format/mod.rs

zeppelin_embed/format/
mod.rs
