rete_core/
header.rs

1//! The fixed-size 1024-byte file header (SPEC.md §4.1).
2//!
3//! A client's very first request is `bytes=0-1023`; this struct is what those
4//! bytes decode to, and it points at every other section of the file via a
5//! **typed section directory**, so new top-level sections (e.g. a future text
6//! index) are added as a new directory entry without reshaping the header.
7//!
8//! Layout: a fixed 64-byte **core** (magic, version, flags, content hash, counts,
9//! codecs, `section_count`, `schema_meta_len`) followed by up to
10//! [`MAX_SECTIONS`] **directory entries** of 24 bytes each `(kind, flags, offset,
11//! length)`, zero-padded to 1024. The known section kinds populate the named
12//! convenience fields below; unknown kinds (written by a newer build) are
13//! preserved verbatim in [`Header::extra_sections`].
14
15use std::convert::TryInto;
16
17/// Magic bytes at offset 0: ASCII `RETE`.
18pub const MAGIC: [u8; 4] = *b"RETE";
19
20/// Current format generation written by this crate.
21///
22/// `0x05` is stable format generation 1, introduced by Rete 1.0.0. It retains
23/// the six index permutations and 1 KiB section-directory layout finalized in
24/// the last experimental generation.
25pub const CURRENT_FORMAT_VERSION: u8 = 0x05;
26
27/// Oldest stable format generation accepted by this reader.
28///
29/// Files written before Rete 1.0.0 used experimental generations `0x01` through
30/// `0x04` and must be rebuilt from their RDF source.
31pub const MIN_STABLE_READ_VERSION: u8 = 0x05;
32
33/// Fixed header size in bytes.
34pub const HEADER_LEN: usize = 1024;
35
36/// Byte offset of the first section-directory entry (i.e. the core size).
37const SECTION_DIR_OFFSET: usize = 64;
38/// Size of one section-directory entry.
39const SECTION_ENTRY_LEN: usize = 24;
40/// How many directory entries fit in the 1 KB frame.
41pub const MAX_SECTIONS: usize = (HEADER_LEN - SECTION_DIR_OFFSET) / SECTION_ENTRY_LEN;
42
43/// Flag bit: the file contains named graphs (quads) rather than triples only.
44pub const FLAG_HAS_QUADS: u8 = 0b0000_0001;
45
46/// Flag bit: each tiled permutation section carries a **tile-synopsis trailer**
47/// — per-tile min/max of the two non-leading columns, appended after the tile
48/// payloads. It lets a range reader prune a routed tile by a bound secondary
49/// component *before* fetching it. See `file.rs::encode_tiled_section`.
50pub const FLAG_TILE_SYNOPSIS: u8 = 0b0000_0010;
51
52/// Flag bit: the file contains **RDF-star quoted triples** (`<< s p o >>`) as
53/// dictionary terms. Purely informational for compatibility — a plain-RDF
54/// consumer can detect from the header alone that some terms are quoted triples
55/// (which it may not understand) without scanning the dictionary. The file is
56/// otherwise a normal `.rete`: quoted triples are stored like any other term, so
57/// this needs no format-version bump and old readers stay forward-compatible.
58pub const FLAG_HAS_QUOTED_TRIPLES: u8 = 0b0000_0100;
59
60/// A top-level file section, addressed by [`SectionKind`] in the header directory.
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
62pub enum SectionKind {
63    /// Dataset-card metadata.
64    Metadata,
65    /// Dictionary container (front-coded term sections).
66    Dictionary,
67    /// Permutation index container (SPO/POS/OSP).
68    Index,
69    /// Community + schema pyramid metadata.
70    PyramidMeta,
71    /// Named-graphs section.
72    NamedGraphs,
73    /// Full-text (word) index over literals — `token → subjects`.
74    TextIndex,
75    /// A section kind this build doesn't know — preserved verbatim on round-trip
76    /// so a newer writer's sections survive an older reader.
77    Unknown(u16),
78}
79
80impl SectionKind {
81    fn to_u16(self) -> u16 {
82        match self {
83            SectionKind::Metadata => 1,
84            SectionKind::Dictionary => 2,
85            SectionKind::Index => 3,
86            SectionKind::PyramidMeta => 4,
87            SectionKind::NamedGraphs => 5,
88            SectionKind::TextIndex => 6,
89            SectionKind::Unknown(k) => k,
90        }
91    }
92
93    fn from_u16(k: u16) -> Self {
94        match k {
95            1 => SectionKind::Metadata,
96            2 => SectionKind::Dictionary,
97            3 => SectionKind::Index,
98            4 => SectionKind::PyramidMeta,
99            5 => SectionKind::NamedGraphs,
100            6 => SectionKind::TextIndex,
101            other => SectionKind::Unknown(other),
102        }
103    }
104}
105
106/// One parsed/encoded section-directory entry: a typed `(offset, length)` into
107/// the file, plus 16 bits of per-section flags (reserved).
108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
109pub struct Section {
110    pub kind: SectionKind,
111    pub flags: u16,
112    pub offset: u64,
113    pub length: u64,
114}
115
116#[derive(Debug, thiserror::Error)]
117#[non_exhaustive]
118pub enum HeaderError {
119    #[error("buffer too small: need {HEADER_LEN} bytes, got {0}")]
120    TooSmall(usize),
121    #[error("bad magic: expected RETE")]
122    BadMagic,
123    #[error(
124        "unsupported .rete format {found:#04x}; this Rete build reads {min:#04x}..={max:#04x}. Pre-1.0 files must be rebuilt from RDF source with `rete build`"
125    )]
126    UnsupportedVersion { found: u8, min: u8, max: u8 },
127    #[error("section count {0} overruns the header frame")]
128    BadSectionCount(usize),
129}
130
131/// Decoded file header. All multi-byte fields are little-endian on disk. The
132/// `*_offset` / `*_len` fields are a convenience view over the section directory
133/// (populated from it on parse, emitted back to it on serialize).
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub struct Header {
136    /// Format version of the parsed file (currently
137    /// [`MIN_STABLE_READ_VERSION`] through [`CURRENT_FORMAT_VERSION`]).
138    pub version: u8,
139    pub flags: u8,
140    pub metadata_offset: u64,
141    pub metadata_len: u64,
142    pub dictionary_offset: u64,
143    pub dictionary_len: u64,
144    pub root_dir_offset: u64,
145    pub root_dir_len: u64,
146    pub pyramid_meta_offset: u64,
147    pub pyramid_meta_len: u64,
148    pub dict_codec: u8,
149    pub block_codec: u8,
150    pub pyramid_levels: u16,
151    pub quad_count: u64,
152    pub term_count: u64,
153    /// First 16 bytes of the blake3 content hash — an immutable validator.
154    pub content_hash: [u8; 16],
155    /// Named-graphs section (0 if the file has only the default graph).
156    pub named_graphs_offset: u64,
157    pub named_graphs_len: u64,
158    /// Byte length of the trailing **schema-pyramid block** within the pyramid-meta
159    /// section (0 if none). A reader fetches *only* that block — at
160    /// `pyramid_meta_offset + pyramid_meta_len - schema_meta_len` — for an
161    /// index/dictionary/summary-free Tier-0 coherence check.
162    pub schema_meta_len: u32,
163    /// Full-text index section (0 if the file has none; built with
164    /// `rete build --text-index`). See [`SectionKind::TextIndex`].
165    pub text_index_offset: u64,
166    pub text_index_len: u64,
167    /// Directory entries whose [`SectionKind`] this build doesn't recognize,
168    /// preserved verbatim. Empty for a file this crate wrote.
169    pub extra_sections: Vec<Section>,
170}
171
172impl Header {
173    /// Serialize into a fixed 1024-byte array.
174    pub fn to_bytes(&self) -> [u8; HEADER_LEN] {
175        let mut b = [0u8; HEADER_LEN];
176        // --- core (64 bytes) ---
177        b[0..4].copy_from_slice(&MAGIC);
178        b[4] = self.version;
179        b[5] = self.flags;
180        b[6..8].copy_from_slice(&(HEADER_LEN as u16).to_le_bytes());
181        b[8..24].copy_from_slice(&self.content_hash);
182        b[24..32].copy_from_slice(&self.quad_count.to_le_bytes());
183        b[32..40].copy_from_slice(&self.term_count.to_le_bytes());
184        b[40..42].copy_from_slice(&self.pyramid_levels.to_le_bytes());
185        b[42] = self.dict_codec;
186        b[43] = self.block_codec;
187        // [44..46) section_count written below.
188        b[46..50].copy_from_slice(&self.schema_meta_len.to_le_bytes());
189        // [50..64) reserved.
190
191        // --- section directory ---
192        // The five always-present sections (verbatim, so the named offsets
193        // round-trip exactly), then the optional text index (only when present, so
194        // a file without one stays byte-identical), then preserved unknown kinds.
195        let entry = |kind, offset, length| Section {
196            kind,
197            flags: 0,
198            offset,
199            length,
200        };
201        let mut entries: Vec<Section> = vec![
202            entry(
203                SectionKind::Metadata,
204                self.metadata_offset,
205                self.metadata_len,
206            ),
207            entry(
208                SectionKind::Dictionary,
209                self.dictionary_offset,
210                self.dictionary_len,
211            ),
212            entry(SectionKind::Index, self.root_dir_offset, self.root_dir_len),
213            entry(
214                SectionKind::PyramidMeta,
215                self.pyramid_meta_offset,
216                self.pyramid_meta_len,
217            ),
218            entry(
219                SectionKind::NamedGraphs,
220                self.named_graphs_offset,
221                self.named_graphs_len,
222            ),
223        ];
224        if self.text_index_len > 0 {
225            entries.push(entry(
226                SectionKind::TextIndex,
227                self.text_index_offset,
228                self.text_index_len,
229            ));
230        }
231        entries.extend(self.extra_sections.iter().copied());
232        debug_assert!(
233            entries.len() <= MAX_SECTIONS,
234            "too many sections for a 1 KB header"
235        );
236        let n = entries.len().min(MAX_SECTIONS);
237        b[44..46].copy_from_slice(&(n as u16).to_le_bytes());
238        for (i, s) in entries.iter().take(n).enumerate() {
239            let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
240            b[p..p + 2].copy_from_slice(&s.kind.to_u16().to_le_bytes());
241            b[p + 2..p + 4].copy_from_slice(&s.flags.to_le_bytes());
242            // [p+4..p+8) reserved.
243            b[p + 8..p + 16].copy_from_slice(&s.offset.to_le_bytes());
244            b[p + 16..p + 24].copy_from_slice(&s.length.to_le_bytes());
245        }
246        b
247    }
248
249    /// Parse a header from the first 1024 bytes of a file.
250    pub fn from_bytes(b: &[u8]) -> Result<Self, HeaderError> {
251        if b.len() < HEADER_LEN {
252            return Err(HeaderError::TooSmall(b.len()));
253        }
254        if b[0..4] != MAGIC {
255            return Err(HeaderError::BadMagic);
256        }
257        if !(MIN_STABLE_READ_VERSION..=CURRENT_FORMAT_VERSION).contains(&b[4]) {
258            return Err(HeaderError::UnsupportedVersion {
259                found: b[4],
260                min: MIN_STABLE_READ_VERSION,
261                max: CURRENT_FORMAT_VERSION,
262            });
263        }
264        let u16_at = |o: usize| u16::from_le_bytes(b[o..o + 2].try_into().unwrap());
265        let u32_at = |o: usize| u32::from_le_bytes(b[o..o + 4].try_into().unwrap());
266        let u64_at = |o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
267
268        let section_count = u16_at(44) as usize;
269        if SECTION_DIR_OFFSET + section_count * SECTION_ENTRY_LEN > HEADER_LEN {
270            return Err(HeaderError::BadSectionCount(section_count));
271        }
272
273        let mut h = Header {
274            version: b[4],
275            flags: b[5],
276            metadata_offset: 0,
277            metadata_len: 0,
278            dictionary_offset: 0,
279            dictionary_len: 0,
280            root_dir_offset: 0,
281            root_dir_len: 0,
282            pyramid_meta_offset: 0,
283            pyramid_meta_len: 0,
284            dict_codec: b[42],
285            block_codec: b[43],
286            pyramid_levels: u16_at(40),
287            quad_count: u64_at(24),
288            term_count: u64_at(32),
289            content_hash: b[8..24].try_into().unwrap(),
290            named_graphs_offset: 0,
291            named_graphs_len: 0,
292            schema_meta_len: u32_at(46),
293            text_index_offset: 0,
294            text_index_len: 0,
295            extra_sections: Vec::new(),
296        };
297        for i in 0..section_count {
298            let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
299            let kind = SectionKind::from_u16(u16_at(p));
300            let offset = u64_at(p + 8);
301            let length = u64_at(p + 16);
302            match kind {
303                SectionKind::Metadata => {
304                    h.metadata_offset = offset;
305                    h.metadata_len = length;
306                }
307                SectionKind::Dictionary => {
308                    h.dictionary_offset = offset;
309                    h.dictionary_len = length;
310                }
311                SectionKind::Index => {
312                    h.root_dir_offset = offset;
313                    h.root_dir_len = length;
314                }
315                SectionKind::PyramidMeta => {
316                    h.pyramid_meta_offset = offset;
317                    h.pyramid_meta_len = length;
318                }
319                SectionKind::NamedGraphs => {
320                    h.named_graphs_offset = offset;
321                    h.named_graphs_len = length;
322                }
323                SectionKind::TextIndex => {
324                    h.text_index_offset = offset;
325                    h.text_index_len = length;
326                }
327                SectionKind::Unknown(_) => h.extra_sections.push(Section {
328                    kind,
329                    flags: u16_at(p + 2),
330                    offset,
331                    length,
332                }),
333            }
334        }
335        Ok(h)
336    }
337
338    pub fn has_quads(&self) -> bool {
339        self.flags & FLAG_HAS_QUADS != 0
340    }
341
342    /// Does the file contain RDF-star quoted triples ([`FLAG_HAS_QUOTED_TRIPLES`])?
343    pub fn has_quoted_triples(&self) -> bool {
344        self.flags & FLAG_HAS_QUOTED_TRIPLES != 0
345    }
346
347    /// Do the tiled index sections carry a [`FLAG_TILE_SYNOPSIS`] trailer?
348    pub fn has_tile_synopsis(&self) -> bool {
349        self.flags & FLAG_TILE_SYNOPSIS != 0
350    }
351
352    /// The directory entry for a section kind, or `None` if absent. Known kinds
353    /// read from the named convenience fields; unknown kinds from
354    /// [`extra_sections`](Self::extra_sections).
355    pub fn section(&self, kind: SectionKind) -> Option<Section> {
356        let (offset, length) = match kind {
357            SectionKind::Metadata => (self.metadata_offset, self.metadata_len),
358            SectionKind::Dictionary => (self.dictionary_offset, self.dictionary_len),
359            SectionKind::Index => (self.root_dir_offset, self.root_dir_len),
360            SectionKind::PyramidMeta => (self.pyramid_meta_offset, self.pyramid_meta_len),
361            SectionKind::NamedGraphs => (self.named_graphs_offset, self.named_graphs_len),
362            SectionKind::TextIndex => (self.text_index_offset, self.text_index_len),
363            SectionKind::Unknown(_) => {
364                return self.extra_sections.iter().find(|s| s.kind == kind).copied()
365            }
366        };
367        Some(Section {
368            kind,
369            flags: 0,
370            offset,
371            length,
372        })
373    }
374
375    /// Attach (or overwrite) a section's `(offset, length)` — the extension point
376    /// for new top-level sections. Known kinds set the named fields; an unknown
377    /// kind is appended to [`extra_sections`](Self::extra_sections).
378    pub fn with_section(mut self, kind: SectionKind, offset: u64, length: u64) -> Self {
379        match kind {
380            SectionKind::Metadata => {
381                self.metadata_offset = offset;
382                self.metadata_len = length;
383            }
384            SectionKind::Dictionary => {
385                self.dictionary_offset = offset;
386                self.dictionary_len = length;
387            }
388            SectionKind::Index => {
389                self.root_dir_offset = offset;
390                self.root_dir_len = length;
391            }
392            SectionKind::PyramidMeta => {
393                self.pyramid_meta_offset = offset;
394                self.pyramid_meta_len = length;
395            }
396            SectionKind::NamedGraphs => {
397                self.named_graphs_offset = offset;
398                self.named_graphs_len = length;
399            }
400            SectionKind::TextIndex => {
401                self.text_index_offset = offset;
402                self.text_index_len = length;
403            }
404            SectionKind::Unknown(_) => self.extra_sections.push(Section {
405                kind,
406                flags: 0,
407                offset,
408                length,
409            }),
410        }
411        self
412    }
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418
419    fn sample() -> Header {
420        Header {
421            version: CURRENT_FORMAT_VERSION,
422            flags: FLAG_HAS_QUADS,
423            metadata_offset: 1024,
424            metadata_len: 42,
425            dictionary_offset: 1066,
426            dictionary_len: 2048,
427            root_dir_offset: 3114,
428            root_dir_len: 256,
429            pyramid_meta_offset: 3370,
430            pyramid_meta_len: 64,
431            dict_codec: 1,
432            block_codec: 2,
433            pyramid_levels: 3,
434            quad_count: 5,
435            term_count: 9,
436            content_hash: [7u8; 16],
437            named_graphs_offset: 3434,
438            named_graphs_len: 48,
439            schema_meta_len: 99,
440            text_index_offset: 0,
441            text_index_len: 0,
442            extra_sections: Vec::new(),
443        }
444    }
445
446    #[test]
447    fn round_trip() {
448        let h = sample();
449        let bytes = h.to_bytes();
450        assert_eq!(bytes.len(), HEADER_LEN);
451        assert_eq!(&bytes[0..4], b"RETE");
452        let back = Header::from_bytes(&bytes).unwrap();
453        assert_eq!(h, back);
454        assert!(back.has_quads());
455    }
456
457    #[test]
458    fn byte_layout_matches_spec() {
459        // Pins the core fields and the first directory entry to exact offsets.
460        let h = Header {
461            content_hash: [0xCC; 16],
462            quad_count: 0x99,
463            term_count: 0xAA,
464            pyramid_levels: 0xABCD,
465            dict_codec: 0xA1,
466            block_codec: 0xA2,
467            schema_meta_len: 0xD00D,
468            metadata_offset: 0x11,
469            metadata_len: 0x22,
470            dictionary_offset: 0x33,
471            dictionary_len: 0x44,
472            ..sample()
473        };
474        let b = h.to_bytes();
475        let u16_at = |o: usize| u16::from_le_bytes(b[o..o + 2].try_into().unwrap());
476        let u64_at = |o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
477
478        // core
479        assert_eq!(&b[0..4], b"RETE");
480        assert_eq!(b[4], CURRENT_FORMAT_VERSION);
481        assert_eq!(b[5], FLAG_HAS_QUADS);
482        assert_eq!(u16_at(6), HEADER_LEN as u16);
483        assert_eq!(&b[8..24], &[0xCC; 16]); // content hash
484        assert_eq!(u64_at(24), 0x99); // quad count
485        assert_eq!(u64_at(32), 0xAA); // term count
486        assert_eq!(u16_at(40), 0xABCD); // pyramid levels
487        assert_eq!(b[42], 0xA1); // dict codec
488        assert_eq!(b[43], 0xA2); // block codec
489        assert_eq!(u16_at(44), 5); // section_count: the 5 known sections
490        assert_eq!(u32::from_le_bytes(b[46..50].try_into().unwrap()), 0xD00D); // schema-meta len
491                                                                               // first directory entry = Metadata, at offset 64
492        assert_eq!(u16_at(64), 1); // kind = Metadata
493        assert_eq!(u64_at(72), 0x11); // metadata offset
494        assert_eq!(u64_at(80), 0x22); // metadata length
495                                      // second entry = Dictionary, at 88
496        assert_eq!(u16_at(88), 2);
497        assert_eq!(u64_at(96), 0x33);
498        assert_eq!(u64_at(104), 0x44);
499        assert_eq!(b.len(), HEADER_LEN);
500    }
501
502    #[test]
503    fn rejects_bad_magic() {
504        let mut bytes = [0u8; HEADER_LEN];
505        bytes[4] = CURRENT_FORMAT_VERSION;
506        assert!(matches!(
507            Header::from_bytes(&bytes),
508            Err(HeaderError::BadMagic)
509        ));
510    }
511
512    #[test]
513    fn stable_reader_accepts_v1_baseline_and_rejects_pre_v1() {
514        let current = sample().to_bytes();
515        assert_eq!(current[4], 0x05);
516        assert_eq!(Header::from_bytes(&current).unwrap().version, 0x05);
517
518        for old in 0x01..=0x04 {
519            let mut bytes = current;
520            bytes[4] = old;
521            let error = Header::from_bytes(&bytes).unwrap_err();
522            assert!(matches!(
523                &error,
524                HeaderError::UnsupportedVersion {
525                    found,
526                    min: 0x05,
527                    max: 0x05
528                } if *found == old
529            ));
530            assert!(error
531                .to_string()
532                .contains("Pre-1.0 files must be rebuilt from RDF source with `rete build`"));
533        }
534
535        for unsupported in [0x00, 0x06, 0xff] {
536            let mut bytes = current;
537            bytes[4] = unsupported;
538            assert!(matches!(
539                Header::from_bytes(&bytes),
540                Err(HeaderError::UnsupportedVersion {
541                    found,
542                    min: 0x05,
543                    max: 0x05
544                }) if found == unsupported
545            ));
546        }
547    }
548
549    #[test]
550    fn rejects_overrunning_section_count() {
551        let mut bad = sample().to_bytes();
552        bad[44..46].copy_from_slice(&9999u16.to_le_bytes());
553        assert!(matches!(
554            Header::from_bytes(&bad),
555            Err(HeaderError::BadSectionCount(9999))
556        ));
557    }
558
559    #[test]
560    fn unknown_section_survives_round_trip() {
561        // A section a future build added (kind 99) must be preserved verbatim by a
562        // reader that doesn't know it, and readable via `section()`.
563        let h = sample().with_section(SectionKind::Unknown(99), 4096, 512);
564        let back = Header::from_bytes(&h.to_bytes()).unwrap();
565        assert_eq!(back.extra_sections.len(), 1);
566        let s = back.section(SectionKind::Unknown(99)).unwrap();
567        assert_eq!((s.offset, s.length), (4096, 512));
568        // Known sections still resolve.
569        let dict = back.section(SectionKind::Dictionary).unwrap();
570        assert_eq!(dict.offset, h.dictionary_offset);
571        assert_eq!(h, back);
572    }
573
574    #[test]
575    fn text_index_section_round_trips_and_is_optional() {
576        // Absent (len 0): only the 5 always-present sections, so a file without a
577        // text index is byte-identical to one built before this section existed.
578        assert_eq!(
579            u16::from_le_bytes(sample().to_bytes()[44..46].try_into().unwrap()),
580            5
581        );
582        assert!(sample().section(SectionKind::TextIndex).unwrap().length == 0);
583
584        // Present: a 6th directory entry that round-trips and resolves.
585        let h = sample().with_section(SectionKind::TextIndex, 5000, 4096);
586        let bytes = h.to_bytes();
587        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
588        let back = Header::from_bytes(&bytes).unwrap();
589        assert_eq!(h, back);
590        let s = back.section(SectionKind::TextIndex).unwrap();
591        assert_eq!((s.offset, s.length), (5000, 4096));
592        assert!(back.extra_sections.is_empty(), "TextIndex is a known kind");
593    }
594}
rete_core/header.rs

rete_core/
header.rs
