rete_core/
header.rs

1//! The fixed-size 1024-byte file header (SPEC.md §4.1).
2//!
3//! A client's very first request is `bytes=0-1023`; this struct is what those
4//! bytes decode to, and it points at every other section of the file via a
5//! **typed section directory**, so new top-level sections (e.g. a future text
6//! index) are added as a new directory entry without reshaping the header.
7//!
8//! Layout: a fixed 64-byte **core** (magic, version, flags, content hash, counts,
9//! codecs, `section_count`, `schema_meta_len`) followed by up to
10//! [`MAX_SECTIONS`] **directory entries** of 24 bytes each `(kind, flags, offset,
11//! length)`, zero-padded to 1024. The known section kinds populate the named
12//! convenience fields below; unknown kinds (written by a newer build) are
13//! preserved verbatim in [`Header::extra_sections`].
14
15use std::convert::TryInto;
16
17/// Magic bytes at offset 0: ASCII `RETE`.
18pub const MAGIC: [u8; 4] = *b"RETE";
19
20/// Current format generation written by this crate.
21///
22/// `0x05` is stable format generation 1, frozen on 2026-07-14 and first
23/// released in Rete 0.3.0. It retains the six-wide index-permutation addressing
24/// and the 1 KiB section-directory layout finalized in the last experimental
25/// generation; *which* of those six a given file stores is [`Header::perms`]
26/// (byte 50), not the generation.
27///
28/// The generation number is not a release version — there is no Rete 1.0.0, and
29/// it is the Rust/CLI/WASM APIs, not the format, that are waiting on one. Nor
30/// does it pin reader capability: writer semantics changed inside `0x05` nine
31/// days after the freeze (#68 split oversized index groups across tiles), so a
32/// `0x05` file can need a reader newer than the one that froze `0x05`.
33pub const CURRENT_FORMAT_VERSION: u8 = 0x05;
34
35/// Oldest stable format generation accepted by this reader.
36///
37/// Files written before the generation-1 freeze used experimental generations
38/// `0x01` through `0x04` and must be rebuilt from their RDF source. `0x05` has
39/// not moved since it froze on 2026-07-14, but that is a track record, not a
40/// promise: no backwards compatibility is guaranteed before 1.0.0, so this
41/// constant may yet be raised and a `0x05` file may yet need a rebuild. See
42/// `docs/compatibility.md`.
43pub const MIN_STABLE_READ_VERSION: u8 = 0x05;
44
45/// Fixed header size in bytes.
46pub const HEADER_LEN: usize = 1024;
47
48/// Byte offset of the first section-directory entry (i.e. the core size).
49const SECTION_DIR_OFFSET: usize = 64;
50/// Size of one section-directory entry.
51const SECTION_ENTRY_LEN: usize = 24;
52/// How many directory entries fit in the 1 KB frame.
53pub const MAX_SECTIONS: usize = (HEADER_LEN - SECTION_DIR_OFFSET) / SECTION_ENTRY_LEN;
54
55/// Flag bit: the file contains named graphs (quads) rather than triples only.
56pub const FLAG_HAS_QUADS: u8 = 0b0000_0001;
57
58/// Flag bit: each tiled permutation section carries a **tile-synopsis trailer**
59/// — per-tile min/max of the two non-leading columns, appended after the tile
60/// payloads. It lets a range reader prune a routed tile by a bound secondary
61/// component *before* fetching it. See `file.rs::encode_tiled_section`.
62pub const FLAG_TILE_SYNOPSIS: u8 = 0b0000_0010;
63
64/// Flag bit: the file contains **RDF-star quoted triples** (`<< s p o >>`) as
65/// dictionary terms. Purely informational for compatibility — a plain-RDF
66/// consumer can detect from the header alone that some terms are quoted triples
67/// (which it may not understand) without scanning the dictionary. The file is
68/// otherwise a normal `.rete`: quoted triples are stored like any other term, so
69/// this needs no format-version bump and old readers stay forward-compatible.
70pub const FLAG_HAS_QUOTED_TRIPLES: u8 = 0b0000_0100;
71
72/// Flag bit: the named graphs live in the **quad index** — the QUAD_INDEX
73/// section ([`SectionKind::QuadIndex`], kind 10) holds every named-graph quad as
74/// the fourth column of graph-leading permutations, the NAMED_GRAPHS section
75/// (kind 5) is absent, and the index container carries one trailing **seal**
76/// section (see `quad.rs`). Readers before this flag do not check it; they
77/// refuse the file anyway, on the seal (the container's section count), so a
78/// quad-index file never opens as one without named graphs
79/// (`docs/compatibility.md`).
80pub const FLAG_QUAD_INDEX: u8 = 0b0000_1000;
81
82/// Flag bit: the full-text index also covers the **named graphs'** literals
83/// (`rete build --text-index-graphs all`, Rete 0.4.0). It lets a query inside
84/// `GRAPH`, under `FROM` or over the union default graph decide from the
85/// header alone whether the index can prune its `FILTER(CONTAINS(…))`: without
86/// the bit it scans and never faults the index's token table. Bit 3
87/// (`0b0000_1000`) belongs to the quad index. Older readers ignore unknown bits.
88pub const FLAG_TEXT_INDEX_NAMED_GRAPHS: u8 = 0b0001_0000;
89
90/// Length of the BLAKE3 self-digest that heads a predicate-restricted text
91/// index section ([`SectionKind::TextIndexPartial`], kind 11):
92/// `digest = blake3(section[16..])[..16]`, the same rule as the LABEL_INDEX and
93/// SCHEMA sections.
94pub const TEXT_INDEX_PARTIAL_DIGEST_LEN: u64 = 16;
95
96/// A top-level file section, addressed by [`SectionKind`] in the header directory.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum SectionKind {
99    /// Dataset-card metadata.
100    Metadata,
101    /// Dictionary container (front-coded term sections).
102    Dictionary,
103    /// Permutation index container (SPO/POS/OSP).
104    Index,
105    /// Community + schema pyramid metadata.
106    PyramidMeta,
107    /// Named-graphs section.
108    NamedGraphs,
109    /// Full-text (word) index over literals — `token → subjects`.
110    TextIndex,
111    /// A full-text index that covers **only some predicates**
112    /// (`rete build --text-index-include/--text-index-exclude`): the same
113    /// layout as [`SectionKind::TextIndex`], under its own kind so that a
114    /// reader which predates text-index scopes does not see it at all. Such a
115    /// reader prunes `FILTER(CONTAINS(…))` with any text index it finds, as if
116    /// the index covered every literal; with this one it would silently drop
117    /// the rows of the predicates left out. Not seeing it, it scans instead.
118    /// The header maps it onto the same `text_index_*` fields, with
119    /// [`Header::text_index_partial`] set. Like kinds 8 and 9 it is **outside
120    /// the content hash** — an older `verify`, which cannot see it, must not
121    /// call the file corrupt — and starts with its own 16-byte digest
122    /// ([`TEXT_INDEX_PARTIAL_DIGEST_LEN`]) instead; the kind-6 layout follows
123    /// it ([`Header::text_index_payload`]).
124    TextIndexPartial,
125    /// Build-conditions record (timestamp, builder version, build parameters,
126    /// measured starter-query costs), stored as an opaque JSON blob owned by the
127    /// application layer (the CLI). **Deliberately excluded from the content
128    /// hash**: two builds of identical data must hash identically, and this
129    /// section is exactly the per-build facts (when, by which binary, how fast)
130    /// that differ between them. `verify` therefore does not cover it.
131    BuildInfo,
132    /// The range-readable **label index** (`rete build --label-index`): every
133    /// labelled subject's display label, sorted for prefix search, as a
134    /// multi-level table a reader walks with a few small range reads. See
135    /// [`crate::label_index`]. Optional; outside the content hash and
136    /// self-digested (SPEC §6.4).
137    LabelIndex,
138    /// The **schema** section: the class hierarchy, the per-level type rollups
139    /// and lateral class links, the coherence axioms, the planner's
140    /// `query_stats` and the characteristic sets — computed without the
141    /// community dendrogram. See [`crate::schema_section`]. Optional; outside
142    /// the content hash and self-digested (SPEC §7.5).
143    Schema,
144    /// The quad index (kind 10): every named-graph quad in graph-leading
145    /// permutations behind a sorted graph directory. Present exactly when the
146    /// header has [`FLAG_QUAD_INDEX`]. Kept in [`Header::extra_sections`] like
147    /// an unknown kind (it has no named convenience fields).
148    QuadIndex,
149    /// A section kind this build doesn't know — preserved verbatim on round-trip
150    /// so a newer writer's sections survive an older reader.
151    Unknown(u16),
152}
153
154impl SectionKind {
155    fn to_u16(self) -> u16 {
156        match self {
157            SectionKind::Metadata => 1,
158            SectionKind::Dictionary => 2,
159            SectionKind::Index => 3,
160            SectionKind::PyramidMeta => 4,
161            SectionKind::NamedGraphs => 5,
162            SectionKind::TextIndex => 6,
163            SectionKind::BuildInfo => 7,
164            SectionKind::LabelIndex => 8,
165            SectionKind::Schema => 9,
166            SectionKind::QuadIndex => 10,
167            SectionKind::TextIndexPartial => 11,
168            SectionKind::Unknown(k) => k,
169        }
170    }
171
172    fn from_u16(k: u16) -> Self {
173        match k {
174            1 => SectionKind::Metadata,
175            2 => SectionKind::Dictionary,
176            3 => SectionKind::Index,
177            4 => SectionKind::PyramidMeta,
178            5 => SectionKind::NamedGraphs,
179            6 => SectionKind::TextIndex,
180            7 => SectionKind::BuildInfo,
181            8 => SectionKind::LabelIndex,
182            9 => SectionKind::Schema,
183            10 => SectionKind::QuadIndex,
184            11 => SectionKind::TextIndexPartial,
185            other => SectionKind::Unknown(other),
186        }
187    }
188}
189
190/// One parsed/encoded section-directory entry: a typed `(offset, length)` into
191/// the file, plus 16 bits of per-section flags (reserved).
192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
193pub struct Section {
194    pub kind: SectionKind,
195    pub flags: u16,
196    pub offset: u64,
197    pub length: u64,
198}
199
200#[derive(Debug, thiserror::Error)]
201#[non_exhaustive]
202pub enum HeaderError {
203    #[error("buffer too small: need {HEADER_LEN} bytes, got {0}")]
204    TooSmall(usize),
205    #[error("bad magic: expected RETE")]
206    BadMagic,
207    #[error(
208        "unsupported .rete format {found:#04x}; this Rete build reads {min:#04x}..={max:#04x}. Pre-1.0 files must be rebuilt from RDF source with `rete build`"
209    )]
210    UnsupportedVersion { found: u8, min: u8, max: u8 },
211    #[error("section count {0} overruns the header frame")]
212    BadSectionCount(usize),
213    #[error("unusable index permutation mask {mask:#04x}: {why}")]
214    BadPermMask { mask: u8, why: &'static str },
215}
216
217/// Decoded file header. All multi-byte fields are little-endian on disk. The
218/// `*_offset` / `*_len` fields are a convenience view over the section directory
219/// (populated from it on parse, emitted back to it on serialize).
220#[derive(Debug, Clone, PartialEq, Eq)]
221pub struct Header {
222    /// Format version of the parsed file (currently
223    /// [`MIN_STABLE_READ_VERSION`] through [`CURRENT_FORMAT_VERSION`]).
224    pub version: u8,
225    pub flags: u8,
226    pub metadata_offset: u64,
227    pub metadata_len: u64,
228    pub dictionary_offset: u64,
229    pub dictionary_len: u64,
230    pub root_dir_offset: u64,
231    pub root_dir_len: u64,
232    pub pyramid_meta_offset: u64,
233    pub pyramid_meta_len: u64,
234    pub dict_codec: u8,
235    pub block_codec: u8,
236    pub pyramid_levels: u16,
237    pub quad_count: u64,
238    pub term_count: u64,
239    /// First 16 bytes of the blake3 content hash — an immutable validator.
240    pub content_hash: [u8; 16],
241    /// Named-graphs section (0 if the file has only the default graph).
242    pub named_graphs_offset: u64,
243    pub named_graphs_len: u64,
244    /// Byte length of the trailing **schema-pyramid block** within the pyramid-meta
245    /// section (0 if none). A reader fetches *only* that block — at
246    /// `pyramid_meta_offset + pyramid_meta_len - schema_meta_len` — for an
247    /// index/dictionary/summary-free Tier-0 coherence check.
248    pub schema_meta_len: u32,
249    /// Full-text index section (0 if the file has none; built with
250    /// `rete build --text-index`). See [`SectionKind::TextIndex`].
251    pub text_index_offset: u64,
252    pub text_index_len: u64,
253    /// The full-text index covers only some predicates: it is written (and was
254    /// read) as [`SectionKind::TextIndexPartial`] rather than
255    /// [`SectionKind::TextIndex`]. `false` for every file without one.
256    pub text_index_partial: bool,
257    /// Build-conditions section (0 if absent). See [`SectionKind::BuildInfo`]:
258    /// an opaque per-build record that is **not** covered by the content hash.
259    pub build_info_offset: u64,
260    pub build_info_len: u64,
261    /// Which index permutations the file's index containers carry
262    /// ([`crate::index::PermSet`]).
263    ///
264    /// On disk this is one byte at `[50]`, inside the reserved core span, and
265    /// **`0` means all six** — so every file written before the mask existed
266    /// decodes as [`PermSet::ALL`] and a full six-permutation build stays
267    /// byte-identical to what it always was. A lean file writes its mask
268    /// (`0b000_0111` for SPO+POS+OSP), and its index container then holds three
269    /// sections rather than six, which is what an older reader trips over.
270    ///
271    /// [`PermSet::ALL`]: crate::index::PermSet::ALL
272    pub perms: crate::index::PermSet,
273    /// Directory entries of the kinds added after the generation-1 freeze —
274    /// [`SectionKind::LabelIndex`] (8), [`SectionKind::Schema`] (9) — and of
275    /// any kind this build doesn't recognize, preserved verbatim in directory
276    /// order. None of them is covered by the header content hash.
277    pub extra_sections: Vec<Section>,
278}
279
280impl Header {
281    /// Serialize into a fixed 1024-byte array.
282    pub fn to_bytes(&self) -> [u8; HEADER_LEN] {
283        let mut b = [0u8; HEADER_LEN];
284        // --- core (64 bytes) ---
285        b[0..4].copy_from_slice(&MAGIC);
286        b[4] = self.version;
287        b[5] = self.flags;
288        b[6..8].copy_from_slice(&(HEADER_LEN as u16).to_le_bytes());
289        b[8..24].copy_from_slice(&self.content_hash);
290        b[24..32].copy_from_slice(&self.quad_count.to_le_bytes());
291        b[32..40].copy_from_slice(&self.term_count.to_le_bytes());
292        b[40..42].copy_from_slice(&self.pyramid_levels.to_le_bytes());
293        b[42] = self.dict_codec;
294        b[43] = self.block_codec;
295        // [44..46) section_count written below.
296        b[46..50].copy_from_slice(&self.schema_meta_len.to_le_bytes());
297        // [50] permutation mask; 0 = all six, so a full build is byte-identical
298        // to every file written before the field existed.
299        b[50] = if self.perms == crate::index::PermSet::ALL {
300            0
301        } else {
302            self.perms.bits()
303        };
304        // [51..64) reserved.
305
306        // --- section directory ---
307        // The five always-present sections (verbatim, so the named offsets
308        // round-trip exactly), then the optional text index (only when present, so
309        // a file without one stays byte-identical), then preserved unknown kinds.
310        let entry = |kind, offset, length| Section {
311            kind,
312            flags: 0,
313            offset,
314            length,
315        };
316        let mut entries: Vec<Section> = vec![
317            entry(
318                SectionKind::Metadata,
319                self.metadata_offset,
320                self.metadata_len,
321            ),
322            entry(
323                SectionKind::Dictionary,
324                self.dictionary_offset,
325                self.dictionary_len,
326            ),
327            entry(SectionKind::Index, self.root_dir_offset, self.root_dir_len),
328            entry(
329                SectionKind::PyramidMeta,
330                self.pyramid_meta_offset,
331                self.pyramid_meta_len,
332            ),
333            entry(
334                SectionKind::NamedGraphs,
335                self.named_graphs_offset,
336                self.named_graphs_len,
337            ),
338        ];
339        if self.text_index_len > 0 {
340            entries.push(entry(
341                if self.text_index_partial {
342                    SectionKind::TextIndexPartial
343                } else {
344                    SectionKind::TextIndex
345                },
346                self.text_index_offset,
347                self.text_index_len,
348            ));
349        }
350        if self.build_info_len > 0 {
351            entries.push(entry(
352                SectionKind::BuildInfo,
353                self.build_info_offset,
354                self.build_info_len,
355            ));
356        }
357        entries.extend(self.extra_sections.iter().copied());
358        debug_assert!(
359            entries.len() <= MAX_SECTIONS,
360            "too many sections for a 1 KB header"
361        );
362        let n = entries.len().min(MAX_SECTIONS);
363        b[44..46].copy_from_slice(&(n as u16).to_le_bytes());
364        for (i, s) in entries.iter().take(n).enumerate() {
365            let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
366            b[p..p + 2].copy_from_slice(&s.kind.to_u16().to_le_bytes());
367            b[p + 2..p + 4].copy_from_slice(&s.flags.to_le_bytes());
368            // [p+4..p+8) reserved.
369            b[p + 8..p + 16].copy_from_slice(&s.offset.to_le_bytes());
370            b[p + 16..p + 24].copy_from_slice(&s.length.to_le_bytes());
371        }
372        b
373    }
374
375    /// Parse a header from the first 1024 bytes of a file.
376    pub fn from_bytes(b: &[u8]) -> Result<Self, HeaderError> {
377        if b.len() < HEADER_LEN {
378            return Err(HeaderError::TooSmall(b.len()));
379        }
380        if b[0..4] != MAGIC {
381            return Err(HeaderError::BadMagic);
382        }
383        if !(MIN_STABLE_READ_VERSION..=CURRENT_FORMAT_VERSION).contains(&b[4]) {
384            return Err(HeaderError::UnsupportedVersion {
385                found: b[4],
386                min: MIN_STABLE_READ_VERSION,
387                max: CURRENT_FORMAT_VERSION,
388            });
389        }
390        let u16_at = |o: usize| u16::from_le_bytes(b[o..o + 2].try_into().unwrap());
391        let u32_at = |o: usize| u32::from_le_bytes(b[o..o + 4].try_into().unwrap());
392        let u64_at = |o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
393
394        let section_count = u16_at(44) as usize;
395        if SECTION_DIR_OFFSET + section_count * SECTION_ENTRY_LEN > HEADER_LEN {
396            return Err(HeaderError::BadSectionCount(section_count));
397        }
398
399        let perms = if b[50] == 0 {
400            crate::index::PermSet::ALL
401        } else {
402            crate::index::PermSet::from_bits(b[50])
403                .map_err(|why| HeaderError::BadPermMask { mask: b[50], why })?
404        };
405
406        let mut h = Header {
407            version: b[4],
408            flags: b[5],
409            metadata_offset: 0,
410            metadata_len: 0,
411            dictionary_offset: 0,
412            dictionary_len: 0,
413            root_dir_offset: 0,
414            root_dir_len: 0,
415            pyramid_meta_offset: 0,
416            pyramid_meta_len: 0,
417            dict_codec: b[42],
418            block_codec: b[43],
419            pyramid_levels: u16_at(40),
420            quad_count: u64_at(24),
421            term_count: u64_at(32),
422            content_hash: b[8..24].try_into().unwrap(),
423            named_graphs_offset: 0,
424            named_graphs_len: 0,
425            schema_meta_len: u32_at(46),
426            perms,
427            text_index_offset: 0,
428            text_index_len: 0,
429            text_index_partial: false,
430            build_info_offset: 0,
431            build_info_len: 0,
432            extra_sections: Vec::new(),
433        };
434        for i in 0..section_count {
435            let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
436            let kind = SectionKind::from_u16(u16_at(p));
437            let offset = u64_at(p + 8);
438            let length = u64_at(p + 16);
439            match kind {
440                SectionKind::Metadata => {
441                    h.metadata_offset = offset;
442                    h.metadata_len = length;
443                }
444                SectionKind::Dictionary => {
445                    h.dictionary_offset = offset;
446                    h.dictionary_len = length;
447                }
448                SectionKind::Index => {
449                    h.root_dir_offset = offset;
450                    h.root_dir_len = length;
451                }
452                SectionKind::PyramidMeta => {
453                    h.pyramid_meta_offset = offset;
454                    h.pyramid_meta_len = length;
455                }
456                SectionKind::NamedGraphs => {
457                    h.named_graphs_offset = offset;
458                    h.named_graphs_len = length;
459                }
460                SectionKind::TextIndex => {
461                    h.text_index_offset = offset;
462                    h.text_index_len = length;
463                    h.text_index_partial = false;
464                }
465                SectionKind::TextIndexPartial => {
466                    h.text_index_offset = offset;
467                    h.text_index_len = length;
468                    h.text_index_partial = true;
469                }
470                SectionKind::BuildInfo => {
471                    h.build_info_offset = offset;
472                    h.build_info_len = length;
473                }
474                SectionKind::LabelIndex
475                | SectionKind::Schema
476                | SectionKind::QuadIndex
477                | SectionKind::Unknown(_) => h.extra_sections.push(Section {
478                    kind,
479                    flags: u16_at(p + 2),
480                    offset,
481                    length,
482                }),
483            }
484        }
485        Ok(h)
486    }
487
488    /// The LABEL_INDEX section (kind 8), if the file carries a non-empty one.
489    pub fn label_index_section(&self) -> Option<Section> {
490        self.section(SectionKind::LabelIndex)
491            .filter(|s| s.length > 0)
492    }
493
494    /// The SCHEMA section (kind 9), if the file carries a non-empty one.
495    pub fn schema_section(&self) -> Option<Section> {
496        self.section(SectionKind::Schema).filter(|s| s.length > 0)
497    }
498
499    pub fn has_quads(&self) -> bool {
500        self.flags & FLAG_HAS_QUADS != 0
501    }
502
503    /// The total file length implied by this header alone: the writer lays the
504    /// sections out back-to-back after the 1 KiB header and ends the file with
505    /// the 4-byte `RETE` footer, so the furthest `offset + length` in the
506    /// section directory plus 4 **is** the file length.
507    ///
508    /// This is the one length signal a *remote* reader can always trust: an
509    /// HTTP host may advertise the size of a compressed representation in
510    /// `Content-Length` (GitHub Pages gzips — a 71 MB file HEADs as 58 MB)
511    /// while serving ranges over the identity bytes, and may hide
512    /// `Content-Range` from cross-origin JS by omitting
513    /// `Access-Control-Expose-Headers`. The header travels *in band* — in the
514    /// first 1 KiB range read — so it cannot be skewed by transport encoding.
515    ///
516    /// `None` if any directory entry overflows `u64` (a crafted or corrupt
517    /// header must yield "unknown", never a panic or a wrapped length — the
518    /// weekly fuzz caught `verify()` on exactly this class of input).
519    pub fn expected_file_len(&self) -> Option<u64> {
520        let mut end = HEADER_LEN as u64;
521        let named = [
522            (self.metadata_offset, self.metadata_len),
523            (self.dictionary_offset, self.dictionary_len),
524            (self.root_dir_offset, self.root_dir_len),
525            (self.pyramid_meta_offset, self.pyramid_meta_len),
526            (self.named_graphs_offset, self.named_graphs_len),
527            (self.text_index_offset, self.text_index_len),
528            (self.build_info_offset, self.build_info_len),
529        ];
530        let extras = self.extra_sections.iter().map(|s| (s.offset, s.length));
531        for (offset, length) in named.into_iter().chain(extras) {
532            if length == 0 {
533                continue; // absent section — its (0, 0) entry says nothing
534            }
535            end = end.max(offset.checked_add(length)?);
536        }
537        end.checked_add(MAGIC.len() as u64)
538    }
539
540    /// Does the file contain RDF-star quoted triples ([`FLAG_HAS_QUOTED_TRIPLES`])?
541    pub fn has_quoted_triples(&self) -> bool {
542        self.flags & FLAG_HAS_QUOTED_TRIPLES != 0
543    }
544
545    /// Do the tiled index sections carry a [`FLAG_TILE_SYNOPSIS`] trailer?
546    pub fn has_tile_synopsis(&self) -> bool {
547        self.flags & FLAG_TILE_SYNOPSIS != 0
548    }
549
550    /// Does the full-text index cover the named graphs' literals too
551    /// ([`FLAG_TEXT_INDEX_NAMED_GRAPHS`])? Known from the header alone.
552    pub fn text_index_covers_named_graphs(&self) -> bool {
553        self.flags & FLAG_TEXT_INDEX_NAMED_GRAPHS != 0
554    }
555
556    /// The `(offset, length)` of the text index's payload — the token-table
557    /// varint, the token table and the postings. That is the whole section for
558    /// kind 6; a kind-11 section starts with its 16-byte self-digest, which this
559    /// skips. `(offset, 0)` past a kind-11 section too short for its digest.
560    pub fn text_index_payload(&self) -> (u64, u64) {
561        if self.text_index_partial {
562            (
563                self.text_index_offset
564                    .saturating_add(TEXT_INDEX_PARTIAL_DIGEST_LEN),
565                self.text_index_len
566                    .saturating_sub(TEXT_INDEX_PARTIAL_DIGEST_LEN),
567            )
568        } else {
569            (self.text_index_offset, self.text_index_len)
570        }
571    }
572
573    /// The directory entry for a section kind, or `None` if absent. Known kinds
574    /// read from the named convenience fields; unknown kinds from
575    /// [`extra_sections`](Self::extra_sections).
576    pub fn section(&self, kind: SectionKind) -> Option<Section> {
577        let (offset, length) = match kind {
578            SectionKind::Metadata => (self.metadata_offset, self.metadata_len),
579            SectionKind::Dictionary => (self.dictionary_offset, self.dictionary_len),
580            SectionKind::Index => (self.root_dir_offset, self.root_dir_len),
581            SectionKind::PyramidMeta => (self.pyramid_meta_offset, self.pyramid_meta_len),
582            SectionKind::NamedGraphs => (self.named_graphs_offset, self.named_graphs_len),
583            SectionKind::TextIndex if !self.text_index_partial => {
584                (self.text_index_offset, self.text_index_len)
585            }
586            SectionKind::TextIndexPartial if self.text_index_partial => {
587                (self.text_index_offset, self.text_index_len)
588            }
589            SectionKind::TextIndex | SectionKind::TextIndexPartial => (0, 0),
590            SectionKind::BuildInfo => (self.build_info_offset, self.build_info_len),
591            SectionKind::LabelIndex
592            | SectionKind::Schema
593            | SectionKind::QuadIndex
594            | SectionKind::Unknown(_) => {
595                return self.extra_sections.iter().find(|s| s.kind == kind).copied()
596            }
597        };
598        Some(Section {
599            kind,
600            flags: 0,
601            offset,
602            length,
603        })
604    }
605
606    /// Attach (or overwrite) a section's `(offset, length)` — the extension point
607    /// for new top-level sections. Known kinds set the named fields; an unknown
608    /// kind is appended to [`extra_sections`](Self::extra_sections).
609    pub fn with_section(mut self, kind: SectionKind, offset: u64, length: u64) -> Self {
610        match kind {
611            SectionKind::Metadata => {
612                self.metadata_offset = offset;
613                self.metadata_len = length;
614            }
615            SectionKind::Dictionary => {
616                self.dictionary_offset = offset;
617                self.dictionary_len = length;
618            }
619            SectionKind::Index => {
620                self.root_dir_offset = offset;
621                self.root_dir_len = length;
622            }
623            SectionKind::PyramidMeta => {
624                self.pyramid_meta_offset = offset;
625                self.pyramid_meta_len = length;
626            }
627            SectionKind::NamedGraphs => {
628                self.named_graphs_offset = offset;
629                self.named_graphs_len = length;
630            }
631            SectionKind::TextIndex => {
632                self.text_index_offset = offset;
633                self.text_index_len = length;
634                self.text_index_partial = false;
635            }
636            SectionKind::TextIndexPartial => {
637                self.text_index_offset = offset;
638                self.text_index_len = length;
639                self.text_index_partial = true;
640            }
641            SectionKind::BuildInfo => {
642                self.build_info_offset = offset;
643                self.build_info_len = length;
644            }
645            SectionKind::LabelIndex | SectionKind::Schema | SectionKind::QuadIndex => {
646                // A known post-freeze kind appears at most once: overwrite.
647                match self.extra_sections.iter_mut().find(|s| s.kind == kind) {
648                    Some(s) => {
649                        s.offset = offset;
650                        s.length = length;
651                    }
652                    None => self.extra_sections.push(Section {
653                        kind,
654                        flags: 0,
655                        offset,
656                        length,
657                    }),
658                }
659            }
660            SectionKind::Unknown(_) => self.extra_sections.push(Section {
661                kind,
662                flags: 0,
663                offset,
664                length,
665            }),
666        }
667        self
668    }
669
670    /// Are the named graphs stored in the quad index ([`FLAG_QUAD_INDEX`])?
671    pub fn has_quad_index(&self) -> bool {
672        self.flags & FLAG_QUAD_INDEX != 0
673    }
674
675    /// The QUAD_INDEX section, when the directory lists a non-empty one.
676    pub fn quad_index_section(&self) -> Option<Section> {
677        self.section(SectionKind::QuadIndex)
678            .filter(|s| s.length > 0)
679    }
680
681    /// How many sections the index container (kind 3) holds: one per stored
682    /// permutation, plus the quad-index seal when [`FLAG_QUAD_INDEX`] is set.
683    pub fn index_container_sections(&self) -> usize {
684        self.perms.len() + usize::from(self.has_quad_index())
685    }
686}
687
688#[cfg(test)]
689mod tests {
690    use super::*;
691
692    fn sample() -> Header {
693        Header {
694            version: CURRENT_FORMAT_VERSION,
695            flags: FLAG_HAS_QUADS,
696            metadata_offset: 1024,
697            metadata_len: 42,
698            dictionary_offset: 1066,
699            dictionary_len: 2048,
700            root_dir_offset: 3114,
701            root_dir_len: 256,
702            pyramid_meta_offset: 3370,
703            pyramid_meta_len: 64,
704            dict_codec: 1,
705            block_codec: 2,
706            pyramid_levels: 3,
707            perms: crate::index::PermSet::ALL,
708            quad_count: 5,
709            term_count: 9,
710            content_hash: [7u8; 16],
711            named_graphs_offset: 3434,
712            named_graphs_len: 48,
713            schema_meta_len: 99,
714            text_index_offset: 0,
715            text_index_len: 0,
716            text_index_partial: false,
717            build_info_offset: 0,
718            build_info_len: 0,
719            extra_sections: Vec::new(),
720        }
721    }
722
723    #[test]
724    fn round_trip() {
725        let h = sample();
726        let bytes = h.to_bytes();
727        assert_eq!(bytes.len(), HEADER_LEN);
728        assert_eq!(&bytes[0..4], b"RETE");
729        let back = Header::from_bytes(&bytes).unwrap();
730        assert_eq!(h, back);
731        assert!(back.has_quads());
732    }
733
734    #[test]
735    fn byte_layout_matches_spec() {
736        // Pins the core fields and the first directory entry to exact offsets.
737        let h = Header {
738            content_hash: [0xCC; 16],
739            quad_count: 0x99,
740            term_count: 0xAA,
741            pyramid_levels: 0xABCD,
742            dict_codec: 0xA1,
743            block_codec: 0xA2,
744            schema_meta_len: 0xD00D,
745            metadata_offset: 0x11,
746            metadata_len: 0x22,
747            dictionary_offset: 0x33,
748            dictionary_len: 0x44,
749            ..sample()
750        };
751        let b = h.to_bytes();
752        let u16_at = |o: usize| u16::from_le_bytes(b[o..o + 2].try_into().unwrap());
753        let u64_at = |o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
754
755        // core
756        assert_eq!(&b[0..4], b"RETE");
757        assert_eq!(b[4], CURRENT_FORMAT_VERSION);
758        assert_eq!(b[5], FLAG_HAS_QUADS);
759        assert_eq!(u16_at(6), HEADER_LEN as u16);
760        assert_eq!(&b[8..24], &[0xCC; 16]); // content hash
761        assert_eq!(u64_at(24), 0x99); // quad count
762        assert_eq!(u64_at(32), 0xAA); // term count
763        assert_eq!(u16_at(40), 0xABCD); // pyramid levels
764        assert_eq!(b[42], 0xA1); // dict codec
765        assert_eq!(b[43], 0xA2); // block codec
766        assert_eq!(u16_at(44), 5); // section_count: the 5 known sections
767        assert_eq!(u32::from_le_bytes(b[46..50].try_into().unwrap()), 0xD00D); // schema-meta len
768                                                                               // first directory entry = Metadata, at offset 64
769        assert_eq!(u16_at(64), 1); // kind = Metadata
770        assert_eq!(u64_at(72), 0x11); // metadata offset
771        assert_eq!(u64_at(80), 0x22); // metadata length
772                                      // second entry = Dictionary, at 88
773        assert_eq!(u16_at(88), 2);
774        assert_eq!(u64_at(96), 0x33);
775        assert_eq!(u64_at(104), 0x44);
776        assert_eq!(b.len(), HEADER_LEN);
777    }
778
779    #[test]
780    fn expected_file_len_is_last_section_end_plus_footer() {
781        // sample(): named graphs end furthest, at 3434 + 48 = 3482.
782        assert_eq!(sample().expected_file_len(), Some(3482 + 4));
783
784        // A zero-length section's offset must not count (absent sections are
785        // written as (0, 0), and a stale offset with len 0 addresses nothing).
786        let mut h = sample();
787        h.text_index_offset = 1 << 40;
788        h.text_index_len = 0;
789        assert_eq!(h.expected_file_len(), Some(3482 + 4));
790
791        // An empty file (header + footer only) is 1028 bytes.
792        let mut e = sample();
793        e.metadata_len = 0;
794        e.dictionary_len = 0;
795        e.root_dir_len = 0;
796        e.pyramid_meta_len = 0;
797        e.named_graphs_len = 0;
798        assert_eq!(e.expected_file_len(), Some(1024 + 4));
799
800        // A crafted header whose entry overflows u64 yields None, not a panic
801        // or a wrapped (tiny) length.
802        let mut c = sample();
803        c.named_graphs_offset = u64::MAX - 8;
804        c.named_graphs_len = 64;
805        assert_eq!(c.expected_file_len(), None);
806
807        // Unknown (future) sections extend the file too.
808        let f = sample().with_section(SectionKind::Unknown(99), 1 << 20, 512);
809        assert_eq!(f.expected_file_len(), Some((1 << 20) + 512 + 4));
810    }
811
812    #[test]
813    fn rejects_bad_magic() {
814        let mut bytes = [0u8; HEADER_LEN];
815        bytes[4] = CURRENT_FORMAT_VERSION;
816        assert!(matches!(
817            Header::from_bytes(&bytes),
818            Err(HeaderError::BadMagic)
819        ));
820    }
821
822    #[test]
823    fn stable_reader_accepts_v1_baseline_and_rejects_pre_v1() {
824        let current = sample().to_bytes();
825        assert_eq!(current[4], 0x05);
826        assert_eq!(Header::from_bytes(&current).unwrap().version, 0x05);
827
828        for old in 0x01..=0x04 {
829            let mut bytes = current;
830            bytes[4] = old;
831            let error = Header::from_bytes(&bytes).unwrap_err();
832            assert!(matches!(
833                &error,
834                HeaderError::UnsupportedVersion {
835                    found,
836                    min: 0x05,
837                    max: 0x05
838                } if *found == old
839            ));
840            assert!(error
841                .to_string()
842                .contains("Pre-1.0 files must be rebuilt from RDF source with `rete build`"));
843        }
844
845        for unsupported in [0x00, 0x06, 0xff] {
846            let mut bytes = current;
847            bytes[4] = unsupported;
848            assert!(matches!(
849                Header::from_bytes(&bytes),
850                Err(HeaderError::UnsupportedVersion {
851                    found,
852                    min: 0x05,
853                    max: 0x05
854                }) if found == unsupported
855            ));
856        }
857    }
858
859    #[test]
860    fn rejects_overrunning_section_count() {
861        let mut bad = sample().to_bytes();
862        bad[44..46].copy_from_slice(&9999u16.to_le_bytes());
863        assert!(matches!(
864            Header::from_bytes(&bad),
865            Err(HeaderError::BadSectionCount(9999))
866        ));
867    }
868
869    #[test]
870    fn unknown_section_survives_round_trip() {
871        // A section a future build added (kind 99) must be preserved verbatim by a
872        // reader that doesn't know it, and readable via `section()`.
873        let h = sample().with_section(SectionKind::Unknown(99), 4096, 512);
874        let back = Header::from_bytes(&h.to_bytes()).unwrap();
875        assert_eq!(back.extra_sections.len(), 1);
876        let s = back.section(SectionKind::Unknown(99)).unwrap();
877        assert_eq!((s.offset, s.length), (4096, 512));
878        // Known sections still resolve.
879        let dict = back.section(SectionKind::Dictionary).unwrap();
880        assert_eq!(dict.offset, h.dictionary_offset);
881        assert_eq!(h, back);
882    }
883
884    #[test]
885    fn build_info_section_round_trips_and_is_optional() {
886        // Absent (len 0): only the 5 always-present sections — a file without a
887        // build-info section is byte-identical to one built before it existed.
888        let plain = sample().to_bytes();
889        assert_eq!(u16::from_le_bytes(plain[44..46].try_into().unwrap()), 5);
890        assert_eq!(sample().section(SectionKind::BuildInfo).unwrap().length, 0);
891
892        // Present: a directory entry that round-trips and resolves as kind 7.
893        let h = sample().with_section(SectionKind::BuildInfo, 1066, 300);
894        let bytes = h.to_bytes();
895        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
896        let back = Header::from_bytes(&bytes).unwrap();
897        assert_eq!(h, back);
898        let s = back.section(SectionKind::BuildInfo).unwrap();
899        assert_eq!((s.offset, s.length), (1066, 300));
900        assert!(back.extra_sections.is_empty(), "BuildInfo is a known kind");
901        // And it extends the expected file length like any other section.
902        let far = sample().with_section(SectionKind::BuildInfo, 1 << 21, 128);
903        assert_eq!(far.expected_file_len(), Some((1 << 21) + 128 + 4));
904    }
905
906    #[test]
907    fn label_index_and_schema_sections_are_known_extras() {
908        // Absent: the directory is unchanged (5 entries), so a file without
909        // them is byte-identical to one built before they existed.
910        assert!(sample().label_index_section().is_none());
911        assert!(sample().schema_section().is_none());
912
913        let h = sample()
914            .with_section(SectionKind::LabelIndex, 8000, 300)
915            .with_section(SectionKind::Schema, 8300, 40);
916        let bytes = h.to_bytes();
917        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 7);
918        // entries 5 and 6 carry kinds 8 and 9 at the directory's byte layout
919        assert_eq!(
920            u16::from_le_bytes(bytes[64 + 5 * 24..][..2].try_into().unwrap()),
921            8
922        );
923        assert_eq!(
924            u16::from_le_bytes(bytes[64 + 6 * 24..][..2].try_into().unwrap()),
925            9
926        );
927        let back = Header::from_bytes(&bytes).unwrap();
928        assert_eq!(h, back);
929        assert_eq!(back.extra_sections.len(), 2, "stored with the extras");
930        let l = back.label_index_section().unwrap();
931        assert_eq!((l.offset, l.length), (8000, 300));
932        let s = back.schema_section().unwrap();
933        assert_eq!((s.offset, s.length), (8300, 40));
934        assert_eq!(back.expected_file_len(), Some(8340 + 4));
935
936        // A known post-freeze kind is overwritten, never duplicated.
937        let again = back.with_section(SectionKind::Schema, 9000, 10);
938        assert_eq!(again.extra_sections.len(), 2);
939        assert_eq!(again.schema_section().unwrap().offset, 9000);
940
941        // A zero-length entry reads as absent.
942        let empty = sample().with_section(SectionKind::LabelIndex, 0, 0);
943        assert!(empty.label_index_section().is_none());
944    }
945
946    #[test]
947    fn quad_index_flag_and_section_round_trip() {
948        // Absent: no flag, no kind-10 entry, the container is the permutations.
949        let plain = sample();
950        assert!(!plain.has_quad_index());
951        assert!(plain.quad_index_section().is_none());
952        assert_eq!(plain.index_container_sections(), 6);
953
954        // Present: flag bit 3, a kind-10 entry kept with the extra sections,
955        // and one more container section (the seal).
956        let mut h = sample().with_section(SectionKind::QuadIndex, 7000, 900);
957        h.flags |= FLAG_QUAD_INDEX;
958        h.named_graphs_offset = 0;
959        h.named_graphs_len = 0;
960        let bytes = h.to_bytes();
961        assert_eq!(bytes[5] & 0b1000, 0b1000, "flag bit 3");
962        let n = u16::from_le_bytes(bytes[44..46].try_into().unwrap()) as usize;
963        let kinds: Vec<u16> = (0..n)
964            .map(|i| {
965                let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
966                u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap())
967            })
968            .collect();
969        assert!(kinds.contains(&10), "a kind-10 directory entry: {kinds:?}");
970        let back = Header::from_bytes(&bytes).unwrap();
971        assert_eq!(back, h);
972        assert!(back.has_quad_index());
973        let s = back.quad_index_section().unwrap();
974        assert_eq!(
975            (s.kind, s.offset, s.length),
976            (SectionKind::QuadIndex, 7000, 900)
977        );
978        assert_eq!(back.index_container_sections(), 7);
979        assert_eq!(back.expected_file_len(), Some(7000 + 900 + 4));
980        // Three permutations + seal = four: never the 3 or 6 an old reader wants.
981        let mut lean = back.clone();
982        lean.perms = crate::index::PermSet::CORE;
983        assert_eq!(lean.index_container_sections(), 4);
984        // Re-attaching overwrites rather than duplicating.
985        let again = back.with_section(SectionKind::QuadIndex, 8000, 10);
986        assert_eq!(again.extra_sections.len(), 1);
987        assert_eq!(again.quad_index_section().unwrap().offset, 8000);
988    }
989
990    #[test]
991    fn text_index_section_round_trips_and_is_optional() {
992        // Absent (len 0): only the 5 always-present sections, so a file without a
993        // text index is byte-identical to one built before this section existed.
994        assert_eq!(
995            u16::from_le_bytes(sample().to_bytes()[44..46].try_into().unwrap()),
996            5
997        );
998        assert!(sample().section(SectionKind::TextIndex).unwrap().length == 0);
999
1000        // Present: a 6th directory entry that round-trips and resolves.
1001        let h = sample().with_section(SectionKind::TextIndex, 5000, 4096);
1002        let bytes = h.to_bytes();
1003        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
1004        let back = Header::from_bytes(&bytes).unwrap();
1005        assert_eq!(h, back);
1006        let s = back.section(SectionKind::TextIndex).unwrap();
1007        assert_eq!((s.offset, s.length), (5000, 4096));
1008        assert!(back.extra_sections.is_empty(), "TextIndex is a known kind");
1009        assert!(!back.text_index_partial);
1010    }
1011
1012    /// A predicate-restricted text index is written as kind 11, maps onto the
1013    /// same `text_index_*` fields, and a reader that only knows kind 6 sees an
1014    /// unknown section — no text index at all — rather than a full one.
1015    #[test]
1016    fn partial_text_index_is_kind_11_and_invisible_as_kind_6() {
1017        let h = sample().with_section(SectionKind::TextIndexPartial, 5000, 4096);
1018        assert!(h.text_index_partial);
1019        let bytes = h.to_bytes();
1020        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
1021        // The 6th entry, at 64 + 5 × 24, carries kind 11.
1022        let p = SECTION_DIR_OFFSET + 5 * SECTION_ENTRY_LEN;
1023        assert_eq!(u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap()), 11);
1024        let back = Header::from_bytes(&bytes).unwrap();
1025        assert_eq!(h, back);
1026        assert!(back.text_index_partial);
1027        assert_eq!((back.text_index_offset, back.text_index_len), (5000, 4096));
1028        let s = back.section(SectionKind::TextIndexPartial).unwrap();
1029        assert_eq!((s.offset, s.length), (5000, 4096));
1030        assert_eq!(back.section(SectionKind::TextIndex).unwrap().length, 0);
1031        assert!(back.extra_sections.is_empty(), "kind 11 is known here");
1032
1033        // What a reader that predates kind 11 decodes: an unknown kind, which
1034        // it preserves, and empty text-index fields — so it neither searches
1035        // nor prunes with the partial index.
1036        let mut old_view = bytes;
1037        old_view[p..p + 2].copy_from_slice(&0x7777u16.to_le_bytes());
1038        let old = Header::from_bytes(&old_view).unwrap();
1039        assert_eq!(old.text_index_len, 0);
1040        assert_eq!(old.extra_sections.len(), 1);
1041    }
1042}
rete_core/header.rs

rete_core/
header.rs
