Skip to main content

rete_core/
header.rs

1//! The fixed-size 1024-byte file header (SPEC.md §4.1).
2//!
3//! A client's very first request is `bytes=0-1023`; this struct is what those
4//! bytes decode to, and it points at every other section of the file via a
5//! **typed section directory**, so new top-level sections (e.g. a future text
6//! index) are added as a new directory entry without reshaping the header.
7//!
8//! Layout: a fixed 64-byte **core** (magic, version, flags, content hash, counts,
9//! codecs, `section_count`, `schema_meta_len`) followed by up to
10//! [`MAX_SECTIONS`] **directory entries** of 24 bytes each `(kind, flags, offset,
11//! length)`, zero-padded to 1024. The known section kinds populate the named
12//! convenience fields below; unknown kinds (written by a newer build) are
13//! preserved verbatim in [`Header::extra_sections`].
14
15use std::convert::TryInto;
16
17/// Magic bytes at offset 0: ASCII `RETE`.
18pub const MAGIC: [u8; 4] = *b"RETE";
19
20/// Current format generation written by this crate.
21///
22/// `0x05` is stable format generation 1, frozen on 2026-07-14 and first
23/// released in Rete 0.3.0. It retains the six-wide index-permutation addressing
24/// and the 1 KiB section-directory layout finalized in the last experimental
25/// generation; *which* of those six a given file stores is [`Header::perms`]
26/// (byte 50), not the generation.
27///
28/// The generation number is not a release version — there is no Rete 1.0.0, and
29/// it is the Rust/CLI/WASM APIs, not the format, that are waiting on one. Nor
30/// does it pin reader capability: writer semantics changed inside `0x05` nine
31/// days after the freeze (#68 split oversized index groups across tiles), so a
32/// `0x05` file can need a reader newer than the one that froze `0x05`.
33pub const CURRENT_FORMAT_VERSION: u8 = 0x08;
34
35/// Format generation `0x05` (stable generation 1, the 1 KiB section-directory
36/// layout this module parses). Every reader still reads it while the catalog is
37/// converted; the memory-bounded build, `merge` and `recard` still write it
38/// until they move to `0x06`.
39pub const LEGACY_FORMAT_VERSION: u8 = 0x05;
40
41/// Oldest stable format generation accepted by this reader.
42///
43/// Files written before the generation-1 freeze used experimental generations
44/// `0x01` through `0x04` and must be rebuilt from their RDF source. `0x05` has
45/// not moved since it froze on 2026-07-14, but that is a track record, not a
46/// promise: no backwards compatibility is guaranteed before 1.0.0, so this
47/// constant may yet be raised and a `0x05` file may yet need a rebuild. See
48/// `docs/compatibility.md`.
49pub const MIN_STABLE_READ_VERSION: u8 = 0x05;
50
51/// Fixed header size in bytes.
52pub const HEADER_LEN: usize = 1024;
53
54/// Byte offset of the first section-directory entry (i.e. the core size).
55const SECTION_DIR_OFFSET: usize = 64;
56/// Size of one section-directory entry.
57const SECTION_ENTRY_LEN: usize = 24;
58/// How many directory entries fit in the 1 KB frame.
59pub const MAX_SECTIONS: usize = (HEADER_LEN - SECTION_DIR_OFFSET) / SECTION_ENTRY_LEN;
60
61/// Flag bit: the file contains named graphs (quads) rather than triples only.
62pub const FLAG_HAS_QUADS: u8 = 0b0000_0001;
63
64/// Flag bit: each tiled permutation section carries a **tile-synopsis trailer**
65/// — per-tile min/max of the two non-leading columns, appended after the tile
66/// payloads. It lets a range reader prune a routed tile by a bound secondary
67/// component *before* fetching it. See `file.rs::encode_tiled_section`.
68pub const FLAG_TILE_SYNOPSIS: u8 = 0b0000_0010;
69
70/// Flag bit: the file contains **RDF-star quoted triples** (`<< s p o >>`) as
71/// dictionary terms. Purely informational for compatibility — a plain-RDF
72/// consumer can detect from the header alone that some terms are quoted triples
73/// (which it may not understand) without scanning the dictionary. The file is
74/// otherwise a normal `.rete`: quoted triples are stored like any other term, so
75/// this needs no format-version bump and old readers stay forward-compatible.
76pub const FLAG_HAS_QUOTED_TRIPLES: u8 = 0b0000_0100;
77
78/// Flag bit: the named graphs live in the **quad index** — the QUAD_INDEX
79/// section ([`SectionKind::QuadIndex`], kind 10) holds every named-graph quad as
80/// the fourth column of graph-leading permutations, the NAMED_GRAPHS section
81/// (kind 5) is absent, and the index container carries one trailing **seal**
82/// section (see `quad.rs`). Readers before this flag do not check it; they
83/// refuse the file anyway, on the seal (the container's section count), so a
84/// quad-index file never opens as one without named graphs
85/// (`docs/compatibility.md`).
86pub const FLAG_QUAD_INDEX: u8 = 0b0000_1000;
87
88/// Flag bit: the full-text index also covers the **named graphs'** literals
89/// (`rete build --text-index-graphs all`, Rete 0.4.0). It lets a query inside
90/// `GRAPH`, under `FROM` or over the union default graph decide from the
91/// header alone whether the index can prune its `FILTER(CONTAINS(…))`: without
92/// the bit it scans and never faults the index's token table. Bit 3
93/// (`0b0000_1000`) belongs to the quad index. Older readers ignore unknown bits.
94pub const FLAG_TEXT_INDEX_NAMED_GRAPHS: u8 = 0b0001_0000;
95
96/// Flag bit of a `0x05` **image**: the dictionary sections are in the SPARQL
97/// term order ([`crate::term_order::TermOrder::Sparql`]), not byte order.
98///
99/// A real `0x05` file never has it (the order of `0x05` is byte order, and no
100/// writer has put a `0x05` file on disk since `0x06`); the in-memory image a
101/// build hands to the `0x08` writer, and the datasets the wasm playground builds
102/// in the browser, do. The writer of a file refuses to stamp an older
103/// generation on such an image.
104pub const FLAG_SPARQL_TERM_ORDER: u8 = 0b0010_0000;
105
106/// Length of the BLAKE3 self-digest that heads a predicate-restricted text
107/// index section ([`SectionKind::TextIndexPartial`], kind 11):
108/// `digest = blake3(section[16..])[..16]`, the same rule as the LABEL_INDEX and
109/// SCHEMA sections.
110pub const TEXT_INDEX_PARTIAL_DIGEST_LEN: u64 = 16;
111
112/// A top-level file section, addressed by [`SectionKind`] in the header directory.
113#[derive(Debug, Clone, Copy, PartialEq, Eq)]
114pub enum SectionKind {
115    /// Dataset-card metadata.
116    Metadata,
117    /// Dictionary container (front-coded term sections).
118    Dictionary,
119    /// Permutation index container (SPO/POS/OSP).
120    Index,
121    /// Community + schema pyramid metadata.
122    PyramidMeta,
123    /// Named-graphs section.
124    NamedGraphs,
125    /// Full-text (word) index over literals — `token → subjects`.
126    TextIndex,
127    /// A full-text index that covers **only some predicates**
128    /// (`rete build --text-index-include/--text-index-exclude`): the same
129    /// layout as [`SectionKind::TextIndex`], under its own kind so that a
130    /// reader which predates text-index scopes does not see it at all. Such a
131    /// reader prunes `FILTER(CONTAINS(…))` with any text index it finds, as if
132    /// the index covered every literal; with this one it would silently drop
133    /// the rows of the predicates left out. Not seeing it, it scans instead.
134    /// The header maps it onto the same `text_index_*` fields, with
135    /// [`Header::text_index_partial`] set. Like kinds 8 and 9 it is **outside
136    /// the content hash** — an older `verify`, which cannot see it, must not
137    /// call the file corrupt — and starts with its own 16-byte digest
138    /// ([`TEXT_INDEX_PARTIAL_DIGEST_LEN`]) instead; the kind-6 layout follows
139    /// it ([`Header::text_index_payload`]).
140    TextIndexPartial,
141    /// Build-conditions record (timestamp, builder version, build parameters,
142    /// measured starter-query costs), stored as an opaque JSON blob owned by the
143    /// application layer (the CLI). **Deliberately excluded from the content
144    /// hash**: two builds of identical data must hash identically, and this
145    /// section is exactly the per-build facts (when, by which binary, how fast)
146    /// that differ between them. `verify` therefore does not cover it.
147    BuildInfo,
148    /// The range-readable **label index** (`rete build --label-index`): every
149    /// labelled subject's display label, sorted for prefix search, as a
150    /// multi-level table a reader walks with a few small range reads. See
151    /// [`crate::label_index`]. Optional; outside the content hash and
152    /// self-digested (SPEC §6.4).
153    LabelIndex,
154    /// The **schema** section: the class hierarchy, the per-level type rollups
155    /// and lateral class links, the coherence axioms, the planner's
156    /// `query_stats` and the characteristic sets — computed without the
157    /// community dendrogram. See [`crate::schema_section`]. Optional; outside
158    /// the content hash and self-digested (SPEC §7.5).
159    Schema,
160    /// The quad index (kind 10): every named-graph quad in graph-leading
161    /// permutations behind a sorted graph directory. Present exactly when the
162    /// header has [`FLAG_QUAD_INDEX`]. Kept in [`Header::extra_sections`] like
163    /// an unknown kind (it has no named convenience fields).
164    QuadIndex,
165    /// A section kind this build doesn't know — preserved verbatim on round-trip
166    /// so a newer writer's sections survive an older reader.
167    Unknown(u16),
168}
169
170impl SectionKind {
171    fn to_u16(self) -> u16 {
172        match self {
173            SectionKind::Metadata => 1,
174            SectionKind::Dictionary => 2,
175            SectionKind::Index => 3,
176            SectionKind::PyramidMeta => 4,
177            SectionKind::NamedGraphs => 5,
178            SectionKind::TextIndex => 6,
179            SectionKind::BuildInfo => 7,
180            SectionKind::LabelIndex => 8,
181            SectionKind::Schema => 9,
182            SectionKind::QuadIndex => 10,
183            SectionKind::TextIndexPartial => 11,
184            SectionKind::Unknown(k) => k,
185        }
186    }
187
188    fn from_u16(k: u16) -> Self {
189        match k {
190            1 => SectionKind::Metadata,
191            2 => SectionKind::Dictionary,
192            3 => SectionKind::Index,
193            4 => SectionKind::PyramidMeta,
194            5 => SectionKind::NamedGraphs,
195            6 => SectionKind::TextIndex,
196            7 => SectionKind::BuildInfo,
197            8 => SectionKind::LabelIndex,
198            9 => SectionKind::Schema,
199            10 => SectionKind::QuadIndex,
200            11 => SectionKind::TextIndexPartial,
201            other => SectionKind::Unknown(other),
202        }
203    }
204}
205
206/// One parsed/encoded section-directory entry: a typed `(offset, length)` into
207/// the file, plus 16 bits of per-section flags (reserved).
208#[derive(Debug, Clone, Copy, PartialEq, Eq)]
209pub struct Section {
210    pub kind: SectionKind,
211    pub flags: u16,
212    pub offset: u64,
213    pub length: u64,
214}
215
216#[derive(Debug, thiserror::Error)]
217#[non_exhaustive]
218pub enum HeaderError {
219    #[error("buffer too small: need {HEADER_LEN} bytes, got {0}")]
220    TooSmall(usize),
221    #[error("bad magic: expected RETE")]
222    BadMagic,
223    #[error(
224        "unsupported .rete format {found:#04x}; this Rete build reads {min:#04x}..={max:#04x}. Pre-1.0 files must be rebuilt from RDF source with `rete build`"
225    )]
226    UnsupportedVersion { found: u8, min: u8, max: u8 },
227    #[error("section count {0} overruns the header frame")]
228    BadSectionCount(usize),
229    #[error("unusable index permutation mask {mask:#04x}: {why}")]
230    BadPermMask { mask: u8, why: &'static str },
231    #[error("malformed 0x06 header: {0}")]
232    Malformed06(&'static str),
233}
234
235/// Decoded file header. All multi-byte fields are little-endian on disk. The
236/// `*_offset` / `*_len` fields are a convenience view over the section directory
237/// (populated from it on parse, emitted back to it on serialize).
238#[derive(Debug, Clone, PartialEq, Eq)]
239pub struct Header {
240    /// Format version of the parsed file (currently
241    /// [`MIN_STABLE_READ_VERSION`] through [`CURRENT_FORMAT_VERSION`]).
242    pub version: u8,
243    pub flags: u8,
244    pub metadata_offset: u64,
245    pub metadata_len: u64,
246    pub dictionary_offset: u64,
247    pub dictionary_len: u64,
248    pub root_dir_offset: u64,
249    pub root_dir_len: u64,
250    pub pyramid_meta_offset: u64,
251    pub pyramid_meta_len: u64,
252    pub dict_codec: u8,
253    pub block_codec: u8,
254    pub pyramid_levels: u16,
255    /// 0x06 only: per-position term-id widths (header byte 77); `0` = all u32.
256    pub id_widths: u8,
257    pub quad_count: u64,
258    pub term_count: u64,
259    /// First 16 bytes of the blake3 content hash — an immutable validator.
260    pub content_hash: [u8; 16],
261    /// Named-graphs section (0 if the file has only the default graph).
262    pub named_graphs_offset: u64,
263    pub named_graphs_len: u64,
264    /// Byte length of the trailing **schema-pyramid block** within the pyramid-meta
265    /// section (0 if none). A reader fetches *only* that block — at
266    /// `pyramid_meta_offset + pyramid_meta_len - schema_meta_len` — for an
267    /// index/dictionary/summary-free Tier-0 coherence check.
268    pub schema_meta_len: u32,
269    /// Full-text index section (0 if the file has none; built with
270    /// `rete build --text-index`). See [`SectionKind::TextIndex`].
271    pub text_index_offset: u64,
272    pub text_index_len: u64,
273    /// The full-text index covers only some predicates: it is written (and was
274    /// read) as [`SectionKind::TextIndexPartial`] rather than
275    /// [`SectionKind::TextIndex`]. `false` for every file without one.
276    pub text_index_partial: bool,
277    /// Build-conditions section (0 if absent). See [`SectionKind::BuildInfo`]:
278    /// an opaque per-build record that is **not** covered by the content hash.
279    pub build_info_offset: u64,
280    pub build_info_len: u64,
281    /// Which index permutations the file's index containers carry
282    /// ([`crate::index::PermSet`]).
283    ///
284    /// On disk this is one byte at `[50]`, inside the reserved core span, and
285    /// **`0` means all six** — so every file written before the mask existed
286    /// decodes as [`PermSet::ALL`] and a full six-permutation build stays
287    /// byte-identical to what it always was. A lean file writes its mask
288    /// (`0b000_0111` for SPO+POS+OSP), and its index container then holds three
289    /// sections rather than six, which is what an older reader trips over.
290    ///
291    /// [`PermSet::ALL`]: crate::index::PermSet::ALL
292    pub perms: crate::index::PermSet,
293    /// Directory entries of the kinds added after the generation-1 freeze —
294    /// [`SectionKind::LabelIndex`] (8), [`SectionKind::Schema`] (9) — and of
295    /// any kind this build doesn't recognize, preserved verbatim in directory
296    /// order. None of them is covered by the header content hash.
297    pub extra_sections: Vec<Section>,
298}
299
300impl Header {
301    /// Serialize into a fixed 1024-byte array.
302    pub fn to_bytes(&self) -> [u8; HEADER_LEN] {
303        let mut b = [0u8; HEADER_LEN];
304        // --- core (64 bytes) ---
305        b[0..4].copy_from_slice(&MAGIC);
306        b[4] = self.version;
307        b[5] = self.flags;
308        b[6..8].copy_from_slice(&(HEADER_LEN as u16).to_le_bytes());
309        b[8..24].copy_from_slice(&self.content_hash);
310        b[24..32].copy_from_slice(&self.quad_count.to_le_bytes());
311        b[32..40].copy_from_slice(&self.term_count.to_le_bytes());
312        b[40..42].copy_from_slice(&self.pyramid_levels.to_le_bytes());
313        b[42] = self.dict_codec;
314        b[43] = self.block_codec;
315        // [44..46) section_count written below.
316        b[46..50].copy_from_slice(&self.schema_meta_len.to_le_bytes());
317        // [50] permutation mask; 0 = all six, so a full build is byte-identical
318        // to every file written before the field existed.
319        b[50] = if self.perms == crate::index::PermSet::ALL {
320            0
321        } else {
322            self.perms.bits()
323        };
324        // [51..64) reserved.
325
326        // --- section directory ---
327        // The five always-present sections (verbatim, so the named offsets
328        // round-trip exactly), then the optional text index (only when present, so
329        // a file without one stays byte-identical), then preserved unknown kinds.
330        let entry = |kind, offset, length| Section {
331            kind,
332            flags: 0,
333            offset,
334            length,
335        };
336        let mut entries: Vec<Section> = vec![
337            entry(
338                SectionKind::Metadata,
339                self.metadata_offset,
340                self.metadata_len,
341            ),
342            entry(
343                SectionKind::Dictionary,
344                self.dictionary_offset,
345                self.dictionary_len,
346            ),
347            entry(SectionKind::Index, self.root_dir_offset, self.root_dir_len),
348            entry(
349                SectionKind::PyramidMeta,
350                self.pyramid_meta_offset,
351                self.pyramid_meta_len,
352            ),
353            entry(
354                SectionKind::NamedGraphs,
355                self.named_graphs_offset,
356                self.named_graphs_len,
357            ),
358        ];
359        if self.text_index_len > 0 {
360            entries.push(entry(
361                if self.text_index_partial {
362                    SectionKind::TextIndexPartial
363                } else {
364                    SectionKind::TextIndex
365                },
366                self.text_index_offset,
367                self.text_index_len,
368            ));
369        }
370        if self.build_info_len > 0 {
371            entries.push(entry(
372                SectionKind::BuildInfo,
373                self.build_info_offset,
374                self.build_info_len,
375            ));
376        }
377        entries.extend(self.extra_sections.iter().copied());
378        debug_assert!(
379            entries.len() <= MAX_SECTIONS,
380            "too many sections for a 1 KB header"
381        );
382        let n = entries.len().min(MAX_SECTIONS);
383        b[44..46].copy_from_slice(&(n as u16).to_le_bytes());
384        for (i, s) in entries.iter().take(n).enumerate() {
385            let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
386            b[p..p + 2].copy_from_slice(&s.kind.to_u16().to_le_bytes());
387            b[p + 2..p + 4].copy_from_slice(&s.flags.to_le_bytes());
388            // [p+4..p+8) reserved.
389            b[p + 8..p + 16].copy_from_slice(&s.offset.to_le_bytes());
390            b[p + 16..p + 24].copy_from_slice(&s.length.to_le_bytes());
391        }
392        b
393    }
394
395    /// Parse a header from the first 1024 bytes of a file.
396    pub fn from_bytes(b: &[u8]) -> Result<Self, HeaderError> {
397        if b.len() < HEADER_LEN {
398            return Err(HeaderError::TooSmall(b.len()));
399        }
400        if b[0..4] != MAGIC {
401            return Err(HeaderError::BadMagic);
402        }
403        if crate::v06::reads(b[4]) {
404            // A 0x06 file: its 128-byte header, seen as a logical header (no
405            // physical offsets — see `v06::logical_header`).
406            return crate::v06::Header06::from_bytes(b)
407                .map(|h| crate::v06::logical_header(&h))
408                .map_err(|e| match e {
409                    crate::file::FileError::Container(m) => HeaderError::Malformed06(m),
410                    _ => HeaderError::Malformed06("unreadable"),
411                });
412        }
413        if !(MIN_STABLE_READ_VERSION..=CURRENT_FORMAT_VERSION).contains(&b[4]) {
414            return Err(HeaderError::UnsupportedVersion {
415                found: b[4],
416                min: MIN_STABLE_READ_VERSION,
417                max: CURRENT_FORMAT_VERSION,
418            });
419        }
420        let u16_at = |o: usize| u16::from_le_bytes(b[o..o + 2].try_into().unwrap());
421        let u32_at = |o: usize| u32::from_le_bytes(b[o..o + 4].try_into().unwrap());
422        let u64_at = |o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
423
424        let section_count = u16_at(44) as usize;
425        if SECTION_DIR_OFFSET + section_count * SECTION_ENTRY_LEN > HEADER_LEN {
426            return Err(HeaderError::BadSectionCount(section_count));
427        }
428
429        let perms = if b[50] == 0 {
430            crate::index::PermSet::ALL
431        } else {
432            crate::index::PermSet::from_bits(b[50])
433                .map_err(|why| HeaderError::BadPermMask { mask: b[50], why })?
434        };
435
436        let mut h = Header {
437            version: b[4],
438            flags: b[5],
439            metadata_offset: 0,
440            metadata_len: 0,
441            dictionary_offset: 0,
442            dictionary_len: 0,
443            root_dir_offset: 0,
444            root_dir_len: 0,
445            pyramid_meta_offset: 0,
446            pyramid_meta_len: 0,
447            dict_codec: b[42],
448            block_codec: b[43],
449            pyramid_levels: u16_at(40),
450            id_widths: 0,
451            quad_count: u64_at(24),
452            term_count: u64_at(32),
453            content_hash: b[8..24].try_into().unwrap(),
454            named_graphs_offset: 0,
455            named_graphs_len: 0,
456            schema_meta_len: u32_at(46),
457            perms,
458            text_index_offset: 0,
459            text_index_len: 0,
460            text_index_partial: false,
461            build_info_offset: 0,
462            build_info_len: 0,
463            extra_sections: Vec::new(),
464        };
465        for i in 0..section_count {
466            let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
467            let kind = SectionKind::from_u16(u16_at(p));
468            let offset = u64_at(p + 8);
469            let length = u64_at(p + 16);
470            match kind {
471                SectionKind::Metadata => {
472                    h.metadata_offset = offset;
473                    h.metadata_len = length;
474                }
475                SectionKind::Dictionary => {
476                    h.dictionary_offset = offset;
477                    h.dictionary_len = length;
478                }
479                SectionKind::Index => {
480                    h.root_dir_offset = offset;
481                    h.root_dir_len = length;
482                }
483                SectionKind::PyramidMeta => {
484                    h.pyramid_meta_offset = offset;
485                    h.pyramid_meta_len = length;
486                }
487                SectionKind::NamedGraphs => {
488                    h.named_graphs_offset = offset;
489                    h.named_graphs_len = length;
490                }
491                SectionKind::TextIndex => {
492                    h.text_index_offset = offset;
493                    h.text_index_len = length;
494                    h.text_index_partial = false;
495                }
496                SectionKind::TextIndexPartial => {
497                    h.text_index_offset = offset;
498                    h.text_index_len = length;
499                    h.text_index_partial = true;
500                }
501                SectionKind::BuildInfo => {
502                    h.build_info_offset = offset;
503                    h.build_info_len = length;
504                }
505                SectionKind::LabelIndex
506                | SectionKind::Schema
507                | SectionKind::QuadIndex
508                | SectionKind::Unknown(_) => h.extra_sections.push(Section {
509                    kind,
510                    flags: u16_at(p + 2),
511                    offset,
512                    length,
513                }),
514            }
515        }
516        Ok(h)
517    }
518
519    /// The LABEL_INDEX section (kind 8), if the file carries a non-empty one.
520    pub fn label_index_section(&self) -> Option<Section> {
521        self.section(SectionKind::LabelIndex)
522            .filter(|s| s.length > 0)
523    }
524
525    /// The SCHEMA section (kind 9), if the file carries a non-empty one.
526    pub fn schema_section(&self) -> Option<Section> {
527        self.section(SectionKind::Schema).filter(|s| s.length > 0)
528    }
529
530    pub fn has_quads(&self) -> bool {
531        self.flags & FLAG_HAS_QUADS != 0
532    }
533
534    /// The total file length implied by this header alone: the writer lays the
535    /// sections out back-to-back after the 1 KiB header and ends the file with
536    /// the 4-byte `RETE` footer, so the furthest `offset + length` in the
537    /// section directory plus 4 **is** the file length.
538    ///
539    /// This is the one length signal a *remote* reader can always trust: an
540    /// HTTP host may advertise the size of a compressed representation in
541    /// `Content-Length` (GitHub Pages gzips — a 71 MB file HEADs as 58 MB)
542    /// while serving ranges over the identity bytes, and may hide
543    /// `Content-Range` from cross-origin JS by omitting
544    /// `Access-Control-Expose-Headers`. The header travels *in band* — in the
545    /// first 1 KiB range read — so it cannot be skewed by transport encoding.
546    ///
547    /// `None` if any directory entry overflows `u64` (a crafted or corrupt
548    /// header must yield "unknown", never a panic or a wrapped length — the
549    /// weekly fuzz caught `verify()` on exactly this class of input).
550    pub fn expected_file_len(&self) -> Option<u64> {
551        if crate::v06::reads(self.version) {
552            // A 0x06 header states its length; its logical view parks every
553            // offset at that length (see `v06::logical_header`).
554            return Some(self.metadata_offset);
555        }
556        let mut end = HEADER_LEN as u64;
557        let named = [
558            (self.metadata_offset, self.metadata_len),
559            (self.dictionary_offset, self.dictionary_len),
560            (self.root_dir_offset, self.root_dir_len),
561            (self.pyramid_meta_offset, self.pyramid_meta_len),
562            (self.named_graphs_offset, self.named_graphs_len),
563            (self.text_index_offset, self.text_index_len),
564            (self.build_info_offset, self.build_info_len),
565        ];
566        let extras = self.extra_sections.iter().map(|s| (s.offset, s.length));
567        for (offset, length) in named.into_iter().chain(extras) {
568            if length == 0 {
569                continue; // absent section — its (0, 0) entry says nothing
570            }
571            end = end.max(offset.checked_add(length)?);
572        }
573        end.checked_add(MAGIC.len() as u64)
574    }
575
576    /// Does the file contain RDF-star quoted triples ([`FLAG_HAS_QUOTED_TRIPLES`])?
577    pub fn has_quoted_triples(&self) -> bool {
578        self.flags & FLAG_HAS_QUOTED_TRIPLES != 0
579    }
580
581    /// How this file orders its dictionary terms: the SPARQL order from `0x08`
582    /// (and in a `0x05` image with [`FLAG_SPARQL_TERM_ORDER`]), byte order
583    /// otherwise.
584    pub fn term_order(&self) -> crate::term_order::TermOrder {
585        if self.version >= 0x08
586            || (self.version == 0x05 && self.flags & FLAG_SPARQL_TERM_ORDER != 0)
587        {
588            crate::term_order::TermOrder::Sparql
589        } else {
590            crate::term_order::TermOrder::Bytewise
591        }
592    }
593
594    /// Do the tiled index sections carry a [`FLAG_TILE_SYNOPSIS`] trailer?
595    pub fn has_tile_synopsis(&self) -> bool {
596        self.flags & FLAG_TILE_SYNOPSIS != 0
597    }
598
599    /// Does the full-text index cover the named graphs' literals too
600    /// ([`FLAG_TEXT_INDEX_NAMED_GRAPHS`])? Known from the header alone.
601    pub fn text_index_covers_named_graphs(&self) -> bool {
602        self.flags & FLAG_TEXT_INDEX_NAMED_GRAPHS != 0
603    }
604
605    /// The `(offset, length)` of the text index's payload — the token-table
606    /// varint, the token table and the postings. That is the whole section for
607    /// kind 6; a kind-11 section starts with its 16-byte self-digest, which this
608    /// skips. `(offset, 0)` past a kind-11 section too short for its digest.
609    pub fn text_index_payload(&self) -> (u64, u64) {
610        if self.text_index_partial {
611            (
612                self.text_index_offset
613                    .saturating_add(TEXT_INDEX_PARTIAL_DIGEST_LEN),
614                self.text_index_len
615                    .saturating_sub(TEXT_INDEX_PARTIAL_DIGEST_LEN),
616            )
617        } else {
618            (self.text_index_offset, self.text_index_len)
619        }
620    }
621
622    /// The directory entry for a section kind, or `None` if absent. Known kinds
623    /// read from the named convenience fields; unknown kinds from
624    /// [`extra_sections`](Self::extra_sections).
625    pub fn section(&self, kind: SectionKind) -> Option<Section> {
626        let (offset, length) = match kind {
627            SectionKind::Metadata => (self.metadata_offset, self.metadata_len),
628            SectionKind::Dictionary => (self.dictionary_offset, self.dictionary_len),
629            SectionKind::Index => (self.root_dir_offset, self.root_dir_len),
630            SectionKind::PyramidMeta => (self.pyramid_meta_offset, self.pyramid_meta_len),
631            SectionKind::NamedGraphs => (self.named_graphs_offset, self.named_graphs_len),
632            SectionKind::TextIndex if !self.text_index_partial => {
633                (self.text_index_offset, self.text_index_len)
634            }
635            SectionKind::TextIndexPartial if self.text_index_partial => {
636                (self.text_index_offset, self.text_index_len)
637            }
638            SectionKind::TextIndex | SectionKind::TextIndexPartial => (0, 0),
639            SectionKind::BuildInfo => (self.build_info_offset, self.build_info_len),
640            SectionKind::LabelIndex
641            | SectionKind::Schema
642            | SectionKind::QuadIndex
643            | SectionKind::Unknown(_) => {
644                return self.extra_sections.iter().find(|s| s.kind == kind).copied()
645            }
646        };
647        Some(Section {
648            kind,
649            flags: 0,
650            offset,
651            length,
652        })
653    }
654
655    /// Attach (or overwrite) a section's `(offset, length)` — the extension point
656    /// for new top-level sections. Known kinds set the named fields; an unknown
657    /// kind is appended to [`extra_sections`](Self::extra_sections).
658    pub fn with_section(mut self, kind: SectionKind, offset: u64, length: u64) -> Self {
659        match kind {
660            SectionKind::Metadata => {
661                self.metadata_offset = offset;
662                self.metadata_len = length;
663            }
664            SectionKind::Dictionary => {
665                self.dictionary_offset = offset;
666                self.dictionary_len = length;
667            }
668            SectionKind::Index => {
669                self.root_dir_offset = offset;
670                self.root_dir_len = length;
671            }
672            SectionKind::PyramidMeta => {
673                self.pyramid_meta_offset = offset;
674                self.pyramid_meta_len = length;
675            }
676            SectionKind::NamedGraphs => {
677                self.named_graphs_offset = offset;
678                self.named_graphs_len = length;
679            }
680            SectionKind::TextIndex => {
681                self.text_index_offset = offset;
682                self.text_index_len = length;
683                self.text_index_partial = false;
684            }
685            SectionKind::TextIndexPartial => {
686                self.text_index_offset = offset;
687                self.text_index_len = length;
688                self.text_index_partial = true;
689            }
690            SectionKind::BuildInfo => {
691                self.build_info_offset = offset;
692                self.build_info_len = length;
693            }
694            SectionKind::LabelIndex | SectionKind::Schema | SectionKind::QuadIndex => {
695                // A known post-freeze kind appears at most once: overwrite.
696                match self.extra_sections.iter_mut().find(|s| s.kind == kind) {
697                    Some(s) => {
698                        s.offset = offset;
699                        s.length = length;
700                    }
701                    None => self.extra_sections.push(Section {
702                        kind,
703                        flags: 0,
704                        offset,
705                        length,
706                    }),
707                }
708            }
709            SectionKind::Unknown(_) => self.extra_sections.push(Section {
710                kind,
711                flags: 0,
712                offset,
713                length,
714            }),
715        }
716        self
717    }
718
719    /// Are the named graphs stored in the quad index ([`FLAG_QUAD_INDEX`])?
720    pub fn has_quad_index(&self) -> bool {
721        self.flags & FLAG_QUAD_INDEX != 0
722    }
723
724    /// The QUAD_INDEX section, when the directory lists a non-empty one.
725    pub fn quad_index_section(&self) -> Option<Section> {
726        self.section(SectionKind::QuadIndex)
727            .filter(|s| s.length > 0)
728    }
729
730    /// How many sections the index container (kind 3) holds: one per stored
731    /// permutation, plus the quad-index seal when [`FLAG_QUAD_INDEX`] is set.
732    pub fn index_container_sections(&self) -> usize {
733        self.perms.len() + usize::from(self.has_quad_index())
734    }
735}
736
737#[cfg(test)]
738mod tests {
739    use super::*;
740
741    fn sample() -> Header {
742        Header {
743            version: LEGACY_FORMAT_VERSION,
744            flags: FLAG_HAS_QUADS,
745            metadata_offset: 1024,
746            metadata_len: 42,
747            dictionary_offset: 1066,
748            dictionary_len: 2048,
749            root_dir_offset: 3114,
750            root_dir_len: 256,
751            pyramid_meta_offset: 3370,
752            pyramid_meta_len: 64,
753            dict_codec: 1,
754            block_codec: 2,
755            pyramid_levels: 3,
756            id_widths: 0,
757            perms: crate::index::PermSet::ALL,
758            quad_count: 5,
759            term_count: 9,
760            content_hash: [7u8; 16],
761            named_graphs_offset: 3434,
762            named_graphs_len: 48,
763            schema_meta_len: 99,
764            text_index_offset: 0,
765            text_index_len: 0,
766            text_index_partial: false,
767            build_info_offset: 0,
768            build_info_len: 0,
769            extra_sections: Vec::new(),
770        }
771    }
772
773    #[test]
774    fn round_trip() {
775        let h = sample();
776        let bytes = h.to_bytes();
777        assert_eq!(bytes.len(), HEADER_LEN);
778        assert_eq!(&bytes[0..4], b"RETE");
779        let back = Header::from_bytes(&bytes).unwrap();
780        assert_eq!(h, back);
781        assert!(back.has_quads());
782    }
783
784    #[test]
785    fn byte_layout_matches_spec() {
786        // Pins the core fields and the first directory entry to exact offsets.
787        let h = Header {
788            content_hash: [0xCC; 16],
789            quad_count: 0x99,
790            term_count: 0xAA,
791            pyramid_levels: 0xABCD,
792            id_widths: 0,
793            dict_codec: 0xA1,
794            block_codec: 0xA2,
795            schema_meta_len: 0xD00D,
796            metadata_offset: 0x11,
797            metadata_len: 0x22,
798            dictionary_offset: 0x33,
799            dictionary_len: 0x44,
800            ..sample()
801        };
802        let b = h.to_bytes();
803        let u16_at = |o: usize| u16::from_le_bytes(b[o..o + 2].try_into().unwrap());
804        let u64_at = |o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
805
806        // core
807        assert_eq!(&b[0..4], b"RETE");
808        assert_eq!(b[4], LEGACY_FORMAT_VERSION);
809        assert_eq!(b[5], FLAG_HAS_QUADS);
810        assert_eq!(u16_at(6), HEADER_LEN as u16);
811        assert_eq!(&b[8..24], &[0xCC; 16]); // content hash
812        assert_eq!(u64_at(24), 0x99); // quad count
813        assert_eq!(u64_at(32), 0xAA); // term count
814        assert_eq!(u16_at(40), 0xABCD); // pyramid levels
815        assert_eq!(b[42], 0xA1); // dict codec
816        assert_eq!(b[43], 0xA2); // block codec
817        assert_eq!(u16_at(44), 5); // section_count: the 5 known sections
818        assert_eq!(u32::from_le_bytes(b[46..50].try_into().unwrap()), 0xD00D); // schema-meta len
819                                                                               // first directory entry = Metadata, at offset 64
820        assert_eq!(u16_at(64), 1); // kind = Metadata
821        assert_eq!(u64_at(72), 0x11); // metadata offset
822        assert_eq!(u64_at(80), 0x22); // metadata length
823                                      // second entry = Dictionary, at 88
824        assert_eq!(u16_at(88), 2);
825        assert_eq!(u64_at(96), 0x33);
826        assert_eq!(u64_at(104), 0x44);
827        assert_eq!(b.len(), HEADER_LEN);
828    }
829
830    #[test]
831    fn expected_file_len_is_last_section_end_plus_footer() {
832        // sample(): named graphs end furthest, at 3434 + 48 = 3482.
833        assert_eq!(sample().expected_file_len(), Some(3482 + 4));
834
835        // A zero-length section's offset must not count (absent sections are
836        // written as (0, 0), and a stale offset with len 0 addresses nothing).
837        let mut h = sample();
838        h.text_index_offset = 1 << 40;
839        h.text_index_len = 0;
840        assert_eq!(h.expected_file_len(), Some(3482 + 4));
841
842        // An empty file (header + footer only) is 1028 bytes.
843        let mut e = sample();
844        e.metadata_len = 0;
845        e.dictionary_len = 0;
846        e.root_dir_len = 0;
847        e.pyramid_meta_len = 0;
848        e.named_graphs_len = 0;
849        assert_eq!(e.expected_file_len(), Some(1024 + 4));
850
851        // A crafted header whose entry overflows u64 yields None, not a panic
852        // or a wrapped (tiny) length.
853        let mut c = sample();
854        c.named_graphs_offset = u64::MAX - 8;
855        c.named_graphs_len = 64;
856        assert_eq!(c.expected_file_len(), None);
857
858        // Unknown (future) sections extend the file too.
859        let f = sample().with_section(SectionKind::Unknown(99), 1 << 20, 512);
860        assert_eq!(f.expected_file_len(), Some((1 << 20) + 512 + 4));
861    }
862
863    #[test]
864    fn rejects_bad_magic() {
865        let mut bytes = [0u8; HEADER_LEN];
866        bytes[4] = CURRENT_FORMAT_VERSION;
867        assert!(matches!(
868            Header::from_bytes(&bytes),
869            Err(HeaderError::BadMagic)
870        ));
871    }
872
873    #[test]
874    fn stable_reader_accepts_v1_baseline_and_rejects_pre_v1() {
875        let current = sample().to_bytes();
876        assert_eq!(current[4], 0x05);
877        assert_eq!(Header::from_bytes(&current).unwrap().version, 0x05);
878
879        for old in 0x01..=0x04 {
880            let mut bytes = current;
881            bytes[4] = old;
882            let error = Header::from_bytes(&bytes).unwrap_err();
883            assert!(matches!(
884                &error,
885                HeaderError::UnsupportedVersion {
886                    found,
887                    min: 0x05,
888                    max: 0x08
889                } if *found == old
890            ));
891            assert!(error
892                .to_string()
893                .contains("Pre-1.0 files must be rebuilt from RDF source with `rete build`"));
894        }
895
896        for unsupported in [0x00, 0x09, 0xff] {
897            let mut bytes = current;
898            bytes[4] = unsupported;
899            assert!(matches!(
900                Header::from_bytes(&bytes),
901                Err(HeaderError::UnsupportedVersion {
902                    found,
903                    min: 0x05,
904                    max: 0x08
905                }) if found == unsupported
906            ));
907        }
908    }
909
910    #[test]
911    fn rejects_overrunning_section_count() {
912        let mut bad = sample().to_bytes();
913        bad[44..46].copy_from_slice(&9999u16.to_le_bytes());
914        assert!(matches!(
915            Header::from_bytes(&bad),
916            Err(HeaderError::BadSectionCount(9999))
917        ));
918    }
919
920    #[test]
921    fn unknown_section_survives_round_trip() {
922        // A section a future build added (kind 99) must be preserved verbatim by a
923        // reader that doesn't know it, and readable via `section()`.
924        let h = sample().with_section(SectionKind::Unknown(99), 4096, 512);
925        let back = Header::from_bytes(&h.to_bytes()).unwrap();
926        assert_eq!(back.extra_sections.len(), 1);
927        let s = back.section(SectionKind::Unknown(99)).unwrap();
928        assert_eq!((s.offset, s.length), (4096, 512));
929        // Known sections still resolve.
930        let dict = back.section(SectionKind::Dictionary).unwrap();
931        assert_eq!(dict.offset, h.dictionary_offset);
932        assert_eq!(h, back);
933    }
934
935    #[test]
936    fn build_info_section_round_trips_and_is_optional() {
937        // Absent (len 0): only the 5 always-present sections — a file without a
938        // build-info section is byte-identical to one built before it existed.
939        let plain = sample().to_bytes();
940        assert_eq!(u16::from_le_bytes(plain[44..46].try_into().unwrap()), 5);
941        assert_eq!(sample().section(SectionKind::BuildInfo).unwrap().length, 0);
942
943        // Present: a directory entry that round-trips and resolves as kind 7.
944        let h = sample().with_section(SectionKind::BuildInfo, 1066, 300);
945        let bytes = h.to_bytes();
946        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
947        let back = Header::from_bytes(&bytes).unwrap();
948        assert_eq!(h, back);
949        let s = back.section(SectionKind::BuildInfo).unwrap();
950        assert_eq!((s.offset, s.length), (1066, 300));
951        assert!(back.extra_sections.is_empty(), "BuildInfo is a known kind");
952        // And it extends the expected file length like any other section.
953        let far = sample().with_section(SectionKind::BuildInfo, 1 << 21, 128);
954        assert_eq!(far.expected_file_len(), Some((1 << 21) + 128 + 4));
955    }
956
957    #[test]
958    fn label_index_and_schema_sections_are_known_extras() {
959        // Absent: the directory is unchanged (5 entries), so a file without
960        // them is byte-identical to one built before they existed.
961        assert!(sample().label_index_section().is_none());
962        assert!(sample().schema_section().is_none());
963
964        let h = sample()
965            .with_section(SectionKind::LabelIndex, 8000, 300)
966            .with_section(SectionKind::Schema, 8300, 40);
967        let bytes = h.to_bytes();
968        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 7);
969        // entries 5 and 6 carry kinds 8 and 9 at the directory's byte layout
970        assert_eq!(
971            u16::from_le_bytes(bytes[64 + 5 * 24..][..2].try_into().unwrap()),
972            8
973        );
974        assert_eq!(
975            u16::from_le_bytes(bytes[64 + 6 * 24..][..2].try_into().unwrap()),
976            9
977        );
978        let back = Header::from_bytes(&bytes).unwrap();
979        assert_eq!(h, back);
980        assert_eq!(back.extra_sections.len(), 2, "stored with the extras");
981        let l = back.label_index_section().unwrap();
982        assert_eq!((l.offset, l.length), (8000, 300));
983        let s = back.schema_section().unwrap();
984        assert_eq!((s.offset, s.length), (8300, 40));
985        assert_eq!(back.expected_file_len(), Some(8340 + 4));
986
987        // A known post-freeze kind is overwritten, never duplicated.
988        let again = back.with_section(SectionKind::Schema, 9000, 10);
989        assert_eq!(again.extra_sections.len(), 2);
990        assert_eq!(again.schema_section().unwrap().offset, 9000);
991
992        // A zero-length entry reads as absent.
993        let empty = sample().with_section(SectionKind::LabelIndex, 0, 0);
994        assert!(empty.label_index_section().is_none());
995    }
996
997    #[test]
998    fn quad_index_flag_and_section_round_trip() {
999        // Absent: no flag, no kind-10 entry, the container is the permutations.
1000        let plain = sample();
1001        assert!(!plain.has_quad_index());
1002        assert!(plain.quad_index_section().is_none());
1003        assert_eq!(plain.index_container_sections(), 6);
1004
1005        // Present: flag bit 3, a kind-10 entry kept with the extra sections,
1006        // and one more container section (the seal).
1007        let mut h = sample().with_section(SectionKind::QuadIndex, 7000, 900);
1008        h.flags |= FLAG_QUAD_INDEX;
1009        h.named_graphs_offset = 0;
1010        h.named_graphs_len = 0;
1011        let bytes = h.to_bytes();
1012        assert_eq!(bytes[5] & 0b1000, 0b1000, "flag bit 3");
1013        let n = u16::from_le_bytes(bytes[44..46].try_into().unwrap()) as usize;
1014        let kinds: Vec<u16> = (0..n)
1015            .map(|i| {
1016                let p = SECTION_DIR_OFFSET + i * SECTION_ENTRY_LEN;
1017                u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap())
1018            })
1019            .collect();
1020        assert!(kinds.contains(&10), "a kind-10 directory entry: {kinds:?}");
1021        let back = Header::from_bytes(&bytes).unwrap();
1022        assert_eq!(back, h);
1023        assert!(back.has_quad_index());
1024        let s = back.quad_index_section().unwrap();
1025        assert_eq!(
1026            (s.kind, s.offset, s.length),
1027            (SectionKind::QuadIndex, 7000, 900)
1028        );
1029        assert_eq!(back.index_container_sections(), 7);
1030        assert_eq!(back.expected_file_len(), Some(7000 + 900 + 4));
1031        // Three permutations + seal = four: never the 3 or 6 an old reader wants.
1032        let mut lean = back.clone();
1033        lean.perms = crate::index::PermSet::CORE;
1034        assert_eq!(lean.index_container_sections(), 4);
1035        // Re-attaching overwrites rather than duplicating.
1036        let again = back.with_section(SectionKind::QuadIndex, 8000, 10);
1037        assert_eq!(again.extra_sections.len(), 1);
1038        assert_eq!(again.quad_index_section().unwrap().offset, 8000);
1039    }
1040
1041    #[test]
1042    fn text_index_section_round_trips_and_is_optional() {
1043        // Absent (len 0): only the 5 always-present sections, so a file without a
1044        // text index is byte-identical to one built before this section existed.
1045        assert_eq!(
1046            u16::from_le_bytes(sample().to_bytes()[44..46].try_into().unwrap()),
1047            5
1048        );
1049        assert!(sample().section(SectionKind::TextIndex).unwrap().length == 0);
1050
1051        // Present: a 6th directory entry that round-trips and resolves.
1052        let h = sample().with_section(SectionKind::TextIndex, 5000, 4096);
1053        let bytes = h.to_bytes();
1054        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
1055        let back = Header::from_bytes(&bytes).unwrap();
1056        assert_eq!(h, back);
1057        let s = back.section(SectionKind::TextIndex).unwrap();
1058        assert_eq!((s.offset, s.length), (5000, 4096));
1059        assert!(back.extra_sections.is_empty(), "TextIndex is a known kind");
1060        assert!(!back.text_index_partial);
1061    }
1062
1063    /// A predicate-restricted text index is written as kind 11, maps onto the
1064    /// same `text_index_*` fields, and a reader that only knows kind 6 sees an
1065    /// unknown section — no text index at all — rather than a full one.
1066    #[test]
1067    fn partial_text_index_is_kind_11_and_invisible_as_kind_6() {
1068        let h = sample().with_section(SectionKind::TextIndexPartial, 5000, 4096);
1069        assert!(h.text_index_partial);
1070        let bytes = h.to_bytes();
1071        assert_eq!(u16::from_le_bytes(bytes[44..46].try_into().unwrap()), 6);
1072        // The 6th entry, at 64 + 5 × 24, carries kind 11.
1073        let p = SECTION_DIR_OFFSET + 5 * SECTION_ENTRY_LEN;
1074        assert_eq!(u16::from_le_bytes(bytes[p..p + 2].try_into().unwrap()), 11);
1075        let back = Header::from_bytes(&bytes).unwrap();
1076        assert_eq!(h, back);
1077        assert!(back.text_index_partial);
1078        assert_eq!((back.text_index_offset, back.text_index_len), (5000, 4096));
1079        let s = back.section(SectionKind::TextIndexPartial).unwrap();
1080        assert_eq!((s.offset, s.length), (5000, 4096));
1081        assert_eq!(back.section(SectionKind::TextIndex).unwrap().length, 0);
1082        assert!(back.extra_sections.is_empty(), "kind 11 is known here");
1083
1084        // What a reader that predates kind 11 decodes: an unknown kind, which
1085        // it preserves, and empty text-index fields — so it neither searches
1086        // nor prunes with the partial index.
1087        let mut old_view = bytes;
1088        old_view[p..p + 2].copy_from_slice(&0x7777u16.to_le_bytes());
1089        let old = Header::from_bytes(&old_view).unwrap();
1090        assert_eq!(old.text_index_len, 0);
1091        assert_eq!(old.extra_sections.len(), 1);
1092    }
1093}