Skip to main content

rete_core/v06/
open.rs

1//! Reading a `0x06` file: the header and the root come with the first read;
2//! everything else is built lazily from them — the dictionary and the index as
3//! two-level directories whose leaves fault on demand and stay for the session,
4//! the text index likewise, the label index and the quad index through their
5//! own lazy machinery.
6
7use std::sync::Arc;
8
9use super::*;
10use crate::derived;
11use crate::dict::{ChunkLeaf, ChunkedSection, LeafedChunks, SectionMeta};
12use crate::dictionary::Dictionary;
13use crate::file::{decompress_bounded, ByteRange, FileError};
14use crate::index::{GraphIndex, LeafFetch, LeafedTiles, TileLeaf, NUM_PERMS};
15use crate::reader::RangeReader;
16
17/// Coalescing gaps for the batched faults (as for `0x05`).
18const TILE_GAP: u64 = 4096;
19const DICT_GAP: u64 = 64 * 1024;
20
21/// An open `0x06` file: its header and root, and the reader.
22pub struct V06File {
23    /// The 128-byte header.
24    pub h: Header06,
25    /// The decoded root block.
26    pub root: Root06,
27    /// Where everything else is read from.
28    pub reader: Arc<dyn RangeReader + Send + Sync>,
29}
30
31/// Read `(offset, len)` ranges as one coalesced batch.
32fn read_coalesced(
33    r: &(dyn RangeReader + Send + Sync),
34    ranges: &[(u64, u64)],
35    gap: u64,
36) -> Option<Vec<Vec<u8>>> {
37    let rs: Vec<ByteRange> = ranges
38        .iter()
39        .map(|&(offset, len)| ByteRange { offset, len })
40        .collect();
41    crate::file::read_coalesced(r, &rs, gap)
42}
43
44/// [`read_coalesced`] for tiles far apart: the reader gets the exact ranges.
45fn read_scattered(
46    r: &(dyn RangeReader + Send + Sync),
47    ranges: &[(u64, u64)],
48) -> Option<Vec<Vec<u8>>> {
49    let rs: Vec<ByteRange> = ranges
50        .iter()
51        .map(|&(offset, len)| ByteRange { offset, len })
52        .collect();
53    crate::file::read_coalesced_scattered(r, &rs, 0)
54}
55
56impl V06File {
57    /// Open from `first`, the file's leading bytes (at least the header; the
58    /// root is read from it when it fits, else fetched).
59    pub fn open(
60        reader: Arc<dyn RangeReader + Send + Sync>,
61        first: &[u8],
62    ) -> Result<Self, FileError> {
63        let h = Header06::from_bytes(first)?;
64        if h.file_len != reader.len() {
65            return Err(FileError::Container(
66                "0x06 file length does not match the header (truncated?)",
67            ));
68        }
69        let rs = HEADER_LEN;
70        let re = rs + h.root_len as usize;
71        let root_bytes = if re <= first.len() {
72            first[rs..re].to_vec()
73        } else {
74            reader.read_at(rs as u64, h.root_len as u64)?
75        };
76        if digest16(&root_bytes) != h.root_digest {
77            return Err(FileError::Container("0x06 root digest mismatch"));
78        }
79        let root = Root06::decode(&root_bytes, h.internal_codec)?;
80        for s in &root.sections {
81            if h.version == VERSION_06 && kind::is_v07_only(s.kind) {
82                return Err(FileError::Container(
83                    "0x06 file has a section only 0x07 defines",
84                ));
85            }
86            let a = s.abs(&h)?;
87            let end = a.offset + a.len;
88            let ok = if kind::is_front(s.kind) {
89                a.offset >= h.front_base() && end <= h.data_offset
90            } else {
91                a.offset >= h.data_offset && end <= h.file_len - FOOTER.len() as u64
92            };
93            if !ok {
94                return Err(FileError::Container("0x06 section out of its region"));
95            }
96        }
97        Ok(V06File { h, root, reader })
98    }
99
100    /// The decompression bound of section `k`'s blocks: its recorded largest
101    /// block, or [`BLOCK_CEILING`] where none is recorded.
102    pub fn bound(&self, k: u8) -> u64 {
103        section_bound(&self.root, k)
104    }
105
106    /// The absolute range of section `k`, if present.
107    pub fn section(&self, k: u8) -> Option<ByteRange> {
108        self.root.section(k).and_then(|s| s.abs(&self.h).ok())
109    }
110
111    /// A batched leaf fetcher over this file's reader (leaves are stored with
112    /// the internal codec; decoding happens in the directory types).
113    pub fn fetch(&self) -> LeafFetch {
114        let r = self.reader.clone();
115        Arc::new(move |ranges: &[(u64, u64)]| read_coalesced(&*r, ranges, 0))
116    }
117
118    /// The Dataset Card (decompressed), if the file has one.
119    pub fn card(&self) -> Result<Option<Vec<u8>>, FileError> {
120        let r = self.h.card_range();
121        if r.len == 0 {
122            return Ok(None);
123        }
124        let stored = self.reader.read_at(r.offset, r.len)?;
125        if digest16(&stored) != self.h.card_digest {
126            return Err(FileError::Container("0x06 card digest mismatch"));
127        }
128        Ok(Some(decompress_bounded(
129            self.h.internal_codec,
130            &stored,
131            CARD_MAX,
132        )?))
133    }
134
135    /// The build record (raw), if the file has one.
136    pub fn build_record(&self) -> Result<Option<Vec<u8>>, FileError> {
137        let r = self.h.build_range();
138        if r.len == 0 {
139            return Ok(None);
140        }
141        Ok(Some(self.reader.read_at(r.offset, r.len)?))
142    }
143
144    /// The label index's root node (`0x07`: its own cold section), digest
145    /// checked. Fetched only when a label search runs.
146    pub fn label_root_node(&self) -> Result<Vec<u8>, FileError> {
147        let r = self
148            .section(kind::LABEL_ROOT)
149            .ok_or(FileError::Container("0x07 file has no label root section"))?;
150        let b = self.reader.read_at(r.offset, r.len)?;
151        check_digest(&self.root, kind::LABEL_ROOT, &b)?;
152        Ok(b)
153    }
154
155    /// The card annex (raw JSON, decompressed), if the file has one.
156    pub fn annex(&self) -> Result<Option<Vec<u8>>, FileError> {
157        read_annex(&*self.reader, &self.h, &self.root)
158    }
159
160    /// The full card: the hot card with its annex merged back in.
161    pub fn card_full(&self) -> Result<Option<Vec<u8>>, FileError> {
162        let Some(hot) = self.card()? else {
163            return Ok(None);
164        };
165        match self.annex()? {
166            None => Ok(Some(hot)),
167            Some(a) => annex::merge(&hot, &a)
168                .map(Some)
169                .map_err(|_| FileError::Container("0x07 card annex does not merge")),
170        }
171    }
172
173    /// The stats + schema section, as the `0x05` SCHEMA section bytes
174    /// (digest-prefixed) its decoder takes.
175    pub fn schema_section_bytes(&self) -> Option<Vec<u8>> {
176        let r = self.section(kind::SCHEMA)?;
177        let stored = self.reader.read_at(r.offset, r.len).ok()?;
178        let body =
179            decompress_bounded(self.h.internal_codec, &stored, self.bound(kind::SCHEMA)).ok()?;
180        let mut out = digest16(&body).to_vec();
181        out.extend_from_slice(&body);
182        Some(out)
183    }
184
185    /// The decoded stats + schema section.
186    pub fn schema(&self) -> Option<crate::schema_section::SchemaMeta> {
187        crate::schema_section::SchemaMeta::decode(&self.schema_section_bytes()?).ok()
188    }
189
190    /// The community pyramid's bytes.
191    pub fn pyramid_bytes(&self) -> Option<Vec<u8>> {
192        let r = self.section(kind::PYRAMID)?;
193        self.reader.read_at(r.offset, r.len).ok()
194    }
195
196    /// The derived IRI families the root lists (SPEC.md section 5.3), checked
197    /// against the header flag and the chunk count of the subject-only section.
198    pub fn derived_families(&self, n_chunks: usize) -> Result<Vec<derived::Family>, FileError> {
199        let flagged = self.h.flags & F_DERIVED != 0;
200        let Some(unit) = self.root.unit(tag::DERIVED) else {
201            if flagged {
202                return Err(FileError::Container(
203                    "0x08 derived-IRI flag without a derived-IRI root",
204                ));
205            }
206            return Ok(Vec::new());
207        };
208        if !flagged || self.h.version < VERSION {
209            return Err(FileError::Container(
210                "derived-IRI root without the derived-IRI header flag",
211            ));
212        }
213        derived::decode_families(unit, n_chunks)
214            .map_err(|_| FileError::Container("0x08 derived-IRI root is malformed"))
215    }
216
217    /// The four dictionary sections, two-level, chunks faulted on demand.
218    /// `want[si] == false` leaves that section empty (a search-only open).
219    pub fn dictionary(&self, want: [bool; 4]) -> Result<Dictionary, FileError> {
220        let unit = self
221            .root
222            .unit(tag::DICT)
223            .ok_or(FileError::Container("0x06 file has no dictionary"))?;
224        let roots = decode_dict_root(unit, self.h.version)?;
225        let order = term_order(self.h.version);
226        let fams = self.derived_families(roots[derived::SECTION].n_chunks)?;
227        let elided = derived::elided_runs(&fams);
228        let leaves_base = self.section(kind::DICT_LEAVES).map_or(0, |r| r.offset);
229        let data_base = self.section(kind::DICT_DATA).map_or(0, |r| r.offset);
230        let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
231        let codec = self.h.dict_codec;
232        let max = self.bound(kind::DICT_DATA);
233        let mut dirs: Vec<Option<Arc<LeafedChunks>>> = Vec::with_capacity(4);
234        let mut sections = Vec::with_capacity(4);
235        for (si, s) in roots.iter().enumerate() {
236            let meta = SectionMeta {
237                term_count: if want[si] { s.term_count } else { 0 },
238                restart_interval: s.restart_interval.max(1),
239                restart_offsets: Vec::new(),
240            };
241            if !want[si] {
242                sections.push(
243                    ChunkedSection::from_parts(meta, Vec::new(), None, cache.clone(), si as u8)
244                        .with_order(order),
245                );
246                dirs.push(None);
247                continue;
248            }
249            let leaves: Vec<ChunkLeaf> = s
250                .leaves
251                .iter()
252                .map(|(l, run, coff, key)| ChunkLeaf {
253                    first: l.first,
254                    n: l.n,
255                    first_run: *run,
256                    key: key.clone(),
257                    leaf: (leaves_base + l.off, l.len),
258                    chunk_base: data_base + s.data_off + coff,
259                })
260                .collect();
261            let dir = Arc::new(
262                LeafedChunks::new(leaves, s.n_chunks, self.fetch(), self.h.internal_codec)
263                    .with_max_block(self.bound(kind::DICT_LEAVES)),
264            );
265            let (r, d) = (self.reader.clone(), dir.clone());
266            let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
267                let (off, len) = d.range(ci)?;
268                let b = r.read_at(off, len).ok()?;
269                decompress_bounded(codec, &b, max).ok()
270            });
271            let (r, d) = (self.reader.clone(), dir.clone());
272            let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
273                d.ensure_leaves(&d.leaves_of(cis));
274                let ranges: Option<Vec<(u64, u64)>> = cis.iter().map(|&ci| d.range(ci)).collect();
275                let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
276                blobs
277                    .iter()
278                    .map(|b| decompress_bounded(codec, b, max).ok())
279                    .collect()
280            });
281            let mut section =
282                ChunkedSection::leafed(meta, dir.clone(), loader, cache.clone(), si as u8)
283                    .with_bulk_loader(bulk)
284                    .with_order(order);
285            if order == crate::term_order::TermOrder::Sparql {
286                section = section.with_groups(s.groups);
287            }
288            if si == derived::SECTION {
289                section = section.with_elided(elided.clone());
290            }
291            sections.push(section);
292            dirs.push(Some(dir));
293        }
294        let arr: [ChunkedSection; 4] = sections
295            .try_into()
296            .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
297        let r = self.reader.clone();
298        let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
299            // Fault every section's leaves first (one batch), then the chunks.
300            let mut per: [Vec<usize>; 4] = Default::default();
301            for &(si, ci) in pairs {
302                if let Some(v) = per.get_mut(si as usize) {
303                    v.push(ci);
304                }
305            }
306            // One read for the leaves of every section: a batch that names
307            // terms in three sections used to cost three sequential reads here.
308            let wanted: Vec<(usize, Vec<usize>)> = per
309                .iter()
310                .enumerate()
311                .filter_map(|(si, cis)| Some((si, dirs.get(si)?.as_ref()?.leaves_of(cis))))
312                .collect();
313            let across: Vec<(&dyn crate::index::LeafDir, &[usize])> = wanted
314                .iter()
315                .filter_map(|(si, lis)| {
316                    let d: &dyn crate::index::LeafDir = dirs.get(*si)?.as_deref()?;
317                    Some((d, lis.as_slice()))
318                })
319                .collect();
320            crate::index::ensure_leaves_across(&across);
321            let ranges: Option<Vec<(u64, u64)>> = pairs
322                .iter()
323                .map(|&(si, ci)| dirs.get(si as usize)?.as_ref()?.range(ci))
324                .collect();
325            let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
326            blobs
327                .iter()
328                .map(|b| decompress_bounded(codec, b, max).ok())
329                .collect()
330        });
331        let mut dict = Dictionary::from_chunked_sections(arr).with_multi_bulk_loader(multi);
332        dict.set_has_quoted_triples(self.h.flags & F_QUOTED != 0);
333        if want[derived::SECTION] {
334            dict = dict.with_derived(fams);
335        }
336        Ok(dict)
337    }
338
339    /// The class clusters the cluster root lists, checked against the index root:
340    /// every run is whole leaves, and the runs cover the permutation.
341    fn clusters(&self, perms: &[PermRoot]) -> Result<Vec<ClusterOrder>, FileError> {
342        let Some(unit) = self.root.unit(tag::CLUSTER) else {
343            if self.h.flags & F_CLUSTERED != 0 {
344                return Err(FileError::Container(
345                    "0x07 clustered file without a cluster root",
346                ));
347            }
348            return Ok(Vec::new());
349        };
350        let orders = decode_cluster_root(unit)?;
351        let quad = self.quad_root().transpose()?;
352        for c in &orders {
353            // The order's tile count and its leaves, `(first tile, tiles)`.
354            let (n_tiles, leaves): (usize, Vec<(usize, usize)>) = if c.si == QUAD_SPOG {
355                let o = quad
356                    .as_ref()
357                    .and_then(|q| q.orders.iter().find(|o| o.si == c.si))
358                    .ok_or(FileError::Container(
359                        "0x07 cluster root names an order the quad index lacks",
360                    ))?;
361                (
362                    o.n_tiles,
363                    o.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
364                )
365            } else {
366                let p = perms
367                    .iter()
368                    .find(|p| p.si == c.si)
369                    .ok_or(FileError::Container(
370                        "0x07 cluster root names a permutation the index lacks",
371                    ))?;
372                (
373                    p.n_tiles,
374                    p.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
375                )
376            };
377            if c.run_tiles.iter().sum::<usize>() != n_tiles {
378                return Err(FileError::Container(
379                    "0x07 cluster runs do not cover the tiles",
380                ));
381            }
382            // A leaf holds the tiles of one run.
383            let mut ends = Vec::with_capacity(c.run_tiles.len());
384            let mut at = 0usize;
385            for &n in &c.run_tiles {
386                at += n;
387                ends.push(at);
388            }
389            for (first, n) in leaves {
390                let r = ends.partition_point(|&e| e <= first);
391                if first + n > ends[r.min(ends.len() - 1)] {
392                    return Err(FileError::Container(
393                        "0x07 a directory leaf spans two cluster runs",
394                    ));
395                }
396            }
397        }
398        Ok(orders)
399    }
400
401    /// The default graph's index, two-level, tiles faulted on demand.
402    pub fn index(&self, read_concurrency: usize) -> Result<GraphIndex, FileError> {
403        let unit = self
404            .root
405            .unit(tag::INDEX)
406            .ok_or(FileError::Container("0x06 file has no index"))?;
407        let perms_root = decode_index_root(unit)?;
408        let leaves_base = self.section(kind::INDEX_LEAVES).map_or(0, |r| r.offset);
409        let data_base = self.section(kind::INDEX_DATA).map_or(0, |r| r.offset);
410        let perms = self.h.perms();
411        let mut lists: [Option<Arc<LeafedTiles>>; NUM_PERMS] = Default::default();
412        for p in &perms_root {
413            if p.si >= NUM_PERMS {
414                return Err(FileError::Container("0x06 index: bad permutation"));
415            }
416            let leaves: Vec<TileLeaf> = p
417                .leaves
418                .iter()
419                .map(|(l, mn, mx, toff)| TileLeaf {
420                    first: l.first,
421                    n: l.n,
422                    min_a: *mn,
423                    max_a: *mx,
424                    leaf: (leaves_base + l.off, l.len),
425                    tile_base: data_base + p.data_off + toff,
426                })
427                .collect();
428            lists[p.si] = Some(Arc::new(
429                LeafedTiles::new(
430                    leaves,
431                    p.n_tiles,
432                    p.data_len,
433                    self.fetch(),
434                    self.h.internal_codec,
435                )
436                .with_max_block(self.bound(kind::INDEX_LEAVES)),
437            ));
438        }
439        for perm in perms.iter() {
440            if lists[perm.section_index()].is_none() {
441                return Err(FileError::Container(
442                    "0x06 index lacks a stored permutation",
443                ));
444            }
445        }
446        let codec = self.h.block_codec;
447        let max = self.bound(kind::INDEX_DATA);
448        let (r, l) = (self.reader.clone(), lists.clone());
449        let loader: crate::index::TileLoader = Box::new(move |si, ti| {
450            let (off, len) = l.get(si)?.as_ref()?.range(ti)?;
451            let b = r.read_at(off, len).ok()?;
452            decompress_bounded(codec, &b, max).ok()
453        });
454        let (r, l) = (self.reader.clone(), lists.clone());
455        let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
456            let list = l.get(si)?.as_ref()?;
457            if let (Some(&a), Some(&b)) = (tis.first(), tis.last()) {
458                list.ensure_span(a, b + 1);
459            }
460            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
461            let blobs = read_coalesced(&*r, &ranges?, TILE_GAP)?;
462            blobs
463                .iter()
464                .map(|b| decompress_bounded(codec, b, max).ok())
465                .collect()
466        });
467        let (r, l) = (self.reader.clone(), lists.clone());
468        let scattered: crate::index::TileBulkLoader = Box::new(move |si, tis| {
469            let list = l.get(si)?.as_ref()?;
470            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
471            let blobs = read_scattered(&*r, &ranges?)?;
472            blobs
473                .iter()
474                .map(|b| decompress_bounded(codec, b, max).ok())
475                .collect()
476        });
477        let mut index = GraphIndex::from_leafed(lists, perms, loader)
478            .with_bulk_loader(bulk)
479            .with_scattered_loader(scattered);
480        for c in self
481            .clusters(&perms_root)?
482            .into_iter()
483            .filter(|c| c.si < NUM_PERMS)
484        {
485            index = index.with_clusters(
486                c.si,
487                crate::index::ClusterRuns::new(c.classes, &c.run_tiles),
488            );
489        }
490        index.set_read_concurrency(read_concurrency);
491        Ok(index)
492    }
493
494    /// The text index (leafed token table), if the file has one.
495    pub fn text_index(&self) -> Option<Result<crate::text_index::TextIndex, FileError>> {
496        let unit = self.root.unit(tag::TEXT)?;
497        Some((|| {
498            let t = decode_text_root(unit)?;
499            let leaves_base = self.section(kind::TEXT_LEAVES).map_or(0, |r| r.offset);
500            let data = self
501                .section(kind::TEXT_DATA)
502                .unwrap_or(ByteRange { offset: 0, len: 0 });
503            let leaves = t
504                .leaves
505                .iter()
506                .map(|(l, tok)| crate::text_index::TokenLeaf {
507                    first: l.first,
508                    n: l.n,
509                    first_token: String::from_utf8_lossy(tok).into_owned(),
510                    leaf: (leaves_base + l.off, l.len),
511                })
512                .collect();
513            let tokens = Arc::new(
514                crate::text_index::LeafedTokens::new(
515                    leaves,
516                    t.n_tokens,
517                    self.fetch(),
518                    self.h.internal_codec,
519                )
520                .with_max_block(self.bound(kind::TEXT_LEAVES)),
521            );
522            let r = self.reader.clone();
523            let loader = Box::new(move |off: u64, len: u64| {
524                if off.checked_add(len)? > data.len {
525                    return None;
526                }
527                r.read_at(data.offset + off, len).ok()
528            });
529            crate::text_index::TextIndex::leafed(tokens, &t.scope, loader)
530        })())
531    }
532
533    /// Stored bytes of the token table (all leaves) — what a substring search
534    /// would read; a word lookup reads one leaf.
535    pub fn token_table_bytes(&self) -> Option<u64> {
536        self.root.unit(tag::TEXT)?;
537        Some(self.section(kind::TEXT_LEAVES).map_or(0, |r| r.len))
538    }
539
540    /// The label index, if the file has one.
541    pub fn label_index(&self) -> Option<Result<crate::label_index::LabelIndex, FileError>> {
542        let unit = self.root.unit(tag::LABELS)?;
543        Some((|| {
544            let mut l = decode_label_root(unit)?;
545            if self.section(kind::LABEL_ROOT).is_some() {
546                l.head.extend_from_slice(&self.label_root_node()?);
547            }
548            let data = self
549                .section(kind::LABEL_DATA)
550                .unwrap_or(ByteRange { offset: 0, len: 0 });
551            let nodes = self
552                .section(kind::LABEL_LEAVES)
553                .unwrap_or(ByteRange { offset: 0, len: 0 });
554            if data.len != l.data_len {
555                return Err(FileError::Container("0x06 label data length mismatch"));
556            }
557            let head_len = l.head.len() as u64;
558            let section_len = head_len + data.len + nodes.len;
559            let r = self.reader.clone();
560            let head = l.head.clone();
561            let loader = Box::new(move |off: u64, len: u64| -> Option<Vec<u8>> {
562                let end = off.checked_add(len)?;
563                if end <= head_len {
564                    return head.get(off as usize..end as usize).map(<[u8]>::to_vec);
565                }
566                let bo = off.checked_sub(head_len)?;
567                if bo + len <= data.len {
568                    r.read_at(data.offset + bo, len).ok()
569                } else if bo >= data.len && bo - data.len + len <= nodes.len {
570                    r.read_at(nodes.offset + (bo - data.len), len).ok()
571                } else {
572                    None
573                }
574            });
575            let max = self
576                .bound(kind::LABEL_LEAVES)
577                .max(self.bound(kind::LABEL_DATA));
578            crate::label_index::LabelIndex::from_head(
579                &l.head,
580                section_len,
581                self.h.block_codec,
582                loader,
583            )
584            .map(|li| li.with_max_block(max))
585        })())
586    }
587
588    /// The label index's entry count, from the root alone.
589    pub fn label_entry_count(&self) -> Option<u64> {
590        let l = decode_label_root(self.root.unit(tag::LABELS)?).ok()?;
591        crate::label_index::LabelIndexPreamble::parse(&l.head)
592            .ok()
593            .map(|p| p.entry_count)
594    }
595
596    /// The quad index's root, if the file has named graphs.
597    pub fn quad_root(&self) -> Option<Result<QuadRoot, FileError>> {
598        self.root.unit(tag::QUAD).map(decode_quad_root)
599    }
600
601    /// The named graphs (quad index), if any.
602    pub(crate) fn quad(
603        &self,
604        read_concurrency: usize,
605    ) -> Option<Result<crate::quad::QuadNamed, FileError>> {
606        let root = match self.quad_root()? {
607            Ok(r) => r,
608            Err(e) => return Some(Err(e)),
609        };
610        let leaves = self.section(kind::QUAD_LEAVES).map_or(0, |r| r.offset);
611        let data = self.section(kind::QUAD_DATA).map_or(0, |r| r.offset);
612        // The class runs of SPOG, when the file clusters it.
613        let clusters = match self.root.unit(tag::CLUSTER).map(decode_cluster_root) {
614            Some(Ok(c)) => c,
615            Some(Err(e)) => return Some(Err(e)),
616            None => Vec::new(),
617        };
618        Some(
619            crate::quad::V06Quad::new(
620                &root,
621                leaves,
622                data,
623                self.h.internal_codec,
624                has_v07(self.h.version),
625            )
626            .and_then(|v| v.with_clusters(&clusters))
627            .map(|mut v| {
628                v.tile_max = self.bound(kind::QUAD_DATA);
629                v.leaf_max = self.bound(kind::QUAD_LEAVES);
630                crate::quad::QuadNamed::new_v06(
631                    self.reader.clone(),
632                    v,
633                    self.h.block_codec,
634                    self.h.perms(),
635                    read_concurrency,
636                )
637            }),
638        )
639    }
640
641    /// Verify every digest: the root, the card, each section (streamed), and
642    /// the content hash. `only` restricts the section checks to one kind.
643    /// `Ok(false)` names nothing; [`verify_report`](Self::verify_report) does.
644    pub fn verify(&self, only: Option<u8>) -> Result<bool, FileError> {
645        Ok(self.verify_report(only)?.is_empty())
646    }
647
648    /// Like [`verify`](Self::verify), returning what failed (empty = all good).
649    pub fn verify_report(&self, only: Option<u8>) -> Result<Vec<String>, FileError> {
650        const SLICE: u64 = 8 << 20;
651        let mut bad = Vec::new();
652        if content_hash(&self.h) != self.h.content_hash {
653            bad.push("content hash".to_string());
654        }
655        let card = self.h.card_range();
656        if only.is_none() && card.len > 0 {
657            let stored = self.reader.read_at(card.offset, card.len)?;
658            if digest16(&stored) != self.h.card_digest {
659                bad.push("card".to_string());
660            }
661        }
662        for s in &self.root.sections {
663            if only.is_some_and(|k| k != s.kind) {
664                continue;
665            }
666            let a = s.abs(&self.h)?;
667            let mut h = blake3::Hasher::new();
668            let mut at = a.offset;
669            while at < a.offset + a.len {
670                let n = (a.offset + a.len - at).min(SLICE);
671                h.update(&self.reader.read_at(at, n)?);
672                at += n;
673            }
674            if h.finalize().as_bytes()[..16] != s.digest {
675                bad.push(kind::name(s.kind).to_string());
676            }
677        }
678        if only.is_none() {
679            let want = footer(self.h.version);
680            let foot = self
681                .reader
682                .read_at(self.h.file_len - want.len() as u64, want.len() as u64)?;
683            if foot != want {
684                bad.push("footer".to_string());
685            }
686        }
687        Ok(bad)
688    }
689}
690
691/// Is this file (by its first bytes) a `0x06` or `0x07` file?
692pub fn is_v06(first: &[u8]) -> bool {
693    first.len() > 4 && first[0..4] == crate::header::MAGIC && reads(first[4])
694}
695
696/// Read a `0x06` file's header and root through any reader (two reads at most;
697/// over a seeded or prefix-holding reader, none).
698pub fn read_front<R: RangeReader + ?Sized>(r: &R) -> Result<(Header06, Root06), FileError> {
699    let head = r.read_at(0, HEADER_LEN as u64)?;
700    let h = Header06::from_bytes(&head)?;
701    let root_bytes = r.read_at(HEADER_LEN as u64, h.root_len as u64)?;
702    if digest16(&root_bytes) != h.root_digest {
703        return Err(FileError::Container("0x06 root digest mismatch"));
704    }
705    let root = Root06::decode(&root_bytes, h.internal_codec)?;
706    Ok((h, root))
707}
708
709/// The stored bytes of section `k`, if present.
710pub fn read_section<R: RangeReader + ?Sized>(
711    r: &R,
712    h: &Header06,
713    root: &Root06,
714    k: u8,
715) -> Result<Option<Vec<u8>>, FileError> {
716    match root.section(k) {
717        Some(s) => {
718            let a = s.abs(h)?;
719            Ok(Some(r.read_at(a.offset, a.len)?))
720        }
721        None => Ok(None),
722    }
723}
724
725/// The card (decompressed) and the build record (raw): one read of the card
726/// region, which sits right after the root.
727#[allow(clippy::type_complexity)]
728pub fn read_card_build<R: RangeReader + ?Sized>(
729    r: &R,
730    h: &Header06,
731) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
732    read_card_build_with(r, h, true)
733}
734
735/// [`read_card_build`], reading the build record only when `want_build`: a
736/// caller that never prints it stops its read at the end of the card.
737#[allow(clippy::type_complexity)]
738pub fn read_card_build_with<R: RangeReader + ?Sized>(
739    r: &R,
740    h: &Header06,
741    want_build: bool,
742) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
743    let c = h.card_range();
744    let mut b = h.build_range();
745    if !want_build {
746        b.len = 0;
747    }
748    if c.len == 0 && b.len == 0 {
749        return Ok((None, None));
750    }
751    let lo = if c.len == 0 {
752        b.offset
753    } else if b.len == 0 {
754        c.offset
755    } else {
756        c.offset.min(b.offset)
757    };
758    let hi = (c.offset + c.len).max(b.offset + b.len);
759    let span = r.read_at(lo, hi - lo)?;
760    let card = if c.len > 0 {
761        let stored = &span[(c.offset - lo) as usize..(c.offset - lo + c.len) as usize];
762        if digest16(stored) != h.card_digest {
763            return Err(FileError::Container("0x06 card digest mismatch"));
764        }
765        Some(decompress_bounded(h.internal_codec, stored, CARD_MAX)?)
766    } else {
767        None
768    };
769    let build = (b.len > 0)
770        .then(|| span[(b.offset - lo) as usize..(b.offset - lo + b.len) as usize].to_vec());
771    Ok((card, build))
772}
773
774/// Section `k`'s decompression bound in `root`: its recorded largest block,
775/// or [`BLOCK_CEILING`].
776pub fn section_bound(root: &Root06, k: u8) -> u64 {
777    root.section(k)
778        .map(|s| s.max_block as u64)
779        .filter(|&m| m > 0)
780        .unwrap_or(BLOCK_CEILING)
781}
782
783/// The stats + schema section as the digest-prefixed bytes its decoder takes.
784pub fn read_schema_bytes<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
785    let (h, root) = read_front(r)?;
786    let Some(stored) = read_section(r, &h, &root, kind::SCHEMA)? else {
787        return Ok(None);
788    };
789    let body = decompress_bounded(
790        h.internal_codec,
791        &stored,
792        section_bound(&root, kind::SCHEMA),
793    )?;
794    let mut out = digest16(&body).to_vec();
795    out.extend_from_slice(&body);
796    Ok(Some(out))
797}
798
799/// The community pyramid's bytes, if any.
800pub fn read_pyramid<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
801    let (h, root) = read_front(r)?;
802    read_section(r, &h, &root, kind::PYRAMID)
803}
804
805/// Stored bytes of the text index's token table (all leaves).
806pub fn read_token_table_bytes<R: RangeReader + ?Sized>(r: &R) -> Option<u64> {
807    let (_, root) = read_front(r).ok()?;
808    root.unit(tag::TEXT)?;
809    Some(root.section(kind::TEXT_LEAVES).map_or(0, |s| s.len))
810}
811
812/// A reader holding only some ranges of a file in memory (the borrowed-reader
813/// opens of a `0x06` file read what they need into one of these).
814pub struct SparseReader {
815    len: u64,
816    parts: Vec<(u64, Vec<u8>)>,
817}
818
819impl SparseReader {
820    /// Fetch `ranges` from `r` (plus the first bytes `first`) into memory —
821    /// the ones `first` does not already hold, in one batch.
822    pub fn fetch<R: RangeReader + ?Sized>(
823        r: &R,
824        first: Vec<u8>,
825        ranges: &[ByteRange],
826    ) -> Result<Self, FileError> {
827        let held = first.len() as u64;
828        let want: Vec<(u64, u64)> = ranges
829            .iter()
830            .filter(|g| g.len > 0 && g.offset.saturating_add(g.len) > held)
831            .map(|g| (g.offset, g.len))
832            .collect();
833        let mut parts = vec![(0u64, first)];
834        if !want.is_empty() {
835            for ((o, _), b) in want.iter().zip(r.read_many(&want)?) {
836                parts.push((*o, b));
837            }
838        }
839        Ok(SparseReader {
840            len: r.len(),
841            parts,
842        })
843    }
844}
845
846/// A borrowed reader whose first bytes are already in memory: reads inside
847/// them cost nothing, the rest go to `r`.
848pub struct Prefixed<'a, R: RangeReader + ?Sized> {
849    /// The file's first bytes.
850    pub first: &'a [u8],
851    /// The reader behind them.
852    pub r: &'a R,
853}
854
855impl<R: RangeReader + ?Sized> RangeReader for Prefixed<'_, R> {
856    fn is_local(&self) -> bool {
857        self.r.is_local()
858    }
859
860    fn len(&self) -> u64 {
861        self.r.len()
862    }
863
864    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
865        match offset.checked_add(len) {
866            Some(end) if end <= self.first.len() as u64 => {
867                Ok(self.first[offset as usize..end as usize].to_vec())
868            }
869            _ => self.r.read_at(offset, len),
870        }
871    }
872}
873
874impl RangeReader for SparseReader {
875    fn len(&self) -> u64 {
876        self.len
877    }
878
879    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
880        for (o, b) in &self.parts {
881            if offset >= *o && offset + len <= o + b.len() as u64 {
882                let s = (offset - o) as usize;
883                return Ok(b[s..s + len as usize].to_vec());
884            }
885        }
886        Err(std::io::Error::other("0x06: range not held by this reader"))
887    }
888}
889
890/// What the card's measured signals need from a `0x06` file's root: the text
891/// index's `(bytes, token-table bytes)` and the label index's `(bytes, entries)`.
892#[allow(clippy::type_complexity)]
893pub fn read_signals<R: RangeReader + ?Sized>(
894    r: &R,
895) -> Result<(Option<(u64, u64)>, Option<(u64, Option<u64>)>), FileError> {
896    let (_, root) = read_front(r)?;
897    let len = |k: u8| root.section(k).map_or(0, |s| s.len);
898    let text = root.unit(tag::TEXT).map(|_| {
899        (
900            len(kind::TEXT_LEAVES) + len(kind::TEXT_DATA),
901            len(kind::TEXT_LEAVES),
902        )
903    });
904    let labels = match root.unit(tag::LABELS) {
905        None => None,
906        Some(u) => {
907            let l = decode_label_root(u)?;
908            let entries = crate::label_index::LabelIndexPreamble::parse(&l.head)
909                .ok()
910                .map(|p| p.entry_count);
911            Some((
912                l.head.len() as u64
913                    + len(kind::LABEL_ROOT)
914                    + len(kind::LABEL_DATA)
915                    + len(kind::LABEL_LEAVES),
916                entries,
917            ))
918        }
919    };
920    Ok((text, labels))
921}
922
923/// Check `bytes` against the digest the root records for section `k`.
924pub fn check_digest(root: &Root06, k: u8, bytes: &[u8]) -> Result<(), FileError> {
925    match root.section(k) {
926        Some(s) if s.digest == digest16(bytes) => Ok(()),
927        Some(_) => Err(FileError::Container("0x07 section digest mismatch")),
928        None => Ok(()),
929    }
930}
931
932/// The card annex of a `0x07` file (raw JSON, decompressed, digest checked).
933pub fn read_annex<R: RangeReader + ?Sized>(
934    r: &R,
935    h: &Header06,
936    root: &Root06,
937) -> Result<Option<Vec<u8>>, FileError> {
938    let Some(stored) = read_section(r, h, root, kind::CARD_ANNEX)? else {
939        return Ok(None);
940    };
941    check_digest(root, kind::CARD_ANNEX, &stored)?;
942    let max = root
943        .section(kind::CARD_ANNEX)
944        .map(|s| s.max_block as u64)
945        .filter(|&m| m > 0)
946        .unwrap_or(ANNEX_MAX);
947    Ok(Some(decompress_bounded(h.internal_codec, &stored, max)?))
948}
949
950/// The card of the file read through `r` with its annex merged back in (what
951/// `rete card --full` shows), and whether an annex was merged.
952pub fn read_card_full<R: RangeReader + ?Sized>(
953    r: &R,
954) -> Result<(Option<Vec<u8>>, bool), FileError> {
955    let (h, root) = read_front(r)?;
956    let Some(hot) = read_card_build(r, &h)?.0 else {
957        return Ok((None, false));
958    };
959    match read_annex(r, &h, &root)? {
960        None => Ok((Some(hot), false)),
961        Some(a) => annex::merge(&hot, &a)
962            .map(|c| (Some(c), true))
963            .map_err(|_| FileError::Container("0x07 card annex does not merge")),
964    }
965}