Skip to main content

rete_core/v06/
open.rs

1//! Reading a `0x06` file: the header and the root come with the first read;
2//! everything else is built lazily from them — the dictionary and the index as
3//! two-level directories whose leaves fault on demand and stay for the session,
4//! the text index likewise, the label index and the quad index through their
5//! own lazy machinery.
6
7use std::sync::Arc;
8
9use super::*;
10use crate::dict::{ChunkLeaf, ChunkedSection, LeafedChunks, SectionMeta};
11use crate::dictionary::Dictionary;
12use crate::file::{decompress_bounded, ByteRange, FileError};
13use crate::index::{GraphIndex, LeafFetch, LeafedTiles, TileLeaf, NUM_PERMS};
14use crate::reader::RangeReader;
15
16/// Coalescing gaps for the batched faults (as for `0x05`).
17const TILE_GAP: u64 = 4096;
18const DICT_GAP: u64 = 64 * 1024;
19
20/// An open `0x06` file: its header and root, and the reader.
21pub struct V06File {
22    /// The 128-byte header.
23    pub h: Header06,
24    /// The decoded root block.
25    pub root: Root06,
26    /// Where everything else is read from.
27    pub reader: Arc<dyn RangeReader + Send + Sync>,
28}
29
30/// Read `(offset, len)` ranges as one coalesced batch.
31fn read_coalesced(
32    r: &(dyn RangeReader + Send + Sync),
33    ranges: &[(u64, u64)],
34    gap: u64,
35) -> Option<Vec<Vec<u8>>> {
36    let rs: Vec<ByteRange> = ranges
37        .iter()
38        .map(|&(offset, len)| ByteRange { offset, len })
39        .collect();
40    crate::file::read_coalesced(r, &rs, gap)
41}
42
43/// [`read_coalesced`] for tiles far apart: the reader gets the exact ranges.
44fn read_scattered(
45    r: &(dyn RangeReader + Send + Sync),
46    ranges: &[(u64, u64)],
47) -> Option<Vec<Vec<u8>>> {
48    let rs: Vec<ByteRange> = ranges
49        .iter()
50        .map(|&(offset, len)| ByteRange { offset, len })
51        .collect();
52    crate::file::read_coalesced_scattered(r, &rs, 0)
53}
54
55impl V06File {
56    /// Open from `first`, the file's leading bytes (at least the header; the
57    /// root is read from it when it fits, else fetched).
58    pub fn open(
59        reader: Arc<dyn RangeReader + Send + Sync>,
60        first: &[u8],
61    ) -> Result<Self, FileError> {
62        let h = Header06::from_bytes(first)?;
63        if h.file_len != reader.len() {
64            return Err(FileError::Container(
65                "0x06 file length does not match the header (truncated?)",
66            ));
67        }
68        let rs = HEADER_LEN;
69        let re = rs + h.root_len as usize;
70        let root_bytes = if re <= first.len() {
71            first[rs..re].to_vec()
72        } else {
73            reader.read_at(rs as u64, h.root_len as u64)?
74        };
75        if digest16(&root_bytes) != h.root_digest {
76            return Err(FileError::Container("0x06 root digest mismatch"));
77        }
78        let root = Root06::decode(&root_bytes, h.internal_codec)?;
79        for s in &root.sections {
80            if h.version == VERSION_06 && kind::is_v07_only(s.kind) {
81                return Err(FileError::Container(
82                    "0x06 file has a section only 0x07 defines",
83                ));
84            }
85            let a = s.abs(&h)?;
86            let end = a.offset + a.len;
87            let ok = if kind::is_front(s.kind) {
88                a.offset >= h.front_base() && end <= h.data_offset
89            } else {
90                a.offset >= h.data_offset && end <= h.file_len - FOOTER.len() as u64
91            };
92            if !ok {
93                return Err(FileError::Container("0x06 section out of its region"));
94            }
95        }
96        Ok(V06File { h, root, reader })
97    }
98
99    /// The decompression bound of section `k`'s blocks: its recorded largest
100    /// block, or [`BLOCK_CEILING`] where none is recorded.
101    pub fn bound(&self, k: u8) -> u64 {
102        section_bound(&self.root, k)
103    }
104
105    /// The absolute range of section `k`, if present.
106    pub fn section(&self, k: u8) -> Option<ByteRange> {
107        self.root.section(k).and_then(|s| s.abs(&self.h).ok())
108    }
109
110    /// A batched leaf fetcher over this file's reader (leaves are stored with
111    /// the internal codec; decoding happens in the directory types).
112    pub fn fetch(&self) -> LeafFetch {
113        let r = self.reader.clone();
114        Arc::new(move |ranges: &[(u64, u64)]| read_coalesced(&*r, ranges, 0))
115    }
116
117    /// The Dataset Card (decompressed), if the file has one.
118    pub fn card(&self) -> Result<Option<Vec<u8>>, FileError> {
119        let r = self.h.card_range();
120        if r.len == 0 {
121            return Ok(None);
122        }
123        let stored = self.reader.read_at(r.offset, r.len)?;
124        if digest16(&stored) != self.h.card_digest {
125            return Err(FileError::Container("0x06 card digest mismatch"));
126        }
127        Ok(Some(decompress_bounded(
128            self.h.internal_codec,
129            &stored,
130            CARD_MAX,
131        )?))
132    }
133
134    /// The build record (raw), if the file has one.
135    pub fn build_record(&self) -> Result<Option<Vec<u8>>, FileError> {
136        let r = self.h.build_range();
137        if r.len == 0 {
138            return Ok(None);
139        }
140        Ok(Some(self.reader.read_at(r.offset, r.len)?))
141    }
142
143    /// The label index's root node (`0x07`: its own cold section), digest
144    /// checked. Fetched only when a label search runs.
145    pub fn label_root_node(&self) -> Result<Vec<u8>, FileError> {
146        let r = self
147            .section(kind::LABEL_ROOT)
148            .ok_or(FileError::Container("0x07 file has no label root section"))?;
149        let b = self.reader.read_at(r.offset, r.len)?;
150        check_digest(&self.root, kind::LABEL_ROOT, &b)?;
151        Ok(b)
152    }
153
154    /// The card annex (raw JSON, decompressed), if the file has one.
155    pub fn annex(&self) -> Result<Option<Vec<u8>>, FileError> {
156        read_annex(&*self.reader, &self.h, &self.root)
157    }
158
159    /// The full card: the hot card with its annex merged back in.
160    pub fn card_full(&self) -> Result<Option<Vec<u8>>, FileError> {
161        let Some(hot) = self.card()? else {
162            return Ok(None);
163        };
164        match self.annex()? {
165            None => Ok(Some(hot)),
166            Some(a) => annex::merge(&hot, &a)
167                .map(Some)
168                .map_err(|_| FileError::Container("0x07 card annex does not merge")),
169        }
170    }
171
172    /// The stats + schema section, as the `0x05` SCHEMA section bytes
173    /// (digest-prefixed) its decoder takes.
174    pub fn schema_section_bytes(&self) -> Option<Vec<u8>> {
175        let r = self.section(kind::SCHEMA)?;
176        let stored = self.reader.read_at(r.offset, r.len).ok()?;
177        let body =
178            decompress_bounded(self.h.internal_codec, &stored, self.bound(kind::SCHEMA)).ok()?;
179        let mut out = digest16(&body).to_vec();
180        out.extend_from_slice(&body);
181        Some(out)
182    }
183
184    /// The decoded stats + schema section.
185    pub fn schema(&self) -> Option<crate::schema_section::SchemaMeta> {
186        crate::schema_section::SchemaMeta::decode(&self.schema_section_bytes()?).ok()
187    }
188
189    /// The community pyramid's bytes.
190    pub fn pyramid_bytes(&self) -> Option<Vec<u8>> {
191        let r = self.section(kind::PYRAMID)?;
192        self.reader.read_at(r.offset, r.len).ok()
193    }
194
195    /// The four dictionary sections, two-level, chunks faulted on demand.
196    /// `want[si] == false` leaves that section empty (a search-only open).
197    pub fn dictionary(&self, want: [bool; 4]) -> Result<Dictionary, FileError> {
198        let unit = self
199            .root
200            .unit(tag::DICT)
201            .ok_or(FileError::Container("0x06 file has no dictionary"))?;
202        let roots = decode_dict_root(unit)?;
203        let leaves_base = self.section(kind::DICT_LEAVES).map_or(0, |r| r.offset);
204        let data_base = self.section(kind::DICT_DATA).map_or(0, |r| r.offset);
205        let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
206        let codec = self.h.dict_codec;
207        let max = self.bound(kind::DICT_DATA);
208        let mut dirs: Vec<Option<Arc<LeafedChunks>>> = Vec::with_capacity(4);
209        let mut sections = Vec::with_capacity(4);
210        for (si, s) in roots.iter().enumerate() {
211            let meta = SectionMeta {
212                term_count: if want[si] { s.term_count } else { 0 },
213                restart_interval: s.restart_interval.max(1),
214                restart_offsets: Vec::new(),
215            };
216            if !want[si] {
217                sections.push(ChunkedSection::from_parts(
218                    meta,
219                    Vec::new(),
220                    None,
221                    cache.clone(),
222                    si as u8,
223                ));
224                dirs.push(None);
225                continue;
226            }
227            let leaves: Vec<ChunkLeaf> = s
228                .leaves
229                .iter()
230                .map(|(l, run, coff, key)| ChunkLeaf {
231                    first: l.first,
232                    n: l.n,
233                    first_run: *run,
234                    key: key.clone(),
235                    leaf: (leaves_base + l.off, l.len),
236                    chunk_base: data_base + s.data_off + coff,
237                })
238                .collect();
239            let dir = Arc::new(
240                LeafedChunks::new(leaves, s.n_chunks, self.fetch(), self.h.internal_codec)
241                    .with_max_block(self.bound(kind::DICT_LEAVES)),
242            );
243            let (r, d) = (self.reader.clone(), dir.clone());
244            let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
245                let (off, len) = d.range(ci)?;
246                let b = r.read_at(off, len).ok()?;
247                decompress_bounded(codec, &b, max).ok()
248            });
249            let (r, d) = (self.reader.clone(), dir.clone());
250            let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
251                d.ensure_leaves(&d.leaves_of(cis));
252                let ranges: Option<Vec<(u64, u64)>> = cis.iter().map(|&ci| d.range(ci)).collect();
253                let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
254                blobs
255                    .iter()
256                    .map(|b| decompress_bounded(codec, b, max).ok())
257                    .collect()
258            });
259            sections.push(
260                ChunkedSection::leafed(meta, dir.clone(), loader, cache.clone(), si as u8)
261                    .with_bulk_loader(bulk),
262            );
263            dirs.push(Some(dir));
264        }
265        let arr: [ChunkedSection; 4] = sections
266            .try_into()
267            .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
268        let r = self.reader.clone();
269        let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
270            // Fault every section's leaves first (one batch), then the chunks.
271            let mut per: [Vec<usize>; 4] = Default::default();
272            for &(si, ci) in pairs {
273                if let Some(v) = per.get_mut(si as usize) {
274                    v.push(ci);
275                }
276            }
277            // One read for the leaves of every section: a batch that names
278            // terms in three sections used to cost three sequential reads here.
279            let wanted: Vec<(usize, Vec<usize>)> = per
280                .iter()
281                .enumerate()
282                .filter_map(|(si, cis)| Some((si, dirs.get(si)?.as_ref()?.leaves_of(cis))))
283                .collect();
284            let across: Vec<(&dyn crate::index::LeafDir, &[usize])> = wanted
285                .iter()
286                .filter_map(|(si, lis)| {
287                    let d: &dyn crate::index::LeafDir = dirs.get(*si)?.as_deref()?;
288                    Some((d, lis.as_slice()))
289                })
290                .collect();
291            crate::index::ensure_leaves_across(&across);
292            let ranges: Option<Vec<(u64, u64)>> = pairs
293                .iter()
294                .map(|&(si, ci)| dirs.get(si as usize)?.as_ref()?.range(ci))
295                .collect();
296            let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
297            blobs
298                .iter()
299                .map(|b| decompress_bounded(codec, b, max).ok())
300                .collect()
301        });
302        let mut dict = Dictionary::from_chunked_sections(arr).with_multi_bulk_loader(multi);
303        dict.set_has_quoted_triples(self.h.flags & F_QUOTED != 0);
304        Ok(dict)
305    }
306
307    /// The class clusters the cluster root lists, checked against the index root:
308    /// every run is whole leaves, and the runs cover the permutation.
309    fn clusters(&self, perms: &[PermRoot]) -> Result<Vec<ClusterOrder>, FileError> {
310        let Some(unit) = self.root.unit(tag::CLUSTER) else {
311            if self.h.flags & F_CLUSTERED != 0 {
312                return Err(FileError::Container(
313                    "0x07 clustered file without a cluster root",
314                ));
315            }
316            return Ok(Vec::new());
317        };
318        let orders = decode_cluster_root(unit)?;
319        let quad = self.quad_root().transpose()?;
320        for c in &orders {
321            // The order's tile count and its leaves, `(first tile, tiles)`.
322            let (n_tiles, leaves): (usize, Vec<(usize, usize)>) = if c.si == QUAD_SPOG {
323                let o = quad
324                    .as_ref()
325                    .and_then(|q| q.orders.iter().find(|o| o.si == c.si))
326                    .ok_or(FileError::Container(
327                        "0x07 cluster root names an order the quad index lacks",
328                    ))?;
329                (
330                    o.n_tiles,
331                    o.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
332                )
333            } else {
334                let p = perms
335                    .iter()
336                    .find(|p| p.si == c.si)
337                    .ok_or(FileError::Container(
338                        "0x07 cluster root names a permutation the index lacks",
339                    ))?;
340                (
341                    p.n_tiles,
342                    p.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
343                )
344            };
345            if c.run_tiles.iter().sum::<usize>() != n_tiles {
346                return Err(FileError::Container(
347                    "0x07 cluster runs do not cover the tiles",
348                ));
349            }
350            // A leaf holds the tiles of one run.
351            let mut ends = Vec::with_capacity(c.run_tiles.len());
352            let mut at = 0usize;
353            for &n in &c.run_tiles {
354                at += n;
355                ends.push(at);
356            }
357            for (first, n) in leaves {
358                let r = ends.partition_point(|&e| e <= first);
359                if first + n > ends[r.min(ends.len() - 1)] {
360                    return Err(FileError::Container(
361                        "0x07 a directory leaf spans two cluster runs",
362                    ));
363                }
364            }
365        }
366        Ok(orders)
367    }
368
369    /// The default graph's index, two-level, tiles faulted on demand.
370    pub fn index(&self, read_concurrency: usize) -> Result<GraphIndex, FileError> {
371        let unit = self
372            .root
373            .unit(tag::INDEX)
374            .ok_or(FileError::Container("0x06 file has no index"))?;
375        let perms_root = decode_index_root(unit)?;
376        let leaves_base = self.section(kind::INDEX_LEAVES).map_or(0, |r| r.offset);
377        let data_base = self.section(kind::INDEX_DATA).map_or(0, |r| r.offset);
378        let perms = self.h.perms();
379        let mut lists: [Option<Arc<LeafedTiles>>; NUM_PERMS] = Default::default();
380        for p in &perms_root {
381            if p.si >= NUM_PERMS {
382                return Err(FileError::Container("0x06 index: bad permutation"));
383            }
384            let leaves: Vec<TileLeaf> = p
385                .leaves
386                .iter()
387                .map(|(l, mn, mx, toff)| TileLeaf {
388                    first: l.first,
389                    n: l.n,
390                    min_a: *mn,
391                    max_a: *mx,
392                    leaf: (leaves_base + l.off, l.len),
393                    tile_base: data_base + p.data_off + toff,
394                })
395                .collect();
396            lists[p.si] = Some(Arc::new(
397                LeafedTiles::new(
398                    leaves,
399                    p.n_tiles,
400                    p.data_len,
401                    self.fetch(),
402                    self.h.internal_codec,
403                )
404                .with_max_block(self.bound(kind::INDEX_LEAVES)),
405            ));
406        }
407        for perm in perms.iter() {
408            if lists[perm.section_index()].is_none() {
409                return Err(FileError::Container(
410                    "0x06 index lacks a stored permutation",
411                ));
412            }
413        }
414        let codec = self.h.block_codec;
415        let max = self.bound(kind::INDEX_DATA);
416        let (r, l) = (self.reader.clone(), lists.clone());
417        let loader: crate::index::TileLoader = Box::new(move |si, ti| {
418            let (off, len) = l.get(si)?.as_ref()?.range(ti)?;
419            let b = r.read_at(off, len).ok()?;
420            decompress_bounded(codec, &b, max).ok()
421        });
422        let (r, l) = (self.reader.clone(), lists.clone());
423        let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
424            let list = l.get(si)?.as_ref()?;
425            if let (Some(&a), Some(&b)) = (tis.first(), tis.last()) {
426                list.ensure_span(a, b + 1);
427            }
428            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
429            let blobs = read_coalesced(&*r, &ranges?, TILE_GAP)?;
430            blobs
431                .iter()
432                .map(|b| decompress_bounded(codec, b, max).ok())
433                .collect()
434        });
435        let (r, l) = (self.reader.clone(), lists.clone());
436        let scattered: crate::index::TileBulkLoader = Box::new(move |si, tis| {
437            let list = l.get(si)?.as_ref()?;
438            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
439            let blobs = read_scattered(&*r, &ranges?)?;
440            blobs
441                .iter()
442                .map(|b| decompress_bounded(codec, b, max).ok())
443                .collect()
444        });
445        let mut index = GraphIndex::from_leafed(lists, perms, loader)
446            .with_bulk_loader(bulk)
447            .with_scattered_loader(scattered);
448        for c in self
449            .clusters(&perms_root)?
450            .into_iter()
451            .filter(|c| c.si < NUM_PERMS)
452        {
453            index = index.with_clusters(
454                c.si,
455                crate::index::ClusterRuns::new(c.classes, &c.run_tiles),
456            );
457        }
458        index.set_read_concurrency(read_concurrency);
459        Ok(index)
460    }
461
462    /// The text index (leafed token table), if the file has one.
463    pub fn text_index(&self) -> Option<Result<crate::text_index::TextIndex, FileError>> {
464        let unit = self.root.unit(tag::TEXT)?;
465        Some((|| {
466            let t = decode_text_root(unit)?;
467            let leaves_base = self.section(kind::TEXT_LEAVES).map_or(0, |r| r.offset);
468            let data = self
469                .section(kind::TEXT_DATA)
470                .unwrap_or(ByteRange { offset: 0, len: 0 });
471            let leaves = t
472                .leaves
473                .iter()
474                .map(|(l, tok)| crate::text_index::TokenLeaf {
475                    first: l.first,
476                    n: l.n,
477                    first_token: String::from_utf8_lossy(tok).into_owned(),
478                    leaf: (leaves_base + l.off, l.len),
479                })
480                .collect();
481            let tokens = Arc::new(
482                crate::text_index::LeafedTokens::new(
483                    leaves,
484                    t.n_tokens,
485                    self.fetch(),
486                    self.h.internal_codec,
487                )
488                .with_max_block(self.bound(kind::TEXT_LEAVES)),
489            );
490            let r = self.reader.clone();
491            let loader = Box::new(move |off: u64, len: u64| {
492                if off.checked_add(len)? > data.len {
493                    return None;
494                }
495                r.read_at(data.offset + off, len).ok()
496            });
497            crate::text_index::TextIndex::leafed(tokens, &t.scope, loader)
498        })())
499    }
500
501    /// Stored bytes of the token table (all leaves) — what a substring search
502    /// would read; a word lookup reads one leaf.
503    pub fn token_table_bytes(&self) -> Option<u64> {
504        self.root.unit(tag::TEXT)?;
505        Some(self.section(kind::TEXT_LEAVES).map_or(0, |r| r.len))
506    }
507
508    /// The label index, if the file has one.
509    pub fn label_index(&self) -> Option<Result<crate::label_index::LabelIndex, FileError>> {
510        let unit = self.root.unit(tag::LABELS)?;
511        Some((|| {
512            let mut l = decode_label_root(unit)?;
513            if self.section(kind::LABEL_ROOT).is_some() {
514                l.head.extend_from_slice(&self.label_root_node()?);
515            }
516            let data = self
517                .section(kind::LABEL_DATA)
518                .unwrap_or(ByteRange { offset: 0, len: 0 });
519            let nodes = self
520                .section(kind::LABEL_LEAVES)
521                .unwrap_or(ByteRange { offset: 0, len: 0 });
522            if data.len != l.data_len {
523                return Err(FileError::Container("0x06 label data length mismatch"));
524            }
525            let head_len = l.head.len() as u64;
526            let section_len = head_len + data.len + nodes.len;
527            let r = self.reader.clone();
528            let head = l.head.clone();
529            let loader = Box::new(move |off: u64, len: u64| -> Option<Vec<u8>> {
530                let end = off.checked_add(len)?;
531                if end <= head_len {
532                    return head.get(off as usize..end as usize).map(<[u8]>::to_vec);
533                }
534                let bo = off.checked_sub(head_len)?;
535                if bo + len <= data.len {
536                    r.read_at(data.offset + bo, len).ok()
537                } else if bo >= data.len && bo - data.len + len <= nodes.len {
538                    r.read_at(nodes.offset + (bo - data.len), len).ok()
539                } else {
540                    None
541                }
542            });
543            let max = self
544                .bound(kind::LABEL_LEAVES)
545                .max(self.bound(kind::LABEL_DATA));
546            crate::label_index::LabelIndex::from_head(
547                &l.head,
548                section_len,
549                self.h.block_codec,
550                loader,
551            )
552            .map(|li| li.with_max_block(max))
553        })())
554    }
555
556    /// The label index's entry count, from the root alone.
557    pub fn label_entry_count(&self) -> Option<u64> {
558        let l = decode_label_root(self.root.unit(tag::LABELS)?).ok()?;
559        crate::label_index::LabelIndexPreamble::parse(&l.head)
560            .ok()
561            .map(|p| p.entry_count)
562    }
563
564    /// The quad index's root, if the file has named graphs.
565    pub fn quad_root(&self) -> Option<Result<QuadRoot, FileError>> {
566        self.root.unit(tag::QUAD).map(decode_quad_root)
567    }
568
569    /// The named graphs (quad index), if any.
570    pub(crate) fn quad(
571        &self,
572        read_concurrency: usize,
573    ) -> Option<Result<crate::quad::QuadNamed, FileError>> {
574        let root = match self.quad_root()? {
575            Ok(r) => r,
576            Err(e) => return Some(Err(e)),
577        };
578        let leaves = self.section(kind::QUAD_LEAVES).map_or(0, |r| r.offset);
579        let data = self.section(kind::QUAD_DATA).map_or(0, |r| r.offset);
580        // The class runs of SPOG, when the file clusters it.
581        let clusters = match self.root.unit(tag::CLUSTER).map(decode_cluster_root) {
582            Some(Ok(c)) => c,
583            Some(Err(e)) => return Some(Err(e)),
584            None => Vec::new(),
585        };
586        Some(
587            crate::quad::V06Quad::new(
588                &root,
589                leaves,
590                data,
591                self.h.internal_codec,
592                self.h.version == VERSION,
593            )
594            .and_then(|v| v.with_clusters(&clusters))
595            .map(|mut v| {
596                v.tile_max = self.bound(kind::QUAD_DATA);
597                v.leaf_max = self.bound(kind::QUAD_LEAVES);
598                crate::quad::QuadNamed::new_v06(
599                    self.reader.clone(),
600                    v,
601                    self.h.block_codec,
602                    self.h.perms(),
603                    read_concurrency,
604                )
605            }),
606        )
607    }
608
609    /// Verify every digest: the root, the card, each section (streamed), and
610    /// the content hash. `only` restricts the section checks to one kind.
611    /// `Ok(false)` names nothing; [`verify_report`](Self::verify_report) does.
612    pub fn verify(&self, only: Option<u8>) -> Result<bool, FileError> {
613        Ok(self.verify_report(only)?.is_empty())
614    }
615
616    /// Like [`verify`](Self::verify), returning what failed (empty = all good).
617    pub fn verify_report(&self, only: Option<u8>) -> Result<Vec<String>, FileError> {
618        const SLICE: u64 = 8 << 20;
619        let mut bad = Vec::new();
620        if content_hash(&self.h) != self.h.content_hash {
621            bad.push("content hash".to_string());
622        }
623        let card = self.h.card_range();
624        if only.is_none() && card.len > 0 {
625            let stored = self.reader.read_at(card.offset, card.len)?;
626            if digest16(&stored) != self.h.card_digest {
627                bad.push("card".to_string());
628            }
629        }
630        for s in &self.root.sections {
631            if only.is_some_and(|k| k != s.kind) {
632                continue;
633            }
634            let a = s.abs(&self.h)?;
635            let mut h = blake3::Hasher::new();
636            let mut at = a.offset;
637            while at < a.offset + a.len {
638                let n = (a.offset + a.len - at).min(SLICE);
639                h.update(&self.reader.read_at(at, n)?);
640                at += n;
641            }
642            if h.finalize().as_bytes()[..16] != s.digest {
643                bad.push(kind::name(s.kind).to_string());
644            }
645        }
646        if only.is_none() {
647            let want = footer(self.h.version);
648            let foot = self
649                .reader
650                .read_at(self.h.file_len - want.len() as u64, want.len() as u64)?;
651            if foot != want {
652                bad.push("footer".to_string());
653            }
654        }
655        Ok(bad)
656    }
657}
658
659/// Is this file (by its first bytes) a `0x06` or `0x07` file?
660pub fn is_v06(first: &[u8]) -> bool {
661    first.len() > 4 && first[0..4] == crate::header::MAGIC && reads(first[4])
662}
663
664/// Read a `0x06` file's header and root through any reader (two reads at most;
665/// over a seeded or prefix-holding reader, none).
666pub fn read_front<R: RangeReader + ?Sized>(r: &R) -> Result<(Header06, Root06), FileError> {
667    let head = r.read_at(0, HEADER_LEN as u64)?;
668    let h = Header06::from_bytes(&head)?;
669    let root_bytes = r.read_at(HEADER_LEN as u64, h.root_len as u64)?;
670    if digest16(&root_bytes) != h.root_digest {
671        return Err(FileError::Container("0x06 root digest mismatch"));
672    }
673    let root = Root06::decode(&root_bytes, h.internal_codec)?;
674    Ok((h, root))
675}
676
677/// The stored bytes of section `k`, if present.
678pub fn read_section<R: RangeReader + ?Sized>(
679    r: &R,
680    h: &Header06,
681    root: &Root06,
682    k: u8,
683) -> Result<Option<Vec<u8>>, FileError> {
684    match root.section(k) {
685        Some(s) => {
686            let a = s.abs(h)?;
687            Ok(Some(r.read_at(a.offset, a.len)?))
688        }
689        None => Ok(None),
690    }
691}
692
693/// The card (decompressed) and the build record (raw): one read of the card
694/// region, which sits right after the root.
695#[allow(clippy::type_complexity)]
696pub fn read_card_build<R: RangeReader + ?Sized>(
697    r: &R,
698    h: &Header06,
699) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
700    read_card_build_with(r, h, true)
701}
702
703/// [`read_card_build`], reading the build record only when `want_build`: a
704/// caller that never prints it stops its read at the end of the card.
705#[allow(clippy::type_complexity)]
706pub fn read_card_build_with<R: RangeReader + ?Sized>(
707    r: &R,
708    h: &Header06,
709    want_build: bool,
710) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
711    let c = h.card_range();
712    let mut b = h.build_range();
713    if !want_build {
714        b.len = 0;
715    }
716    if c.len == 0 && b.len == 0 {
717        return Ok((None, None));
718    }
719    let lo = if c.len == 0 {
720        b.offset
721    } else if b.len == 0 {
722        c.offset
723    } else {
724        c.offset.min(b.offset)
725    };
726    let hi = (c.offset + c.len).max(b.offset + b.len);
727    let span = r.read_at(lo, hi - lo)?;
728    let card = if c.len > 0 {
729        let stored = &span[(c.offset - lo) as usize..(c.offset - lo + c.len) as usize];
730        if digest16(stored) != h.card_digest {
731            return Err(FileError::Container("0x06 card digest mismatch"));
732        }
733        Some(decompress_bounded(h.internal_codec, stored, CARD_MAX)?)
734    } else {
735        None
736    };
737    let build = (b.len > 0)
738        .then(|| span[(b.offset - lo) as usize..(b.offset - lo + b.len) as usize].to_vec());
739    Ok((card, build))
740}
741
742/// Section `k`'s decompression bound in `root`: its recorded largest block,
743/// or [`BLOCK_CEILING`].
744pub fn section_bound(root: &Root06, k: u8) -> u64 {
745    root.section(k)
746        .map(|s| s.max_block as u64)
747        .filter(|&m| m > 0)
748        .unwrap_or(BLOCK_CEILING)
749}
750
751/// The stats + schema section as the digest-prefixed bytes its decoder takes.
752pub fn read_schema_bytes<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
753    let (h, root) = read_front(r)?;
754    let Some(stored) = read_section(r, &h, &root, kind::SCHEMA)? else {
755        return Ok(None);
756    };
757    let body = decompress_bounded(
758        h.internal_codec,
759        &stored,
760        section_bound(&root, kind::SCHEMA),
761    )?;
762    let mut out = digest16(&body).to_vec();
763    out.extend_from_slice(&body);
764    Ok(Some(out))
765}
766
767/// The community pyramid's bytes, if any.
768pub fn read_pyramid<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
769    let (h, root) = read_front(r)?;
770    read_section(r, &h, &root, kind::PYRAMID)
771}
772
773/// Stored bytes of the text index's token table (all leaves).
774pub fn read_token_table_bytes<R: RangeReader + ?Sized>(r: &R) -> Option<u64> {
775    let (_, root) = read_front(r).ok()?;
776    root.unit(tag::TEXT)?;
777    Some(root.section(kind::TEXT_LEAVES).map_or(0, |s| s.len))
778}
779
780/// A reader holding only some ranges of a file in memory (the borrowed-reader
781/// opens of a `0x06` file read what they need into one of these).
782pub struct SparseReader {
783    len: u64,
784    parts: Vec<(u64, Vec<u8>)>,
785}
786
787impl SparseReader {
788    /// Fetch `ranges` from `r` (plus the first bytes `first`) into memory —
789    /// the ones `first` does not already hold, in one batch.
790    pub fn fetch<R: RangeReader + ?Sized>(
791        r: &R,
792        first: Vec<u8>,
793        ranges: &[ByteRange],
794    ) -> Result<Self, FileError> {
795        let held = first.len() as u64;
796        let want: Vec<(u64, u64)> = ranges
797            .iter()
798            .filter(|g| g.len > 0 && g.offset.saturating_add(g.len) > held)
799            .map(|g| (g.offset, g.len))
800            .collect();
801        let mut parts = vec![(0u64, first)];
802        if !want.is_empty() {
803            for ((o, _), b) in want.iter().zip(r.read_many(&want)?) {
804                parts.push((*o, b));
805            }
806        }
807        Ok(SparseReader {
808            len: r.len(),
809            parts,
810        })
811    }
812}
813
814/// A borrowed reader whose first bytes are already in memory: reads inside
815/// them cost nothing, the rest go to `r`.
816pub struct Prefixed<'a, R: RangeReader + ?Sized> {
817    /// The file's first bytes.
818    pub first: &'a [u8],
819    /// The reader behind them.
820    pub r: &'a R,
821}
822
823impl<R: RangeReader + ?Sized> RangeReader for Prefixed<'_, R> {
824    fn len(&self) -> u64 {
825        self.r.len()
826    }
827
828    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
829        match offset.checked_add(len) {
830            Some(end) if end <= self.first.len() as u64 => {
831                Ok(self.first[offset as usize..end as usize].to_vec())
832            }
833            _ => self.r.read_at(offset, len),
834        }
835    }
836}
837
838impl RangeReader for SparseReader {
839    fn len(&self) -> u64 {
840        self.len
841    }
842
843    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
844        for (o, b) in &self.parts {
845            if offset >= *o && offset + len <= o + b.len() as u64 {
846                let s = (offset - o) as usize;
847                return Ok(b[s..s + len as usize].to_vec());
848            }
849        }
850        Err(std::io::Error::other("0x06: range not held by this reader"))
851    }
852}
853
854/// What the card's measured signals need from a `0x06` file's root: the text
855/// index's `(bytes, token-table bytes)` and the label index's `(bytes, entries)`.
856#[allow(clippy::type_complexity)]
857pub fn read_signals<R: RangeReader + ?Sized>(
858    r: &R,
859) -> Result<(Option<(u64, u64)>, Option<(u64, Option<u64>)>), FileError> {
860    let (_, root) = read_front(r)?;
861    let len = |k: u8| root.section(k).map_or(0, |s| s.len);
862    let text = root.unit(tag::TEXT).map(|_| {
863        (
864            len(kind::TEXT_LEAVES) + len(kind::TEXT_DATA),
865            len(kind::TEXT_LEAVES),
866        )
867    });
868    let labels = match root.unit(tag::LABELS) {
869        None => None,
870        Some(u) => {
871            let l = decode_label_root(u)?;
872            let entries = crate::label_index::LabelIndexPreamble::parse(&l.head)
873                .ok()
874                .map(|p| p.entry_count);
875            Some((
876                l.head.len() as u64
877                    + len(kind::LABEL_ROOT)
878                    + len(kind::LABEL_DATA)
879                    + len(kind::LABEL_LEAVES),
880                entries,
881            ))
882        }
883    };
884    Ok((text, labels))
885}
886
887/// Check `bytes` against the digest the root records for section `k`.
888pub fn check_digest(root: &Root06, k: u8, bytes: &[u8]) -> Result<(), FileError> {
889    match root.section(k) {
890        Some(s) if s.digest == digest16(bytes) => Ok(()),
891        Some(_) => Err(FileError::Container("0x07 section digest mismatch")),
892        None => Ok(()),
893    }
894}
895
896/// The card annex of a `0x07` file (raw JSON, decompressed, digest checked).
897pub fn read_annex<R: RangeReader + ?Sized>(
898    r: &R,
899    h: &Header06,
900    root: &Root06,
901) -> Result<Option<Vec<u8>>, FileError> {
902    let Some(stored) = read_section(r, h, root, kind::CARD_ANNEX)? else {
903        return Ok(None);
904    };
905    check_digest(root, kind::CARD_ANNEX, &stored)?;
906    let max = root
907        .section(kind::CARD_ANNEX)
908        .map(|s| s.max_block as u64)
909        .filter(|&m| m > 0)
910        .unwrap_or(ANNEX_MAX);
911    Ok(Some(decompress_bounded(h.internal_codec, &stored, max)?))
912}
913
914/// The card of the file read through `r` with its annex merged back in (what
915/// `rete card --full` shows), and whether an annex was merged.
916pub fn read_card_full<R: RangeReader + ?Sized>(
917    r: &R,
918) -> Result<(Option<Vec<u8>>, bool), FileError> {
919    let (h, root) = read_front(r)?;
920    let Some(hot) = read_card_build(r, &h)?.0 else {
921        return Ok((None, false));
922    };
923    match read_annex(r, &h, &root)? {
924        None => Ok((Some(hot), false)),
925        Some(a) => annex::merge(&hot, &a)
926            .map(|c| (Some(c), true))
927            .map_err(|_| FileError::Container("0x07 card annex does not merge")),
928    }
929}