Skip to main content

rete_core/v06/
open.rs

1//! Reading a `0x06` file: the header and the root come with the first read;
2//! everything else is built lazily from them — the dictionary and the index as
3//! two-level directories whose leaves fault on demand and stay for the session,
4//! the text index likewise, the label index and the quad index through their
5//! own lazy machinery.
6
7use std::sync::Arc;
8
9use super::*;
10use crate::dict::{ChunkLeaf, ChunkedSection, LeafedChunks, SectionMeta};
11use crate::dictionary::Dictionary;
12use crate::file::{decompress_bounded, ByteRange, FileError};
13use crate::index::{GraphIndex, LeafFetch, LeafedTiles, TileLeaf, NUM_PERMS};
14use crate::reader::RangeReader;
15
16/// Coalescing gaps for the batched faults (as for `0x05`).
17const TILE_GAP: u64 = 4096;
18const DICT_GAP: u64 = 64 * 1024;
19
20/// An open `0x06` file: its header and root, and the reader.
21pub struct V06File {
22    /// The 128-byte header.
23    pub h: Header06,
24    /// The decoded root block.
25    pub root: Root06,
26    /// Where everything else is read from.
27    pub reader: Arc<dyn RangeReader + Send + Sync>,
28}
29
30/// Read `(offset, len)` ranges as one coalesced batch.
31fn read_coalesced(
32    r: &(dyn RangeReader + Send + Sync),
33    ranges: &[(u64, u64)],
34    gap: u64,
35) -> Option<Vec<Vec<u8>>> {
36    let rs: Vec<ByteRange> = ranges
37        .iter()
38        .map(|&(offset, len)| ByteRange { offset, len })
39        .collect();
40    crate::file::read_coalesced(r, &rs, gap)
41}
42
43/// [`read_coalesced`] for tiles far apart: the reader gets the exact ranges.
44fn read_scattered(
45    r: &(dyn RangeReader + Send + Sync),
46    ranges: &[(u64, u64)],
47) -> Option<Vec<Vec<u8>>> {
48    let rs: Vec<ByteRange> = ranges
49        .iter()
50        .map(|&(offset, len)| ByteRange { offset, len })
51        .collect();
52    crate::file::read_coalesced_scattered(r, &rs, 0)
53}
54
55impl V06File {
56    /// Open from `first`, the file's leading bytes (at least the header; the
57    /// root is read from it when it fits, else fetched).
58    pub fn open(
59        reader: Arc<dyn RangeReader + Send + Sync>,
60        first: &[u8],
61    ) -> Result<Self, FileError> {
62        let h = Header06::from_bytes(first)?;
63        if h.file_len != reader.len() {
64            return Err(FileError::Container(
65                "0x06 file length does not match the header (truncated?)",
66            ));
67        }
68        let rs = HEADER_LEN;
69        let re = rs + h.root_len as usize;
70        let root_bytes = if re <= first.len() {
71            first[rs..re].to_vec()
72        } else {
73            reader.read_at(rs as u64, h.root_len as u64)?
74        };
75        if digest16(&root_bytes) != h.root_digest {
76            return Err(FileError::Container("0x06 root digest mismatch"));
77        }
78        let root = Root06::decode(&root_bytes, h.internal_codec)?;
79        for s in &root.sections {
80            let a = s.abs(&h)?;
81            let end = a.offset + a.len;
82            let ok = if kind::is_front(s.kind) {
83                a.offset >= h.front_base() && end <= h.data_offset
84            } else {
85                a.offset >= h.data_offset && end <= h.file_len - FOOTER.len() as u64
86            };
87            if !ok {
88                return Err(FileError::Container("0x06 section out of its region"));
89            }
90        }
91        Ok(V06File { h, root, reader })
92    }
93
94    /// The decompression bound of section `k`'s blocks: its recorded largest
95    /// block, or [`BLOCK_CEILING`] where none is recorded.
96    pub fn bound(&self, k: u8) -> u64 {
97        section_bound(&self.root, k)
98    }
99
100    /// The absolute range of section `k`, if present.
101    pub fn section(&self, k: u8) -> Option<ByteRange> {
102        self.root.section(k).and_then(|s| s.abs(&self.h).ok())
103    }
104
105    /// A batched leaf fetcher over this file's reader (leaves are stored with
106    /// the internal codec; decoding happens in the directory types).
107    pub fn fetch(&self) -> LeafFetch {
108        let r = self.reader.clone();
109        Arc::new(move |ranges: &[(u64, u64)]| read_coalesced(&*r, ranges, 0))
110    }
111
112    /// The Dataset Card (decompressed), if the file has one.
113    pub fn card(&self) -> Result<Option<Vec<u8>>, FileError> {
114        let r = self.h.card_range();
115        if r.len == 0 {
116            return Ok(None);
117        }
118        let stored = self.reader.read_at(r.offset, r.len)?;
119        if digest16(&stored) != self.h.card_digest {
120            return Err(FileError::Container("0x06 card digest mismatch"));
121        }
122        Ok(Some(decompress_bounded(
123            self.h.internal_codec,
124            &stored,
125            CARD_MAX,
126        )?))
127    }
128
129    /// The build record (raw), if the file has one.
130    pub fn build_record(&self) -> Result<Option<Vec<u8>>, FileError> {
131        let r = self.h.build_range();
132        if r.len == 0 {
133            return Ok(None);
134        }
135        Ok(Some(self.reader.read_at(r.offset, r.len)?))
136    }
137
138    /// The stats + schema section, as the `0x05` SCHEMA section bytes
139    /// (digest-prefixed) its decoder takes.
140    pub fn schema_section_bytes(&self) -> Option<Vec<u8>> {
141        let r = self.section(kind::SCHEMA)?;
142        let stored = self.reader.read_at(r.offset, r.len).ok()?;
143        let body =
144            decompress_bounded(self.h.internal_codec, &stored, self.bound(kind::SCHEMA)).ok()?;
145        let mut out = digest16(&body).to_vec();
146        out.extend_from_slice(&body);
147        Some(out)
148    }
149
150    /// The decoded stats + schema section.
151    pub fn schema(&self) -> Option<crate::schema_section::SchemaMeta> {
152        crate::schema_section::SchemaMeta::decode(&self.schema_section_bytes()?).ok()
153    }
154
155    /// The community pyramid's bytes.
156    pub fn pyramid_bytes(&self) -> Option<Vec<u8>> {
157        let r = self.section(kind::PYRAMID)?;
158        self.reader.read_at(r.offset, r.len).ok()
159    }
160
161    /// The four dictionary sections, two-level, chunks faulted on demand.
162    /// `want[si] == false` leaves that section empty (a search-only open).
163    pub fn dictionary(&self, want: [bool; 4]) -> Result<Dictionary, FileError> {
164        let unit = self
165            .root
166            .unit(tag::DICT)
167            .ok_or(FileError::Container("0x06 file has no dictionary"))?;
168        let roots = decode_dict_root(unit)?;
169        let leaves_base = self.section(kind::DICT_LEAVES).map_or(0, |r| r.offset);
170        let data_base = self.section(kind::DICT_DATA).map_or(0, |r| r.offset);
171        let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
172        let codec = self.h.dict_codec;
173        let max = self.bound(kind::DICT_DATA);
174        let mut dirs: Vec<Option<Arc<LeafedChunks>>> = Vec::with_capacity(4);
175        let mut sections = Vec::with_capacity(4);
176        for (si, s) in roots.iter().enumerate() {
177            let meta = SectionMeta {
178                term_count: if want[si] { s.term_count } else { 0 },
179                restart_interval: s.restart_interval.max(1),
180                restart_offsets: Vec::new(),
181            };
182            if !want[si] {
183                sections.push(ChunkedSection::from_parts(
184                    meta,
185                    Vec::new(),
186                    None,
187                    cache.clone(),
188                    si as u8,
189                ));
190                dirs.push(None);
191                continue;
192            }
193            let leaves: Vec<ChunkLeaf> = s
194                .leaves
195                .iter()
196                .map(|(l, run, coff, key)| ChunkLeaf {
197                    first: l.first,
198                    n: l.n,
199                    first_run: *run,
200                    key: key.clone(),
201                    leaf: (leaves_base + l.off, l.len),
202                    chunk_base: data_base + s.data_off + coff,
203                })
204                .collect();
205            let dir = Arc::new(
206                LeafedChunks::new(leaves, s.n_chunks, self.fetch(), self.h.internal_codec)
207                    .with_max_block(self.bound(kind::DICT_LEAVES)),
208            );
209            let (r, d) = (self.reader.clone(), dir.clone());
210            let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
211                let (off, len) = d.range(ci)?;
212                let b = r.read_at(off, len).ok()?;
213                decompress_bounded(codec, &b, max).ok()
214            });
215            let (r, d) = (self.reader.clone(), dir.clone());
216            let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
217                d.ensure_leaves(&d.leaves_of(cis));
218                let ranges: Option<Vec<(u64, u64)>> = cis.iter().map(|&ci| d.range(ci)).collect();
219                let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
220                blobs
221                    .iter()
222                    .map(|b| decompress_bounded(codec, b, max).ok())
223                    .collect()
224            });
225            sections.push(
226                ChunkedSection::leafed(meta, dir.clone(), loader, cache.clone(), si as u8)
227                    .with_bulk_loader(bulk),
228            );
229            dirs.push(Some(dir));
230        }
231        let arr: [ChunkedSection; 4] = sections
232            .try_into()
233            .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
234        let r = self.reader.clone();
235        let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
236            // Fault every section's leaves first (one batch), then the chunks.
237            let mut per: [Vec<usize>; 4] = Default::default();
238            for &(si, ci) in pairs {
239                if let Some(v) = per.get_mut(si as usize) {
240                    v.push(ci);
241                }
242            }
243            // One read for the leaves of every section: a batch that names
244            // terms in three sections used to cost three sequential reads here.
245            let wanted: Vec<(usize, Vec<usize>)> = per
246                .iter()
247                .enumerate()
248                .filter_map(|(si, cis)| Some((si, dirs.get(si)?.as_ref()?.leaves_of(cis))))
249                .collect();
250            let across: Vec<(&dyn crate::index::LeafDir, &[usize])> = wanted
251                .iter()
252                .filter_map(|(si, lis)| {
253                    let d: &dyn crate::index::LeafDir = dirs.get(*si)?.as_deref()?;
254                    Some((d, lis.as_slice()))
255                })
256                .collect();
257            crate::index::ensure_leaves_across(&across);
258            let ranges: Option<Vec<(u64, u64)>> = pairs
259                .iter()
260                .map(|&(si, ci)| dirs.get(si as usize)?.as_ref()?.range(ci))
261                .collect();
262            let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
263            blobs
264                .iter()
265                .map(|b| decompress_bounded(codec, b, max).ok())
266                .collect()
267        });
268        let mut dict = Dictionary::from_chunked_sections(arr).with_multi_bulk_loader(multi);
269        dict.set_has_quoted_triples(self.h.flags & F_QUOTED != 0);
270        Ok(dict)
271    }
272
273    /// The default graph's index, two-level, tiles faulted on demand.
274    pub fn index(&self, read_concurrency: usize) -> Result<GraphIndex, FileError> {
275        let unit = self
276            .root
277            .unit(tag::INDEX)
278            .ok_or(FileError::Container("0x06 file has no index"))?;
279        let perms_root = decode_index_root(unit)?;
280        let leaves_base = self.section(kind::INDEX_LEAVES).map_or(0, |r| r.offset);
281        let data_base = self.section(kind::INDEX_DATA).map_or(0, |r| r.offset);
282        let perms = self.h.perms();
283        let mut lists: [Option<Arc<LeafedTiles>>; NUM_PERMS] = Default::default();
284        for p in &perms_root {
285            if p.si >= NUM_PERMS {
286                return Err(FileError::Container("0x06 index: bad permutation"));
287            }
288            let leaves: Vec<TileLeaf> = p
289                .leaves
290                .iter()
291                .map(|(l, mn, mx, toff)| TileLeaf {
292                    first: l.first,
293                    n: l.n,
294                    min_a: *mn,
295                    max_a: *mx,
296                    leaf: (leaves_base + l.off, l.len),
297                    tile_base: data_base + p.data_off + toff,
298                })
299                .collect();
300            lists[p.si] = Some(Arc::new(
301                LeafedTiles::new(
302                    leaves,
303                    p.n_tiles,
304                    p.data_len,
305                    self.fetch(),
306                    self.h.internal_codec,
307                )
308                .with_max_block(self.bound(kind::INDEX_LEAVES)),
309            ));
310        }
311        for perm in perms.iter() {
312            if lists[perm.section_index()].is_none() {
313                return Err(FileError::Container(
314                    "0x06 index lacks a stored permutation",
315                ));
316            }
317        }
318        let codec = self.h.block_codec;
319        let max = self.bound(kind::INDEX_DATA);
320        let (r, l) = (self.reader.clone(), lists.clone());
321        let loader: crate::index::TileLoader = Box::new(move |si, ti| {
322            let (off, len) = l.get(si)?.as_ref()?.range(ti)?;
323            let b = r.read_at(off, len).ok()?;
324            decompress_bounded(codec, &b, max).ok()
325        });
326        let (r, l) = (self.reader.clone(), lists.clone());
327        let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
328            let list = l.get(si)?.as_ref()?;
329            if let (Some(&a), Some(&b)) = (tis.first(), tis.last()) {
330                list.ensure_span(a, b + 1);
331            }
332            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
333            let blobs = read_coalesced(&*r, &ranges?, TILE_GAP)?;
334            blobs
335                .iter()
336                .map(|b| decompress_bounded(codec, b, max).ok())
337                .collect()
338        });
339        let (r, l) = (self.reader.clone(), lists.clone());
340        let scattered: crate::index::TileBulkLoader = Box::new(move |si, tis| {
341            let list = l.get(si)?.as_ref()?;
342            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
343            let blobs = read_scattered(&*r, &ranges?)?;
344            blobs
345                .iter()
346                .map(|b| decompress_bounded(codec, b, max).ok())
347                .collect()
348        });
349        let mut index = GraphIndex::from_leafed(lists, perms, loader)
350            .with_bulk_loader(bulk)
351            .with_scattered_loader(scattered);
352        index.set_read_concurrency(read_concurrency);
353        Ok(index)
354    }
355
356    /// The text index (leafed token table), if the file has one.
357    pub fn text_index(&self) -> Option<Result<crate::text_index::TextIndex, FileError>> {
358        let unit = self.root.unit(tag::TEXT)?;
359        Some((|| {
360            let t = decode_text_root(unit)?;
361            let leaves_base = self.section(kind::TEXT_LEAVES).map_or(0, |r| r.offset);
362            let data = self
363                .section(kind::TEXT_DATA)
364                .unwrap_or(ByteRange { offset: 0, len: 0 });
365            let leaves = t
366                .leaves
367                .iter()
368                .map(|(l, tok)| crate::text_index::TokenLeaf {
369                    first: l.first,
370                    n: l.n,
371                    first_token: String::from_utf8_lossy(tok).into_owned(),
372                    leaf: (leaves_base + l.off, l.len),
373                })
374                .collect();
375            let tokens = Arc::new(
376                crate::text_index::LeafedTokens::new(
377                    leaves,
378                    t.n_tokens,
379                    self.fetch(),
380                    self.h.internal_codec,
381                )
382                .with_max_block(self.bound(kind::TEXT_LEAVES)),
383            );
384            let r = self.reader.clone();
385            let loader = Box::new(move |off: u64, len: u64| {
386                if off.checked_add(len)? > data.len {
387                    return None;
388                }
389                r.read_at(data.offset + off, len).ok()
390            });
391            crate::text_index::TextIndex::leafed(tokens, &t.scope, loader)
392        })())
393    }
394
395    /// Stored bytes of the token table (all leaves) — what a substring search
396    /// would read; a word lookup reads one leaf.
397    pub fn token_table_bytes(&self) -> Option<u64> {
398        self.root.unit(tag::TEXT)?;
399        Some(self.section(kind::TEXT_LEAVES).map_or(0, |r| r.len))
400    }
401
402    /// The label index, if the file has one.
403    pub fn label_index(&self) -> Option<Result<crate::label_index::LabelIndex, FileError>> {
404        let unit = self.root.unit(tag::LABELS)?;
405        Some((|| {
406            let l = decode_label_root(unit)?;
407            let data = self
408                .section(kind::LABEL_DATA)
409                .unwrap_or(ByteRange { offset: 0, len: 0 });
410            let nodes = self
411                .section(kind::LABEL_LEAVES)
412                .unwrap_or(ByteRange { offset: 0, len: 0 });
413            if data.len != l.data_len {
414                return Err(FileError::Container("0x06 label data length mismatch"));
415            }
416            let head_len = l.head.len() as u64;
417            let section_len = head_len + data.len + nodes.len;
418            let r = self.reader.clone();
419            let head = l.head.clone();
420            let loader = Box::new(move |off: u64, len: u64| -> Option<Vec<u8>> {
421                let end = off.checked_add(len)?;
422                if end <= head_len {
423                    return head.get(off as usize..end as usize).map(<[u8]>::to_vec);
424                }
425                let bo = off.checked_sub(head_len)?;
426                if bo + len <= data.len {
427                    r.read_at(data.offset + bo, len).ok()
428                } else if bo >= data.len && bo - data.len + len <= nodes.len {
429                    r.read_at(nodes.offset + (bo - data.len), len).ok()
430                } else {
431                    None
432                }
433            });
434            let max = self
435                .bound(kind::LABEL_LEAVES)
436                .max(self.bound(kind::LABEL_DATA));
437            crate::label_index::LabelIndex::from_head(
438                &l.head,
439                section_len,
440                self.h.block_codec,
441                loader,
442            )
443            .map(|li| li.with_max_block(max))
444        })())
445    }
446
447    /// The label index's entry count, from the root alone.
448    pub fn label_entry_count(&self) -> Option<u64> {
449        let l = decode_label_root(self.root.unit(tag::LABELS)?).ok()?;
450        crate::label_index::LabelIndexPreamble::parse(&l.head)
451            .ok()
452            .map(|p| p.entry_count)
453    }
454
455    /// The quad index's root, if the file has named graphs.
456    pub fn quad_root(&self) -> Option<Result<QuadRoot, FileError>> {
457        self.root.unit(tag::QUAD).map(decode_quad_root)
458    }
459
460    /// The named graphs (quad index), if any.
461    pub(crate) fn quad(
462        &self,
463        read_concurrency: usize,
464    ) -> Option<Result<crate::quad::QuadNamed, FileError>> {
465        let root = match self.quad_root()? {
466            Ok(r) => r,
467            Err(e) => return Some(Err(e)),
468        };
469        let leaves = self.section(kind::QUAD_LEAVES).map_or(0, |r| r.offset);
470        let data = self.section(kind::QUAD_DATA).map_or(0, |r| r.offset);
471        Some(
472            crate::quad::V06Quad::new(&root, leaves, data, self.h.internal_codec).map(|mut v| {
473                v.tile_max = self.bound(kind::QUAD_DATA);
474                v.leaf_max = self.bound(kind::QUAD_LEAVES);
475                crate::quad::QuadNamed::new_v06(
476                    self.reader.clone(),
477                    v,
478                    self.h.block_codec,
479                    self.h.perms(),
480                    read_concurrency,
481                )
482            }),
483        )
484    }
485
486    /// Verify every digest: the root, the card, each section (streamed), and
487    /// the content hash. `only` restricts the section checks to one kind.
488    /// `Ok(false)` names nothing; [`verify_report`](Self::verify_report) does.
489    pub fn verify(&self, only: Option<u8>) -> Result<bool, FileError> {
490        Ok(self.verify_report(only)?.is_empty())
491    }
492
493    /// Like [`verify`](Self::verify), returning what failed (empty = all good).
494    pub fn verify_report(&self, only: Option<u8>) -> Result<Vec<String>, FileError> {
495        const SLICE: u64 = 8 << 20;
496        let mut bad = Vec::new();
497        if content_hash(&self.h) != self.h.content_hash {
498            bad.push("content hash".to_string());
499        }
500        let card = self.h.card_range();
501        if only.is_none() && card.len > 0 {
502            let stored = self.reader.read_at(card.offset, card.len)?;
503            if digest16(&stored) != self.h.card_digest {
504                bad.push("card".to_string());
505            }
506        }
507        for s in &self.root.sections {
508            if only.is_some_and(|k| k != s.kind) {
509                continue;
510            }
511            let a = s.abs(&self.h)?;
512            let mut h = blake3::Hasher::new();
513            let mut at = a.offset;
514            while at < a.offset + a.len {
515                let n = (a.offset + a.len - at).min(SLICE);
516                h.update(&self.reader.read_at(at, n)?);
517                at += n;
518            }
519            if h.finalize().as_bytes()[..16] != s.digest {
520                bad.push(kind::name(s.kind).to_string());
521            }
522        }
523        if only.is_none() {
524            let foot = self
525                .reader
526                .read_at(self.h.file_len - FOOTER.len() as u64, FOOTER.len() as u64)?;
527            if foot != FOOTER {
528                bad.push("footer".to_string());
529            }
530        }
531        Ok(bad)
532    }
533}
534
535/// Is this file (by its first bytes) a `0x06` file?
536pub fn is_v06(first: &[u8]) -> bool {
537    first.len() > 4 && first[0..4] == crate::header::MAGIC && first[4] == VERSION
538}
539
540/// Read a `0x06` file's header and root through any reader (two reads at most;
541/// over a seeded or prefix-holding reader, none).
542pub fn read_front<R: RangeReader + ?Sized>(r: &R) -> Result<(Header06, Root06), FileError> {
543    let head = r.read_at(0, HEADER_LEN as u64)?;
544    let h = Header06::from_bytes(&head)?;
545    let root_bytes = r.read_at(HEADER_LEN as u64, h.root_len as u64)?;
546    if digest16(&root_bytes) != h.root_digest {
547        return Err(FileError::Container("0x06 root digest mismatch"));
548    }
549    let root = Root06::decode(&root_bytes, h.internal_codec)?;
550    Ok((h, root))
551}
552
553/// The stored bytes of section `k`, if present.
554pub fn read_section<R: RangeReader + ?Sized>(
555    r: &R,
556    h: &Header06,
557    root: &Root06,
558    k: u8,
559) -> Result<Option<Vec<u8>>, FileError> {
560    match root.section(k) {
561        Some(s) => {
562            let a = s.abs(h)?;
563            Ok(Some(r.read_at(a.offset, a.len)?))
564        }
565        None => Ok(None),
566    }
567}
568
569/// The card (decompressed) and the build record (raw): one read of the card
570/// region, which sits right after the root.
571#[allow(clippy::type_complexity)]
572pub fn read_card_build<R: RangeReader + ?Sized>(
573    r: &R,
574    h: &Header06,
575) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
576    read_card_build_with(r, h, true)
577}
578
579/// [`read_card_build`], reading the build record only when `want_build`: a
580/// caller that never prints it stops its read at the end of the card.
581#[allow(clippy::type_complexity)]
582pub fn read_card_build_with<R: RangeReader + ?Sized>(
583    r: &R,
584    h: &Header06,
585    want_build: bool,
586) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
587    let c = h.card_range();
588    let mut b = h.build_range();
589    if !want_build {
590        b.len = 0;
591    }
592    if c.len == 0 && b.len == 0 {
593        return Ok((None, None));
594    }
595    let lo = if c.len == 0 {
596        b.offset
597    } else if b.len == 0 {
598        c.offset
599    } else {
600        c.offset.min(b.offset)
601    };
602    let hi = (c.offset + c.len).max(b.offset + b.len);
603    let span = r.read_at(lo, hi - lo)?;
604    let card = if c.len > 0 {
605        let stored = &span[(c.offset - lo) as usize..(c.offset - lo + c.len) as usize];
606        if digest16(stored) != h.card_digest {
607            return Err(FileError::Container("0x06 card digest mismatch"));
608        }
609        Some(decompress_bounded(h.internal_codec, stored, CARD_MAX)?)
610    } else {
611        None
612    };
613    let build = (b.len > 0)
614        .then(|| span[(b.offset - lo) as usize..(b.offset - lo + b.len) as usize].to_vec());
615    Ok((card, build))
616}
617
618/// Section `k`'s decompression bound in `root`: its recorded largest block,
619/// or [`BLOCK_CEILING`].
620pub fn section_bound(root: &Root06, k: u8) -> u64 {
621    root.section(k)
622        .map(|s| s.max_block as u64)
623        .filter(|&m| m > 0)
624        .unwrap_or(BLOCK_CEILING)
625}
626
627/// The stats + schema section as the digest-prefixed bytes its decoder takes.
628pub fn read_schema_bytes<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
629    let (h, root) = read_front(r)?;
630    let Some(stored) = read_section(r, &h, &root, kind::SCHEMA)? else {
631        return Ok(None);
632    };
633    let body = decompress_bounded(
634        h.internal_codec,
635        &stored,
636        section_bound(&root, kind::SCHEMA),
637    )?;
638    let mut out = digest16(&body).to_vec();
639    out.extend_from_slice(&body);
640    Ok(Some(out))
641}
642
643/// The community pyramid's bytes, if any.
644pub fn read_pyramid<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
645    let (h, root) = read_front(r)?;
646    read_section(r, &h, &root, kind::PYRAMID)
647}
648
649/// Stored bytes of the text index's token table (all leaves).
650pub fn read_token_table_bytes<R: RangeReader + ?Sized>(r: &R) -> Option<u64> {
651    let (_, root) = read_front(r).ok()?;
652    root.unit(tag::TEXT)?;
653    Some(root.section(kind::TEXT_LEAVES).map_or(0, |s| s.len))
654}
655
656/// A reader holding only some ranges of a file in memory (the borrowed-reader
657/// opens of a `0x06` file read what they need into one of these).
658pub struct SparseReader {
659    len: u64,
660    parts: Vec<(u64, Vec<u8>)>,
661}
662
663impl SparseReader {
664    /// Fetch `ranges` from `r` (plus the first bytes `first`) into memory —
665    /// the ones `first` does not already hold, in one batch.
666    pub fn fetch<R: RangeReader + ?Sized>(
667        r: &R,
668        first: Vec<u8>,
669        ranges: &[ByteRange],
670    ) -> Result<Self, FileError> {
671        let held = first.len() as u64;
672        let want: Vec<(u64, u64)> = ranges
673            .iter()
674            .filter(|g| g.len > 0 && g.offset.saturating_add(g.len) > held)
675            .map(|g| (g.offset, g.len))
676            .collect();
677        let mut parts = vec![(0u64, first)];
678        if !want.is_empty() {
679            for ((o, _), b) in want.iter().zip(r.read_many(&want)?) {
680                parts.push((*o, b));
681            }
682        }
683        Ok(SparseReader {
684            len: r.len(),
685            parts,
686        })
687    }
688}
689
690/// A borrowed reader whose first bytes are already in memory: reads inside
691/// them cost nothing, the rest go to `r`.
692pub struct Prefixed<'a, R: RangeReader + ?Sized> {
693    /// The file's first bytes.
694    pub first: &'a [u8],
695    /// The reader behind them.
696    pub r: &'a R,
697}
698
699impl<R: RangeReader + ?Sized> RangeReader for Prefixed<'_, R> {
700    fn len(&self) -> u64 {
701        self.r.len()
702    }
703
704    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
705        match offset.checked_add(len) {
706            Some(end) if end <= self.first.len() as u64 => {
707                Ok(self.first[offset as usize..end as usize].to_vec())
708            }
709            _ => self.r.read_at(offset, len),
710        }
711    }
712}
713
714impl RangeReader for SparseReader {
715    fn len(&self) -> u64 {
716        self.len
717    }
718
719    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
720        for (o, b) in &self.parts {
721            if offset >= *o && offset + len <= o + b.len() as u64 {
722                let s = (offset - o) as usize;
723                return Ok(b[s..s + len as usize].to_vec());
724            }
725        }
726        Err(std::io::Error::other("0x06: range not held by this reader"))
727    }
728}
729
730/// What the card's measured signals need from a `0x06` file's root: the text
731/// index's `(bytes, token-table bytes)` and the label index's `(bytes, entries)`.
732#[allow(clippy::type_complexity)]
733pub fn read_signals<R: RangeReader + ?Sized>(
734    r: &R,
735) -> Result<(Option<(u64, u64)>, Option<(u64, Option<u64>)>), FileError> {
736    let (_, root) = read_front(r)?;
737    let len = |k: u8| root.section(k).map_or(0, |s| s.len);
738    let text = root.unit(tag::TEXT).map(|_| {
739        (
740            len(kind::TEXT_LEAVES) + len(kind::TEXT_DATA),
741            len(kind::TEXT_LEAVES),
742        )
743    });
744    let labels = match root.unit(tag::LABELS) {
745        None => None,
746        Some(u) => {
747            let l = decode_label_root(u)?;
748            let entries = crate::label_index::LabelIndexPreamble::parse(&l.head)
749                .ok()
750                .map(|p| p.entry_count);
751            Some((
752                l.head.len() as u64 + len(kind::LABEL_DATA) + len(kind::LABEL_LEAVES),
753                entries,
754            ))
755        }
756    };
757    Ok((text, labels))
758}