Skip to main content

rete_core/v06/
open.rs

1//! Reading a `0x06` file: the header and the root come with the first read;
2//! everything else is built lazily from them — the dictionary and the index as
3//! two-level directories whose leaves fault on demand and stay for the session,
4//! the text index likewise, the label index and the quad index through their
5//! own lazy machinery.
6
7use std::sync::Arc;
8
9use super::*;
10use crate::dict::{ChunkLeaf, ChunkedSection, LeafedChunks, SectionMeta};
11use crate::dictionary::Dictionary;
12use crate::file::{decompress_bounded, ByteRange, FileError};
13use crate::index::{GraphIndex, LeafFetch, LeafedTiles, TileLeaf, NUM_PERMS};
14use crate::reader::RangeReader;
15
16/// Coalescing gaps for the batched faults (as for `0x05`).
17const TILE_GAP: u64 = 4096;
18const DICT_GAP: u64 = 64 * 1024;
19
20/// An open `0x06` file: its header and root, and the reader.
21pub struct V06File {
22    /// The 128-byte header.
23    pub h: Header06,
24    /// The decoded root block.
25    pub root: Root06,
26    /// Where everything else is read from.
27    pub reader: Arc<dyn RangeReader + Send + Sync>,
28}
29
30/// Read `(offset, len)` ranges as one coalesced batch.
31fn read_coalesced(
32    r: &(dyn RangeReader + Send + Sync),
33    ranges: &[(u64, u64)],
34    gap: u64,
35) -> Option<Vec<Vec<u8>>> {
36    let rs: Vec<ByteRange> = ranges
37        .iter()
38        .map(|&(offset, len)| ByteRange { offset, len })
39        .collect();
40    crate::file::read_coalesced(r, &rs, gap)
41}
42
43impl V06File {
44    /// Open from `first`, the file's leading bytes (at least the header; the
45    /// root is read from it when it fits, else fetched).
46    pub fn open(
47        reader: Arc<dyn RangeReader + Send + Sync>,
48        first: &[u8],
49    ) -> Result<Self, FileError> {
50        let h = Header06::from_bytes(first)?;
51        if h.file_len != reader.len() {
52            return Err(FileError::Container(
53                "0x06 file length does not match the header (truncated?)",
54            ));
55        }
56        let rs = HEADER_LEN;
57        let re = rs + h.root_len as usize;
58        let root_bytes = if re <= first.len() {
59            first[rs..re].to_vec()
60        } else {
61            reader.read_at(rs as u64, h.root_len as u64)?
62        };
63        if digest16(&root_bytes) != h.root_digest {
64            return Err(FileError::Container("0x06 root digest mismatch"));
65        }
66        let root = Root06::decode(&root_bytes, h.internal_codec)?;
67        for s in &root.sections {
68            let a = s.abs(&h)?;
69            let end = a.offset + a.len;
70            let ok = if kind::is_front(s.kind) {
71                a.offset >= h.front_base() && end <= h.data_offset
72            } else {
73                a.offset >= h.data_offset && end <= h.file_len - FOOTER.len() as u64
74            };
75            if !ok {
76                return Err(FileError::Container("0x06 section out of its region"));
77            }
78        }
79        Ok(V06File { h, root, reader })
80    }
81
82    /// The decompression bound of section `k`'s blocks: its recorded largest
83    /// block, or [`BLOCK_CEILING`] where none is recorded.
84    pub fn bound(&self, k: u8) -> u64 {
85        section_bound(&self.root, k)
86    }
87
88    /// The absolute range of section `k`, if present.
89    pub fn section(&self, k: u8) -> Option<ByteRange> {
90        self.root.section(k).and_then(|s| s.abs(&self.h).ok())
91    }
92
93    /// A batched leaf fetcher over this file's reader (leaves are stored with
94    /// the internal codec; decoding happens in the directory types).
95    pub fn fetch(&self) -> LeafFetch {
96        let r = self.reader.clone();
97        Arc::new(move |ranges: &[(u64, u64)]| read_coalesced(&*r, ranges, 0))
98    }
99
100    /// The Dataset Card (decompressed), if the file has one.
101    pub fn card(&self) -> Result<Option<Vec<u8>>, FileError> {
102        let r = self.h.card_range();
103        if r.len == 0 {
104            return Ok(None);
105        }
106        let stored = self.reader.read_at(r.offset, r.len)?;
107        if digest16(&stored) != self.h.card_digest {
108            return Err(FileError::Container("0x06 card digest mismatch"));
109        }
110        Ok(Some(decompress_bounded(
111            self.h.internal_codec,
112            &stored,
113            CARD_MAX,
114        )?))
115    }
116
117    /// The build record (raw), if the file has one.
118    pub fn build_record(&self) -> Result<Option<Vec<u8>>, FileError> {
119        let r = self.h.build_range();
120        if r.len == 0 {
121            return Ok(None);
122        }
123        Ok(Some(self.reader.read_at(r.offset, r.len)?))
124    }
125
126    /// The stats + schema section, as the `0x05` SCHEMA section bytes
127    /// (digest-prefixed) its decoder takes.
128    pub fn schema_section_bytes(&self) -> Option<Vec<u8>> {
129        let r = self.section(kind::SCHEMA)?;
130        let stored = self.reader.read_at(r.offset, r.len).ok()?;
131        let body =
132            decompress_bounded(self.h.internal_codec, &stored, self.bound(kind::SCHEMA)).ok()?;
133        let mut out = digest16(&body).to_vec();
134        out.extend_from_slice(&body);
135        Some(out)
136    }
137
138    /// The decoded stats + schema section.
139    pub fn schema(&self) -> Option<crate::schema_section::SchemaMeta> {
140        crate::schema_section::SchemaMeta::decode(&self.schema_section_bytes()?).ok()
141    }
142
143    /// The community pyramid's bytes.
144    pub fn pyramid_bytes(&self) -> Option<Vec<u8>> {
145        let r = self.section(kind::PYRAMID)?;
146        self.reader.read_at(r.offset, r.len).ok()
147    }
148
149    /// The four dictionary sections, two-level, chunks faulted on demand.
150    /// `want[si] == false` leaves that section empty (a search-only open).
151    pub fn dictionary(&self, want: [bool; 4]) -> Result<Dictionary, FileError> {
152        let unit = self
153            .root
154            .unit(tag::DICT)
155            .ok_or(FileError::Container("0x06 file has no dictionary"))?;
156        let roots = decode_dict_root(unit)?;
157        let leaves_base = self.section(kind::DICT_LEAVES).map_or(0, |r| r.offset);
158        let data_base = self.section(kind::DICT_DATA).map_or(0, |r| r.offset);
159        let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
160        let codec = self.h.dict_codec;
161        let max = self.bound(kind::DICT_DATA);
162        let mut dirs: Vec<Option<Arc<LeafedChunks>>> = Vec::with_capacity(4);
163        let mut sections = Vec::with_capacity(4);
164        for (si, s) in roots.iter().enumerate() {
165            let meta = SectionMeta {
166                term_count: if want[si] { s.term_count } else { 0 },
167                restart_interval: s.restart_interval.max(1),
168                restart_offsets: Vec::new(),
169            };
170            if !want[si] {
171                sections.push(ChunkedSection::from_parts(
172                    meta,
173                    Vec::new(),
174                    None,
175                    cache.clone(),
176                    si as u8,
177                ));
178                dirs.push(None);
179                continue;
180            }
181            let leaves: Vec<ChunkLeaf> = s
182                .leaves
183                .iter()
184                .map(|(l, run, coff, key)| ChunkLeaf {
185                    first: l.first,
186                    n: l.n,
187                    first_run: *run,
188                    key: key.clone(),
189                    leaf: (leaves_base + l.off, l.len),
190                    chunk_base: data_base + s.data_off + coff,
191                })
192                .collect();
193            let dir = Arc::new(
194                LeafedChunks::new(leaves, s.n_chunks, self.fetch(), self.h.internal_codec)
195                    .with_max_block(self.bound(kind::DICT_LEAVES)),
196            );
197            let (r, d) = (self.reader.clone(), dir.clone());
198            let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
199                let (off, len) = d.range(ci)?;
200                let b = r.read_at(off, len).ok()?;
201                decompress_bounded(codec, &b, max).ok()
202            });
203            let (r, d) = (self.reader.clone(), dir.clone());
204            let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
205                d.ensure_leaves(&d.leaves_of(cis));
206                let ranges: Option<Vec<(u64, u64)>> = cis.iter().map(|&ci| d.range(ci)).collect();
207                let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
208                blobs
209                    .iter()
210                    .map(|b| decompress_bounded(codec, b, max).ok())
211                    .collect()
212            });
213            sections.push(
214                ChunkedSection::leafed(meta, dir.clone(), loader, cache.clone(), si as u8)
215                    .with_bulk_loader(bulk),
216            );
217            dirs.push(Some(dir));
218        }
219        let arr: [ChunkedSection; 4] = sections
220            .try_into()
221            .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
222        let r = self.reader.clone();
223        let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
224            // Fault every section's leaves first (one batch), then the chunks.
225            let mut per: [Vec<usize>; 4] = Default::default();
226            for &(si, ci) in pairs {
227                if let Some(v) = per.get_mut(si as usize) {
228                    v.push(ci);
229                }
230            }
231            // One read for the leaves of every section: a batch that names
232            // terms in three sections used to cost three sequential reads here.
233            let wanted: Vec<(usize, Vec<usize>)> = per
234                .iter()
235                .enumerate()
236                .filter_map(|(si, cis)| Some((si, dirs.get(si)?.as_ref()?.leaves_of(cis))))
237                .collect();
238            let across: Vec<(&dyn crate::index::LeafDir, &[usize])> = wanted
239                .iter()
240                .filter_map(|(si, lis)| {
241                    let d: &dyn crate::index::LeafDir = dirs.get(*si)?.as_deref()?;
242                    Some((d, lis.as_slice()))
243                })
244                .collect();
245            crate::index::ensure_leaves_across(&across);
246            let ranges: Option<Vec<(u64, u64)>> = pairs
247                .iter()
248                .map(|&(si, ci)| dirs.get(si as usize)?.as_ref()?.range(ci))
249                .collect();
250            let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
251            blobs
252                .iter()
253                .map(|b| decompress_bounded(codec, b, max).ok())
254                .collect()
255        });
256        let mut dict = Dictionary::from_chunked_sections(arr).with_multi_bulk_loader(multi);
257        dict.set_has_quoted_triples(self.h.flags & F_QUOTED != 0);
258        Ok(dict)
259    }
260
261    /// The default graph's index, two-level, tiles faulted on demand.
262    pub fn index(&self, read_concurrency: usize) -> Result<GraphIndex, FileError> {
263        let unit = self
264            .root
265            .unit(tag::INDEX)
266            .ok_or(FileError::Container("0x06 file has no index"))?;
267        let perms_root = decode_index_root(unit)?;
268        let leaves_base = self.section(kind::INDEX_LEAVES).map_or(0, |r| r.offset);
269        let data_base = self.section(kind::INDEX_DATA).map_or(0, |r| r.offset);
270        let perms = self.h.perms();
271        let mut lists: [Option<Arc<LeafedTiles>>; NUM_PERMS] = Default::default();
272        for p in &perms_root {
273            if p.si >= NUM_PERMS {
274                return Err(FileError::Container("0x06 index: bad permutation"));
275            }
276            let leaves: Vec<TileLeaf> = p
277                .leaves
278                .iter()
279                .map(|(l, mn, mx, toff)| TileLeaf {
280                    first: l.first,
281                    n: l.n,
282                    min_a: *mn,
283                    max_a: *mx,
284                    leaf: (leaves_base + l.off, l.len),
285                    tile_base: data_base + p.data_off + toff,
286                })
287                .collect();
288            lists[p.si] = Some(Arc::new(
289                LeafedTiles::new(
290                    leaves,
291                    p.n_tiles,
292                    p.data_len,
293                    self.fetch(),
294                    self.h.internal_codec,
295                )
296                .with_max_block(self.bound(kind::INDEX_LEAVES)),
297            ));
298        }
299        for perm in perms.iter() {
300            if lists[perm.section_index()].is_none() {
301                return Err(FileError::Container(
302                    "0x06 index lacks a stored permutation",
303                ));
304            }
305        }
306        let codec = self.h.block_codec;
307        let max = self.bound(kind::INDEX_DATA);
308        let (r, l) = (self.reader.clone(), lists.clone());
309        let loader: crate::index::TileLoader = Box::new(move |si, ti| {
310            let (off, len) = l.get(si)?.as_ref()?.range(ti)?;
311            let b = r.read_at(off, len).ok()?;
312            decompress_bounded(codec, &b, max).ok()
313        });
314        let (r, l) = (self.reader.clone(), lists.clone());
315        let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
316            let list = l.get(si)?.as_ref()?;
317            if let (Some(&a), Some(&b)) = (tis.first(), tis.last()) {
318                list.ensure_span(a, b + 1);
319            }
320            let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
321            let blobs = read_coalesced(&*r, &ranges?, TILE_GAP)?;
322            blobs
323                .iter()
324                .map(|b| decompress_bounded(codec, b, max).ok())
325                .collect()
326        });
327        let mut index = GraphIndex::from_leafed(lists, perms, loader).with_bulk_loader(bulk);
328        index.set_read_concurrency(read_concurrency);
329        Ok(index)
330    }
331
332    /// The text index (leafed token table), if the file has one.
333    pub fn text_index(&self) -> Option<Result<crate::text_index::TextIndex, FileError>> {
334        let unit = self.root.unit(tag::TEXT)?;
335        Some((|| {
336            let t = decode_text_root(unit)?;
337            let leaves_base = self.section(kind::TEXT_LEAVES).map_or(0, |r| r.offset);
338            let data = self
339                .section(kind::TEXT_DATA)
340                .unwrap_or(ByteRange { offset: 0, len: 0 });
341            let leaves = t
342                .leaves
343                .iter()
344                .map(|(l, tok)| crate::text_index::TokenLeaf {
345                    first: l.first,
346                    n: l.n,
347                    first_token: String::from_utf8_lossy(tok).into_owned(),
348                    leaf: (leaves_base + l.off, l.len),
349                })
350                .collect();
351            let tokens = Arc::new(
352                crate::text_index::LeafedTokens::new(
353                    leaves,
354                    t.n_tokens,
355                    self.fetch(),
356                    self.h.internal_codec,
357                )
358                .with_max_block(self.bound(kind::TEXT_LEAVES)),
359            );
360            let r = self.reader.clone();
361            let loader = Box::new(move |off: u64, len: u64| {
362                if off.checked_add(len)? > data.len {
363                    return None;
364                }
365                r.read_at(data.offset + off, len).ok()
366            });
367            crate::text_index::TextIndex::leafed(tokens, &t.scope, loader)
368        })())
369    }
370
371    /// Stored bytes of the token table (all leaves) — what a substring search
372    /// would read; a word lookup reads one leaf.
373    pub fn token_table_bytes(&self) -> Option<u64> {
374        self.root.unit(tag::TEXT)?;
375        Some(self.section(kind::TEXT_LEAVES).map_or(0, |r| r.len))
376    }
377
378    /// The label index, if the file has one.
379    pub fn label_index(&self) -> Option<Result<crate::label_index::LabelIndex, FileError>> {
380        let unit = self.root.unit(tag::LABELS)?;
381        Some((|| {
382            let l = decode_label_root(unit)?;
383            let data = self
384                .section(kind::LABEL_DATA)
385                .unwrap_or(ByteRange { offset: 0, len: 0 });
386            let nodes = self
387                .section(kind::LABEL_LEAVES)
388                .unwrap_or(ByteRange { offset: 0, len: 0 });
389            if data.len != l.data_len {
390                return Err(FileError::Container("0x06 label data length mismatch"));
391            }
392            let head_len = l.head.len() as u64;
393            let section_len = head_len + data.len + nodes.len;
394            let r = self.reader.clone();
395            let head = l.head.clone();
396            let loader = Box::new(move |off: u64, len: u64| -> Option<Vec<u8>> {
397                let end = off.checked_add(len)?;
398                if end <= head_len {
399                    return head.get(off as usize..end as usize).map(<[u8]>::to_vec);
400                }
401                let bo = off.checked_sub(head_len)?;
402                if bo + len <= data.len {
403                    r.read_at(data.offset + bo, len).ok()
404                } else if bo >= data.len && bo - data.len + len <= nodes.len {
405                    r.read_at(nodes.offset + (bo - data.len), len).ok()
406                } else {
407                    None
408                }
409            });
410            let max = self
411                .bound(kind::LABEL_LEAVES)
412                .max(self.bound(kind::LABEL_DATA));
413            crate::label_index::LabelIndex::from_head(
414                &l.head,
415                section_len,
416                self.h.block_codec,
417                loader,
418            )
419            .map(|li| li.with_max_block(max))
420        })())
421    }
422
423    /// The label index's entry count, from the root alone.
424    pub fn label_entry_count(&self) -> Option<u64> {
425        let l = decode_label_root(self.root.unit(tag::LABELS)?).ok()?;
426        crate::label_index::LabelIndexPreamble::parse(&l.head)
427            .ok()
428            .map(|p| p.entry_count)
429    }
430
431    /// The quad index's root, if the file has named graphs.
432    pub fn quad_root(&self) -> Option<Result<QuadRoot, FileError>> {
433        self.root.unit(tag::QUAD).map(decode_quad_root)
434    }
435
436    /// The named graphs (quad index), if any.
437    pub(crate) fn quad(
438        &self,
439        read_concurrency: usize,
440    ) -> Option<Result<crate::quad::QuadNamed, FileError>> {
441        let root = match self.quad_root()? {
442            Ok(r) => r,
443            Err(e) => return Some(Err(e)),
444        };
445        let leaves = self.section(kind::QUAD_LEAVES).map_or(0, |r| r.offset);
446        let data = self.section(kind::QUAD_DATA).map_or(0, |r| r.offset);
447        Some(
448            crate::quad::V06Quad::new(&root, leaves, data, self.h.internal_codec).map(|mut v| {
449                v.tile_max = self.bound(kind::QUAD_DATA);
450                v.leaf_max = self.bound(kind::QUAD_LEAVES);
451                crate::quad::QuadNamed::new_v06(
452                    self.reader.clone(),
453                    v,
454                    self.h.block_codec,
455                    self.h.perms(),
456                    read_concurrency,
457                )
458            }),
459        )
460    }
461
462    /// Verify every digest: the root, the card, each section (streamed), and
463    /// the content hash. `only` restricts the section checks to one kind.
464    /// `Ok(false)` names nothing; [`verify_report`](Self::verify_report) does.
465    pub fn verify(&self, only: Option<u8>) -> Result<bool, FileError> {
466        Ok(self.verify_report(only)?.is_empty())
467    }
468
469    /// Like [`verify`](Self::verify), returning what failed (empty = all good).
470    pub fn verify_report(&self, only: Option<u8>) -> Result<Vec<String>, FileError> {
471        const SLICE: u64 = 8 << 20;
472        let mut bad = Vec::new();
473        if content_hash(&self.h) != self.h.content_hash {
474            bad.push("content hash".to_string());
475        }
476        let card = self.h.card_range();
477        if only.is_none() && card.len > 0 {
478            let stored = self.reader.read_at(card.offset, card.len)?;
479            if digest16(&stored) != self.h.card_digest {
480                bad.push("card".to_string());
481            }
482        }
483        for s in &self.root.sections {
484            if only.is_some_and(|k| k != s.kind) {
485                continue;
486            }
487            let a = s.abs(&self.h)?;
488            let mut h = blake3::Hasher::new();
489            let mut at = a.offset;
490            while at < a.offset + a.len {
491                let n = (a.offset + a.len - at).min(SLICE);
492                h.update(&self.reader.read_at(at, n)?);
493                at += n;
494            }
495            if h.finalize().as_bytes()[..16] != s.digest {
496                bad.push(kind::name(s.kind).to_string());
497            }
498        }
499        if only.is_none() {
500            let foot = self
501                .reader
502                .read_at(self.h.file_len - FOOTER.len() as u64, FOOTER.len() as u64)?;
503            if foot != FOOTER {
504                bad.push("footer".to_string());
505            }
506        }
507        Ok(bad)
508    }
509}
510
511/// Is this file (by its first bytes) a `0x06` file?
512pub fn is_v06(first: &[u8]) -> bool {
513    first.len() > 4 && first[0..4] == crate::header::MAGIC && first[4] == VERSION
514}
515
516/// Read a `0x06` file's header and root through any reader (two reads at most;
517/// over a seeded or prefix-holding reader, none).
518pub fn read_front<R: RangeReader + ?Sized>(r: &R) -> Result<(Header06, Root06), FileError> {
519    let head = r.read_at(0, HEADER_LEN as u64)?;
520    let h = Header06::from_bytes(&head)?;
521    let root_bytes = r.read_at(HEADER_LEN as u64, h.root_len as u64)?;
522    if digest16(&root_bytes) != h.root_digest {
523        return Err(FileError::Container("0x06 root digest mismatch"));
524    }
525    let root = Root06::decode(&root_bytes, h.internal_codec)?;
526    Ok((h, root))
527}
528
529/// The stored bytes of section `k`, if present.
530pub fn read_section<R: RangeReader + ?Sized>(
531    r: &R,
532    h: &Header06,
533    root: &Root06,
534    k: u8,
535) -> Result<Option<Vec<u8>>, FileError> {
536    match root.section(k) {
537        Some(s) => {
538            let a = s.abs(h)?;
539            Ok(Some(r.read_at(a.offset, a.len)?))
540        }
541        None => Ok(None),
542    }
543}
544
545/// The card (decompressed) and the build record (raw): one read of the card
546/// region, which sits right after the root.
547#[allow(clippy::type_complexity)]
548pub fn read_card_build<R: RangeReader + ?Sized>(
549    r: &R,
550    h: &Header06,
551) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
552    let c = h.card_range();
553    let b = h.build_range();
554    if c.len == 0 && b.len == 0 {
555        return Ok((None, None));
556    }
557    let lo = if c.len == 0 {
558        b.offset
559    } else if b.len == 0 {
560        c.offset
561    } else {
562        c.offset.min(b.offset)
563    };
564    let hi = (c.offset + c.len).max(b.offset + b.len);
565    let span = r.read_at(lo, hi - lo)?;
566    let card = if c.len > 0 {
567        let stored = &span[(c.offset - lo) as usize..(c.offset - lo + c.len) as usize];
568        if digest16(stored) != h.card_digest {
569            return Err(FileError::Container("0x06 card digest mismatch"));
570        }
571        Some(decompress_bounded(h.internal_codec, stored, CARD_MAX)?)
572    } else {
573        None
574    };
575    let build = (b.len > 0)
576        .then(|| span[(b.offset - lo) as usize..(b.offset - lo + b.len) as usize].to_vec());
577    Ok((card, build))
578}
579
580/// Section `k`'s decompression bound in `root`: its recorded largest block,
581/// or [`BLOCK_CEILING`].
582pub fn section_bound(root: &Root06, k: u8) -> u64 {
583    root.section(k)
584        .map(|s| s.max_block as u64)
585        .filter(|&m| m > 0)
586        .unwrap_or(BLOCK_CEILING)
587}
588
589/// The stats + schema section as the digest-prefixed bytes its decoder takes.
590pub fn read_schema_bytes<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
591    let (h, root) = read_front(r)?;
592    let Some(stored) = read_section(r, &h, &root, kind::SCHEMA)? else {
593        return Ok(None);
594    };
595    let body = decompress_bounded(
596        h.internal_codec,
597        &stored,
598        section_bound(&root, kind::SCHEMA),
599    )?;
600    let mut out = digest16(&body).to_vec();
601    out.extend_from_slice(&body);
602    Ok(Some(out))
603}
604
605/// The community pyramid's bytes, if any.
606pub fn read_pyramid<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
607    let (h, root) = read_front(r)?;
608    read_section(r, &h, &root, kind::PYRAMID)
609}
610
611/// Stored bytes of the text index's token table (all leaves).
612pub fn read_token_table_bytes<R: RangeReader + ?Sized>(r: &R) -> Option<u64> {
613    let (_, root) = read_front(r).ok()?;
614    root.unit(tag::TEXT)?;
615    Some(root.section(kind::TEXT_LEAVES).map_or(0, |s| s.len))
616}
617
618/// A reader holding only some ranges of a file in memory (the borrowed-reader
619/// opens of a `0x06` file read what they need into one of these).
620pub struct SparseReader {
621    len: u64,
622    parts: Vec<(u64, Vec<u8>)>,
623}
624
625impl SparseReader {
626    /// Fetch `ranges` from `r` (plus the first bytes `first`) into memory —
627    /// the ones `first` does not already hold, in one batch.
628    pub fn fetch<R: RangeReader + ?Sized>(
629        r: &R,
630        first: Vec<u8>,
631        ranges: &[ByteRange],
632    ) -> Result<Self, FileError> {
633        let held = first.len() as u64;
634        let want: Vec<(u64, u64)> = ranges
635            .iter()
636            .filter(|g| g.len > 0 && g.offset.saturating_add(g.len) > held)
637            .map(|g| (g.offset, g.len))
638            .collect();
639        let mut parts = vec![(0u64, first)];
640        if !want.is_empty() {
641            for ((o, _), b) in want.iter().zip(r.read_many(&want)?) {
642                parts.push((*o, b));
643            }
644        }
645        Ok(SparseReader {
646            len: r.len(),
647            parts,
648        })
649    }
650}
651
652/// A borrowed reader whose first bytes are already in memory: reads inside
653/// them cost nothing, the rest go to `r`.
654pub struct Prefixed<'a, R: RangeReader + ?Sized> {
655    /// The file's first bytes.
656    pub first: &'a [u8],
657    /// The reader behind them.
658    pub r: &'a R,
659}
660
661impl<R: RangeReader + ?Sized> RangeReader for Prefixed<'_, R> {
662    fn len(&self) -> u64 {
663        self.r.len()
664    }
665
666    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
667        match offset.checked_add(len) {
668            Some(end) if end <= self.first.len() as u64 => {
669                Ok(self.first[offset as usize..end as usize].to_vec())
670            }
671            _ => self.r.read_at(offset, len),
672        }
673    }
674}
675
676impl RangeReader for SparseReader {
677    fn len(&self) -> u64 {
678        self.len
679    }
680
681    fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
682        for (o, b) in &self.parts {
683            if offset >= *o && offset + len <= o + b.len() as u64 {
684                let s = (offset - o) as usize;
685                return Ok(b[s..s + len as usize].to_vec());
686            }
687        }
688        Err(std::io::Error::other("0x06: range not held by this reader"))
689    }
690}
691
692/// What the card's measured signals need from a `0x06` file's root: the text
693/// index's `(bytes, token-table bytes)` and the label index's `(bytes, entries)`.
694#[allow(clippy::type_complexity)]
695pub fn read_signals<R: RangeReader + ?Sized>(
696    r: &R,
697) -> Result<(Option<(u64, u64)>, Option<(u64, Option<u64>)>), FileError> {
698    let (_, root) = read_front(r)?;
699    let len = |k: u8| root.section(k).map_or(0, |s| s.len);
700    let text = root.unit(tag::TEXT).map(|_| {
701        (
702            len(kind::TEXT_LEAVES) + len(kind::TEXT_DATA),
703            len(kind::TEXT_LEAVES),
704        )
705    });
706    let labels = match root.unit(tag::LABELS) {
707        None => None,
708        Some(u) => {
709            let l = decode_label_root(u)?;
710            let entries = crate::label_index::LabelIndexPreamble::parse(&l.head)
711                .ok()
712                .map(|p| p.entry_count);
713            Some((
714                l.head.len() as u64 + len(kind::LABEL_DATA) + len(kind::LABEL_LEAVES),
715                entries,
716            ))
717        }
718    };
719    Ok((text, labels))
720}