1use std::sync::Arc;
8
9use super::*;
10use crate::dict::{ChunkLeaf, ChunkedSection, LeafedChunks, SectionMeta};
11use crate::dictionary::Dictionary;
12use crate::file::{decompress_bounded, ByteRange, FileError};
13use crate::index::{GraphIndex, LeafFetch, LeafedTiles, TileLeaf, NUM_PERMS};
14use crate::reader::RangeReader;
15
16const TILE_GAP: u64 = 4096;
18const DICT_GAP: u64 = 64 * 1024;
19
20pub struct V06File {
22 pub h: Header06,
24 pub root: Root06,
26 pub reader: Arc<dyn RangeReader + Send + Sync>,
28}
29
30fn read_coalesced(
32 r: &(dyn RangeReader + Send + Sync),
33 ranges: &[(u64, u64)],
34 gap: u64,
35) -> Option<Vec<Vec<u8>>> {
36 let rs: Vec<ByteRange> = ranges
37 .iter()
38 .map(|&(offset, len)| ByteRange { offset, len })
39 .collect();
40 crate::file::read_coalesced(r, &rs, gap)
41}
42
43fn read_scattered(
45 r: &(dyn RangeReader + Send + Sync),
46 ranges: &[(u64, u64)],
47) -> Option<Vec<Vec<u8>>> {
48 let rs: Vec<ByteRange> = ranges
49 .iter()
50 .map(|&(offset, len)| ByteRange { offset, len })
51 .collect();
52 crate::file::read_coalesced_scattered(r, &rs, 0)
53}
54
55impl V06File {
56 pub fn open(
59 reader: Arc<dyn RangeReader + Send + Sync>,
60 first: &[u8],
61 ) -> Result<Self, FileError> {
62 let h = Header06::from_bytes(first)?;
63 if h.file_len != reader.len() {
64 return Err(FileError::Container(
65 "0x06 file length does not match the header (truncated?)",
66 ));
67 }
68 let rs = HEADER_LEN;
69 let re = rs + h.root_len as usize;
70 let root_bytes = if re <= first.len() {
71 first[rs..re].to_vec()
72 } else {
73 reader.read_at(rs as u64, h.root_len as u64)?
74 };
75 if digest16(&root_bytes) != h.root_digest {
76 return Err(FileError::Container("0x06 root digest mismatch"));
77 }
78 let root = Root06::decode(&root_bytes, h.internal_codec)?;
79 for s in &root.sections {
80 if h.version == VERSION_06 && kind::is_v07_only(s.kind) {
81 return Err(FileError::Container(
82 "0x06 file has a section only 0x07 defines",
83 ));
84 }
85 let a = s.abs(&h)?;
86 let end = a.offset + a.len;
87 let ok = if kind::is_front(s.kind) {
88 a.offset >= h.front_base() && end <= h.data_offset
89 } else {
90 a.offset >= h.data_offset && end <= h.file_len - FOOTER.len() as u64
91 };
92 if !ok {
93 return Err(FileError::Container("0x06 section out of its region"));
94 }
95 }
96 Ok(V06File { h, root, reader })
97 }
98
99 pub fn bound(&self, k: u8) -> u64 {
102 section_bound(&self.root, k)
103 }
104
105 pub fn section(&self, k: u8) -> Option<ByteRange> {
107 self.root.section(k).and_then(|s| s.abs(&self.h).ok())
108 }
109
110 pub fn fetch(&self) -> LeafFetch {
113 let r = self.reader.clone();
114 Arc::new(move |ranges: &[(u64, u64)]| read_coalesced(&*r, ranges, 0))
115 }
116
117 pub fn card(&self) -> Result<Option<Vec<u8>>, FileError> {
119 let r = self.h.card_range();
120 if r.len == 0 {
121 return Ok(None);
122 }
123 let stored = self.reader.read_at(r.offset, r.len)?;
124 if digest16(&stored) != self.h.card_digest {
125 return Err(FileError::Container("0x06 card digest mismatch"));
126 }
127 Ok(Some(decompress_bounded(
128 self.h.internal_codec,
129 &stored,
130 CARD_MAX,
131 )?))
132 }
133
134 pub fn build_record(&self) -> Result<Option<Vec<u8>>, FileError> {
136 let r = self.h.build_range();
137 if r.len == 0 {
138 return Ok(None);
139 }
140 Ok(Some(self.reader.read_at(r.offset, r.len)?))
141 }
142
143 pub fn label_root_node(&self) -> Result<Vec<u8>, FileError> {
146 let r = self
147 .section(kind::LABEL_ROOT)
148 .ok_or(FileError::Container("0x07 file has no label root section"))?;
149 let b = self.reader.read_at(r.offset, r.len)?;
150 check_digest(&self.root, kind::LABEL_ROOT, &b)?;
151 Ok(b)
152 }
153
154 pub fn annex(&self) -> Result<Option<Vec<u8>>, FileError> {
156 read_annex(&*self.reader, &self.h, &self.root)
157 }
158
159 pub fn card_full(&self) -> Result<Option<Vec<u8>>, FileError> {
161 let Some(hot) = self.card()? else {
162 return Ok(None);
163 };
164 match self.annex()? {
165 None => Ok(Some(hot)),
166 Some(a) => annex::merge(&hot, &a)
167 .map(Some)
168 .map_err(|_| FileError::Container("0x07 card annex does not merge")),
169 }
170 }
171
172 pub fn schema_section_bytes(&self) -> Option<Vec<u8>> {
175 let r = self.section(kind::SCHEMA)?;
176 let stored = self.reader.read_at(r.offset, r.len).ok()?;
177 let body =
178 decompress_bounded(self.h.internal_codec, &stored, self.bound(kind::SCHEMA)).ok()?;
179 let mut out = digest16(&body).to_vec();
180 out.extend_from_slice(&body);
181 Some(out)
182 }
183
184 pub fn schema(&self) -> Option<crate::schema_section::SchemaMeta> {
186 crate::schema_section::SchemaMeta::decode(&self.schema_section_bytes()?).ok()
187 }
188
189 pub fn pyramid_bytes(&self) -> Option<Vec<u8>> {
191 let r = self.section(kind::PYRAMID)?;
192 self.reader.read_at(r.offset, r.len).ok()
193 }
194
195 pub fn dictionary(&self, want: [bool; 4]) -> Result<Dictionary, FileError> {
198 let unit = self
199 .root
200 .unit(tag::DICT)
201 .ok_or(FileError::Container("0x06 file has no dictionary"))?;
202 let roots = decode_dict_root(unit)?;
203 let leaves_base = self.section(kind::DICT_LEAVES).map_or(0, |r| r.offset);
204 let data_base = self.section(kind::DICT_DATA).map_or(0, |r| r.offset);
205 let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
206 let codec = self.h.dict_codec;
207 let max = self.bound(kind::DICT_DATA);
208 let mut dirs: Vec<Option<Arc<LeafedChunks>>> = Vec::with_capacity(4);
209 let mut sections = Vec::with_capacity(4);
210 for (si, s) in roots.iter().enumerate() {
211 let meta = SectionMeta {
212 term_count: if want[si] { s.term_count } else { 0 },
213 restart_interval: s.restart_interval.max(1),
214 restart_offsets: Vec::new(),
215 };
216 if !want[si] {
217 sections.push(ChunkedSection::from_parts(
218 meta,
219 Vec::new(),
220 None,
221 cache.clone(),
222 si as u8,
223 ));
224 dirs.push(None);
225 continue;
226 }
227 let leaves: Vec<ChunkLeaf> = s
228 .leaves
229 .iter()
230 .map(|(l, run, coff, key)| ChunkLeaf {
231 first: l.first,
232 n: l.n,
233 first_run: *run,
234 key: key.clone(),
235 leaf: (leaves_base + l.off, l.len),
236 chunk_base: data_base + s.data_off + coff,
237 })
238 .collect();
239 let dir = Arc::new(
240 LeafedChunks::new(leaves, s.n_chunks, self.fetch(), self.h.internal_codec)
241 .with_max_block(self.bound(kind::DICT_LEAVES)),
242 );
243 let (r, d) = (self.reader.clone(), dir.clone());
244 let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
245 let (off, len) = d.range(ci)?;
246 let b = r.read_at(off, len).ok()?;
247 decompress_bounded(codec, &b, max).ok()
248 });
249 let (r, d) = (self.reader.clone(), dir.clone());
250 let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
251 d.ensure_leaves(&d.leaves_of(cis));
252 let ranges: Option<Vec<(u64, u64)>> = cis.iter().map(|&ci| d.range(ci)).collect();
253 let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
254 blobs
255 .iter()
256 .map(|b| decompress_bounded(codec, b, max).ok())
257 .collect()
258 });
259 sections.push(
260 ChunkedSection::leafed(meta, dir.clone(), loader, cache.clone(), si as u8)
261 .with_bulk_loader(bulk),
262 );
263 dirs.push(Some(dir));
264 }
265 let arr: [ChunkedSection; 4] = sections
266 .try_into()
267 .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
268 let r = self.reader.clone();
269 let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
270 let mut per: [Vec<usize>; 4] = Default::default();
272 for &(si, ci) in pairs {
273 if let Some(v) = per.get_mut(si as usize) {
274 v.push(ci);
275 }
276 }
277 let wanted: Vec<(usize, Vec<usize>)> = per
280 .iter()
281 .enumerate()
282 .filter_map(|(si, cis)| Some((si, dirs.get(si)?.as_ref()?.leaves_of(cis))))
283 .collect();
284 let across: Vec<(&dyn crate::index::LeafDir, &[usize])> = wanted
285 .iter()
286 .filter_map(|(si, lis)| {
287 let d: &dyn crate::index::LeafDir = dirs.get(*si)?.as_deref()?;
288 Some((d, lis.as_slice()))
289 })
290 .collect();
291 crate::index::ensure_leaves_across(&across);
292 let ranges: Option<Vec<(u64, u64)>> = pairs
293 .iter()
294 .map(|&(si, ci)| dirs.get(si as usize)?.as_ref()?.range(ci))
295 .collect();
296 let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
297 blobs
298 .iter()
299 .map(|b| decompress_bounded(codec, b, max).ok())
300 .collect()
301 });
302 let mut dict = Dictionary::from_chunked_sections(arr).with_multi_bulk_loader(multi);
303 dict.set_has_quoted_triples(self.h.flags & F_QUOTED != 0);
304 Ok(dict)
305 }
306
307 fn clusters(&self, perms: &[PermRoot]) -> Result<Vec<ClusterOrder>, FileError> {
310 let Some(unit) = self.root.unit(tag::CLUSTER) else {
311 if self.h.flags & F_CLUSTERED != 0 {
312 return Err(FileError::Container(
313 "0x07 clustered file without a cluster root",
314 ));
315 }
316 return Ok(Vec::new());
317 };
318 let orders = decode_cluster_root(unit)?;
319 let quad = self.quad_root().transpose()?;
320 for c in &orders {
321 let (n_tiles, leaves): (usize, Vec<(usize, usize)>) = if c.si == QUAD_SPOG {
323 let o = quad
324 .as_ref()
325 .and_then(|q| q.orders.iter().find(|o| o.si == c.si))
326 .ok_or(FileError::Container(
327 "0x07 cluster root names an order the quad index lacks",
328 ))?;
329 (
330 o.n_tiles,
331 o.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
332 )
333 } else {
334 let p = perms
335 .iter()
336 .find(|p| p.si == c.si)
337 .ok_or(FileError::Container(
338 "0x07 cluster root names a permutation the index lacks",
339 ))?;
340 (
341 p.n_tiles,
342 p.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
343 )
344 };
345 if c.run_tiles.iter().sum::<usize>() != n_tiles {
346 return Err(FileError::Container(
347 "0x07 cluster runs do not cover the tiles",
348 ));
349 }
350 let mut ends = Vec::with_capacity(c.run_tiles.len());
352 let mut at = 0usize;
353 for &n in &c.run_tiles {
354 at += n;
355 ends.push(at);
356 }
357 for (first, n) in leaves {
358 let r = ends.partition_point(|&e| e <= first);
359 if first + n > ends[r.min(ends.len() - 1)] {
360 return Err(FileError::Container(
361 "0x07 a directory leaf spans two cluster runs",
362 ));
363 }
364 }
365 }
366 Ok(orders)
367 }
368
369 pub fn index(&self, read_concurrency: usize) -> Result<GraphIndex, FileError> {
371 let unit = self
372 .root
373 .unit(tag::INDEX)
374 .ok_or(FileError::Container("0x06 file has no index"))?;
375 let perms_root = decode_index_root(unit)?;
376 let leaves_base = self.section(kind::INDEX_LEAVES).map_or(0, |r| r.offset);
377 let data_base = self.section(kind::INDEX_DATA).map_or(0, |r| r.offset);
378 let perms = self.h.perms();
379 let mut lists: [Option<Arc<LeafedTiles>>; NUM_PERMS] = Default::default();
380 for p in &perms_root {
381 if p.si >= NUM_PERMS {
382 return Err(FileError::Container("0x06 index: bad permutation"));
383 }
384 let leaves: Vec<TileLeaf> = p
385 .leaves
386 .iter()
387 .map(|(l, mn, mx, toff)| TileLeaf {
388 first: l.first,
389 n: l.n,
390 min_a: *mn,
391 max_a: *mx,
392 leaf: (leaves_base + l.off, l.len),
393 tile_base: data_base + p.data_off + toff,
394 })
395 .collect();
396 lists[p.si] = Some(Arc::new(
397 LeafedTiles::new(
398 leaves,
399 p.n_tiles,
400 p.data_len,
401 self.fetch(),
402 self.h.internal_codec,
403 )
404 .with_max_block(self.bound(kind::INDEX_LEAVES)),
405 ));
406 }
407 for perm in perms.iter() {
408 if lists[perm.section_index()].is_none() {
409 return Err(FileError::Container(
410 "0x06 index lacks a stored permutation",
411 ));
412 }
413 }
414 let codec = self.h.block_codec;
415 let max = self.bound(kind::INDEX_DATA);
416 let (r, l) = (self.reader.clone(), lists.clone());
417 let loader: crate::index::TileLoader = Box::new(move |si, ti| {
418 let (off, len) = l.get(si)?.as_ref()?.range(ti)?;
419 let b = r.read_at(off, len).ok()?;
420 decompress_bounded(codec, &b, max).ok()
421 });
422 let (r, l) = (self.reader.clone(), lists.clone());
423 let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
424 let list = l.get(si)?.as_ref()?;
425 if let (Some(&a), Some(&b)) = (tis.first(), tis.last()) {
426 list.ensure_span(a, b + 1);
427 }
428 let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
429 let blobs = read_coalesced(&*r, &ranges?, TILE_GAP)?;
430 blobs
431 .iter()
432 .map(|b| decompress_bounded(codec, b, max).ok())
433 .collect()
434 });
435 let (r, l) = (self.reader.clone(), lists.clone());
436 let scattered: crate::index::TileBulkLoader = Box::new(move |si, tis| {
437 let list = l.get(si)?.as_ref()?;
438 let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
439 let blobs = read_scattered(&*r, &ranges?)?;
440 blobs
441 .iter()
442 .map(|b| decompress_bounded(codec, b, max).ok())
443 .collect()
444 });
445 let mut index = GraphIndex::from_leafed(lists, perms, loader)
446 .with_bulk_loader(bulk)
447 .with_scattered_loader(scattered);
448 for c in self
449 .clusters(&perms_root)?
450 .into_iter()
451 .filter(|c| c.si < NUM_PERMS)
452 {
453 index = index.with_clusters(
454 c.si,
455 crate::index::ClusterRuns::new(c.classes, &c.run_tiles),
456 );
457 }
458 index.set_read_concurrency(read_concurrency);
459 Ok(index)
460 }
461
462 pub fn text_index(&self) -> Option<Result<crate::text_index::TextIndex, FileError>> {
464 let unit = self.root.unit(tag::TEXT)?;
465 Some((|| {
466 let t = decode_text_root(unit)?;
467 let leaves_base = self.section(kind::TEXT_LEAVES).map_or(0, |r| r.offset);
468 let data = self
469 .section(kind::TEXT_DATA)
470 .unwrap_or(ByteRange { offset: 0, len: 0 });
471 let leaves = t
472 .leaves
473 .iter()
474 .map(|(l, tok)| crate::text_index::TokenLeaf {
475 first: l.first,
476 n: l.n,
477 first_token: String::from_utf8_lossy(tok).into_owned(),
478 leaf: (leaves_base + l.off, l.len),
479 })
480 .collect();
481 let tokens = Arc::new(
482 crate::text_index::LeafedTokens::new(
483 leaves,
484 t.n_tokens,
485 self.fetch(),
486 self.h.internal_codec,
487 )
488 .with_max_block(self.bound(kind::TEXT_LEAVES)),
489 );
490 let r = self.reader.clone();
491 let loader = Box::new(move |off: u64, len: u64| {
492 if off.checked_add(len)? > data.len {
493 return None;
494 }
495 r.read_at(data.offset + off, len).ok()
496 });
497 crate::text_index::TextIndex::leafed(tokens, &t.scope, loader)
498 })())
499 }
500
501 pub fn token_table_bytes(&self) -> Option<u64> {
504 self.root.unit(tag::TEXT)?;
505 Some(self.section(kind::TEXT_LEAVES).map_or(0, |r| r.len))
506 }
507
508 pub fn label_index(&self) -> Option<Result<crate::label_index::LabelIndex, FileError>> {
510 let unit = self.root.unit(tag::LABELS)?;
511 Some((|| {
512 let mut l = decode_label_root(unit)?;
513 if self.section(kind::LABEL_ROOT).is_some() {
514 l.head.extend_from_slice(&self.label_root_node()?);
515 }
516 let data = self
517 .section(kind::LABEL_DATA)
518 .unwrap_or(ByteRange { offset: 0, len: 0 });
519 let nodes = self
520 .section(kind::LABEL_LEAVES)
521 .unwrap_or(ByteRange { offset: 0, len: 0 });
522 if data.len != l.data_len {
523 return Err(FileError::Container("0x06 label data length mismatch"));
524 }
525 let head_len = l.head.len() as u64;
526 let section_len = head_len + data.len + nodes.len;
527 let r = self.reader.clone();
528 let head = l.head.clone();
529 let loader = Box::new(move |off: u64, len: u64| -> Option<Vec<u8>> {
530 let end = off.checked_add(len)?;
531 if end <= head_len {
532 return head.get(off as usize..end as usize).map(<[u8]>::to_vec);
533 }
534 let bo = off.checked_sub(head_len)?;
535 if bo + len <= data.len {
536 r.read_at(data.offset + bo, len).ok()
537 } else if bo >= data.len && bo - data.len + len <= nodes.len {
538 r.read_at(nodes.offset + (bo - data.len), len).ok()
539 } else {
540 None
541 }
542 });
543 let max = self
544 .bound(kind::LABEL_LEAVES)
545 .max(self.bound(kind::LABEL_DATA));
546 crate::label_index::LabelIndex::from_head(
547 &l.head,
548 section_len,
549 self.h.block_codec,
550 loader,
551 )
552 .map(|li| li.with_max_block(max))
553 })())
554 }
555
556 pub fn label_entry_count(&self) -> Option<u64> {
558 let l = decode_label_root(self.root.unit(tag::LABELS)?).ok()?;
559 crate::label_index::LabelIndexPreamble::parse(&l.head)
560 .ok()
561 .map(|p| p.entry_count)
562 }
563
564 pub fn quad_root(&self) -> Option<Result<QuadRoot, FileError>> {
566 self.root.unit(tag::QUAD).map(decode_quad_root)
567 }
568
569 pub(crate) fn quad(
571 &self,
572 read_concurrency: usize,
573 ) -> Option<Result<crate::quad::QuadNamed, FileError>> {
574 let root = match self.quad_root()? {
575 Ok(r) => r,
576 Err(e) => return Some(Err(e)),
577 };
578 let leaves = self.section(kind::QUAD_LEAVES).map_or(0, |r| r.offset);
579 let data = self.section(kind::QUAD_DATA).map_or(0, |r| r.offset);
580 let clusters = match self.root.unit(tag::CLUSTER).map(decode_cluster_root) {
582 Some(Ok(c)) => c,
583 Some(Err(e)) => return Some(Err(e)),
584 None => Vec::new(),
585 };
586 Some(
587 crate::quad::V06Quad::new(
588 &root,
589 leaves,
590 data,
591 self.h.internal_codec,
592 self.h.version == VERSION,
593 )
594 .and_then(|v| v.with_clusters(&clusters))
595 .map(|mut v| {
596 v.tile_max = self.bound(kind::QUAD_DATA);
597 v.leaf_max = self.bound(kind::QUAD_LEAVES);
598 crate::quad::QuadNamed::new_v06(
599 self.reader.clone(),
600 v,
601 self.h.block_codec,
602 self.h.perms(),
603 read_concurrency,
604 )
605 }),
606 )
607 }
608
609 pub fn verify(&self, only: Option<u8>) -> Result<bool, FileError> {
613 Ok(self.verify_report(only)?.is_empty())
614 }
615
616 pub fn verify_report(&self, only: Option<u8>) -> Result<Vec<String>, FileError> {
618 const SLICE: u64 = 8 << 20;
619 let mut bad = Vec::new();
620 if content_hash(&self.h) != self.h.content_hash {
621 bad.push("content hash".to_string());
622 }
623 let card = self.h.card_range();
624 if only.is_none() && card.len > 0 {
625 let stored = self.reader.read_at(card.offset, card.len)?;
626 if digest16(&stored) != self.h.card_digest {
627 bad.push("card".to_string());
628 }
629 }
630 for s in &self.root.sections {
631 if only.is_some_and(|k| k != s.kind) {
632 continue;
633 }
634 let a = s.abs(&self.h)?;
635 let mut h = blake3::Hasher::new();
636 let mut at = a.offset;
637 while at < a.offset + a.len {
638 let n = (a.offset + a.len - at).min(SLICE);
639 h.update(&self.reader.read_at(at, n)?);
640 at += n;
641 }
642 if h.finalize().as_bytes()[..16] != s.digest {
643 bad.push(kind::name(s.kind).to_string());
644 }
645 }
646 if only.is_none() {
647 let want = footer(self.h.version);
648 let foot = self
649 .reader
650 .read_at(self.h.file_len - want.len() as u64, want.len() as u64)?;
651 if foot != want {
652 bad.push("footer".to_string());
653 }
654 }
655 Ok(bad)
656 }
657}
658
659pub fn is_v06(first: &[u8]) -> bool {
661 first.len() > 4 && first[0..4] == crate::header::MAGIC && reads(first[4])
662}
663
664pub fn read_front<R: RangeReader + ?Sized>(r: &R) -> Result<(Header06, Root06), FileError> {
667 let head = r.read_at(0, HEADER_LEN as u64)?;
668 let h = Header06::from_bytes(&head)?;
669 let root_bytes = r.read_at(HEADER_LEN as u64, h.root_len as u64)?;
670 if digest16(&root_bytes) != h.root_digest {
671 return Err(FileError::Container("0x06 root digest mismatch"));
672 }
673 let root = Root06::decode(&root_bytes, h.internal_codec)?;
674 Ok((h, root))
675}
676
677pub fn read_section<R: RangeReader + ?Sized>(
679 r: &R,
680 h: &Header06,
681 root: &Root06,
682 k: u8,
683) -> Result<Option<Vec<u8>>, FileError> {
684 match root.section(k) {
685 Some(s) => {
686 let a = s.abs(h)?;
687 Ok(Some(r.read_at(a.offset, a.len)?))
688 }
689 None => Ok(None),
690 }
691}
692
693#[allow(clippy::type_complexity)]
696pub fn read_card_build<R: RangeReader + ?Sized>(
697 r: &R,
698 h: &Header06,
699) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
700 read_card_build_with(r, h, true)
701}
702
703#[allow(clippy::type_complexity)]
706pub fn read_card_build_with<R: RangeReader + ?Sized>(
707 r: &R,
708 h: &Header06,
709 want_build: bool,
710) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
711 let c = h.card_range();
712 let mut b = h.build_range();
713 if !want_build {
714 b.len = 0;
715 }
716 if c.len == 0 && b.len == 0 {
717 return Ok((None, None));
718 }
719 let lo = if c.len == 0 {
720 b.offset
721 } else if b.len == 0 {
722 c.offset
723 } else {
724 c.offset.min(b.offset)
725 };
726 let hi = (c.offset + c.len).max(b.offset + b.len);
727 let span = r.read_at(lo, hi - lo)?;
728 let card = if c.len > 0 {
729 let stored = &span[(c.offset - lo) as usize..(c.offset - lo + c.len) as usize];
730 if digest16(stored) != h.card_digest {
731 return Err(FileError::Container("0x06 card digest mismatch"));
732 }
733 Some(decompress_bounded(h.internal_codec, stored, CARD_MAX)?)
734 } else {
735 None
736 };
737 let build = (b.len > 0)
738 .then(|| span[(b.offset - lo) as usize..(b.offset - lo + b.len) as usize].to_vec());
739 Ok((card, build))
740}
741
742pub fn section_bound(root: &Root06, k: u8) -> u64 {
745 root.section(k)
746 .map(|s| s.max_block as u64)
747 .filter(|&m| m > 0)
748 .unwrap_or(BLOCK_CEILING)
749}
750
751pub fn read_schema_bytes<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
753 let (h, root) = read_front(r)?;
754 let Some(stored) = read_section(r, &h, &root, kind::SCHEMA)? else {
755 return Ok(None);
756 };
757 let body = decompress_bounded(
758 h.internal_codec,
759 &stored,
760 section_bound(&root, kind::SCHEMA),
761 )?;
762 let mut out = digest16(&body).to_vec();
763 out.extend_from_slice(&body);
764 Ok(Some(out))
765}
766
767pub fn read_pyramid<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
769 let (h, root) = read_front(r)?;
770 read_section(r, &h, &root, kind::PYRAMID)
771}
772
773pub fn read_token_table_bytes<R: RangeReader + ?Sized>(r: &R) -> Option<u64> {
775 let (_, root) = read_front(r).ok()?;
776 root.unit(tag::TEXT)?;
777 Some(root.section(kind::TEXT_LEAVES).map_or(0, |s| s.len))
778}
779
780pub struct SparseReader {
783 len: u64,
784 parts: Vec<(u64, Vec<u8>)>,
785}
786
787impl SparseReader {
788 pub fn fetch<R: RangeReader + ?Sized>(
791 r: &R,
792 first: Vec<u8>,
793 ranges: &[ByteRange],
794 ) -> Result<Self, FileError> {
795 let held = first.len() as u64;
796 let want: Vec<(u64, u64)> = ranges
797 .iter()
798 .filter(|g| g.len > 0 && g.offset.saturating_add(g.len) > held)
799 .map(|g| (g.offset, g.len))
800 .collect();
801 let mut parts = vec![(0u64, first)];
802 if !want.is_empty() {
803 for ((o, _), b) in want.iter().zip(r.read_many(&want)?) {
804 parts.push((*o, b));
805 }
806 }
807 Ok(SparseReader {
808 len: r.len(),
809 parts,
810 })
811 }
812}
813
814pub struct Prefixed<'a, R: RangeReader + ?Sized> {
817 pub first: &'a [u8],
819 pub r: &'a R,
821}
822
823impl<R: RangeReader + ?Sized> RangeReader for Prefixed<'_, R> {
824 fn len(&self) -> u64 {
825 self.r.len()
826 }
827
828 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
829 match offset.checked_add(len) {
830 Some(end) if end <= self.first.len() as u64 => {
831 Ok(self.first[offset as usize..end as usize].to_vec())
832 }
833 _ => self.r.read_at(offset, len),
834 }
835 }
836}
837
838impl RangeReader for SparseReader {
839 fn len(&self) -> u64 {
840 self.len
841 }
842
843 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
844 for (o, b) in &self.parts {
845 if offset >= *o && offset + len <= o + b.len() as u64 {
846 let s = (offset - o) as usize;
847 return Ok(b[s..s + len as usize].to_vec());
848 }
849 }
850 Err(std::io::Error::other("0x06: range not held by this reader"))
851 }
852}
853
854#[allow(clippy::type_complexity)]
857pub fn read_signals<R: RangeReader + ?Sized>(
858 r: &R,
859) -> Result<(Option<(u64, u64)>, Option<(u64, Option<u64>)>), FileError> {
860 let (_, root) = read_front(r)?;
861 let len = |k: u8| root.section(k).map_or(0, |s| s.len);
862 let text = root.unit(tag::TEXT).map(|_| {
863 (
864 len(kind::TEXT_LEAVES) + len(kind::TEXT_DATA),
865 len(kind::TEXT_LEAVES),
866 )
867 });
868 let labels = match root.unit(tag::LABELS) {
869 None => None,
870 Some(u) => {
871 let l = decode_label_root(u)?;
872 let entries = crate::label_index::LabelIndexPreamble::parse(&l.head)
873 .ok()
874 .map(|p| p.entry_count);
875 Some((
876 l.head.len() as u64
877 + len(kind::LABEL_ROOT)
878 + len(kind::LABEL_DATA)
879 + len(kind::LABEL_LEAVES),
880 entries,
881 ))
882 }
883 };
884 Ok((text, labels))
885}
886
887pub fn check_digest(root: &Root06, k: u8, bytes: &[u8]) -> Result<(), FileError> {
889 match root.section(k) {
890 Some(s) if s.digest == digest16(bytes) => Ok(()),
891 Some(_) => Err(FileError::Container("0x07 section digest mismatch")),
892 None => Ok(()),
893 }
894}
895
896pub fn read_annex<R: RangeReader + ?Sized>(
898 r: &R,
899 h: &Header06,
900 root: &Root06,
901) -> Result<Option<Vec<u8>>, FileError> {
902 let Some(stored) = read_section(r, h, root, kind::CARD_ANNEX)? else {
903 return Ok(None);
904 };
905 check_digest(root, kind::CARD_ANNEX, &stored)?;
906 let max = root
907 .section(kind::CARD_ANNEX)
908 .map(|s| s.max_block as u64)
909 .filter(|&m| m > 0)
910 .unwrap_or(ANNEX_MAX);
911 Ok(Some(decompress_bounded(h.internal_codec, &stored, max)?))
912}
913
914pub fn read_card_full<R: RangeReader + ?Sized>(
917 r: &R,
918) -> Result<(Option<Vec<u8>>, bool), FileError> {
919 let (h, root) = read_front(r)?;
920 let Some(hot) = read_card_build(r, &h)?.0 else {
921 return Ok((None, false));
922 };
923 match read_annex(r, &h, &root)? {
924 None => Ok((Some(hot), false)),
925 Some(a) => annex::merge(&hot, &a)
926 .map(|c| (Some(c), true))
927 .map_err(|_| FileError::Container("0x07 card annex does not merge")),
928 }
929}