1use std::sync::Arc;
8
9use super::*;
10use crate::derived;
11use crate::dict::{ChunkLeaf, ChunkedSection, LeafedChunks, SectionMeta};
12use crate::dictionary::Dictionary;
13use crate::file::{decompress_bounded, ByteRange, FileError};
14use crate::index::{GraphIndex, LeafFetch, LeafedTiles, TileLeaf, NUM_PERMS};
15use crate::reader::RangeReader;
16
17const TILE_GAP: u64 = 4096;
19const DICT_GAP: u64 = 64 * 1024;
20
21pub struct V06File {
23 pub h: Header06,
25 pub root: Root06,
27 pub reader: Arc<dyn RangeReader + Send + Sync>,
29}
30
31fn read_coalesced(
33 r: &(dyn RangeReader + Send + Sync),
34 ranges: &[(u64, u64)],
35 gap: u64,
36) -> Option<Vec<Vec<u8>>> {
37 let rs: Vec<ByteRange> = ranges
38 .iter()
39 .map(|&(offset, len)| ByteRange { offset, len })
40 .collect();
41 crate::file::read_coalesced(r, &rs, gap)
42}
43
44fn read_scattered(
46 r: &(dyn RangeReader + Send + Sync),
47 ranges: &[(u64, u64)],
48) -> Option<Vec<Vec<u8>>> {
49 let rs: Vec<ByteRange> = ranges
50 .iter()
51 .map(|&(offset, len)| ByteRange { offset, len })
52 .collect();
53 crate::file::read_coalesced_scattered(r, &rs, 0)
54}
55
56impl V06File {
57 pub fn open(
60 reader: Arc<dyn RangeReader + Send + Sync>,
61 first: &[u8],
62 ) -> Result<Self, FileError> {
63 let h = Header06::from_bytes(first)?;
64 if h.file_len != reader.len() {
65 return Err(FileError::Container(
66 "0x06 file length does not match the header (truncated?)",
67 ));
68 }
69 let rs = HEADER_LEN;
70 let re = rs + h.root_len as usize;
71 let root_bytes = if re <= first.len() {
72 first[rs..re].to_vec()
73 } else {
74 reader.read_at(rs as u64, h.root_len as u64)?
75 };
76 if digest16(&root_bytes) != h.root_digest {
77 return Err(FileError::Container("0x06 root digest mismatch"));
78 }
79 let root = Root06::decode(&root_bytes, h.internal_codec)?;
80 for s in &root.sections {
81 if h.version == VERSION_06 && kind::is_v07_only(s.kind) {
82 return Err(FileError::Container(
83 "0x06 file has a section only 0x07 defines",
84 ));
85 }
86 let a = s.abs(&h)?;
87 let end = a.offset + a.len;
88 let ok = if kind::is_front(s.kind) {
89 a.offset >= h.front_base() && end <= h.data_offset
90 } else {
91 a.offset >= h.data_offset && end <= h.file_len - FOOTER.len() as u64
92 };
93 if !ok {
94 return Err(FileError::Container("0x06 section out of its region"));
95 }
96 }
97 Ok(V06File { h, root, reader })
98 }
99
100 pub fn bound(&self, k: u8) -> u64 {
103 section_bound(&self.root, k)
104 }
105
106 pub fn section(&self, k: u8) -> Option<ByteRange> {
108 self.root.section(k).and_then(|s| s.abs(&self.h).ok())
109 }
110
111 pub fn fetch(&self) -> LeafFetch {
114 let r = self.reader.clone();
115 Arc::new(move |ranges: &[(u64, u64)]| read_coalesced(&*r, ranges, 0))
116 }
117
118 pub fn card(&self) -> Result<Option<Vec<u8>>, FileError> {
120 let r = self.h.card_range();
121 if r.len == 0 {
122 return Ok(None);
123 }
124 let stored = self.reader.read_at(r.offset, r.len)?;
125 if digest16(&stored) != self.h.card_digest {
126 return Err(FileError::Container("0x06 card digest mismatch"));
127 }
128 Ok(Some(decompress_bounded(
129 self.h.internal_codec,
130 &stored,
131 CARD_MAX,
132 )?))
133 }
134
135 pub fn build_record(&self) -> Result<Option<Vec<u8>>, FileError> {
137 let r = self.h.build_range();
138 if r.len == 0 {
139 return Ok(None);
140 }
141 Ok(Some(self.reader.read_at(r.offset, r.len)?))
142 }
143
144 pub fn label_root_node(&self) -> Result<Vec<u8>, FileError> {
147 let r = self
148 .section(kind::LABEL_ROOT)
149 .ok_or(FileError::Container("0x07 file has no label root section"))?;
150 let b = self.reader.read_at(r.offset, r.len)?;
151 check_digest(&self.root, kind::LABEL_ROOT, &b)?;
152 Ok(b)
153 }
154
155 pub fn annex(&self) -> Result<Option<Vec<u8>>, FileError> {
157 read_annex(&*self.reader, &self.h, &self.root)
158 }
159
160 pub fn card_full(&self) -> Result<Option<Vec<u8>>, FileError> {
162 let Some(hot) = self.card()? else {
163 return Ok(None);
164 };
165 match self.annex()? {
166 None => Ok(Some(hot)),
167 Some(a) => annex::merge(&hot, &a)
168 .map(Some)
169 .map_err(|_| FileError::Container("0x07 card annex does not merge")),
170 }
171 }
172
173 pub fn schema_section_bytes(&self) -> Option<Vec<u8>> {
176 let r = self.section(kind::SCHEMA)?;
177 let stored = self.reader.read_at(r.offset, r.len).ok()?;
178 let body =
179 decompress_bounded(self.h.internal_codec, &stored, self.bound(kind::SCHEMA)).ok()?;
180 let mut out = digest16(&body).to_vec();
181 out.extend_from_slice(&body);
182 Some(out)
183 }
184
185 pub fn schema(&self) -> Option<crate::schema_section::SchemaMeta> {
187 crate::schema_section::SchemaMeta::decode(&self.schema_section_bytes()?).ok()
188 }
189
190 pub fn pyramid_bytes(&self) -> Option<Vec<u8>> {
192 let r = self.section(kind::PYRAMID)?;
193 self.reader.read_at(r.offset, r.len).ok()
194 }
195
196 pub fn derived_families(&self, n_chunks: usize) -> Result<Vec<derived::Family>, FileError> {
199 let flagged = self.h.flags & F_DERIVED != 0;
200 let Some(unit) = self.root.unit(tag::DERIVED) else {
201 if flagged {
202 return Err(FileError::Container(
203 "0x08 derived-IRI flag without a derived-IRI root",
204 ));
205 }
206 return Ok(Vec::new());
207 };
208 if !flagged || self.h.version < VERSION {
209 return Err(FileError::Container(
210 "derived-IRI root without the derived-IRI header flag",
211 ));
212 }
213 derived::decode_families(unit, n_chunks)
214 .map_err(|_| FileError::Container("0x08 derived-IRI root is malformed"))
215 }
216
217 pub fn dictionary(&self, want: [bool; 4]) -> Result<Dictionary, FileError> {
220 let unit = self
221 .root
222 .unit(tag::DICT)
223 .ok_or(FileError::Container("0x06 file has no dictionary"))?;
224 let roots = decode_dict_root(unit, self.h.version)?;
225 let order = term_order(self.h.version);
226 let fams = self.derived_families(roots[derived::SECTION].n_chunks)?;
227 let elided = derived::elided_runs(&fams);
228 let leaves_base = self.section(kind::DICT_LEAVES).map_or(0, |r| r.offset);
229 let data_base = self.section(kind::DICT_DATA).map_or(0, |r| r.offset);
230 let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
231 let codec = self.h.dict_codec;
232 let max = self.bound(kind::DICT_DATA);
233 let mut dirs: Vec<Option<Arc<LeafedChunks>>> = Vec::with_capacity(4);
234 let mut sections = Vec::with_capacity(4);
235 for (si, s) in roots.iter().enumerate() {
236 let meta = SectionMeta {
237 term_count: if want[si] { s.term_count } else { 0 },
238 restart_interval: s.restart_interval.max(1),
239 restart_offsets: Vec::new(),
240 };
241 if !want[si] {
242 sections.push(
243 ChunkedSection::from_parts(meta, Vec::new(), None, cache.clone(), si as u8)
244 .with_order(order),
245 );
246 dirs.push(None);
247 continue;
248 }
249 let leaves: Vec<ChunkLeaf> = s
250 .leaves
251 .iter()
252 .map(|(l, run, coff, key)| ChunkLeaf {
253 first: l.first,
254 n: l.n,
255 first_run: *run,
256 key: key.clone(),
257 leaf: (leaves_base + l.off, l.len),
258 chunk_base: data_base + s.data_off + coff,
259 })
260 .collect();
261 let dir = Arc::new(
262 LeafedChunks::new(leaves, s.n_chunks, self.fetch(), self.h.internal_codec)
263 .with_max_block(self.bound(kind::DICT_LEAVES)),
264 );
265 let (r, d) = (self.reader.clone(), dir.clone());
266 let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
267 let (off, len) = d.range(ci)?;
268 let b = r.read_at(off, len).ok()?;
269 decompress_bounded(codec, &b, max).ok()
270 });
271 let (r, d) = (self.reader.clone(), dir.clone());
272 let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
273 d.ensure_leaves(&d.leaves_of(cis));
274 let ranges: Option<Vec<(u64, u64)>> = cis.iter().map(|&ci| d.range(ci)).collect();
275 let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
276 blobs
277 .iter()
278 .map(|b| decompress_bounded(codec, b, max).ok())
279 .collect()
280 });
281 let mut section =
282 ChunkedSection::leafed(meta, dir.clone(), loader, cache.clone(), si as u8)
283 .with_bulk_loader(bulk)
284 .with_order(order);
285 if order == crate::term_order::TermOrder::Sparql {
286 section = section.with_groups(s.groups);
287 }
288 if si == derived::SECTION {
289 section = section.with_elided(elided.clone());
290 }
291 sections.push(section);
292 dirs.push(Some(dir));
293 }
294 let arr: [ChunkedSection; 4] = sections
295 .try_into()
296 .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
297 let r = self.reader.clone();
298 let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
299 let mut per: [Vec<usize>; 4] = Default::default();
301 for &(si, ci) in pairs {
302 if let Some(v) = per.get_mut(si as usize) {
303 v.push(ci);
304 }
305 }
306 let wanted: Vec<(usize, Vec<usize>)> = per
309 .iter()
310 .enumerate()
311 .filter_map(|(si, cis)| Some((si, dirs.get(si)?.as_ref()?.leaves_of(cis))))
312 .collect();
313 let across: Vec<(&dyn crate::index::LeafDir, &[usize])> = wanted
314 .iter()
315 .filter_map(|(si, lis)| {
316 let d: &dyn crate::index::LeafDir = dirs.get(*si)?.as_deref()?;
317 Some((d, lis.as_slice()))
318 })
319 .collect();
320 crate::index::ensure_leaves_across(&across);
321 let ranges: Option<Vec<(u64, u64)>> = pairs
322 .iter()
323 .map(|&(si, ci)| dirs.get(si as usize)?.as_ref()?.range(ci))
324 .collect();
325 let blobs = read_coalesced(&*r, &ranges?, DICT_GAP)?;
326 blobs
327 .iter()
328 .map(|b| decompress_bounded(codec, b, max).ok())
329 .collect()
330 });
331 let mut dict = Dictionary::from_chunked_sections(arr).with_multi_bulk_loader(multi);
332 dict.set_has_quoted_triples(self.h.flags & F_QUOTED != 0);
333 if want[derived::SECTION] {
334 dict = dict.with_derived(fams);
335 }
336 Ok(dict)
337 }
338
339 fn clusters(&self, perms: &[PermRoot]) -> Result<Vec<ClusterOrder>, FileError> {
342 let Some(unit) = self.root.unit(tag::CLUSTER) else {
343 if self.h.flags & F_CLUSTERED != 0 {
344 return Err(FileError::Container(
345 "0x07 clustered file without a cluster root",
346 ));
347 }
348 return Ok(Vec::new());
349 };
350 let orders = decode_cluster_root(unit)?;
351 let quad = self.quad_root().transpose()?;
352 for c in &orders {
353 let (n_tiles, leaves): (usize, Vec<(usize, usize)>) = if c.si == QUAD_SPOG {
355 let o = quad
356 .as_ref()
357 .and_then(|q| q.orders.iter().find(|o| o.si == c.si))
358 .ok_or(FileError::Container(
359 "0x07 cluster root names an order the quad index lacks",
360 ))?;
361 (
362 o.n_tiles,
363 o.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
364 )
365 } else {
366 let p = perms
367 .iter()
368 .find(|p| p.si == c.si)
369 .ok_or(FileError::Container(
370 "0x07 cluster root names a permutation the index lacks",
371 ))?;
372 (
373 p.n_tiles,
374 p.leaves.iter().map(|(l, _, _, _)| (l.first, l.n)).collect(),
375 )
376 };
377 if c.run_tiles.iter().sum::<usize>() != n_tiles {
378 return Err(FileError::Container(
379 "0x07 cluster runs do not cover the tiles",
380 ));
381 }
382 let mut ends = Vec::with_capacity(c.run_tiles.len());
384 let mut at = 0usize;
385 for &n in &c.run_tiles {
386 at += n;
387 ends.push(at);
388 }
389 for (first, n) in leaves {
390 let r = ends.partition_point(|&e| e <= first);
391 if first + n > ends[r.min(ends.len() - 1)] {
392 return Err(FileError::Container(
393 "0x07 a directory leaf spans two cluster runs",
394 ));
395 }
396 }
397 }
398 Ok(orders)
399 }
400
401 pub fn index(&self, read_concurrency: usize) -> Result<GraphIndex, FileError> {
403 let unit = self
404 .root
405 .unit(tag::INDEX)
406 .ok_or(FileError::Container("0x06 file has no index"))?;
407 let perms_root = decode_index_root(unit)?;
408 let leaves_base = self.section(kind::INDEX_LEAVES).map_or(0, |r| r.offset);
409 let data_base = self.section(kind::INDEX_DATA).map_or(0, |r| r.offset);
410 let perms = self.h.perms();
411 let mut lists: [Option<Arc<LeafedTiles>>; NUM_PERMS] = Default::default();
412 for p in &perms_root {
413 if p.si >= NUM_PERMS {
414 return Err(FileError::Container("0x06 index: bad permutation"));
415 }
416 let leaves: Vec<TileLeaf> = p
417 .leaves
418 .iter()
419 .map(|(l, mn, mx, toff)| TileLeaf {
420 first: l.first,
421 n: l.n,
422 min_a: *mn,
423 max_a: *mx,
424 leaf: (leaves_base + l.off, l.len),
425 tile_base: data_base + p.data_off + toff,
426 })
427 .collect();
428 lists[p.si] = Some(Arc::new(
429 LeafedTiles::new(
430 leaves,
431 p.n_tiles,
432 p.data_len,
433 self.fetch(),
434 self.h.internal_codec,
435 )
436 .with_max_block(self.bound(kind::INDEX_LEAVES)),
437 ));
438 }
439 for perm in perms.iter() {
440 if lists[perm.section_index()].is_none() {
441 return Err(FileError::Container(
442 "0x06 index lacks a stored permutation",
443 ));
444 }
445 }
446 let codec = self.h.block_codec;
447 let max = self.bound(kind::INDEX_DATA);
448 let (r, l) = (self.reader.clone(), lists.clone());
449 let loader: crate::index::TileLoader = Box::new(move |si, ti| {
450 let (off, len) = l.get(si)?.as_ref()?.range(ti)?;
451 let b = r.read_at(off, len).ok()?;
452 decompress_bounded(codec, &b, max).ok()
453 });
454 let (r, l) = (self.reader.clone(), lists.clone());
455 let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
456 let list = l.get(si)?.as_ref()?;
457 if let (Some(&a), Some(&b)) = (tis.first(), tis.last()) {
458 list.ensure_span(a, b + 1);
459 }
460 let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
461 let blobs = read_coalesced(&*r, &ranges?, TILE_GAP)?;
462 blobs
463 .iter()
464 .map(|b| decompress_bounded(codec, b, max).ok())
465 .collect()
466 });
467 let (r, l) = (self.reader.clone(), lists.clone());
468 let scattered: crate::index::TileBulkLoader = Box::new(move |si, tis| {
469 let list = l.get(si)?.as_ref()?;
470 let ranges: Option<Vec<(u64, u64)>> = tis.iter().map(|&ti| list.range(ti)).collect();
471 let blobs = read_scattered(&*r, &ranges?)?;
472 blobs
473 .iter()
474 .map(|b| decompress_bounded(codec, b, max).ok())
475 .collect()
476 });
477 let mut index = GraphIndex::from_leafed(lists, perms, loader)
478 .with_bulk_loader(bulk)
479 .with_scattered_loader(scattered);
480 for c in self
481 .clusters(&perms_root)?
482 .into_iter()
483 .filter(|c| c.si < NUM_PERMS)
484 {
485 index = index.with_clusters(
486 c.si,
487 crate::index::ClusterRuns::new(c.classes, &c.run_tiles),
488 );
489 }
490 index.set_read_concurrency(read_concurrency);
491 Ok(index)
492 }
493
494 pub fn text_index(&self) -> Option<Result<crate::text_index::TextIndex, FileError>> {
496 let unit = self.root.unit(tag::TEXT)?;
497 Some((|| {
498 let t = decode_text_root(unit)?;
499 let leaves_base = self.section(kind::TEXT_LEAVES).map_or(0, |r| r.offset);
500 let data = self
501 .section(kind::TEXT_DATA)
502 .unwrap_or(ByteRange { offset: 0, len: 0 });
503 let leaves = t
504 .leaves
505 .iter()
506 .map(|(l, tok)| crate::text_index::TokenLeaf {
507 first: l.first,
508 n: l.n,
509 first_token: String::from_utf8_lossy(tok).into_owned(),
510 leaf: (leaves_base + l.off, l.len),
511 })
512 .collect();
513 let tokens = Arc::new(
514 crate::text_index::LeafedTokens::new(
515 leaves,
516 t.n_tokens,
517 self.fetch(),
518 self.h.internal_codec,
519 )
520 .with_max_block(self.bound(kind::TEXT_LEAVES)),
521 );
522 let r = self.reader.clone();
523 let loader = Box::new(move |off: u64, len: u64| {
524 if off.checked_add(len)? > data.len {
525 return None;
526 }
527 r.read_at(data.offset + off, len).ok()
528 });
529 crate::text_index::TextIndex::leafed(tokens, &t.scope, loader)
530 })())
531 }
532
533 pub fn token_table_bytes(&self) -> Option<u64> {
536 self.root.unit(tag::TEXT)?;
537 Some(self.section(kind::TEXT_LEAVES).map_or(0, |r| r.len))
538 }
539
540 pub fn label_index(&self) -> Option<Result<crate::label_index::LabelIndex, FileError>> {
542 let unit = self.root.unit(tag::LABELS)?;
543 Some((|| {
544 let mut l = decode_label_root(unit)?;
545 if self.section(kind::LABEL_ROOT).is_some() {
546 l.head.extend_from_slice(&self.label_root_node()?);
547 }
548 let data = self
549 .section(kind::LABEL_DATA)
550 .unwrap_or(ByteRange { offset: 0, len: 0 });
551 let nodes = self
552 .section(kind::LABEL_LEAVES)
553 .unwrap_or(ByteRange { offset: 0, len: 0 });
554 if data.len != l.data_len {
555 return Err(FileError::Container("0x06 label data length mismatch"));
556 }
557 let head_len = l.head.len() as u64;
558 let section_len = head_len + data.len + nodes.len;
559 let r = self.reader.clone();
560 let head = l.head.clone();
561 let loader = Box::new(move |off: u64, len: u64| -> Option<Vec<u8>> {
562 let end = off.checked_add(len)?;
563 if end <= head_len {
564 return head.get(off as usize..end as usize).map(<[u8]>::to_vec);
565 }
566 let bo = off.checked_sub(head_len)?;
567 if bo + len <= data.len {
568 r.read_at(data.offset + bo, len).ok()
569 } else if bo >= data.len && bo - data.len + len <= nodes.len {
570 r.read_at(nodes.offset + (bo - data.len), len).ok()
571 } else {
572 None
573 }
574 });
575 let max = self
576 .bound(kind::LABEL_LEAVES)
577 .max(self.bound(kind::LABEL_DATA));
578 crate::label_index::LabelIndex::from_head(
579 &l.head,
580 section_len,
581 self.h.block_codec,
582 loader,
583 )
584 .map(|li| li.with_max_block(max))
585 })())
586 }
587
588 pub fn label_entry_count(&self) -> Option<u64> {
590 let l = decode_label_root(self.root.unit(tag::LABELS)?).ok()?;
591 crate::label_index::LabelIndexPreamble::parse(&l.head)
592 .ok()
593 .map(|p| p.entry_count)
594 }
595
596 pub fn quad_root(&self) -> Option<Result<QuadRoot, FileError>> {
598 self.root.unit(tag::QUAD).map(decode_quad_root)
599 }
600
601 pub(crate) fn quad(
603 &self,
604 read_concurrency: usize,
605 ) -> Option<Result<crate::quad::QuadNamed, FileError>> {
606 let root = match self.quad_root()? {
607 Ok(r) => r,
608 Err(e) => return Some(Err(e)),
609 };
610 let leaves = self.section(kind::QUAD_LEAVES).map_or(0, |r| r.offset);
611 let data = self.section(kind::QUAD_DATA).map_or(0, |r| r.offset);
612 let clusters = match self.root.unit(tag::CLUSTER).map(decode_cluster_root) {
614 Some(Ok(c)) => c,
615 Some(Err(e)) => return Some(Err(e)),
616 None => Vec::new(),
617 };
618 Some(
619 crate::quad::V06Quad::new(
620 &root,
621 leaves,
622 data,
623 self.h.internal_codec,
624 has_v07(self.h.version),
625 )
626 .and_then(|v| v.with_clusters(&clusters))
627 .map(|mut v| {
628 v.tile_max = self.bound(kind::QUAD_DATA);
629 v.leaf_max = self.bound(kind::QUAD_LEAVES);
630 crate::quad::QuadNamed::new_v06(
631 self.reader.clone(),
632 v,
633 self.h.block_codec,
634 self.h.perms(),
635 read_concurrency,
636 )
637 }),
638 )
639 }
640
641 pub fn verify(&self, only: Option<u8>) -> Result<bool, FileError> {
645 Ok(self.verify_report(only)?.is_empty())
646 }
647
648 pub fn verify_report(&self, only: Option<u8>) -> Result<Vec<String>, FileError> {
650 const SLICE: u64 = 8 << 20;
651 let mut bad = Vec::new();
652 if content_hash(&self.h) != self.h.content_hash {
653 bad.push("content hash".to_string());
654 }
655 let card = self.h.card_range();
656 if only.is_none() && card.len > 0 {
657 let stored = self.reader.read_at(card.offset, card.len)?;
658 if digest16(&stored) != self.h.card_digest {
659 bad.push("card".to_string());
660 }
661 }
662 for s in &self.root.sections {
663 if only.is_some_and(|k| k != s.kind) {
664 continue;
665 }
666 let a = s.abs(&self.h)?;
667 let mut h = blake3::Hasher::new();
668 let mut at = a.offset;
669 while at < a.offset + a.len {
670 let n = (a.offset + a.len - at).min(SLICE);
671 h.update(&self.reader.read_at(at, n)?);
672 at += n;
673 }
674 if h.finalize().as_bytes()[..16] != s.digest {
675 bad.push(kind::name(s.kind).to_string());
676 }
677 }
678 if only.is_none() {
679 let want = footer(self.h.version);
680 let foot = self
681 .reader
682 .read_at(self.h.file_len - want.len() as u64, want.len() as u64)?;
683 if foot != want {
684 bad.push("footer".to_string());
685 }
686 }
687 Ok(bad)
688 }
689}
690
691pub fn is_v06(first: &[u8]) -> bool {
693 first.len() > 4 && first[0..4] == crate::header::MAGIC && reads(first[4])
694}
695
696pub fn read_front<R: RangeReader + ?Sized>(r: &R) -> Result<(Header06, Root06), FileError> {
699 let head = r.read_at(0, HEADER_LEN as u64)?;
700 let h = Header06::from_bytes(&head)?;
701 let root_bytes = r.read_at(HEADER_LEN as u64, h.root_len as u64)?;
702 if digest16(&root_bytes) != h.root_digest {
703 return Err(FileError::Container("0x06 root digest mismatch"));
704 }
705 let root = Root06::decode(&root_bytes, h.internal_codec)?;
706 Ok((h, root))
707}
708
709pub fn read_section<R: RangeReader + ?Sized>(
711 r: &R,
712 h: &Header06,
713 root: &Root06,
714 k: u8,
715) -> Result<Option<Vec<u8>>, FileError> {
716 match root.section(k) {
717 Some(s) => {
718 let a = s.abs(h)?;
719 Ok(Some(r.read_at(a.offset, a.len)?))
720 }
721 None => Ok(None),
722 }
723}
724
725#[allow(clippy::type_complexity)]
728pub fn read_card_build<R: RangeReader + ?Sized>(
729 r: &R,
730 h: &Header06,
731) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
732 read_card_build_with(r, h, true)
733}
734
735#[allow(clippy::type_complexity)]
738pub fn read_card_build_with<R: RangeReader + ?Sized>(
739 r: &R,
740 h: &Header06,
741 want_build: bool,
742) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
743 let c = h.card_range();
744 let mut b = h.build_range();
745 if !want_build {
746 b.len = 0;
747 }
748 if c.len == 0 && b.len == 0 {
749 return Ok((None, None));
750 }
751 let lo = if c.len == 0 {
752 b.offset
753 } else if b.len == 0 {
754 c.offset
755 } else {
756 c.offset.min(b.offset)
757 };
758 let hi = (c.offset + c.len).max(b.offset + b.len);
759 let span = r.read_at(lo, hi - lo)?;
760 let card = if c.len > 0 {
761 let stored = &span[(c.offset - lo) as usize..(c.offset - lo + c.len) as usize];
762 if digest16(stored) != h.card_digest {
763 return Err(FileError::Container("0x06 card digest mismatch"));
764 }
765 Some(decompress_bounded(h.internal_codec, stored, CARD_MAX)?)
766 } else {
767 None
768 };
769 let build = (b.len > 0)
770 .then(|| span[(b.offset - lo) as usize..(b.offset - lo + b.len) as usize].to_vec());
771 Ok((card, build))
772}
773
774pub fn section_bound(root: &Root06, k: u8) -> u64 {
777 root.section(k)
778 .map(|s| s.max_block as u64)
779 .filter(|&m| m > 0)
780 .unwrap_or(BLOCK_CEILING)
781}
782
783pub fn read_schema_bytes<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
785 let (h, root) = read_front(r)?;
786 let Some(stored) = read_section(r, &h, &root, kind::SCHEMA)? else {
787 return Ok(None);
788 };
789 let body = decompress_bounded(
790 h.internal_codec,
791 &stored,
792 section_bound(&root, kind::SCHEMA),
793 )?;
794 let mut out = digest16(&body).to_vec();
795 out.extend_from_slice(&body);
796 Ok(Some(out))
797}
798
799pub fn read_pyramid<R: RangeReader + ?Sized>(r: &R) -> Result<Option<Vec<u8>>, FileError> {
801 let (h, root) = read_front(r)?;
802 read_section(r, &h, &root, kind::PYRAMID)
803}
804
805pub fn read_token_table_bytes<R: RangeReader + ?Sized>(r: &R) -> Option<u64> {
807 let (_, root) = read_front(r).ok()?;
808 root.unit(tag::TEXT)?;
809 Some(root.section(kind::TEXT_LEAVES).map_or(0, |s| s.len))
810}
811
812pub struct SparseReader {
815 len: u64,
816 parts: Vec<(u64, Vec<u8>)>,
817}
818
819impl SparseReader {
820 pub fn fetch<R: RangeReader + ?Sized>(
823 r: &R,
824 first: Vec<u8>,
825 ranges: &[ByteRange],
826 ) -> Result<Self, FileError> {
827 let held = first.len() as u64;
828 let want: Vec<(u64, u64)> = ranges
829 .iter()
830 .filter(|g| g.len > 0 && g.offset.saturating_add(g.len) > held)
831 .map(|g| (g.offset, g.len))
832 .collect();
833 let mut parts = vec![(0u64, first)];
834 if !want.is_empty() {
835 for ((o, _), b) in want.iter().zip(r.read_many(&want)?) {
836 parts.push((*o, b));
837 }
838 }
839 Ok(SparseReader {
840 len: r.len(),
841 parts,
842 })
843 }
844}
845
846pub struct Prefixed<'a, R: RangeReader + ?Sized> {
849 pub first: &'a [u8],
851 pub r: &'a R,
853}
854
855impl<R: RangeReader + ?Sized> RangeReader for Prefixed<'_, R> {
856 fn is_local(&self) -> bool {
857 self.r.is_local()
858 }
859
860 fn len(&self) -> u64 {
861 self.r.len()
862 }
863
864 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
865 match offset.checked_add(len) {
866 Some(end) if end <= self.first.len() as u64 => {
867 Ok(self.first[offset as usize..end as usize].to_vec())
868 }
869 _ => self.r.read_at(offset, len),
870 }
871 }
872}
873
874impl RangeReader for SparseReader {
875 fn len(&self) -> u64 {
876 self.len
877 }
878
879 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
880 for (o, b) in &self.parts {
881 if offset >= *o && offset + len <= o + b.len() as u64 {
882 let s = (offset - o) as usize;
883 return Ok(b[s..s + len as usize].to_vec());
884 }
885 }
886 Err(std::io::Error::other("0x06: range not held by this reader"))
887 }
888}
889
890#[allow(clippy::type_complexity)]
893pub fn read_signals<R: RangeReader + ?Sized>(
894 r: &R,
895) -> Result<(Option<(u64, u64)>, Option<(u64, Option<u64>)>), FileError> {
896 let (_, root) = read_front(r)?;
897 let len = |k: u8| root.section(k).map_or(0, |s| s.len);
898 let text = root.unit(tag::TEXT).map(|_| {
899 (
900 len(kind::TEXT_LEAVES) + len(kind::TEXT_DATA),
901 len(kind::TEXT_LEAVES),
902 )
903 });
904 let labels = match root.unit(tag::LABELS) {
905 None => None,
906 Some(u) => {
907 let l = decode_label_root(u)?;
908 let entries = crate::label_index::LabelIndexPreamble::parse(&l.head)
909 .ok()
910 .map(|p| p.entry_count);
911 Some((
912 l.head.len() as u64
913 + len(kind::LABEL_ROOT)
914 + len(kind::LABEL_DATA)
915 + len(kind::LABEL_LEAVES),
916 entries,
917 ))
918 }
919 };
920 Ok((text, labels))
921}
922
923pub fn check_digest(root: &Root06, k: u8, bytes: &[u8]) -> Result<(), FileError> {
925 match root.section(k) {
926 Some(s) if s.digest == digest16(bytes) => Ok(()),
927 Some(_) => Err(FileError::Container("0x07 section digest mismatch")),
928 None => Ok(()),
929 }
930}
931
932pub fn read_annex<R: RangeReader + ?Sized>(
934 r: &R,
935 h: &Header06,
936 root: &Root06,
937) -> Result<Option<Vec<u8>>, FileError> {
938 let Some(stored) = read_section(r, h, root, kind::CARD_ANNEX)? else {
939 return Ok(None);
940 };
941 check_digest(root, kind::CARD_ANNEX, &stored)?;
942 let max = root
943 .section(kind::CARD_ANNEX)
944 .map(|s| s.max_block as u64)
945 .filter(|&m| m > 0)
946 .unwrap_or(ANNEX_MAX);
947 Ok(Some(decompress_bounded(h.internal_codec, &stored, max)?))
948}
949
950pub fn read_card_full<R: RangeReader + ?Sized>(
953 r: &R,
954) -> Result<(Option<Vec<u8>>, bool), FileError> {
955 let (h, root) = read_front(r)?;
956 let Some(hot) = read_card_build(r, &h)?.0 else {
957 return Ok((None, false));
958 };
959 match read_annex(r, &h, &root)? {
960 None => Ok((Some(hot), false)),
961 Some(a) => annex::merge(&hot, &a)
962 .map(|c| (Some(c), true))
963 .map_err(|_| FileError::Container("0x07 card annex does not merge")),
964 }
965}