1use crate::dictionary::Dictionary;
18use crate::header::{
19 Header, Section, SectionKind, FLAG_HAS_QUADS, FLAG_HAS_QUOTED_TRIPLES, FLAG_TILE_SYNOPSIS,
20 HEADER_LEN, MAGIC,
21};
22use crate::index::{GraphIndex, IndexPermutation, Pattern, PermSet, NUM_PERMS};
23use crate::meta::{ClassNode, CommunityDescriptor, LevelLinks, LevelRollup, PyramidMeta};
24use crate::pyramid::{build_dendrogram, project_graph, PyramidAlgo};
25use crate::reader::RangeReader;
26use crate::tiling::{choose_round_for_budget, summarize, SuperEdge};
27use crate::triples::Triple;
28use crate::varint::{read_uvarint, write_uvarint};
29
30#[cfg(test)]
31mod hardening_tests;
32
33pub const DEFAULT_TILE_BUDGET: usize = 64 * 1024;
35
36pub fn build_pyramid_meta(
45 dict: &Dictionary,
46 triples: &[(u32, u32, u32)],
47 budget: usize,
48) -> (Vec<u8>, u16) {
49 build_pyramid_meta_with(dict, triples, budget, None)
50}
51
52pub fn build_pyramid_meta_with(
56 dict: &Dictionary,
57 triples: &[(u32, u32, u32)],
58 budget: usize,
59 type_override: Option<&str>,
60) -> (Vec<u8>, u16) {
61 build_pyramid_meta_algo(dict, triples, budget, type_override, PyramidAlgo::Louvain)
62}
63
64pub fn build_pyramid_meta_algo(
71 dict: &Dictionary,
72 triples: &[(u32, u32, u32)],
73 budget: usize,
74 type_override: Option<&str>,
75 algo: PyramidAlgo,
76) -> (Vec<u8>, u16) {
77 build_pyramid_meta_layout(
78 dict,
79 triples,
80 budget,
81 type_override,
82 algo,
83 PyramidLayout::Combined,
84 )
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
89pub enum PyramidLayout {
90 #[default]
94 Combined,
95 CommunityOnly,
100}
101
102pub fn build_pyramid_meta_layout(
104 dict: &Dictionary,
105 triples: &[(u32, u32, u32)],
106 budget: usize,
107 type_override: Option<&str>,
108 algo: PyramidAlgo,
109 layout: PyramidLayout,
110) -> (Vec<u8>, u16) {
111 let timing = std::env::var_os("RETE_BUILD_TIMING").is_some();
117 let mut t = timing.then(std::time::Instant::now);
118 let mut lap = |label: &str| {
119 if let Some(t0) = &mut t {
120 eprintln!(
121 " [pyramid] {label}: {:.0} ms",
122 t0.elapsed().as_secs_f64() * 1000.0
123 );
124 *t0 = std::time::Instant::now();
125 }
126 };
127
128 let louvain = |lap: &mut dyn FnMut(&str)| {
130 let g = project_graph(dict, triples);
131 lap("project_graph");
132 let d = build_dendrogram(&g);
133 lap("build_dendrogram (Louvain)");
134 d
135 };
136 let dend = match algo {
137 PyramidAlgo::Louvain => louvain(&mut lap),
138 PyramidAlgo::Types => {
139 match crate::schema_pyramid::build_type_dendrogram(dict, triples, type_override) {
140 Some(d) => {
141 lap("build_type_dendrogram");
142 d
143 }
144 None => {
145 eprintln!(
146 " [pyramid] --pyramid-algo types: no usable rdf:type \
147 predicate — falling back to louvain"
148 );
149 louvain(&mut lap)
150 }
151 }
152 }
153 };
154 let round = choose_round_for_budget(dict, triples, &dend, budget);
155 lap("choose_round_for_budget");
156 let summary = summarize(dict, triples, &dend, round);
157 lap("summarize");
158 if layout == PyramidLayout::CommunityOnly {
163 let descriptors = crate::schema_pyramid::build_community_descriptors(
166 dict,
167 triples,
168 &dend,
169 round,
170 type_override,
171 );
172 lap("build_community_descriptors");
173 let meta = PyramidMeta::new(round as u32, summary, &[]).with_schema(
174 Vec::new(),
175 Vec::new(),
176 Vec::new(),
177 descriptors,
178 Vec::new(),
179 Vec::new(),
180 Vec::new(),
181 );
182 let out = (meta.encode(), dend.rounds() as u16);
183 lap("encode");
184 return out;
185 }
186 let sp = crate::schema_pyramid::build_schema_pyramid_with(
187 dict,
188 triples,
189 &dend,
190 round,
191 type_override,
192 );
193 lap("build_schema_pyramid");
194 let predicate_stats = compute_predicate_stats(triples);
195 lap("compute_predicate_stats");
196 let char_sets = compute_char_sets(triples);
197 lap("compute_char_sets");
198 let label_index = compute_label_index(dict, triples);
199 lap("compute_label_index");
200 let meta = PyramidMeta::new(round as u32, summary, &[])
201 .with_schema(
202 sp.class_hierarchy,
203 sp.level_rollups,
204 sp.level_links,
205 sp.descriptors,
206 sp.subclass_cycles,
207 sp.disjoint_pairs,
208 sp.equivalent_pairs,
209 )
210 .with_predicate_stats(predicate_stats)
211 .with_char_sets(char_sets)
212 .with_label_index(label_index);
213 let out = (meta.encode(), dend.rounds() as u16);
214 lap("encode");
215 out
216}
217
218const LABEL_PREDICATES: &[&str] = crate::label_index::DIRECT_LABEL_PREDICATES;
223
224fn compute_label_index(
232 dict: &Dictionary,
233 triples: &[(u32, u32, u32)],
234) -> Vec<crate::meta::LabelEntry> {
235 use crate::terms::{is_literal, literal_lexical};
236 use std::collections::{HashMap, HashSet};
237 const MAX_LABELS: usize = 8192;
238
239 let label_pids: HashSet<u32> = LABEL_PREDICATES
241 .iter()
242 .filter_map(|p| dict.predicate_id(p))
243 .collect();
244 if label_pids.is_empty() {
245 return Vec::new();
246 }
247 let mut degree: HashMap<u32, u32> = HashMap::new();
249 for &(s, _p, _o) in triples {
250 *degree.entry(s).or_insert(0) += 1;
251 }
252 let mut seen: HashSet<(u32, String)> = HashSet::new();
254 let mut candidates: Vec<(u32, String, u32)> = Vec::new(); for &(s, p, o) in triples {
256 if !label_pids.contains(&p) {
257 continue;
258 }
259 let Some(term) = dict.object_term(o) else {
260 continue;
261 };
262 if !is_literal(&term) {
263 continue;
264 }
265 let Some(label) = literal_lexical(&term) else {
266 continue;
267 };
268 if label.is_empty() {
269 continue;
270 }
271 if seen.insert((s, label.to_lowercase())) {
272 candidates.push((*degree.get(&s).unwrap_or(&0), label, s));
273 }
274 }
275 if candidates.len() > MAX_LABELS {
278 candidates.sort_by(|a, b| {
279 b.0.cmp(&a.0)
280 .then_with(|| a.2.cmp(&b.2))
281 .then_with(|| a.1.cmp(&b.1))
282 });
283 candidates.truncate(MAX_LABELS);
284 }
285 candidates.sort_by(|a, b| {
287 a.1.to_lowercase()
288 .cmp(&b.1.to_lowercase())
289 .then_with(|| a.1.cmp(&b.1))
290 .then_with(|| a.2.cmp(&b.2))
291 });
292 candidates
293 .into_iter()
294 .map(|(_deg, label, subject)| crate::meta::LabelEntry { label, subject })
295 .collect()
296}
297
298pub fn compute_text_index(dict: &Dictionary, triples: &[(u32, u32, u32)]) -> Vec<u8> {
307 crate::text_index::build_section(
308 dict,
309 &crate::text_index::TextIndexScope::default(),
310 triples,
311 std::iter::empty(),
312 writer_codec(),
313 )
314}
315
316fn compute_char_sets(triples: &[(u32, u32, u32)]) -> Vec<crate::meta::CharSet> {
321 use std::collections::{BTreeSet, HashMap};
322 const MAX_CHAR_SETS: usize = 128;
323 let mut by_subject: HashMap<u32, BTreeSet<u32>> = HashMap::new();
324 for &(s, p, _o) in triples {
325 by_subject.entry(s).or_default().insert(p);
326 }
327 let mut shapes: HashMap<Vec<u32>, u64> = HashMap::new();
328 for set in by_subject.into_values() {
329 *shapes.entry(set.into_iter().collect()).or_insert(0) += 1;
330 }
331 let mut v: Vec<crate::meta::CharSet> = shapes
332 .into_iter()
333 .map(|(predicates, subjects)| crate::meta::CharSet {
334 predicates,
335 subjects,
336 })
337 .collect();
338 v.sort_by(|a, b| {
339 b.subjects
340 .cmp(&a.subjects)
341 .then_with(|| a.predicates.cmp(&b.predicates))
342 });
343 v.truncate(MAX_CHAR_SETS);
344 v
345}
346
347fn compute_predicate_stats(triples: &[(u32, u32, u32)]) -> Vec<crate::meta::PredStat> {
353 use std::collections::HashMap;
354 #[allow(clippy::type_complexity)]
355 let mut acc: HashMap<u32, (HashMap<u32, u32>, HashMap<u32, u32>, u64)> = HashMap::new();
356 for &(s, p, o) in triples {
357 let e = acc.entry(p).or_default();
358 *e.0.entry(s).or_insert(0) += 1;
359 *e.1.entry(o).or_insert(0) += 1;
360 e.2 += 1;
361 }
362 let mut stats: Vec<crate::meta::PredStat> = acc
363 .into_iter()
364 .map(|(predicate, (subj, obj, count))| crate::meta::PredStat {
365 predicate,
366 count,
367 distinct_subjects: subj.len() as u64,
368 distinct_objects: obj.len() as u64,
369 max_objects_per_subject: subj.values().copied().max().unwrap_or(0),
370 max_subjects_per_object: obj.values().copied().max().unwrap_or(0),
371 })
372 .collect();
373 stats.sort_by_key(|p| p.predicate);
374 stats
375}
376
377pub const CODEC_NONE: u8 = 0;
379pub const CODEC_ZSTD: u8 = 1;
381
382#[derive(Debug, thiserror::Error)]
383#[non_exhaustive]
384pub enum FileError {
385 #[error("header: {0}")]
386 Header(#[from] crate::header::HeaderError),
387 #[error("malformed container: {0}")]
388 Container(&'static str),
389 #[error("unknown codec: {0}")]
390 UnknownCodec(u8),
391 #[error("decompression failed: {0}")]
392 Decompress(std::io::Error),
393 #[error("io: {0}")]
394 Io(#[from] std::io::Error),
395}
396
397pub fn writer_codec() -> u8 {
400 if cfg!(feature = "compression") {
401 CODEC_ZSTD
402 } else {
403 CODEC_NONE
404 }
405}
406
407fn intersect_sorted(a: &[u32], b: &[u32]) -> Vec<u32> {
410 let mut out = Vec::with_capacity(a.len().min(b.len()));
411 let (mut i, mut j) = (0, 0);
412 while i < a.len() && j < b.len() {
413 match a[i].cmp(&b[j]) {
414 std::cmp::Ordering::Less => i += 1,
415 std::cmp::Ordering::Greater => j += 1,
416 std::cmp::Ordering::Equal => {
417 out.push(a[i]);
418 i += 1;
419 j += 1;
420 }
421 }
422 }
423 out
424}
425
426pub(crate) fn compress(codec: u8, bytes: &[u8]) -> Vec<u8> {
427 match codec {
428 #[cfg(feature = "compression")]
431 CODEC_ZSTD => {
432 let level = crate::write_opts::zstd_level();
433 match crate::write_opts::frame_shape() {
434 crate::write_opts::FrameShape::Stream => zstd::encode_all(bytes, level),
435 crate::write_opts::FrameShape::Sized => zstd::bulk::compress(bytes, level),
436 }
437 .expect("zstd encode is infallible in-memory")
438 }
439 _ => bytes.to_vec(),
440 }
441}
442
443#[cfg(not(target_arch = "wasm32"))]
461pub(crate) fn compress_to(
462 codec: u8,
463 src_len: u64,
464 src: &mut dyn std::io::Read,
465 dst: &mut dyn std::io::Write,
466) -> std::io::Result<()> {
467 match codec {
468 #[cfg(feature = "compression")]
469 CODEC_ZSTD => zstd_stream(src_len, src, dst),
470 _ => {
471 let _ = src_len;
472 std::io::copy(src, dst).map(|_| ())
473 }
474 }
475}
476
477pub(crate) fn compress_streamed(codec: u8, bytes: &[u8]) -> Vec<u8> {
483 match codec {
484 #[cfg(feature = "compression")]
485 CODEC_ZSTD => {
486 let mut out = Vec::with_capacity(bytes.len() / 4 + 64);
487 zstd_stream(bytes.len() as u64, &mut &bytes[..], &mut out)
488 .expect("zstd encode is infallible in-memory");
489 out
490 }
491 _ => bytes.to_vec(),
492 }
493}
494
495#[cfg(feature = "compression")]
497fn zstd_stream(
498 src_len: u64,
499 src: &mut dyn std::io::Read,
500 dst: &mut dyn std::io::Write,
501) -> std::io::Result<()> {
502 let mut enc = zstd::stream::Encoder::new(dst, crate::write_opts::zstd_level())?;
503 if crate::write_opts::frame_shape() == crate::write_opts::FrameShape::Sized {
504 enc.set_pledged_src_size(Some(src_len))?;
505 enc.include_contentsize(true)?;
506 }
507 let copied = std::io::copy(src, &mut enc)?;
508 if copied != src_len {
509 return Err(std::io::Error::other(format!(
510 "zstd stream: {copied} bytes read, {src_len} announced"
511 )));
512 }
513 enc.finish()?;
514 Ok(())
515}
516
517pub(crate) fn decompress(codec: u8, bytes: &[u8]) -> Result<Vec<u8>, FileError> {
518 match codec {
519 CODEC_NONE => Ok(bytes.to_vec()),
520 #[cfg(feature = "compression")]
529 CODEC_ZSTD => zstd::decode_all(bytes).map_err(FileError::Decompress),
530 #[cfg(not(feature = "compression"))]
535 CODEC_ZSTD => {
536 use std::io::Read;
537 let mut dec = ruzstd::decoding::StreamingDecoder::new(bytes)
538 .map_err(|e| FileError::Decompress(std::io::Error::other(e.to_string())))?;
539 let mut out = Vec::new();
540 dec.read_to_end(&mut out).map_err(FileError::Decompress)?;
541 Ok(out)
542 }
543 other => Err(FileError::UnknownCodec(other)),
544 }
545}
546
547pub(crate) fn zstd_frame_content_size(b: &[u8]) -> Option<u64> {
551 if b.len() < 5 || b[..4] != [0x28, 0xB5, 0x2F, 0xFD] {
552 return None;
553 }
554 let fhd = b[4];
555 let fcs_flag = fhd >> 6;
556 let single_segment = fhd & 0x20 != 0;
557 let did_size = [0usize, 1, 2, 4][(fhd & 0x03) as usize];
558 let mut pos = 5 + usize::from(!single_segment) + did_size;
559 let fcs_size = match (fcs_flag, single_segment) {
560 (0, false) => return None,
561 (0, true) => 1,
562 (1, _) => 2,
563 (2, _) => 4,
564 _ => 8,
565 };
566 let f = b.get(pos..pos + fcs_size)?;
567 pos += fcs_size;
568 let _ = pos;
569 let mut v = 0u64;
570 for (i, x) in f.iter().enumerate() {
571 v |= (*x as u64) << (8 * i);
572 }
573 Some(if fcs_size == 2 { v + 256 } else { v })
574}
575
576pub(crate) fn decompress_bounded(codec: u8, bytes: &[u8], max: u64) -> Result<Vec<u8>, FileError> {
582 let too_big = |n: u64| {
583 FileError::Decompress(std::io::Error::other(format!(
584 "a block of {n} bytes exceeds its section's {max}-byte maximum"
585 )))
586 };
587 match codec {
588 CODEC_NONE => {
589 if bytes.len() as u64 > max {
590 return Err(too_big(bytes.len() as u64));
591 }
592 Ok(bytes.to_vec())
593 }
594 CODEC_ZSTD => {
595 let declared = zstd_frame_content_size(bytes);
596 if let Some(n) = declared.filter(|&n| n > max) {
597 return Err(too_big(n));
598 }
599 #[cfg(feature = "compression")]
600 {
601 if let Some(n) = declared {
602 return zstd::bulk::decompress(bytes, n as usize)
605 .map_err(FileError::Decompress);
606 }
607 use std::io::Read;
608 let mut out = Vec::new();
609 let mut dec = zstd::stream::read::Decoder::with_buffer(bytes)
610 .map_err(FileError::Decompress)?;
611 dec.window_log_max(27).map_err(FileError::Decompress)?;
612 dec.take(max.saturating_add(1))
613 .read_to_end(&mut out)
614 .map_err(FileError::Decompress)?;
615 if out.len() as u64 > max {
616 return Err(too_big(out.len() as u64));
617 }
618 Ok(out)
619 }
620 #[cfg(not(feature = "compression"))]
621 {
622 use std::io::Read;
623 let dec = ruzstd::decoding::StreamingDecoder::new(bytes)
624 .map_err(|e| FileError::Decompress(std::io::Error::other(e.to_string())))?;
625 let mut out = Vec::new();
626 dec.take(max.saturating_add(1))
627 .read_to_end(&mut out)
628 .map_err(FileError::Decompress)?;
629 if out.len() as u64 > max {
630 return Err(too_big(out.len() as u64));
631 }
632 Ok(out)
633 }
634 }
635 other => Err(FileError::UnknownCodec(other)),
636 }
637}
638
639pub(crate) fn block_raw_len(codec: u8, bytes: &[u8], ceiling: u64) -> Result<u64, FileError> {
643 match codec {
644 CODEC_NONE => Ok(bytes.len() as u64),
645 CODEC_ZSTD => match zstd_frame_content_size(bytes) {
646 Some(n) => Ok(n),
647 None => Ok(decompress_bounded(codec, bytes, ceiling)?.len() as u64),
648 },
649 other => Err(FileError::UnknownCodec(other)),
650 }
651}
652
653const TILE_COALESCE_GAP: u64 = 4096;
657
658const DICT_COALESCE_GAP: u64 = 64 * 1024;
664
665pub const DEFAULT_EXPORT_BUDGET_MB: u64 = 4096;
672
673pub const DEFAULT_QUERY_BUDGET_MB: u64 = DEFAULT_EXPORT_BUDGET_MB;
685
686pub fn spill_threshold_for(budget: u64) -> u64 {
690 budget / 8
691}
692
693const MIN_DICT_CACHE_CAP: u64 = 64 << 20; const TRANSIENT_RESERVE_MAX: u64 = 128 << 20; #[derive(Debug, Clone, Copy)]
714pub struct MemoryBudget {
715 pub block_cache: u64,
717 pub dict_cache: u64,
719 pub tile_cache: u64,
721}
722
723pub fn split_memory_budget(total: u64) -> MemoryBudget {
734 if total == u64::MAX {
735 return MemoryBudget {
736 block_cache: u64::MAX,
737 dict_cache: u64::MAX,
738 tile_cache: u64::MAX,
739 };
740 }
741 let block_cache = (total / 8).min(crate::block_cache::DEFAULT_CACHE_CAP);
744 let tile_cache = (total / 16).min(256 << 20);
747 let reserve = (total / 32).min(TRANSIENT_RESERVE_MAX);
753 let dict_cache = total
756 .saturating_sub(block_cache)
757 .saturating_sub(tile_cache)
758 .saturating_sub(reserve)
759 .max(MIN_DICT_CACHE_CAP);
760 MemoryBudget {
761 block_cache,
762 dict_cache,
763 tile_cache,
764 }
765}
766
767#[derive(Debug, Clone, Copy, Default)]
770pub struct FixedOverhead {
771 pub dict_directory: u64,
773 pub index_directory: u64,
775 pub named_graphs: u64,
777}
778
779impl FixedOverhead {
780 pub fn total(&self) -> u64 {
782 self.dict_directory + self.index_directory + self.named_graphs
783 }
784}
785
786fn fmt_cap(cap: u64) -> String {
788 if cap == u64::MAX {
789 "unlimited".to_string()
790 } else {
791 format!("{}MiB", cap >> 20)
792 }
793}
794
795pub(crate) fn read_coalesced<R: RangeReader + ?Sized>(
801 reader: &R,
802 ranges: &[ByteRange],
803 gap: u64,
804) -> Option<Vec<Vec<u8>>> {
805 coalesced(reader, ranges, gap, false)
806}
807
808pub(crate) fn read_coalesced_scattered<R: RangeReader + ?Sized>(
812 reader: &R,
813 ranges: &[ByteRange],
814 gap: u64,
815) -> Option<Vec<Vec<u8>>> {
816 coalesced(reader, ranges, gap, true)
817}
818
819fn coalesced<R: RangeReader + ?Sized>(
820 reader: &R,
821 ranges: &[ByteRange],
822 gap: u64,
823 scattered: bool,
824) -> Option<Vec<Vec<u8>>> {
825 let mut spans: Vec<(u64, u64)> = Vec::new();
828 let mut span_of: Vec<usize> = Vec::with_capacity(ranges.len());
829 let mut i = 0;
830 while i < ranges.len() {
831 let start = ranges[i].offset;
832 let mut end = ranges[i].offset.checked_add(ranges[i].len)?;
833 let mut j = i + 1;
834 while j < ranges.len() {
835 let r = &ranges[j];
836 if r.offset < end || r.offset - end > gap {
837 break;
838 }
839 end = r.offset.checked_add(r.len)?;
840 j += 1;
841 }
842 let si = spans.len();
843 spans.push((start, end - start));
844 for _ in i..j {
845 span_of.push(si);
846 }
847 i = j;
848 }
849 let blobs = if scattered {
850 reader.read_many_scattered(&spans)
851 } else {
852 reader.read_many(&spans)
853 }
854 .ok()?;
855 if blobs.len() != spans.len() {
856 return None;
857 }
858 let mut out = Vec::with_capacity(ranges.len());
859 for (k, r) in ranges.iter().enumerate() {
860 let (span_start, _) = spans[span_of[k]];
861 let blob = &blobs[span_of[k]];
862 out.push(checked_slice(blob, r.offset - span_start, r.len)?.to_vec());
863 }
864 Some(out)
865}
866
867fn content_hash(parts: &[&[u8]]) -> [u8; 16] {
870 let mut h = blake3::Hasher::new();
871 for p in parts {
872 h.update(p);
873 }
874 let mut out = [0u8; 16];
875 out.copy_from_slice(&h.finalize().as_bytes()[..16]);
876 out
877}
878
879pub type TermTriple = (String, String, String);
881
882#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct DumpPlan {
886 pub scan: Option<crate::index::ScanPlan>,
890 pub dictionary_bytes: u64,
899}
900
901#[derive(Debug, Clone)]
905pub struct LayoutSegment {
906 pub kind: &'static str,
907 pub label: String,
908 pub offset: u64,
909 pub len: u64,
910}
911
912#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
914pub struct ByteRange {
915 pub offset: u64,
916 pub len: u64,
917}
918
919impl ByteRange {
920 pub fn end(self) -> u64 {
921 self.offset + self.len
922 }
923}
924
925#[derive(Debug, Clone, PartialEq, Eq)]
927pub struct TripleProvenance {
928 pub terms: TermTriple,
930 pub ids: Triple,
932 pub graph: Option<String>,
934 pub matched_pattern: Pattern,
936 pub index_permutation: IndexPermutation,
938 pub dictionary_range: ByteRange,
940 pub index_range: ByteRange,
942 pub index_section_range: ByteRange,
945 pub pyramid_range: Option<ByteRange>,
947 pub tile: Option<String>,
951 pub tile_range: Option<ByteRange>,
954}
955
956fn encode_container(sections: &[&[u8]], codec: u8) -> Vec<u8> {
960 let mut out = Vec::new();
961 write_uvarint(&mut out, sections.len() as u64);
962 for s in sections {
963 let payload = compress(codec, s);
964 write_uvarint(&mut out, payload.len() as u64);
965 out.extend_from_slice(&payload);
966 }
967 out
968}
969
970fn decode_container(bytes: &[u8], codec: u8) -> Result<Vec<Vec<u8>>, FileError> {
972 let (n, mut pos) = read_uvarint(bytes).ok_or(FileError::Container("truncated count"))?;
973 let n = wire_usize(n, "section count too large")?;
977 let mut out = Vec::with_capacity(n.min(bytes.len()));
978 for _ in 0..n {
979 let (len, used) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
980 .ok_or(FileError::Container("truncated length"))?;
981 pos += used;
982 let end = pos
985 .checked_add(wire_usize(len, "section length too large")?)
986 .filter(|&end| end <= bytes.len())
987 .ok_or(FileError::Container("section overruns buffer"))?;
988 out.push(decompress(codec, &bytes[pos..end])?);
989 pos = end;
990 }
991 Ok(out)
992}
993
994fn checked_end(off: u64, len: u64) -> Result<u64, FileError> {
995 off.checked_add(len)
996 .ok_or(FileError::Container("section range overflows"))
997}
998
999fn wire_usize(v: u64, what: &'static str) -> Result<usize, FileError> {
1003 usize::try_from(v).map_err(|_| FileError::Container(what))
1004}
1005
1006fn checked_slice(bytes: &[u8], off: u64, len: u64) -> Option<&[u8]> {
1009 let start = usize::try_from(off).ok()?;
1010 let end = start.checked_add(usize::try_from(len).ok()?)?;
1011 bytes.get(start..end)
1012}
1013
1014const DICT_CHUNK_BUDGET: usize = crate::write_opts::DEFAULT_DICT_CHUNK_BYTES;
1017
1018fn encode_chunked_dict_section(raw: &[u8], codec: u8) -> Vec<u8> {
1035 let meta = crate::dict::parse_meta(raw).unwrap_or(crate::dict::SectionMeta {
1036 term_count: 0,
1037 restart_interval: 1,
1038 restart_offsets: Vec::new(),
1039 });
1040 let body_start = meta
1041 .restart_offsets
1042 .first()
1043 .copied()
1044 .unwrap_or(raw.len() as u64);
1045 let header = &raw[..(body_start.min(raw.len() as u64)) as usize];
1046
1047 let chunk_budget = crate::write_opts::dict_chunk_budget() as u64;
1049 let n_runs = meta.restart_offsets.len();
1050 let mut bounds: Vec<(usize, u64, u64)> = Vec::new(); let mut r = 0;
1052 while r < n_runs {
1053 let start = meta.restart_offsets[r];
1054 let mut r2 = r + 1;
1055 while r2 < n_runs && meta.restart_offsets[r2] - start < chunk_budget {
1056 r2 += 1;
1057 }
1058 let end = if r2 < n_runs {
1059 meta.restart_offsets[r2]
1060 } else {
1061 raw.len() as u64
1062 };
1063 bounds.push((r, start, end));
1064 r = r2;
1065 }
1066
1067 #[cfg(feature = "parallel")]
1071 let compressed: Vec<Vec<u8>> = {
1072 use rayon::prelude::*;
1073 bounds
1074 .par_iter()
1075 .map(|&(_, s, e)| compress(codec, &raw[s as usize..e as usize]))
1076 .collect()
1077 };
1078 #[cfg(not(feature = "parallel"))]
1079 let compressed: Vec<Vec<u8>> = bounds
1080 .iter()
1081 .map(|&(_, s, e)| compress(codec, &raw[s as usize..e as usize]))
1082 .collect();
1083 let mut out = Vec::new();
1084 write_uvarint(&mut out, header.len() as u64);
1085 out.extend_from_slice(header);
1086 write_uvarint(&mut out, bounds.len() as u64);
1087 let mut prev_run = 0usize;
1088 let mut prev_last: Option<Vec<u8>> = None;
1092 for (i, (&(first_run, start, end), comp)) in bounds.iter().zip(&compressed).enumerate() {
1093 let key = if i == 0 {
1094 Vec::new()
1096 } else {
1097 let first = crate::dict::run_first_term(raw, start as usize).unwrap_or_default();
1098 match &prev_last {
1099 None => first,
1102 Some(pl) => crate::dict::shortest_separator(pl, &first),
1103 }
1104 };
1105 if i + 1 < bounds.len() {
1106 let last_run_off = meta.restart_offsets[bounds[i + 1].0 - 1] as usize;
1108 prev_last = crate::dict::run_last_term(raw, last_run_off, end as usize);
1109 }
1110 write_uvarint(&mut out, (first_run - prev_run) as u64);
1111 write_uvarint(&mut out, key.len() as u64);
1112 out.extend_from_slice(&key);
1113 write_uvarint(&mut out, comp.len() as u64);
1114 prev_run = first_run;
1115 }
1116 for comp in &compressed {
1117 out.extend_from_slice(comp);
1118 }
1119 out
1120}
1121
1122pub(crate) struct DictChunkEntry {
1126 pub(crate) first_run: usize,
1127 pub(crate) key: Vec<u8>,
1128 body_start: u64,
1129 pub(crate) start: u64,
1130 pub(crate) end: u64,
1131}
1132
1133fn parse_chunked_dict_dir(
1137 bytes: &[u8],
1138 total_len: u64,
1139) -> Result<(crate::dict::SectionMeta, Vec<DictChunkEntry>), FileError> {
1140 let mut pos = 0usize;
1141 let take = |pos: &mut usize| -> Result<u64, FileError> {
1142 let (v, n) = read_uvarint(bytes.get(*pos..).unwrap_or(&[]))
1143 .ok_or(FileError::Container("truncated dict chunk directory"))?;
1144 *pos += n;
1145 Ok(v)
1146 };
1147 let header_len = wire_usize(take(&mut pos)?, "dict header too large")?;
1148 let header = bytes
1149 .get(pos..pos.saturating_add(header_len))
1150 .ok_or(FileError::Container("truncated dict header"))?;
1151 let meta = crate::dict::parse_meta(header)
1152 .map_err(|_| FileError::Container("malformed dict header"))?;
1153 pos += header_len;
1154
1155 let num_chunks = wire_usize(take(&mut pos)?, "dict chunk count too large")?;
1156 let mut entries = Vec::with_capacity(num_chunks.min(bytes.len()));
1157 let mut lens = Vec::with_capacity(num_chunks.min(bytes.len()));
1158 let mut prev_run = 0usize;
1159 for i in 0..num_chunks {
1160 let drun = wire_usize(take(&mut pos)?, "dict chunk run too large")?;
1161 let klen = wire_usize(take(&mut pos)?, "dict chunk key too large")?;
1162 let key = bytes
1163 .get(pos..pos.saturating_add(klen))
1164 .ok_or(FileError::Container("truncated dict chunk key"))?
1165 .to_vec();
1166 pos += klen;
1167 let clen = take(&mut pos)?;
1168 let first_run = next_chunk_run(i, prev_run, drun)?;
1169 let body_start = meta
1170 .restart_offsets
1171 .get(first_run)
1172 .copied()
1173 .ok_or(FileError::Container("dict chunk run out of range"))?;
1174 entries.push(DictChunkEntry {
1175 first_run,
1176 key,
1177 body_start,
1178 start: 0,
1179 end: 0,
1180 });
1181 lens.push(clen);
1182 prev_run = first_run;
1183 }
1184 let mut start = pos as u64;
1185 for (e, len) in entries.iter_mut().zip(lens) {
1186 let end = start
1187 .checked_add(len)
1188 .filter(|&e| e <= total_len)
1189 .ok_or(FileError::Container("dict chunk overruns section"))?;
1190 e.start = start;
1191 e.end = end;
1192 start = end;
1193 }
1194 Ok((meta, entries))
1195}
1196
1197fn next_chunk_run(i: usize, prev_run: usize, drun: usize) -> Result<usize, FileError> {
1204 match (i, drun) {
1205 (0, 0) => Ok(0),
1206 (0, _) => Err(FileError::Container(
1207 "first dict chunk does not start at run 0",
1208 )),
1209 (_, 0) => Err(FileError::Container("dict chunk runs do not ascend")),
1210 _ => prev_run
1211 .checked_add(drun)
1212 .ok_or(FileError::Container("dict chunk run overflows")),
1213 }
1214}
1215
1216pub(crate) fn read_dict_dir_ranged<R: RangeReader + ?Sized>(
1228 reader: &R,
1229 section: ByteRange,
1230) -> Result<(crate::dict::SectionMeta, Vec<DictChunkEntry>), FileError> {
1231 let total = section.len;
1232 let init = 8192.min(total); let head = reader.read_at(section.offset, init)?;
1240 let (header_len, n0) =
1241 read_uvarint(&head).ok_or(FileError::Container("truncated dict header len"))?;
1242 let hbase = n0; let (term_count, n1) = read_uvarint(head.get(hbase..).unwrap_or(&[]))
1244 .ok_or(FileError::Container("truncated dict term_count"))?;
1245 let (restart_interval, _n2) = read_uvarint(head.get(hbase + n1..).unwrap_or(&[]))
1246 .ok_or(FileError::Container("truncated dict interval"))?;
1247 if restart_interval == 0 {
1248 return Err(FileError::Container("zero restart interval"));
1249 }
1250 let dir_start = (hbase as u64)
1253 .checked_add(header_len)
1254 .filter(|&d| d <= total)
1255 .ok_or(FileError::Container("dict header overruns section"))?;
1256 let dir_total = total - dir_start;
1257 let meta = crate::dict::SectionMeta {
1258 term_count: u32::try_from(term_count)
1259 .map_err(|_| FileError::Container("dict term count too large"))?,
1260 restart_interval: u32::try_from(restart_interval)
1261 .map_err(|_| FileError::Container("dict restart interval too large"))?,
1262 restart_offsets: Vec::new(),
1263 };
1264 let num_runs = (meta.term_count as usize).div_ceil(meta.restart_interval as usize);
1266 let finish = |mut entries: Vec<DictChunkEntry>| {
1267 for e in &mut entries {
1268 e.start += dir_start; e.end += dir_start;
1270 }
1271 (meta.clone(), entries)
1272 };
1273 let mut have: Vec<u8> = Vec::new();
1277 if dir_start < head.len() as u64 {
1278 match parse_chunk_dir_only(&head[dir_start as usize..], dir_total, num_runs)? {
1279 ChunkDirParse::Done(entries) => return Ok(finish(entries)),
1280 ChunkDirParse::Truncated { .. } => have = head[dir_start as usize..].to_vec(),
1281 }
1282 }
1283 let mut want = 4096u64.min(dir_total).max(1);
1296 loop {
1297 let held = have.len() as u64;
1298 if want > held {
1299 let extra = reader.read_at(section.offset + dir_start + held, want - held)?;
1300 if extra.is_empty() {
1301 return Err(FileError::Container("truncated dict chunk directory"));
1302 }
1303 have.extend_from_slice(&extra);
1304 }
1305 let held = have.len() as u64;
1306 match parse_chunk_dir_only(&have, dir_total, num_runs)? {
1307 ChunkDirParse::Done(entries) => return Ok(finish(entries)),
1308 ChunkDirParse::Truncated { .. } if held >= dir_total => {
1309 return Err(FileError::Container("truncated dict chunk directory"));
1310 }
1311 ChunkDirParse::Truncated {
1312 parsed,
1313 used,
1314 total,
1315 } => {
1316 let est = if parsed >= 16 && used > 0 {
1321 let whole = ((used as u64) / (parsed as u64)).saturating_mul(total as u64);
1322 whole.saturating_add(whole / 8).saturating_add(64)
1323 } else {
1324 held.saturating_mul(2)
1326 };
1327 want = est
1328 .max(held.saturating_add(1))
1329 .min(held.saturating_mul(4))
1330 .min(dir_total);
1331 }
1332 }
1333 }
1334}
1335
1336enum ChunkDirParse {
1341 Done(Vec<DictChunkEntry>),
1342 Truncated {
1343 parsed: usize,
1345 used: usize,
1347 total: usize,
1349 },
1350}
1351
1352fn parse_chunk_dir_only(
1362 dir: &[u8],
1363 dir_total: u64,
1364 num_runs: usize,
1365) -> Result<ChunkDirParse, FileError> {
1366 let mut pos = 0usize;
1367 let take = |pos: &mut usize| -> Option<u64> {
1368 let (v, n) = read_uvarint(dir.get(*pos..).unwrap_or(&[]))?;
1369 *pos += n;
1370 Some(v)
1371 };
1372 let Some(num_chunks) = take(&mut pos) else {
1373 return Ok(ChunkDirParse::Truncated {
1374 parsed: 0,
1375 used: 0,
1376 total: 0,
1377 });
1378 };
1379 let num_chunks = wire_usize(num_chunks, "dict chunk count too large")?;
1380 let mut entries = Vec::with_capacity(num_chunks.min(dir.len()));
1381 let mut lens: Vec<u64> = Vec::with_capacity(num_chunks.min(dir.len()));
1382 let mut prev_run = 0usize;
1383 for i in 0..num_chunks {
1384 let entry_start = pos;
1385 let short = ChunkDirParse::Truncated {
1386 parsed: entries.len(),
1387 used: entry_start,
1388 total: num_chunks,
1389 };
1390 let (Some(drun), Some(klen)) = (take(&mut pos), take(&mut pos)) else {
1391 return Ok(short);
1392 };
1393 let drun = wire_usize(drun, "dict chunk run too large")?;
1394 let klen = wire_usize(klen, "dict chunk key too large")?;
1395 let Some(key) = dir.get(pos..pos.saturating_add(klen)) else {
1396 return Ok(short);
1397 };
1398 let key = key.to_vec();
1399 pos += klen;
1400 let Some(clen) = take(&mut pos) else {
1401 return Ok(short);
1402 };
1403 let first_run = next_chunk_run(i, prev_run, drun)?;
1404 if first_run >= num_runs {
1405 return Err(FileError::Container("dict chunk run out of range"));
1406 }
1407 entries.push(DictChunkEntry {
1408 first_run,
1409 key,
1410 body_start: 0,
1411 start: 0,
1412 end: 0,
1413 });
1414 lens.push(clen);
1415 prev_run = first_run;
1416 }
1417 let mut start = pos as u64;
1418 for (e, len) in entries.iter_mut().zip(lens) {
1419 let end = start
1420 .checked_add(len)
1421 .filter(|&e| e <= dir_total)
1422 .ok_or(FileError::Container("dict chunk overruns section"))?;
1423 e.start = start;
1424 e.end = end;
1425 start = end;
1426 }
1427 Ok(ChunkDirParse::Done(entries))
1428}
1429
1430fn decode_chunked_dict_section(
1434 payload: &[u8],
1435 codec: u8,
1436 cache: std::sync::Arc<crate::chunk_cache::ChunkCache>,
1437 section_index: u8,
1438) -> Result<crate::dict::ChunkedSection, FileError> {
1439 let (meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64)?;
1440 let chunks = entries
1441 .into_iter()
1442 .map(|e| {
1443 let body = decompress(codec, &payload[e.start as usize..e.end as usize])?;
1444 Ok((
1445 crate::dict::SectionChunk::new(e.first_run, e.key, e.body_start),
1446 body,
1447 ))
1448 })
1449 .collect::<Result<Vec<_>, FileError>>()?;
1450 Ok(crate::dict::ChunkedSection::resident(
1451 meta,
1452 chunks,
1453 cache,
1454 section_index,
1455 ))
1456}
1457
1458fn decode_dictionary_container(bytes: &[u8], codec: u8) -> Result<Dictionary, FileError> {
1459 let dsecs = decode_container(bytes, CODEC_NONE)?;
1460 if dsecs.len() != 4 {
1461 return Err(FileError::Container("expected 4 dictionary sections"));
1462 }
1463 let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
1466 let mut sections = Vec::with_capacity(4);
1467 for (si, sec) in dsecs.iter().enumerate() {
1468 sections.push(decode_chunked_dict_section(
1469 sec,
1470 codec,
1471 cache.clone(),
1472 si as u8,
1473 )?);
1474 }
1475 let arr: [crate::dict::ChunkedSection; 4] = sections
1476 .try_into()
1477 .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
1478 Ok(Dictionary::from_chunked_sections(arr))
1479}
1480
1481fn encode_tiled_section(index: &GraphIndex, si: usize, codec: u8) -> Vec<u8> {
1487 let tiles = index.tile_sections()[si];
1488 let n = tiles.len();
1489 #[cfg(feature = "parallel")]
1496 let compressed: Vec<Vec<u8>> = {
1497 use rayon::prelude::*;
1498 (0..n)
1499 .into_par_iter()
1500 .map(|ti| compress(codec, &index.tile_body(si, ti)))
1501 .collect()
1502 };
1503 #[cfg(not(feature = "parallel"))]
1504 let compressed: Vec<Vec<u8>> = (0..n)
1505 .map(|ti| compress(codec, &index.tile_body(si, ti)))
1506 .collect();
1507 let mut out = Vec::new();
1508 write_uvarint(&mut out, tiles.len() as u64);
1509 let mut prev_min = 0u32;
1510 for (tile, comp) in tiles.iter().zip(&compressed) {
1511 let (min_a, max_a) = tile.leading_range();
1512 write_uvarint(&mut out, (min_a - prev_min) as u64);
1513 write_uvarint(&mut out, (max_a - min_a) as u64);
1514 write_uvarint(&mut out, comp.len() as u64);
1515 prev_min = min_a;
1516 }
1517 for comp in &compressed {
1518 out.extend_from_slice(comp);
1519 }
1520 for ti in 0..n {
1528 let body = index.tile_body(si, ti);
1529 let (min_b, max_b, min_c, max_c) = match crate::triples::TripleBlock::parse(&body) {
1530 Ok(b) => {
1531 let z = b.zone();
1532 (z.min_b, z.max_b, z.min_c, z.max_c)
1533 }
1534 Err(_) => (0, u32::MAX, 0, u32::MAX),
1535 };
1536 write_uvarint(&mut out, min_b as u64);
1537 write_uvarint(&mut out, (max_b - min_b) as u64);
1538 write_uvarint(&mut out, min_c as u64);
1539 write_uvarint(&mut out, (max_c - min_c) as u64);
1540 }
1541 out
1542}
1543
1544pub(crate) struct TileDirEntry {
1547 pub(crate) min_a: u32,
1548 pub(crate) max_a: u32,
1549 pub(crate) start: u64,
1550 pub(crate) end: u64,
1551}
1552
1553type TileSynopsis = (u32, u32, u32, u32);
1555
1556fn parse_tile_synopsis(
1564 payload: &[u8],
1565 trailer_start: usize,
1566 num_tiles: usize,
1567) -> Option<Vec<TileSynopsis>> {
1568 let mut pos = trailer_start;
1569 let take = |pos: &mut usize| -> Option<u32> {
1570 let (v, n) = read_uvarint(payload.get(*pos..)?)?;
1571 *pos += n;
1572 u32::try_from(v).ok()
1573 };
1574 let mut out = Vec::with_capacity(num_tiles.min(payload.len()));
1575 for _ in 0..num_tiles {
1576 let min_b = take(&mut pos)?;
1577 let max_b = min_b.checked_add(take(&mut pos)?)?;
1578 let min_c = take(&mut pos)?;
1579 let max_c = min_c.checked_add(take(&mut pos)?)?;
1580 out.push((min_b, max_b, min_c, max_c));
1581 }
1582 Some(out)
1583}
1584
1585fn parse_tile_directory(bytes: &[u8], total_len: u64) -> Result<Vec<TileDirEntry>, FileError> {
1590 let mut pos = 0usize;
1591 let take = |pos: &mut usize| -> Result<u64, FileError> {
1592 let (v, n) = read_uvarint(bytes.get(*pos..).unwrap_or(&[]))
1593 .ok_or(FileError::Container("truncated tile directory"))?;
1594 *pos += n;
1595 Ok(v)
1596 };
1597 let num_tiles = wire_usize(take(&mut pos)?, "tile count too large")?;
1598 let mut entries = Vec::with_capacity(num_tiles.min(bytes.len()));
1599 let mut prev_min = 0u32;
1600 let mut lens = Vec::with_capacity(num_tiles.min(bytes.len()));
1601 let id = |v: u64| u32::try_from(v).map_err(|_| FileError::Container("tile id out of range"));
1605 for _ in 0..num_tiles {
1606 let dmin = id(take(&mut pos)?)?;
1607 let span = id(take(&mut pos)?)?;
1608 let len = take(&mut pos)?;
1609 let min_a = prev_min
1610 .checked_add(dmin)
1611 .ok_or(FileError::Container("tile id out of range"))?;
1612 let max_a = min_a
1613 .checked_add(span)
1614 .ok_or(FileError::Container("tile id out of range"))?;
1615 entries.push(TileDirEntry {
1616 min_a,
1617 max_a,
1618 start: 0,
1619 end: 0,
1620 });
1621 lens.push(len);
1622 prev_min = min_a;
1623 }
1624 let mut start = pos as u64;
1625 for (e, len) in entries.iter_mut().zip(lens) {
1626 let end = start
1627 .checked_add(len)
1628 .filter(|&e| e <= total_len)
1629 .ok_or(FileError::Container("tile overruns section"))?;
1630 e.start = start;
1631 e.end = end;
1632 start = end;
1633 }
1634 Ok(entries)
1635}
1636
1637pub(crate) fn read_tile_directory_ranged<R: RangeReader + ?Sized>(
1641 reader: &R,
1642 section: ByteRange,
1643) -> Result<Vec<TileDirEntry>, FileError> {
1644 let total = section.len;
1645 let mut prefetch = 4096u64.min(total);
1646 loop {
1647 let prefix = reader.read_at(section.offset, prefetch)?;
1648 match parse_tile_directory(&prefix, total) {
1649 Ok(dir) => return Ok(dir),
1650 Err(_) if prefetch < total => prefetch = prefetch.saturating_mul(2).min(total),
1651 Err(e) => return Err(e),
1652 }
1653 }
1654}
1655
1656pub(crate) fn read_tile_synopsis_ranged<R: RangeReader + ?Sized>(
1663 reader: &R,
1664 section: ByteRange,
1665 dir: &[TileDirEntry],
1666) -> Vec<Option<TileSynopsis>> {
1667 let n = dir.len();
1668 let none = vec![None; n];
1669 let trailer_start = dir.iter().map(|e| e.end).max().unwrap_or(0);
1670 let total = section.len;
1671 if n == 0 || trailer_start >= total {
1672 return none; }
1674 let max_trailer = (n as u64).saturating_mul(4 * 5);
1678 let trailer_len = (total - trailer_start).min(max_trailer);
1679 let Some(offset) = section.offset.checked_add(trailer_start) else {
1680 return none;
1681 };
1682 let Ok(bytes) = reader.read_at(offset, trailer_len) else {
1683 return none;
1684 };
1685 match parse_tile_synopsis(&bytes, 0, n) {
1686 Some(v) => v.into_iter().map(Some).collect(),
1687 None => none,
1688 }
1689}
1690
1691fn tile_file_ranges(
1695 index_bytes: &[u8],
1696 container_offset: u64,
1697 section_ranges: &[ByteRange; NUM_PERMS],
1698) -> [Vec<(u32, u32, ByteRange)>; NUM_PERMS] {
1699 let mut out: [Vec<(u32, u32, ByteRange)>; NUM_PERMS] = Default::default();
1700 for (section, range) in out.iter_mut().zip(section_ranges) {
1701 if range.len == 0 || range.offset < container_offset {
1705 continue;
1706 }
1707 let Some(payload) = checked_slice(index_bytes, range.offset - container_offset, range.len)
1708 else {
1709 continue;
1710 };
1711 if let Ok(dir) = parse_tile_directory(payload, payload.len() as u64) {
1712 *section = dir
1713 .into_iter()
1714 .map(|e| {
1715 (
1716 e.min_a,
1717 e.max_a,
1718 ByteRange {
1719 offset: range.offset + e.start,
1720 len: (e.end - e.start),
1721 },
1722 )
1723 })
1724 .collect();
1725 }
1726 }
1727 out
1728}
1729
1730fn decode_tiled_section(payload: &[u8], codec: u8) -> Result<Vec<(u32, u32, Vec<u8>)>, FileError> {
1733 parse_tile_directory(payload, payload.len() as u64)?
1734 .into_iter()
1735 .map(|e| {
1736 Ok((
1737 e.min_a,
1738 e.max_a,
1739 decompress(codec, &payload[e.start as usize..e.end as usize])?,
1740 ))
1741 })
1742 .collect()
1743}
1744
1745fn decode_index_container(
1755 bytes: &[u8],
1756 codec: u8,
1757 perms: PermSet,
1758) -> Result<GraphIndex, FileError> {
1759 let isecs = decode_container(bytes, CODEC_NONE)?;
1760 if isecs.len() != perms.len() {
1761 return Err(FileError::Container(
1762 "index container section count does not match the header permutation mask",
1763 ));
1764 }
1765 decode_index_sections(isecs, codec, perms)
1766}
1767
1768fn decode_index_sections(
1769 mut isecs: Vec<Vec<u8>>,
1770 codec: u8,
1771 perms: PermSet,
1772) -> Result<GraphIndex, FileError> {
1773 let mut sections: [Vec<(u32, u32, Vec<u8>)>; NUM_PERMS] = Default::default();
1774 for (perm, sec) in perms.iter().zip(isecs.iter_mut()) {
1775 sections[perm.section_index()] = decode_tiled_section(sec, codec)?;
1776 }
1777 Ok(GraphIndex::from_tiles(sections, perms))
1778}
1779
1780fn decode_root_index_container(
1785 bytes: &[u8],
1786 header: &Header,
1787) -> Result<(GraphIndex, Option<[u8; 16]>), FileError> {
1788 if !header.has_quad_index() {
1789 return Ok((
1790 decode_index_container(bytes, header.block_codec, header.perms)?,
1791 None,
1792 ));
1793 }
1794 let mut isecs = decode_container(bytes, CODEC_NONE)?;
1795 if isecs.len() != header.index_container_sections() {
1796 return Err(FileError::Container(
1797 "index container section count does not match the header permutation mask",
1798 ));
1799 }
1800 let seal = isecs.pop().unwrap_or_default();
1801 let digest = crate::quad::parse_seal(&seal)?;
1802 Ok((
1803 decode_index_sections(isecs, header.block_codec, header.perms)?,
1804 Some(digest),
1805 ))
1806}
1807
1808fn check_named_layout(header: &Header) -> Result<(), FileError> {
1813 let quad = header.quad_index_section().is_some();
1814 if header.has_quad_index() {
1815 if !quad {
1816 return Err(FileError::Container(
1817 "quad-index flag set but the file has no quad-index section",
1818 ));
1819 }
1820 if header.named_graphs_len > 0 {
1821 return Err(FileError::Container(
1822 "a file cannot carry both a quad index and a named-graphs section",
1823 ));
1824 }
1825 crate::quad::check_perms(header.perms)?;
1826 } else if quad {
1827 return Err(FileError::Container(
1828 "quad-index section present but the header does not flag it",
1829 ));
1830 }
1831 Ok(())
1832}
1833
1834fn container_section_payload_ranges(
1835 bytes: &[u8],
1836 container_offset: u64,
1837 expected_sections: usize,
1838) -> Result<Vec<ByteRange>, FileError> {
1839 let (section_count, mut pos) =
1840 read_uvarint(bytes).ok_or(FileError::Container("truncated count"))?;
1841 let section_count = usize::try_from(section_count)
1842 .map_err(|_| FileError::Container("section count too large"))?;
1843 if section_count != expected_sections {
1844 return Err(FileError::Container("unexpected section count"));
1845 }
1846
1847 let mut ranges = Vec::with_capacity(section_count);
1848 for _ in 0..section_count {
1849 let remaining = bytes
1850 .get(pos..)
1851 .ok_or(FileError::Container("truncated length"))?;
1852 let (payload_len, used) =
1853 read_uvarint(remaining).ok_or(FileError::Container("truncated length"))?;
1854 pos = pos
1855 .checked_add(used)
1856 .ok_or(FileError::Container("section range overflows"))?;
1857 let payload_len_usize = usize::try_from(payload_len)
1858 .map_err(|_| FileError::Container("section length too large"))?;
1859 let payload_end = pos
1860 .checked_add(payload_len_usize)
1861 .ok_or(FileError::Container("section range overflows"))?;
1862 if payload_end > bytes.len() {
1863 return Err(FileError::Container("section overruns buffer"));
1864 }
1865 ranges.push(ByteRange {
1866 offset: checked_end(container_offset, pos as u64)?,
1867 len: payload_len,
1868 });
1869 pos = payload_end;
1870 }
1871
1872 Ok(ranges)
1873}
1874
1875fn decode_index_section_ranges(
1879 bytes: &[u8],
1880 container_offset: u64,
1881 perms: PermSet,
1882 sections: usize,
1883) -> Result<[ByteRange; NUM_PERMS], FileError> {
1884 let ranges = container_section_payload_ranges(bytes, container_offset, sections)?;
1885 let mut out = [ByteRange { offset: 0, len: 0 }; NUM_PERMS];
1886 for (perm, range) in perms.iter().zip(ranges) {
1887 out[perm.section_index()] = range;
1888 }
1889 Ok(out)
1890}
1891
1892fn read_uvarint_at<R: RangeReader + ?Sized>(
1893 reader: &R,
1894 absolute_offset: u64,
1895 container_end: u64,
1896) -> Result<(u64, u64), FileError> {
1897 if absolute_offset >= container_end {
1898 return Err(FileError::Container("truncated container varint"));
1899 }
1900 let remaining = container_end - absolute_offset;
1901 let probe_len = remaining.min(10);
1902 let bytes = reader.read_at(absolute_offset, probe_len)?;
1903 read_uvarint(&bytes)
1904 .map(|(value, used)| (value, used as u64))
1905 .ok_or(FileError::Container("truncated container varint"))
1906}
1907
1908pub(crate) fn locate_container_section_ranged<R: RangeReader + ?Sized>(
1911 reader: &R,
1912 container_offset: u64,
1913 container_len: u64,
1914 section_index: usize,
1915 expected_sections: u64,
1916) -> Result<ByteRange, FileError> {
1917 let container_end = checked_end(container_offset, container_len)?;
1918 let (section_count, used) = read_uvarint_at(reader, container_offset, container_end)?;
1919 if section_count != expected_sections {
1920 return Err(FileError::Container("unexpected container section count"));
1921 }
1922 if section_index >= section_count as usize {
1923 return Err(FileError::Container(
1924 "container section index out of bounds",
1925 ));
1926 }
1927
1928 let mut pos = checked_end(container_offset, used)?;
1929 for i in 0..section_count as usize {
1930 let (payload_len, len_used) = read_uvarint_at(reader, pos, container_end)?;
1931 pos = checked_end(pos, len_used)?;
1932 let payload_end = checked_end(pos, payload_len)?;
1933 if payload_end > container_end {
1934 return Err(FileError::Container("section overruns buffer"));
1935 }
1936 if i == section_index {
1937 return Ok(ByteRange {
1938 offset: pos,
1939 len: payload_len,
1940 });
1941 }
1942 pos = payload_end;
1943 }
1944 Err(FileError::Container("container section not found"))
1945}
1946
1947#[allow(clippy::type_complexity)]
1957fn open_index_container_lazy(
1958 reader: &std::sync::Arc<dyn RangeReader + Send + Sync>,
1959 container: ByteRange,
1960 block_codec: u8,
1961 has_synopsis: bool,
1962 read_concurrency: usize,
1963 perms: PermSet,
1964 sections: usize,
1965) -> Result<
1966 (
1967 GraphIndex,
1968 [ByteRange; NUM_PERMS],
1969 [Vec<(u32, u32, ByteRange)>; NUM_PERMS],
1970 ),
1971 FileError,
1972> {
1973 let mut index_section_ranges = [ByteRange { offset: 0, len: 0 }; NUM_PERMS];
1974 let mut tile_ranges: [Vec<(u32, u32, ByteRange)>; NUM_PERMS] = Default::default();
1975 #[allow(clippy::type_complexity)]
1976 let mut directories: [Vec<(u32, u32, Option<TileSynopsis>)>; NUM_PERMS] = Default::default();
1977 for (pos, perm) in perms.iter().enumerate() {
1978 let si = perm.section_index();
1979 let section = locate_container_section_ranged(
1980 reader,
1981 container.offset,
1982 container.len,
1983 pos,
1984 sections as u64,
1985 )?;
1986 index_section_ranges[si] = section;
1987 let dir = read_tile_directory_ranged(reader, section)?;
1988 let syn = if has_synopsis {
1991 read_tile_synopsis_ranged(reader, section, &dir)
1992 } else {
1993 vec![None; dir.len()]
1994 };
1995 directories[si] = dir
1996 .iter()
1997 .zip(syn)
1998 .map(|(e, s)| (e.min_a, e.max_a, s))
1999 .collect();
2000 tile_ranges[si] = dir
2001 .into_iter()
2002 .map(|e| {
2003 (
2004 e.min_a,
2005 e.max_a,
2006 ByteRange {
2007 offset: section.offset + e.start,
2008 len: (e.end - e.start),
2009 },
2010 )
2011 })
2012 .collect();
2013 }
2014
2015 let codec = block_codec;
2025 let loader_ranges = tile_ranges.clone();
2026 let loader_reader = reader.clone();
2027 let loader: crate::index::TileLoader = Box::new(move |si, ti| {
2028 let (_, _, range) = loader_ranges.get(si)?.get(ti)?;
2029 let bytes = loader_reader.read_at(range.offset, range.len).ok()?;
2030 if bytes.len() as u64 != range.len {
2031 return None;
2032 }
2033 checked_tile(codec, &bytes)
2034 });
2035 let bulk_ranges = tile_ranges.clone();
2036 let bulk_reader = reader.clone();
2037 let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
2038 let section = bulk_ranges.get(si)?;
2039 let want: Option<Vec<ByteRange>> = tis
2040 .iter()
2041 .map(|&ti| section.get(ti).map(|&(_, _, r)| r))
2042 .collect();
2043 let blobs = read_coalesced(bulk_reader.as_ref(), &want?, TILE_COALESCE_GAP)?;
2044 blobs.iter().map(|b| checked_tile(codec, b)).collect()
2045 });
2046 let mut index =
2047 GraphIndex::from_remote_directories(directories, perms, loader).with_bulk_loader(bulk);
2048 index.set_tile_lens(std::array::from_fn(|si| {
2051 tile_ranges[si]
2052 .iter()
2053 .map(|&(_, _, r)| r.len.min(u32::MAX as u64) as u32)
2054 .collect()
2055 }));
2056 index.set_read_concurrency(read_concurrency);
2060 Ok((index, index_section_ranges, tile_ranges))
2061}
2062
2063fn checked_tile(codec: u8, bytes: &[u8]) -> Option<Vec<u8>> {
2067 let tile = decompress(codec, bytes).ok()?;
2068 crate::triples::TripleBlock::parse(&tile).ok()?;
2069 Some(tile)
2070}
2071
2072pub fn write_file(
2079 dict: &Dictionary,
2080 index: &GraphIndex,
2081 has_quads: bool,
2082 pyramid_meta: &[u8],
2083 pyramid_levels: u16,
2084) -> Vec<u8> {
2085 write_dataset(dict, index, &[], has_quads, pyramid_meta, pyramid_levels)
2086}
2087
2088#[doc(hidden)]
2093pub fn write_file_v05(
2094 dict: &Dictionary,
2095 index: &GraphIndex,
2096 has_quads: bool,
2097 pyramid_meta: &[u8],
2098 pyramid_levels: u16,
2099) -> Vec<u8> {
2100 write_dataset_v05(dict, index, &[], has_quads, pyramid_meta, pyramid_levels)
2101}
2102
2103fn into_v06(image: Vec<u8>) -> Vec<u8> {
2106 crate::build_record::finish_in_ram(image, None)
2107 .unwrap_or_else(|e| panic!("writing format 0x06: {e}"))
2108}
2109
2110pub(crate) fn encode_index_container(index: &GraphIndex, codec: u8) -> Vec<u8> {
2116 encode_index_container_sealed(index, codec, None)
2117}
2118
2119fn encode_index_container_sealed(index: &GraphIndex, codec: u8, seal: Option<&[u8]>) -> Vec<u8> {
2124 let payloads: Vec<Vec<u8>> = index
2125 .perms()
2126 .iter()
2127 .map(|perm| encode_tiled_section(index, perm.section_index(), codec))
2128 .collect();
2129 let mut refs: Vec<&[u8]> = payloads.iter().map(|p| p.as_slice()).collect();
2130 if let Some(seal) = seal {
2131 refs.push(seal);
2132 }
2133 encode_container(&refs, CODEC_NONE)
2134}
2135
2136fn encode_named_graphs(named: &[(String, GraphIndex)], codec: u8) -> Vec<u8> {
2138 let mut out = Vec::new();
2139 write_uvarint(&mut out, named.len() as u64);
2140 for (iri, index) in named {
2141 write_uvarint(&mut out, iri.len() as u64);
2142 out.extend_from_slice(iri.as_bytes());
2143 let container = encode_index_container(index, codec);
2144 write_uvarint(&mut out, container.len() as u64);
2145 out.extend_from_slice(&container);
2146 }
2147 out
2148}
2149
2150fn decode_named_graphs(
2151 bytes: &[u8],
2152 codec: u8,
2153 perms: PermSet,
2154) -> Result<Vec<(String, GraphIndex)>, FileError> {
2155 let (n, mut pos) = read_uvarint(bytes).ok_or(FileError::Container("truncated graph count"))?;
2156 let bound = |start: usize, len: u64| -> Result<usize, FileError> {
2159 usize::try_from(len)
2160 .ok()
2161 .and_then(|len| start.checked_add(len))
2162 .filter(|&e| e <= bytes.len())
2163 .ok_or(FileError::Container("named-graph field overruns buffer"))
2164 };
2165 let n = wire_usize(n, "named-graph count too large")?;
2166 let mut out = Vec::with_capacity(n.min(bytes.len()));
2167 for _ in 0..n {
2168 let (ilen, u1) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
2169 .ok_or(FileError::Container("truncated iri len"))?;
2170 pos += u1;
2171 let iend = bound(pos, ilen)?;
2172 let iri = String::from_utf8_lossy(&bytes[pos..iend]).into_owned();
2173 pos = iend;
2174 let (clen, u2) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
2175 .ok_or(FileError::Container("truncated container len"))?;
2176 pos += u2;
2177 let cend = bound(pos, clen)?;
2178 let index = decode_index_container(&bytes[pos..cend], codec, perms)?;
2179 index.set_size_hint(u64::from(index.triple_count()));
2182 out.push((iri, index));
2183 pos = cend;
2184 }
2185 Ok(out)
2186}
2187
2188pub fn write_dataset(
2193 dict: &Dictionary,
2194 default_index: &GraphIndex,
2195 named: &[(String, GraphIndex)],
2196 has_quads: bool,
2197 pyramid_meta: &[u8],
2198 pyramid_levels: u16,
2199) -> Vec<u8> {
2200 write_dataset_with_metadata(
2201 dict,
2202 default_index,
2203 named,
2204 has_quads,
2205 pyramid_meta,
2206 pyramid_levels,
2207 &[],
2208 &[],
2209 )
2210}
2211
2212#[doc(hidden)]
2214pub fn write_dataset_v05(
2215 dict: &Dictionary,
2216 default_index: &GraphIndex,
2217 named: &[(String, GraphIndex)],
2218 has_quads: bool,
2219 pyramid_meta: &[u8],
2220 pyramid_levels: u16,
2221) -> Vec<u8> {
2222 write_dataset_with_metadata_v05(
2223 dict,
2224 default_index,
2225 named,
2226 has_quads,
2227 pyramid_meta,
2228 pyramid_levels,
2229 &[],
2230 &[],
2231 )
2232}
2233
2234pub(crate) fn encode_dict_container(dict: &Dictionary, codec: u8) -> Vec<u8> {
2239 let raw_sections = dict.sections();
2240 let dict_payloads: Vec<Vec<u8>> = raw_sections
2241 .iter()
2242 .map(|raw| encode_chunked_dict_section(raw, codec))
2243 .collect();
2244 encode_container(
2245 &[
2246 dict_payloads[0].as_slice(),
2247 dict_payloads[1].as_slice(),
2248 dict_payloads[2].as_slice(),
2249 dict_payloads[3].as_slice(),
2250 ],
2251 CODEC_NONE,
2252 )
2253}
2254
2255#[allow(clippy::too_many_arguments)]
2266pub fn write_dataset_with_metadata(
2267 dict: &Dictionary,
2268 default_index: &GraphIndex,
2269 named: &[(String, GraphIndex)],
2270 has_quads: bool,
2271 pyramid_meta: &[u8],
2272 pyramid_levels: u16,
2273 metadata: &[u8],
2274 text_index: &[u8],
2275) -> Vec<u8> {
2276 into_v06(write_dataset_with_metadata_v05(
2277 dict,
2278 default_index,
2279 named,
2280 has_quads,
2281 pyramid_meta,
2282 pyramid_levels,
2283 metadata,
2284 text_index,
2285 ))
2286}
2287
2288#[doc(hidden)]
2291#[allow(clippy::too_many_arguments)]
2292pub fn write_dataset_with_metadata_v05(
2293 dict: &Dictionary,
2294 default_index: &GraphIndex,
2295 named: &[(String, GraphIndex)],
2296 has_quads: bool,
2297 pyramid_meta: &[u8],
2298 pyramid_levels: u16,
2299 metadata: &[u8],
2300 text_index: &[u8],
2301) -> Vec<u8> {
2302 let codec = writer_codec();
2303 let dict_container = encode_dict_container(dict, codec);
2304 let scope = (!text_index.is_empty())
2308 .then(|| crate::text_index::TextIndex::from_section(text_index, codec).ok())
2309 .flatten()
2310 .map(|ti| ti.scope().clone());
2311 let text_index_partial = scope
2312 .as_ref()
2313 .is_some_and(crate::text_index::TextIndexScope::restricts_predicates);
2314 let text_index_named = scope
2315 .as_ref()
2316 .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs);
2317 write_dataset_from_parts(
2318 &dict_container,
2319 dict.term_count() as u64,
2320 default_index,
2321 NamedPart::PerGraph(named),
2322 has_quads,
2323 dict.has_quoted_triples(),
2324 pyramid_meta,
2325 pyramid_levels,
2326 metadata,
2327 text_index,
2328 text_index_partial,
2329 text_index_named,
2330 codec,
2331 )
2332}
2333
2334pub(crate) enum NamedPart<'a> {
2337 PerGraph(&'a [(String, GraphIndex)]),
2338 Quad(&'a crate::quad::EncodedQuadIndex),
2339}
2340
2341#[allow(clippy::too_many_arguments)]
2346pub(crate) fn write_dataset_from_parts(
2347 dict_container: &[u8],
2348 term_count: u64,
2349 default_index: &GraphIndex,
2350 named: NamedPart<'_>,
2351 has_quads: bool,
2352 has_quoted_triples: bool,
2353 pyramid_meta: &[u8],
2354 pyramid_levels: u16,
2355 metadata: &[u8],
2356 text_index: &[u8],
2357 text_index_partial: bool,
2358 text_index_named: bool,
2359 codec: u8,
2360) -> Vec<u8> {
2361 let (named_section, quad, named_quads) = match named {
2365 NamedPart::PerGraph(named) => (
2366 if named.is_empty() {
2367 Vec::new()
2368 } else {
2369 encode_named_graphs(named, codec)
2370 },
2371 None,
2372 named
2373 .iter()
2374 .map(|(_, idx)| idx.triple_count() as u64)
2375 .sum::<u64>(),
2376 ),
2377 NamedPart::Quad(q) if q.graph_count == 0 => (Vec::new(), None, 0),
2378 NamedPart::Quad(q) => (Vec::new(), Some(q), q.quad_count),
2379 };
2380 let seal = quad.map(|q| crate::quad::encode_seal(crate::quad::digest(&q.bytes)));
2381 let index_container = encode_index_container_sealed(default_index, codec, seal.as_deref());
2382 let text_index_partial = text_index_partial && !text_index.is_empty();
2387 let digested;
2388 let text_index: &[u8] = if text_index_partial {
2389 digested = crate::text_index::with_partial_digest(text_index);
2390 &digested
2391 } else {
2392 text_index
2393 };
2394
2395 let meta_section_len = metadata.len() as u64;
2398 let dict_offset = HEADER_LEN as u64 + meta_section_len;
2399 let dict_len = dict_container.len() as u64;
2400 let index_offset = dict_offset + dict_len;
2401 let index_len = index_container.len() as u64;
2402 let pyr_offset = index_offset + index_len;
2403 let pyr_len = pyramid_meta.len() as u64;
2404 let text_offset = pyr_offset + pyr_len;
2406 let text_len = text_index.len() as u64;
2407 let named_offset = text_offset + text_len;
2408 let named_len = named_section.len() as u64;
2409 let quad_len = quad.map_or(0, |q| q.bytes.len() as u64);
2412
2413 let mut parts: Vec<&[u8]> = Vec::with_capacity(5);
2419 if meta_section_len > 0 {
2420 parts.push(metadata);
2421 }
2422 parts.push(dict_container);
2423 parts.push(&index_container);
2424 parts.push(pyramid_meta);
2425 if text_len > 0 && !text_index_partial {
2426 parts.push(text_index);
2427 }
2428 if named_len > 0 {
2429 parts.push(&named_section);
2430 }
2431
2432 let schema_meta_len = crate::meta::schema_block_len(pyramid_meta);
2435
2436 let header = Header {
2440 version: crate::header::LEGACY_FORMAT_VERSION,
2441 flags: FLAG_TILE_SYNOPSIS
2442 | if has_quads { FLAG_HAS_QUADS } else { 0 }
2443 | if quad.is_some() {
2444 crate::header::FLAG_QUAD_INDEX
2445 } else {
2446 0
2447 }
2448 | if has_quoted_triples {
2449 FLAG_HAS_QUOTED_TRIPLES
2450 } else {
2451 0
2452 }
2453 | if text_len > 0 && text_index_named {
2454 crate::header::FLAG_TEXT_INDEX_NAMED_GRAPHS
2455 } else {
2456 0
2457 },
2458 metadata_offset: HEADER_LEN as u64,
2459 metadata_len: meta_section_len,
2460 dictionary_offset: dict_offset,
2461 dictionary_len: dict_len,
2462 root_dir_offset: index_offset,
2463 root_dir_len: index_len,
2464 pyramid_meta_offset: if pyr_len > 0 { pyr_offset } else { 0 },
2465 pyramid_meta_len: pyr_len,
2466 dict_codec: codec,
2467 block_codec: codec,
2468 pyramid_levels,
2469 id_widths: 0,
2470 perms: default_index.perms(),
2474 quad_count: default_index.triple_count() as u64 + named_quads,
2475 term_count,
2476 content_hash: content_hash(&parts),
2477 named_graphs_offset: if named_len > 0 { named_offset } else { 0 },
2478 named_graphs_len: named_len,
2479 schema_meta_len,
2480 text_index_offset: if text_len > 0 { text_offset } else { 0 },
2481 text_index_len: text_len,
2482 text_index_partial: text_len > 0 && text_index_partial,
2483 build_info_offset: 0,
2484 build_info_len: 0,
2485 extra_sections: if quad_len > 0 {
2486 vec![crate::header::Section {
2487 kind: crate::header::SectionKind::QuadIndex,
2488 flags: 0,
2489 offset: named_offset,
2490 length: quad_len,
2491 }]
2492 } else {
2493 Vec::new()
2494 },
2495 };
2496
2497 let mut out = Vec::with_capacity(
2498 HEADER_LEN
2499 + metadata.len()
2500 + dict_container.len()
2501 + index_container.len()
2502 + pyramid_meta.len()
2503 + text_index.len()
2504 + named_section.len()
2505 + quad_len as usize
2506 + MAGIC.len(),
2507 );
2508 out.extend_from_slice(&header.to_bytes());
2509 if meta_section_len > 0 {
2510 out.extend_from_slice(metadata);
2511 }
2512 out.extend_from_slice(dict_container);
2513 out.extend_from_slice(&index_container);
2514 out.extend_from_slice(pyramid_meta);
2515 if text_len > 0 {
2516 out.extend_from_slice(text_index);
2517 }
2518 if named_len > 0 {
2519 out.extend_from_slice(&named_section);
2520 }
2521 if let Some(q) = quad {
2522 out.extend_from_slice(&q.bytes);
2523 }
2524 out.extend_from_slice(&MAGIC); out
2526}
2527
2528pub const RDF_TYPE: &str = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
2530
2531pub fn schema_summary(rete: &Rete) -> Vec<(String, String, String, u32)> {
2538 use std::collections::{BTreeMap, HashMap};
2539 let triples = rete.dump(None);
2540
2541 let mut class_of: HashMap<&str, &str> = HashMap::new();
2542 for (s, p, o) in &triples {
2543 if p == RDF_TYPE {
2544 class_of.insert(s.as_str(), o.as_str());
2545 }
2546 }
2547 let classify = |t: &str| -> String {
2548 if let Some(c) = class_of.get(t) {
2549 (*c).to_string()
2550 } else if t.starts_with('"') {
2551 "(literal)".to_string()
2552 } else {
2553 "(untyped)".to_string()
2554 }
2555 };
2556
2557 let mut counts: BTreeMap<(String, String, String), u32> = BTreeMap::new();
2558 for (s, p, o) in &triples {
2559 if p == RDF_TYPE {
2560 continue; }
2562 *counts
2563 .entry((classify(s), p.clone(), classify(o)))
2564 .or_default() += 1;
2565 }
2566 counts
2567 .into_iter()
2568 .map(|((a, p, b), c)| (a, p, b, c))
2569 .collect()
2570}
2571
2572pub fn schema_classes(rete: &Rete) -> Vec<(String, u32)> {
2576 use std::collections::BTreeMap;
2577 let mut counts: BTreeMap<String, u32> = BTreeMap::new();
2578 for (_s, p, o) in rete.dump(None) {
2579 if p == RDF_TYPE {
2580 *counts.entry(o).or_default() += 1;
2581 }
2582 }
2583 let mut out: Vec<(String, u32)> = counts.into_iter().collect();
2584 out.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
2585 out
2586}
2587
2588pub fn read_metadata_ranged<R: RangeReader>(reader: &R) -> Result<Option<Vec<u8>>, FileError> {
2598 let head = reader.read_at(0, HEADER_LEN as u64)?;
2599 if crate::v06::open::is_v06(&head) {
2600 let h = crate::v06::Header06::from_bytes(&head)?;
2601 return Ok(crate::v06::open::read_card_build(reader, &h)?.0);
2602 }
2603 let header = Header::from_bytes(&head)?;
2604 if header.metadata_len == 0 {
2605 return Ok(None);
2606 }
2607 let bytes = reader.read_at(header.metadata_offset, header.metadata_len)?;
2608 Ok(Some(bytes))
2609}
2610
2611pub fn read_card_full_ranged<R: RangeReader>(reader: &R) -> Result<Option<Vec<u8>>, FileError> {
2616 let head = reader.read_at(0, HEADER_LEN as u64)?;
2617 if crate::v06::open::is_v06(&head) {
2618 return Ok(crate::v06::open::read_card_full(reader)?.0);
2619 }
2620 read_metadata_ranged(reader)
2621}
2622
2623#[allow(clippy::type_complexity)]
2633pub fn read_card_and_build_info_ranged<R: RangeReader>(
2634 reader: &R,
2635) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
2636 read_card_and_build_info_with_header(reader).map(|(_, m, b)| (m, b))
2637}
2638
2639#[allow(clippy::type_complexity)]
2650pub fn read_card_and_build_info_with_header<R: RangeReader>(
2651 reader: &R,
2652) -> Result<(Header, Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
2653 let head = reader.read_at(0, HEADER_LEN as u64)?;
2654 let header = Header::from_bytes(&head)?;
2655 if crate::v06::reads(header.version) {
2656 let h = crate::v06::Header06::from_bytes(&head)?;
2657 let (card, build) = crate::v06::open::read_card_build(reader, &h)?;
2658 return Ok((header, card, build));
2659 }
2660 let meta = (header.metadata_offset, header.metadata_len);
2661 let build = (header.build_info_offset, header.build_info_len);
2662 if meta.1 > 0 && build.1 > 0 && build.0 == meta.0 + meta.1 {
2663 let both = reader.read_at(meta.0, meta.1 + build.1)?;
2665 let (m, b) = usize::try_from(meta.1)
2666 .ok()
2667 .and_then(|mid| both.split_at_checked(mid))
2668 .ok_or(FileError::Container("short card read"))?;
2669 return Ok((header, Some(m.to_vec()), Some(b.to_vec())));
2670 }
2671 let fetch = |off: u64, len: u64| -> Result<Option<Vec<u8>>, FileError> {
2672 if len == 0 {
2673 return Ok(None);
2674 }
2675 Ok(Some(reader.read_at(off, len)?))
2676 };
2677 Ok((header, fetch(meta.0, meta.1)?, fetch(build.0, build.1)?))
2678}
2679
2680pub fn read_build_info(bytes: &[u8]) -> Result<Option<Vec<u8>>, FileError> {
2685 if crate::v06::open::is_v06(bytes) {
2686 let h = crate::v06::Header06::from_bytes(bytes)?;
2687 let r = crate::reader::SliceReader::new(bytes);
2688 return Ok(crate::v06::open::read_card_build(&r, &h)?.1);
2689 }
2690 let header = Header::from_bytes(bytes)?;
2691 if header.build_info_len == 0 {
2692 return Ok(None);
2693 }
2694 let section = checked_slice(bytes, header.build_info_offset, header.build_info_len)
2695 .ok_or(FileError::Container("build-info section overruns buffer"))?;
2696 Ok(Some(section.to_vec()))
2697}
2698
2699#[derive(Debug, Clone, PartialEq, Eq)]
2708pub struct BuildInfoPlan {
2709 pub header: [u8; HEADER_LEN],
2711 pub insert: u64,
2714 pub tail_start: u64,
2718 pub new_len: u64,
2720}
2721
2722pub fn plan_build_info(
2744 head: &[u8],
2745 file_len: u64,
2746 info_len: u64,
2747) -> Result<BuildInfoPlan, FileError> {
2748 let mut header = Header::from_bytes(head)?;
2749 let insert = if header.metadata_len > 0 {
2754 header
2755 .metadata_offset
2756 .checked_add(header.metadata_len)
2757 .filter(|&end| header.metadata_offset >= HEADER_LEN as u64 && end <= file_len)
2758 .ok_or(FileError::Container(
2759 "the metadata section lies outside the file",
2760 ))?
2761 } else {
2762 HEADER_LEN as u64
2763 };
2764 let old_len = header.build_info_len;
2765 if old_len > 0 && header.build_info_offset != insert {
2766 return Err(FileError::Container(
2767 "existing build-info section is not adjacent to the metadata",
2768 ));
2769 }
2770 let tail_start = insert.saturating_add(old_len);
2771 if tail_start > file_len || insert > file_len {
2772 return Err(FileError::Container("build-info splice out of bounds"));
2773 }
2774
2775 let shift = |off: &mut u64, len: u64| {
2777 if len > 0 && *off >= tail_start {
2778 *off = *off - old_len + info_len;
2779 }
2780 };
2781 shift(&mut header.dictionary_offset, header.dictionary_len);
2782 shift(&mut header.root_dir_offset, header.root_dir_len);
2783 shift(&mut header.pyramid_meta_offset, header.pyramid_meta_len);
2784 shift(&mut header.named_graphs_offset, header.named_graphs_len);
2785 shift(&mut header.text_index_offset, header.text_index_len);
2786 for s in &mut header.extra_sections {
2787 if s.length > 0 && s.offset >= tail_start {
2788 s.offset = s.offset - old_len + info_len;
2789 }
2790 }
2791 if info_len == 0 {
2792 header.build_info_offset = 0;
2793 header.build_info_len = 0;
2794 } else {
2795 header.build_info_offset = insert;
2796 header.build_info_len = info_len;
2797 }
2798
2799 Ok(BuildInfoPlan {
2800 header: header.to_bytes(),
2801 insert,
2802 tail_start,
2803 new_len: file_len - old_len + info_len,
2804 })
2805}
2806
2807pub fn attach_build_info(image: &[u8], info: &[u8]) -> Result<Vec<u8>, FileError> {
2813 if crate::v06::open::is_v06(image) {
2814 return crate::v06::write::replace_card_with(
2816 image,
2817 crate::v06::write::NewCard::Keep,
2818 (!info.is_empty()).then_some(info),
2819 );
2820 }
2821 let plan = plan_build_info(image, image.len() as u64, info.len() as u64)?;
2822 let mut out = Vec::with_capacity(plan.new_len as usize);
2823 out.extend_from_slice(&plan.header);
2824 out.extend_from_slice(&image[HEADER_LEN..plan.insert as usize]);
2825 out.extend_from_slice(info);
2826 out.extend_from_slice(&image[plan.tail_start as usize..]);
2827 debug_assert_eq!(out.len() as u64, plan.new_len);
2828 Ok(out)
2829}
2830
2831pub fn replace_metadata(image: &[u8], metadata: &[u8]) -> Result<Vec<u8>, FileError> {
2851 if crate::v06::open::is_v06(image) {
2852 let h = crate::v06::Header06::from_bytes(image)?;
2854 let (_, build) =
2855 crate::v06::open::read_card_build(&crate::reader::SliceReader::new(image), &h)?;
2856 return crate::v06::write::replace_card(
2857 image,
2858 (!metadata.is_empty()).then_some(metadata),
2859 build.as_deref(),
2860 );
2861 }
2862 let mut header = Header::from_bytes(image)?;
2863 let insert = HEADER_LEN as u64;
2866 if header.metadata_len > 0 && header.metadata_offset != insert {
2867 return Err(FileError::Container(
2868 "metadata section does not sit immediately after the header",
2869 ));
2870 }
2871 let old_len = header.metadata_len;
2872 let tail_start = (insert + old_len) as usize;
2873 if tail_start > image.len() {
2874 return Err(FileError::Container("metadata splice out of bounds"));
2875 }
2876 let new_len = metadata.len() as u64;
2877
2878 let shift = |off: &mut u64, len: u64| {
2879 if len > 0 && *off >= insert + old_len {
2880 *off = *off - old_len + new_len;
2881 }
2882 };
2883 shift(&mut header.dictionary_offset, header.dictionary_len);
2884 shift(&mut header.root_dir_offset, header.root_dir_len);
2885 shift(&mut header.pyramid_meta_offset, header.pyramid_meta_len);
2886 shift(&mut header.named_graphs_offset, header.named_graphs_len);
2887 shift(&mut header.text_index_offset, header.text_index_len);
2888 shift(&mut header.build_info_offset, header.build_info_len);
2889 for s in &mut header.extra_sections {
2890 if s.length > 0 && s.offset >= insert + old_len {
2891 s.offset = s.offset - old_len + new_len;
2892 }
2893 }
2894 header.metadata_offset = insert;
2895 header.metadata_len = new_len;
2896
2897 let mut out = Vec::with_capacity(image.len() - old_len as usize + metadata.len());
2898 out.extend_from_slice(&header.to_bytes());
2899 out.extend_from_slice(metadata);
2900 out.extend_from_slice(&image[tail_start..]);
2901 header.content_hash = hash_of_sections(&header, &out)?;
2905 out[..HEADER_LEN].copy_from_slice(&header.to_bytes());
2906 Ok(out)
2907}
2908
2909fn hash_of_sections(header: &Header, bytes: &[u8]) -> Result<[u8; 16], FileError> {
2915 let slice = |off: u64, len: u64| -> Result<&[u8], FileError> {
2916 checked_slice(bytes, off, len).ok_or(FileError::Container("section overruns buffer"))
2925 };
2926 let parts = content_hash_parts(header)
2927 .into_iter()
2928 .map(|(off, len)| slice(off, len))
2929 .collect::<Result<Vec<&[u8]>, FileError>>()?;
2930 Ok(content_hash(&parts))
2931}
2932
2933pub fn content_hash_parts(header: &Header) -> Vec<(u64, u64)> {
2946 let mut parts: Vec<(u64, u64)> = Vec::with_capacity(6);
2947 if header.metadata_len > 0 {
2948 parts.push((header.metadata_offset, header.metadata_len));
2949 }
2950 parts.push((header.dictionary_offset, header.dictionary_len));
2951 parts.push((header.root_dir_offset, header.root_dir_len));
2952 if header.pyramid_meta_len > 0 {
2955 parts.push((header.pyramid_meta_offset, header.pyramid_meta_len));
2956 }
2957 if header.text_index_len > 0 && !header.text_index_partial {
2959 parts.push((header.text_index_offset, header.text_index_len));
2960 }
2961 if header.named_graphs_len > 0 {
2962 parts.push((header.named_graphs_offset, header.named_graphs_len));
2963 }
2964 parts
2965}
2966
2967pub fn verify_ranged_v06(
2970 reader: &dyn crate::reader::RangeReader,
2971 head: &[u8],
2972 only: Option<u8>,
2973) -> Result<Vec<String>, FileError> {
2974 const SLICE: u64 = 8 << 20;
2975 let h = crate::v06::Header06::from_bytes(head)?;
2976 if h.file_len != reader.len() {
2977 return Ok(vec!["file length".to_string()]);
2978 }
2979 let mut bad = Vec::new();
2980 let root_bytes = reader.read_at(crate::v06::HEADER_LEN as u64, h.root_len as u64)?;
2981 if crate::v06::digest16(&root_bytes) != h.root_digest {
2982 bad.push("root".to_string());
2983 return Ok(bad);
2984 }
2985 let root = crate::v06::Root06::decode(&root_bytes, h.internal_codec)?;
2986 if crate::v06::content_hash(&h) != h.content_hash {
2987 bad.push("content hash".to_string());
2988 }
2989 let card = h.card_range();
2990 if only.is_none() && card.len > 0 {
2991 let stored = reader.read_at(card.offset, card.len)?;
2992 if crate::v06::digest16(&stored) != h.card_digest {
2993 bad.push("card".to_string());
2994 }
2995 }
2996 for s in &root.sections {
2997 if only.is_some_and(|k| k != s.kind) {
2998 continue;
2999 }
3000 let a = match s.abs(&h) {
3001 Ok(a) if a.offset + a.len <= h.file_len => a,
3002 _ => {
3003 bad.push(crate::v06::kind::name(s.kind).to_string());
3004 continue;
3005 }
3006 };
3007 let mut hs = blake3::Hasher::new();
3008 let mut at = a.offset;
3009 while at < a.offset + a.len {
3010 let n = (a.offset + a.len - at).min(SLICE);
3011 hs.update(&reader.read_at(at, n)?);
3012 at += n;
3013 }
3014 if hs.finalize().as_bytes()[..16] != s.digest {
3015 bad.push(crate::v06::kind::name(s.kind).to_string());
3016 }
3017 }
3018 if only.is_none() {
3019 let n = crate::v06::FOOTER.len() as u64;
3020 if reader.read_at(h.file_len - n, n)? != crate::v06::footer(h.version) {
3021 bad.push("footer".to_string());
3022 }
3023 }
3024 Ok(bad)
3025}
3026
3027pub fn verify(bytes: &[u8]) -> Result<bool, FileError> {
3035 if crate::v06::open::is_v06(bytes) {
3036 let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
3037 std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(bytes)));
3038 return crate::v06::open::V06File::open(r, bytes)?.verify(None);
3039 }
3040 let header = Header::from_bytes(bytes)?;
3041 if hash_of_sections(&header, bytes)? != header.content_hash {
3042 return Ok(false);
3043 }
3044 if header.has_quad_index() || header.quad_index_section().is_some() {
3047 check_named_layout(&header)?;
3048 let slice = |off: u64, len: u64| -> Result<&[u8], FileError> {
3049 checked_slice(bytes, off, len).ok_or(FileError::Container("section overruns buffer"))
3050 };
3051 let root = slice(header.root_dir_offset, header.root_dir_len)?;
3052 let ranges = container_section_payload_ranges(
3053 root,
3054 header.root_dir_offset,
3055 header.index_container_sections(),
3056 )?;
3057 let seal = ranges
3058 .last()
3059 .ok_or(FileError::Container("missing quad-index seal"))?;
3060 let want = crate::quad::parse_seal(slice(seal.offset, seal.len)?)?;
3061 let q = header
3062 .quad_index_section()
3063 .ok_or(FileError::Container("missing quad-index section"))?;
3064 if crate::quad::digest(slice(q.offset, q.length)?) != want {
3065 return Ok(false);
3066 }
3067 }
3068 for s in self_digested_sections(&header) {
3069 let section = checked_slice(bytes, s.offset, s.length)
3070 .ok_or(FileError::Container("section overruns buffer"))?;
3071 if !crate::label_index::digest_ok(section) {
3072 return Ok(false);
3073 }
3074 }
3075 Ok(true)
3076}
3077
3078fn self_digested_sections(header: &Header) -> Vec<Section> {
3082 [
3083 header.label_index_section(),
3084 header.schema_section(),
3085 header
3086 .section(crate::header::SectionKind::TextIndexPartial)
3087 .filter(|s| s.length > 0),
3088 ]
3089 .into_iter()
3090 .flatten()
3091 .collect()
3092}
3093
3094pub fn append_sections(
3102 image: &[u8],
3103 sections: &[(SectionKind, &[u8])],
3104) -> Result<Vec<u8>, FileError> {
3105 let lens: Vec<(SectionKind, u64)> =
3106 sections.iter().map(|(k, b)| (*k, b.len() as u64)).collect();
3107 let (header, body_end) = plan_append(image, image.len() as u64, &lens)?;
3108 let extra: usize = sections.iter().map(|(_, b)| b.len()).sum();
3109 let mut out = Vec::with_capacity(image.len() + extra);
3110 out.extend_from_slice(&header.to_bytes());
3111 out.extend_from_slice(&image[HEADER_LEN..body_end as usize]);
3112 for (_, b) in sections {
3113 out.extend_from_slice(b);
3114 }
3115 out.extend_from_slice(&MAGIC);
3116 Ok(out)
3117}
3118
3119fn plan_append(
3121 head: &[u8],
3122 file_len: u64,
3123 sections: &[(SectionKind, u64)],
3124) -> Result<(Header, u64), FileError> {
3125 let mut header = Header::from_bytes(head)?;
3126 if header.expected_file_len() != Some(file_len) {
3127 return Err(FileError::Container(
3128 "cannot append sections: the file length does not match its header",
3129 ));
3130 }
3131 let body_end = file_len - MAGIC.len() as u64;
3132 let mut at = body_end;
3133 for &(kind, len) in sections {
3134 if !matches!(kind, SectionKind::LabelIndex | SectionKind::Schema) {
3135 return Err(FileError::Container(
3136 "only LABEL_INDEX and SCHEMA sections can be appended",
3137 ));
3138 }
3139 if len == 0 {
3140 continue;
3141 }
3142 if header.section(kind).is_some_and(|s| s.length > 0) {
3143 return Err(FileError::Container(
3144 "the file already carries that section",
3145 ));
3146 }
3147 header = header.with_section(kind, at, len);
3148 at += len;
3149 }
3150 Ok((header, body_end))
3151}
3152
3153#[cfg(not(target_arch = "wasm32"))]
3158pub(crate) fn open_dictionary_lazy<R: RangeReader + Send + Sync + 'static>(
3159 reader: R,
3160) -> Result<Dictionary, FileError> {
3161 let head = reader.read_at(0, HEADER_LEN as u64)?;
3162 let header = Header::from_bytes(&head)?;
3163 ranged_chunked_dictionary(&std::sync::Arc::new(reader), &header, [true; 4])
3164}
3165
3166pub fn verify_ranged(reader: &dyn crate::reader::RangeReader) -> Result<bool, FileError> {
3172 const SLICE: u64 = 8 << 20;
3175 let len = reader.len();
3176 if len < HEADER_LEN as u64 {
3177 return Err(FileError::Container("file shorter than its header"));
3178 }
3179 let head = reader.read_at(0, HEADER_LEN as u64)?;
3180 if crate::v06::open::is_v06(&head) {
3181 return verify_ranged_v06(reader, &head, None).map(|bad| bad.is_empty());
3182 }
3183 let header = Header::from_bytes(&head)?;
3184 let mut h = blake3::Hasher::new();
3185 for (off, n) in content_hash_parts(&header) {
3186 if off.checked_add(n).is_none_or(|end| end > len) {
3189 return Err(FileError::Container("section overruns buffer"));
3190 }
3191 let mut at = off;
3192 while at < off + n {
3193 let take = (off + n - at).min(SLICE);
3194 h.update(&reader.read_at(at, take)?);
3195 at += take;
3196 }
3197 }
3198 let mut out = [0u8; 16];
3199 out.copy_from_slice(&h.finalize().as_bytes()[..16]);
3200 if out != header.content_hash {
3201 return Ok(false);
3202 }
3203 if header.has_quad_index() || header.quad_index_section().is_some() {
3206 check_named_layout(&header)?;
3207 let sections = header.index_container_sections();
3208 let seal = locate_container_section_ranged(
3209 reader,
3210 header.root_dir_offset,
3211 header.root_dir_len,
3212 sections - 1,
3213 sections as u64,
3214 )?;
3215 let want = crate::quad::parse_seal(&reader.read_at(seal.offset, seal.len)?)?;
3216 let q = header
3217 .quad_index_section()
3218 .ok_or(FileError::Container("missing quad-index section"))?;
3219 if q.offset.checked_add(q.length).is_none_or(|end| end > len) {
3220 return Err(FileError::Container("section overruns buffer"));
3221 }
3222 let mut h = blake3::Hasher::new();
3223 let mut at = q.offset;
3224 while at < q.offset + q.length {
3225 let take = (q.offset + q.length - at).min(SLICE);
3226 h.update(&reader.read_at(at, take)?);
3227 at += take;
3228 }
3229 let mut got = [0u8; 16];
3230 got.copy_from_slice(&h.finalize().as_bytes()[..16]);
3231 if got != want {
3232 return Ok(false);
3233 }
3234 }
3235 for s in self_digested_sections(&header) {
3237 if s.length < 16 || s.offset.checked_add(s.length).is_none_or(|end| end > len) {
3238 return Err(FileError::Container("section overruns buffer"));
3239 }
3240 let want = reader.read_at(s.offset, 16)?;
3241 let mut h = blake3::Hasher::new();
3242 let mut at = s.offset + 16;
3243 while at < s.offset + s.length {
3244 let take = (s.offset + s.length - at).min(SLICE);
3245 h.update(&reader.read_at(at, take)?);
3246 at += take;
3247 }
3248 if h.finalize().as_bytes()[..16] != want[..] {
3249 return Ok(false);
3250 }
3251 }
3252 Ok(true)
3253}
3254
3255type PyramidLoader = Box<dyn Fn() -> Option<PyramidMeta> + Send + Sync>;
3257
3258enum PyramidSlot {
3264 Resident(Option<PyramidMeta>),
3265 Lazy {
3266 loader: PyramidLoader,
3267 cell: std::sync::OnceLock<Option<PyramidMeta>>,
3268 },
3269}
3270
3271pub(crate) enum SectionSlot<T> {
3276 Resident(Option<T>),
3277 Lazy {
3278 loader: Box<dyn Fn() -> Option<T> + Send + Sync>,
3279 cell: std::sync::OnceLock<Option<T>>,
3280 },
3281}
3282
3283impl<T> SectionSlot<T> {
3284 pub(crate) fn get(&self) -> Option<&T> {
3286 match self {
3287 SectionSlot::Resident(v) => v.as_ref(),
3288 SectionSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
3289 }
3290 }
3291
3292 pub(crate) fn get_if_loaded(&self) -> Option<&T> {
3294 match self {
3295 SectionSlot::Resident(v) => v.as_ref(),
3296 SectionSlot::Lazy { cell, .. } => cell.get().and_then(|o| o.as_ref()),
3297 }
3298 }
3299}
3300
3301fn resident_label_slot<'a>(
3303 header: &Header,
3304 region: &dyn Fn(u64, u64) -> Result<&'a [u8], FileError>,
3305) -> SectionSlot<crate::label_index::LabelIndex> {
3306 SectionSlot::Resident(header.label_index_section().and_then(|s| {
3307 let bytes = region(s.offset, s.length).ok()?;
3308 crate::label_index::LabelIndex::from_section(bytes, header.block_codec).ok()
3309 }))
3310}
3311
3312fn resident_schema_slot<'a>(
3314 header: &Header,
3315 region: &dyn Fn(u64, u64) -> Result<&'a [u8], FileError>,
3316) -> SectionSlot<crate::schema_section::SchemaMeta> {
3317 SectionSlot::Resident(header.schema_section().and_then(|s| {
3318 crate::schema_section::SchemaMeta::decode(region(s.offset, s.length).ok()?).ok()
3319 }))
3320}
3321
3322fn ranged_label_slot<R: RangeReader + Send + Sync + 'static>(
3326 reader: &std::sync::Arc<R>,
3327 header: &Header,
3328) -> SectionSlot<crate::label_index::LabelIndex> {
3329 let Some(s) = header.label_index_section() else {
3330 return SectionSlot::Resident(None);
3331 };
3332 let r = reader.clone();
3333 let codec = header.block_codec;
3334 SectionSlot::Lazy {
3335 loader: Box::new(move || {
3336 let head = r
3337 .read_at(s.offset, crate::label_index::HEAD_READ.min(s.length))
3338 .ok()?;
3339 let rr = r.clone();
3340 let loader = Box::new(move |off: u64, len: u64| rr.read_at(s.offset + off, len).ok());
3341 crate::label_index::LabelIndex::from_head(&head, s.length, codec, loader).ok()
3342 }),
3343 cell: std::sync::OnceLock::new(),
3344 }
3345}
3346
3347fn ranged_schema_slot<R: RangeReader + Send + Sync + 'static>(
3349 reader: &std::sync::Arc<R>,
3350 header: &Header,
3351) -> SectionSlot<crate::schema_section::SchemaMeta> {
3352 let Some(s) = header.schema_section() else {
3353 return SectionSlot::Resident(None);
3354 };
3355 let r = reader.clone();
3356 SectionSlot::Lazy {
3357 loader: Box::new(move || {
3358 let bytes = r.read_at(s.offset, s.length).ok()?;
3359 crate::schema_section::SchemaMeta::decode(&bytes).ok()
3360 }),
3361 cell: std::sync::OnceLock::new(),
3362 }
3363}
3364
3365pub fn read_schema_ranged<R: RangeReader + ?Sized>(
3369 reader: &R,
3370) -> Result<Option<crate::schema_section::SchemaMeta>, FileError> {
3371 let head = reader.read_at(0, HEADER_LEN as u64)?;
3372 let header = Header::from_bytes(&head)?;
3373 Ok(read_schema_with_header(reader, &header))
3374}
3375
3376fn read_schema_with_header<R: RangeReader + ?Sized>(
3377 reader: &R,
3378 header: &Header,
3379) -> Option<crate::schema_section::SchemaMeta> {
3380 if crate::v06::reads(header.version) {
3381 let b = crate::v06::open::read_schema_bytes(reader).ok()??;
3382 return crate::schema_section::SchemaMeta::decode(&b).ok();
3383 }
3384 let s = header.schema_section()?;
3385 let bytes = reader.read_at(s.offset, s.length).ok()?;
3386 crate::schema_section::SchemaMeta::decode(&bytes).ok()
3387}
3388
3389type TextIndexLoader = Box<dyn Fn() -> Option<crate::text_index::TextIndex> + Send + Sync>;
3392
3393type TokenTableProbe = Box<dyn Fn() -> Option<u64> + Send + Sync>;
3396
3397enum TextIndexSlot {
3407 Resident {
3408 index: Option<crate::text_index::TextIndex>,
3409 token_table_len: Option<u64>,
3412 },
3413 Lazy {
3414 loader: TextIndexLoader,
3415 cell: std::sync::OnceLock<Option<crate::text_index::TextIndex>>,
3416 token_table: TokenTableProbe,
3418 token_table_cell: std::sync::OnceLock<Option<u64>>,
3419 },
3420}
3421
3422impl TextIndexSlot {
3423 fn index(&self) -> Option<&crate::text_index::TextIndex> {
3425 match self {
3426 TextIndexSlot::Resident { index, .. } => index.as_ref(),
3427 TextIndexSlot::Lazy { loader, cell, .. } => cell.get_or_init(loader).as_ref(),
3428 }
3429 }
3430
3431 fn token_table_len(&self) -> Option<u64> {
3434 match self {
3435 TextIndexSlot::Resident {
3436 token_table_len, ..
3437 } => *token_table_len,
3438 TextIndexSlot::Lazy {
3439 token_table,
3440 token_table_cell,
3441 ..
3442 } => *token_table_cell.get_or_init(token_table),
3443 }
3444 }
3445}
3446
3447fn resident_text_index_slot(section: Option<&[u8]>, codec: u8) -> Result<TextIndexSlot, FileError> {
3450 let Some(section) = section else {
3451 return Ok(TextIndexSlot::Resident {
3452 index: None,
3453 token_table_len: None,
3454 });
3455 };
3456 Ok(TextIndexSlot::Resident {
3457 index: Some(
3458 crate::text_index::TextIndex::from_section(section, codec)
3459 .map_err(|_| FileError::Container("malformed text index"))?,
3460 ),
3461 token_table_len: crate::text_index::TextIndex::postings_base(section).map(|b| b as u64),
3462 })
3463}
3464
3465pub fn read_text_index_token_table_len_ranged<R: RangeReader + ?Sized>(
3478 reader: &R,
3479 header: &Header,
3480) -> Option<u64> {
3481 if crate::v06::reads(header.version) {
3482 return crate::v06::open::read_token_table_bytes(reader);
3483 }
3484 let (ti_off, ti_len) = header.text_index_payload();
3485 if ti_len == 0 {
3486 return None;
3487 }
3488 read_token_table_len(reader, ti_off, ti_len)
3489}
3490
3491fn read_token_table_len<R: RangeReader + ?Sized>(reader: &R, off: u64, len: u64) -> Option<u64> {
3495 let head = reader.read_at(off, 10u64.min(len)).ok()?;
3496 let (ttlen, n) = crate::varint::read_uvarint(&head)?;
3497 Some((n as u64 + ttlen).min(len))
3498}
3499
3500const NAMED_GRAPH_RESIDENT_MAX: u64 = 1 << 20; pub const TRANSIENT_GRAPH_QUADS: u64 = 1024;
3512
3513const NAMED_SLAB: usize = 1024;
3518
3519const NAMED_WALK_CHUNK: u64 = 64 * 1024;
3523
3524const NAMED_WALK_CHUNK_MAX: u64 = 8 * 1024 * 1024;
3537
3538#[derive(Default)]
3540struct NamedEntry {
3541 meta: std::sync::OnceLock<(String, ByteRange)>,
3544 index: std::sync::OnceLock<Box<GraphIndex>>,
3547}
3548
3549#[derive(Default)]
3553struct NamedWalk {
3554 next: usize,
3556 pos: u64,
3560 buf: Vec<u8>,
3567 buf_off: u64,
3568 chunk: u64,
3575 big_seen: bool,
3582}
3583
3584struct LazyNamedGraphs {
3598 reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
3599 section: ByteRange,
3601 codec: u8,
3602 has_synopsis: bool,
3603 read_concurrency: usize,
3604 perms: PermSet,
3607 #[allow(clippy::type_complexity)]
3611 dir: std::sync::OnceLock<(usize, Box<[std::sync::OnceLock<Box<[NamedEntry]>>]>)>,
3612 walk: std::sync::Mutex<NamedWalk>,
3613 failed: std::sync::atomic::AtomicBool,
3614 tile_cap: std::sync::atomic::AtomicU64,
3620}
3621
3622impl LazyNamedGraphs {
3623 fn new(
3624 reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
3625 section: ByteRange,
3626 codec: u8,
3627 has_synopsis: bool,
3628 read_concurrency: usize,
3629 perms: PermSet,
3630 ) -> Self {
3631 LazyNamedGraphs {
3632 reader,
3633 section,
3634 codec,
3635 has_synopsis,
3636 read_concurrency,
3637 perms,
3638 dir: std::sync::OnceLock::new(),
3639 walk: std::sync::Mutex::new(NamedWalk::default()),
3640 failed: std::sync::atomic::AtomicBool::new(false),
3641 tile_cap: std::sync::atomic::AtomicU64::new(u64::MAX),
3642 }
3643 }
3644
3645 fn set_tile_cap(&self, cap: u64) {
3649 self.tile_cap
3650 .store(cap, std::sync::atomic::Ordering::Relaxed);
3651 self.for_each_opened(|g| {
3652 if g.is_lazy() {
3653 g.set_cache_cap(cap);
3654 }
3655 });
3656 }
3657
3658 fn fail(&self) {
3660 self.failed
3661 .store(true, std::sync::atomic::Ordering::Relaxed);
3662 }
3663
3664 #[allow(clippy::type_complexity)]
3666 fn directory(&self) -> Option<&(usize, Box<[std::sync::OnceLock<Box<[NamedEntry]>>]>)> {
3667 if let Some(d) = self.dir.get() {
3668 return Some(d);
3669 }
3670 let end = self.section.offset.checked_add(self.section.len)?;
3671 let (n, used) = match read_uvarint_at(&self.reader, self.section.offset, end) {
3672 Ok(v) => v,
3673 Err(_) => {
3674 self.fail();
3675 return None;
3676 }
3677 };
3678 if n > self.section.len / 2 {
3681 self.fail();
3682 return None;
3683 }
3684 let Ok(n) = usize::try_from(n) else {
3685 self.fail();
3686 return None;
3687 };
3688 let slabs = n.div_ceil(NAMED_SLAB);
3689 let table: Box<[std::sync::OnceLock<Box<[NamedEntry]>>]> =
3690 (0..slabs).map(|_| std::sync::OnceLock::new()).collect();
3691 let _ = self.dir.set((n, table));
3692 {
3693 let mut w = self.walk.lock().unwrap();
3696 if w.pos == 0 {
3697 w.pos = self.section.offset + used;
3698 }
3699 }
3700 self.dir.get()
3701 }
3702
3703 fn count(&self) -> usize {
3704 self.directory().map(|(n, _)| *n).unwrap_or(0)
3705 }
3706
3707 fn entry(&self, i: usize) -> Option<&NamedEntry> {
3709 let (n, table) = self.directory()?;
3710 if i >= *n {
3711 return None;
3712 }
3713 let slab = table[i / NAMED_SLAB].get_or_init(|| {
3714 let len = NAMED_SLAB.min(n - (i / NAMED_SLAB) * NAMED_SLAB);
3715 (0..len).map(|_| NamedEntry::default()).collect()
3716 });
3717 slab.get(i % NAMED_SLAB)
3718 }
3719
3720 fn next_read_len(w: &mut NamedWalk, pos: u64, end: u64, need: u64) -> u64 {
3728 let cur = w.chunk.max(NAMED_WALK_CHUNK);
3729 w.chunk = (cur * 2).min(NAMED_WALK_CHUNK_MAX);
3730 cur.max(need).min(end - pos)
3731 }
3732
3733 fn ensure_meta(&self, upto: usize, exhaustive: bool) -> Option<()> {
3748 let target = self.entry(upto)?;
3749 if target.meta.get().is_some() {
3750 return Some(());
3751 }
3752 let end = self.section.offset.checked_add(self.section.len)?;
3753 let mut w = self.walk.lock().unwrap();
3754 if exhaustive && !w.big_seen {
3755 w.chunk = NAMED_WALK_CHUNK_MAX;
3756 }
3757 while w.next <= upto {
3758 let pos = w.pos;
3759 if pos >= end {
3760 self.fail();
3762 return None;
3763 }
3764 let have =
3767 |b: &[u8], off: u64, need: u64| pos >= off && pos + need <= off + b.len() as u64;
3768 if !have(&w.buf, w.buf_off, 20.min(end - pos)) {
3769 let len = Self::next_read_len(&mut w, pos, end, 20.min(end - pos));
3770 w.buf = match self.reader.read_at(pos, len) {
3771 Ok(b) => b,
3772 Err(_) => {
3773 self.fail();
3774 return None;
3775 }
3776 };
3777 w.buf_off = pos;
3778 }
3779 let rel = (pos - w.buf_off) as usize;
3782 let Some((ilen, u1)) = w.buf.get(rel..).and_then(read_uvarint) else {
3783 self.fail();
3784 return None;
3785 };
3786 let Some(iri_end) = (u1 as u64)
3787 .checked_add(ilen)
3788 .and_then(|n| pos.checked_add(n))
3789 .filter(|&e| e <= end)
3790 else {
3791 self.fail(); return None;
3793 };
3794 let header_need = (iri_end - pos).saturating_add(10);
3796 if !have(&w.buf, w.buf_off, header_need.min(end - pos)) {
3797 let len = Self::next_read_len(&mut w, pos, end, header_need);
3798 w.buf = match self.reader.read_at(pos, len) {
3799 Ok(b) => b,
3800 Err(_) => {
3801 self.fail();
3802 return None;
3803 }
3804 };
3805 w.buf_off = pos;
3806 }
3807 let rel = (pos - w.buf_off) as usize;
3808 let istart = rel + u1;
3809 let Some(iend) = usize::try_from(ilen)
3810 .ok()
3811 .and_then(|l| istart.checked_add(l))
3812 .filter(|&e| e <= w.buf.len())
3813 else {
3814 self.fail(); return None;
3816 };
3817 let iri = String::from_utf8_lossy(&w.buf[istart..iend]).into_owned();
3818 let Some((clen, u2)) = read_uvarint(&w.buf[iend..]) else {
3819 self.fail();
3820 return None;
3821 };
3822 let cend = match iri_end
3823 .checked_add(u2 as u64)
3824 .and_then(|cstart| cstart.checked_add(clen))
3825 {
3826 Some(e) if e <= end => e,
3827 _ => {
3828 self.fail(); return None;
3830 }
3831 };
3832 if clen > NAMED_GRAPH_RESIDENT_MAX {
3833 w.big_seen = true;
3838 w.chunk = NAMED_WALK_CHUNK;
3839 }
3840 let range = ByteRange {
3841 offset: cend - clen,
3842 len: clen,
3843 };
3844 if let Some(e) = self.entry(w.next) {
3845 let _ = e.meta.set((iri, range));
3846 }
3847 w.pos = cend;
3848 w.next += 1;
3849 }
3850 Some(())
3851 }
3852
3853 fn name_at(&self, i: usize, exhaustive: bool) -> Option<&str> {
3856 self.ensure_meta(i, exhaustive)?;
3857 self.entry(i)?.meta.get().map(|(iri, _)| iri.as_str())
3858 }
3859
3860 fn graph_at(&self, i: usize, exhaustive: bool) -> Option<(&str, &GraphIndex)> {
3863 self.ensure_meta(i, exhaustive)?;
3864 let e = self.entry(i)?;
3865 let (iri, range) = e.meta.get()?;
3866 if e.index.get().is_none() {
3867 let opened = self.open_graph(*range)?;
3868 let _ = e.index.set(Box::new(opened));
3869 }
3870 Some((iri.as_str(), e.index.get()?.as_ref()))
3871 }
3872
3873 fn container_bytes(&self, range: ByteRange) -> Option<Vec<u8>> {
3877 {
3878 let w = self.walk.lock().unwrap();
3879 let buf_end = w.buf_off + w.buf.len() as u64;
3880 if range.offset >= w.buf_off && range.offset + range.len <= buf_end {
3881 let a = (range.offset - w.buf_off) as usize;
3882 return Some(w.buf[a..a + range.len as usize].to_vec());
3883 }
3884 }
3885 match self.reader.read_at(range.offset, range.len) {
3886 Ok(b) if b.len() as u64 == range.len => Some(b),
3888 _ => {
3889 self.fail();
3890 None
3891 }
3892 }
3893 }
3894
3895 fn open_graph(&self, range: ByteRange) -> Option<GraphIndex> {
3900 if range.len <= NAMED_GRAPH_RESIDENT_MAX {
3901 let bytes = self.container_bytes(range)?;
3902 match decode_index_container(&bytes, self.codec, self.perms) {
3903 Ok(g) => {
3904 g.set_size_hint(u64::from(g.triple_count()));
3907 Some(g)
3908 }
3909 Err(_) => {
3910 self.fail();
3911 None
3912 }
3913 }
3914 } else {
3915 match open_index_container_lazy(
3916 &self.reader,
3917 range,
3918 self.codec,
3919 self.has_synopsis,
3920 self.read_concurrency,
3921 self.perms,
3922 self.perms.len(),
3923 ) {
3924 Ok((g, _, _)) => {
3925 let cap = self.tile_cap.load(std::sync::atomic::Ordering::Relaxed);
3929 if cap != u64::MAX {
3930 g.set_cache_cap(cap);
3931 }
3932 Some(g)
3933 }
3934 Err(_) => {
3935 self.fail();
3936 None
3937 }
3938 }
3939 }
3940 }
3941
3942 fn release(&mut self, iri: &str) {
3948 let Some((_, table)) = self.dir.get_mut() else {
3949 return;
3950 };
3951 for slab in table.iter_mut() {
3952 let Some(entries) = slab.get_mut() else {
3953 continue;
3954 };
3955 for e in entries.iter_mut() {
3956 let matches = e
3957 .meta
3958 .get()
3959 .map(|(name, _)| name.as_str() == iri)
3960 .unwrap_or(false);
3961 if matches {
3962 e.index.take();
3963 return;
3964 }
3965 }
3966 }
3967 }
3968
3969 fn for_each_opened(&self, mut f: impl FnMut(&GraphIndex)) {
3971 if let Some((_, table)) = self.dir.get() {
3972 for slab in table.iter().filter_map(|s| s.get()) {
3973 for e in slab.iter() {
3974 if let Some(g) = e.index.get() {
3975 f(g);
3976 }
3977 }
3978 }
3979 }
3980 }
3981}
3982
3983enum NamedGraphsSlot {
3988 Resident(Vec<(String, GraphIndex)>),
3989 Lazy(LazyNamedGraphs),
3990 Quad(crate::quad::QuadNamed),
3993}
3994
3995impl NamedGraphsSlot {
3996 fn directory_bytes(&self) -> u64 {
4000 match self {
4001 NamedGraphsSlot::Resident(v) => v
4002 .iter()
4003 .map(|(iri, g)| iri.capacity() as u64 + g.directory_bytes())
4004 .sum(),
4005 NamedGraphsSlot::Lazy(l) => {
4006 let mut bytes = l.walk.lock().map(|w| w.buf.capacity() as u64).unwrap_or(0);
4007 if let Some((_, table)) = l.dir.get() {
4008 bytes += table.len() as u64
4009 * std::mem::size_of::<std::sync::OnceLock<Box<[NamedEntry]>>>() as u64;
4010 for slab in table.iter().filter_map(|s| s.get()) {
4011 bytes += slab.len() as u64 * std::mem::size_of::<NamedEntry>() as u64;
4012 for e in slab.iter() {
4013 if let Some((iri, _)) = e.meta.get() {
4014 bytes += iri.capacity() as u64;
4015 }
4016 if let Some(g) = e.index.get() {
4017 bytes += g.directory_bytes();
4018 }
4019 }
4020 }
4021 }
4022 bytes
4023 }
4024 NamedGraphsSlot::Quad(q) => q.directory_bytes(),
4025 }
4026 }
4027
4028 fn count(&self) -> usize {
4029 match self {
4030 NamedGraphsSlot::Resident(v) => v.len(),
4031 NamedGraphsSlot::Lazy(l) => l.count(),
4032 NamedGraphsSlot::Quad(q) => q.count(),
4033 }
4034 }
4035
4036 fn name_at<'a>(&'a self, dict: &Dictionary, i: usize, exhaustive: bool) -> Option<&'a str> {
4040 match self {
4041 NamedGraphsSlot::Resident(v) => v.get(i).map(|(iri, _)| iri.as_str()),
4042 NamedGraphsSlot::Lazy(l) => l.name_at(i, exhaustive),
4043 NamedGraphsSlot::Quad(q) => q.name_at(dict, i, exhaustive),
4044 }
4045 }
4046
4047 fn name_owned(&self, dict: &Dictionary, i: usize, exhaustive: bool) -> Option<String> {
4050 match self {
4051 NamedGraphsSlot::Quad(q) => q.name_owned(dict, i, exhaustive),
4052 _ => self.name_at(dict, i, exhaustive).map(str::to_owned),
4053 }
4054 }
4055
4056 fn graph_at<'a>(
4057 &'a self,
4058 dict: &Dictionary,
4059 i: usize,
4060 exhaustive: bool,
4061 ) -> Option<(&'a str, &'a GraphIndex)> {
4062 match self {
4063 NamedGraphsSlot::Resident(v) => v.get(i).map(|(iri, g)| (iri.as_str(), g)),
4064 NamedGraphsSlot::Lazy(l) => l.graph_at(i, exhaustive),
4065 NamedGraphsSlot::Quad(q) => q.graph_at(dict, i, exhaustive),
4066 }
4067 }
4068
4069 fn find<'a>(&'a self, dict: &Dictionary, iri: &str) -> Option<&'a GraphIndex> {
4070 match self {
4071 NamedGraphsSlot::Quad(q) => q.find(dict, iri),
4072 NamedGraphsSlot::Resident(v) => v.iter().find(|(name, _)| name == iri).map(|(_, g)| g),
4073 NamedGraphsSlot::Lazy(l) => {
4074 for i in 0..l.count() {
4079 if l.name_at(i, false)? == iri {
4080 return l.graph_at(i, false).map(|(_, g)| g);
4081 }
4082 }
4083 None
4084 }
4085 }
4086 }
4087
4088 fn load_incomplete(&self) -> bool {
4089 match self {
4090 NamedGraphsSlot::Resident(v) => v.iter().any(|(_, g)| g.load_incomplete()),
4091 NamedGraphsSlot::Lazy(l) => {
4092 if l.failed.load(std::sync::atomic::Ordering::Relaxed) {
4093 return true;
4094 }
4095 let mut bad = false;
4096 l.for_each_opened(|g| bad |= g.load_incomplete());
4097 bad
4098 }
4099 NamedGraphsSlot::Quad(q) => q.load_incomplete(),
4100 }
4101 }
4102
4103 fn reset_load_failures(&self) {
4104 match self {
4105 NamedGraphsSlot::Resident(v) => {
4106 for (_, g) in v {
4107 g.reset_load_failure();
4108 }
4109 }
4110 NamedGraphsSlot::Lazy(l) => {
4111 l.failed.store(false, std::sync::atomic::Ordering::Relaxed);
4112 l.for_each_opened(|g| g.reset_load_failure());
4113 }
4114 NamedGraphsSlot::Quad(q) => q.reset_load_failures(),
4115 }
4116 }
4117
4118 fn resident(
4122 header: &Header,
4123 quad_section: impl FnOnce(u64, u64) -> Result<Vec<u8>, FileError>,
4124 per_graph: impl FnOnce() -> Result<Vec<(String, GraphIndex)>, FileError>,
4125 ) -> Result<Self, FileError> {
4126 check_named_layout(header)?;
4127 if let Some(q) = header.quad_index_section() {
4128 let bytes = quad_section(q.offset, q.length)?;
4129 return Ok(NamedGraphsSlot::Quad(crate::quad::QuadNamed::resident(
4130 bytes,
4131 header.block_codec,
4132 header.perms,
4133 )));
4134 }
4135 Ok(NamedGraphsSlot::Resident(if header.named_graphs_len > 0 {
4136 per_graph()?
4137 } else {
4138 Vec::new()
4139 }))
4140 }
4141}
4142
4143pub struct Rete {
4145 header: Header,
4146 dict: Dictionary,
4147 index: GraphIndex,
4148 index_section_ranges: [ByteRange; NUM_PERMS],
4149 tile_ranges: [Vec<(u32, u32, ByteRange)>; NUM_PERMS],
4153 pyramid: PyramidSlot,
4154 text_index: TextIndexSlot,
4155 labels: SectionSlot<crate::label_index::LabelIndex>,
4157 schema: SectionSlot<crate::schema_section::SchemaMeta>,
4159 named_graphs: NamedGraphsSlot,
4160 metadata: Vec<u8>,
4165 service_client: Option<Box<dyn crate::service::ServiceClient>>,
4170 service_error: std::sync::Mutex<Option<String>>,
4174 spill_threshold: std::sync::atomic::AtomicU64,
4177 v06: Option<std::sync::Arc<crate::v06::open::V06File>>,
4179}
4180
4181impl Rete {
4182 pub fn open(bytes: &[u8]) -> Result<Self, FileError> {
4185 #[cfg(feature = "v06-parity")]
4186 if let Some(v6) = parity_image(bytes) {
4187 return Self::open(&v6);
4188 }
4189 if crate::v06::open::is_v06(bytes) {
4190 let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
4193 std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(bytes)));
4194 let mut rete = Self::open_v06(r, bytes, false)?;
4195 rete.metadata = rete
4196 .v06
4197 .as_ref()
4198 .and_then(|f| f.card().ok().flatten())
4199 .unwrap_or_default();
4200 return Ok(rete);
4201 }
4202 let header = Header::from_bytes(bytes)?;
4203
4204 let region = |off: u64, len: u64| -> Result<&[u8], FileError> {
4208 checked_slice(bytes, off, len)
4209 .ok_or(FileError::Container("section range out of bounds"))
4210 };
4211
4212 let dict = decode_dictionary_container(
4213 region(header.dictionary_offset, header.dictionary_len)?,
4214 header.dict_codec,
4215 )?;
4216
4217 check_named_layout(&header)?;
4218 let index_bytes = region(header.root_dir_offset, header.root_dir_len)?;
4219 let (index, _) = decode_root_index_container(index_bytes, &header)?;
4220 let index_section_ranges = decode_index_section_ranges(
4221 index_bytes,
4222 header.root_dir_offset,
4223 header.perms,
4224 header.index_container_sections(),
4225 )?;
4226
4227 let pyramid = PyramidSlot::Resident(if header.pyramid_meta_len > 0 {
4228 Some(
4229 PyramidMeta::decode(region(header.pyramid_meta_offset, header.pyramid_meta_len)?)
4230 .map_err(|_| FileError::Container("malformed pyramid meta"))?,
4231 )
4232 } else {
4233 None
4234 });
4235
4236 let (ti_off, ti_len) = header.text_index_payload();
4239 let text_index = resident_text_index_slot(
4240 if ti_len > 0 {
4241 Some(region(ti_off, ti_len)?)
4242 } else {
4243 None
4244 },
4245 header.block_codec,
4246 )?;
4247 let labels = resident_label_slot(&header, ®ion);
4248 let schema = resident_schema_slot(&header, ®ion);
4249
4250 let named_graphs = NamedGraphsSlot::resident(
4251 &header,
4252 |off, len| Ok(region(off, len)?.to_vec()),
4253 || {
4254 decode_named_graphs(
4255 region(header.named_graphs_offset, header.named_graphs_len)?,
4256 header.block_codec,
4257 header.perms,
4258 )
4259 },
4260 )?;
4261
4262 let metadata = if header.metadata_len > 0 {
4263 region(header.metadata_offset, header.metadata_len)?.to_vec()
4264 } else {
4265 Vec::new()
4266 };
4267
4268 let tile_ranges =
4269 tile_file_ranges(index_bytes, header.root_dir_offset, &index_section_ranges);
4270 Ok(Self {
4271 header,
4272 dict,
4273 index,
4274 index_section_ranges,
4275 tile_ranges,
4276 pyramid,
4277 text_index,
4278 labels,
4279 schema,
4280 named_graphs,
4281 metadata,
4282 service_client: None,
4283 service_error: std::sync::Mutex::new(None),
4284 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
4285 v06: None,
4286 })
4287 }
4288
4289 pub fn set_service_client(&mut self, client: Box<dyn crate::service::ServiceClient>) {
4294 self.service_client = Some(client);
4295 }
4296
4297 pub(crate) fn service_client(&self) -> Option<&dyn crate::service::ServiceClient> {
4298 self.service_client.as_deref()
4299 }
4300
4301 pub(crate) fn record_service_error(&self, msg: &str) {
4303 let mut e = self.service_error.lock().unwrap();
4304 if e.is_none() {
4305 *e = Some(msg.to_string());
4306 }
4307 }
4308
4309 pub(crate) fn take_service_error(&self) -> Option<String> {
4312 self.service_error.lock().unwrap().take()
4313 }
4314
4315 pub fn header(&self) -> &Header {
4316 &self.header
4317 }
4318
4319 pub fn file_layout(&self) -> Vec<LayoutSegment> {
4325 if let Some(f) = &self.v06 {
4326 return v06_layout(f);
4327 }
4328 let h = &self.header;
4329 let seg = |kind: &'static str, label: String, offset: u64, len: u64| LayoutSegment {
4330 kind,
4331 label,
4332 offset,
4333 len,
4334 };
4335 let mut out = vec![seg(
4341 "header",
4342 format!("header (fixed {} bytes)", crate::header::HEADER_LEN),
4343 0,
4344 crate::header::HEADER_LEN as u64,
4345 )];
4346 if h.metadata_len > 0 {
4347 out.push(seg(
4348 "metadata",
4349 "metadata (dataset card)".into(),
4350 h.metadata_offset,
4351 h.metadata_len,
4352 ));
4353 }
4354 out.push(seg(
4355 "dictionary",
4356 "dictionary (4 front-coded term sections)".into(),
4357 h.dictionary_offset,
4358 h.dictionary_len,
4359 ));
4360 for (si, perm) in crate::index::ALL_PERMS.into_iter().enumerate() {
4361 let sec = self.index_section_ranges[si];
4362 if sec.len == 0 {
4363 continue;
4364 }
4365 let first_tile = self.tile_ranges[si]
4366 .first()
4367 .map(|&(_, _, r)| r.offset)
4368 .unwrap_or(sec.offset + sec.len);
4369 if first_tile > sec.offset {
4370 out.push(seg(
4371 "directory",
4372 format!("{} tile directory", perm.name()),
4373 sec.offset,
4374 first_tile - sec.offset,
4375 ));
4376 }
4377 for (ti, &(min_a, max_a, r)) in self.tile_ranges[si].iter().enumerate() {
4378 out.push(seg(
4379 "tile",
4380 format!("{} tile {ti} (leading ids {min_a}..{max_a})", perm.name()),
4381 r.offset,
4382 r.len,
4383 ));
4384 }
4385 }
4386 if h.pyramid_meta_len > 0 {
4387 out.push(seg(
4388 "pyramid",
4389 "pyramid summary (communities + superedges)".into(),
4390 h.pyramid_meta_offset,
4391 h.pyramid_meta_len,
4392 ));
4393 }
4394 if h.named_graphs_len > 0 {
4395 out.push(seg(
4396 "named-graphs",
4397 format!("named graphs ({})", self.named_graphs.count()),
4398 h.named_graphs_offset,
4399 h.named_graphs_len,
4400 ));
4401 }
4402 if let Some(s) = h.label_index_section() {
4403 let label = match self.labels.get_if_loaded() {
4404 Some(li) => format!("label index ({} entries)", li.entry_count()),
4405 None => "label index".to_string(),
4406 };
4407 out.push(seg("label-index", label, s.offset, s.length));
4408 }
4409 if let Some(s) = h.schema_section() {
4410 out.push(seg(
4411 "schema",
4412 "schema (classes, query_stats, shapes)".into(),
4413 s.offset,
4414 s.length,
4415 ));
4416 }
4417 if let Some(q) = h.quad_index_section() {
4418 out.push(seg(
4419 "quad-index",
4420 format!("quad index ({} named graphs)", self.named_graphs.count()),
4421 q.offset,
4422 q.length,
4423 ));
4424 }
4425 out.sort_by_key(|s| s.offset);
4426 out
4427 }
4428
4429 pub fn metadata(&self) -> Option<&[u8]> {
4434 if self.metadata.is_empty() {
4435 None
4436 } else {
4437 Some(&self.metadata)
4438 }
4439 }
4440
4441 pub fn dictionary(&self) -> &Dictionary {
4442 &self.dict
4443 }
4444
4445 pub fn pyramid(&self) -> Option<&PyramidMeta> {
4447 match &self.pyramid {
4448 PyramidSlot::Resident(p) => p.as_ref(),
4449 PyramidSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
4451 }
4452 }
4453
4454 pub fn pyramid_if_loaded(&self) -> Option<&PyramidMeta> {
4459 match &self.pyramid {
4460 PyramidSlot::Resident(p) => p.as_ref(),
4461 PyramidSlot::Lazy { cell, .. } => cell.get().and_then(|o| o.as_ref()),
4462 }
4463 }
4464
4465 pub fn schema(&self) -> Option<&crate::schema_section::SchemaMeta> {
4471 self.schema.get()
4472 }
4473
4474 pub fn schema_or_pyramid(
4478 &self,
4479 ) -> Option<std::borrow::Cow<'_, crate::schema_section::SchemaMeta>> {
4480 if let Some(s) = self.schema() {
4481 return Some(std::borrow::Cow::Borrowed(s));
4482 }
4483 self.pyramid()
4484 .map(|p| std::borrow::Cow::Owned(crate::schema_section::SchemaMeta::from_pyramid(p)))
4485 }
4486
4487 pub fn predicate_stats(&self) -> &[crate::meta::PredStat] {
4492 if let Some(s) = self.schema().filter(|s| !s.predicate_stats.is_empty()) {
4493 return &s.predicate_stats;
4494 }
4495 self.pyramid_if_loaded()
4496 .map(|p| p.predicate_stats.as_slice())
4497 .unwrap_or(&[])
4498 }
4499
4500 pub(crate) fn planner_stats(&self, fault: bool) -> &[crate::meta::PredStat] {
4505 let schema = if fault {
4506 self.schema.get()
4507 } else {
4508 self.schema.get_if_loaded()
4509 };
4510 if let Some(s) = schema.filter(|s| !s.predicate_stats.is_empty()) {
4511 return &s.predicate_stats;
4512 }
4513 self.pyramid_if_loaded()
4514 .map(|p| p.predicate_stats.as_slice())
4515 .unwrap_or(&[])
4516 }
4517
4518 pub fn char_sets(&self) -> &[crate::meta::CharSet] {
4521 if let Some(s) = self.schema().filter(|s| !s.char_sets.is_empty()) {
4522 return &s.char_sets;
4523 }
4524 self.pyramid_if_loaded()
4525 .map(|p| p.char_sets.as_slice())
4526 .unwrap_or(&[])
4527 }
4528
4529 pub fn label_index(&self) -> &[crate::meta::LabelEntry] {
4535 self.pyramid_if_loaded()
4536 .map(|p| p.label_index.as_slice())
4537 .unwrap_or(&[])
4538 }
4539
4540 pub fn has_label_index(&self) -> bool {
4542 self.header.label_index_section().is_some()
4543 }
4544
4545 pub fn label_index_section(&self) -> Option<&crate::label_index::LabelIndex> {
4547 self.labels.get()
4548 }
4549
4550 pub fn prefix_search(&self, prefix: &str, limit: usize) -> Vec<(String, String)> {
4557 if let Some(li) = self.labels.get() {
4558 return li.prefix_search(prefix, limit);
4559 }
4560 let Some(pyr) = self.pyramid() else {
4561 return Vec::new();
4562 };
4563 resolve_label_hits(&self.dict, pyr.prefix_search(prefix, limit))
4564 }
4565
4566 pub(crate) fn text_index(&self) -> Option<&crate::text_index::TextIndex> {
4569 self.text_index.index()
4570 }
4571
4572 pub fn has_text_index(&self) -> bool {
4575 self.header.text_index_len > 0
4576 }
4577
4578 pub fn text_index_scope(&self) -> Option<crate::text_index::TextIndexScope> {
4582 self.text_index().map(|ti| ti.scope().clone())
4583 }
4584
4585 pub fn text_index_token_table_len(&self) -> Option<u64> {
4600 self.text_index.token_table_len()
4601 }
4602
4603 pub fn text_search(&self, words: &[&str], prefix: Option<&str>, limit: usize) -> Vec<String> {
4613 text_search_in(self.text_index(), &self.dict, words, prefix, limit)
4614 }
4615
4616 pub fn default_index(&self) -> &GraphIndex {
4618 &self.index
4619 }
4620
4621 pub fn dump(&self, graph: Option<&str>) -> Vec<TermTriple> {
4623 let index = match graph {
4624 None => &self.index,
4625 Some(g) => match self.graph_index(g) {
4626 Some(i) => i,
4627 None => return Vec::new(),
4628 },
4629 };
4630 index
4631 .match_pattern((None, None, None))
4632 .into_iter()
4633 .filter_map(|(s, p, o)| {
4634 Some((
4635 self.dict.subject_term(s)?,
4636 self.dict.predicate_term(p)?,
4637 self.dict.object_term(o)?,
4638 ))
4639 })
4640 .collect()
4641 }
4642
4643 pub fn dump_each<F: FnMut(&str, &str, &str)>(&self, graph: Option<&str>, f: F) {
4659 self.dump_filtered_each(graph, None, None, None, f)
4660 }
4661
4662 pub fn dump_filtered_each<F: FnMut(&str, &str, &str)>(
4733 &self,
4734 graph: Option<&str>,
4735 s: Option<&str>,
4736 p: Option<&str>,
4737 o: Option<&str>,
4738 mut f: F,
4739 ) {
4740 let index = match graph {
4741 None => Some(&self.index),
4742 Some(g) => self.graph_index(g),
4743 };
4744 let Some(index) = index else { return };
4745 let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
4748 return;
4749 };
4750 let dict = &self.dict;
4751 const WINDOW: usize = 4096;
4796 let cap = dict.cache_cap();
4807 let prefetch_window =
4808 cap == u64::MAX || cap >= (WINDOW as u64) * 2 * (DICT_CHUNK_BUDGET as u64);
4809 const PREFETCH_BATCH: usize = 64;
4816 let prefetch_batch = if cap == u64::MAX {
4817 usize::MAX
4818 } else {
4819 PREFETCH_BATCH
4820 };
4821 let mut ids: Vec<(u32, u32, u32)> = Vec::with_capacity(WINDOW);
4822 let mut nodes: Vec<u32> = Vec::with_capacity(WINDOW * 2);
4823 let mut preds: Vec<u32> = Vec::with_capacity(WINDOW);
4824 let mut resolver = crate::dictionary::WindowResolver::default();
4825 let mut flush = |ids: &mut Vec<(u32, u32, u32)>,
4826 resolver: &mut crate::dictionary::WindowResolver,
4827 f: &mut F| {
4828 if ids.is_empty() {
4829 return;
4830 }
4831 if prefetch_window {
4835 nodes.clear();
4836 preds.clear();
4837 for &(s, p, o) in ids.iter() {
4838 nodes.push(dict.subject_node(s));
4839 nodes.push(dict.object_node(o));
4840 preds.push(p);
4841 }
4842 dict.prefetch_terms_batched(&nodes, &preds, prefetch_batch);
4843 }
4844 dict.resolve_window(ids, resolver, |s, p, o| f(s, p, o));
4845 ids.clear();
4846 };
4847 for t in index.scan_iter(pattern) {
4848 ids.push(t);
4849 if ids.len() == WINDOW {
4850 flush(&mut ids, &mut resolver, &mut f);
4851 }
4852 }
4853 flush(&mut ids, &mut resolver, &mut f);
4854 }
4855
4856 pub fn dump_plan(
4864 &self,
4865 graph: Option<&str>,
4866 s: Option<&str>,
4867 p: Option<&str>,
4868 o: Option<&str>,
4869 ) -> DumpPlan {
4870 let index = match graph {
4871 None => Some(&self.index),
4872 Some(g) => self.graph_index(g),
4873 };
4874 let scan = index
4875 .zip(self.resolve_query_pattern(s, p, o))
4876 .map(|(ix, pattern)| ix.scan_plan(pattern));
4877 DumpPlan {
4878 scan,
4879 dictionary_bytes: match &self.v06 {
4880 Some(f) => f.section(crate::v06::kind::DICT_DATA).map_or(0, |r| r.len),
4881 None => self.header.dictionary_len,
4882 },
4883 }
4884 }
4885
4886 pub fn dump_iter(&self, graph: Option<&str>) -> impl Iterator<Item = TermTriple> + '_ {
4904 self.query_iter(graph, None, None, None)
4905 }
4906
4907 pub fn query_iter(
4918 &self,
4919 graph: Option<&str>,
4920 s: Option<&str>,
4921 p: Option<&str>,
4922 o: Option<&str>,
4923 ) -> impl Iterator<Item = TermTriple> + '_ {
4924 let index = match graph {
4925 None => Some(&self.index),
4926 Some(g) => self.graph_index(g),
4927 };
4928 index
4929 .zip(self.resolve_query_pattern(s, p, o))
4930 .into_iter()
4931 .flat_map(|(ix, pattern)| ix.scan_iter(pattern))
4932 .filter_map(move |(s, p, o)| {
4933 Some((
4934 self.dict.subject_term(s)?,
4935 self.dict.predicate_term(p)?,
4936 self.dict.object_term(o)?,
4937 ))
4938 })
4939 }
4940
4941 pub fn query_batch(
4960 &self,
4961 graph: Option<&str>,
4962 s: Option<&str>,
4963 p: Option<&str>,
4964 o: Option<&str>,
4965 cursor: u64,
4966 max_quads: usize,
4967 ) -> (Vec<TermTriple>, u64, bool) {
4968 let index = match graph {
4969 None => &self.index,
4970 Some(g) => match self.graph_index(g) {
4973 Some(i) => i,
4974 None => return (Vec::new(), cursor, true),
4975 },
4976 };
4977 let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
4980 return (Vec::new(), cursor, true);
4981 };
4982 let (ids, next, done) = index.scan_batch(pattern, cursor, max_quads);
4983 if !ids.is_empty() {
4987 let mut nodes = Vec::with_capacity(ids.len() * 2);
4988 let mut preds = Vec::with_capacity(ids.len());
4989 for &(s, p, o) in &ids {
4990 nodes.push(self.dict.subject_node(s));
4991 nodes.push(self.dict.object_node(o));
4992 preds.push(p);
4993 }
4994 self.dict.prefetch_terms(&nodes, &preds);
4995 }
4996 let triples = ids
4997 .into_iter()
4998 .filter_map(|(s, p, o)| {
4999 Some((
5000 self.dict.subject_term(s)?,
5001 self.dict.predicate_term(p)?,
5002 self.dict.object_term(o)?,
5003 ))
5004 })
5005 .collect();
5006 (triples, next, done)
5007 }
5008
5009 pub fn dump_batch(
5035 &self,
5036 graph: Option<&str>,
5037 cursor: u32,
5038 max_quads: usize,
5039 ) -> (Vec<TermTriple>, u32, bool) {
5040 let index = match graph {
5041 None => self.default_index(),
5042 Some(g) => match self.graph_index(g) {
5045 Some(i) => i,
5046 None => return (Vec::new(), cursor, true),
5047 },
5048 };
5049 let spo = IndexPermutation::Spo.section_index();
5053 let Some((first, last)) = index.leading_bounds(spo) else {
5054 return (Vec::new(), cursor, true); };
5056
5057 let mut sid = cursor.max(first);
5058 let mut probes = max_quads.saturating_mul(4).max(1 << 16);
5059 let mut ids: Vec<(u32, u32, u32)> = Vec::new();
5060 let mut exhausted = false;
5061 while sid <= last && probes > 0 {
5062 probes -= 1;
5063 let hits = index.match_pattern((Some(sid), None, None));
5064 if hits.is_empty() {
5065 match index.next_leading(IndexPermutation::Spo.section_index(), sid) {
5075 Some(next) if next > sid => {
5076 sid = next;
5077 continue;
5078 }
5079 None if !index.load_incomplete() => {
5080 exhausted = true;
5081 break;
5082 }
5083 _ => {}
5084 }
5085 match index.next_tile_start(spo, sid) {
5086 None => {
5087 exhausted = true;
5088 break;
5089 }
5090 Some(next) if next > sid => {
5091 sid = next;
5092 continue;
5093 }
5094 Some(_) => {}
5095 }
5096 } else {
5097 ids.extend(hits);
5098 }
5099 if sid == u32::MAX {
5100 exhausted = true;
5101 break;
5102 }
5103 sid += 1;
5104 if ids.len() >= max_quads {
5105 break;
5106 }
5107 }
5108 let done = exhausted || sid > last;
5109
5110 let dict = self.dictionary();
5117 if !ids.is_empty() {
5118 let mut nodes = Vec::with_capacity(ids.len() * 2);
5119 let mut preds = Vec::with_capacity(ids.len());
5120 for &(s, p, o) in &ids {
5121 nodes.push(dict.subject_node(s));
5122 nodes.push(dict.object_node(o));
5123 preds.push(p);
5124 }
5125 dict.prefetch_terms(&nodes, &preds);
5126 }
5127 let triples = ids
5128 .into_iter()
5129 .filter_map(|(s, p, o)| {
5130 Some((
5131 dict.subject_term(s)?,
5132 dict.predicate_term(p)?,
5133 dict.object_term(o)?,
5134 ))
5135 })
5136 .collect();
5137 (triples, sid, done)
5138 }
5139
5140 pub fn named_graph_count(&self) -> usize {
5143 self.named_graphs.count()
5144 }
5145
5146 pub fn named_graph_quad_count(&self, i: usize) -> Option<u64> {
5151 match &self.named_graphs {
5152 NamedGraphsSlot::Quad(q) => q.quad_count_at(i),
5153 _ => None,
5154 }
5155 }
5156
5157 pub fn named_graph_transient(&self, i: usize) -> Option<GraphIndex> {
5166 match &self.named_graphs {
5167 NamedGraphsSlot::Quad(q) => q.transient_view(i, TRANSIENT_GRAPH_QUADS),
5168 _ => None,
5169 }
5170 }
5171
5172 pub(crate) fn named_graphs_with_subjects(
5179 &self,
5180 subjects: &std::collections::BTreeSet<u32>,
5181 ) -> Option<Vec<usize>> {
5182 match &self.named_graphs {
5183 NamedGraphsSlot::Quad(q) => q.graphs_with_subjects(subjects),
5184 _ => None,
5185 }
5186 }
5187
5188 pub fn named_quad_count(&self) -> Option<u64> {
5193 match &self.named_graphs {
5194 NamedGraphsSlot::Quad(q) => Some(q.quad_count()),
5195 _ => None,
5196 }
5197 }
5198
5199 pub(crate) fn index_triples(&self, index: &GraphIndex) -> u64 {
5205 index.size_hint().unwrap_or(self.header().quad_count)
5206 }
5207
5208 pub fn named_graph_name_at(&self, i: usize) -> Option<&str> {
5211 self.named_graphs.name_at(&self.dict, i, false)
5212 }
5213
5214 pub fn named_graph_at(&self, i: usize) -> Option<(&str, &GraphIndex)> {
5218 self.named_graphs.graph_at(&self.dict, i, false)
5219 }
5220
5221 pub(crate) fn named_graph_name_owned_demand(
5229 &self,
5230 i: usize,
5231 exhaustive: bool,
5232 ) -> Option<String> {
5233 self.named_graphs.name_owned(&self.dict, i, exhaustive)
5234 }
5235
5236 pub(crate) fn named_graph_node_demand(&self, i: usize, exhaustive: bool) -> Option<u32> {
5240 match &self.named_graphs {
5241 NamedGraphsSlot::Quad(q) => q.node_at(&self.dict, i, exhaustive),
5242 _ => None,
5243 }
5244 }
5245
5246 pub(crate) fn named_graph_ordinal(&self, iri: &str) -> Option<usize> {
5250 match &self.named_graphs {
5251 NamedGraphsSlot::Quad(q) => q.ordinal_of_iri(&self.dict, iri),
5252 _ => None,
5253 }
5254 }
5255
5256 pub(crate) fn named_graph_ordinal_of_node(&self, node: u32) -> Option<usize> {
5260 match &self.named_graphs {
5261 NamedGraphsSlot::Quad(q) => q.ordinal_of_node(&self.dict, node),
5262 _ => None,
5263 }
5264 }
5265
5266 pub(crate) fn named_graph_ordinal_of_id(&self, g: u32) -> Option<usize> {
5269 match &self.named_graphs {
5270 NamedGraphsSlot::Quad(q) => q.ordinal_of_id(g),
5271 _ => None,
5272 }
5273 }
5274
5275 pub(crate) fn named_quad_scan(
5280 &self,
5281 pat: (Option<u32>, Option<u32>, Option<u32>),
5282 ) -> Option<crate::quad::QuadScan> {
5283 match &self.named_graphs {
5284 NamedGraphsSlot::Quad(q) => q.scan(pat),
5285 _ => None,
5286 }
5287 }
5288
5289 pub(crate) fn quad_named(&self) -> Option<&crate::quad::QuadNamed> {
5291 match &self.named_graphs {
5292 NamedGraphsSlot::Quad(q) => Some(q),
5293 _ => None,
5294 }
5295 }
5296
5297 pub(crate) fn named_graph_at_demand(
5300 &self,
5301 i: usize,
5302 exhaustive: bool,
5303 ) -> Option<(&str, &GraphIndex)> {
5304 self.named_graphs.graph_at(&self.dict, i, exhaustive)
5305 }
5306
5307 pub fn graph_names(&self) -> Vec<&str> {
5312 (0..self.named_graphs.count())
5313 .filter_map(|i| self.named_graphs.name_at(&self.dict, i, false))
5314 .collect()
5315 }
5316
5317 pub fn graph_index(&self, iri: &str) -> Option<&GraphIndex> {
5319 self.named_graphs.find(&self.dict, iri)
5320 }
5321
5322 pub fn match_ids(
5325 &self,
5326 pattern: (Option<u32>, Option<u32>, Option<u32>),
5327 ) -> Vec<(u32, u32, u32)> {
5328 self.index.match_pattern(pattern)
5329 }
5330
5331 pub fn predicate_pairs(&self, predicate: &str) -> Vec<(u32, u32)> {
5334 let pid = match self.dict.predicate_id(predicate) {
5335 Some(p) => p,
5336 None => return Vec::new(),
5337 };
5338 self.index
5339 .match_pattern((None, Some(pid), None))
5340 .into_iter()
5341 .map(|(s, _p, o)| (self.dict.subject_node(s), self.dict.object_node(o)))
5342 .collect()
5343 }
5344
5345 pub fn open_ranged<R: RangeReader>(reader: &R) -> Result<Self, FileError> {
5349 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
5350 if crate::v06::open::is_v06(&head) {
5351 let mut all = head;
5354 let at = all.len() as u64;
5355 if reader.len() > at {
5356 all.extend_from_slice(&reader.read_at(at, reader.len() - at)?);
5357 }
5358 return Self::open(&all);
5359 }
5360 let header = Header::from_bytes(&head)?;
5361
5362 let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
5363 let dict = decode_dictionary_container(&dict_bytes, header.dict_codec)?;
5364
5365 check_named_layout(&header)?;
5366 let index_bytes = reader.read_at(header.root_dir_offset, header.root_dir_len)?;
5367 let (index, _) = decode_root_index_container(&index_bytes, &header)?;
5368 let index_section_ranges = decode_index_section_ranges(
5369 &index_bytes,
5370 header.root_dir_offset,
5371 header.perms,
5372 header.index_container_sections(),
5373 )?;
5374
5375 let pyramid = PyramidSlot::Resident(if header.pyramid_meta_len > 0 {
5376 let mb = reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?;
5377 Some(
5378 PyramidMeta::decode(&mb)
5379 .map_err(|_| FileError::Container("malformed pyramid meta"))?,
5380 )
5381 } else {
5382 None
5383 });
5384
5385 let (ti_off, ti_len) = header.text_index_payload();
5388 let text_index_bytes = if ti_len > 0 {
5389 Some(reader.read_at(ti_off, ti_len)?)
5390 } else {
5391 None
5392 };
5393 let text_index = resident_text_index_slot(text_index_bytes.as_deref(), header.block_codec)?;
5394 let labels = SectionSlot::Resident(header.label_index_section().and_then(|s| {
5397 let b = reader.read_at(s.offset, s.length).ok()?;
5398 crate::label_index::LabelIndex::from_section(&b, header.block_codec).ok()
5399 }));
5400 let schema = SectionSlot::Resident(read_schema_with_header(reader, &header));
5401
5402 let named_graphs = NamedGraphsSlot::resident(
5403 &header,
5404 |off, len| Ok(reader.read_at(off, len)?),
5405 || {
5406 let nb = reader.read_at(header.named_graphs_offset, header.named_graphs_len)?;
5407 decode_named_graphs(&nb, header.block_codec, header.perms)
5408 },
5409 )?;
5410
5411 let tile_ranges =
5415 tile_file_ranges(&index_bytes, header.root_dir_offset, &index_section_ranges);
5416 Ok(Self {
5417 header,
5418 dict,
5419 index,
5420 index_section_ranges,
5421 tile_ranges,
5422 pyramid,
5423 text_index,
5424 labels,
5425 schema,
5426 named_graphs,
5427 metadata: Vec::new(),
5428 service_client: None,
5429 service_error: std::sync::Mutex::new(None),
5430 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5431 v06: None,
5432 })
5433 }
5434
5435 fn open_v06(
5441 reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
5442 first: &[u8],
5443 budget: bool,
5444 ) -> Result<Self, FileError> {
5445 let f = std::sync::Arc::new(crate::v06::open::V06File::open(reader.clone(), first)?);
5446 let header = crate::v06::logical_header(&f.h);
5447 let rc = reader.concurrency();
5448 let dict = f.dictionary([true; 4])?;
5449 let index = f.index(rc)?;
5450 let (pyramid, text_index, labels, schema) = v06_slots(&f);
5451 let named_graphs = match f.quad(rc) {
5452 Some(q) => NamedGraphsSlot::Quad(q?),
5453 None => NamedGraphsSlot::Resident(Vec::new()),
5454 };
5455 let rete = Self {
5456 header,
5457 dict,
5458 index,
5459 index_section_ranges: [ByteRange::default(); NUM_PERMS],
5460 tile_ranges: Default::default(),
5461 pyramid,
5462 text_index,
5463 labels,
5464 schema,
5465 named_graphs,
5466 metadata: Vec::new(),
5467 service_client: None,
5468 service_error: std::sync::Mutex::new(None),
5469 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5470 v06: Some(f),
5471 };
5472 #[cfg(not(target_arch = "wasm32"))]
5473 if budget {
5474 rete.set_memory_budget(match crate::sysmem::default_query_budget_mb() {
5475 0 => None,
5476 mb => Some(mb << 20),
5477 });
5478 }
5479 #[cfg(target_arch = "wasm32")]
5480 let _ = budget;
5481 Ok(rete)
5482 }
5483
5484 pub fn v06(&self) -> Option<&crate::v06::open::V06File> {
5486 self.v06.as_deref()
5487 }
5488
5489 pub fn open_ranged_lazy<R: RangeReader + Send + Sync + 'static>(
5501 reader: R,
5502 ) -> Result<Self, FileError> {
5503 Self::open_ranged_lazy_with(reader, crate::v06::FIRST_READ)
5504 }
5505
5506 pub fn open_ranged_lazy_with<R: RangeReader + Send + Sync + 'static>(
5511 reader: R,
5512 first_read: u64,
5513 ) -> Result<Self, FileError> {
5514 let first_len = first_read.max(HEADER_LEN as u64).min(reader.len());
5517 let first: std::sync::Arc<[u8]> = std::sync::Arc::from(reader.read_at(0, first_len)?);
5518 #[cfg(feature = "v06-parity")]
5519 if !crate::v06::open::is_v06(&first) {
5520 let all = reader.read_at(0, reader.len())?;
5521 if let Some(v6) = parity_image(&all) {
5522 let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
5523 std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(v6)));
5524 let head = r.read_at(0, crate::v06::FIRST_READ.min(r.len()))?;
5525 return Self::open_v06(r, &head, true);
5526 }
5527 }
5528 let reader = crate::reader::PrefixReader::new(reader, first.clone());
5529 if crate::v06::open::is_v06(&first) {
5530 let r: std::sync::Arc<dyn RangeReader + Send + Sync> = std::sync::Arc::new(reader);
5531 return Self::open_v06(r, &first, true);
5532 }
5533 let head = reader.read_at(0, HEADER_LEN as u64)?;
5534 let header = Header::from_bytes(&head)?;
5535 check_named_layout(&header)?;
5536 let reader = std::sync::Arc::new(reader);
5537 let read_concurrency = reader.concurrency();
5540
5541 let open_dict = || ranged_chunked_dictionary(&reader, &header, [true; 4]);
5545
5546 let reader_dyn: std::sync::Arc<dyn RangeReader + Send + Sync> = reader.clone();
5550 let open_index = || {
5551 open_index_container_lazy(
5552 &reader_dyn,
5553 ByteRange {
5554 offset: header.root_dir_offset,
5555 len: header.root_dir_len,
5556 },
5557 header.block_codec,
5558 header.has_tile_synopsis(),
5559 read_concurrency,
5560 header.perms,
5561 header.index_container_sections(),
5562 )
5563 };
5564
5565 let (dict, index) = open_side_by_side(read_concurrency, open_dict, open_index);
5572 let dict = dict?;
5573 let (index, index_section_ranges, tile_ranges) = index?;
5574
5575 let pyramid = ranged_pyramid_slot(&reader, &header);
5579
5580 let text_index = ranged_text_index_slot(&reader, &header);
5585
5586 let labels = ranged_label_slot(&reader, &header);
5590 let schema = ranged_schema_slot(&reader, &header);
5591
5592 let named_graphs = if let Some(q) = header.quad_index_section() {
5599 NamedGraphsSlot::Quad(crate::quad::QuadNamed::new(
5603 reader_dyn,
5604 ByteRange {
5605 offset: q.offset,
5606 len: q.length,
5607 },
5608 header.block_codec,
5609 header.perms,
5610 read_concurrency,
5611 ))
5612 } else if header.named_graphs_len > 0 {
5613 NamedGraphsSlot::Lazy(LazyNamedGraphs::new(
5614 reader_dyn,
5615 ByteRange {
5616 offset: header.named_graphs_offset,
5617 len: header.named_graphs_len,
5618 },
5619 header.block_codec,
5620 header.has_tile_synopsis(),
5621 read_concurrency,
5622 header.perms,
5623 ))
5624 } else {
5625 NamedGraphsSlot::Resident(Vec::new())
5626 };
5627
5628 let rete = Self {
5629 header,
5630 dict,
5631 index,
5632 index_section_ranges,
5633 tile_ranges,
5634 pyramid,
5635 text_index,
5636 labels,
5637 schema,
5638 named_graphs,
5639 metadata: Vec::new(),
5640 service_client: None,
5641 service_error: std::sync::Mutex::new(None),
5642 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5643 v06: None,
5644 };
5645 #[cfg(not(target_arch = "wasm32"))]
5657 rete.set_memory_budget(match crate::sysmem::default_query_budget_mb() {
5658 0 => None,
5659 mb => Some(mb << 20),
5660 });
5661 Ok(rete)
5662 }
5663
5664 pub fn index_incomplete(&self) -> bool {
5669 self.index.load_incomplete()
5670 || self.dict.load_incomplete()
5671 || self.named_graphs.load_incomplete()
5672 }
5673
5674 pub fn reset_load_failures(&self) {
5682 self.index.reset_load_failure();
5683 self.dict.reset_load_failure();
5684 self.named_graphs.reset_load_failures();
5685 }
5686
5687 pub fn set_memory_budget(&self, budget: Option<u64>) -> MemoryBudget {
5718 let total = budget.unwrap_or(u64::MAX);
5719 self.set_spill_threshold(budget.map(spill_threshold_for));
5720 let mut split = split_memory_budget(total);
5721 let fixed = if total == u64::MAX {
5722 0
5723 } else {
5724 self.fixed_overhead().total()
5725 };
5726 if split.dict_cache != u64::MAX {
5727 split.dict_cache = split
5728 .dict_cache
5729 .saturating_sub(fixed)
5730 .max(MIN_DICT_CACHE_CAP);
5731 }
5732 self.dict.set_cache_cap(split.dict_cache);
5733 self.index.set_cache_cap(split.tile_cache);
5734 match &self.named_graphs {
5739 NamedGraphsSlot::Lazy(lazy) => lazy.set_tile_cap(split.tile_cache),
5740 NamedGraphsSlot::Quad(q) => q.set_tile_cap(split.tile_cache),
5743 NamedGraphsSlot::Resident(_) => {}
5744 }
5745 if std::env::var("RETE_OPEN_DEBUG").is_ok() {
5746 eprintln!(
5747 "[memory-budget] total={} dict_cache={} tile_cache={} block_cache={} fixed={:.1}MiB",
5748 if total == u64::MAX {
5749 "unlimited".to_string()
5750 } else {
5751 format!("{}MiB", total >> 20)
5752 },
5753 fmt_cap(split.dict_cache),
5754 fmt_cap(split.tile_cache),
5755 fmt_cap(split.block_cache),
5756 fixed as f64 / (1u64 << 20) as f64,
5757 );
5758 }
5759 split
5760 }
5761
5762 pub fn spill_threshold(&self) -> u64 {
5769 self.spill_threshold
5770 .load(std::sync::atomic::Ordering::Relaxed)
5771 }
5772
5773 pub fn set_spill_threshold(&self, bytes: Option<u64>) {
5777 self.spill_threshold.store(
5778 bytes.unwrap_or(u64::MAX),
5779 std::sync::atomic::Ordering::Relaxed,
5780 );
5781 }
5782
5783 pub fn dict_cache_stats(&self) -> crate::chunk_cache::CacheStats {
5787 self.dict.cache_stats()
5788 }
5789
5790 pub fn fixed_overhead(&self) -> FixedOverhead {
5794 FixedOverhead {
5795 dict_directory: self.dict.directory_bytes(),
5796 index_directory: self.index.directory_bytes(),
5797 named_graphs: self.named_graphs.directory_bytes(),
5798 }
5799 }
5800
5801 pub fn index_cache_stats(&self) -> crate::chunk_cache::CacheStats {
5803 self.index.cache_stats()
5804 }
5805
5806 pub fn dict_cache_cap(&self) -> u64 {
5808 self.dict.cache_cap()
5809 }
5810
5811 pub fn release_named_graph(&mut self, iri: &str) {
5824 match &mut self.named_graphs {
5825 NamedGraphsSlot::Lazy(lazy) => lazy.release(iri),
5826 NamedGraphsSlot::Quad(q) => q.release(&self.dict, iri),
5827 NamedGraphsSlot::Resident(_) => {}
5828 }
5829 }
5830
5831 fn resolve_query_pattern(
5832 &self,
5833 s: Option<&str>,
5834 p: Option<&str>,
5835 o: Option<&str>,
5836 ) -> Option<Pattern> {
5837 let sid = match s {
5838 Some(t) => match self.dict.subject_id(t) {
5839 Some(id) => Some(id),
5840 None => return None,
5841 },
5842 None => None,
5843 };
5844 let pid = match p {
5845 Some(t) => match self.dict.predicate_id(t) {
5846 Some(id) => Some(id),
5847 None => return None,
5848 },
5849 None => None,
5850 };
5851 let oid = match o {
5852 Some(t) => match self.dict.object_id(t) {
5853 Some(id) => Some(id),
5854 None => return None,
5855 },
5856 None => None,
5857 };
5858 Some((sid, pid, oid))
5859 }
5860
5861 pub fn query_with_provenance(
5865 &self,
5866 s: Option<&str>,
5867 p: Option<&str>,
5868 o: Option<&str>,
5869 ) -> Vec<TripleProvenance> {
5870 let pattern = match self.resolve_query_pattern(s, p, o) {
5871 Some(pattern) => pattern,
5872 None => return Vec::new(),
5873 };
5874
5875 let index_permutation = GraphIndex::best_permutation_in(self.header.perms, pattern);
5876 let si = index_permutation.section_index();
5877 let (dictionary_range, index_range, index_section_range, pyramid_range) = match &self.v06 {
5878 Some(f) => {
5879 use crate::v06::kind;
5880 let sec = |k| f.section(k).unwrap_or_default();
5881 let perm = f
5882 .root
5883 .unit(crate::v06::tag::INDEX)
5884 .and_then(|u| crate::v06::decode_index_root(u).ok())
5885 .and_then(|v| v.into_iter().find(|p| p.si == si))
5886 .map(|p| ByteRange {
5887 offset: sec(kind::INDEX_DATA).offset + p.data_off,
5888 len: p.data_len,
5889 })
5890 .unwrap_or_default();
5891 (
5892 sec(kind::DICT_DATA),
5893 sec(kind::INDEX_DATA),
5894 perm,
5895 f.section(kind::PYRAMID),
5896 )
5897 }
5898 None => (
5899 ByteRange {
5900 offset: self.header.dictionary_offset,
5901 len: self.header.dictionary_len,
5902 },
5903 ByteRange {
5904 offset: self.header.root_dir_offset,
5905 len: self.header.root_dir_len,
5906 },
5907 self.index_section_ranges[si],
5908 (self.header.pyramid_meta_len > 0).then_some(ByteRange {
5909 offset: self.header.pyramid_meta_offset,
5910 len: self.header.pyramid_meta_len,
5911 }),
5912 ),
5913 };
5914
5915 let tiles = &self.tile_ranges[si];
5916 let v06_tiles = self.v06.is_some().then(|| self.index.tile_sections()[si]);
5917 self.index
5918 .match_pattern(pattern)
5919 .into_iter()
5920 .filter_map(|(s, p, o)| {
5921 let terms = (
5922 self.dict.subject_term(s)?,
5923 self.dict.predicate_term(p)?,
5924 self.dict.object_term(o)?,
5925 );
5926 let a = index_permutation.forward((s, p, o)).0;
5929 let (tile, tile_range) = if let Some(tl) = v06_tiles {
5930 let (ti, tj) = self.index.tile_span(si, Some(a));
5931 match (ti < tj && ti < tl.len()).then(|| tl.range(ti)).flatten() {
5932 Some((offset, len)) => (
5933 Some(format!("{}/{ti}", index_permutation.name())),
5934 Some(ByteRange { offset, len }),
5935 ),
5936 None => (None, None),
5937 }
5938 } else {
5939 let ti = tiles.partition_point(|&(_, max_a, _)| max_a < a);
5940 match tiles.get(ti) {
5941 Some(&(min_a, _, range)) if min_a <= a => (
5942 Some(format!("{}/{ti}", index_permutation.name())),
5943 Some(range),
5944 ),
5945 _ => (None, None),
5946 }
5947 };
5948 Some(TripleProvenance {
5949 terms,
5950 ids: (s, p, o),
5951 graph: None,
5952 matched_pattern: pattern,
5953 index_permutation,
5954 dictionary_range,
5955 index_range,
5956 index_section_range,
5957 pyramid_range,
5958 tile,
5959 tile_range,
5960 })
5961 })
5962 .collect()
5963 }
5964
5965 pub fn query(&self, s: Option<&str>, p: Option<&str>, o: Option<&str>) -> Vec<TermTriple> {
5969 self.query_with_provenance(s, p, o)
5970 .into_iter()
5971 .map(|m| m.terms)
5972 .collect()
5973 }
5974
5975 pub fn query_in_graph(
5983 &self,
5984 graph: Option<&str>,
5985 s: Option<&str>,
5986 p: Option<&str>,
5987 o: Option<&str>,
5988 ) -> Vec<TermTriple> {
5989 let pattern = match self.resolve_query_pattern(s, p, o) {
5990 Some(pattern) => pattern,
5991 None => return Vec::new(),
5992 };
5993 let index = match graph {
5994 None => &self.index,
5995 Some(g) => match self.graph_index(g) {
5996 Some(i) => i,
5997 None => return Vec::new(),
5998 },
5999 };
6000 self.resolve_matches(index.match_pattern(pattern))
6001 }
6002
6003 fn resolve_matches(&self, ids: Vec<Triple>) -> Vec<TermTriple> {
6006 if !ids.is_empty() {
6014 let mut nodes = Vec::with_capacity(ids.len() * 2);
6015 let mut preds = Vec::with_capacity(ids.len());
6016 for &(s, p, o) in &ids {
6017 nodes.push(self.dict.subject_node(s));
6018 nodes.push(self.dict.object_node(o));
6019 preds.push(p);
6020 }
6021 self.dict.prefetch_terms(&nodes, &preds);
6022 }
6023 ids.into_iter()
6024 .filter_map(|(s, p, o)| {
6025 Some((
6026 self.dict.subject_term(s)?,
6027 self.dict.predicate_term(p)?,
6028 self.dict.object_term(o)?,
6029 ))
6030 })
6031 .collect()
6032 }
6033
6034 pub fn query_quads(
6039 &self,
6040 s: Option<&str>,
6041 p: Option<&str>,
6042 o: Option<&str>,
6043 ) -> Vec<(TermTriple, Option<String>)> {
6044 let mut out: Vec<(TermTriple, Option<String>)> = self
6045 .query_in_graph(None, s, p, o)
6046 .into_iter()
6047 .map(|t| (t, None))
6048 .collect();
6049 let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
6050 return out;
6051 };
6052 for i in 0..self.named_graphs.count() {
6053 let Some(iri) = self.named_graphs.name_at(&self.dict, i, true) else {
6056 continue;
6057 };
6058 let owned = self.named_graph_transient(i);
6061 let index = match &owned {
6062 Some(view) => view,
6063 None => match self.named_graphs.graph_at(&self.dict, i, true) {
6064 Some((_, g)) => g,
6065 None => continue,
6066 },
6067 };
6068 for triple in self.resolve_matches(index.match_pattern(pattern)) {
6069 out.push((triple, Some(iri.to_string())));
6070 }
6071 }
6072 out
6073 }
6074
6075 pub fn query_ranged<R: RangeReader>(
6082 reader: &R,
6083 s: Option<&str>,
6084 p: Option<&str>,
6085 o: Option<&str>,
6086 ) -> Result<Vec<TermTriple>, FileError> {
6087 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6088 if crate::v06::open::is_v06(&head) {
6089 return Ok(v06_query_ranged(reader, head, s, p, o, true)?.unwrap_or_default());
6090 }
6091 let routed = match route_pattern(reader, head, s, p, o)? {
6092 Some(routed) => routed,
6093 None => return Ok(Vec::new()),
6094 };
6095 let matches = fetch_routed_matches(reader, &routed)?;
6096 Ok(matches
6097 .into_iter()
6098 .filter_map(|(s, p, o)| {
6099 Some((
6100 routed.dict.subject_term(s)?,
6101 routed.dict.predicate_term(p)?,
6102 routed.dict.object_term(o)?,
6103 ))
6104 })
6105 .collect())
6106 }
6107
6108 pub fn route_pattern_ranged<R: RangeReader>(
6112 reader: &R,
6113 s: Option<&str>,
6114 p: Option<&str>,
6115 o: Option<&str>,
6116 ) -> Result<bool, FileError> {
6117 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6118 if crate::v06::open::is_v06(&head) {
6119 return Ok(v06_query_ranged(reader, head, s, p, o, false)?.is_some());
6120 }
6121 Ok(route_pattern(reader, head, s, p, o)?.is_some())
6122 }
6123}
6124
6125fn v06_query_ranged<R: RangeReader>(
6131 reader: &R,
6132 head: Vec<u8>,
6133 s: Option<&str>,
6134 p: Option<&str>,
6135 o: Option<&str>,
6136 fetch: bool,
6137) -> Result<Option<Vec<TermTriple>>, FileError> {
6138 use crate::v06::{kind, open::SparseReader, Header06, Root06, HEADER_LEN};
6139 let len = reader.len();
6140 let h = Header06::from_bytes(&head)?;
6141 let root_end = (HEADER_LEN as u64 + h.root_len as u64).min(len);
6142 let root_bytes = if root_end <= head.len() as u64 {
6143 head[HEADER_LEN..root_end as usize].to_vec()
6144 } else {
6145 reader.read_at(HEADER_LEN as u64, root_end - HEADER_LEN as u64)?
6146 };
6147 let root = Root06::decode(&root_bytes, h.internal_codec)?;
6148 let dict = root.section(kind::DICT_DATA).and_then(|d| d.abs(&h).ok());
6149 let end = dict
6152 .map_or(h.data_offset, |d| d.offset + d.len)
6153 .max(crate::header::HEADER_LEN as u64)
6154 .min(len);
6155 let mut held = head;
6156 if end > held.len() as u64 {
6157 let at = held.len() as u64;
6158 held.extend_from_slice(&reader.read_at(at, end - at)?);
6159 }
6160 let sparse = SparseReader::fetch(reader, held, &[])?;
6161 let rete = Rete::open_ranged_lazy_with(sparse, 0)?;
6162 let Some(pattern) = resolve_query_pattern(rete.dictionary(), s, p, o) else {
6163 return Ok(None);
6164 };
6165 if !fetch {
6166 return Ok(Some(Vec::new()));
6167 }
6168 let perm = GraphIndex::best_permutation_in(h.perms(), pattern);
6169 let si = perm.section_index();
6170 let index = rete.default_index();
6171 let [pa, _, _] = perm.order_pattern(pattern);
6172 let mut out = Vec::new();
6175 for (start, stop) in index.routed_spans(si, pa) {
6176 let ranges: Vec<(u64, u64)> = (start..stop)
6177 .filter_map(|i| index.tile_range(si, i))
6178 .collect();
6179 if let (Some(&(lo, _)), Some(&(last, last_len))) = (ranges.first(), ranges.last()) {
6180 let body = reader.read_at(lo, last + last_len - lo)?;
6181 for &(off, l) in &ranges {
6182 let at = (off - lo) as usize;
6183 let tile = decompress(h.block_codec, &body[at..at + l as usize])?;
6184 out.extend(GraphIndex::match_serialized_block(&tile, perm, pattern));
6185 }
6186 }
6187 }
6188 out.sort_unstable();
6189 let d = rete.dictionary();
6190 Ok(Some(
6191 out.into_iter()
6192 .filter_map(|(s, p, o)| {
6193 Some((d.subject_term(s)?, d.predicate_term(p)?, d.object_term(o)?))
6194 })
6195 .collect(),
6196 ))
6197}
6198
6199struct RoutedPattern {
6202 dict: Dictionary,
6203 pattern: Pattern,
6204 permutation: IndexPermutation,
6205 header: Header,
6206 section: ByteRange,
6208}
6209
6210fn route_pattern<R: RangeReader>(
6213 reader: &R,
6214 head: Vec<u8>,
6215 s: Option<&str>,
6216 p: Option<&str>,
6217 o: Option<&str>,
6218) -> Result<Option<RoutedPattern>, FileError> {
6219 let header = Header::from_bytes(&head)?;
6220
6221 let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
6222 let dict = decode_dictionary_container(&dict_bytes, header.dict_codec)?;
6223
6224 let Some(pattern) = resolve_query_pattern(&dict, s, p, o) else {
6225 return Ok(None);
6226 };
6227 let permutation = GraphIndex::best_permutation_in(header.perms, pattern);
6228 let section = locate_container_section_ranged(
6229 reader,
6230 header.root_dir_offset,
6231 header.root_dir_len,
6232 header
6233 .perms
6234 .position(permutation)
6235 .ok_or(FileError::Container("routed to an absent permutation"))?,
6236 header.index_container_sections() as u64,
6237 )?;
6238 Ok(Some(RoutedPattern {
6239 dict,
6240 pattern,
6241 permutation,
6242 header,
6243 section,
6244 }))
6245}
6246
6247fn fetch_routed_matches<R: RangeReader>(
6252 reader: &R,
6253 routed: &RoutedPattern,
6254) -> Result<Vec<Triple>, FileError> {
6255 let dir = read_tile_directory_ranged(reader, routed.section)?;
6256 let [pa, _, _] = routed.permutation.order_pattern(routed.pattern);
6257 let codec = routed.header.block_codec;
6258 let mut out = Vec::new();
6259 match pa {
6260 Some(a) => {
6263 for e in dir.iter().filter(|e| e.min_a <= a && a <= e.max_a) {
6264 let len = e.end - e.start;
6265 let bytes = reader.read_at(checked_end(routed.section.offset, e.start)?, len)?;
6266 if bytes.len() as u64 != len {
6267 return Err(FileError::Container("short tile read"));
6268 }
6269 let tile = decompress(codec, &bytes)?;
6270 out.extend(GraphIndex::match_serialized_block(
6271 &tile,
6272 routed.permutation,
6273 routed.pattern,
6274 ));
6275 }
6276 }
6277 None => {
6280 if let (Some(first), Some(last)) = (dir.first(), dir.last()) {
6281 let base = first.start;
6282 let body =
6283 reader.read_at(checked_end(routed.section.offset, base)?, last.end - base)?;
6284 for e in &dir {
6285 let stored = checked_slice(&body, e.start - base, e.end - e.start)
6287 .ok_or(FileError::Container("short tile read"))?;
6288 let tile = decompress(codec, stored)?;
6289 out.extend(GraphIndex::match_serialized_block(
6290 &tile,
6291 routed.permutation,
6292 routed.pattern,
6293 ));
6294 }
6295 }
6296 }
6297 }
6298 out.sort_unstable();
6299 Ok(out)
6300}
6301
6302fn resolve_query_pattern(
6303 dict: &Dictionary,
6304 s: Option<&str>,
6305 p: Option<&str>,
6306 o: Option<&str>,
6307) -> Option<Pattern> {
6308 let sid = match s {
6309 Some(t) => Some(dict.subject_id(t)?),
6310 None => None,
6311 };
6312 let pid = match p {
6313 Some(t) => Some(dict.predicate_id(t)?),
6314 None => None,
6315 };
6316 let oid = match o {
6317 Some(t) => Some(dict.object_id(t)?),
6318 None => None,
6319 };
6320 Some((sid, pid, oid))
6321}
6322
6323fn ranged_chunked_dictionary<R: RangeReader + Send + Sync + 'static>(
6338 reader: &std::sync::Arc<R>,
6339 header: &Header,
6340 want: [bool; 4],
6341) -> Result<Dictionary, FileError> {
6342 let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
6345 let mut dict_sections: Vec<crate::dict::ChunkedSection> = Vec::with_capacity(4);
6346 let mut section_ranges: [std::sync::Arc<Vec<ByteRange>>; 4] = Default::default();
6347 for si in 0..4 {
6348 if !want[si] {
6349 dict_sections.push(crate::dict::ChunkedSection::from_parts(
6350 crate::dict::SectionMeta {
6351 term_count: 0,
6352 restart_interval: 1,
6353 restart_offsets: Vec::new(),
6354 },
6355 Vec::new(),
6356 None,
6357 cache.clone(),
6358 si as u8,
6359 ));
6360 continue;
6361 }
6362 let section = locate_container_section_ranged(
6363 reader.as_ref(),
6364 header.dictionary_offset,
6365 header.dictionary_len,
6366 si,
6367 4,
6368 )?;
6369 let (meta, entries) = read_dict_dir_ranged(reader.as_ref(), section)?;
6370 let ranges: std::sync::Arc<Vec<ByteRange>> = std::sync::Arc::new(
6373 entries
6374 .iter()
6375 .map(|e| ByteRange {
6376 offset: section.offset + e.start,
6377 len: (e.end - e.start),
6378 })
6379 .collect(),
6380 );
6381 section_ranges[si] = ranges.clone();
6382 let chunks: Vec<crate::dict::SectionChunk> = entries
6383 .into_iter()
6384 .map(|e| crate::dict::SectionChunk::new(e.first_run, e.key, e.body_start))
6385 .collect();
6386 let chunk_reader = reader.clone();
6387 let codec = header.dict_codec;
6388 let loader_ranges = ranges.clone();
6389 let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
6390 let range = loader_ranges.get(ci)?;
6391 let bytes = chunk_reader.read_at(range.offset, range.len).ok()?;
6392 decompress(codec, &bytes).ok()
6393 });
6394 let bulk_reader = reader.clone();
6397 let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
6398 let want: Option<Vec<ByteRange>> =
6399 cis.iter().map(|&ci| ranges.get(ci).copied()).collect();
6400 let blobs = read_coalesced(bulk_reader.as_ref(), &want?, DICT_COALESCE_GAP)?;
6401 blobs.iter().map(|b| decompress(codec, b).ok()).collect()
6402 });
6403 dict_sections.push(
6404 crate::dict::ChunkedSection::from_parts(
6405 meta,
6406 chunks,
6407 Some(loader),
6408 cache.clone(),
6409 si as u8,
6410 )
6411 .with_bulk_loader(bulk),
6412 );
6413 }
6414 let dict_arr: [crate::dict::ChunkedSection; 4] = dict_sections
6415 .try_into()
6416 .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
6417 let multi_reader = reader.clone();
6421 let codec = header.dict_codec;
6422 let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
6423 let want: Option<Vec<ByteRange>> = pairs
6424 .iter()
6425 .map(|&(si, ci)| section_ranges.get(si as usize)?.get(ci).copied())
6426 .collect();
6427 let blobs = read_coalesced(multi_reader.as_ref(), &want?, DICT_COALESCE_GAP)?;
6428 blobs.iter().map(|b| decompress(codec, b).ok()).collect()
6429 });
6430 Ok(Dictionary::from_chunked_sections(dict_arr).with_multi_bulk_loader(multi))
6431}
6432
6433#[cfg(not(target_arch = "wasm32"))]
6438fn open_side_by_side<A, B: Send>(
6439 concurrency: usize,
6440 a: impl FnOnce() -> A,
6441 b: impl FnOnce() -> B + Send,
6442) -> (A, B) {
6443 if concurrency <= 1 {
6444 return (a(), b());
6445 }
6446 std::thread::scope(|scope| {
6447 let hb = scope.spawn(b);
6448 let ra = a();
6449 let rb = hb
6450 .join()
6451 .unwrap_or_else(|panic| std::panic::resume_unwind(panic));
6452 (ra, rb)
6453 })
6454}
6455
6456#[cfg(target_arch = "wasm32")]
6458fn open_side_by_side<A, B>(
6459 _concurrency: usize,
6460 a: impl FnOnce() -> A,
6461 b: impl FnOnce() -> B,
6462) -> (A, B) {
6463 (a(), b())
6464}
6465
6466#[cfg(feature = "v06-parity")]
6469fn parity_image(bytes: &[u8]) -> Option<Vec<u8>> {
6470 if crate::v06::open::is_v06(bytes) || bytes.len() < HEADER_LEN {
6471 return None;
6472 }
6473 crate::v06::write::convert_image(bytes, crate::v06::write::WriteOpts::default()).ok()
6474}
6475
6476fn v06_layout(f: &crate::v06::open::V06File) -> Vec<LayoutSegment> {
6479 let h = &f.h;
6480 let seg = |kind: &'static str, label: String, offset: u64, len: u64| LayoutSegment {
6481 kind,
6482 label,
6483 offset,
6484 len,
6485 };
6486 let mut out = vec![
6487 seg(
6488 "header",
6489 format!("header (fixed {} bytes)", crate::v06::HEADER_LEN),
6490 0,
6491 crate::v06::HEADER_LEN as u64,
6492 ),
6493 seg(
6494 "directory",
6495 "root block (section table, directory roots)".into(),
6496 h.root_offset(),
6497 h.root_len as u64,
6498 ),
6499 ];
6500 let c = h.card_range();
6501 if c.len > 0 {
6502 out.push(seg(
6503 "metadata",
6504 "metadata (dataset card, zstd)".into(),
6505 c.offset,
6506 c.len,
6507 ));
6508 }
6509 let b = h.build_range();
6510 if b.len > 0 {
6511 out.push(seg("build-info", "build record".into(), b.offset, b.len));
6512 }
6513 for s in &f.root.sections {
6514 let Ok(a) = s.abs(h) else { continue };
6515 let kind: &'static str = match s.kind {
6516 crate::v06::kind::SCHEMA => "schema",
6517 crate::v06::kind::DICT_DATA => "dictionary",
6518 crate::v06::kind::INDEX_DATA => "tile",
6519 crate::v06::kind::QUAD_DATA => "named-graphs",
6520 crate::v06::kind::LABEL_DATA => "label-index",
6521 crate::v06::kind::TEXT_DATA => "text-index",
6522 crate::v06::kind::PYRAMID => "pyramid",
6523 _ => "directory",
6524 };
6525 out.push(seg(
6526 kind,
6527 crate::v06::kind::name(s.kind).to_string(),
6528 a.offset,
6529 a.len,
6530 ));
6531 }
6532 out.sort_by_key(|s| s.offset);
6533 out
6534}
6535
6536#[allow(clippy::type_complexity)]
6539fn v06_slots(
6540 f: &std::sync::Arc<crate::v06::open::V06File>,
6541) -> (
6542 PyramidSlot,
6543 TextIndexSlot,
6544 SectionSlot<crate::label_index::LabelIndex>,
6545 SectionSlot<crate::schema_section::SchemaMeta>,
6546) {
6547 use crate::v06::{P_LABELS, P_PYRAMID, P_SCHEMA, P_TEXT};
6548 let pyramid = if f.h.has(P_PYRAMID) {
6549 let a = f.clone();
6550 PyramidSlot::Lazy {
6551 loader: Box::new(move || PyramidMeta::decode(&a.pyramid_bytes()?).ok()),
6552 cell: std::sync::OnceLock::new(),
6553 }
6554 } else {
6555 PyramidSlot::Resident(None)
6556 };
6557 let text_index = if f.h.has(P_TEXT) {
6558 let (a, b) = (f.clone(), f.clone());
6559 TextIndexSlot::Lazy {
6560 loader: Box::new(move || a.text_index()?.ok()),
6561 cell: std::sync::OnceLock::new(),
6562 token_table: Box::new(move || b.token_table_bytes()),
6563 token_table_cell: std::sync::OnceLock::new(),
6564 }
6565 } else {
6566 TextIndexSlot::Resident {
6567 index: None,
6568 token_table_len: None,
6569 }
6570 };
6571 let labels = if f.h.has(P_LABELS) {
6572 let a = f.clone();
6573 SectionSlot::Lazy {
6574 loader: Box::new(move || a.label_index()?.ok()),
6575 cell: std::sync::OnceLock::new(),
6576 }
6577 } else {
6578 SectionSlot::Resident(None)
6579 };
6580 let schema = if f.h.has(P_SCHEMA) {
6581 let a = f.clone();
6582 SectionSlot::Lazy {
6583 loader: Box::new(move || a.schema()),
6584 cell: std::sync::OnceLock::new(),
6585 }
6586 } else {
6587 SectionSlot::Resident(None)
6588 };
6589 (pyramid, text_index, labels, schema)
6590}
6591
6592fn ranged_pyramid_slot<R: RangeReader + Send + Sync + 'static>(
6596 reader: &std::sync::Arc<R>,
6597 header: &Header,
6598) -> PyramidSlot {
6599 if header.pyramid_meta_len == 0 {
6600 return PyramidSlot::Resident(None);
6601 }
6602 let pyr_reader = reader.clone();
6603 let pyr_off = header.pyramid_meta_offset;
6604 let pyr_len = header.pyramid_meta_len;
6605 PyramidSlot::Lazy {
6606 loader: Box::new(move || {
6607 let mb = pyr_reader.read_at(pyr_off, pyr_len).ok()?;
6608 PyramidMeta::decode(&mb).ok()
6609 }),
6610 cell: std::sync::OnceLock::new(),
6611 }
6612}
6613
6614fn ranged_text_index_slot<R: RangeReader + Send + Sync + 'static>(
6619 reader: &std::sync::Arc<R>,
6620 header: &Header,
6621) -> TextIndexSlot {
6622 let (ti_off, ti_len) = header.text_index_payload();
6623 if ti_len == 0 {
6624 return TextIndexSlot::Resident {
6625 index: None,
6626 token_table_len: None,
6627 };
6628 }
6629 let ti_reader = reader.clone();
6630 let probe_reader = reader.clone();
6631 let codec = header.block_codec;
6632 TextIndexSlot::Lazy {
6633 loader: Box::new(move || {
6634 let prefix_len = read_token_table_len(&*ti_reader, ti_off, ti_len)?;
6638 let prefix = ti_reader.read_at(ti_off, prefix_len).ok()?;
6639 let postings_base = crate::text_index::TextIndex::postings_base(&prefix)? as u64;
6640 let postings_abs = ti_off + postings_base;
6641 let pr = ti_reader.clone();
6642 let posting_loader =
6643 Box::new(move |off: u64, len: u64| pr.read_at(postings_abs + off, len).ok());
6644 crate::text_index::TextIndex::from_token_table(&prefix, codec, posting_loader).ok()
6645 }),
6646 cell: std::sync::OnceLock::new(),
6647 token_table: Box::new(move || read_token_table_len(&*probe_reader, ti_off, ti_len)),
6650 token_table_cell: std::sync::OnceLock::new(),
6651 }
6652}
6653
6654fn resolve_label_hits(
6661 dict: &Dictionary,
6662 hits: Vec<&crate::meta::LabelEntry>,
6663) -> Vec<(String, String)> {
6664 let ids: Vec<u32> = hits.iter().map(|e| e.subject).collect();
6665 dict.prefetch_subject_terms(&ids);
6666 hits.iter()
6667 .zip(dict.subject_terms(&ids))
6668 .filter_map(|(e, iri)| Some((e.label.clone(), iri?)))
6669 .collect()
6670}
6671
6672fn text_search_in(
6675 ti: Option<&crate::text_index::TextIndex>,
6676 dict: &Dictionary,
6677 words: &[&str],
6678 prefix: Option<&str>,
6679 limit: usize,
6680) -> Vec<String> {
6681 let Some(ti) = ti else {
6682 return Vec::new();
6683 };
6684 let mut acc: Option<Vec<u32>> = None;
6688 if let Some(p) = prefix {
6689 acc = Some(ti.prefix(&p.to_lowercase()));
6690 }
6691 for w in words {
6692 for tok in crate::text_index::tokenize(w) {
6693 let posting = ti.lookup(&tok);
6694 acc = Some(match acc {
6695 Some(a) => intersect_sorted(&a, &posting),
6696 None => posting,
6697 });
6698 if acc.as_ref().is_some_and(|a| a.is_empty()) {
6699 return Vec::new();
6700 }
6701 }
6702 }
6703 let ids = acc.unwrap_or_default();
6704 let mut out = Vec::with_capacity(if limit > 0 {
6705 limit.min(ids.len())
6706 } else {
6707 ids.len()
6708 });
6709 let page = if limit > 0 {
6713 limit.min(ids.len())
6714 } else {
6715 ids.len()
6716 };
6717 dict.prefetch_subject_terms(&ids[..page]);
6718 for id in ids {
6719 if let Some(iri) = dict.subject_term(id) {
6720 out.push(iri);
6721 if limit > 0 && out.len() >= limit {
6722 break;
6723 }
6724 }
6725 }
6726 out
6727}
6728
6729pub struct SearchView {
6752 header: Header,
6753 dict: SectionSlot<Dictionary>,
6754 pyramid: PyramidSlot,
6755 text_index: TextIndexSlot,
6756 labels: SectionSlot<crate::label_index::LabelIndex>,
6757}
6758
6759impl SearchView {
6760 pub fn open_ranged<R: RangeReader + Send + Sync + 'static>(
6765 reader: R,
6766 ) -> Result<Self, FileError> {
6767 Self::open_ranged_with(reader, crate::v06::FIRST_READ)
6768 }
6769
6770 pub fn open_ranged_with<R: RangeReader + Send + Sync + 'static>(
6773 reader: R,
6774 first_read: u64,
6775 ) -> Result<Self, FileError> {
6776 let first_len = first_read.max(HEADER_LEN as u64).min(reader.len());
6777 let first: std::sync::Arc<[u8]> = std::sync::Arc::from(reader.read_at(0, first_len)?);
6778 let reader = crate::reader::PrefixReader::new(reader, first.clone());
6779 if crate::v06::open::is_v06(&first) {
6780 let r: std::sync::Arc<dyn RangeReader + Send + Sync> = std::sync::Arc::new(reader);
6781 let f = std::sync::Arc::new(crate::v06::open::V06File::open(r, &first)?);
6782 let header = crate::v06::logical_header(&f.h);
6783 let (pyramid, text_index, labels, _) = v06_slots(&f);
6784 let want = [true, true, false, false];
6785 let dict = if f.h.has(crate::v06::P_LABELS) {
6786 let a = f.clone();
6787 SectionSlot::Lazy {
6788 loader: Box::new(move || a.dictionary(want).ok()),
6789 cell: std::sync::OnceLock::new(),
6790 }
6791 } else {
6792 SectionSlot::Resident(Some(f.dictionary(want)?))
6793 };
6794 return Ok(Self {
6795 header,
6796 dict,
6797 pyramid,
6798 text_index,
6799 labels,
6800 });
6801 }
6802 let head = reader.read_at(0, HEADER_LEN as u64)?;
6803 let header = Header::from_bytes(&head)?;
6804 let reader = std::sync::Arc::new(reader);
6805 let dict = if header.label_index_section().is_some() {
6809 let (r, h) = (reader.clone(), header.clone());
6810 SectionSlot::Lazy {
6811 loader: Box::new(move || {
6812 ranged_chunked_dictionary(&r, &h, [true, true, false, false]).ok()
6813 }),
6814 cell: std::sync::OnceLock::new(),
6815 }
6816 } else {
6817 SectionSlot::Resident(Some(ranged_chunked_dictionary(
6818 &reader,
6819 &header,
6820 [true, true, false, false],
6821 )?))
6822 };
6823 let pyramid = ranged_pyramid_slot(&reader, &header);
6824 let text_index = ranged_text_index_slot(&reader, &header);
6825 let labels = ranged_label_slot(&reader, &header);
6826 Ok(Self {
6827 header,
6828 dict,
6829 pyramid,
6830 text_index,
6831 labels,
6832 })
6833 }
6834
6835 pub fn has_labels(&self) -> bool {
6839 self.header.label_index_section().is_some() || self.header.pyramid_meta_len > 0
6840 }
6841
6842 pub fn has_label_index(&self) -> bool {
6844 self.header.label_index_section().is_some()
6845 }
6846
6847 pub fn header(&self) -> &Header {
6849 &self.header
6850 }
6851
6852 pub fn has_text_index(&self) -> bool {
6854 self.header.text_index_len > 0
6855 }
6856
6857 pub fn has_pyramid(&self) -> bool {
6860 self.header.pyramid_meta_len > 0
6861 }
6862
6863 pub fn text_search(&self, words: &[&str], prefix: Option<&str>, limit: usize) -> Vec<String> {
6866 let Some(dict) = self.dict.get() else {
6867 return Vec::new();
6868 };
6869 text_search_in(self.text_index.index(), dict, words, prefix, limit)
6870 }
6871
6872 pub fn text_index_token_table_len(&self) -> Option<u64> {
6876 self.text_index.token_table_len()
6877 }
6878
6879 pub fn prefix_search(&self, prefix: &str, limit: usize) -> Vec<(String, String)> {
6883 if let Some(li) = self.labels.get() {
6884 return li.prefix_search(prefix, limit);
6885 }
6886 let pyr = match &self.pyramid {
6887 PyramidSlot::Resident(p) => p.as_ref(),
6888 PyramidSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
6889 };
6890 let (Some(pyr), Some(dict)) = (pyr, self.dict.get()) else {
6891 return Vec::new();
6892 };
6893 resolve_label_hits(dict, pyr.prefix_search(prefix, limit))
6894 }
6895}
6896
6897#[must_use]
6902pub struct SummaryView {
6903 pub round: u32,
6904 pub summary: Vec<SuperEdge>,
6905 pub class_hierarchy: Vec<ClassNode>,
6907 pub level_rollups: Vec<LevelRollup>,
6909 pub level_links: Vec<LevelLinks>,
6911 pub descriptors: Vec<CommunityDescriptor>,
6913 pub subclass_cycles: Vec<Vec<String>>,
6915 pub disjoint_pairs: Vec<(String, String)>,
6917 pub equivalent_pairs: Vec<(String, String)>,
6919 dict: Dictionary,
6920}
6921
6922impl SummaryView {
6923 pub fn open_ranged<R: RangeReader>(reader: &R) -> Result<Option<Self>, FileError> {
6925 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6926 let header = Header::from_bytes(&head)?;
6927 if header.pyramid_meta_len == 0 {
6928 return Ok(None);
6929 }
6930 if crate::v06::reads(header.version) {
6931 return Self::open_ranged_v06(reader, &header, head).map(Some);
6932 }
6933
6934 let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
6935 let dict = decode_dictionary_container(&dict_bytes, header.dict_codec)?;
6936
6937 let mb = reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?;
6938 let mut meta =
6939 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
6940 if let Some(s) = read_schema_with_header(reader, &header) {
6943 meta.class_hierarchy = s.class_hierarchy;
6944 meta.level_rollups = s.level_rollups;
6945 meta.level_links = s.level_links;
6946 meta.subclass_cycles = s.subclass_cycles;
6947 meta.disjoint_pairs = s.disjoint_pairs;
6948 meta.equivalent_pairs = s.equivalent_pairs;
6949 }
6950
6951 Ok(Some(SummaryView {
6952 round: meta.round,
6953 summary: meta.summary,
6954 class_hierarchy: meta.class_hierarchy,
6955 level_rollups: meta.level_rollups,
6956 level_links: meta.level_links,
6957 descriptors: meta.descriptors,
6958 subclass_cycles: meta.subclass_cycles,
6959 disjoint_pairs: meta.disjoint_pairs,
6960 equivalent_pairs: meta.equivalent_pairs,
6961 dict,
6962 }))
6963 }
6964
6965 fn open_ranged_v06<R: RangeReader>(
6969 reader: &R,
6970 header: &Header,
6971 head: Vec<u8>,
6972 ) -> Result<Self, FileError> {
6973 use crate::v06::{kind, open};
6974 let mut first = head;
6978 let want_first = crate::v06::FIRST_READ.min(reader.len());
6979 if want_first > first.len() as u64 {
6980 let at = first.len() as u64;
6981 first.extend_from_slice(&reader.read_at(at, want_first - at)?);
6982 }
6983 let pre = open::Prefixed {
6984 first: &first,
6985 r: reader,
6986 };
6987 let (h, root) = open::read_front(&pre)?;
6988 let want: Vec<ByteRange> = [
6989 kind::PYRAMID,
6990 kind::SCHEMA,
6991 kind::DICT_LEAVES,
6992 kind::DICT_DATA,
6993 ]
6994 .iter()
6995 .filter_map(|&k| root.section(k).and_then(|s| s.abs(&h).ok()))
6996 .collect();
6997 let sparse = open::SparseReader::fetch(&pre, first.clone(), &want)?;
6998 let mb = open::read_section(&sparse, &h, &root, kind::PYRAMID)?
6999 .ok_or(FileError::Container("0x06 file has no pyramid"))?;
7000 let mut meta =
7001 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7002 if let Some(s) = read_schema_with_header(&sparse, header) {
7003 meta.class_hierarchy = s.class_hierarchy;
7004 meta.level_rollups = s.level_rollups;
7005 meta.level_links = s.level_links;
7006 meta.subclass_cycles = s.subclass_cycles;
7007 meta.disjoint_pairs = s.disjoint_pairs;
7008 meta.equivalent_pairs = s.equivalent_pairs;
7009 }
7010 let f = open::V06File::open(std::sync::Arc::new(sparse), &first)?;
7011 let dict = f.dictionary([true; 4])?;
7012 Ok(SummaryView {
7013 round: meta.round,
7014 summary: meta.summary,
7015 class_hierarchy: meta.class_hierarchy,
7016 level_rollups: meta.level_rollups,
7017 level_links: meta.level_links,
7018 descriptors: meta.descriptors,
7019 subclass_cycles: meta.subclass_cycles,
7020 disjoint_pairs: meta.disjoint_pairs,
7021 equivalent_pairs: meta.equivalent_pairs,
7022 dict,
7023 })
7024 }
7025
7026 pub fn open_schema_ranged<R: RangeReader>(reader: &R) -> Result<Option<Self>, FileError> {
7034 let head = reader.read_at(0, HEADER_LEN as u64)?;
7035 let header = Header::from_bytes(&head)?;
7036 let Some(s) = read_schema_with_header(reader, &header) else {
7037 return Ok(None);
7038 };
7039 let dict = crate::DictionaryBuilder::new().build();
7040 Ok(Some(SummaryView {
7041 round: 0,
7042 summary: Vec::new(),
7043 class_hierarchy: s.class_hierarchy,
7044 level_rollups: s.level_rollups,
7045 level_links: s.level_links,
7046 descriptors: Vec::new(),
7047 subclass_cycles: s.subclass_cycles,
7048 disjoint_pairs: s.disjoint_pairs,
7049 equivalent_pairs: s.equivalent_pairs,
7050 dict,
7051 }))
7052 }
7053
7054 pub fn level_count(&self) -> usize {
7056 self.level_rollups.len()
7057 }
7058
7059 pub fn level_rollup(&self, k: usize) -> Option<&LevelRollup> {
7062 self.level_rollups.get(k)
7063 }
7064
7065 pub fn predicate_term(&self, id: u32) -> Option<String> {
7067 self.dict.predicate_term(id)
7068 }
7069
7070 pub fn predicate_total(&self, predicate: &str) -> u32 {
7073 match self.dict.predicate_id(predicate) {
7074 Some(pid) => self
7075 .summary
7076 .iter()
7077 .filter(|e| e.predicate == pid)
7078 .map(|e| e.count)
7079 .sum(),
7080 None => 0,
7081 }
7082 }
7083
7084 pub fn predicate_totals(&self) -> Vec<(String, u32)> {
7086 let mut by_pred: std::collections::BTreeMap<u32, u32> = std::collections::BTreeMap::new();
7087 for e in &self.summary {
7088 *by_pred.entry(e.predicate).or_default() += e.count;
7089 }
7090 let mut out: Vec<(String, u32)> = by_pred
7091 .into_iter()
7092 .filter_map(|(pid, c)| self.dict.predicate_term(pid).map(|t| (t, c)))
7093 .collect();
7094 out.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
7095 out
7096 }
7097
7098 pub fn community_count(&self) -> usize {
7100 let mut comms = std::collections::BTreeSet::new();
7101 for e in &self.summary {
7102 comms.insert(e.s_comm);
7103 comms.insert(e.o_comm);
7104 }
7105 comms.len()
7106 }
7107
7108 pub fn tbox_coherence(&self) -> Vec<crate::reason::Inconsistency> {
7123 schema_coherence(
7124 &self.class_hierarchy,
7125 &self.subclass_cycles,
7126 &self.disjoint_pairs,
7127 &self.equivalent_pairs,
7128 )
7129 }
7130
7131 pub fn tbox_is_coherent(&self) -> bool {
7134 self.tbox_coherence().is_empty()
7135 }
7136}
7137
7138pub fn schema_coherence(
7144 class_hierarchy: &[ClassNode],
7145 subclass_cycles: &[Vec<String>],
7146 disjoint_pairs: &[(String, String)],
7147 equivalent_pairs: &[(String, String)],
7148) -> Vec<crate::reason::Inconsistency> {
7149 use crate::reason::Inconsistency;
7150 use std::collections::{BTreeMap, BTreeSet, VecDeque};
7151 const MAX_REACH: usize = 100_000;
7152
7153 let mut out: Vec<Inconsistency> = Vec::new();
7154
7155 for cyc in subclass_cycles {
7156 let detail = if cyc.len() == 1 {
7157 format!("{} is rdfs:subClassOf itself (a cycle)", cyc[0])
7158 } else {
7159 format!(
7160 "classes {{{}}} are mutually rdfs:subClassOf (a cycle)",
7161 cyc.join(", ")
7162 )
7163 };
7164 out.push(Inconsistency {
7165 kind: "subclass-cycle",
7166 detail,
7167 });
7168 }
7169
7170 if !disjoint_pairs.is_empty() {
7171 let mut adj: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
7173 for n in class_hierarchy {
7174 let e = adj.entry(n.class.as_str()).or_default();
7175 for p in &n.parents {
7176 e.push(p.as_str());
7177 }
7178 }
7179 for (a, b) in equivalent_pairs {
7180 adj.entry(a.as_str()).or_default().push(b.as_str());
7181 adj.entry(b.as_str()).or_default().push(a.as_str());
7182 }
7183
7184 let mut focuses: BTreeSet<&str> =
7186 class_hierarchy.iter().map(|n| n.class.as_str()).collect();
7187 for (a, b) in disjoint_pairs.iter().chain(equivalent_pairs) {
7188 focuses.insert(a.as_str());
7189 focuses.insert(b.as_str());
7190 }
7191
7192 let mut seen: BTreeSet<&str> = BTreeSet::new();
7193 for &c in &focuses {
7194 let mut reach: BTreeSet<&str> = BTreeSet::new();
7196 let mut q: VecDeque<&str> = VecDeque::new();
7197 reach.insert(c);
7198 q.push_back(c);
7199 while let Some(x) = q.pop_front() {
7200 if reach.len() > MAX_REACH {
7201 break;
7202 }
7203 if let Some(ns) = adj.get(x) {
7204 for &p in ns {
7205 if reach.insert(p) {
7206 q.push_back(p);
7207 }
7208 }
7209 }
7210 }
7211 for (x, y) in disjoint_pairs {
7212 if reach.contains(x.as_str()) && reach.contains(y.as_str()) && seen.insert(c) {
7213 out.push(Inconsistency {
7214 kind: "unsatisfiable-class",
7215 detail: format!(
7216 "{c} is a subclass of both {x} and {y}, which are \
7217 owl:disjointWith — no individual can be a {c}"
7218 ),
7219 });
7220 break;
7221 }
7222 }
7223 }
7224 }
7225
7226 out.sort_by(|a, b| (a.kind, &a.detail).cmp(&(b.kind, &b.detail)));
7227 out
7228}
7229
7230pub fn read_schema_coherence_ranged<R: RangeReader>(
7238 reader: &R,
7239) -> Result<Option<Vec<crate::reason::Inconsistency>>, FileError> {
7240 let head = reader.read_at(0, HEADER_LEN as u64)?;
7241 let header = Header::from_bytes(&head)?;
7242 if let Some(s) = read_schema_with_header(reader, &header) {
7244 return Ok(Some(schema_coherence(
7245 &s.class_hierarchy,
7246 &s.subclass_cycles,
7247 &s.disjoint_pairs,
7248 &s.equivalent_pairs,
7249 )));
7250 }
7251 if header.pyramid_meta_len == 0 {
7252 return Ok(None);
7253 }
7254 if header.schema_meta_len > 0 && (header.schema_meta_len as u64) <= header.pyramid_meta_len {
7258 let off =
7259 header.pyramid_meta_offset + header.pyramid_meta_len - header.schema_meta_len as u64;
7260 let block = reader.read_at(off, header.schema_meta_len as u64)?;
7261 let (hierarchy, cycles, disjoint, equivalent) = crate::meta::decode_schema_block(&block)
7262 .map_err(|_| FileError::Container("malformed schema block"))?;
7263 return Ok(Some(schema_coherence(
7264 &hierarchy,
7265 &cycles,
7266 &disjoint,
7267 &equivalent,
7268 )));
7269 }
7270 let mb = if crate::v06::reads(header.version) {
7272 crate::v06::open::read_pyramid(reader)?.unwrap_or_default()
7273 } else {
7274 reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?
7275 };
7276 let meta =
7277 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7278 Ok(Some(schema_coherence(
7279 &meta.class_hierarchy,
7280 &meta.subclass_cycles,
7281 &meta.disjoint_pairs,
7282 &meta.equivalent_pairs,
7283 )))
7284}
7285
7286#[allow(clippy::type_complexity)]
7294pub fn read_schema_summary_ranged<R: RangeReader>(
7295 reader: &R,
7296) -> Result<Option<(Vec<(String, u64)>, Vec<(String, String, String, u64)>)>, FileError> {
7297 let head = reader.read_at(0, HEADER_LEN as u64)?;
7298 let header = Header::from_bytes(&head)?;
7299 let meta = if let Some(s) = read_schema_with_header(reader, &header) {
7301 PyramidMeta::new(0, Vec::new(), &[]).with_schema(
7302 Vec::new(),
7303 s.level_rollups,
7304 s.level_links,
7305 Vec::new(),
7306 Vec::new(),
7307 Vec::new(),
7308 Vec::new(),
7309 )
7310 } else {
7311 if header.pyramid_meta_len == 0 {
7312 return Ok(None);
7313 }
7314 if header.schema_meta_len > 0 && (header.schema_meta_len as u64) <= header.pyramid_meta_len
7315 {
7316 let off = header.pyramid_meta_offset + header.pyramid_meta_len
7317 - header.schema_meta_len as u64;
7318 let block = reader.read_at(off, header.schema_meta_len as u64)?;
7319 let summary = crate::meta::decode_schema_block_summary(&block)
7320 .map_err(|_| FileError::Container("malformed schema block"))?;
7321 return Ok(Some(summary));
7322 }
7323 let mb = if crate::v06::reads(header.version) {
7325 crate::v06::open::read_pyramid(reader)?.unwrap_or_default()
7326 } else {
7327 reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?
7328 };
7329 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?
7330 };
7331 if meta.level_rollups.is_empty() && meta.level_links.is_empty() {
7332 return Ok(None);
7333 }
7334 let classes = meta
7335 .level_rollups
7336 .iter()
7337 .max_by_key(|r| r.depth)
7338 .map(|r| r.classes.clone())
7339 .unwrap_or_default();
7340 let relations = meta
7341 .level_links
7342 .iter()
7343 .max_by_key(|l| l.depth)
7344 .map(|l| {
7345 l.links
7346 .iter()
7347 .map(|c| {
7348 (
7349 c.s_class.clone(),
7350 c.predicate.clone(),
7351 c.o_class.clone(),
7352 c.count,
7353 )
7354 })
7355 .collect()
7356 })
7357 .unwrap_or_default();
7358 Ok(Some((classes, relations)))
7359}
7360
7361#[cfg(test)]
7366pub(crate) fn dict_chunk_keys_for_test(image: &[u8]) -> Vec<Vec<Vec<u8>>> {
7367 if crate::v06::open::is_v06(image) {
7368 use crate::v06::{decode_chunk_leaf, decode_dict_root, kind, open, tag};
7370 let r = crate::reader::SliceReader::new(image);
7371 let (h, root) = open::read_front(&r).unwrap();
7372 let roots = decode_dict_root(root.unit(tag::DICT).unwrap()).unwrap();
7373 let leaves = open::read_section(&r, &h, &root, kind::DICT_LEAVES)
7374 .unwrap()
7375 .unwrap();
7376 return roots
7377 .iter()
7378 .map(|s| {
7379 s.leaves
7380 .iter()
7381 .flat_map(|(l, _, _, _)| {
7382 let stored = &leaves[l.off as usize..(l.off + l.len) as usize];
7383 let raw = decompress(h.internal_codec, stored).unwrap();
7384 decode_chunk_leaf(&raw, l.n).unwrap()
7385 })
7386 .map(|c| c.key)
7387 .collect()
7388 })
7389 .collect();
7390 }
7391 let header = Header::from_bytes(&image[..HEADER_LEN]).unwrap();
7392 let s = header.dictionary_offset as usize;
7393 let e = s + header.dictionary_len as usize;
7394 decode_container(&image[s..e], CODEC_NONE)
7395 .unwrap()
7396 .iter()
7397 .map(|payload| {
7398 let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
7399 entries.into_iter().map(|c| c.key).collect()
7400 })
7401 .collect()
7402}
7403
7404#[cfg(test)]
7405mod tests {
7406 use super::*;
7407 use super::{
7408 write_dataset_v05 as write_dataset,
7409 write_dataset_with_metadata_v05 as write_dataset_with_metadata,
7410 write_file_v05 as write_file,
7411 };
7412 use crate::dictionary::DictionaryBuilder;
7413 use crate::index::GraphIndexBuilder;
7414
7415 #[cfg(not(feature = "compression"))]
7418 #[test]
7419 fn ruzstd_decodes_a_zstd_frame() {
7420 let data: Vec<u8> = (0..50_000u32)
7421 .flat_map(|i| (i % 251).to_le_bytes())
7422 .collect();
7423 let frame = ruzstd::encoding::compress_to_vec(
7424 &data[..],
7425 ruzstd::encoding::CompressionLevel::Fastest,
7426 );
7427 assert!(frame.len() < data.len());
7428 assert_eq!(decompress(CODEC_ZSTD, &frame).unwrap(), data);
7429 }
7430
7431 #[cfg(feature = "compression")]
7436 #[test]
7437 fn decompression_is_bounded() {
7438 let data: Vec<u8> = (0..100_000u32).map(|i| (i % 251) as u8).collect();
7439 let sized = zstd::bulk::compress(&data, 3).unwrap();
7440 assert_eq!(zstd_frame_content_size(&sized), Some(100_000));
7441 assert_eq!(block_raw_len(CODEC_ZSTD, &sized, 1 << 20).unwrap(), 100_000);
7442 assert_eq!(
7443 decompress_bounded(CODEC_ZSTD, &sized, 100_000).unwrap(),
7444 data
7445 );
7446 assert!(decompress_bounded(CODEC_ZSTD, &sized, 99_999).is_err());
7447 let stream = zstd::encode_all(&data[..], 3).unwrap();
7449 assert_eq!(zstd_frame_content_size(&stream), None);
7450 assert_eq!(
7451 block_raw_len(CODEC_ZSTD, &stream, 1 << 20).unwrap(),
7452 100_000
7453 );
7454 assert_eq!(
7455 decompress_bounded(CODEC_ZSTD, &stream, 100_000).unwrap(),
7456 data
7457 );
7458 assert!(decompress_bounded(CODEC_ZSTD, &stream, 50_000).is_err());
7459 let mut liar = sized.clone();
7461 let fhd = liar[4];
7462 let fcs_at = 5 + usize::from(fhd & 0x20 == 0);
7463 liar[fcs_at] = liar[fcs_at].wrapping_sub(1);
7464 assert!(decompress_bounded(CODEC_ZSTD, &liar, 200_000).is_err());
7465 assert!(decompress_bounded(CODEC_NONE, &data, 99_999).is_err());
7467 assert_eq!(
7468 decompress_bounded(CODEC_NONE, &data, 100_000).unwrap(),
7469 data
7470 );
7471 assert_eq!(zstd_frame_content_size(b"RETE"), None);
7472 }
7473
7474 #[test]
7480 fn three_permutation_file_round_trips_every_open_path() {
7481 use crate::index::PermSet;
7482 use crate::reader::SliceReader;
7483
7484 let build = |perms: PermSet| {
7485 let mut db = crate::DictionaryBuilder::new();
7486 let mut triples = Vec::new();
7487 for i in 0..60u32 {
7488 let (s, p, o) = (
7489 format!("<http://ex/s{}>", i % 11),
7490 format!("<http://ex/p{}>", i % 3),
7491 format!("<http://ex/o{}>", i % 7),
7492 );
7493 db.observe(&s, &p, &o);
7494 triples.push((s, p, o));
7495 }
7496 let dict = db.build();
7497 let ids: Vec<(u32, u32, u32)> = triples
7498 .iter()
7499 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
7500 .collect();
7501 let def = GraphIndexBuilder::from_triples(ids.clone())
7502 .with_tile_budget(96)
7503 .with_perms(perms)
7504 .build();
7505 let named = GraphIndexBuilder::from_triples(ids)
7506 .with_tile_budget(96)
7507 .with_perms(perms)
7508 .build();
7509 write_dataset(
7510 &dict,
7511 &def,
7512 &[("<http://ex/g>".to_string(), named)],
7513 true,
7514 &[],
7515 0,
7516 )
7517 };
7518
7519 let six = build(PermSet::ALL);
7520 let three = build(PermSet::CORE);
7521 assert!(three.len() < six.len());
7522 assert_eq!(Header::from_bytes(&three).unwrap().perms, PermSet::CORE);
7523 assert_eq!(Header::from_bytes(&six).unwrap().perms, PermSet::ALL);
7524
7525 let shapes: Vec<(Option<&str>, Option<&str>, Option<&str>)> = vec![
7526 (None, None, None),
7527 (Some("<http://ex/s3>"), None, None),
7528 (None, Some("<http://ex/p1>"), None),
7529 (None, None, Some("<http://ex/o5>")),
7530 (Some("<http://ex/s3>"), Some("<http://ex/p1>"), None),
7531 (Some("<http://ex/s3>"), None, Some("<http://ex/o5>")),
7532 (None, Some("<http://ex/p1>"), Some("<http://ex/o5>")),
7533 (
7534 Some("<http://ex/s3>"),
7535 Some("<http://ex/p1>"),
7536 Some("<http://ex/o5>"),
7537 ),
7538 ];
7539
7540 for (image, tag) in [(&six, "six"), (&three, "three")] {
7541 let resident = Rete::open(image).unwrap();
7542 let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
7543 let ranged = Rete::open_ranged(&SliceReader::new(leaked)).unwrap();
7544 let lazy =
7545 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
7546 for (s, p, o) in &shapes {
7547 let want: Vec<_> = {
7548 let mut v = Rete::open(&six).unwrap().query(*s, *p, *o);
7549 v.sort();
7550 v
7551 };
7552 for (got, path) in [
7553 (resident.query(*s, *p, *o), "resident"),
7554 (ranged.query(*s, *p, *o), "ranged"),
7555 (lazy.query(*s, *p, *o), "lazy"),
7556 ] {
7557 let mut got = got;
7558 got.sort();
7559 assert_eq!(got, want, "{tag}/{path} disagreed on {:?}", (s, p, o));
7560 }
7561 let mut g = resident.query_in_graph(Some("<http://ex/g>"), *s, *p, *o);
7563 g.sort();
7564 assert_eq!(g, want, "{tag}/named disagreed on {:?}", (s, p, o));
7565 }
7566 }
7567
7568 let leaked: &'static [u8] = Box::leak(three.clone().into_boxed_slice());
7572 let reader = SliceReader::new(leaked);
7573 let routed = Rete::query_ranged(&reader, None, None, Some("<http://ex/o5>")).unwrap();
7574 let mut want = Rete::open(&six)
7575 .unwrap()
7576 .query(None, None, Some("<http://ex/o5>"));
7577 want.sort();
7578 let mut got = routed;
7579 got.sort();
7580 assert_eq!(got, want, "routed single-pattern read on a lean file");
7581 }
7582
7583 #[test]
7584 fn read_coalesced_merges_within_gap_and_splits_beyond() {
7585 use crate::reader::{CountingReader, SliceReader};
7586 let bytes = vec![0u8; 4096];
7587 let ranges = [
7589 ByteRange { offset: 0, len: 16 },
7590 ByteRange {
7591 offset: 48,
7592 len: 16,
7593 },
7594 ByteRange {
7595 offset: 1088,
7596 len: 16,
7597 },
7598 ];
7599 let r = CountingReader::new(SliceReader::new(&bytes));
7601 let out = read_coalesced(&r, &ranges, 16).unwrap();
7602 assert_eq!(out.len(), 3);
7603 assert_eq!(r.requests(), 3);
7604 let r = CountingReader::new(SliceReader::new(&bytes));
7606 read_coalesced(&r, &ranges, 64).unwrap();
7607 assert_eq!(r.requests(), 2);
7608 let r = CountingReader::new(SliceReader::new(&bytes));
7610 read_coalesced(&r, &ranges, 4096).unwrap();
7611 assert_eq!(r.requests(), 1);
7612 }
7613
7614 fn build_image() -> Vec<u8> {
7615 let triples = [
7616 ("Alice", "knows", "Bob"),
7617 ("Bob", "knows", "Carol"),
7618 ("Alice", "age", "30"),
7619 ];
7620 let mut db = DictionaryBuilder::new();
7621 for (s, p, o) in triples {
7622 db.observe(s, p, o);
7623 }
7624 let dict = db.build();
7625
7626 let mut ib = GraphIndexBuilder::new();
7627 for (s, p, o) in triples {
7628 ib.push(dict.encode(s, p, o).unwrap());
7629 }
7630 let index = ib.build();
7631
7632 let (meta, levels) = build_pyramid_meta(&dict, &triples_ids(&dict), DEFAULT_TILE_BUDGET);
7633 write_file(&dict, &index, false, &meta, levels)
7634 }
7635
7636 fn triples_ids(dict: &Dictionary) -> Vec<(u32, u32, u32)> {
7637 [
7638 ("Alice", "knows", "Bob"),
7639 ("Bob", "knows", "Carol"),
7640 ("Alice", "age", "30"),
7641 ]
7642 .iter()
7643 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
7644 .collect()
7645 }
7646
7647 #[test]
7648 fn file_round_trips_header_and_counts() {
7649 let bytes = build_image();
7650 let rete = Rete::open(&bytes).unwrap();
7651 assert_eq!(rete.header().quad_count, 3);
7652 assert!(rete.header().term_count >= 5);
7653 let expected_codec = writer_codec();
7654 assert_eq!(rete.header().dict_codec, expected_codec);
7655 assert_eq!(rete.header().block_codec, expected_codec);
7656 assert_eq!(&bytes[bytes.len() - 4..], &MAGIC); }
7658
7659 #[test]
7664 fn multi_tile_file_round_trips_and_routes() {
7665 let triples: Vec<(String, String, String)> = (0..200)
7666 .map(|i| {
7667 (
7668 format!("<http://ex/s/{i}>"),
7669 format!("<http://ex/p/{}>", i % 5),
7670 format!("<http://ex/o/{}>", i % 23),
7671 )
7672 })
7673 .collect();
7674 let mut db = DictionaryBuilder::new();
7675 for (s, p, o) in &triples {
7676 db.observe(s, p, o);
7677 }
7678 let dict = db.build();
7679 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7680 for (s, p, o) in &triples {
7681 ib.push(dict.encode(s, p, o).unwrap());
7682 }
7683 let index = ib.build();
7684 assert!(
7685 index.tile_sections()[0].len() > 3,
7686 "tiny budget must force many tiles"
7687 );
7688 let bytes = write_file(&dict, &index, false, &[], 0);
7689
7690 let rete = Rete::open(&bytes).unwrap();
7691 assert_eq!(rete.header().version, crate::header::LEGACY_FORMAT_VERSION);
7692 assert_eq!(rete.query(None, None, None).len(), 200);
7693 assert_eq!(rete.query(Some("<http://ex/s/7>"), None, None).len(), 1);
7694 assert_eq!(
7695 rete.query(None, Some("<http://ex/p/3>"), None).len(),
7696 40,
7697 "predicate extent spans tiles"
7698 );
7699 assert_eq!(
7700 rete.query(None, None, Some("<http://ex/o/22>")).len(),
7701 8 );
7703
7704 use crate::reader::SliceReader;
7706 let reader = SliceReader::new(&bytes);
7707 let routed = Rete::query_ranged(&reader, Some("<http://ex/s/7>"), None, None).unwrap();
7708 assert_eq!(routed.len(), 1);
7709 let routed = Rete::query_ranged(&reader, None, Some("<http://ex/p/3>"), None).unwrap();
7710 assert_eq!(routed.len(), 40);
7711 let routed = Rete::query_ranged(&reader, None, None, Some("<http://ex/o/22>")).unwrap();
7712 assert_eq!(routed.len(), 8);
7713 }
7714
7715 #[test]
7724 fn tile_directory_offsets_survive_past_4gib() {
7725 let mut dir = Vec::new();
7726 write_uvarint(&mut dir, 2); write_uvarint(&mut dir, 5); write_uvarint(&mut dir, 0); write_uvarint(&mut dir, 3 << 30); write_uvarint(&mut dir, 1); write_uvarint(&mut dir, 0);
7732 write_uvarint(&mut dir, 2 << 30); let total = dir.len() as u64 + (3u64 << 30) + (2u64 << 30) + 64;
7734 let entries = parse_tile_directory(&dir, total).unwrap();
7735 assert_eq!(entries.len(), 2);
7736 assert_eq!(entries[1].start, dir.len() as u64 + (3u64 << 30));
7737 assert!(
7738 entries[1].end > u32::MAX as u64,
7739 "tail tile sits past 4 GiB"
7740 );
7741 assert!(parse_tile_directory(&dir, 1 << 20).is_err());
7743 }
7744
7745 #[test]
7746 fn tile_synopsis_trailer_round_trips() {
7747 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7748 for i in 0..200u32 {
7749 ib.push((i, i % 7, i % 13));
7750 }
7751 let index = ib.build();
7752 let tile_count = index.tile_sections()[0].len();
7753 assert!(tile_count > 3, "tiny budget forces many tiles");
7754
7755 let payload = encode_tiled_section(&index, 0, CODEC_NONE);
7756 let dir = parse_tile_directory(&payload, payload.len() as u64).unwrap();
7757 assert_eq!(dir.len(), tile_count);
7758 let trailer_start = dir.iter().map(|e| e.end).max().unwrap();
7760 assert!(
7761 trailer_start < payload.len() as u64,
7762 "a trailer follows the tiles"
7763 );
7764 for e in &dir {
7765 assert!(
7766 e.end <= payload.len() as u64,
7767 "tiles still located within the payload"
7768 );
7769 }
7770 let syn = parse_tile_synopsis(&payload, trailer_start as usize, dir.len()).unwrap();
7771 for (e, (min_b, max_b, min_c, max_c)) in dir.iter().zip(syn) {
7772 let block = decompress(CODEC_NONE, &payload[e.start as usize..e.end as usize]).unwrap();
7773 let z = *crate::triples::TripleBlock::parse(&block).unwrap().zone();
7774 assert_eq!(
7775 (min_b, max_b, min_c, max_c),
7776 (z.min_b, z.max_b, z.min_c, z.max_c),
7777 "synopsis equals the tile's own zone"
7778 );
7779 }
7780 }
7781
7782 #[test]
7786 fn tile_synopsis_lazy_matches_reference_every_shape() {
7787 use crate::reader::{CountingReader, SliceReader};
7788 let triples: Vec<(String, String, String)> = (0..200u32)
7789 .map(|i| {
7790 (
7791 format!("<http://ex/s/{i:04}>"),
7792 format!("<http://ex/p/{}>", i % 7),
7793 format!("<http://ex/o/{:04}>", i % 13),
7794 )
7795 })
7796 .collect();
7797 let mut db = DictionaryBuilder::new();
7798 for (s, p, o) in &triples {
7799 db.observe(s, p, o);
7800 }
7801 let dict = db.build();
7802 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7803 for (s, p, o) in &triples {
7804 ib.push(dict.encode(s, p, o).unwrap());
7805 }
7806 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
7807
7808 let eager = Rete::open(&bytes).unwrap();
7809 assert!(
7810 eager.header().has_tile_synopsis(),
7811 "new files set the synopsis flag"
7812 );
7813
7814 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
7817 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
7818 let lazy = Rete::open_ranged_lazy_with(reader, HEADER_LEN as u64).unwrap();
7819
7820 let brute = |s: Option<&str>, p: Option<&str>, o: Option<&str>| {
7821 let mut v: Vec<(String, String, String)> = triples
7822 .iter()
7823 .filter(|(a, b, c)| {
7824 s.is_none_or(|x| x == a) && p.is_none_or(|x| x == b) && o.is_none_or(|x| x == c)
7825 })
7826 .cloned()
7827 .collect();
7828 v.sort();
7829 v
7830 };
7831 let sv = [
7833 None,
7834 Some("<http://ex/s/0007>"),
7835 Some("<http://ex/s/0130>"),
7836 Some("<http://ex/s/9999>"),
7837 ];
7838 let pv = [
7839 None,
7840 Some("<http://ex/p/3>"),
7841 Some("<http://ex/p/6>"),
7842 Some("<http://ex/p/999>"),
7843 ];
7844 let ov = [
7845 None,
7846 Some("<http://ex/o/0000>"),
7847 Some("<http://ex/o/0012>"),
7848 Some("<http://ex/o/9999>"),
7849 ];
7850 for &s in &sv {
7851 for &p in &pv {
7852 for &o in &ov {
7853 let mut e = eager.query(s, p, o);
7854 e.sort();
7855 let mut l = lazy.query(s, p, o);
7856 l.sort();
7857 let r = brute(s, p, o);
7858 assert_eq!(e, r, "eager {s:?} {p:?} {o:?}");
7859 assert_eq!(l, r, "lazy {s:?} {p:?} {o:?} — synopsis over-pruned");
7860 }
7861 }
7862 }
7863 assert!(!lazy.index_incomplete(), "no lazy fetch failed");
7864 }
7865
7866 #[test]
7870 fn polyglot_offset_reads_lazily() {
7871 use crate::reader::{
7872 detect_polyglot_base, CountingReader, OffsetReader, SliceReader, POLYGLOT_DIGITS,
7873 POLYGLOT_MARKER,
7874 };
7875
7876 let image = build_image();
7880 let mut shell = Vec::new();
7881 shell.extend_from_slice(b"<!DOCTYPE html><html><head><!--");
7882 shell.extend_from_slice(POLYGLOT_MARKER);
7883 let digits_at = shell.len();
7884 shell.extend_from_slice(&[b'0'; POLYGLOT_DIGITS]); shell.extend_from_slice(b"--></head><body>a web page</body></html>\n");
7886 shell.resize(50_000, b' '); let base = shell.len() as u64;
7888 let digits = format!("{base:0width$}", width = POLYGLOT_DIGITS);
7889 shell[digits_at..digits_at + POLYGLOT_DIGITS].copy_from_slice(digits.as_bytes());
7890
7891 let mut poly = shell;
7892 poly.extend_from_slice(&image);
7893
7894 assert_ne!(
7896 &poly[0..4],
7897 b"RETE",
7898 "polyglot must not start with the magic"
7899 );
7900 let detected = detect_polyglot_base(&poly[..crate::header::HEADER_LEN]).unwrap();
7902 assert_eq!(detected, base);
7903
7904 let leaked: &'static [u8] = Box::leak(poly.into_boxed_slice());
7906 let counting = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
7907 let lazy = Rete::open_ranged_lazy_with(
7908 OffsetReader::new(counting.clone(), detected),
7909 HEADER_LEN as u64,
7910 )
7911 .unwrap();
7912
7913 let plain = Rete::open(&image).unwrap();
7915 let mut want = plain.query(None, None, None);
7916 want.sort();
7917 let mut got = lazy.query(None, None, None);
7918 got.sort();
7919 assert_eq!(
7920 got, want,
7921 "polyglot lazy query differs from the plain .rete"
7922 );
7923 assert!(!got.is_empty(), "expected some triples");
7924 assert!(!lazy.index_incomplete(), "a lazy fetch failed");
7925
7926 let read = counting.bytes_read();
7929 eprintln!(
7930 "polyglot lazy read: HTML shell {base} B + .rete image {} B; \
7931 the query touched only {read} B (never the shell).",
7932 image.len()
7933 );
7934 assert!(
7935 read < base,
7936 "read {read} bytes; the HTML shell alone is {base} — the reader \
7937 touched the prefix instead of only the embedded .rete"
7938 );
7939 }
7940
7941 #[cfg(test)]
7944 fn build_text_indexed(triples: &[(String, String, String)]) -> Vec<u8> {
7945 let mut db = DictionaryBuilder::new();
7946 for (s, p, o) in triples {
7947 db.observe(s, p, o);
7948 }
7949 let dict = db.build();
7950 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7951 let mut id_triples: Vec<(u32, u32, u32)> = Vec::with_capacity(triples.len());
7952 for (s, p, o) in triples {
7953 let t = dict.encode(s, p, o).unwrap();
7954 ib.push(t);
7955 id_triples.push(t);
7956 }
7957 let index = ib.build();
7958 let text_index = compute_text_index(&dict, &id_triples);
7959 assert!(
7960 !text_index.is_empty(),
7961 "literals should produce a text index"
7962 );
7963 write_dataset_with_metadata(&dict, &index, &[], false, &[], 0, &[], &text_index)
7964 }
7965
7966 #[test]
7970 fn text_index_eager_matches_brute_force() {
7971 let triples: Vec<(String, String, String)> = vec![
7972 (
7973 "<http://ex/s0>",
7974 "<http://ex/label>",
7975 "\"alpha glucose phosphate\"",
7976 ),
7977 ("<http://ex/s1>", "<http://ex/label>", "\"beta Glucose\""),
7978 ("<http://ex/s2>", "<http://ex/label>", "\"gamma fructose\""),
7979 (
7980 "<http://ex/s3>",
7981 "<http://ex/note>",
7982 "\"einstein relativity\"",
7983 ),
7984 (
7985 "<http://ex/s4>",
7986 "<http://ex/ref>",
7987 "<http://ex/not-a-literal>",
7988 ),
7989 ]
7990 .into_iter()
7991 .map(|(s, p, o)| (s.to_string(), p.to_string(), o.to_string()))
7992 .collect();
7993 let bytes = build_text_indexed(&triples);
7994 let rete = Rete::open(&bytes).unwrap();
7995 assert!(rete.has_text_index());
7996
7997 let brute = |words: &[&str]| -> Vec<String> {
7999 let mut v: Vec<String> = triples
8000 .iter()
8001 .filter(|(_, _, o)| {
8002 crate::terms::is_literal(o)
8003 && words.iter().all(|w| {
8004 let wl = w.to_lowercase();
8005 crate::terms::literal_lexical(o)
8006 .unwrap()
8007 .split(|c: char| !c.is_alphanumeric())
8008 .any(|t| t.to_lowercase() == wl)
8009 })
8010 })
8011 .map(|(s, _, _)| s.clone())
8012 .collect();
8013 v.sort();
8014 v.dedup();
8015 v
8016 };
8017
8018 let mut got = rete.text_search(&["glucose"], None, 0);
8019 got.sort();
8020 assert_eq!(got, brute(&["glucose"]), "case-insensitive single word");
8021
8022 let mut got = rete.text_search(&["glucose", "phosphate"], None, 0);
8024 got.sort();
8025 assert_eq!(got, brute(&["glucose", "phosphate"]));
8026
8027 assert!(rete.text_search(&["zzznope"], None, 0).is_empty());
8029
8030 let got = rete.text_search(&[], Some("ein"), 0);
8032 assert_eq!(got, vec!["<http://ex/s3>".to_string()]);
8033
8034 let mut db = DictionaryBuilder::new();
8036 for (s, p, o) in &triples {
8037 db.observe(s, p, o);
8038 }
8039 let dict = db.build();
8040 let mut ib = GraphIndexBuilder::new();
8041 for (s, p, o) in &triples {
8042 ib.push(dict.encode(s, p, o).unwrap());
8043 }
8044 let plain = write_dataset(&dict, &ib.build(), &[], false, &[], 0);
8045 let plain_rete = Rete::open(&plain).unwrap();
8046 assert!(!plain_rete.has_text_index());
8047 assert!(plain_rete.text_search(&["glucose"], None, 0).is_empty());
8048 }
8049
8050 #[test]
8054 fn text_index_lazy_faults_only_queried_postings() {
8055 use crate::reader::{CountingReader, SliceReader};
8056 let mut triples: Vec<(String, String, String)> = (0..300u32)
8059 .map(|i| {
8060 (
8061 format!("<http://ex/s/{i:04}>"),
8062 "<http://ex/label>".to_string(),
8063 format!("\"common word number {i}\""),
8064 )
8065 })
8066 .collect();
8067 for i in [3u32, 77, 250] {
8068 triples.push((
8069 format!("<http://ex/s/{i:04}>"),
8070 "<http://ex/tag>".to_string(),
8071 "\"raretoken\"".to_string(),
8072 ));
8073 }
8074 let bytes = build_text_indexed(&triples);
8075 let eager = Rete::open(&bytes).unwrap();
8076 let mut want = eager.text_search(&["raretoken"], None, 0);
8077 want.sort();
8078 assert_eq!(want.len(), 3);
8079
8080 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8081 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8082 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
8083 let before = reader.bytes_read();
8086 let mut got = lazy.text_search(&["raretoken"], None, 0);
8087 got.sort();
8088 assert_eq!(got, want, "lazy search matches eager");
8089 let pulled = reader.bytes_read() - before;
8090 let ti_len = eager.header().text_index_len;
8093 assert!(
8094 pulled < ti_len,
8095 "search pulled {pulled} B but the section is {ti_len} B — faulted too much"
8096 );
8097 assert!(!lazy.index_incomplete());
8098 }
8099
8100 #[test]
8105 fn search_view_matches_full_open_for_far_fewer_bytes() {
8106 use crate::reader::{CountingReader, SliceReader};
8107 let filler = "lorem ipsum dolor sit amet consectetur adipiscing elit ".repeat(20);
8110 let mut triples: Vec<(String, String, String)> = (0..300u32)
8111 .map(|i| {
8112 (
8113 format!("<http://ex/s/{i:04}>"),
8114 "<http://ex/abstract>".to_string(),
8115 format!("\"{filler} number {i}\""),
8116 )
8117 })
8118 .collect();
8119 for i in [3u32, 77, 250] {
8120 triples.push((
8121 format!("<http://ex/s/{i:04}>"),
8122 "<http://ex/tag>".to_string(),
8123 "\"raretoken\"".to_string(),
8124 ));
8125 }
8126 let bytes = build_text_indexed(&triples);
8127 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8128
8129 let full_reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8130 let full = Rete::open_ranged_lazy_with(full_reader.clone(), HEADER_LEN as u64).unwrap();
8131 let mut want = full.text_search(&["raretoken"], None, 0);
8132 want.sort();
8133 assert_eq!(want.len(), 3);
8134
8135 let search_reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8136 let view = SearchView::open_ranged_with(search_reader.clone(), HEADER_LEN as u64).unwrap();
8137 assert!(view.has_text_index());
8138 let mut got = view.text_search(&["raretoken"], None, 0);
8139 got.sort();
8140 assert_eq!(got, want, "search view matches the full open");
8141 assert!(
8142 search_reader.bytes_read() < full_reader.bytes_read(),
8143 "search read {} B, full open read {} B — the narrow open bought nothing",
8144 search_reader.bytes_read(),
8145 full_reader.bytes_read()
8146 );
8147
8148 let mut prefix_hits = view.prefix_search("", 5);
8151 prefix_hits.sort();
8152 assert!(!prefix_hits.is_empty() || !view.has_pyramid());
8153 }
8154
8155 #[test]
8165 fn dict_chunk_directory_probe_costs_about_one_directory() {
8166 use crate::reader::{CountingReader, SliceReader};
8167 let head = "x".repeat(3000);
8181 let triples: Vec<(String, String, String)> = (0..2000u32)
8182 .map(|i| {
8183 let tail = char::from(b'a' + (i % 26) as u8).to_string().repeat(1000);
8184 (
8185 format!("<http://ex/s/{i:04}>"),
8186 "<http://ex/abstract>".to_string(),
8187 format!("\"{head}{i:04}{tail}\""),
8188 )
8189 })
8190 .collect();
8191 let mut db = DictionaryBuilder::new();
8192 for (s, p, o) in &triples {
8193 db.observe(s, p, o);
8194 }
8195 let dict = db.build();
8196 let mut ib = GraphIndexBuilder::new();
8197 for (s, p, o) in &triples {
8198 ib.push(dict.encode(s, p, o).unwrap());
8199 }
8200 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8201
8202 let reader = CountingReader::new(SliceReader::new(&bytes));
8203 let header = Header::from_bytes(&reader.read_at(0, HEADER_LEN as u64).unwrap()).unwrap();
8204 let section = locate_container_section_ranged(
8206 &reader,
8207 header.dictionary_offset,
8208 header.dictionary_len,
8209 2,
8210 4,
8211 )
8212 .unwrap();
8213
8214 let before = reader.bytes_read();
8215 let (_meta, entries) = read_dict_dir_ranged(&reader, section).unwrap();
8216 let spent = reader.bytes_read() - before;
8217
8218 let dir_end = entries[0].start;
8220 let head = reader.read_at(section.offset, 64).unwrap();
8221 let (header_len, n0) = read_uvarint(&head).unwrap();
8222 let dir_start = n0 as u64 + header_len;
8223 let dir_len = dir_end - dir_start;
8224 let dir_total = section.len - dir_start;
8225 assert!(
8226 dir_start + dir_len > 8192,
8227 "directory fits the header prefix ({dir_len} B) — the probe never runs"
8228 );
8229
8230 let mut old = 0u64;
8232 let mut p = 4096u64.min(dir_total).max(1);
8233 loop {
8234 let got = p.min(dir_total);
8235 old += got;
8236 if got >= dir_len || p >= dir_total {
8237 break;
8238 }
8239 p = p.saturating_mul(2).min(dir_total);
8240 }
8241 assert!(
8242 spent <= dir_len * 2,
8243 "probe read {spent} B for a {dir_len} B directory — past the append-only bound"
8244 );
8245 assert!(
8246 spent < old,
8247 "probe read {spent} B; the re-reading loop it replaced spent {old} B"
8248 );
8249 }
8250
8251 fn verbatim_keyed_twin(payload: &[u8], codec: u8) -> Vec<u8> {
8261 let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
8262 let (header_len, n0) = read_uvarint(payload).unwrap();
8263 let mut out = Vec::new();
8264 write_uvarint(&mut out, header_len);
8265 out.extend_from_slice(&payload[n0..n0 + header_len as usize]);
8266 write_uvarint(&mut out, entries.len() as u64);
8267 let mut prev_run = 0usize;
8268 for e in &entries {
8269 let body = decompress(codec, &payload[e.start as usize..e.end as usize]).unwrap();
8270 let first = crate::dict::run_first_term(&body, 0).unwrap();
8271 write_uvarint(&mut out, (e.first_run - prev_run) as u64);
8272 write_uvarint(&mut out, first.len() as u64);
8273 out.extend_from_slice(&first);
8274 write_uvarint(&mut out, e.end - e.start);
8275 prev_run = e.first_run;
8276 }
8277 for e in &entries {
8278 out.extend_from_slice(&payload[e.start as usize..e.end as usize]);
8279 }
8280 out
8281 }
8282
8283 fn chunk_bounds_of(payload: &[u8], codec: u8) -> Vec<(Vec<u8>, Vec<u8>)> {
8285 let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
8286 entries
8287 .iter()
8288 .map(|e| {
8289 let body = decompress(codec, &payload[e.start as usize..e.end as usize]).unwrap();
8290 (
8291 crate::dict::run_first_term(&body, 0).unwrap(),
8292 crate::dict::run_last_term(&body, 0, body.len()).unwrap(),
8293 )
8294 })
8295 .collect()
8296 }
8297
8298 fn adversarial_terms() -> Vec<String> {
8304 let mut terms: Vec<String> = Vec::new();
8305 for i in 0..6_000 {
8306 terms.push(format!("<http://example.org/work/{i:06}>"));
8307 }
8308 let shared = "X".repeat(900);
8309 for i in 0..4_000 {
8310 terms.push(format!("\"{shared}-{i:05}-tail\""));
8311 }
8312 let deep = "Y".repeat(3_000); for i in 0..120 {
8314 terms.push(format!("\"{deep}{i:03}\""));
8315 }
8316 for i in 0..6 {
8317 terms.push(format!("\"BIG-{i:02}-{}\"", "Z".repeat(90_000)));
8319 }
8320 for i in 0..800 {
8321 terms.push(format!(
8322 "\"h\u{e9}llo-{}-\u{1F600}{i:04}\"",
8323 "\u{e9}".repeat(40)
8324 ));
8325 }
8326 for i in 0..2_000 {
8327 terms.push(format!("\"{i:05}-{}\"", "T".repeat(1_800)));
8328 }
8329 terms.sort();
8330 terms.dedup();
8331 terms
8332 }
8333
8334 fn chunked_section_of(terms: &[String]) -> Vec<u8> {
8335 let mut b = crate::dict::DictSectionBuilder::new().with_restart_interval(16);
8336 for t in terms {
8337 b.push(t.clone());
8338 }
8339 b.build()
8340 }
8341
8342 #[test]
8352 fn chunk_directory_keys_are_separators_not_terms() {
8353 let terms = adversarial_terms();
8354 let raw = chunked_section_of(&terms);
8355 let payload = encode_chunked_dict_section(&raw, CODEC_NONE);
8356 let (_meta, entries) = parse_chunked_dict_dir(&payload, payload.len() as u64).unwrap();
8357 let bounds = chunk_bounds_of(&payload, CODEC_NONE);
8358 assert!(
8359 entries.len() > 20,
8360 "need a many-chunk section, got {}",
8361 entries.len()
8362 );
8363
8364 assert!(entries[0].key.is_empty(), "chunk 0 needs no separator");
8365 let mut key_bytes = 0usize;
8366 let mut term_bytes = 0usize;
8367 let mut differ = 0usize;
8368 for (i, e) in entries.iter().enumerate() {
8369 key_bytes += e.key.len();
8370 term_bytes += bounds[i].0.len();
8371 if e.key != bounds[i].0 {
8372 differ += 1;
8373 }
8374 assert!(
8375 e.key <= bounds[i].0,
8376 "chunk {i}: key must be <= its own first term"
8377 );
8378 if i > 0 {
8379 assert!(
8380 e.key.as_slice() > bounds[i - 1].1.as_slice(),
8381 "chunk {i}: key must be > the previous chunk's last term"
8382 );
8383 assert!(
8384 e.key > entries[i - 1].key,
8385 "chunk {i}: keys must ascend for partition_point"
8386 );
8387 assert_eq!(
8389 e.key,
8390 crate::dict::shortest_separator(&bounds[i - 1].1, &bounds[i].0),
8391 "chunk {i}: key is not the SHORTEST separator"
8392 );
8393 }
8394 }
8395 assert!(
8396 differ > 0,
8397 "every key equalled its first term — the writer stores terms again"
8398 );
8399 assert!(
8400 key_bytes * 4 < term_bytes,
8401 "separators bought almost nothing: {key_bytes} B vs {term_bytes} B of first terms"
8402 );
8403 eprintln!(
8404 "chunks={} first-term keys={term_bytes} B separator keys={key_bytes} B ({:.1}x)",
8405 entries.len(),
8406 term_bytes as f64 / key_bytes.max(1) as f64
8407 );
8408 }
8409
8410 #[test]
8421 fn separator_keys_route_identically_to_verbatim_first_terms() {
8422 let terms = adversarial_terms();
8423 let raw = chunked_section_of(&terms);
8424 let sep_payload = encode_chunked_dict_section(&raw, CODEC_NONE);
8425 let ver_payload = verbatim_keyed_twin(&sep_payload, CODEC_NONE);
8426
8427 let (_ms, es) = parse_chunked_dict_dir(&sep_payload, sep_payload.len() as u64).unwrap();
8429 let (_mv, ev) = parse_chunked_dict_dir(&ver_payload, ver_payload.len() as u64).unwrap();
8430 assert_eq!(es.len(), ev.len());
8431 for (a, b) in es.iter().zip(ev.iter()) {
8432 assert_eq!(a.first_run, b.first_run);
8433 assert_eq!(
8434 &sep_payload[a.start as usize..a.end as usize],
8435 &ver_payload[b.start as usize..b.end as usize],
8436 "chunk bodies must be identical"
8437 );
8438 }
8439 assert!(
8440 sep_payload.len() < ver_payload.len(),
8441 "separator payload is not smaller"
8442 );
8443
8444 let sec_s = decode_chunked_dict_section(
8445 &sep_payload,
8446 CODEC_NONE,
8447 crate::chunk_cache::ChunkCache::unlimited_arc(),
8448 0,
8449 )
8450 .unwrap();
8451 let sec_v = decode_chunked_dict_section(
8452 &ver_payload,
8453 CODEC_NONE,
8454 crate::chunk_cache::ChunkCache::unlimited_arc(),
8455 0,
8456 )
8457 .unwrap();
8458
8459 for (i, t) in terms.iter().enumerate() {
8461 let want = Some(i as u32 + 1);
8462 assert_eq!(sec_v.id(t), want, "verbatim id() lost a term");
8463 assert_eq!(sec_s.id(t), want, "separator id() lost a term");
8464 }
8465 for i in 0..terms.len() as u32 {
8467 assert_eq!(sec_s.term(i + 1), sec_v.term(i + 1));
8468 assert_eq!(
8469 sec_s.term(i + 1).as_deref(),
8470 Some(terms[i as usize].as_str())
8471 );
8472 }
8473 let bounds = chunk_bounds_of(&sep_payload, CODEC_NONE);
8475 let mut probes: Vec<String> = Vec::new();
8476 let push = |bytes: &[u8], probes: &mut Vec<String>| {
8477 if let Ok(s) = std::str::from_utf8(bytes) {
8478 probes.push(s.to_string());
8479 }
8480 };
8481 for (first, last) in &bounds {
8482 for base in [first, last] {
8483 push(base, &mut probes); let mut after = base.clone();
8485 after.push(0x01); push(&after, &mut probes);
8487 let mut far = base.clone();
8488 far.push(b'~');
8489 push(&far, &mut probes);
8490 if !base.is_empty() {
8491 push(&base[..base.len() - 1], &mut probes); let mut down = base.clone();
8493 *down.last_mut().unwrap() = down.last().unwrap().wrapping_sub(1);
8494 push(&down, &mut probes);
8495 let mut up = base.clone();
8496 *up.last_mut().unwrap() = up.last().unwrap().wrapping_add(1);
8497 push(&up, &mut probes);
8498 }
8499 }
8500 }
8501 for e in &es {
8503 push(&e.key, &mut probes);
8504 let mut plus = e.key.clone();
8505 plus.push(0);
8506 push(&plus, &mut probes);
8507 }
8508 probes.push(String::new()); probes.push("\u{1}".to_string()); probes.push("\u{10FFFF}\u{10FFFF}".to_string()); probes.push(terms.first().unwrap().clone());
8512 probes.push(terms.last().unwrap().clone());
8513
8514 let mut hits = 0usize;
8515 for p in &probes {
8516 let (a, b) = (sec_v.id(p), sec_s.id(p));
8517 assert_eq!(
8518 a,
8519 b,
8520 "probe diverged: {:?}",
8521 p.get(..p.len().min(48)).unwrap_or(p.as_str())
8522 );
8523 hits += usize::from(a.is_some());
8524 }
8525 assert!(
8526 probes.len() > 500 && hits > 50,
8527 "probe set is too thin: {} probes, {hits} of them present",
8528 probes.len()
8529 );
8530 eprintln!(
8531 "{} boundary probes, {hits} present, all identical",
8532 probes.len()
8533 );
8534 }
8535
8536 mod separator_props {
8542 use super::*;
8543 use proptest::prelude::*;
8544
8545 fn term_set() -> impl Strategy<Value = Vec<String>> {
8552 prop::collection::vec(("[a-c]{1,6}", 0u8..3, 300usize..1200), 60..250).prop_map(
8553 |specs| {
8554 let mut terms: Vec<String> = specs
8555 .into_iter()
8556 .enumerate()
8557 .map(|(i, (core, kind, pad))| {
8558 let block = "z".repeat(pad);
8559 match kind {
8560 0 => format!("<http://ex/{core}/{i:05}>"),
8561 1 => format!("\"{block}{core}{i:05}\""),
8562 _ => format!("\"{core}{i:05}{block}\""),
8563 }
8564 })
8565 .collect();
8566 terms.sort();
8567 terms.dedup();
8568 terms
8569 },
8570 )
8571 }
8572
8573 proptest! {
8574 #[test]
8575 fn prop_separator_keyed_section_answers_like_verbatim(
8576 terms in term_set(),
8577 extra in prop::collection::vec(any::<String>(), 0..8),
8578 ) {
8579 let raw = chunked_section_of(&terms);
8580 let sep = encode_chunked_dict_section(&raw, CODEC_NONE);
8581 let ver = verbatim_keyed_twin(&sep, CODEC_NONE);
8582 prop_assert!(sep.len() <= ver.len());
8583
8584 let (_m, es) = parse_chunked_dict_dir(&sep, sep.len() as u64).unwrap();
8586 let bounds = chunk_bounds_of(&sep, CODEC_NONE);
8587 prop_assert!(es[0].key.is_empty());
8588 for i in 1..es.len() {
8589 prop_assert!(es[i].key.as_slice() > bounds[i - 1].1.as_slice());
8590 prop_assert!(es[i].key <= bounds[i].0);
8591 prop_assert!(es[i].key > es[i - 1].key);
8592 }
8593
8594 let sec_s = decode_chunked_dict_section(
8595 &sep,
8596 CODEC_NONE,
8597 crate::chunk_cache::ChunkCache::unlimited_arc(),
8598 0,
8599 )
8600 .unwrap();
8601 let sec_v = decode_chunked_dict_section(
8602 &ver,
8603 CODEC_NONE,
8604 crate::chunk_cache::ChunkCache::unlimited_arc(),
8605 0,
8606 )
8607 .unwrap();
8608
8609 for (i, t) in terms.iter().enumerate() {
8610 prop_assert_eq!(sec_s.id(t), Some(i as u32 + 1));
8611 prop_assert_eq!(sec_v.id(t), Some(i as u32 + 1));
8612 prop_assert_eq!(sec_s.term(i as u32 + 1), sec_v.term(i as u32 + 1));
8613 }
8614
8615 let mut probes: Vec<String> = extra;
8617 probes.push(String::new());
8618 for (first, last) in &bounds {
8619 for base in [first, last] {
8620 if let Ok(s) = std::str::from_utf8(base) {
8621 probes.push(s.to_string());
8622 probes.push(format!("{s}\u{1}"));
8623 probes.push(s[..s.len() - s.chars().next_back()
8624 .map(char::len_utf8).unwrap_or(0)].to_string());
8625 }
8626 }
8627 }
8628 for e in &es {
8629 if let Ok(s) = std::str::from_utf8(&e.key) {
8630 probes.push(s.to_string());
8631 }
8632 }
8633 for p in &probes {
8634 prop_assert_eq!(sec_s.id(p), sec_v.id(p), "probe diverged: {:?}", p);
8635 }
8636 }
8637 }
8638 }
8639
8640 #[test]
8645 fn text_index_is_tamper_evident_and_verifies() {
8646 let triples: Vec<(String, String, String)> = vec![(
8647 "<http://ex/s0>".to_string(),
8648 "<http://ex/label>".to_string(),
8649 "\"alpha glucose phosphate\"".to_string(),
8650 )];
8651 let bytes = build_text_indexed(&triples);
8652 let header = Rete::open(&bytes).unwrap().header().clone();
8653 assert!(header.text_index_len > 0);
8654 assert!(verify(&bytes).unwrap(), "a text-indexed build must verify");
8655
8656 let mut tampered = bytes.clone();
8657 tampered[header.text_index_offset as usize] ^= 0xff;
8658 assert!(
8659 !verify(&tampered).unwrap(),
8660 "tampering with the text index must break verify()"
8661 );
8662 }
8663
8664 #[test]
8668 fn synopsis_cuts_remote_fetch_bytes() {
8669 use crate::header::FLAG_TILE_SYNOPSIS;
8670 use crate::reader::{CountingReader, SliceReader};
8671
8672 let triples: Vec<(String, String, String)> = (0..400u32)
8676 .map(|i| {
8677 (
8678 format!("<http://ex/s/{i:04}>"),
8679 "<http://ex/p>".to_string(),
8680 format!("<http://ex/o/{i:04}>"),
8681 )
8682 })
8683 .collect();
8684 let mut db = DictionaryBuilder::new();
8685 for (s, p, o) in &triples {
8686 db.observe(s, p, o);
8687 }
8688 let dict = db.build();
8689 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
8690 for (s, p, o) in &triples {
8691 ib.push(dict.encode(s, p, o).unwrap());
8692 }
8693 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8694
8695 let q = (Some("<http://ex/s/0395>"), None, Some("<http://ex/o/0005>"));
8699 let query_bytes = |image: &[u8]| -> (u64, usize) {
8703 let leaked: &'static [u8] = Box::leak(image.to_vec().into_boxed_slice());
8704 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8705 let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
8706 let before = reader.bytes_read(); let n = rete.query(q.0, q.1, q.2).len();
8708 assert!(!rete.index_incomplete());
8709 (reader.bytes_read() - before, n)
8710 };
8711
8712 let (on_bytes, on_n) = query_bytes(&bytes);
8713 let mut off = bytes.clone();
8715 off[5] &= !FLAG_TILE_SYNOPSIS;
8716 let (off_bytes, off_n) = query_bytes(&off);
8717
8718 assert_eq!(on_n, 0, "the pair never co-occurs");
8719 assert_eq!(off_n, 0, "same answer without the synopsis");
8720 assert!(
8724 on_bytes < off_bytes,
8725 "synopsis skips the routed tile fetch: {on_bytes} < {off_bytes}"
8726 );
8727 }
8728
8729 #[test]
8735 fn double_bound_object_join_eager_matches_lazy() {
8736 use crate::reader::SliceReader;
8737 let occ = "<http://ex/occ>";
8738 let phys = "<http://ex/physicist>";
8739 let phil = "<http://ex/philosopher>";
8740 let label = "<http://www.w3.org/2000/01/rdf-schema#label>";
8741 let mut triples: Vec<(String, String, String)> = Vec::new();
8743 for i in 0..20u32 {
8744 triples.push((format!("<http://ex/p/{i:02}>"), occ.into(), phys.into()));
8745 if i < 10 {
8746 triples.push((format!("<http://ex/p/{i:02}>"), occ.into(), phil.into()));
8747 }
8748 triples.push((
8749 format!("<http://ex/p/{i:02}>"),
8750 label.into(),
8751 format!("\"Name {i:02}\""),
8752 ));
8753 }
8754 let mut db = DictionaryBuilder::new();
8755 for (s, p, o) in &triples {
8756 db.observe(s, p, o);
8757 }
8758 let dict = db.build();
8759 let mut ib = GraphIndexBuilder::new().with_tile_budget(16);
8760 for (s, p, o) in &triples {
8761 ib.push(dict.encode(s, p, o).unwrap());
8762 }
8763 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8764
8765 let q = "SELECT ?l WHERE { \
8766 ?p <http://ex/occ> <http://ex/physicist> ; \
8767 <http://ex/occ> <http://ex/philosopher> ; \
8768 <http://www.w3.org/2000/01/rdf-schema#label> ?l }";
8769 let run = |rete: &Rete| -> Vec<String> {
8770 let (_, sols) = crate::eval_sparql(rete, q).unwrap();
8771 let mut v: Vec<String> = sols.iter().filter_map(|b| b.get("l").cloned()).collect();
8772 v.sort();
8773 v
8774 };
8775
8776 let eager_rows = run(&Rete::open(&bytes).unwrap());
8777 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8778 let lazy = Rete::open_ranged_lazy_with(
8779 std::sync::Arc::new(SliceReader::new(leaked)),
8780 HEADER_LEN as u64,
8781 )
8782 .unwrap();
8783 let lazy_rows = run(&lazy);
8784 assert!(!lazy.index_incomplete());
8785
8786 assert_eq!(eager_rows.len(), 10, "the 10 physicist∩philosopher labels");
8787 assert_eq!(eager_rows, lazy_rows, "eager and lazy must agree exactly");
8788 }
8789
8790 #[test]
8794 fn multi_chunk_dictionary_round_trips() {
8795 let mut db = DictionaryBuilder::new();
8796 let term = |i: u32| format!("<http://example.org/some/long/prefix/entity/{i:06}>");
8797 for i in 0..6000u32 {
8798 db.observe(&term(i), "<http://ex/p>", &term(i + 1));
8799 }
8800 let dict = db.build();
8801 let mut ib = GraphIndexBuilder::new();
8802 for i in 0..6000u32 {
8803 ib.push(
8804 dict.encode(&term(i), "<http://ex/p>", &term(i + 1))
8805 .unwrap(),
8806 );
8807 }
8808 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8809 let rete = Rete::open(&bytes).unwrap();
8810 let d = rete.dictionary();
8811 assert_eq!(d.term_count(), dict.term_count());
8812 for i in (0..6000).step_by(97).chain([0, 1, 5999, 6000]) {
8813 let t = term(i);
8814 let sid = dict.subject_id(&t);
8815 assert_eq!(d.subject_id(&t), sid, "subject_id({t})");
8816 if let Some(id) = sid {
8817 assert_eq!(d.subject_term(id).as_deref(), Some(t.as_str()));
8818 }
8819 let oid = dict.object_id(&t);
8820 assert_eq!(d.object_id(&t), oid, "object_id({t})");
8821 }
8822 assert_eq!(d.subject_id("<http://example.org/absent>"), None);
8823 assert_eq!(d.predicate_id("<http://ex/p>"), Some(1));
8824 assert_eq!(d.predicate_term(1).as_deref(), Some("<http://ex/p>"));
8825 }
8826
8827 #[test]
8828 #[cfg(feature = "compression")]
8829 fn compression_shrinks_repetitive_data() {
8830 let mut db = DictionaryBuilder::new();
8834 let triples: Vec<(String, String, String)> = (0..500)
8835 .map(|i| {
8836 (
8837 format!("<http://example.org/entity/{i}>"),
8838 "<http://example.org/p/relatedTo>".to_string(),
8839 format!("<http://example.org/entity/{}>", (i + 1) % 500),
8840 )
8841 })
8842 .collect();
8843 for (s, p, o) in &triples {
8844 db.observe(s, p, o);
8845 }
8846 let dict = db.build();
8847 let mut ib = GraphIndexBuilder::new();
8848 for (s, p, o) in &triples {
8849 ib.push(dict.encode(s, p, o).unwrap());
8850 }
8851 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8852
8853 let raw: usize = triples
8854 .iter()
8855 .map(|(s, p, o)| s.len() + p.len() + o.len())
8856 .sum();
8857 assert!(
8858 bytes.len() < raw / 2,
8859 "expected strong compression: file {} vs raw terms {raw}",
8860 bytes.len()
8861 );
8862
8863 let rete = Rete::open(&bytes).unwrap();
8865 let r = rete.query(Some("<http://example.org/entity/0>"), None, None);
8866 assert_eq!(r.len(), 1);
8867 assert_eq!(r[0].2, "<http://example.org/entity/1>");
8868 }
8869
8870 fn big_file_with_pyramid() -> Vec<u8> {
8871 let triples: Vec<(String, String, String)> = (0..300)
8873 .map(|i| {
8874 (
8875 format!("<http://ex/e{i}>"),
8876 "<http://ex/next>".to_string(),
8877 format!("<http://ex/e{}>", (i + 1) % 300),
8878 )
8879 })
8880 .collect();
8881 let mut db = DictionaryBuilder::new();
8882 for (s, p, o) in &triples {
8883 db.observe(s, p, o);
8884 }
8885 let dict = db.build();
8886 let ids: Vec<_> = triples
8887 .iter()
8888 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
8889 .collect();
8890 let mut ib = GraphIndexBuilder::new();
8891 for &t in &ids {
8892 ib.push(t);
8893 }
8894 let (meta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
8895 write_file(&dict, &ib.build(), false, &meta, levels)
8896 }
8897
8898 #[test]
8899 fn ranged_open_is_minimal_and_correct() {
8900 use crate::reader::{CountingReader, SliceReader};
8901 let bytes = big_file_with_pyramid();
8902
8903 let full = CountingReader::new(SliceReader::new(&bytes));
8904 let rete = Rete::open_ranged(&full).unwrap();
8905 assert!(full.requests() <= 4, "requests = {}", full.requests());
8907 assert_eq!(
8908 rete.query(Some("<http://ex/e0>"), None, None)[0].2,
8909 "<http://ex/e1>"
8910 );
8911
8912 let summ_reader = CountingReader::new(SliceReader::new(&bytes));
8914 let view = SummaryView::open_ranged(&summ_reader).unwrap().unwrap();
8915 assert!(!view.summary.is_empty());
8916 assert!(
8917 summ_reader.bytes_read() < bytes.len() as u64,
8918 "summary read {} of {} bytes",
8919 summ_reader.bytes_read(),
8920 bytes.len()
8921 );
8922 assert!(summ_reader.bytes_read() < full.bytes_read());
8924 }
8925
8926 #[test]
8927 fn content_hash_is_set_and_verifies() {
8928 let bytes = build_image();
8929 let rete = Rete::open(&bytes).unwrap();
8930 assert_ne!(
8931 rete.header().content_hash,
8932 [0u8; 16],
8933 "hash must be populated"
8934 );
8935 assert!(verify(&bytes).unwrap(), "freshly built file verifies");
8936 assert!(
8937 verify_ranged(&crate::reader::SliceReader::new(&bytes)).unwrap(),
8938 "the streaming verify agrees"
8939 );
8940
8941 assert_eq!(
8943 Rete::open(&build_image()).unwrap().header().content_hash,
8944 rete.header().content_hash
8945 );
8946
8947 let mut tampered = bytes.clone();
8949 let last = tampered.len() - 5; tampered[last] ^= 0xff;
8951 assert!(!verify(&tampered).unwrap());
8952 assert!(!verify_ranged(&crate::reader::SliceReader::new(&tampered)).unwrap());
8953 let short = &bytes[..bytes.len() / 2];
8955 assert!(verify_ranged(&crate::reader::SliceReader::new(short)).is_err());
8956 assert!(verify_ranged(&crate::reader::SliceReader::new(&bytes[..10])).is_err());
8957 }
8958
8959 fn build_with_metadata(meta: &[u8]) -> Vec<u8> {
8961 let triples = [
8962 ("Alice", "knows", "Bob"),
8963 ("Bob", "knows", "Carol"),
8964 ("Alice", "age", "30"),
8965 ];
8966 let mut db = DictionaryBuilder::new();
8967 for (s, p, o) in triples {
8968 db.observe(s, p, o);
8969 }
8970 let dict = db.build();
8971 let ids: Vec<_> = triples
8972 .iter()
8973 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
8974 .collect();
8975 let mut ib = GraphIndexBuilder::new();
8976 for &t in &ids {
8977 ib.push(t);
8978 }
8979 let (pmeta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
8980 write_dataset_with_metadata(&dict, &ib.build(), &[], false, &pmeta, levels, meta, &[])
8981 }
8982
8983 #[test]
8984 fn metadata_round_trips_and_shifts_offsets() {
8985 let card = br#"{"title":"My Dataset"}"#;
8986 let bytes = build_with_metadata(card);
8987 let rete = Rete::open(&bytes).unwrap();
8988
8989 assert_eq!(rete.metadata(), Some(card.as_slice()));
8991 let h = rete.header();
8992 assert_eq!(h.metadata_offset, HEADER_LEN as u64);
8993 assert_eq!(h.metadata_len, card.len() as u64);
8994 assert_eq!(h.dictionary_offset, HEADER_LEN as u64 + card.len() as u64);
8996
8997 assert_eq!(
8999 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9000 1
9001 );
9002 assert!(verify(&bytes).unwrap());
9004 }
9005
9006 #[test]
9007 fn empty_metadata_is_byte_identical_to_plain_writer() {
9008 assert_eq!(
9011 build_with_metadata(&[]),
9012 build_image(),
9013 "empty-metadata output must equal the plain writer byte-for-byte"
9014 );
9015 }
9016
9017 #[test]
9022 fn replace_metadata_matches_a_direct_write() {
9023 let first = br#"{"title":"My Dataset","queries":[1,2,3]}"#;
9024 let bytes = build_with_metadata(first);
9025 for target in [
9026 br#"{"title":"My Dataset","queries":[1,2]}"#.as_slice(), br#"{"title":"My Dataset","queries":[1,2,3,4,5,6,7]}"#.as_slice(), b"".as_slice(), first.as_slice(), ] {
9031 let spliced = replace_metadata(&bytes, target).unwrap();
9032 assert_eq!(
9033 spliced,
9034 build_with_metadata(target),
9035 "splicing {} bytes of metadata must equal a direct write",
9036 target.len()
9037 );
9038 assert!(
9039 verify(&spliced).unwrap(),
9040 "the new hash covers the new card"
9041 );
9042 let rete = Rete::open(&spliced).unwrap();
9043 assert_eq!(
9044 rete.metadata(),
9045 (!target.is_empty()).then_some(target),
9046 "the new payload reads back verbatim"
9047 );
9048 assert_eq!(
9050 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9051 1
9052 );
9053 }
9054 }
9055
9056 #[test]
9059 fn replace_metadata_keeps_build_info_adjacent() {
9060 let with_info =
9061 attach_build_info(&build_with_metadata(br#"{"a":1}"#), br#"{"builder":"x"}"#).unwrap();
9062 let spliced = replace_metadata(&with_info, br#"{"a":1,"b":2}"#).unwrap();
9063 let h = Header::from_bytes(&spliced).unwrap();
9064 assert_eq!(h.metadata_offset, HEADER_LEN as u64);
9065 assert_eq!(h.build_info_offset, HEADER_LEN as u64 + h.metadata_len);
9066 assert_eq!(
9067 read_build_info(&spliced).unwrap().as_deref(),
9068 Some(br#"{"builder":"x"}"#.as_slice())
9069 );
9070 assert!(verify(&spliced).unwrap());
9071 assert_eq!(
9073 h.content_hash,
9074 Header::from_bytes(&build_with_metadata(br#"{"a":1,"b":2}"#))
9075 .unwrap()
9076 .content_hash
9077 );
9078 }
9079
9080 #[test]
9081 fn metadata_is_tamper_evident() {
9082 let card = br#"{"title":"x"}"#;
9083 let mut bytes = build_with_metadata(card);
9084 assert!(verify(&bytes).unwrap());
9085 bytes[HEADER_LEN + 2] ^= 0xff;
9087 assert!(
9088 !verify(&bytes).unwrap(),
9089 "tampering with the card must break verify()"
9090 );
9091 }
9092
9093 #[test]
9094 fn ranged_opens_do_not_fetch_metadata() {
9095 use crate::reader::{CountingReader, SliceReader};
9096 let card = vec![0xABu8; 512]; let bytes = build_with_metadata(&card);
9098 let total = bytes.len() as u64;
9099
9100 let r = CountingReader::new(SliceReader::new(&bytes));
9102 let rete = Rete::open_ranged(&r).unwrap();
9103 assert!(
9104 rete.metadata().is_none(),
9105 "open_ranged must not load the card"
9106 );
9107 assert!(r.requests() <= 4, "requests = {}", r.requests());
9108 assert!(
9109 r.bytes_read() <= total - card.len() as u64,
9110 "read {} of {} bytes; the {}-byte card must be skipped",
9111 r.bytes_read(),
9112 total,
9113 card.len()
9114 );
9115
9116 let rs = CountingReader::new(SliceReader::new(&bytes));
9118 let view = SummaryView::open_ranged(&rs).unwrap().unwrap();
9119 assert!(!view.summary.is_empty());
9120 assert!(rs.bytes_read() <= total - card.len() as u64);
9121 }
9122
9123 #[test]
9124 fn metadata_ranged_fetches_only_header_and_card() {
9125 use crate::reader::{CountingReader, SliceReader};
9126 let card = vec![0xCDu8; 384];
9129 let bytes = build_with_metadata(&card);
9130
9131 let r = CountingReader::new(SliceReader::new(&bytes));
9132 let got = read_metadata_ranged(&r).unwrap().unwrap();
9133 assert_eq!(got, card, "the card reads back verbatim");
9134 assert_eq!(r.requests(), 2, "exactly header + metadata ranges");
9135 assert_eq!(
9136 r.bytes_read(),
9137 HEADER_LEN as u64 + card.len() as u64,
9138 "no dictionary/index/pyramid bytes are touched"
9139 );
9140
9141 let plain = build_image();
9143 let rp = CountingReader::new(SliceReader::new(&plain));
9144 assert!(read_metadata_ranged(&rp).unwrap().is_none());
9145 assert_eq!(rp.requests(), 1, "header only for a cardless file");
9146 assert_eq!(rp.bytes_read(), HEADER_LEN as u64);
9147 }
9148
9149 #[test]
9150 fn build_info_attaches_outside_the_hash() {
9151 let card = br#"{"title":"My Dataset"}"#;
9152 let base = build_with_metadata(card);
9153 let base_hash = Header::from_bytes(&base).unwrap().content_hash;
9154
9155 let info = br#"{"built_at":"2026-08-04T00:00:00Z","builder":"rete-cli 0.3.2"}"#;
9156 let with = attach_build_info(&base, info).unwrap();
9157
9158 assert_eq!(read_build_info(&with).unwrap().unwrap(), info);
9160 let h = Header::from_bytes(&with).unwrap();
9161 assert_eq!(h.build_info_offset, HEADER_LEN as u64 + card.len() as u64);
9162 assert_eq!(h.build_info_len, info.len() as u64);
9163 assert_eq!(h.expected_file_len(), Some(with.len() as u64));
9164
9165 assert_eq!(h.content_hash, base_hash);
9169 assert!(verify(&with).unwrap());
9170 let other = attach_build_info(&base, br#"{"built_at":"1999-01-01T00:00:00Z"}"#).unwrap();
9171 assert_eq!(Header::from_bytes(&other).unwrap().content_hash, base_hash);
9172
9173 let rete = Rete::open(&with).unwrap();
9175 assert_eq!(
9176 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9177 1
9178 );
9179 assert_eq!(rete.metadata(), Some(card.as_slice()));
9180
9181 let replaced = attach_build_info(&with, br#"{"builder":"other"}"#).unwrap();
9184 assert_eq!(
9185 read_build_info(&replaced).unwrap().unwrap(),
9186 br#"{"builder":"other"}"#
9187 );
9188 assert!(verify(&replaced).unwrap());
9189 let stripped = attach_build_info(&replaced, &[]).unwrap();
9190 assert_eq!(stripped, base, "stripping build-info restores the image");
9191 }
9192
9193 #[test]
9194 fn build_info_attaches_on_a_metadata_free_file() {
9195 let base = build_image();
9197 let info = b"{\"builder\":\"x\"}";
9198 let with = attach_build_info(&base, info).unwrap();
9199 let h = Header::from_bytes(&with).unwrap();
9200 assert_eq!(h.build_info_offset, HEADER_LEN as u64);
9201 assert_eq!(h.metadata_len, 0);
9202 assert_eq!(read_build_info(&with).unwrap().unwrap(), info);
9203 assert!(verify(&with).unwrap());
9204 assert_eq!(
9205 h.content_hash,
9206 Header::from_bytes(&base).unwrap().content_hash
9207 );
9208 Rete::open(&with).unwrap();
9209 }
9210
9211 #[test]
9218 fn a_plan_rebuilds_exactly_what_the_in_memory_splice_produces() {
9219 let card = br#"{"title":"My Dataset"}"#;
9220 let base = build_with_metadata(card);
9221 let base_hash = Header::from_bytes(&base).unwrap().content_hash;
9222 for start in [
9223 base.clone(),
9224 attach_build_info(&base, b"{\"a\":1}").unwrap(),
9225 ] {
9226 for info in [
9227 br#"{"builder":"rete-cli 0.3.2","query_costs":{}}"#.as_slice(), b"{}".as_slice(), b"".as_slice(), ] {
9231 let plan = plan_build_info(&start, start.len() as u64, info.len() as u64).unwrap();
9232 let mut streamed = Vec::new();
9233 streamed.extend_from_slice(&plan.header);
9234 streamed.extend_from_slice(&start[HEADER_LEN..plan.insert as usize]);
9235 streamed.extend_from_slice(info);
9236 streamed.extend_from_slice(&start[plan.tail_start as usize..]);
9237 assert_eq!(streamed.len() as u64, plan.new_len);
9238 assert_eq!(
9239 streamed,
9240 attach_build_info(&start, info).unwrap(),
9241 "a {}-byte section streamed must equal the same section spliced",
9242 info.len()
9243 );
9244 assert!(verify(&streamed).unwrap());
9246 let h = Header::from_bytes(&streamed).unwrap();
9247 assert_eq!(h.content_hash, base_hash);
9248 assert_eq!(h.expected_file_len(), Some(plan.new_len));
9249 let rete = Rete::open(&streamed).unwrap();
9250 assert_eq!(rete.metadata(), Some(card.as_slice()));
9251 assert_eq!(
9252 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9253 1
9254 );
9255 }
9256 }
9257 }
9258
9259 #[test]
9262 fn a_plan_refuses_a_header_that_overruns_the_file() {
9263 let base = build_with_metadata(br#"{"title":"x"}"#);
9264 assert!(plan_build_info(&base, 8, 16).is_err());
9265 assert!(plan_build_info(&base[..HEADER_LEN], base.len() as u64, 16).is_ok());
9266 }
9267
9268 #[test]
9269 fn card_and_build_info_ranged_is_one_header_plus_one_range() {
9270 use crate::reader::{CountingReader, SliceReader};
9271 let card = vec![0xCDu8; 384];
9272 let info = vec![0xEFu8; 200];
9273 let bytes = attach_build_info(&build_with_metadata(&card), &info).unwrap();
9274
9275 let r = CountingReader::new(SliceReader::new(&bytes));
9278 let (m, b) = read_card_and_build_info_ranged(&r).unwrap();
9279 assert_eq!(m.unwrap(), card);
9280 assert_eq!(b.unwrap(), info);
9281 assert_eq!(r.requests(), 2, "header + one coalesced range");
9282 assert_eq!(
9283 r.bytes_read(),
9284 HEADER_LEN as u64 + card.len() as u64 + info.len() as u64
9285 );
9286
9287 let plain = build_with_metadata(&card);
9289 let rp = CountingReader::new(SliceReader::new(&plain));
9290 let (m, b) = read_card_and_build_info_ranged(&rp).unwrap();
9291 assert_eq!(m.unwrap(), card);
9292 assert!(b.is_none());
9293 assert_eq!(rp.requests(), 2);
9294 assert_eq!(
9299 r.requests(),
9300 rp.requests(),
9301 "build info must not cost an extra request",
9302 );
9303
9304 let none = build_image();
9306 let rn = CountingReader::new(SliceReader::new(&none));
9307 let (m, b) = read_card_and_build_info_ranged(&rn).unwrap();
9308 assert!(m.is_none() && b.is_none());
9309 assert_eq!(rn.requests(), 1);
9310 }
9311
9312 #[test]
9313 fn schema_summary_groups_by_type() {
9314 let rt = RDF_TYPE;
9315 let bytes = build_from(&[
9316 ("Alice", rt, "Person"),
9317 ("Bob", rt, "Person"),
9318 ("NYC", rt, "City"),
9319 ("Alice", "knows", "Bob"),
9320 ("Alice", "livesIn", "NYC"),
9321 ("Alice", "name", "\"Alice\""),
9322 ]);
9323 let rete = Rete::open(&bytes).unwrap();
9324 let summary = schema_summary(&rete);
9325 assert!(summary.contains(&("Person".into(), "knows".into(), "Person".into(), 1)));
9327 assert!(summary.contains(&("Person".into(), "livesIn".into(), "City".into(), 1)));
9328 assert!(summary.contains(&("Person".into(), "name".into(), "(literal)".into(), 1)));
9329 assert!(!summary.iter().any(|(_, p, _, _)| p == RDF_TYPE));
9331
9332 let classes = schema_classes(&rete);
9334 assert_eq!(
9335 classes,
9336 vec![("Person".into(), 2u32), ("City".into(), 1u32)]
9337 );
9338 }
9339
9340 fn build_from(triples: &[(&str, &str, &str)]) -> Vec<u8> {
9341 let mut db = DictionaryBuilder::new();
9342 for (s, p, o) in triples {
9343 db.observe(s, p, o);
9344 }
9345 let dict = db.build();
9346 let mut ib = GraphIndexBuilder::new();
9347 for (s, p, o) in triples {
9348 ib.push(dict.encode(s, p, o).unwrap());
9349 }
9350 write_file(&dict, &ib.build(), false, &[], 0)
9351 }
9352
9353 fn build_with_pyramid(triples: &[(&str, &str, &str)]) -> Vec<u8> {
9355 let mut db = DictionaryBuilder::new();
9356 for (s, p, o) in triples {
9357 db.observe(s, p, o);
9358 }
9359 let dict = db.build();
9360 let encoded: Vec<_> = triples
9361 .iter()
9362 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9363 .collect();
9364 let mut ib = GraphIndexBuilder::new();
9365 for t in &encoded {
9366 ib.push(*t);
9367 }
9368 let (meta, levels) = build_pyramid_meta(&dict, &encoded, DEFAULT_TILE_BUDGET);
9369 write_dataset(&dict, &ib.build(), &[], false, &meta, levels)
9370 }
9371
9372 #[test]
9373 fn tbox_coherence_flags_unsatisfiable_class_index_free() {
9374 use crate::reader::{CountingReader, SliceReader};
9375 let rt = RDF_TYPE;
9376 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9377 let disj = "<http://www.w3.org/2002/07/owl#disjointWith>";
9378 let bytes = build_with_pyramid(&[
9381 ("<http://ex/C>", sub, "<http://ex/D>"),
9382 ("<http://ex/C>", sub, "<http://ex/E>"),
9383 ("<http://ex/D>", disj, "<http://ex/E>"),
9384 ("<http://ex/x>", rt, "<http://ex/C>"),
9385 ]);
9386
9387 let r = CountingReader::new(SliceReader::new(&bytes));
9388 let view = SummaryView::open_ranged(&r).unwrap().unwrap();
9389 let points = view.tbox_coherence();
9390 assert!(
9391 points
9392 .iter()
9393 .any(|i| i.kind == "unsatisfiable-class" && i.detail.contains("http://ex/C>")),
9394 "expected C unsatisfiable from the schema alone, got {points:?}"
9395 );
9396
9397 let header = Header::from_bytes(&bytes[..HEADER_LEN]).unwrap();
9400 assert!(
9401 r.bytes_read() <= bytes.len() as u64 - header.root_dir_len,
9402 "tbox_coherence must not read the triple index"
9403 );
9404 }
9405
9406 #[test]
9407 fn schema_coherence_reads_only_the_schema_block() {
9408 use crate::reader::{CountingReader, SliceReader};
9409 let rt = RDF_TYPE;
9410 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9411 let disj = "<http://www.w3.org/2002/07/owl#disjointWith>";
9412 let mut triples: Vec<(String, String, String)> = vec![
9416 ("<http://ex/C>".into(), sub.into(), "<http://ex/D>".into()),
9417 ("<http://ex/C>".into(), sub.into(), "<http://ex/E>".into()),
9418 ("<http://ex/D>".into(), disj.into(), "<http://ex/E>".into()),
9419 ];
9420 for i in 0..500 {
9421 let s = format!("<http://ex/x{i}>");
9422 triples.push((s.clone(), rt.into(), "<http://ex/C>".into()));
9423 triples.push((
9424 s,
9425 "<http://ex/label>".into(),
9426 format!("\"unique label {i}\""),
9427 ));
9428 }
9429 let trefs: Vec<(&str, &str, &str)> = triples
9430 .iter()
9431 .map(|(s, p, o)| (s.as_str(), p.as_str(), o.as_str()))
9432 .collect();
9433 let bytes = build_with_pyramid(&trefs);
9434
9435 let header = Header::from_bytes(&bytes[..HEADER_LEN]).unwrap();
9436 assert!(
9437 header.schema_meta_len > 0,
9438 "the writer recorded a schema-block length"
9439 );
9440 assert!(
9441 (header.schema_meta_len as u64) < header.pyramid_meta_len,
9442 "schema block ({}) should be far smaller than the whole pyramid-meta ({})",
9443 header.schema_meta_len,
9444 header.pyramid_meta_len
9445 );
9446
9447 let r = CountingReader::new(SliceReader::new(&bytes));
9448 let points = read_schema_coherence_ranged(&r).unwrap().unwrap();
9449 assert!(points.iter().any(|i| i.kind == "unsatisfiable-class"));
9450 assert!(
9452 r.bytes_read() <= HEADER_LEN as u64 + header.schema_meta_len as u64,
9453 "read {} bytes; expected <= header + schema block ({})",
9454 r.bytes_read(),
9455 HEADER_LEN as u64 + header.schema_meta_len as u64
9456 );
9457 }
9458
9459 #[test]
9460 fn tbox_coherence_clean_schema_is_coherent() {
9461 use crate::reader::SliceReader;
9462 let rt = RDF_TYPE;
9463 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9464 let bytes = build_with_pyramid(&[
9465 ("<http://ex/Dog>", sub, "<http://ex/Animal>"),
9466 ("<http://ex/x>", rt, "<http://ex/Dog>"),
9467 ]);
9468 let view = SummaryView::open_ranged(&SliceReader::new(&bytes))
9469 .unwrap()
9470 .unwrap();
9471 assert!(view.tbox_is_coherent(), "a plain hierarchy is coherent");
9472 }
9473
9474 #[test]
9482 fn readers_before_the_quad_index_refuse_its_files() {
9483 for perms in [PermSet::ALL, PermSet::CORE] {
9484 let quads: Vec<crate::ingest::RawQuad> = (0..50)
9485 .map(|i| {
9486 (
9487 format!("<http://ex/s{i}>"),
9488 "<http://ex/p>".to_string(),
9489 format!("\"{i}\""),
9490 Some(format!("<http://ex/g{}>", i % 7)),
9491 )
9492 })
9493 .collect();
9494 let (bytes, _) = crate::ingest::assemble_dataset_with_layout(
9495 quads,
9496 false,
9497 false,
9498 None,
9499 crate::PyramidAlgo::Louvain,
9500 perms,
9501 crate::ingest::GraphLayout::Quad,
9502 |_, _| Vec::<u8>::new(),
9503 );
9504 let h = Header::from_bytes(&bytes).unwrap();
9505 assert!(h.has_quad_index());
9506 let root =
9507 &bytes[h.root_dir_offset as usize..(h.root_dir_offset + h.root_dir_len) as usize];
9508 assert!(decode_index_container(root, h.block_codec, h.perms).is_err());
9510 assert!(
9511 container_section_payload_ranges(root, h.root_dir_offset, h.perms.len()).is_err()
9512 );
9513 assert_ne!(decode_container(root, CODEC_NONE).unwrap().len(), NUM_PERMS);
9515 assert_ne!(
9516 decode_container(root, CODEC_NONE).unwrap().len(),
9517 h.perms.len()
9518 );
9519 let r = crate::reader::SliceReader::new(&bytes);
9521 for expect in [h.perms.len(), NUM_PERMS] {
9522 assert!(locate_container_section_ranged(
9523 &r,
9524 h.root_dir_offset,
9525 h.root_dir_len,
9526 0,
9527 expect as u64
9528 )
9529 .is_err());
9530 }
9531 assert_eq!(hash_of_sections(&h, &bytes).unwrap(), h.content_hash);
9533 let rete = Rete::open(&bytes).unwrap();
9535 assert_eq!(rete.named_graph_count(), 7);
9536 assert_eq!(rete.dump(Some("<http://ex/g3>")).len(), 7);
9537 }
9538 }
9539
9540 #[test]
9541 fn named_graphs_round_trip() {
9542 let all = [
9544 ("Alice", "knows", "Bob"), ("Bob", "age", "30"), ];
9547 let mut db = DictionaryBuilder::new();
9548 for (s, p, o) in all {
9549 db.observe(s, p, o);
9550 }
9551 let dict = db.build();
9552
9553 let mut def = GraphIndexBuilder::new();
9554 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
9555 let mut g1 = GraphIndexBuilder::new();
9556 g1.push(dict.encode("Bob", "age", "30").unwrap());
9557
9558 let named = vec![("http://ex/g1".to_string(), g1.build())];
9559 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9560
9561 assert!(verify(&bytes).unwrap());
9562 let rete = Rete::open(&bytes).unwrap();
9563 assert_eq!(rete.graph_names(), vec!["http://ex/g1"]);
9564 let gi = rete.graph_index("http://ex/g1").unwrap();
9566 assert_eq!(gi.triple_count(), 1);
9567 assert!(rete.graph_index("http://ex/missing").is_none());
9568
9569 assert_eq!(rete.header().quad_count, 2);
9572
9573 assert_eq!(rete.query(Some("Alice"), None, None).len(), 1);
9575
9576 assert_eq!(
9578 rete.dump(None),
9579 vec![("Alice".into(), "knows".into(), "Bob".into())]
9580 );
9581 assert_eq!(
9582 rete.dump(Some("http://ex/g1")),
9583 vec![("Bob".into(), "age".into(), "30".into())]
9584 );
9585 }
9586
9587 #[test]
9593 fn named_graph_container_opens_tile_lazily_like_the_root() {
9594 let mut db = DictionaryBuilder::new();
9595 let node = |n: u32| format!("<http://ex/n{n}>");
9596 let p = "<http://ex/p>".to_string();
9597 db.observe(&node(0), &p, &node(1));
9598 for i in 0..500u32 {
9599 db.observe(&node(1000 + i), &p, &node(2000 + i));
9600 }
9601 let dict = db.build();
9602 let mut def = GraphIndexBuilder::new();
9603 def.push(dict.encode(&node(0), &p, &node(1)).unwrap());
9604 let mut g = GraphIndexBuilder::new().with_tile_budget(256); for i in 0..500u32 {
9606 g.push(dict.encode(&node(1000 + i), &p, &node(2000 + i)).unwrap());
9607 }
9608 let named = vec![("<http://ex/g>".to_string(), g.build())];
9609 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9610 let rete = Rete::open(&bytes).unwrap();
9611 let header = rete.header();
9612
9613 let soff = header.named_graphs_offset as usize;
9616 let send = soff + header.named_graphs_len as usize;
9617 let sec = &bytes[soff..send];
9618 let (n, mut pos) = read_uvarint(sec).unwrap();
9619 assert_eq!(n, 1);
9620 let (ilen, u1) = read_uvarint(&sec[pos..]).unwrap();
9621 pos += u1 + ilen as usize;
9622 let (clen, u2) = read_uvarint(&sec[pos..]).unwrap();
9623 pos += u2;
9624 let container = ByteRange {
9625 offset: (soff + pos) as u64,
9626 len: clen,
9627 };
9628
9629 use crate::reader::SliceReader;
9630 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9631 let reader: std::sync::Arc<dyn RangeReader + Send + Sync> =
9632 std::sync::Arc::new(SliceReader::new(leaked));
9633 let (lazy_idx, _, _) = open_index_container_lazy(
9634 &reader,
9635 container,
9636 header.block_codec,
9637 header.has_tile_synopsis(),
9638 1,
9639 header.perms,
9640 header.perms.len(),
9641 )
9642 .unwrap();
9643 let mut want = rete
9644 .graph_index("<http://ex/g>")
9645 .unwrap()
9646 .match_pattern((None, None, None));
9647 let mut got = lazy_idx.match_pattern((None, None, None));
9648 want.sort_unstable();
9649 got.sort_unstable();
9650 assert_eq!(got.len(), 500);
9651 assert_eq!(got, want);
9652 assert!(!lazy_idx.load_incomplete());
9653 }
9654
9655 #[test]
9661 fn dump_iter_matches_dump_and_stops_early() {
9662 use crate::reader::{CountingReader, SliceReader};
9663
9664 let triples: Vec<(String, String, String)> = (0..2000u32)
9666 .map(|i| {
9667 (
9668 format!("<http://ex/s/{i:04}>"),
9669 "<http://ex/p>".to_string(),
9670 format!("<http://ex/o/{i:04}>"),
9671 )
9672 })
9673 .collect();
9674 let mut db = DictionaryBuilder::new();
9675 for (s, p, o) in &triples {
9676 db.observe(s, p, o);
9677 }
9678 let dict = db.build();
9679 let mut ib = GraphIndexBuilder::new().with_tile_budget(256);
9680 for (s, p, o) in &triples {
9681 ib.push(dict.encode(s, p, o).unwrap());
9682 }
9683 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9684
9685 let rete = Rete::open(&bytes).unwrap();
9687 assert_eq!(rete.dump_iter(None).collect::<Vec<_>>(), rete.dump(None));
9688 assert_eq!(rete.dump_iter(Some("http://ex/nope")).count(), 0);
9690
9691 let read_bytes = |take: usize| -> u64 {
9693 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9694 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
9695 let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
9696 let before = reader.bytes_read();
9697 let n = rete.dump_iter(None).take(take).count();
9698 assert_eq!(n, take.min(triples.len()));
9699 assert!(!rete.index_incomplete());
9700 reader.bytes_read() - before
9701 };
9702 let first_only = read_bytes(1);
9703 let everything = read_bytes(triples.len());
9704 assert!(
9706 first_only < everything,
9707 "dump_iter is not lazy: one triple read {first_only} B, a full scan {everything} B"
9708 );
9709 #[cfg(feature = "compression")]
9716 assert!(
9717 first_only * 2 < everything,
9718 "dump_iter is not lazy: one triple read {first_only} B of the {everything} B a full scan reads"
9719 );
9720
9721 let fresh = Rete::open(&bytes).unwrap();
9726 let drain = |max_quads: usize| {
9731 let (mut out, mut cursor, mut calls) = (Vec::new(), 0u32, 0);
9732 loop {
9733 let (batch, next, done) = fresh.dump_batch(None, cursor, max_quads);
9734 out.extend(batch);
9735 cursor = next;
9736 calls += 1;
9737 assert!(calls < 10_000, "cursor failed to advance");
9738 if done {
9739 break out;
9740 }
9741 }
9742 };
9743 let eager = fresh.dump(None);
9744 for max in [1usize, 7, 128, 100_000] {
9745 assert_eq!(drain(max), eager, "batch size {max} changed the dump");
9746 }
9747 let (t, _, done) = fresh.dump_batch(Some("http://ex/nope"), 0, 16);
9750 assert!(t.is_empty() && done);
9751 }
9752
9753 #[test]
9761 fn query_batch_matches_query_in_graph_and_stops_early() {
9762 use crate::reader::{CountingReader, SliceReader};
9763
9764 let triples: Vec<(String, String, String)> = (0..2000u32)
9767 .map(|i| {
9768 (
9769 format!("<http://ex/s/{:04}>", i / 2),
9770 format!("<http://ex/p/{}>", i % 2),
9771 format!("<http://ex/o/{i:04}>"),
9772 )
9773 })
9774 .collect();
9775 let mut db = DictionaryBuilder::new();
9776 for (s, p, o) in &triples {
9777 db.observe(s, p, o);
9778 }
9779 let dict = db.build();
9780 let mut ib = GraphIndexBuilder::new().with_tile_budget(256);
9781 for (s, p, o) in &triples {
9782 ib.push(dict.encode(s, p, o).unwrap());
9783 }
9784 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9785 let rete = Rete::open(&bytes).unwrap();
9786
9787 let drain = |s: Option<&str>, p: Option<&str>, o: Option<&str>, max: usize| {
9788 let (mut out, mut cursor, mut calls) = (Vec::new(), 0u64, 0);
9789 loop {
9790 let (batch, next, done) = rete.query_batch(None, s, p, o, cursor, max);
9791 assert!(!batch.is_empty() || done, "empty non-final batch");
9794 out.extend(batch);
9795 cursor = next;
9796 calls += 1;
9797 assert!(calls < 20_000, "cursor failed to advance");
9798 if done {
9799 break out;
9800 }
9801 }
9802 };
9803 let shapes: [(Option<&str>, Option<&str>, Option<&str>); 6] = [
9804 (None, None, None),
9805 (Some("<http://ex/s/0007>"), None, None),
9806 (None, Some("<http://ex/p/1>"), None),
9807 (None, None, Some("<http://ex/o/0042>")),
9808 (Some("<http://ex/s/0007>"), Some("<http://ex/p/0>"), None),
9809 (
9810 Some("<http://ex/s/0007>"),
9811 Some("<http://ex/p/0>"),
9812 Some("<http://ex/o/0014>"),
9813 ),
9814 ];
9815 for (s, p, o) in shapes {
9816 let eager = rete.query_in_graph(None, s, p, o);
9817 assert!(
9818 !eager.is_empty(),
9819 "fixture shape {s:?} {p:?} {o:?} matches nothing"
9820 );
9821 assert_eq!(
9823 rete.query_iter(None, s, p, o).collect::<Vec<_>>(),
9824 eager,
9825 "query_iter disagrees for {s:?} {p:?} {o:?}"
9826 );
9827 for max in [1usize, 3, 64, 100_000] {
9828 let mut got = drain(s, p, o, max);
9829 let mut want = eager.clone();
9830 got.sort();
9833 want.sort();
9834 assert_eq!(got, want, "batch size {max} changed {s:?} {p:?} {o:?}");
9835 }
9836 if s.is_none() && p.is_none() && o.is_none() {
9838 assert_eq!(drain(s, p, o, 7), eager, "unbound batching reordered rows");
9839 }
9840 }
9841
9842 let (t, _, done) = rete.query_batch(None, Some("<http://ex/nope>"), None, None, 0, 16);
9845 assert!(t.is_empty() && done);
9846 let (t, _, done) = rete.query_batch(Some("http://ex/nope"), None, None, None, 0, 16);
9847 assert!(t.is_empty() && done);
9848 assert_eq!(
9849 rete.query_iter(None, Some("<http://ex/nope>"), None, None)
9850 .count(),
9851 0
9852 );
9853
9854 let read_bytes = |take: usize| -> u64 {
9857 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9858 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
9859 let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
9860 let before = reader.bytes_read();
9861 let (mut n, mut cursor) = (0usize, 0u64);
9862 loop {
9863 let (batch, next, done) = rete.query_batch(None, None, None, None, cursor, 1);
9864 n += batch.len();
9865 cursor = next;
9866 if done || n >= take {
9867 break;
9868 }
9869 }
9870 assert!(n >= take.min(triples.len()));
9871 assert!(!rete.index_incomplete());
9872 reader.bytes_read() - before
9873 };
9874 let first_only = read_bytes(1);
9875 let everything = read_bytes(triples.len());
9876 assert!(
9877 first_only < everything,
9878 "query_batch is not lazy: one row read {first_only} B, a full scan {everything} B"
9879 );
9880 #[cfg(feature = "compression")]
9881 assert!(
9882 first_only * 2 < everything,
9883 "query_batch is not lazy: one row read {first_only} B of the {everything} B a full scan reads"
9884 );
9885 }
9886
9887 #[test]
9890 fn query_batch_is_graph_scoped() {
9891 let mut db = DictionaryBuilder::new();
9892 for (s, p, o) in [
9893 ("Alice", "knows", "Bob"),
9894 ("Alice", "knows", "Carol"),
9895 ("Alice", "knows", "Dave"),
9896 ("Erin", "knows", "Frank"),
9897 ("Alice", "knows", "Zoe"),
9898 ] {
9899 db.observe(s, p, o);
9900 }
9901 let dict = db.build();
9902 let mut def = GraphIndexBuilder::new();
9903 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
9904 def.push(dict.encode("Alice", "knows", "Carol").unwrap());
9905 let mut g1 = GraphIndexBuilder::new();
9906 g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
9907 g1.push(dict.encode("Erin", "knows", "Frank").unwrap());
9908 let mut g2 = GraphIndexBuilder::new();
9909 g2.push(dict.encode("Alice", "knows", "Zoe").unwrap());
9910 let named = vec![
9911 ("http://ex/g1".to_string(), g1.build()),
9912 ("http://ex/g2".to_string(), g2.build()),
9913 ];
9914 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9915 let rete = Rete::open(&bytes).unwrap();
9916 for graph in [None, Some("http://ex/g1"), Some("http://ex/g2")] {
9917 let eager = rete.query_in_graph(graph, None, None, None);
9918 let (mut out, mut cursor) = (Vec::new(), 0u64);
9919 loop {
9920 let (batch, next, done) = rete.query_batch(graph, None, None, None, cursor, 1);
9921 out.extend(batch);
9922 cursor = next;
9923 if done {
9924 break;
9925 }
9926 }
9927 assert_eq!(out, eager, "graph {graph:?} batched differently");
9928 }
9929 }
9930
9931 #[test]
9932 fn query_in_graph_is_graph_scoped() {
9933 let mut db = DictionaryBuilder::new();
9936 for (s, p, o) in [
9937 ("Alice", "knows", "Bob"),
9938 ("Alice", "knows", "Carol"),
9939 ("Alice", "knows", "Dave"),
9940 ] {
9941 db.observe(s, p, o);
9942 }
9943 let dict = db.build();
9944
9945 let mut def = GraphIndexBuilder::new();
9946 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
9947 def.push(dict.encode("Alice", "knows", "Carol").unwrap());
9948 let mut g1 = GraphIndexBuilder::new();
9949 g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
9950
9951 let named = vec![("http://ex/g1".to_string(), g1.build())];
9952 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9953 let rete = Rete::open(&bytes).unwrap();
9954
9955 let mut def_objs: Vec<String> = rete
9957 .query_in_graph(None, Some("Alice"), Some("knows"), None)
9958 .into_iter()
9959 .map(|(_, _, o)| o)
9960 .collect();
9961 def_objs.sort();
9962 assert_eq!(def_objs, vec!["Bob".to_string(), "Carol".to_string()]);
9963
9964 assert_eq!(
9966 rete.query_in_graph(Some("http://ex/g1"), Some("Alice"), None, None),
9967 vec![("Alice".into(), "knows".into(), "Dave".into())]
9968 );
9969
9970 assert_eq!(rete.query_in_graph(None, None, None, None).len(), 2);
9972 assert_eq!(
9973 rete.query_in_graph(Some("http://ex/g1"), None, None, None)
9974 .len(),
9975 1
9976 );
9977
9978 assert!(rete
9980 .query_in_graph(Some("http://ex/missing"), None, None, None)
9981 .is_empty());
9982 }
9983
9984 #[test]
9997 fn bounded_query_in_graph_does_not_fault_the_whole_dictionary() {
9998 use crate::reader::{CountingReader, SliceReader};
9999
10000 let payload = |i: u32| -> String {
10006 let mut s = String::with_capacity(800);
10007 let mut x = i as u64 ^ 0x9e37_79b9_7f4a_7c15;
10008 while s.len() < 800 {
10009 x = x
10010 .wrapping_mul(6364136223846793005)
10011 .wrapping_add(1442695040888963407);
10012 s.push_str(&format!("{:016x}", x ^ (x >> 29)));
10013 }
10014 s
10015 };
10016 let quads: Vec<(String, String, String, bool)> = (0..2000u32)
10023 .map(|i| {
10024 let named = i % 2 == 0;
10025 let ns = if named { 'a' } else { 'b' };
10026 (
10027 format!("<http://ex/{ns}/s/{i:04}>"),
10028 "<http://ex/abstract>".to_string(),
10029 format!("\"{ns}{} number {i}\"", payload(i)),
10030 named,
10031 )
10032 })
10033 .collect();
10034 let mut db = DictionaryBuilder::new();
10035 for (s, p, o, _) in &quads {
10036 db.observe(s, p, o);
10037 }
10038 let dict = db.build();
10039 let mut def = GraphIndexBuilder::new().with_tile_budget(256);
10040 let mut g1 = GraphIndexBuilder::new().with_tile_budget(256);
10041 for (s, p, o, named) in &quads {
10042 let t = dict.encode(s, p, o).unwrap();
10043 if *named {
10044 g1.push(t);
10045 } else {
10046 def.push(t);
10047 }
10048 }
10049 let named = vec![("http://ex/g1".to_string(), g1.build())];
10050 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10051
10052 let eager = Rete::open(&bytes).unwrap();
10053 let dict_len = eager.header().dictionary_len;
10054
10055 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
10058 let probe = |f: &dyn Fn(&Rete) -> Vec<TermTriple>| -> (Vec<TermTriple>, u64) {
10059 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10060 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10061 let before = reader.bytes_read();
10062 let out = f(&lazy);
10063 assert!(!lazy.index_incomplete(), "a chunk fetch failed");
10064 (out, reader.bytes_read() - before)
10065 };
10066
10067 let want = eager.query_in_graph(
10069 Some("http://ex/g1"),
10070 Some("<http://ex/a/s/0100>"),
10071 None,
10072 None,
10073 );
10074 assert_eq!(want.len(), 1, "fixture: expected exactly one match");
10075 let (got, pulled) = probe(&|r| {
10076 r.query_in_graph(
10077 Some("http://ex/g1"),
10078 Some("<http://ex/a/s/0100>"),
10079 None,
10080 None,
10081 )
10082 });
10083 assert_eq!(got, want, "lazy query_in_graph disagrees with eager");
10084 assert!(
10085 pulled * 4 < dict_len,
10086 "a 1-row query_in_graph faulted {pulled} B of a {dict_len} B dictionary — \
10087 it is prefetching the whole thing again"
10088 );
10089
10090 let (empty, q_pulled) =
10098 probe(&|r| r.query_in_graph(Some("http://ex/nope"), None, None, None));
10099 assert!(empty.is_empty());
10100 let (_, d_pulled) = probe(&|r| {
10101 let mut n = 0usize;
10102 r.dump_each(Some("http://ex/nope"), |_, _, _| n += 1);
10103 assert_eq!(n, 0, "an absent graph yielded triples");
10104 Vec::new()
10105 });
10106 assert_eq!(
10107 q_pulled, d_pulled,
10108 "query_in_graph and dump_each disagree on what an absent graph costs"
10109 );
10110 assert!(
10111 q_pulled * 4 < dict_len,
10112 "an absent graph faulted {q_pulled} B of a {dict_len} B dictionary"
10113 );
10114
10115 let want: Vec<TermTriple> = eager.dump(Some("http://ex/g1"));
10118 assert_eq!(want.len(), 1000);
10119 let (got, pulled) = probe(&|r| {
10120 let mut out = Vec::new();
10121 r.dump_each(Some("http://ex/g1"), |s, p, o| {
10122 out.push((s.to_string(), p.to_string(), o.to_string()))
10123 });
10124 out
10125 });
10126 assert_eq!(got, want, "lazy dump_each disagrees with eager dump");
10127 assert!(
10128 pulled * 3 < dict_len * 2,
10129 "dumping half the graph faulted {pulled} B of a {dict_len} B dictionary — \
10130 the other graph's chunks are being pulled in too"
10131 );
10132 }
10133
10134 #[cfg(test)]
10139 fn build_multichunk_image() -> Vec<u8> {
10140 use crate::index::PermSet;
10141 let mut db = DictionaryBuilder::new();
10142 let mut triples: Vec<(String, String, String)> = (0..4000u32)
10143 .map(|i| {
10144 (
10145 format!("<http://ex/s/{i:07}>"),
10146 format!("<http://ex/p/{}>", i % 8),
10147 format!("\"object literal number {i:07} with padding text to widen the term\""),
10148 )
10149 })
10150 .collect();
10151 triples.push((
10152 "<http://ex/s/big>".to_string(),
10153 "<http://ex/p/big>".to_string(),
10154 format!("\"{}\"", "Z".repeat(8000)),
10155 ));
10156 for (s, p, o) in &triples {
10157 db.observe(s, p, o);
10158 }
10159 let dict = db.build();
10160 let mut idx = GraphIndexBuilder::new()
10161 .with_tile_budget(64)
10162 .with_perms(PermSet::ALL);
10163 for (s, p, o) in &triples {
10164 idx.push(dict.encode(s, p, o).unwrap());
10165 }
10166 write_dataset(&dict, &idx.build(), &[], true, &[], 0)
10167 }
10168
10169 #[test]
10178 fn tiny_dict_cache_evicts_yet_dump_is_byte_identical() {
10179 use crate::reader::SliceReader;
10180 let leaked: &'static [u8] = Box::leak(build_multichunk_image().into_boxed_slice());
10181 let dump = |cap: u64| -> (
10182 Vec<(String, String, String)>,
10183 crate::chunk_cache::CacheStats,
10184 ) {
10185 let rete =
10186 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10187 rete.dict.set_cache_cap(cap);
10188 let mut out = Vec::new();
10189 rete.dump_filtered_each(None, None, None, None, |s, p, o| {
10190 out.push((s.to_string(), p.to_string(), o.to_string()))
10191 });
10192 assert!(
10193 !rete.index_incomplete(),
10194 "no lazy fault should fail at cap {cap}"
10195 );
10196 let stats = rete.dict_cache_stats();
10197 (out, stats)
10198 };
10199
10200 let (unlimited, us) = dump(u64::MAX);
10201 assert_eq!(unlimited.len(), 4001, "every triple dumped");
10202 assert_eq!(us.evictions, 0, "unlimited cap evicts nothing");
10203
10204 let (tiny, ts) = dump(128 * 1024);
10207 assert_eq!(tiny, unlimited, "128 KiB-cap dump differs from unlimited");
10208 assert!(ts.evictions > 0, "a tiny cap must evict (stats {ts:?})");
10209
10210 let (starved, ss) = dump(1);
10212 assert_eq!(starved, unlimited, "1-byte-cap dump differs from unlimited");
10213 assert!(
10214 ss.evictions > 0,
10215 "1-byte cap evicts every insert (stats {ss:?})"
10216 );
10217 }
10218
10219 #[test]
10230 fn memory_budget_charges_the_fixed_directories() {
10231 use crate::reader::SliceReader;
10232 let leaked: &'static [u8] = Box::leak(build_multichunk_image().into_boxed_slice());
10233 let rete =
10234 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10235
10236 let fixed = rete.fixed_overhead();
10237 assert!(
10240 fixed.dict_directory > 0 && fixed.index_directory > 0,
10241 "{fixed:?}"
10242 );
10243 assert_eq!(fixed.named_graphs, 0, "no named graphs in this file");
10244 assert_eq!(
10245 fixed.total(),
10246 fixed.dict_directory + fixed.index_directory + fixed.named_graphs
10247 );
10248
10249 let total: u64 = 512 << 20;
10252 let applied = rete.set_memory_budget(Some(total));
10253 let plain = split_memory_budget(total);
10254 assert_eq!(applied.dict_cache, plain.dict_cache - fixed.total());
10255 assert_eq!(rete.dict_cache_cap(), applied.dict_cache);
10256 assert_eq!(applied.tile_cache, plain.tile_cache);
10257 assert_eq!(applied.block_cache, plain.block_cache);
10258 assert!(
10259 applied.dict_cache + applied.tile_cache + applied.block_cache + fixed.total() <= total,
10260 "caps + fixed directories exceed the budget: {applied:?} + {fixed:?} > {total}"
10261 );
10262
10263 let applied = rete.set_memory_budget(Some(1 << 20));
10265 assert_eq!(applied.dict_cache, MIN_DICT_CACHE_CAP);
10266
10267 let applied = rete.set_memory_budget(None);
10269 assert_eq!(applied.dict_cache, u64::MAX);
10270 assert_eq!(rete.dict_cache_cap(), u64::MAX);
10271
10272 let rete =
10275 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10276 rete.dict.set_cache_cap(128 * 1024);
10277 let mut rows = 0usize;
10278 rete.dump_filtered_each(None, None, None, None, |_, _, _| rows += 1);
10279 assert_eq!(rows, 4001);
10280 let s = rete.dict_cache_stats();
10281 assert!(s.evictions > 0, "the cap must bind: {s:?}");
10282 assert!(
10285 s.entries <= 1 || s.resident_bytes <= 128 * 1024,
10286 "dictionary cache over its cap after the dump: {s:?}"
10287 );
10288 }
10289
10290 #[test]
10295 fn releasing_a_named_graph_drops_and_reopens_its_index() {
10296 use crate::index::PermSet;
10297 use crate::reader::SliceReader;
10298 let mut db = DictionaryBuilder::new();
10299 let all: Vec<(String, String, String)> = (0..40u32)
10300 .map(|i| {
10301 (
10302 format!("<http://ex/s{}>", i % 7),
10303 format!("<http://ex/p{}>", i % 3),
10304 format!("<http://ex/o{}>", i % 5),
10305 )
10306 })
10307 .collect();
10308 for (s, p, o) in &all {
10309 db.observe(s, p, o);
10310 }
10311 let dict = db.build();
10312 let mk = |take: fn(u32) -> bool| {
10313 let mut g = GraphIndexBuilder::new()
10314 .with_tile_budget(16)
10315 .with_perms(PermSet::ALL);
10316 for (i, (s, p, o)) in all.iter().enumerate() {
10317 if take(i as u32) {
10318 g.push(dict.encode(s, p, o).unwrap());
10319 }
10320 }
10321 g.build()
10322 };
10323 let def = mk(|_| true);
10324 let g1 = mk(|i| i % 2 == 0);
10325 let g2 = mk(|i| i % 3 == 0);
10326 let image = write_dataset(
10327 &dict,
10328 &def,
10329 &[
10330 ("<http://ex/g1>".to_string(), g1),
10331 ("<http://ex/g2>".to_string(), g2),
10332 ],
10333 true,
10334 &[],
10335 0,
10336 );
10337 let leaked: &'static [u8] = Box::leak(image.into_boxed_slice());
10338 let mut rete =
10339 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10340
10341 let opened = |r: &Rete| -> usize {
10342 match &r.named_graphs {
10343 NamedGraphsSlot::Lazy(l) => {
10344 let mut n = 0;
10345 l.for_each_opened(|_| n += 1);
10346 n
10347 }
10348 _ => 0,
10349 }
10350 };
10351
10352 assert_eq!(opened(&rete), 0, "nothing opened before first touch");
10353 assert!(rete.graph_index("<http://ex/g1>").is_some());
10355 assert_eq!(opened(&rete), 1, "g1 opened");
10356 rete.release_named_graph("<http://ex/g1>");
10358 assert_eq!(opened(&rete), 0, "g1 released");
10359 rete.release_named_graph("<http://ex/absent>");
10361 rete.release_named_graph("<http://ex/g1>");
10362 assert_eq!(opened(&rete), 0);
10363 assert!(rete.graph_index("<http://ex/g1>").is_some());
10365 assert_eq!(opened(&rete), 1, "g1 re-opened after release");
10366 }
10367
10368 #[test]
10386 fn filtered_dump_is_exactly_the_unfiltered_dump_filtered() {
10387 use crate::index::PermSet;
10388 use crate::reader::SliceReader;
10389
10390 let terms = |i: u32| {
10391 (
10392 format!("<http://ex/s{}>", i % 11),
10393 format!("<http://ex/p{}>", i % 3),
10394 if i.is_multiple_of(4) {
10398 format!("\"lit {}\"@en", i % 5)
10399 } else {
10400 format!("<http://ex/o{}>", i % 7)
10401 },
10402 )
10403 };
10404 let build = |perms: PermSet| {
10405 let mut db = DictionaryBuilder::new();
10406 let all: Vec<_> = (0..90u32).map(terms).collect();
10407 for (s, p, o) in &all {
10408 db.observe(s, p, o);
10409 }
10410 let dict = db.build();
10411 let mut def = GraphIndexBuilder::new()
10412 .with_tile_budget(16)
10413 .with_perms(perms);
10414 let mut g1 = GraphIndexBuilder::new()
10415 .with_tile_budget(16)
10416 .with_perms(perms);
10417 for (i, (s, p, o)) in all.iter().enumerate() {
10418 let t = dict.encode(s, p, o).unwrap();
10419 if i % 2 == 0 {
10423 g1.push(t);
10424 }
10425 def.push(t);
10426 }
10427 write_dataset(
10428 &dict,
10429 &def.build(),
10430 &[("<http://ex/g1>".to_string(), g1.build())],
10431 true,
10432 &[],
10433 0,
10434 )
10435 };
10436
10437 let shapes: Vec<(Option<&str>, Option<&str>, Option<&str>)> = vec![
10438 (None, None, None),
10439 (Some("<http://ex/s3>"), None, None),
10440 (None, Some("<http://ex/p1>"), None),
10441 (None, None, Some("<http://ex/o5>")),
10442 (None, None, Some("\"lit 0\"@en")),
10443 (Some("<http://ex/s3>"), Some("<http://ex/p0>"), None),
10444 (Some("<http://ex/s4>"), None, Some("<http://ex/o4>")),
10445 (None, Some("<http://ex/p1>"), Some("<http://ex/o5>")),
10446 (
10447 Some("<http://ex/s3>"),
10448 Some("<http://ex/p0>"),
10449 Some("<http://ex/o3>"),
10450 ),
10451 (Some("<http://ex/nope>"), None, None),
10455 (None, Some("<http://ex/nope>"), None),
10456 (None, None, Some("<http://ex/nope>")),
10457 ];
10458
10459 let mut any_pruned = false;
10460 for perms in [PermSet::ALL, PermSet::CORE] {
10461 let image = build(perms);
10462 let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
10463 let opens: Vec<(&str, Rete)> = vec![
10464 ("resident", Rete::open(&image).unwrap()),
10465 (
10466 "ranged",
10467 Rete::open_ranged(&SliceReader::new(leaked)).unwrap(),
10468 ),
10469 (
10470 "lazy",
10471 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64)
10472 .unwrap(),
10473 ),
10474 ];
10475 for graph in [None, Some("<http://ex/g1>"), Some("<http://ex/absent>")] {
10476 let full: Vec<TermTriple> = {
10478 let mut v = Vec::new();
10479 Rete::open(&image).unwrap().dump_each(graph, |s, p, o| {
10480 v.push((s.to_string(), p.to_string(), o.to_string()))
10481 });
10482 v.sort();
10483 v
10484 };
10485 for &(s, p, o) in &shapes {
10486 let mut want = full.clone();
10487 want.retain(|(ts, tp, to)| {
10488 s.is_none_or(|x| x == ts)
10489 && p.is_none_or(|x| x == tp)
10490 && o.is_none_or(|x| x == to)
10491 });
10492 for (path, rete) in &opens {
10493 let mut got = Vec::new();
10494 rete.dump_filtered_each(graph, s, p, o, |s, p, o| {
10495 got.push((s.to_string(), p.to_string(), o.to_string()))
10496 });
10497 got.sort();
10498 assert_eq!(
10499 got,
10500 want,
10501 "{path}/{perms:?} graph={graph:?} pattern={:?} — filtered dump \
10502 is not the unfiltered dump filtered",
10503 (s, p, o)
10504 );
10505 assert!(!rete.index_incomplete(), "{path}: a fetch failed");
10506 }
10507 let plan = opens[2].1.dump_plan(graph, s, p, o);
10511 if graph == Some("<http://ex/absent>")
10512 || [s, p, o].contains(&Some("<http://ex/nope>"))
10513 {
10514 assert!(
10515 plan.scan.is_none(),
10516 "an unmatchable dump still planned a scan: {:?}",
10517 (graph, s, p, o)
10518 );
10519 } else {
10520 let scan = plan.scan.expect("a matchable dump plans a scan");
10521 assert!(
10522 PermSet::CORE.contains(scan.permutation),
10523 "a dump routed to {:?}, outside PermSet::CORE — a \
10524 --permutations 3 file has no such section",
10525 scan.permutation
10526 );
10527 assert!(scan.tiles_admitted <= scan.tiles_routed);
10528 assert!(scan.tiles_routed <= scan.tiles_total);
10529 assert!(scan.tile_bytes <= scan.section_bytes);
10530 if scan.tiles_admitted < scan.tiles_total {
10531 any_pruned = true;
10532 }
10533 }
10534 }
10535 }
10536 }
10537 assert!(
10538 any_pruned,
10539 "no shape pruned a single tile — the fixture is too small to be \
10540 measuring anything"
10541 );
10542 }
10543
10544 #[test]
10554 fn a_predicate_scoped_dump_fetches_less_than_the_graph() {
10555 use crate::reader::{CountingReader, SliceReader};
10556
10557 let mut db = DictionaryBuilder::new();
10563 let mut all = Vec::new();
10564 for s in 0..200u32 {
10565 for p in 0..20u32 {
10566 let t = (
10567 format!("<http://ex/s/{s:04}>"),
10568 format!("<http://ex/p/{p:02}>"),
10569 format!("\"object {s:04}/{p:02}\""),
10570 );
10571 db.observe(&t.0, &t.1, &t.2);
10572 all.push(t);
10573 }
10574 }
10575 let dict = db.build();
10576 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
10577 for (s, p, o) in &all {
10578 ib.push(dict.encode(s, p, o).unwrap());
10579 }
10580 let image = write_dataset(&dict, &ib.build(), &[], false, &[], 0);
10581 let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
10582
10583 let probe = |s: Option<&str>, p: Option<&str>, o: Option<&str>| -> (usize, u64) {
10585 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10586 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10587 let before = reader.bytes_read();
10588 let mut n = 0usize;
10589 lazy.dump_filtered_each(None, s, p, o, |_, _, _| n += 1);
10590 assert!(!lazy.index_incomplete());
10591 (n, reader.bytes_read() - before)
10592 };
10593
10594 let (full_rows, full_bytes) = probe(None, None, None);
10595 let (pred_rows, pred_bytes) = probe(None, Some("<http://ex/p/07>"), None);
10596 let (subj_rows, subj_bytes) = probe(Some("<http://ex/s/0021>"), None, None);
10597 assert_eq!(full_rows, 4000);
10598 assert_eq!(pred_rows, 200, "one predicate covers every subject once");
10599 assert_eq!(subj_rows, 20);
10600 assert!(
10602 pred_bytes < full_bytes && subj_bytes < full_bytes,
10603 "a slice fetched {pred_bytes} / {subj_bytes} B where the whole graph fetched \
10604 {full_bytes} B — the scan is not being pruned at all"
10605 );
10606 #[cfg(feature = "compression")]
10617 assert!(
10618 pred_bytes * 4 < full_bytes && subj_bytes * 4 < full_bytes,
10619 "a 1-in-20 predicate slice fetched {pred_bytes} B and one subject \
10620 {subj_bytes} B where the whole graph fetched {full_bytes} B"
10621 );
10622
10623 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10625 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10626 let before = reader.bytes_read();
10627 let plan = lazy
10628 .dump_plan(None, None, Some("<http://ex/p/07>"), None)
10629 .scan
10630 .expect("a known predicate plans a scan");
10631 assert!(
10632 reader.bytes_read() - before < 64 * 1024,
10633 "planning fetched {} B — it is supposed to read directories, not tiles",
10634 reader.bytes_read() - before
10635 );
10636 assert!(plan.tiles_admitted < plan.tiles_total);
10637 assert!(
10643 plan.tile_bytes * 5 < plan.section_bytes,
10644 "one predicate of twenty planned {} B of a {} B section",
10645 plan.tile_bytes,
10646 plan.section_bytes
10647 );
10648 }
10649
10650 #[test]
10651 fn query_quads_tags_every_graph() {
10652 let mut db = DictionaryBuilder::new();
10653 for (s, p, o) in [("Alice", "knows", "Bob"), ("Alice", "knows", "Dave")] {
10654 db.observe(s, p, o);
10655 }
10656 let dict = db.build();
10657 let mut def = GraphIndexBuilder::new();
10658 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10659 let mut g1 = GraphIndexBuilder::new();
10660 g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10661 let named = vec![("http://ex/g1".to_string(), g1.build())];
10662 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10663 let rete = Rete::open(&bytes).unwrap();
10664
10665 let quads = rete.query_quads(Some("Alice"), Some("knows"), None);
10667 assert_eq!(quads.len(), 2);
10668 assert_eq!(
10669 quads[0],
10670 (("Alice".into(), "knows".into(), "Bob".into()), None)
10671 );
10672 assert_eq!(
10673 quads[1],
10674 (
10675 ("Alice".into(), "knows".into(), "Dave".into()),
10676 Some("http://ex/g1".to_string())
10677 )
10678 );
10679
10680 assert!(rete.query_quads(Some("Nobody"), None, None).is_empty());
10682 }
10683
10684 #[test]
10685 fn pyramid_meta_round_trips_in_file() {
10686 let rete = Rete::open(&build_image()).unwrap();
10687 let pyr = rete.pyramid().expect("file has a pyramid");
10688 let total: u32 = pyr.summary.iter().map(|e| e.count).sum();
10690 assert_eq!(total, 3);
10691 assert!(!pyr.summary.is_empty());
10692 assert!(pyr.tiles.is_empty());
10693 }
10694
10695 #[test]
10696 fn schema_pyramid_round_trips_through_file_index_free() {
10697 use crate::reader::{CountingReader, SliceReader};
10698 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
10699 let q = |s: &str, p: &str, o: &str| {
10700 (s.to_string(), p.to_string(), o.to_string(), None::<String>)
10701 };
10702 let quads = vec![
10704 q("<a>", RDF_TYPE, "<Astronomer>"),
10705 q("<b>", RDF_TYPE, "<Astronomer>"),
10706 q("<c>", RDF_TYPE, "<Person>"),
10707 q("<Astronomer>", sub, "<Scientist>"),
10708 q("<Scientist>", sub, "<Person>"),
10709 q("<Person>", sub, "<Agent>"),
10710 q("<a>", "<knows>", "<b>"),
10711 q("<b>", "<knows>", "<c>"),
10712 ];
10713 let (bytes, _) =
10714 crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
10715
10716 let rete = Rete::open(&bytes).unwrap();
10718 let pyr = rete.pyramid().expect("pyramid present");
10719 assert!(!pyr.level_rollups.is_empty(), "schema pyramid shipped");
10720 assert!(pyr
10721 .class_hierarchy
10722 .iter()
10723 .any(|n| n.class == "<Agent>" && n.depth == 0));
10724
10725 let r = CountingReader::new(SliceReader::new(&bytes));
10727 let view = SummaryView::open_ranged(&r).unwrap().unwrap();
10728 assert!(view.level_count() >= 2, "multi-level pyramid");
10729 let coarse = view.level_rollup(0).unwrap();
10730 assert!(
10731 coarse.classes.iter().any(|(c, _)| c == "<Agent>"),
10732 "coarsest level rolls up to the root Agent"
10733 );
10734 let h = Header::from_bytes(&bytes).unwrap();
10735 assert!(
10736 r.bytes_read() <= bytes.len() as u64 - h.root_dir_len,
10737 "summary read {} bytes; the {}-byte index section must be skipped",
10738 r.bytes_read(),
10739 h.root_dir_len
10740 );
10741 }
10742
10743 #[test]
10744 fn predicate_totals_from_summary_only() {
10745 use crate::reader::SliceReader;
10746 let bytes = build_image();
10748 let reader = SliceReader::new(&bytes);
10749 let view = SummaryView::open_ranged(&reader).unwrap().unwrap();
10750 assert_eq!(view.predicate_total("knows"), 2);
10751 assert_eq!(view.predicate_total("age"), 1);
10752 assert_eq!(view.predicate_total("missing"), 0);
10753 let totals = view.predicate_totals();
10754 assert_eq!(totals[0], ("knows".to_string(), 2)); }
10756
10757 #[test]
10758 fn query_patterns_resolve_to_terms() {
10759 let rete = Rete::open(&build_image()).unwrap();
10760
10761 assert_eq!(rete.query(None, None, None).len(), 3);
10763
10764 let mut alice = rete.query(Some("Alice"), None, None);
10766 alice.sort();
10767 assert_eq!(
10768 alice,
10769 vec![
10770 ("Alice".into(), "age".into(), "30".into()),
10771 ("Alice".into(), "knows".into(), "Bob".into()),
10772 ]
10773 );
10774
10775 assert_eq!(rete.query(None, Some("knows"), None).len(), 2);
10777
10778 assert_eq!(
10780 rete.query(Some("Bob"), Some("knows"), Some("Carol")),
10781 vec![("Bob".into(), "knows".into(), "Carol".into())]
10782 );
10783 assert!(rete.query(Some("Nobody"), None, None).is_empty());
10784 assert!(rete.query(None, Some("likes"), None).is_empty());
10785 }
10786
10787 #[test]
10788 fn query_provenance_reports_terms_ids_sections_and_index_choice() {
10789 let bytes = build_image();
10790 let rete = Rete::open(&bytes).unwrap();
10791
10792 let mut matches = rete.query_with_provenance(None, Some("knows"), None);
10793 matches.sort_by(|a, b| a.terms.cmp(&b.terms));
10794
10795 assert_eq!(matches.len(), 2);
10796 assert_eq!(
10797 matches[0].terms,
10798 ("Alice".into(), "knows".into(), "Bob".into())
10799 );
10800 assert_eq!(
10801 matches[0].ids,
10802 rete.dictionary().encode("Alice", "knows", "Bob").unwrap()
10803 );
10804 assert_eq!(matches[0].graph.as_deref(), None);
10805 assert_eq!(
10806 matches[0].matched_pattern,
10807 (None, Some(matches[0].ids.1), None)
10808 );
10809 assert_eq!(
10810 matches[0].index_permutation,
10811 crate::index::IndexPermutation::Pos
10812 );
10813
10814 let h = rete.header();
10815 assert_eq!(matches[0].dictionary_range.offset, h.dictionary_offset);
10816 assert_eq!(matches[0].dictionary_range.len, h.dictionary_len);
10817 assert_eq!(matches[0].index_range.offset, h.root_dir_offset);
10818 assert_eq!(matches[0].index_range.len, h.root_dir_len);
10819 assert!(
10820 matches[0].index_section_range.offset > h.root_dir_offset,
10821 "POS is section 1, so its payload starts after the container header and SPO payload"
10822 );
10823 assert!(matches[0].index_section_range.len > 0);
10824 assert!(matches[0].index_section_range.end() <= matches[0].index_range.end());
10825 assert!(matches[0].index_section_range.len < matches[0].index_range.len);
10826 assert_eq!(
10827 matches[0].pyramid_range.as_ref().map(|r| (r.offset, r.len)),
10828 Some((h.pyramid_meta_offset, h.pyramid_meta_len))
10829 );
10830 let tile_range = matches[0].tile_range.expect("tiled file reports a tile");
10833 assert!(matches[0]
10834 .tile
10835 .as_deref()
10836 .unwrap()
10837 .starts_with(matches[0].index_permutation.name()));
10838 assert!(matches[0].index_section_range.offset <= tile_range.offset);
10839 assert!(tile_range.end() <= matches[0].index_section_range.end());
10840 }
10841
10842 fn build_labeled(n: usize) -> Vec<u8> {
10846 const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
10847 const WORDS: &[&str] = &[
10848 "alanine",
10849 "benzene",
10850 "glucose",
10851 "dextrose",
10852 "ethanol",
10853 "formate",
10854 "heptane",
10855 "isoleucine",
10856 ];
10857 let triples: Vec<(String, String, String)> = (0..n)
10858 .flat_map(|i| {
10859 let s = format!("<http://ex/e{i}>");
10860 let w = WORDS[i % WORDS.len()];
10861 [
10862 (s.clone(), LABEL.to_string(), format!("\"{w}-{i:06}\"")),
10863 (
10864 s,
10865 "<http://ex/p>".to_string(),
10866 format!("<http://ex/c{}>", i % 64),
10867 ),
10868 ]
10869 })
10870 .collect();
10871 let mut db = DictionaryBuilder::new();
10872 for (s, p, o) in &triples {
10873 db.observe(s, p, o);
10874 }
10875 let dict = db.build();
10876 let ids: Vec<(u32, u32, u32)> = triples
10877 .iter()
10878 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
10879 .collect();
10880 let mut ib = GraphIndexBuilder::new();
10881 for &t in &ids {
10882 ib.push(t);
10883 }
10884 let (meta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
10885 write_file(&dict, &ib.build(), false, &meta, levels)
10886 }
10887
10888 #[test]
10889 fn prefix_search_matches_a_filter_scan() {
10890 let bytes = build_labeled(800);
10893 let rete = Rete::open(&bytes).unwrap();
10894 let idx_subjects: std::collections::BTreeSet<String> = rete
10895 .prefix_search("glucose", 10_000)
10896 .into_iter()
10897 .map(|(_label, subject)| subject)
10898 .collect();
10899 let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
10901 FILTER(STRSTARTS(LCASE(?l), \"glucose\")) }";
10902 let crate::QueryOutput::Select(_, rows) = crate::eval_query(&rete, q).unwrap() else {
10903 panic!("expected SELECT");
10904 };
10905 let scan_subjects: std::collections::BTreeSet<String> =
10906 rows.iter().map(|r| r.get("s").cloned().unwrap()).collect();
10907 assert_eq!(idx_subjects, scan_subjects, "index agrees with the scan");
10908 assert_eq!(
10909 idx_subjects.len(),
10910 100,
10911 "800/8 words = 100 glucose-* labels"
10912 );
10913 }
10914
10915 #[test]
10919 fn legacy_label_hits_resolve_in_one_batched_read() {
10920 use std::sync::{Arc, Mutex};
10921 type CallLog = Arc<Mutex<Vec<Vec<(u64, u64)>>>>;
10922 #[derive(Clone)]
10923 struct Calls(Arc<Vec<u8>>, CallLog);
10924 impl crate::RangeReader for Calls {
10925 fn len(&self) -> u64 {
10926 self.0.len() as u64
10927 }
10928 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
10929 self.1.lock().unwrap().push(vec![(offset, len)]);
10930 crate::SliceReader::new(&self.0).read_at(offset, len)
10931 }
10932 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
10933 self.1.lock().unwrap().push(ranges.to_vec());
10934 let r = crate::SliceReader::new(&self.0);
10935 ranges.iter().map(|&(o, l)| r.read_at(o, l)).collect()
10936 }
10937 }
10938 let bytes = build_labeled(20_000);
10939 let h = Header::from_bytes(&bytes).unwrap();
10940 let (d0, dl) = (h.dictionary_offset, h.dictionary_len);
10941 let want = Rete::open(&bytes).unwrap().prefix_search("glucose", 1000);
10942 assert_eq!(want.len(), 1000);
10943 for search_view in [false, true] {
10944 let calls = Calls(Arc::new(bytes.clone()), Arc::new(Mutex::new(Vec::new())));
10945 let got = if search_view {
10946 let view = SearchView::open_ranged_with(calls.clone(), HEADER_LEN as u64).unwrap();
10947 calls.1.lock().unwrap().clear();
10948 view.prefix_search("glucose", 1000)
10949 } else {
10950 let rete = Rete::open_ranged_lazy_with(calls.clone(), HEADER_LEN as u64).unwrap();
10951 calls.1.lock().unwrap().clear();
10952 rete.prefix_search("glucose", 1000)
10953 };
10954 assert_eq!(got, want, "search view = {search_view}");
10955 let dict_reads = calls
10956 .1
10957 .lock()
10958 .unwrap()
10959 .iter()
10960 .filter(|c| c.iter().any(|&(o, l)| o < d0 + dl && d0 < o + l))
10961 .count();
10962 assert_eq!(dict_reads, 1, "search view = {search_view}");
10963 }
10964 }
10965
10966 #[test]
10970 #[ignore]
10971 fn bench_prefix_search_vs_filter_scan() {
10972 use std::time::Instant;
10973 let n = 6000; let bytes = build_labeled(n);
10975 let rete = Rete::open(&bytes).unwrap();
10976 let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
10977 FILTER(STRSTARTS(LCASE(?l), \"glucose\")) }";
10978 let reps = 200;
10979 let idx_n = rete.prefix_search("glucose", 100_000).len();
10980 let t = Instant::now();
10981 for _ in 0..reps {
10982 std::hint::black_box(rete.prefix_search("glucose", 100_000));
10983 }
10984 let idx_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
10985 let t = Instant::now();
10986 for _ in 0..reps {
10987 let _ = std::hint::black_box(crate::eval_query(&rete, q).unwrap());
10988 }
10989 let scan_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
10990 println!(
10991 "label prefix search over {n} labeled subjects ({idx_n} matches): \
10992 index {idx_ms:.4} ms vs FILTER scan {scan_ms:.3} ms ({:.0}× faster)",
10993 scan_ms / idx_ms
10994 );
10995 }
10996
10997 #[test]
11001 #[ignore]
11002 fn bench_text_search_vs_contains_scan() {
11003 use std::time::Instant;
11004 const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
11005 const WORDS: &[&str] = &[
11006 "alanine",
11007 "benzene",
11008 "glucose",
11009 "dextrose",
11010 "ethanol",
11011 "formate",
11012 "heptane",
11013 "isoleucine",
11014 ];
11015 let n = 6000;
11016 let triples: Vec<(String, String, String)> = (0..n)
11017 .map(|i| {
11018 (
11019 format!("<http://ex/e{i}>"),
11020 LABEL.to_string(),
11021 format!("\"{} sample number {i:06}\"", WORDS[i % WORDS.len()]),
11022 )
11023 })
11024 .collect();
11025 let mut db = DictionaryBuilder::new();
11026 for (s, p, o) in &triples {
11027 db.observe(s, p, o);
11028 }
11029 let dict = db.build();
11030 let ids: Vec<(u32, u32, u32)> = triples
11031 .iter()
11032 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11033 .collect();
11034 let mut ib = GraphIndexBuilder::new();
11035 for &t in &ids {
11036 ib.push(t);
11037 }
11038 let ti = compute_text_index(&dict, &ids);
11039 let bytes = write_dataset_with_metadata(&dict, &ib.build(), &[], false, &[], 0, &[], &ti);
11040 let rete = Rete::open(&bytes).unwrap();
11041
11042 let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11043 FILTER(CONTAINS(LCASE(?l), \"glucose\")) }";
11044 let reps = 200;
11045 let idx_n = rete.text_search(&["glucose"], None, 100_000).len();
11046 let t = Instant::now();
11047 for _ in 0..reps {
11048 std::hint::black_box(rete.text_search(&["glucose"], None, 100_000));
11049 }
11050 let idx_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11051 let t = Instant::now();
11052 for _ in 0..reps {
11053 let _ = std::hint::black_box(crate::eval_query(&rete, q).unwrap());
11054 }
11055 let scan_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11056 println!(
11057 "text search over {n} literals ({idx_n} matches): \
11058 index {idx_ms:.4} ms vs FILTER(CONTAINS) scan {scan_ms:.3} ms ({:.0}× faster)",
11059 scan_ms / idx_ms
11060 );
11061 }
11062
11063 #[test]
11067 #[ignore = "operational tool, driven by RETE_DEBUG_* env vars"]
11068 fn debug_bound_po_routing() {
11069 struct FR(std::fs::File);
11070 impl crate::RangeReader for FR {
11071 fn len(&self) -> u64 {
11072 self.0.metadata().map(|m| m.len()).unwrap_or(0)
11073 }
11074 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
11075 use std::os::unix::fs::FileExt;
11076 let mut buf = vec![0u8; len as usize];
11077 self.0.read_exact_at(&mut buf, offset)?;
11078 Ok(buf)
11079 }
11080 }
11081 let path = std::env::var("RETE_DEBUG_FILE").expect("RETE_DEBUG_FILE");
11082 let p_iri = std::env::var("RETE_DEBUG_P").expect("RETE_DEBUG_P");
11083 let o_iri = std::env::var("RETE_DEBUG_O").expect("RETE_DEBUG_O");
11084 let rete = Rete::open_ranged_lazy_with(
11085 std::sync::Arc::new(FR(std::fs::File::open(&path).unwrap())),
11086 HEADER_LEN as u64,
11087 )
11088 .unwrap();
11089 let pid = rete.dict.predicate_id(&p_iri).expect("p resolves");
11090 let oid = rete.dict.object_id(&o_iri).expect("o resolves");
11091 eprintln!("pid={pid} oid={oid}");
11092 let pattern = (None, Some(pid), Some(oid));
11093 let perm = GraphIndex::best_permutation(pattern);
11094 eprintln!("best_permutation = {}", perm.name());
11095 let si = perm.section_index();
11096 let tiles = &rete.index.sections[si];
11097 eprintln!("section {} tiles = {}", perm.name(), tiles.len());
11098 let [pa, pb, pc] = perm.order_pattern(pattern);
11099 eprintln!("permuted pattern pa={pa:?} pb={pb:?} pc={pc:?}");
11100 let (start, end) = rete.index.tile_span(si, pa);
11101 eprintln!("tile_span = [{start}, {end}) -> {} tiles", end - start);
11102 let mut admitted = 0usize;
11103 for (ti, t) in tiles.iter().enumerate().take(end).skip(start) {
11104 if t.syn_admits(pb, pc) {
11105 admitted += 1;
11106 if admitted <= 10 {
11107 let (lo, hi) = t.leading_range();
11108 eprintln!(" admit tile {ti}: a=[{lo},{hi}] syn={:?}", t.syn);
11109 }
11110 }
11111 }
11112 eprintln!("admitted {admitted} tile(s) by synopsis");
11113 let n = rete.index.scan_iter(pattern).count();
11114 eprintln!("scan_iter matches = {n}");
11115 let hi_res = rete.query(None, Some(&p_iri), Some(&o_iri));
11116 eprintln!("high-level query matches = {}", hi_res.len());
11117 }
11118
11119 #[test]
11127 fn layout_header_label_matches_its_span() {
11128 let mut db = DictionaryBuilder::new();
11129 let mut triples = Vec::new();
11130 for i in 0..24u32 {
11131 let (s, p, o) = (
11132 format!("<http://ex/s{}>", i % 5),
11133 format!("<http://ex/p{}>", i % 3),
11134 format!("<http://ex/o{i}>"),
11135 );
11136 db.observe(&s, &p, &o);
11137 triples.push((s, p, o));
11138 }
11139 let dict = db.build();
11140 let ids: Vec<(u32, u32, u32)> = triples
11141 .iter()
11142 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11143 .collect();
11144 let index = GraphIndexBuilder::from_triples(ids).build();
11145 let image = write_dataset_with_metadata(
11146 &dict,
11147 &index,
11148 &[],
11149 false,
11150 &[],
11151 0,
11152 br#"{"name":"layout-label-fixture"}"#,
11153 &[],
11154 );
11155 let rete = Rete::open(&image).unwrap();
11156
11157 let layout = rete.file_layout();
11158 let header = layout
11159 .iter()
11160 .find(|s| s.kind == "header")
11161 .expect("every layout starts with a header segment");
11162
11163 assert_eq!(
11164 header.offset, 0,
11165 "the header is the first thing in the file"
11166 );
11167 assert_eq!(
11168 header.len, HEADER_LEN as u64,
11169 "the header segment must span exactly HEADER_LEN"
11170 );
11171
11172 let digits: String = header
11175 .label
11176 .chars()
11177 .skip_while(|c| !c.is_ascii_digit())
11178 .take_while(|c| c.is_ascii_digit())
11179 .collect();
11180 assert!(
11181 !digits.is_empty(),
11182 "header label {:?} states no size at all",
11183 header.label
11184 );
11185 assert_eq!(
11186 digits.parse::<u64>().unwrap(),
11187 header.len,
11188 "header label {:?} disagrees with the {}-byte span it describes",
11189 header.label,
11190 header.len
11191 );
11192
11193 assert!(
11196 layout.windows(2).all(|w| w[0].offset <= w[1].offset),
11197 "file_layout must return segments sorted by offset"
11198 );
11199 }
11200}