1use crate::dictionary::Dictionary;
18use crate::header::{
19 Header, Section, SectionKind, FLAG_HAS_QUADS, FLAG_HAS_QUOTED_TRIPLES, FLAG_TILE_SYNOPSIS,
20 HEADER_LEN, MAGIC,
21};
22use crate::index::{GraphIndex, IndexPermutation, Pattern, PermSet, NUM_PERMS};
23use crate::meta::{ClassNode, CommunityDescriptor, LevelLinks, LevelRollup, PyramidMeta};
24use crate::pyramid::{build_dendrogram, project_graph, PyramidAlgo};
25use crate::reader::RangeReader;
26use crate::tiling::{choose_round_for_budget, summarize, SuperEdge};
27use crate::triples::Triple;
28use crate::varint::{read_uvarint, write_uvarint};
29
30#[cfg(test)]
31mod hardening_tests;
32
33pub const DEFAULT_TILE_BUDGET: usize = 64 * 1024;
35
36pub fn build_pyramid_meta(
45 dict: &Dictionary,
46 triples: &[(u32, u32, u32)],
47 budget: usize,
48) -> (Vec<u8>, u16) {
49 build_pyramid_meta_with(dict, triples, budget, None)
50}
51
52pub fn build_pyramid_meta_with(
56 dict: &Dictionary,
57 triples: &[(u32, u32, u32)],
58 budget: usize,
59 type_override: Option<&str>,
60) -> (Vec<u8>, u16) {
61 build_pyramid_meta_algo(dict, triples, budget, type_override, PyramidAlgo::Louvain)
62}
63
64pub fn build_pyramid_meta_algo(
71 dict: &Dictionary,
72 triples: &[(u32, u32, u32)],
73 budget: usize,
74 type_override: Option<&str>,
75 algo: PyramidAlgo,
76) -> (Vec<u8>, u16) {
77 build_pyramid_meta_layout(
78 dict,
79 triples,
80 budget,
81 type_override,
82 algo,
83 PyramidLayout::Combined,
84 )
85}
86
87#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
89pub enum PyramidLayout {
90 #[default]
94 Combined,
95 CommunityOnly,
100}
101
102pub fn build_pyramid_meta_layout(
104 dict: &Dictionary,
105 triples: &[(u32, u32, u32)],
106 budget: usize,
107 type_override: Option<&str>,
108 algo: PyramidAlgo,
109 layout: PyramidLayout,
110) -> (Vec<u8>, u16) {
111 let timing = std::env::var_os("RETE_BUILD_TIMING").is_some();
117 let mut t = timing.then(std::time::Instant::now);
118 let mut lap = |label: &str| {
119 if let Some(t0) = &mut t {
120 eprintln!(
121 " [pyramid] {label}: {:.0} ms",
122 t0.elapsed().as_secs_f64() * 1000.0
123 );
124 *t0 = std::time::Instant::now();
125 }
126 };
127
128 let louvain = |lap: &mut dyn FnMut(&str)| {
130 let g = project_graph(dict, triples);
131 lap("project_graph");
132 let d = build_dendrogram(&g);
133 lap("build_dendrogram (Louvain)");
134 d
135 };
136 let dend = match algo {
137 PyramidAlgo::Louvain => louvain(&mut lap),
138 PyramidAlgo::Types => {
139 match crate::schema_pyramid::build_type_dendrogram(dict, triples, type_override) {
140 Some(d) => {
141 lap("build_type_dendrogram");
142 d
143 }
144 None => {
145 eprintln!(
146 " [pyramid] --pyramid-algo types: no usable rdf:type \
147 predicate — falling back to louvain"
148 );
149 louvain(&mut lap)
150 }
151 }
152 }
153 };
154 let round = choose_round_for_budget(dict, triples, &dend, budget);
155 lap("choose_round_for_budget");
156 let summary = summarize(dict, triples, &dend, round);
157 lap("summarize");
158 if layout == PyramidLayout::CommunityOnly {
163 let descriptors = crate::schema_pyramid::build_community_descriptors(
166 dict,
167 triples,
168 &dend,
169 round,
170 type_override,
171 );
172 lap("build_community_descriptors");
173 let meta = PyramidMeta::new(round as u32, summary, &[]).with_schema(
174 Vec::new(),
175 Vec::new(),
176 Vec::new(),
177 descriptors,
178 Vec::new(),
179 Vec::new(),
180 Vec::new(),
181 );
182 let out = (meta.encode(), dend.rounds() as u16);
183 lap("encode");
184 return out;
185 }
186 let sp = crate::schema_pyramid::build_schema_pyramid_with(
187 dict,
188 triples,
189 &dend,
190 round,
191 type_override,
192 );
193 lap("build_schema_pyramid");
194 let predicate_stats = compute_predicate_stats(triples);
195 lap("compute_predicate_stats");
196 let char_sets = compute_char_sets(triples);
197 lap("compute_char_sets");
198 let label_index = compute_label_index(dict, triples);
199 lap("compute_label_index");
200 let meta = PyramidMeta::new(round as u32, summary, &[])
201 .with_schema(
202 sp.class_hierarchy,
203 sp.level_rollups,
204 sp.level_links,
205 sp.descriptors,
206 sp.subclass_cycles,
207 sp.disjoint_pairs,
208 sp.equivalent_pairs,
209 )
210 .with_predicate_stats(predicate_stats)
211 .with_char_sets(char_sets)
212 .with_label_index(label_index);
213 let out = (meta.encode(), dend.rounds() as u16);
214 lap("encode");
215 out
216}
217
218const LABEL_PREDICATES: &[&str] = crate::label_index::DIRECT_LABEL_PREDICATES;
223
224fn compute_label_index(
232 dict: &Dictionary,
233 triples: &[(u32, u32, u32)],
234) -> Vec<crate::meta::LabelEntry> {
235 use crate::terms::{is_literal, literal_lexical};
236 use std::collections::{HashMap, HashSet};
237 const MAX_LABELS: usize = 8192;
238
239 let label_pids: HashSet<u32> = LABEL_PREDICATES
241 .iter()
242 .filter_map(|p| dict.predicate_id(p))
243 .collect();
244 if label_pids.is_empty() {
245 return Vec::new();
246 }
247 let mut degree: HashMap<u32, u32> = HashMap::new();
249 for &(s, _p, _o) in triples {
250 *degree.entry(s).or_insert(0) += 1;
251 }
252 let mut seen: HashSet<(u32, String)> = HashSet::new();
254 let mut candidates: Vec<(u32, String, u32)> = Vec::new(); for &(s, p, o) in triples {
256 if !label_pids.contains(&p) {
257 continue;
258 }
259 let Some(term) = dict.object_term(o) else {
260 continue;
261 };
262 if !is_literal(&term) {
263 continue;
264 }
265 let Some(label) = literal_lexical(&term) else {
266 continue;
267 };
268 if label.is_empty() {
269 continue;
270 }
271 if seen.insert((s, label.to_lowercase())) {
272 candidates.push((*degree.get(&s).unwrap_or(&0), label, s));
273 }
274 }
275 if candidates.len() > MAX_LABELS {
278 candidates.sort_by(|a, b| {
279 b.0.cmp(&a.0)
280 .then_with(|| a.2.cmp(&b.2))
281 .then_with(|| a.1.cmp(&b.1))
282 });
283 candidates.truncate(MAX_LABELS);
284 }
285 candidates.sort_by(|a, b| {
287 a.1.to_lowercase()
288 .cmp(&b.1.to_lowercase())
289 .then_with(|| a.1.cmp(&b.1))
290 .then_with(|| a.2.cmp(&b.2))
291 });
292 candidates
293 .into_iter()
294 .map(|(_deg, label, subject)| crate::meta::LabelEntry { label, subject })
295 .collect()
296}
297
298pub fn compute_text_index(dict: &Dictionary, triples: &[(u32, u32, u32)]) -> Vec<u8> {
307 crate::text_index::build_section(
308 dict,
309 &crate::text_index::TextIndexScope::default(),
310 triples,
311 std::iter::empty(),
312 writer_codec(),
313 )
314}
315
316fn compute_char_sets(triples: &[(u32, u32, u32)]) -> Vec<crate::meta::CharSet> {
321 use std::collections::{BTreeSet, HashMap};
322 const MAX_CHAR_SETS: usize = 128;
323 let mut by_subject: HashMap<u32, BTreeSet<u32>> = HashMap::new();
324 for &(s, p, _o) in triples {
325 by_subject.entry(s).or_default().insert(p);
326 }
327 let mut shapes: HashMap<Vec<u32>, u64> = HashMap::new();
328 for set in by_subject.into_values() {
329 *shapes.entry(set.into_iter().collect()).or_insert(0) += 1;
330 }
331 let mut v: Vec<crate::meta::CharSet> = shapes
332 .into_iter()
333 .map(|(predicates, subjects)| crate::meta::CharSet {
334 predicates,
335 subjects,
336 })
337 .collect();
338 v.sort_by(|a, b| {
339 b.subjects
340 .cmp(&a.subjects)
341 .then_with(|| a.predicates.cmp(&b.predicates))
342 });
343 v.truncate(MAX_CHAR_SETS);
344 v
345}
346
347fn compute_predicate_stats(triples: &[(u32, u32, u32)]) -> Vec<crate::meta::PredStat> {
353 use std::collections::HashMap;
354 #[allow(clippy::type_complexity)]
355 let mut acc: HashMap<u32, (HashMap<u32, u32>, HashMap<u32, u32>, u64)> = HashMap::new();
356 for &(s, p, o) in triples {
357 let e = acc.entry(p).or_default();
358 *e.0.entry(s).or_insert(0) += 1;
359 *e.1.entry(o).or_insert(0) += 1;
360 e.2 += 1;
361 }
362 let mut stats: Vec<crate::meta::PredStat> = acc
363 .into_iter()
364 .map(|(predicate, (subj, obj, count))| crate::meta::PredStat {
365 predicate,
366 count,
367 distinct_subjects: subj.len() as u64,
368 distinct_objects: obj.len() as u64,
369 max_objects_per_subject: subj.values().copied().max().unwrap_or(0),
370 max_subjects_per_object: obj.values().copied().max().unwrap_or(0),
371 })
372 .collect();
373 stats.sort_by_key(|p| p.predicate);
374 stats
375}
376
377pub const CODEC_NONE: u8 = 0;
379pub const CODEC_ZSTD: u8 = 1;
381
382#[derive(Debug, thiserror::Error)]
383#[non_exhaustive]
384pub enum FileError {
385 #[error("header: {0}")]
386 Header(#[from] crate::header::HeaderError),
387 #[error("malformed container: {0}")]
388 Container(&'static str),
389 #[error("unknown codec: {0}")]
390 UnknownCodec(u8),
391 #[error("decompression failed: {0}")]
392 Decompress(std::io::Error),
393 #[error("io: {0}")]
394 Io(#[from] std::io::Error),
395}
396
397pub fn writer_codec() -> u8 {
400 if cfg!(feature = "compression") {
401 CODEC_ZSTD
402 } else {
403 CODEC_NONE
404 }
405}
406
407fn intersect_sorted(a: &[u32], b: &[u32]) -> Vec<u32> {
410 let mut out = Vec::with_capacity(a.len().min(b.len()));
411 let (mut i, mut j) = (0, 0);
412 while i < a.len() && j < b.len() {
413 match a[i].cmp(&b[j]) {
414 std::cmp::Ordering::Less => i += 1,
415 std::cmp::Ordering::Greater => j += 1,
416 std::cmp::Ordering::Equal => {
417 out.push(a[i]);
418 i += 1;
419 j += 1;
420 }
421 }
422 }
423 out
424}
425
426pub(crate) fn compress(codec: u8, bytes: &[u8]) -> Vec<u8> {
427 match codec {
428 #[cfg(feature = "compression")]
431 CODEC_ZSTD => {
432 let level = crate::write_opts::zstd_level();
433 match crate::write_opts::frame_shape() {
434 crate::write_opts::FrameShape::Stream => zstd::encode_all(bytes, level),
435 crate::write_opts::FrameShape::Sized => zstd::bulk::compress(bytes, level),
436 }
437 .expect("zstd encode is infallible in-memory")
438 }
439 _ => bytes.to_vec(),
440 }
441}
442
443#[cfg(not(target_arch = "wasm32"))]
461pub(crate) fn compress_to(
462 codec: u8,
463 src_len: u64,
464 src: &mut dyn std::io::Read,
465 dst: &mut dyn std::io::Write,
466) -> std::io::Result<()> {
467 match codec {
468 #[cfg(feature = "compression")]
469 CODEC_ZSTD => zstd_stream(src_len, src, dst),
470 _ => {
471 let _ = src_len;
472 std::io::copy(src, dst).map(|_| ())
473 }
474 }
475}
476
477pub(crate) fn compress_streamed(codec: u8, bytes: &[u8]) -> Vec<u8> {
483 match codec {
484 #[cfg(feature = "compression")]
485 CODEC_ZSTD => {
486 let mut out = Vec::with_capacity(bytes.len() / 4 + 64);
487 zstd_stream(bytes.len() as u64, &mut &bytes[..], &mut out)
488 .expect("zstd encode is infallible in-memory");
489 out
490 }
491 _ => bytes.to_vec(),
492 }
493}
494
495#[cfg(feature = "compression")]
497fn zstd_stream(
498 src_len: u64,
499 src: &mut dyn std::io::Read,
500 dst: &mut dyn std::io::Write,
501) -> std::io::Result<()> {
502 let mut enc = zstd::stream::Encoder::new(dst, crate::write_opts::zstd_level())?;
503 if crate::write_opts::frame_shape() == crate::write_opts::FrameShape::Sized {
504 enc.set_pledged_src_size(Some(src_len))?;
505 enc.include_contentsize(true)?;
506 }
507 let copied = std::io::copy(src, &mut enc)?;
508 if copied != src_len {
509 return Err(std::io::Error::other(format!(
510 "zstd stream: {copied} bytes read, {src_len} announced"
511 )));
512 }
513 enc.finish()?;
514 Ok(())
515}
516
517pub(crate) fn decompress(codec: u8, bytes: &[u8]) -> Result<Vec<u8>, FileError> {
518 match codec {
519 CODEC_NONE => Ok(bytes.to_vec()),
520 #[cfg(feature = "compression")]
529 CODEC_ZSTD => zstd::decode_all(bytes).map_err(FileError::Decompress),
530 #[cfg(not(feature = "compression"))]
535 CODEC_ZSTD => {
536 use std::io::Read;
537 let mut dec = ruzstd::decoding::StreamingDecoder::new(bytes)
538 .map_err(|e| FileError::Decompress(std::io::Error::other(e.to_string())))?;
539 let mut out = Vec::new();
540 dec.read_to_end(&mut out).map_err(FileError::Decompress)?;
541 Ok(out)
542 }
543 other => Err(FileError::UnknownCodec(other)),
544 }
545}
546
547pub(crate) fn zstd_frame_content_size(b: &[u8]) -> Option<u64> {
551 if b.len() < 5 || b[..4] != [0x28, 0xB5, 0x2F, 0xFD] {
552 return None;
553 }
554 let fhd = b[4];
555 let fcs_flag = fhd >> 6;
556 let single_segment = fhd & 0x20 != 0;
557 let did_size = [0usize, 1, 2, 4][(fhd & 0x03) as usize];
558 let mut pos = 5 + usize::from(!single_segment) + did_size;
559 let fcs_size = match (fcs_flag, single_segment) {
560 (0, false) => return None,
561 (0, true) => 1,
562 (1, _) => 2,
563 (2, _) => 4,
564 _ => 8,
565 };
566 let f = b.get(pos..pos + fcs_size)?;
567 pos += fcs_size;
568 let _ = pos;
569 let mut v = 0u64;
570 for (i, x) in f.iter().enumerate() {
571 v |= (*x as u64) << (8 * i);
572 }
573 Some(if fcs_size == 2 { v + 256 } else { v })
574}
575
576pub(crate) fn decompress_bounded(codec: u8, bytes: &[u8], max: u64) -> Result<Vec<u8>, FileError> {
582 let too_big = |n: u64| {
583 FileError::Decompress(std::io::Error::other(format!(
584 "a block of {n} bytes exceeds its section's {max}-byte maximum"
585 )))
586 };
587 match codec {
588 CODEC_NONE => {
589 if bytes.len() as u64 > max {
590 return Err(too_big(bytes.len() as u64));
591 }
592 Ok(bytes.to_vec())
593 }
594 CODEC_ZSTD => {
595 let declared = zstd_frame_content_size(bytes);
596 if let Some(n) = declared.filter(|&n| n > max) {
597 return Err(too_big(n));
598 }
599 #[cfg(feature = "compression")]
600 {
601 if let Some(n) = declared {
602 return zstd::bulk::decompress(bytes, n as usize)
605 .map_err(FileError::Decompress);
606 }
607 use std::io::Read;
608 let mut out = Vec::new();
609 let mut dec = zstd::stream::read::Decoder::with_buffer(bytes)
610 .map_err(FileError::Decompress)?;
611 dec.window_log_max(27).map_err(FileError::Decompress)?;
612 dec.take(max.saturating_add(1))
613 .read_to_end(&mut out)
614 .map_err(FileError::Decompress)?;
615 if out.len() as u64 > max {
616 return Err(too_big(out.len() as u64));
617 }
618 Ok(out)
619 }
620 #[cfg(not(feature = "compression"))]
621 {
622 use std::io::Read;
623 let dec = ruzstd::decoding::StreamingDecoder::new(bytes)
624 .map_err(|e| FileError::Decompress(std::io::Error::other(e.to_string())))?;
625 let mut out = Vec::new();
626 dec.take(max.saturating_add(1))
627 .read_to_end(&mut out)
628 .map_err(FileError::Decompress)?;
629 if out.len() as u64 > max {
630 return Err(too_big(out.len() as u64));
631 }
632 Ok(out)
633 }
634 }
635 other => Err(FileError::UnknownCodec(other)),
636 }
637}
638
639pub(crate) fn block_raw_len(codec: u8, bytes: &[u8], ceiling: u64) -> Result<u64, FileError> {
643 match codec {
644 CODEC_NONE => Ok(bytes.len() as u64),
645 CODEC_ZSTD => match zstd_frame_content_size(bytes) {
646 Some(n) => Ok(n),
647 None => Ok(decompress_bounded(codec, bytes, ceiling)?.len() as u64),
648 },
649 other => Err(FileError::UnknownCodec(other)),
650 }
651}
652
653const TILE_COALESCE_GAP: u64 = 4096;
657
658const DICT_COALESCE_GAP: u64 = 64 * 1024;
664
665pub const DEFAULT_EXPORT_BUDGET_MB: u64 = 4096;
672
673pub const DEFAULT_QUERY_BUDGET_MB: u64 = DEFAULT_EXPORT_BUDGET_MB;
685
686pub fn spill_threshold_for(budget: u64) -> u64 {
690 budget / 8
691}
692
693const MIN_DICT_CACHE_CAP: u64 = 64 << 20; const TRANSIENT_RESERVE_MAX: u64 = 128 << 20; #[derive(Debug, Clone, Copy)]
714pub struct MemoryBudget {
715 pub block_cache: u64,
717 pub dict_cache: u64,
719 pub tile_cache: u64,
721}
722
723pub fn split_memory_budget(total: u64) -> MemoryBudget {
734 if total == u64::MAX {
735 return MemoryBudget {
736 block_cache: u64::MAX,
737 dict_cache: u64::MAX,
738 tile_cache: u64::MAX,
739 };
740 }
741 let block_cache = (total / 8).min(crate::block_cache::DEFAULT_CACHE_CAP);
744 let tile_cache = (total / 16).min(256 << 20);
747 let reserve = (total / 32).min(TRANSIENT_RESERVE_MAX);
753 let dict_cache = total
756 .saturating_sub(block_cache)
757 .saturating_sub(tile_cache)
758 .saturating_sub(reserve)
759 .max(MIN_DICT_CACHE_CAP);
760 MemoryBudget {
761 block_cache,
762 dict_cache,
763 tile_cache,
764 }
765}
766
767#[derive(Debug, Clone, Copy, Default)]
770pub struct FixedOverhead {
771 pub dict_directory: u64,
773 pub index_directory: u64,
775 pub named_graphs: u64,
777}
778
779impl FixedOverhead {
780 pub fn total(&self) -> u64 {
782 self.dict_directory + self.index_directory + self.named_graphs
783 }
784}
785
786fn fmt_cap(cap: u64) -> String {
788 if cap == u64::MAX {
789 "unlimited".to_string()
790 } else {
791 format!("{}MiB", cap >> 20)
792 }
793}
794
795pub(crate) fn read_coalesced<R: RangeReader + ?Sized>(
801 reader: &R,
802 ranges: &[ByteRange],
803 gap: u64,
804) -> Option<Vec<Vec<u8>>> {
805 coalesced(reader, ranges, gap, false)
806}
807
808pub(crate) fn read_coalesced_scattered<R: RangeReader + ?Sized>(
812 reader: &R,
813 ranges: &[ByteRange],
814 gap: u64,
815) -> Option<Vec<Vec<u8>>> {
816 coalesced(reader, ranges, gap, true)
817}
818
819fn coalesced<R: RangeReader + ?Sized>(
820 reader: &R,
821 ranges: &[ByteRange],
822 gap: u64,
823 scattered: bool,
824) -> Option<Vec<Vec<u8>>> {
825 let mut spans: Vec<(u64, u64)> = Vec::new();
828 let mut span_of: Vec<usize> = Vec::with_capacity(ranges.len());
829 let mut i = 0;
830 while i < ranges.len() {
831 let start = ranges[i].offset;
832 let mut end = ranges[i].offset.checked_add(ranges[i].len)?;
833 let mut j = i + 1;
834 while j < ranges.len() {
835 let r = &ranges[j];
836 if r.offset < end || r.offset - end > gap {
837 break;
838 }
839 end = r.offset.checked_add(r.len)?;
840 j += 1;
841 }
842 let si = spans.len();
843 spans.push((start, end - start));
844 for _ in i..j {
845 span_of.push(si);
846 }
847 i = j;
848 }
849 let blobs = if scattered {
850 reader.read_many_scattered(&spans)
851 } else {
852 reader.read_many(&spans)
853 }
854 .ok()?;
855 if blobs.len() != spans.len() {
856 return None;
857 }
858 let mut out = Vec::with_capacity(ranges.len());
859 for (k, r) in ranges.iter().enumerate() {
860 let (span_start, _) = spans[span_of[k]];
861 let blob = &blobs[span_of[k]];
862 out.push(checked_slice(blob, r.offset - span_start, r.len)?.to_vec());
863 }
864 Some(out)
865}
866
867fn content_hash(parts: &[&[u8]]) -> [u8; 16] {
870 let mut h = blake3::Hasher::new();
871 for p in parts {
872 h.update(p);
873 }
874 let mut out = [0u8; 16];
875 out.copy_from_slice(&h.finalize().as_bytes()[..16]);
876 out
877}
878
879pub type TermTriple = (String, String, String);
881
882#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct DumpPlan {
886 pub scan: Option<crate::index::ScanPlan>,
890 pub dictionary_bytes: u64,
899}
900
901#[derive(Debug, Clone)]
905pub struct LayoutSegment {
906 pub kind: &'static str,
907 pub label: String,
908 pub offset: u64,
909 pub len: u64,
910}
911
912#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
914pub struct ByteRange {
915 pub offset: u64,
916 pub len: u64,
917}
918
919impl ByteRange {
920 pub fn end(self) -> u64 {
921 self.offset + self.len
922 }
923}
924
925#[derive(Debug, Clone, PartialEq, Eq)]
927pub struct TripleProvenance {
928 pub terms: TermTriple,
930 pub ids: Triple,
932 pub graph: Option<String>,
934 pub matched_pattern: Pattern,
936 pub index_permutation: IndexPermutation,
938 pub dictionary_range: ByteRange,
940 pub index_range: ByteRange,
942 pub index_section_range: ByteRange,
945 pub pyramid_range: Option<ByteRange>,
947 pub tile: Option<String>,
951 pub tile_range: Option<ByteRange>,
954}
955
956fn encode_container(sections: &[&[u8]], codec: u8) -> Vec<u8> {
960 let mut out = Vec::new();
961 write_uvarint(&mut out, sections.len() as u64);
962 for s in sections {
963 let payload = compress(codec, s);
964 write_uvarint(&mut out, payload.len() as u64);
965 out.extend_from_slice(&payload);
966 }
967 out
968}
969
970fn decode_container(bytes: &[u8], codec: u8) -> Result<Vec<Vec<u8>>, FileError> {
972 let (n, mut pos) = read_uvarint(bytes).ok_or(FileError::Container("truncated count"))?;
973 let n = wire_usize(n, "section count too large")?;
977 let mut out = Vec::with_capacity(n.min(bytes.len()));
978 for _ in 0..n {
979 let (len, used) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
980 .ok_or(FileError::Container("truncated length"))?;
981 pos += used;
982 let end = pos
985 .checked_add(wire_usize(len, "section length too large")?)
986 .filter(|&end| end <= bytes.len())
987 .ok_or(FileError::Container("section overruns buffer"))?;
988 out.push(decompress(codec, &bytes[pos..end])?);
989 pos = end;
990 }
991 Ok(out)
992}
993
994fn checked_end(off: u64, len: u64) -> Result<u64, FileError> {
995 off.checked_add(len)
996 .ok_or(FileError::Container("section range overflows"))
997}
998
999fn wire_usize(v: u64, what: &'static str) -> Result<usize, FileError> {
1003 usize::try_from(v).map_err(|_| FileError::Container(what))
1004}
1005
1006fn checked_slice(bytes: &[u8], off: u64, len: u64) -> Option<&[u8]> {
1009 let start = usize::try_from(off).ok()?;
1010 let end = start.checked_add(usize::try_from(len).ok()?)?;
1011 bytes.get(start..end)
1012}
1013
1014const DICT_CHUNK_BUDGET: usize = crate::write_opts::DEFAULT_DICT_CHUNK_BYTES;
1017
1018#[cfg(test)]
1035fn encode_chunked_dict_section(raw: &[u8], codec: u8) -> Vec<u8> {
1036 encode_chunked_dict_section_ordered(raw, codec, crate::term_order::TermOrder::Bytewise)
1037}
1038
1039fn encode_chunked_dict_section_ordered(
1042 raw: &[u8],
1043 codec: u8,
1044 order: crate::term_order::TermOrder,
1045) -> Vec<u8> {
1046 let meta = crate::dict::parse_meta(raw).unwrap_or(crate::dict::SectionMeta {
1047 term_count: 0,
1048 restart_interval: 1,
1049 restart_offsets: Vec::new(),
1050 });
1051 let body_start = meta
1052 .restart_offsets
1053 .first()
1054 .copied()
1055 .unwrap_or(raw.len() as u64);
1056 let header = &raw[..(body_start.min(raw.len() as u64)) as usize];
1057
1058 let chunk_budget = crate::write_opts::dict_chunk_budget() as u64;
1060 let n_runs = meta.restart_offsets.len();
1061 let mut bounds: Vec<(usize, u64, u64)> = Vec::new(); let mut r = 0;
1063 while r < n_runs {
1064 let start = meta.restart_offsets[r];
1065 let mut r2 = r + 1;
1066 while r2 < n_runs && meta.restart_offsets[r2] - start < chunk_budget {
1067 r2 += 1;
1068 }
1069 let end = if r2 < n_runs {
1070 meta.restart_offsets[r2]
1071 } else {
1072 raw.len() as u64
1073 };
1074 bounds.push((r, start, end));
1075 r = r2;
1076 }
1077
1078 #[cfg(feature = "parallel")]
1082 let compressed: Vec<Vec<u8>> = {
1083 use rayon::prelude::*;
1084 bounds
1085 .par_iter()
1086 .map(|&(_, s, e)| compress(codec, &raw[s as usize..e as usize]))
1087 .collect()
1088 };
1089 #[cfg(not(feature = "parallel"))]
1090 let compressed: Vec<Vec<u8>> = bounds
1091 .iter()
1092 .map(|&(_, s, e)| compress(codec, &raw[s as usize..e as usize]))
1093 .collect();
1094 let mut out = Vec::new();
1095 write_uvarint(&mut out, header.len() as u64);
1096 out.extend_from_slice(header);
1097 write_uvarint(&mut out, bounds.len() as u64);
1098 let mut prev_run = 0usize;
1099 let mut prev_last: Option<Vec<u8>> = None;
1103 for (i, (&(first_run, start, end), comp)) in bounds.iter().zip(&compressed).enumerate() {
1104 let key = if i == 0 {
1105 Vec::new()
1107 } else {
1108 let first = crate::dict::run_first_term(raw, start as usize).unwrap_or_default();
1109 match &prev_last {
1110 None => first,
1113 Some(pl) => order.separator(pl, &first),
1114 }
1115 };
1116 if i + 1 < bounds.len() {
1117 let last_run_off = meta.restart_offsets[bounds[i + 1].0 - 1] as usize;
1119 prev_last = crate::dict::run_last_term(raw, last_run_off, end as usize);
1120 }
1121 write_uvarint(&mut out, (first_run - prev_run) as u64);
1122 write_uvarint(&mut out, key.len() as u64);
1123 out.extend_from_slice(&key);
1124 write_uvarint(&mut out, comp.len() as u64);
1125 prev_run = first_run;
1126 }
1127 for comp in &compressed {
1128 out.extend_from_slice(comp);
1129 }
1130 out
1131}
1132
1133pub(crate) struct DictChunkEntry {
1137 pub(crate) first_run: usize,
1138 pub(crate) key: Vec<u8>,
1139 body_start: u64,
1140 pub(crate) start: u64,
1141 pub(crate) end: u64,
1142}
1143
1144fn parse_chunked_dict_dir(
1148 bytes: &[u8],
1149 total_len: u64,
1150) -> Result<(crate::dict::SectionMeta, Vec<DictChunkEntry>), FileError> {
1151 let mut pos = 0usize;
1152 let take = |pos: &mut usize| -> Result<u64, FileError> {
1153 let (v, n) = read_uvarint(bytes.get(*pos..).unwrap_or(&[]))
1154 .ok_or(FileError::Container("truncated dict chunk directory"))?;
1155 *pos += n;
1156 Ok(v)
1157 };
1158 let header_len = wire_usize(take(&mut pos)?, "dict header too large")?;
1159 let header = bytes
1160 .get(pos..pos.saturating_add(header_len))
1161 .ok_or(FileError::Container("truncated dict header"))?;
1162 let meta = crate::dict::parse_meta(header)
1163 .map_err(|_| FileError::Container("malformed dict header"))?;
1164 pos += header_len;
1165
1166 let num_chunks = wire_usize(take(&mut pos)?, "dict chunk count too large")?;
1167 let mut entries = Vec::with_capacity(num_chunks.min(bytes.len()));
1168 let mut lens = Vec::with_capacity(num_chunks.min(bytes.len()));
1169 let mut prev_run = 0usize;
1170 for i in 0..num_chunks {
1171 let drun = wire_usize(take(&mut pos)?, "dict chunk run too large")?;
1172 let klen = wire_usize(take(&mut pos)?, "dict chunk key too large")?;
1173 let key = bytes
1174 .get(pos..pos.saturating_add(klen))
1175 .ok_or(FileError::Container("truncated dict chunk key"))?
1176 .to_vec();
1177 pos += klen;
1178 let clen = take(&mut pos)?;
1179 let first_run = next_chunk_run(i, prev_run, drun)?;
1180 let body_start = meta
1181 .restart_offsets
1182 .get(first_run)
1183 .copied()
1184 .ok_or(FileError::Container("dict chunk run out of range"))?;
1185 entries.push(DictChunkEntry {
1186 first_run,
1187 key,
1188 body_start,
1189 start: 0,
1190 end: 0,
1191 });
1192 lens.push(clen);
1193 prev_run = first_run;
1194 }
1195 let mut start = pos as u64;
1196 for (e, len) in entries.iter_mut().zip(lens) {
1197 let end = start
1198 .checked_add(len)
1199 .filter(|&e| e <= total_len)
1200 .ok_or(FileError::Container("dict chunk overruns section"))?;
1201 e.start = start;
1202 e.end = end;
1203 start = end;
1204 }
1205 Ok((meta, entries))
1206}
1207
1208fn next_chunk_run(i: usize, prev_run: usize, drun: usize) -> Result<usize, FileError> {
1215 match (i, drun) {
1216 (0, 0) => Ok(0),
1217 (0, _) => Err(FileError::Container(
1218 "first dict chunk does not start at run 0",
1219 )),
1220 (_, 0) => Err(FileError::Container("dict chunk runs do not ascend")),
1221 _ => prev_run
1222 .checked_add(drun)
1223 .ok_or(FileError::Container("dict chunk run overflows")),
1224 }
1225}
1226
1227pub(crate) fn read_dict_dir_ranged<R: RangeReader + ?Sized>(
1239 reader: &R,
1240 section: ByteRange,
1241) -> Result<(crate::dict::SectionMeta, Vec<DictChunkEntry>), FileError> {
1242 let total = section.len;
1243 let init = 8192.min(total); let head = reader.read_at(section.offset, init)?;
1251 let (header_len, n0) =
1252 read_uvarint(&head).ok_or(FileError::Container("truncated dict header len"))?;
1253 let hbase = n0; let (term_count, n1) = read_uvarint(head.get(hbase..).unwrap_or(&[]))
1255 .ok_or(FileError::Container("truncated dict term_count"))?;
1256 let (restart_interval, _n2) = read_uvarint(head.get(hbase + n1..).unwrap_or(&[]))
1257 .ok_or(FileError::Container("truncated dict interval"))?;
1258 if restart_interval == 0 {
1259 return Err(FileError::Container("zero restart interval"));
1260 }
1261 let dir_start = (hbase as u64)
1264 .checked_add(header_len)
1265 .filter(|&d| d <= total)
1266 .ok_or(FileError::Container("dict header overruns section"))?;
1267 let dir_total = total - dir_start;
1268 let meta = crate::dict::SectionMeta {
1269 term_count: u32::try_from(term_count)
1270 .map_err(|_| FileError::Container("dict term count too large"))?,
1271 restart_interval: u32::try_from(restart_interval)
1272 .map_err(|_| FileError::Container("dict restart interval too large"))?,
1273 restart_offsets: Vec::new(),
1274 };
1275 let num_runs = (meta.term_count as usize).div_ceil(meta.restart_interval as usize);
1277 let finish = |mut entries: Vec<DictChunkEntry>| {
1278 for e in &mut entries {
1279 e.start += dir_start; e.end += dir_start;
1281 }
1282 (meta.clone(), entries)
1283 };
1284 let mut have: Vec<u8> = Vec::new();
1288 if dir_start < head.len() as u64 {
1289 match parse_chunk_dir_only(&head[dir_start as usize..], dir_total, num_runs)? {
1290 ChunkDirParse::Done(entries) => return Ok(finish(entries)),
1291 ChunkDirParse::Truncated { .. } => have = head[dir_start as usize..].to_vec(),
1292 }
1293 }
1294 let mut want = 4096u64.min(dir_total).max(1);
1307 loop {
1308 let held = have.len() as u64;
1309 if want > held {
1310 let extra = reader.read_at(section.offset + dir_start + held, want - held)?;
1311 if extra.is_empty() {
1312 return Err(FileError::Container("truncated dict chunk directory"));
1313 }
1314 have.extend_from_slice(&extra);
1315 }
1316 let held = have.len() as u64;
1317 match parse_chunk_dir_only(&have, dir_total, num_runs)? {
1318 ChunkDirParse::Done(entries) => return Ok(finish(entries)),
1319 ChunkDirParse::Truncated { .. } if held >= dir_total => {
1320 return Err(FileError::Container("truncated dict chunk directory"));
1321 }
1322 ChunkDirParse::Truncated {
1323 parsed,
1324 used,
1325 total,
1326 } => {
1327 let est = if parsed >= 16 && used > 0 {
1332 let whole = ((used as u64) / (parsed as u64)).saturating_mul(total as u64);
1333 whole.saturating_add(whole / 8).saturating_add(64)
1334 } else {
1335 held.saturating_mul(2)
1337 };
1338 want = est
1339 .max(held.saturating_add(1))
1340 .min(held.saturating_mul(4))
1341 .min(dir_total);
1342 }
1343 }
1344 }
1345}
1346
1347enum ChunkDirParse {
1352 Done(Vec<DictChunkEntry>),
1353 Truncated {
1354 parsed: usize,
1356 used: usize,
1358 total: usize,
1360 },
1361}
1362
1363fn parse_chunk_dir_only(
1373 dir: &[u8],
1374 dir_total: u64,
1375 num_runs: usize,
1376) -> Result<ChunkDirParse, FileError> {
1377 let mut pos = 0usize;
1378 let take = |pos: &mut usize| -> Option<u64> {
1379 let (v, n) = read_uvarint(dir.get(*pos..).unwrap_or(&[]))?;
1380 *pos += n;
1381 Some(v)
1382 };
1383 let Some(num_chunks) = take(&mut pos) else {
1384 return Ok(ChunkDirParse::Truncated {
1385 parsed: 0,
1386 used: 0,
1387 total: 0,
1388 });
1389 };
1390 let num_chunks = wire_usize(num_chunks, "dict chunk count too large")?;
1391 let mut entries = Vec::with_capacity(num_chunks.min(dir.len()));
1392 let mut lens: Vec<u64> = Vec::with_capacity(num_chunks.min(dir.len()));
1393 let mut prev_run = 0usize;
1394 for i in 0..num_chunks {
1395 let entry_start = pos;
1396 let short = ChunkDirParse::Truncated {
1397 parsed: entries.len(),
1398 used: entry_start,
1399 total: num_chunks,
1400 };
1401 let (Some(drun), Some(klen)) = (take(&mut pos), take(&mut pos)) else {
1402 return Ok(short);
1403 };
1404 let drun = wire_usize(drun, "dict chunk run too large")?;
1405 let klen = wire_usize(klen, "dict chunk key too large")?;
1406 let Some(key) = dir.get(pos..pos.saturating_add(klen)) else {
1407 return Ok(short);
1408 };
1409 let key = key.to_vec();
1410 pos += klen;
1411 let Some(clen) = take(&mut pos) else {
1412 return Ok(short);
1413 };
1414 let first_run = next_chunk_run(i, prev_run, drun)?;
1415 if first_run >= num_runs {
1416 return Err(FileError::Container("dict chunk run out of range"));
1417 }
1418 entries.push(DictChunkEntry {
1419 first_run,
1420 key,
1421 body_start: 0,
1422 start: 0,
1423 end: 0,
1424 });
1425 lens.push(clen);
1426 prev_run = first_run;
1427 }
1428 let mut start = pos as u64;
1429 for (e, len) in entries.iter_mut().zip(lens) {
1430 let end = start
1431 .checked_add(len)
1432 .filter(|&e| e <= dir_total)
1433 .ok_or(FileError::Container("dict chunk overruns section"))?;
1434 e.start = start;
1435 e.end = end;
1436 start = end;
1437 }
1438 Ok(ChunkDirParse::Done(entries))
1439}
1440
1441fn decode_chunked_dict_section(
1445 payload: &[u8],
1446 codec: u8,
1447 cache: std::sync::Arc<crate::chunk_cache::ChunkCache>,
1448 section_index: u8,
1449) -> Result<crate::dict::ChunkedSection, FileError> {
1450 let (meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64)?;
1451 let chunks = entries
1452 .into_iter()
1453 .map(|e| {
1454 let body = decompress(codec, &payload[e.start as usize..e.end as usize])?;
1455 Ok((
1456 crate::dict::SectionChunk::new(e.first_run, e.key, e.body_start),
1457 body,
1458 ))
1459 })
1460 .collect::<Result<Vec<_>, FileError>>()?;
1461 Ok(crate::dict::ChunkedSection::resident(
1462 meta,
1463 chunks,
1464 cache,
1465 section_index,
1466 ))
1467}
1468
1469fn decode_dictionary_container(
1470 bytes: &[u8],
1471 codec: u8,
1472 order: crate::term_order::TermOrder,
1473) -> Result<Dictionary, FileError> {
1474 let dsecs = decode_container(bytes, CODEC_NONE)?;
1475 if dsecs.len() != 4 {
1476 return Err(FileError::Container("expected 4 dictionary sections"));
1477 }
1478 let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
1481 let mut sections = Vec::with_capacity(4);
1482 for (si, sec) in dsecs.iter().enumerate() {
1483 sections.push(
1484 decode_chunked_dict_section(sec, codec, cache.clone(), si as u8)?.with_order(order),
1485 );
1486 }
1487 let arr: [crate::dict::ChunkedSection; 4] = sections
1488 .try_into()
1489 .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
1490 Ok(Dictionary::from_chunked_sections(arr))
1491}
1492
1493fn encode_tiled_section(index: &GraphIndex, si: usize, codec: u8) -> Vec<u8> {
1499 let tiles = index.tile_sections()[si];
1500 let n = tiles.len();
1501 #[cfg(feature = "parallel")]
1508 let compressed: Vec<Vec<u8>> = {
1509 use rayon::prelude::*;
1510 (0..n)
1511 .into_par_iter()
1512 .map(|ti| compress(codec, &index.tile_body(si, ti)))
1513 .collect()
1514 };
1515 #[cfg(not(feature = "parallel"))]
1516 let compressed: Vec<Vec<u8>> = (0..n)
1517 .map(|ti| compress(codec, &index.tile_body(si, ti)))
1518 .collect();
1519 let mut out = Vec::new();
1520 write_uvarint(&mut out, tiles.len() as u64);
1521 let mut prev_min = 0u32;
1522 for (tile, comp) in tiles.iter().zip(&compressed) {
1523 let (min_a, max_a) = tile.leading_range();
1524 write_uvarint(&mut out, (min_a - prev_min) as u64);
1525 write_uvarint(&mut out, (max_a - min_a) as u64);
1526 write_uvarint(&mut out, comp.len() as u64);
1527 prev_min = min_a;
1528 }
1529 for comp in &compressed {
1530 out.extend_from_slice(comp);
1531 }
1532 for ti in 0..n {
1540 let body = index.tile_body(si, ti);
1541 let (min_b, max_b, min_c, max_c) = match crate::triples::TripleBlock::parse(&body) {
1542 Ok(b) => {
1543 let z = b.zone();
1544 (z.min_b, z.max_b, z.min_c, z.max_c)
1545 }
1546 Err(_) => (0, u32::MAX, 0, u32::MAX),
1547 };
1548 write_uvarint(&mut out, min_b as u64);
1549 write_uvarint(&mut out, (max_b - min_b) as u64);
1550 write_uvarint(&mut out, min_c as u64);
1551 write_uvarint(&mut out, (max_c - min_c) as u64);
1552 }
1553 out
1554}
1555
1556pub(crate) struct TileDirEntry {
1559 pub(crate) min_a: u32,
1560 pub(crate) max_a: u32,
1561 pub(crate) start: u64,
1562 pub(crate) end: u64,
1563}
1564
1565type TileSynopsis = (u32, u32, u32, u32);
1567
1568fn parse_tile_synopsis(
1576 payload: &[u8],
1577 trailer_start: usize,
1578 num_tiles: usize,
1579) -> Option<Vec<TileSynopsis>> {
1580 let mut pos = trailer_start;
1581 let take = |pos: &mut usize| -> Option<u32> {
1582 let (v, n) = read_uvarint(payload.get(*pos..)?)?;
1583 *pos += n;
1584 u32::try_from(v).ok()
1585 };
1586 let mut out = Vec::with_capacity(num_tiles.min(payload.len()));
1587 for _ in 0..num_tiles {
1588 let min_b = take(&mut pos)?;
1589 let max_b = min_b.checked_add(take(&mut pos)?)?;
1590 let min_c = take(&mut pos)?;
1591 let max_c = min_c.checked_add(take(&mut pos)?)?;
1592 out.push((min_b, max_b, min_c, max_c));
1593 }
1594 Some(out)
1595}
1596
1597fn parse_tile_directory(bytes: &[u8], total_len: u64) -> Result<Vec<TileDirEntry>, FileError> {
1602 let mut pos = 0usize;
1603 let take = |pos: &mut usize| -> Result<u64, FileError> {
1604 let (v, n) = read_uvarint(bytes.get(*pos..).unwrap_or(&[]))
1605 .ok_or(FileError::Container("truncated tile directory"))?;
1606 *pos += n;
1607 Ok(v)
1608 };
1609 let num_tiles = wire_usize(take(&mut pos)?, "tile count too large")?;
1610 let mut entries = Vec::with_capacity(num_tiles.min(bytes.len()));
1611 let mut prev_min = 0u32;
1612 let mut lens = Vec::with_capacity(num_tiles.min(bytes.len()));
1613 let id = |v: u64| u32::try_from(v).map_err(|_| FileError::Container("tile id out of range"));
1617 for _ in 0..num_tiles {
1618 let dmin = id(take(&mut pos)?)?;
1619 let span = id(take(&mut pos)?)?;
1620 let len = take(&mut pos)?;
1621 let min_a = prev_min
1622 .checked_add(dmin)
1623 .ok_or(FileError::Container("tile id out of range"))?;
1624 let max_a = min_a
1625 .checked_add(span)
1626 .ok_or(FileError::Container("tile id out of range"))?;
1627 entries.push(TileDirEntry {
1628 min_a,
1629 max_a,
1630 start: 0,
1631 end: 0,
1632 });
1633 lens.push(len);
1634 prev_min = min_a;
1635 }
1636 let mut start = pos as u64;
1637 for (e, len) in entries.iter_mut().zip(lens) {
1638 let end = start
1639 .checked_add(len)
1640 .filter(|&e| e <= total_len)
1641 .ok_or(FileError::Container("tile overruns section"))?;
1642 e.start = start;
1643 e.end = end;
1644 start = end;
1645 }
1646 Ok(entries)
1647}
1648
1649pub(crate) fn read_tile_directory_ranged<R: RangeReader + ?Sized>(
1653 reader: &R,
1654 section: ByteRange,
1655) -> Result<Vec<TileDirEntry>, FileError> {
1656 let total = section.len;
1657 let mut prefetch = 4096u64.min(total);
1658 loop {
1659 let prefix = reader.read_at(section.offset, prefetch)?;
1660 match parse_tile_directory(&prefix, total) {
1661 Ok(dir) => return Ok(dir),
1662 Err(_) if prefetch < total => prefetch = prefetch.saturating_mul(2).min(total),
1663 Err(e) => return Err(e),
1664 }
1665 }
1666}
1667
1668pub(crate) fn read_tile_synopsis_ranged<R: RangeReader + ?Sized>(
1675 reader: &R,
1676 section: ByteRange,
1677 dir: &[TileDirEntry],
1678) -> Vec<Option<TileSynopsis>> {
1679 let n = dir.len();
1680 let none = vec![None; n];
1681 let trailer_start = dir.iter().map(|e| e.end).max().unwrap_or(0);
1682 let total = section.len;
1683 if n == 0 || trailer_start >= total {
1684 return none; }
1686 let max_trailer = (n as u64).saturating_mul(4 * 5);
1690 let trailer_len = (total - trailer_start).min(max_trailer);
1691 let Some(offset) = section.offset.checked_add(trailer_start) else {
1692 return none;
1693 };
1694 let Ok(bytes) = reader.read_at(offset, trailer_len) else {
1695 return none;
1696 };
1697 match parse_tile_synopsis(&bytes, 0, n) {
1698 Some(v) => v.into_iter().map(Some).collect(),
1699 None => none,
1700 }
1701}
1702
1703fn tile_file_ranges(
1707 index_bytes: &[u8],
1708 container_offset: u64,
1709 section_ranges: &[ByteRange; NUM_PERMS],
1710) -> [Vec<(u32, u32, ByteRange)>; NUM_PERMS] {
1711 let mut out: [Vec<(u32, u32, ByteRange)>; NUM_PERMS] = Default::default();
1712 for (section, range) in out.iter_mut().zip(section_ranges) {
1713 if range.len == 0 || range.offset < container_offset {
1717 continue;
1718 }
1719 let Some(payload) = checked_slice(index_bytes, range.offset - container_offset, range.len)
1720 else {
1721 continue;
1722 };
1723 if let Ok(dir) = parse_tile_directory(payload, payload.len() as u64) {
1724 *section = dir
1725 .into_iter()
1726 .map(|e| {
1727 (
1728 e.min_a,
1729 e.max_a,
1730 ByteRange {
1731 offset: range.offset + e.start,
1732 len: (e.end - e.start),
1733 },
1734 )
1735 })
1736 .collect();
1737 }
1738 }
1739 out
1740}
1741
1742fn decode_tiled_section(payload: &[u8], codec: u8) -> Result<Vec<(u32, u32, Vec<u8>)>, FileError> {
1745 parse_tile_directory(payload, payload.len() as u64)?
1746 .into_iter()
1747 .map(|e| {
1748 Ok((
1749 e.min_a,
1750 e.max_a,
1751 decompress(codec, &payload[e.start as usize..e.end as usize])?,
1752 ))
1753 })
1754 .collect()
1755}
1756
1757fn decode_index_container(
1767 bytes: &[u8],
1768 codec: u8,
1769 perms: PermSet,
1770) -> Result<GraphIndex, FileError> {
1771 let isecs = decode_container(bytes, CODEC_NONE)?;
1772 if isecs.len() != perms.len() {
1773 return Err(FileError::Container(
1774 "index container section count does not match the header permutation mask",
1775 ));
1776 }
1777 decode_index_sections(isecs, codec, perms)
1778}
1779
1780fn decode_index_sections(
1781 mut isecs: Vec<Vec<u8>>,
1782 codec: u8,
1783 perms: PermSet,
1784) -> Result<GraphIndex, FileError> {
1785 let mut sections: [Vec<(u32, u32, Vec<u8>)>; NUM_PERMS] = Default::default();
1786 for (perm, sec) in perms.iter().zip(isecs.iter_mut()) {
1787 sections[perm.section_index()] = decode_tiled_section(sec, codec)?;
1788 }
1789 Ok(GraphIndex::from_tiles(sections, perms))
1790}
1791
1792fn decode_root_index_container(
1797 bytes: &[u8],
1798 header: &Header,
1799) -> Result<(GraphIndex, Option<[u8; 16]>), FileError> {
1800 if !header.has_quad_index() {
1801 return Ok((
1802 decode_index_container(bytes, header.block_codec, header.perms)?,
1803 None,
1804 ));
1805 }
1806 let mut isecs = decode_container(bytes, CODEC_NONE)?;
1807 if isecs.len() != header.index_container_sections() {
1808 return Err(FileError::Container(
1809 "index container section count does not match the header permutation mask",
1810 ));
1811 }
1812 let seal = isecs.pop().unwrap_or_default();
1813 let digest = crate::quad::parse_seal(&seal)?;
1814 Ok((
1815 decode_index_sections(isecs, header.block_codec, header.perms)?,
1816 Some(digest),
1817 ))
1818}
1819
1820fn check_named_layout(header: &Header) -> Result<(), FileError> {
1825 let quad = header.quad_index_section().is_some();
1826 if header.has_quad_index() {
1827 if !quad {
1828 return Err(FileError::Container(
1829 "quad-index flag set but the file has no quad-index section",
1830 ));
1831 }
1832 if header.named_graphs_len > 0 {
1833 return Err(FileError::Container(
1834 "a file cannot carry both a quad index and a named-graphs section",
1835 ));
1836 }
1837 crate::quad::check_perms(header.perms)?;
1838 } else if quad {
1839 return Err(FileError::Container(
1840 "quad-index section present but the header does not flag it",
1841 ));
1842 }
1843 Ok(())
1844}
1845
1846fn container_section_payload_ranges(
1847 bytes: &[u8],
1848 container_offset: u64,
1849 expected_sections: usize,
1850) -> Result<Vec<ByteRange>, FileError> {
1851 let (section_count, mut pos) =
1852 read_uvarint(bytes).ok_or(FileError::Container("truncated count"))?;
1853 let section_count = usize::try_from(section_count)
1854 .map_err(|_| FileError::Container("section count too large"))?;
1855 if section_count != expected_sections {
1856 return Err(FileError::Container("unexpected section count"));
1857 }
1858
1859 let mut ranges = Vec::with_capacity(section_count);
1860 for _ in 0..section_count {
1861 let remaining = bytes
1862 .get(pos..)
1863 .ok_or(FileError::Container("truncated length"))?;
1864 let (payload_len, used) =
1865 read_uvarint(remaining).ok_or(FileError::Container("truncated length"))?;
1866 pos = pos
1867 .checked_add(used)
1868 .ok_or(FileError::Container("section range overflows"))?;
1869 let payload_len_usize = usize::try_from(payload_len)
1870 .map_err(|_| FileError::Container("section length too large"))?;
1871 let payload_end = pos
1872 .checked_add(payload_len_usize)
1873 .ok_or(FileError::Container("section range overflows"))?;
1874 if payload_end > bytes.len() {
1875 return Err(FileError::Container("section overruns buffer"));
1876 }
1877 ranges.push(ByteRange {
1878 offset: checked_end(container_offset, pos as u64)?,
1879 len: payload_len,
1880 });
1881 pos = payload_end;
1882 }
1883
1884 Ok(ranges)
1885}
1886
1887fn decode_index_section_ranges(
1891 bytes: &[u8],
1892 container_offset: u64,
1893 perms: PermSet,
1894 sections: usize,
1895) -> Result<[ByteRange; NUM_PERMS], FileError> {
1896 let ranges = container_section_payload_ranges(bytes, container_offset, sections)?;
1897 let mut out = [ByteRange { offset: 0, len: 0 }; NUM_PERMS];
1898 for (perm, range) in perms.iter().zip(ranges) {
1899 out[perm.section_index()] = range;
1900 }
1901 Ok(out)
1902}
1903
1904fn read_uvarint_at<R: RangeReader + ?Sized>(
1905 reader: &R,
1906 absolute_offset: u64,
1907 container_end: u64,
1908) -> Result<(u64, u64), FileError> {
1909 if absolute_offset >= container_end {
1910 return Err(FileError::Container("truncated container varint"));
1911 }
1912 let remaining = container_end - absolute_offset;
1913 let probe_len = remaining.min(10);
1914 let bytes = reader.read_at(absolute_offset, probe_len)?;
1915 read_uvarint(&bytes)
1916 .map(|(value, used)| (value, used as u64))
1917 .ok_or(FileError::Container("truncated container varint"))
1918}
1919
1920pub(crate) fn locate_container_section_ranged<R: RangeReader + ?Sized>(
1923 reader: &R,
1924 container_offset: u64,
1925 container_len: u64,
1926 section_index: usize,
1927 expected_sections: u64,
1928) -> Result<ByteRange, FileError> {
1929 let container_end = checked_end(container_offset, container_len)?;
1930 let (section_count, used) = read_uvarint_at(reader, container_offset, container_end)?;
1931 if section_count != expected_sections {
1932 return Err(FileError::Container("unexpected container section count"));
1933 }
1934 if section_index >= section_count as usize {
1935 return Err(FileError::Container(
1936 "container section index out of bounds",
1937 ));
1938 }
1939
1940 let mut pos = checked_end(container_offset, used)?;
1941 for i in 0..section_count as usize {
1942 let (payload_len, len_used) = read_uvarint_at(reader, pos, container_end)?;
1943 pos = checked_end(pos, len_used)?;
1944 let payload_end = checked_end(pos, payload_len)?;
1945 if payload_end > container_end {
1946 return Err(FileError::Container("section overruns buffer"));
1947 }
1948 if i == section_index {
1949 return Ok(ByteRange {
1950 offset: pos,
1951 len: payload_len,
1952 });
1953 }
1954 pos = payload_end;
1955 }
1956 Err(FileError::Container("container section not found"))
1957}
1958
1959#[allow(clippy::type_complexity)]
1969fn open_index_container_lazy(
1970 reader: &std::sync::Arc<dyn RangeReader + Send + Sync>,
1971 container: ByteRange,
1972 block_codec: u8,
1973 has_synopsis: bool,
1974 read_concurrency: usize,
1975 perms: PermSet,
1976 sections: usize,
1977) -> Result<
1978 (
1979 GraphIndex,
1980 [ByteRange; NUM_PERMS],
1981 [Vec<(u32, u32, ByteRange)>; NUM_PERMS],
1982 ),
1983 FileError,
1984> {
1985 let mut index_section_ranges = [ByteRange { offset: 0, len: 0 }; NUM_PERMS];
1986 let mut tile_ranges: [Vec<(u32, u32, ByteRange)>; NUM_PERMS] = Default::default();
1987 #[allow(clippy::type_complexity)]
1988 let mut directories: [Vec<(u32, u32, Option<TileSynopsis>)>; NUM_PERMS] = Default::default();
1989 for (pos, perm) in perms.iter().enumerate() {
1990 let si = perm.section_index();
1991 let section = locate_container_section_ranged(
1992 reader,
1993 container.offset,
1994 container.len,
1995 pos,
1996 sections as u64,
1997 )?;
1998 index_section_ranges[si] = section;
1999 let dir = read_tile_directory_ranged(reader, section)?;
2000 let syn = if has_synopsis {
2003 read_tile_synopsis_ranged(reader, section, &dir)
2004 } else {
2005 vec![None; dir.len()]
2006 };
2007 directories[si] = dir
2008 .iter()
2009 .zip(syn)
2010 .map(|(e, s)| (e.min_a, e.max_a, s))
2011 .collect();
2012 tile_ranges[si] = dir
2013 .into_iter()
2014 .map(|e| {
2015 (
2016 e.min_a,
2017 e.max_a,
2018 ByteRange {
2019 offset: section.offset + e.start,
2020 len: (e.end - e.start),
2021 },
2022 )
2023 })
2024 .collect();
2025 }
2026
2027 let codec = block_codec;
2037 let loader_ranges = tile_ranges.clone();
2038 let loader_reader = reader.clone();
2039 let loader: crate::index::TileLoader = Box::new(move |si, ti| {
2040 let (_, _, range) = loader_ranges.get(si)?.get(ti)?;
2041 let bytes = loader_reader.read_at(range.offset, range.len).ok()?;
2042 if bytes.len() as u64 != range.len {
2043 return None;
2044 }
2045 checked_tile(codec, &bytes)
2046 });
2047 let bulk_ranges = tile_ranges.clone();
2048 let bulk_reader = reader.clone();
2049 let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
2050 let section = bulk_ranges.get(si)?;
2051 let want: Option<Vec<ByteRange>> = tis
2052 .iter()
2053 .map(|&ti| section.get(ti).map(|&(_, _, r)| r))
2054 .collect();
2055 let blobs = read_coalesced(bulk_reader.as_ref(), &want?, TILE_COALESCE_GAP)?;
2056 blobs.iter().map(|b| checked_tile(codec, b)).collect()
2057 });
2058 let mut index =
2059 GraphIndex::from_remote_directories(directories, perms, loader).with_bulk_loader(bulk);
2060 index.set_tile_lens(std::array::from_fn(|si| {
2063 tile_ranges[si]
2064 .iter()
2065 .map(|&(_, _, r)| r.len.min(u32::MAX as u64) as u32)
2066 .collect()
2067 }));
2068 index.set_read_concurrency(read_concurrency);
2072 Ok((index, index_section_ranges, tile_ranges))
2073}
2074
2075fn checked_tile(codec: u8, bytes: &[u8]) -> Option<Vec<u8>> {
2079 let tile = decompress(codec, bytes).ok()?;
2080 crate::triples::TripleBlock::parse(&tile).ok()?;
2081 Some(tile)
2082}
2083
2084pub fn write_file(
2091 dict: &Dictionary,
2092 index: &GraphIndex,
2093 has_quads: bool,
2094 pyramid_meta: &[u8],
2095 pyramid_levels: u16,
2096) -> Vec<u8> {
2097 write_dataset(dict, index, &[], has_quads, pyramid_meta, pyramid_levels)
2098}
2099
2100#[doc(hidden)]
2105pub fn write_file_v05(
2106 dict: &Dictionary,
2107 index: &GraphIndex,
2108 has_quads: bool,
2109 pyramid_meta: &[u8],
2110 pyramid_levels: u16,
2111) -> Vec<u8> {
2112 write_dataset_v05(dict, index, &[], has_quads, pyramid_meta, pyramid_levels)
2113}
2114
2115fn into_v06(image: Vec<u8>) -> Vec<u8> {
2118 crate::build_record::finish_in_ram(image, None)
2119 .unwrap_or_else(|e| panic!("writing format 0x06: {e}"))
2120}
2121
2122pub(crate) fn encode_index_container(index: &GraphIndex, codec: u8) -> Vec<u8> {
2128 encode_index_container_sealed(index, codec, None)
2129}
2130
2131fn encode_index_container_sealed(index: &GraphIndex, codec: u8, seal: Option<&[u8]>) -> Vec<u8> {
2136 let payloads: Vec<Vec<u8>> = index
2137 .perms()
2138 .iter()
2139 .map(|perm| encode_tiled_section(index, perm.section_index(), codec))
2140 .collect();
2141 let mut refs: Vec<&[u8]> = payloads.iter().map(|p| p.as_slice()).collect();
2142 if let Some(seal) = seal {
2143 refs.push(seal);
2144 }
2145 encode_container(&refs, CODEC_NONE)
2146}
2147
2148fn encode_named_graphs(named: &[(String, GraphIndex)], codec: u8) -> Vec<u8> {
2150 let mut out = Vec::new();
2151 write_uvarint(&mut out, named.len() as u64);
2152 for (iri, index) in named {
2153 write_uvarint(&mut out, iri.len() as u64);
2154 out.extend_from_slice(iri.as_bytes());
2155 let container = encode_index_container(index, codec);
2156 write_uvarint(&mut out, container.len() as u64);
2157 out.extend_from_slice(&container);
2158 }
2159 out
2160}
2161
2162fn decode_named_graphs(
2163 bytes: &[u8],
2164 codec: u8,
2165 perms: PermSet,
2166) -> Result<Vec<(String, GraphIndex)>, FileError> {
2167 let (n, mut pos) = read_uvarint(bytes).ok_or(FileError::Container("truncated graph count"))?;
2168 let bound = |start: usize, len: u64| -> Result<usize, FileError> {
2171 usize::try_from(len)
2172 .ok()
2173 .and_then(|len| start.checked_add(len))
2174 .filter(|&e| e <= bytes.len())
2175 .ok_or(FileError::Container("named-graph field overruns buffer"))
2176 };
2177 let n = wire_usize(n, "named-graph count too large")?;
2178 let mut out = Vec::with_capacity(n.min(bytes.len()));
2179 for _ in 0..n {
2180 let (ilen, u1) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
2181 .ok_or(FileError::Container("truncated iri len"))?;
2182 pos += u1;
2183 let iend = bound(pos, ilen)?;
2184 let iri = String::from_utf8_lossy(&bytes[pos..iend]).into_owned();
2185 pos = iend;
2186 let (clen, u2) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
2187 .ok_or(FileError::Container("truncated container len"))?;
2188 pos += u2;
2189 let cend = bound(pos, clen)?;
2190 let index = decode_index_container(&bytes[pos..cend], codec, perms)?;
2191 index.set_size_hint(u64::from(index.triple_count()));
2194 out.push((iri, index));
2195 pos = cend;
2196 }
2197 Ok(out)
2198}
2199
2200pub fn write_dataset(
2205 dict: &Dictionary,
2206 default_index: &GraphIndex,
2207 named: &[(String, GraphIndex)],
2208 has_quads: bool,
2209 pyramid_meta: &[u8],
2210 pyramid_levels: u16,
2211) -> Vec<u8> {
2212 write_dataset_with_metadata(
2213 dict,
2214 default_index,
2215 named,
2216 has_quads,
2217 pyramid_meta,
2218 pyramid_levels,
2219 &[],
2220 &[],
2221 )
2222}
2223
2224#[doc(hidden)]
2226pub fn write_dataset_v05(
2227 dict: &Dictionary,
2228 default_index: &GraphIndex,
2229 named: &[(String, GraphIndex)],
2230 has_quads: bool,
2231 pyramid_meta: &[u8],
2232 pyramid_levels: u16,
2233) -> Vec<u8> {
2234 write_dataset_with_metadata_v05(
2235 dict,
2236 default_index,
2237 named,
2238 has_quads,
2239 pyramid_meta,
2240 pyramid_levels,
2241 &[],
2242 &[],
2243 )
2244}
2245
2246pub(crate) fn encode_dict_container(dict: &Dictionary, codec: u8) -> Vec<u8> {
2251 let raw_sections = dict.sections();
2252 let dict_payloads: Vec<Vec<u8>> = raw_sections
2253 .iter()
2254 .map(|raw| encode_chunked_dict_section_ordered(raw, codec, dict.order()))
2255 .collect();
2256 encode_container(
2257 &[
2258 dict_payloads[0].as_slice(),
2259 dict_payloads[1].as_slice(),
2260 dict_payloads[2].as_slice(),
2261 dict_payloads[3].as_slice(),
2262 ],
2263 CODEC_NONE,
2264 )
2265}
2266
2267#[allow(clippy::too_many_arguments)]
2278pub fn write_dataset_with_metadata(
2279 dict: &Dictionary,
2280 default_index: &GraphIndex,
2281 named: &[(String, GraphIndex)],
2282 has_quads: bool,
2283 pyramid_meta: &[u8],
2284 pyramid_levels: u16,
2285 metadata: &[u8],
2286 text_index: &[u8],
2287) -> Vec<u8> {
2288 into_v06(write_dataset_with_metadata_v05(
2289 dict,
2290 default_index,
2291 named,
2292 has_quads,
2293 pyramid_meta,
2294 pyramid_levels,
2295 metadata,
2296 text_index,
2297 ))
2298}
2299
2300#[doc(hidden)]
2303#[allow(clippy::too_many_arguments)]
2304pub fn write_dataset_with_metadata_v05(
2305 dict: &Dictionary,
2306 default_index: &GraphIndex,
2307 named: &[(String, GraphIndex)],
2308 has_quads: bool,
2309 pyramid_meta: &[u8],
2310 pyramid_levels: u16,
2311 metadata: &[u8],
2312 text_index: &[u8],
2313) -> Vec<u8> {
2314 let codec = writer_codec();
2315 let dict_container = encode_dict_container(dict, codec);
2316 let scope = (!text_index.is_empty())
2320 .then(|| crate::text_index::TextIndex::from_section(text_index, codec).ok())
2321 .flatten()
2322 .map(|ti| ti.scope().clone());
2323 let text_index_partial = scope
2324 .as_ref()
2325 .is_some_and(crate::text_index::TextIndexScope::restricts_predicates);
2326 let text_index_named = scope
2327 .as_ref()
2328 .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs);
2329 write_dataset_from_parts(
2330 &dict_container,
2331 dict.term_count() as u64,
2332 default_index,
2333 NamedPart::PerGraph(named),
2334 has_quads,
2335 dict.has_quoted_triples(),
2336 pyramid_meta,
2337 pyramid_levels,
2338 metadata,
2339 text_index,
2340 text_index_partial,
2341 text_index_named,
2342 codec,
2343 dict.order(),
2344 )
2345}
2346
2347pub(crate) enum NamedPart<'a> {
2350 PerGraph(&'a [(String, GraphIndex)]),
2351 Quad(&'a crate::quad::EncodedQuadIndex),
2352}
2353
2354#[allow(clippy::too_many_arguments)]
2359pub(crate) fn write_dataset_from_parts(
2360 dict_container: &[u8],
2361 term_count: u64,
2362 default_index: &GraphIndex,
2363 named: NamedPart<'_>,
2364 has_quads: bool,
2365 has_quoted_triples: bool,
2366 pyramid_meta: &[u8],
2367 pyramid_levels: u16,
2368 metadata: &[u8],
2369 text_index: &[u8],
2370 text_index_partial: bool,
2371 text_index_named: bool,
2372 codec: u8,
2373 term_order: crate::term_order::TermOrder,
2374) -> Vec<u8> {
2375 let (named_section, quad, named_quads) = match named {
2379 NamedPart::PerGraph(named) => (
2380 if named.is_empty() {
2381 Vec::new()
2382 } else {
2383 encode_named_graphs(named, codec)
2384 },
2385 None,
2386 named
2387 .iter()
2388 .map(|(_, idx)| idx.triple_count() as u64)
2389 .sum::<u64>(),
2390 ),
2391 NamedPart::Quad(q) if q.graph_count == 0 => (Vec::new(), None, 0),
2392 NamedPart::Quad(q) => (Vec::new(), Some(q), q.quad_count),
2393 };
2394 let seal = quad.map(|q| crate::quad::encode_seal(crate::quad::digest(&q.bytes)));
2395 let index_container = encode_index_container_sealed(default_index, codec, seal.as_deref());
2396 let text_index_partial = text_index_partial && !text_index.is_empty();
2401 let digested;
2402 let text_index: &[u8] = if text_index_partial {
2403 digested = crate::text_index::with_partial_digest(text_index);
2404 &digested
2405 } else {
2406 text_index
2407 };
2408
2409 let meta_section_len = metadata.len() as u64;
2412 let dict_offset = HEADER_LEN as u64 + meta_section_len;
2413 let dict_len = dict_container.len() as u64;
2414 let index_offset = dict_offset + dict_len;
2415 let index_len = index_container.len() as u64;
2416 let pyr_offset = index_offset + index_len;
2417 let pyr_len = pyramid_meta.len() as u64;
2418 let text_offset = pyr_offset + pyr_len;
2420 let text_len = text_index.len() as u64;
2421 let named_offset = text_offset + text_len;
2422 let named_len = named_section.len() as u64;
2423 let quad_len = quad.map_or(0, |q| q.bytes.len() as u64);
2426
2427 let mut parts: Vec<&[u8]> = Vec::with_capacity(5);
2433 if meta_section_len > 0 {
2434 parts.push(metadata);
2435 }
2436 parts.push(dict_container);
2437 parts.push(&index_container);
2438 parts.push(pyramid_meta);
2439 if text_len > 0 && !text_index_partial {
2440 parts.push(text_index);
2441 }
2442 if named_len > 0 {
2443 parts.push(&named_section);
2444 }
2445
2446 let schema_meta_len = crate::meta::schema_block_len(pyramid_meta);
2449
2450 let header = Header {
2454 version: crate::header::LEGACY_FORMAT_VERSION,
2455 flags: FLAG_TILE_SYNOPSIS
2456 | if has_quads { FLAG_HAS_QUADS } else { 0 }
2457 | if quad.is_some() {
2458 crate::header::FLAG_QUAD_INDEX
2459 } else {
2460 0
2461 }
2462 | if has_quoted_triples {
2463 FLAG_HAS_QUOTED_TRIPLES
2464 } else {
2465 0
2466 }
2467 | if text_len > 0 && text_index_named {
2468 crate::header::FLAG_TEXT_INDEX_NAMED_GRAPHS
2469 } else {
2470 0
2471 }
2472 | if term_order == crate::term_order::TermOrder::Sparql {
2473 crate::header::FLAG_SPARQL_TERM_ORDER
2474 } else {
2475 0
2476 },
2477 metadata_offset: HEADER_LEN as u64,
2478 metadata_len: meta_section_len,
2479 dictionary_offset: dict_offset,
2480 dictionary_len: dict_len,
2481 root_dir_offset: index_offset,
2482 root_dir_len: index_len,
2483 pyramid_meta_offset: if pyr_len > 0 { pyr_offset } else { 0 },
2484 pyramid_meta_len: pyr_len,
2485 dict_codec: codec,
2486 block_codec: codec,
2487 pyramid_levels,
2488 id_widths: 0,
2489 perms: default_index.perms(),
2493 quad_count: default_index.triple_count() as u64 + named_quads,
2494 term_count,
2495 content_hash: content_hash(&parts),
2496 named_graphs_offset: if named_len > 0 { named_offset } else { 0 },
2497 named_graphs_len: named_len,
2498 schema_meta_len,
2499 text_index_offset: if text_len > 0 { text_offset } else { 0 },
2500 text_index_len: text_len,
2501 text_index_partial: text_len > 0 && text_index_partial,
2502 build_info_offset: 0,
2503 build_info_len: 0,
2504 extra_sections: if quad_len > 0 {
2505 vec![crate::header::Section {
2506 kind: crate::header::SectionKind::QuadIndex,
2507 flags: 0,
2508 offset: named_offset,
2509 length: quad_len,
2510 }]
2511 } else {
2512 Vec::new()
2513 },
2514 };
2515
2516 let mut out = Vec::with_capacity(
2517 HEADER_LEN
2518 + metadata.len()
2519 + dict_container.len()
2520 + index_container.len()
2521 + pyramid_meta.len()
2522 + text_index.len()
2523 + named_section.len()
2524 + quad_len as usize
2525 + MAGIC.len(),
2526 );
2527 out.extend_from_slice(&header.to_bytes());
2528 if meta_section_len > 0 {
2529 out.extend_from_slice(metadata);
2530 }
2531 out.extend_from_slice(dict_container);
2532 out.extend_from_slice(&index_container);
2533 out.extend_from_slice(pyramid_meta);
2534 if text_len > 0 {
2535 out.extend_from_slice(text_index);
2536 }
2537 if named_len > 0 {
2538 out.extend_from_slice(&named_section);
2539 }
2540 if let Some(q) = quad {
2541 out.extend_from_slice(&q.bytes);
2542 }
2543 out.extend_from_slice(&MAGIC); out
2545}
2546
2547pub const RDF_TYPE: &str = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
2549
2550pub fn schema_summary(rete: &Rete) -> Vec<(String, String, String, u32)> {
2557 use std::collections::{BTreeMap, HashMap};
2558 let triples = rete.dump(None);
2559
2560 let mut class_of: HashMap<&str, &str> = HashMap::new();
2561 for (s, p, o) in &triples {
2562 if p == RDF_TYPE {
2563 class_of.insert(s.as_str(), o.as_str());
2564 }
2565 }
2566 let classify = |t: &str| -> String {
2567 if let Some(c) = class_of.get(t) {
2568 (*c).to_string()
2569 } else if t.starts_with('"') {
2570 "(literal)".to_string()
2571 } else {
2572 "(untyped)".to_string()
2573 }
2574 };
2575
2576 let mut counts: BTreeMap<(String, String, String), u32> = BTreeMap::new();
2577 for (s, p, o) in &triples {
2578 if p == RDF_TYPE {
2579 continue; }
2581 *counts
2582 .entry((classify(s), p.clone(), classify(o)))
2583 .or_default() += 1;
2584 }
2585 counts
2586 .into_iter()
2587 .map(|((a, p, b), c)| (a, p, b, c))
2588 .collect()
2589}
2590
2591pub fn schema_classes(rete: &Rete) -> Vec<(String, u32)> {
2595 use std::collections::BTreeMap;
2596 let mut counts: BTreeMap<String, u32> = BTreeMap::new();
2597 for (_s, p, o) in rete.dump(None) {
2598 if p == RDF_TYPE {
2599 *counts.entry(o).or_default() += 1;
2600 }
2601 }
2602 let mut out: Vec<(String, u32)> = counts.into_iter().collect();
2603 out.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
2604 out
2605}
2606
2607pub fn read_metadata_ranged<R: RangeReader>(reader: &R) -> Result<Option<Vec<u8>>, FileError> {
2617 let head = reader.read_at(0, HEADER_LEN as u64)?;
2618 if crate::v06::open::is_v06(&head) {
2619 let h = crate::v06::Header06::from_bytes(&head)?;
2620 return Ok(crate::v06::open::read_card_build(reader, &h)?.0);
2621 }
2622 let header = Header::from_bytes(&head)?;
2623 if header.metadata_len == 0 {
2624 return Ok(None);
2625 }
2626 let bytes = reader.read_at(header.metadata_offset, header.metadata_len)?;
2627 Ok(Some(bytes))
2628}
2629
2630pub fn read_card_full_ranged<R: RangeReader>(reader: &R) -> Result<Option<Vec<u8>>, FileError> {
2635 let head = reader.read_at(0, HEADER_LEN as u64)?;
2636 if crate::v06::open::is_v06(&head) {
2637 return Ok(crate::v06::open::read_card_full(reader)?.0);
2638 }
2639 read_metadata_ranged(reader)
2640}
2641
2642#[allow(clippy::type_complexity)]
2652pub fn read_card_and_build_info_ranged<R: RangeReader>(
2653 reader: &R,
2654) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
2655 read_card_and_build_info_with_header(reader).map(|(_, m, b)| (m, b))
2656}
2657
2658#[allow(clippy::type_complexity)]
2669pub fn read_card_and_build_info_with_header<R: RangeReader>(
2670 reader: &R,
2671) -> Result<(Header, Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
2672 let head = reader.read_at(0, HEADER_LEN as u64)?;
2673 let header = Header::from_bytes(&head)?;
2674 if crate::v06::reads(header.version) {
2675 let h = crate::v06::Header06::from_bytes(&head)?;
2676 let (card, build) = crate::v06::open::read_card_build(reader, &h)?;
2677 return Ok((header, card, build));
2678 }
2679 let meta = (header.metadata_offset, header.metadata_len);
2680 let build = (header.build_info_offset, header.build_info_len);
2681 if meta.1 > 0 && build.1 > 0 && build.0 == meta.0 + meta.1 {
2682 let both = reader.read_at(meta.0, meta.1 + build.1)?;
2684 let (m, b) = usize::try_from(meta.1)
2685 .ok()
2686 .and_then(|mid| both.split_at_checked(mid))
2687 .ok_or(FileError::Container("short card read"))?;
2688 return Ok((header, Some(m.to_vec()), Some(b.to_vec())));
2689 }
2690 let fetch = |off: u64, len: u64| -> Result<Option<Vec<u8>>, FileError> {
2691 if len == 0 {
2692 return Ok(None);
2693 }
2694 Ok(Some(reader.read_at(off, len)?))
2695 };
2696 Ok((header, fetch(meta.0, meta.1)?, fetch(build.0, build.1)?))
2697}
2698
2699pub fn read_build_info(bytes: &[u8]) -> Result<Option<Vec<u8>>, FileError> {
2704 if crate::v06::open::is_v06(bytes) {
2705 let h = crate::v06::Header06::from_bytes(bytes)?;
2706 let r = crate::reader::SliceReader::new(bytes);
2707 return Ok(crate::v06::open::read_card_build(&r, &h)?.1);
2708 }
2709 let header = Header::from_bytes(bytes)?;
2710 if header.build_info_len == 0 {
2711 return Ok(None);
2712 }
2713 let section = checked_slice(bytes, header.build_info_offset, header.build_info_len)
2714 .ok_or(FileError::Container("build-info section overruns buffer"))?;
2715 Ok(Some(section.to_vec()))
2716}
2717
2718#[derive(Debug, Clone, PartialEq, Eq)]
2727pub struct BuildInfoPlan {
2728 pub header: [u8; HEADER_LEN],
2730 pub insert: u64,
2733 pub tail_start: u64,
2737 pub new_len: u64,
2739}
2740
2741pub fn plan_build_info(
2763 head: &[u8],
2764 file_len: u64,
2765 info_len: u64,
2766) -> Result<BuildInfoPlan, FileError> {
2767 let mut header = Header::from_bytes(head)?;
2768 let insert = if header.metadata_len > 0 {
2773 header
2774 .metadata_offset
2775 .checked_add(header.metadata_len)
2776 .filter(|&end| header.metadata_offset >= HEADER_LEN as u64 && end <= file_len)
2777 .ok_or(FileError::Container(
2778 "the metadata section lies outside the file",
2779 ))?
2780 } else {
2781 HEADER_LEN as u64
2782 };
2783 let old_len = header.build_info_len;
2784 if old_len > 0 && header.build_info_offset != insert {
2785 return Err(FileError::Container(
2786 "existing build-info section is not adjacent to the metadata",
2787 ));
2788 }
2789 let tail_start = insert.saturating_add(old_len);
2790 if tail_start > file_len || insert > file_len {
2791 return Err(FileError::Container("build-info splice out of bounds"));
2792 }
2793
2794 let shift = |off: &mut u64, len: u64| {
2796 if len > 0 && *off >= tail_start {
2797 *off = *off - old_len + info_len;
2798 }
2799 };
2800 shift(&mut header.dictionary_offset, header.dictionary_len);
2801 shift(&mut header.root_dir_offset, header.root_dir_len);
2802 shift(&mut header.pyramid_meta_offset, header.pyramid_meta_len);
2803 shift(&mut header.named_graphs_offset, header.named_graphs_len);
2804 shift(&mut header.text_index_offset, header.text_index_len);
2805 for s in &mut header.extra_sections {
2806 if s.length > 0 && s.offset >= tail_start {
2807 s.offset = s.offset - old_len + info_len;
2808 }
2809 }
2810 if info_len == 0 {
2811 header.build_info_offset = 0;
2812 header.build_info_len = 0;
2813 } else {
2814 header.build_info_offset = insert;
2815 header.build_info_len = info_len;
2816 }
2817
2818 Ok(BuildInfoPlan {
2819 header: header.to_bytes(),
2820 insert,
2821 tail_start,
2822 new_len: file_len - old_len + info_len,
2823 })
2824}
2825
2826pub fn attach_build_info(image: &[u8], info: &[u8]) -> Result<Vec<u8>, FileError> {
2832 if crate::v06::open::is_v06(image) {
2833 return crate::v06::write::replace_card_with(
2835 image,
2836 crate::v06::write::NewCard::Keep,
2837 (!info.is_empty()).then_some(info),
2838 );
2839 }
2840 let plan = plan_build_info(image, image.len() as u64, info.len() as u64)?;
2841 let mut out = Vec::with_capacity(plan.new_len as usize);
2842 out.extend_from_slice(&plan.header);
2843 out.extend_from_slice(&image[HEADER_LEN..plan.insert as usize]);
2844 out.extend_from_slice(info);
2845 out.extend_from_slice(&image[plan.tail_start as usize..]);
2846 debug_assert_eq!(out.len() as u64, plan.new_len);
2847 Ok(out)
2848}
2849
2850pub fn replace_metadata(image: &[u8], metadata: &[u8]) -> Result<Vec<u8>, FileError> {
2870 if crate::v06::open::is_v06(image) {
2871 let h = crate::v06::Header06::from_bytes(image)?;
2873 let (_, build) =
2874 crate::v06::open::read_card_build(&crate::reader::SliceReader::new(image), &h)?;
2875 return crate::v06::write::replace_card(
2876 image,
2877 (!metadata.is_empty()).then_some(metadata),
2878 build.as_deref(),
2879 );
2880 }
2881 let mut header = Header::from_bytes(image)?;
2882 let insert = HEADER_LEN as u64;
2885 if header.metadata_len > 0 && header.metadata_offset != insert {
2886 return Err(FileError::Container(
2887 "metadata section does not sit immediately after the header",
2888 ));
2889 }
2890 let old_len = header.metadata_len;
2891 let tail_start = (insert + old_len) as usize;
2892 if tail_start > image.len() {
2893 return Err(FileError::Container("metadata splice out of bounds"));
2894 }
2895 let new_len = metadata.len() as u64;
2896
2897 let shift = |off: &mut u64, len: u64| {
2898 if len > 0 && *off >= insert + old_len {
2899 *off = *off - old_len + new_len;
2900 }
2901 };
2902 shift(&mut header.dictionary_offset, header.dictionary_len);
2903 shift(&mut header.root_dir_offset, header.root_dir_len);
2904 shift(&mut header.pyramid_meta_offset, header.pyramid_meta_len);
2905 shift(&mut header.named_graphs_offset, header.named_graphs_len);
2906 shift(&mut header.text_index_offset, header.text_index_len);
2907 shift(&mut header.build_info_offset, header.build_info_len);
2908 for s in &mut header.extra_sections {
2909 if s.length > 0 && s.offset >= insert + old_len {
2910 s.offset = s.offset - old_len + new_len;
2911 }
2912 }
2913 header.metadata_offset = insert;
2914 header.metadata_len = new_len;
2915
2916 let mut out = Vec::with_capacity(image.len() - old_len as usize + metadata.len());
2917 out.extend_from_slice(&header.to_bytes());
2918 out.extend_from_slice(metadata);
2919 out.extend_from_slice(&image[tail_start..]);
2920 header.content_hash = hash_of_sections(&header, &out)?;
2924 out[..HEADER_LEN].copy_from_slice(&header.to_bytes());
2925 Ok(out)
2926}
2927
2928fn hash_of_sections(header: &Header, bytes: &[u8]) -> Result<[u8; 16], FileError> {
2934 let slice = |off: u64, len: u64| -> Result<&[u8], FileError> {
2935 checked_slice(bytes, off, len).ok_or(FileError::Container("section overruns buffer"))
2944 };
2945 let parts = content_hash_parts(header)
2946 .into_iter()
2947 .map(|(off, len)| slice(off, len))
2948 .collect::<Result<Vec<&[u8]>, FileError>>()?;
2949 Ok(content_hash(&parts))
2950}
2951
2952pub fn content_hash_parts(header: &Header) -> Vec<(u64, u64)> {
2965 let mut parts: Vec<(u64, u64)> = Vec::with_capacity(6);
2966 if header.metadata_len > 0 {
2967 parts.push((header.metadata_offset, header.metadata_len));
2968 }
2969 parts.push((header.dictionary_offset, header.dictionary_len));
2970 parts.push((header.root_dir_offset, header.root_dir_len));
2971 if header.pyramid_meta_len > 0 {
2974 parts.push((header.pyramid_meta_offset, header.pyramid_meta_len));
2975 }
2976 if header.text_index_len > 0 && !header.text_index_partial {
2978 parts.push((header.text_index_offset, header.text_index_len));
2979 }
2980 if header.named_graphs_len > 0 {
2981 parts.push((header.named_graphs_offset, header.named_graphs_len));
2982 }
2983 parts
2984}
2985
2986pub fn verify_ranged_v06(
2989 reader: &dyn crate::reader::RangeReader,
2990 head: &[u8],
2991 only: Option<u8>,
2992) -> Result<Vec<String>, FileError> {
2993 const SLICE: u64 = 8 << 20;
2994 let h = crate::v06::Header06::from_bytes(head)?;
2995 if h.file_len != reader.len() {
2996 return Ok(vec!["file length".to_string()]);
2997 }
2998 let mut bad = Vec::new();
2999 let root_bytes = reader.read_at(crate::v06::HEADER_LEN as u64, h.root_len as u64)?;
3000 if crate::v06::digest16(&root_bytes) != h.root_digest {
3001 bad.push("root".to_string());
3002 return Ok(bad);
3003 }
3004 let root = crate::v06::Root06::decode(&root_bytes, h.internal_codec)?;
3005 if crate::v06::content_hash(&h) != h.content_hash {
3006 bad.push("content hash".to_string());
3007 }
3008 let card = h.card_range();
3009 if only.is_none() && card.len > 0 {
3010 let stored = reader.read_at(card.offset, card.len)?;
3011 if crate::v06::digest16(&stored) != h.card_digest {
3012 bad.push("card".to_string());
3013 }
3014 }
3015 for s in &root.sections {
3016 if only.is_some_and(|k| k != s.kind) {
3017 continue;
3018 }
3019 let a = match s.abs(&h) {
3020 Ok(a) if a.offset + a.len <= h.file_len => a,
3021 _ => {
3022 bad.push(crate::v06::kind::name(s.kind).to_string());
3023 continue;
3024 }
3025 };
3026 let mut hs = blake3::Hasher::new();
3027 let mut at = a.offset;
3028 while at < a.offset + a.len {
3029 let n = (a.offset + a.len - at).min(SLICE);
3030 hs.update(&reader.read_at(at, n)?);
3031 at += n;
3032 }
3033 if hs.finalize().as_bytes()[..16] != s.digest {
3034 bad.push(crate::v06::kind::name(s.kind).to_string());
3035 }
3036 }
3037 if only.is_none() {
3038 let n = crate::v06::FOOTER.len() as u64;
3039 if reader.read_at(h.file_len - n, n)? != crate::v06::footer(h.version) {
3040 bad.push("footer".to_string());
3041 }
3042 }
3043 Ok(bad)
3044}
3045
3046pub fn verify(bytes: &[u8]) -> Result<bool, FileError> {
3054 if crate::v06::open::is_v06(bytes) {
3055 let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
3056 std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(bytes)));
3057 return crate::v06::open::V06File::open(r, bytes)?.verify(None);
3058 }
3059 let header = Header::from_bytes(bytes)?;
3060 if hash_of_sections(&header, bytes)? != header.content_hash {
3061 return Ok(false);
3062 }
3063 if header.has_quad_index() || header.quad_index_section().is_some() {
3066 check_named_layout(&header)?;
3067 let slice = |off: u64, len: u64| -> Result<&[u8], FileError> {
3068 checked_slice(bytes, off, len).ok_or(FileError::Container("section overruns buffer"))
3069 };
3070 let root = slice(header.root_dir_offset, header.root_dir_len)?;
3071 let ranges = container_section_payload_ranges(
3072 root,
3073 header.root_dir_offset,
3074 header.index_container_sections(),
3075 )?;
3076 let seal = ranges
3077 .last()
3078 .ok_or(FileError::Container("missing quad-index seal"))?;
3079 let want = crate::quad::parse_seal(slice(seal.offset, seal.len)?)?;
3080 let q = header
3081 .quad_index_section()
3082 .ok_or(FileError::Container("missing quad-index section"))?;
3083 if crate::quad::digest(slice(q.offset, q.length)?) != want {
3084 return Ok(false);
3085 }
3086 }
3087 for s in self_digested_sections(&header) {
3088 let section = checked_slice(bytes, s.offset, s.length)
3089 .ok_or(FileError::Container("section overruns buffer"))?;
3090 if !crate::label_index::digest_ok(section) {
3091 return Ok(false);
3092 }
3093 }
3094 Ok(true)
3095}
3096
3097fn self_digested_sections(header: &Header) -> Vec<Section> {
3101 [
3102 header.label_index_section(),
3103 header.schema_section(),
3104 header
3105 .section(crate::header::SectionKind::TextIndexPartial)
3106 .filter(|s| s.length > 0),
3107 ]
3108 .into_iter()
3109 .flatten()
3110 .collect()
3111}
3112
3113pub fn append_sections(
3121 image: &[u8],
3122 sections: &[(SectionKind, &[u8])],
3123) -> Result<Vec<u8>, FileError> {
3124 let lens: Vec<(SectionKind, u64)> =
3125 sections.iter().map(|(k, b)| (*k, b.len() as u64)).collect();
3126 let (header, body_end) = plan_append(image, image.len() as u64, &lens)?;
3127 let extra: usize = sections.iter().map(|(_, b)| b.len()).sum();
3128 let mut out = Vec::with_capacity(image.len() + extra);
3129 out.extend_from_slice(&header.to_bytes());
3130 out.extend_from_slice(&image[HEADER_LEN..body_end as usize]);
3131 for (_, b) in sections {
3132 out.extend_from_slice(b);
3133 }
3134 out.extend_from_slice(&MAGIC);
3135 Ok(out)
3136}
3137
3138fn plan_append(
3140 head: &[u8],
3141 file_len: u64,
3142 sections: &[(SectionKind, u64)],
3143) -> Result<(Header, u64), FileError> {
3144 let mut header = Header::from_bytes(head)?;
3145 if header.expected_file_len() != Some(file_len) {
3146 return Err(FileError::Container(
3147 "cannot append sections: the file length does not match its header",
3148 ));
3149 }
3150 let body_end = file_len - MAGIC.len() as u64;
3151 let mut at = body_end;
3152 for &(kind, len) in sections {
3153 if !matches!(kind, SectionKind::LabelIndex | SectionKind::Schema) {
3154 return Err(FileError::Container(
3155 "only LABEL_INDEX and SCHEMA sections can be appended",
3156 ));
3157 }
3158 if len == 0 {
3159 continue;
3160 }
3161 if header.section(kind).is_some_and(|s| s.length > 0) {
3162 return Err(FileError::Container(
3163 "the file already carries that section",
3164 ));
3165 }
3166 header = header.with_section(kind, at, len);
3167 at += len;
3168 }
3169 Ok((header, body_end))
3170}
3171
3172#[cfg(not(target_arch = "wasm32"))]
3177pub(crate) fn open_dictionary_lazy<R: RangeReader + Send + Sync + 'static>(
3178 reader: R,
3179) -> Result<Dictionary, FileError> {
3180 let head = reader.read_at(0, HEADER_LEN as u64)?;
3181 let header = Header::from_bytes(&head)?;
3182 ranged_chunked_dictionary(&std::sync::Arc::new(reader), &header, [true; 4])
3183}
3184
3185pub fn verify_ranged(reader: &dyn crate::reader::RangeReader) -> Result<bool, FileError> {
3191 const SLICE: u64 = 8 << 20;
3194 let len = reader.len();
3195 if len < HEADER_LEN as u64 {
3196 return Err(FileError::Container("file shorter than its header"));
3197 }
3198 let head = reader.read_at(0, HEADER_LEN as u64)?;
3199 if crate::v06::open::is_v06(&head) {
3200 return verify_ranged_v06(reader, &head, None).map(|bad| bad.is_empty());
3201 }
3202 let header = Header::from_bytes(&head)?;
3203 let mut h = blake3::Hasher::new();
3204 for (off, n) in content_hash_parts(&header) {
3205 if off.checked_add(n).is_none_or(|end| end > len) {
3208 return Err(FileError::Container("section overruns buffer"));
3209 }
3210 let mut at = off;
3211 while at < off + n {
3212 let take = (off + n - at).min(SLICE);
3213 h.update(&reader.read_at(at, take)?);
3214 at += take;
3215 }
3216 }
3217 let mut out = [0u8; 16];
3218 out.copy_from_slice(&h.finalize().as_bytes()[..16]);
3219 if out != header.content_hash {
3220 return Ok(false);
3221 }
3222 if header.has_quad_index() || header.quad_index_section().is_some() {
3225 check_named_layout(&header)?;
3226 let sections = header.index_container_sections();
3227 let seal = locate_container_section_ranged(
3228 reader,
3229 header.root_dir_offset,
3230 header.root_dir_len,
3231 sections - 1,
3232 sections as u64,
3233 )?;
3234 let want = crate::quad::parse_seal(&reader.read_at(seal.offset, seal.len)?)?;
3235 let q = header
3236 .quad_index_section()
3237 .ok_or(FileError::Container("missing quad-index section"))?;
3238 if q.offset.checked_add(q.length).is_none_or(|end| end > len) {
3239 return Err(FileError::Container("section overruns buffer"));
3240 }
3241 let mut h = blake3::Hasher::new();
3242 let mut at = q.offset;
3243 while at < q.offset + q.length {
3244 let take = (q.offset + q.length - at).min(SLICE);
3245 h.update(&reader.read_at(at, take)?);
3246 at += take;
3247 }
3248 let mut got = [0u8; 16];
3249 got.copy_from_slice(&h.finalize().as_bytes()[..16]);
3250 if got != want {
3251 return Ok(false);
3252 }
3253 }
3254 for s in self_digested_sections(&header) {
3256 if s.length < 16 || s.offset.checked_add(s.length).is_none_or(|end| end > len) {
3257 return Err(FileError::Container("section overruns buffer"));
3258 }
3259 let want = reader.read_at(s.offset, 16)?;
3260 let mut h = blake3::Hasher::new();
3261 let mut at = s.offset + 16;
3262 while at < s.offset + s.length {
3263 let take = (s.offset + s.length - at).min(SLICE);
3264 h.update(&reader.read_at(at, take)?);
3265 at += take;
3266 }
3267 if h.finalize().as_bytes()[..16] != want[..] {
3268 return Ok(false);
3269 }
3270 }
3271 Ok(true)
3272}
3273
3274type PyramidLoader = Box<dyn Fn() -> Option<PyramidMeta> + Send + Sync>;
3276
3277enum PyramidSlot {
3283 Resident(Option<PyramidMeta>),
3284 Lazy {
3285 loader: PyramidLoader,
3286 cell: std::sync::OnceLock<Option<PyramidMeta>>,
3287 },
3288}
3289
3290pub(crate) enum SectionSlot<T> {
3295 Resident(Option<T>),
3296 Lazy {
3297 loader: Box<dyn Fn() -> Option<T> + Send + Sync>,
3298 cell: std::sync::OnceLock<Option<T>>,
3299 },
3300}
3301
3302impl<T> SectionSlot<T> {
3303 pub(crate) fn get(&self) -> Option<&T> {
3305 match self {
3306 SectionSlot::Resident(v) => v.as_ref(),
3307 SectionSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
3308 }
3309 }
3310
3311 pub(crate) fn get_if_loaded(&self) -> Option<&T> {
3313 match self {
3314 SectionSlot::Resident(v) => v.as_ref(),
3315 SectionSlot::Lazy { cell, .. } => cell.get().and_then(|o| o.as_ref()),
3316 }
3317 }
3318}
3319
3320fn resident_label_slot<'a>(
3322 header: &Header,
3323 region: &dyn Fn(u64, u64) -> Result<&'a [u8], FileError>,
3324) -> SectionSlot<crate::label_index::LabelIndex> {
3325 SectionSlot::Resident(header.label_index_section().and_then(|s| {
3326 let bytes = region(s.offset, s.length).ok()?;
3327 crate::label_index::LabelIndex::from_section(bytes, header.block_codec).ok()
3328 }))
3329}
3330
3331fn resident_schema_slot<'a>(
3333 header: &Header,
3334 region: &dyn Fn(u64, u64) -> Result<&'a [u8], FileError>,
3335) -> SectionSlot<crate::schema_section::SchemaMeta> {
3336 SectionSlot::Resident(header.schema_section().and_then(|s| {
3337 crate::schema_section::SchemaMeta::decode(region(s.offset, s.length).ok()?).ok()
3338 }))
3339}
3340
3341fn ranged_label_slot<R: RangeReader + Send + Sync + 'static>(
3345 reader: &std::sync::Arc<R>,
3346 header: &Header,
3347) -> SectionSlot<crate::label_index::LabelIndex> {
3348 let Some(s) = header.label_index_section() else {
3349 return SectionSlot::Resident(None);
3350 };
3351 let r = reader.clone();
3352 let codec = header.block_codec;
3353 SectionSlot::Lazy {
3354 loader: Box::new(move || {
3355 let head = r
3356 .read_at(s.offset, crate::label_index::HEAD_READ.min(s.length))
3357 .ok()?;
3358 let rr = r.clone();
3359 let loader = Box::new(move |off: u64, len: u64| rr.read_at(s.offset + off, len).ok());
3360 crate::label_index::LabelIndex::from_head(&head, s.length, codec, loader).ok()
3361 }),
3362 cell: std::sync::OnceLock::new(),
3363 }
3364}
3365
3366fn ranged_schema_slot<R: RangeReader + Send + Sync + 'static>(
3368 reader: &std::sync::Arc<R>,
3369 header: &Header,
3370) -> SectionSlot<crate::schema_section::SchemaMeta> {
3371 let Some(s) = header.schema_section() else {
3372 return SectionSlot::Resident(None);
3373 };
3374 let r = reader.clone();
3375 SectionSlot::Lazy {
3376 loader: Box::new(move || {
3377 let bytes = r.read_at(s.offset, s.length).ok()?;
3378 crate::schema_section::SchemaMeta::decode(&bytes).ok()
3379 }),
3380 cell: std::sync::OnceLock::new(),
3381 }
3382}
3383
3384pub fn read_schema_ranged<R: RangeReader + ?Sized>(
3388 reader: &R,
3389) -> Result<Option<crate::schema_section::SchemaMeta>, FileError> {
3390 let head = reader.read_at(0, HEADER_LEN as u64)?;
3391 let header = Header::from_bytes(&head)?;
3392 Ok(read_schema_with_header(reader, &header))
3393}
3394
3395fn read_schema_with_header<R: RangeReader + ?Sized>(
3396 reader: &R,
3397 header: &Header,
3398) -> Option<crate::schema_section::SchemaMeta> {
3399 if crate::v06::reads(header.version) {
3400 let b = crate::v06::open::read_schema_bytes(reader).ok()??;
3401 return crate::schema_section::SchemaMeta::decode(&b).ok();
3402 }
3403 let s = header.schema_section()?;
3404 let bytes = reader.read_at(s.offset, s.length).ok()?;
3405 crate::schema_section::SchemaMeta::decode(&bytes).ok()
3406}
3407
3408type TextIndexLoader = Box<dyn Fn() -> Option<crate::text_index::TextIndex> + Send + Sync>;
3411
3412type TokenTableProbe = Box<dyn Fn() -> Option<u64> + Send + Sync>;
3415
3416enum TextIndexSlot {
3426 Resident {
3427 index: Option<crate::text_index::TextIndex>,
3428 token_table_len: Option<u64>,
3431 },
3432 Lazy {
3433 loader: TextIndexLoader,
3434 cell: std::sync::OnceLock<Option<crate::text_index::TextIndex>>,
3435 token_table: TokenTableProbe,
3437 token_table_cell: std::sync::OnceLock<Option<u64>>,
3438 },
3439}
3440
3441impl TextIndexSlot {
3442 fn index(&self) -> Option<&crate::text_index::TextIndex> {
3444 match self {
3445 TextIndexSlot::Resident { index, .. } => index.as_ref(),
3446 TextIndexSlot::Lazy { loader, cell, .. } => cell.get_or_init(loader).as_ref(),
3447 }
3448 }
3449
3450 fn token_table_len(&self) -> Option<u64> {
3453 match self {
3454 TextIndexSlot::Resident {
3455 token_table_len, ..
3456 } => *token_table_len,
3457 TextIndexSlot::Lazy {
3458 token_table,
3459 token_table_cell,
3460 ..
3461 } => *token_table_cell.get_or_init(token_table),
3462 }
3463 }
3464}
3465
3466fn resident_text_index_slot(section: Option<&[u8]>, codec: u8) -> Result<TextIndexSlot, FileError> {
3469 let Some(section) = section else {
3470 return Ok(TextIndexSlot::Resident {
3471 index: None,
3472 token_table_len: None,
3473 });
3474 };
3475 Ok(TextIndexSlot::Resident {
3476 index: Some(
3477 crate::text_index::TextIndex::from_section(section, codec)
3478 .map_err(|_| FileError::Container("malformed text index"))?,
3479 ),
3480 token_table_len: crate::text_index::TextIndex::postings_base(section).map(|b| b as u64),
3481 })
3482}
3483
3484pub fn read_text_index_token_table_len_ranged<R: RangeReader + ?Sized>(
3497 reader: &R,
3498 header: &Header,
3499) -> Option<u64> {
3500 if crate::v06::reads(header.version) {
3501 return crate::v06::open::read_token_table_bytes(reader);
3502 }
3503 let (ti_off, ti_len) = header.text_index_payload();
3504 if ti_len == 0 {
3505 return None;
3506 }
3507 read_token_table_len(reader, ti_off, ti_len)
3508}
3509
3510fn read_token_table_len<R: RangeReader + ?Sized>(reader: &R, off: u64, len: u64) -> Option<u64> {
3514 let head = reader.read_at(off, 10u64.min(len)).ok()?;
3515 let (ttlen, n) = crate::varint::read_uvarint(&head)?;
3516 Some((n as u64 + ttlen).min(len))
3517}
3518
3519const NAMED_GRAPH_RESIDENT_MAX: u64 = 1 << 20; pub const TRANSIENT_GRAPH_QUADS: u64 = 1024;
3531
3532const NAMED_SLAB: usize = 1024;
3537
3538const NAMED_WALK_CHUNK: u64 = 64 * 1024;
3542
3543const NAMED_WALK_CHUNK_MAX: u64 = 8 * 1024 * 1024;
3556
3557#[derive(Default)]
3559struct NamedEntry {
3560 meta: std::sync::OnceLock<(String, ByteRange)>,
3563 index: std::sync::OnceLock<Box<GraphIndex>>,
3566}
3567
3568#[derive(Default)]
3572struct NamedWalk {
3573 next: usize,
3575 pos: u64,
3579 buf: Vec<u8>,
3586 buf_off: u64,
3587 chunk: u64,
3594 big_seen: bool,
3601}
3602
3603struct LazyNamedGraphs {
3617 reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
3618 section: ByteRange,
3620 codec: u8,
3621 has_synopsis: bool,
3622 read_concurrency: usize,
3623 perms: PermSet,
3626 #[allow(clippy::type_complexity)]
3630 dir: std::sync::OnceLock<(usize, Box<[std::sync::OnceLock<Box<[NamedEntry]>>]>)>,
3631 walk: std::sync::Mutex<NamedWalk>,
3632 failed: std::sync::atomic::AtomicBool,
3633 tile_cap: std::sync::atomic::AtomicU64,
3639}
3640
3641impl LazyNamedGraphs {
3642 fn new(
3643 reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
3644 section: ByteRange,
3645 codec: u8,
3646 has_synopsis: bool,
3647 read_concurrency: usize,
3648 perms: PermSet,
3649 ) -> Self {
3650 LazyNamedGraphs {
3651 reader,
3652 section,
3653 codec,
3654 has_synopsis,
3655 read_concurrency,
3656 perms,
3657 dir: std::sync::OnceLock::new(),
3658 walk: std::sync::Mutex::new(NamedWalk::default()),
3659 failed: std::sync::atomic::AtomicBool::new(false),
3660 tile_cap: std::sync::atomic::AtomicU64::new(u64::MAX),
3661 }
3662 }
3663
3664 fn set_tile_cap(&self, cap: u64) {
3668 self.tile_cap
3669 .store(cap, std::sync::atomic::Ordering::Relaxed);
3670 self.for_each_opened(|g| {
3671 if g.is_lazy() {
3672 g.set_cache_cap(cap);
3673 }
3674 });
3675 }
3676
3677 fn fail(&self) {
3679 self.failed
3680 .store(true, std::sync::atomic::Ordering::Relaxed);
3681 }
3682
3683 #[allow(clippy::type_complexity)]
3685 fn directory(&self) -> Option<&(usize, Box<[std::sync::OnceLock<Box<[NamedEntry]>>]>)> {
3686 if let Some(d) = self.dir.get() {
3687 return Some(d);
3688 }
3689 let end = self.section.offset.checked_add(self.section.len)?;
3690 let (n, used) = match read_uvarint_at(&self.reader, self.section.offset, end) {
3691 Ok(v) => v,
3692 Err(_) => {
3693 self.fail();
3694 return None;
3695 }
3696 };
3697 if n > self.section.len / 2 {
3700 self.fail();
3701 return None;
3702 }
3703 let Ok(n) = usize::try_from(n) else {
3704 self.fail();
3705 return None;
3706 };
3707 let slabs = n.div_ceil(NAMED_SLAB);
3708 let table: Box<[std::sync::OnceLock<Box<[NamedEntry]>>]> =
3709 (0..slabs).map(|_| std::sync::OnceLock::new()).collect();
3710 let _ = self.dir.set((n, table));
3711 {
3712 let mut w = self.walk.lock().unwrap();
3715 if w.pos == 0 {
3716 w.pos = self.section.offset + used;
3717 }
3718 }
3719 self.dir.get()
3720 }
3721
3722 fn count(&self) -> usize {
3723 self.directory().map(|(n, _)| *n).unwrap_or(0)
3724 }
3725
3726 fn entry(&self, i: usize) -> Option<&NamedEntry> {
3728 let (n, table) = self.directory()?;
3729 if i >= *n {
3730 return None;
3731 }
3732 let slab = table[i / NAMED_SLAB].get_or_init(|| {
3733 let len = NAMED_SLAB.min(n - (i / NAMED_SLAB) * NAMED_SLAB);
3734 (0..len).map(|_| NamedEntry::default()).collect()
3735 });
3736 slab.get(i % NAMED_SLAB)
3737 }
3738
3739 fn next_read_len(w: &mut NamedWalk, pos: u64, end: u64, need: u64) -> u64 {
3747 let cur = w.chunk.max(NAMED_WALK_CHUNK);
3748 w.chunk = (cur * 2).min(NAMED_WALK_CHUNK_MAX);
3749 cur.max(need).min(end - pos)
3750 }
3751
3752 fn ensure_meta(&self, upto: usize, exhaustive: bool) -> Option<()> {
3767 let target = self.entry(upto)?;
3768 if target.meta.get().is_some() {
3769 return Some(());
3770 }
3771 let end = self.section.offset.checked_add(self.section.len)?;
3772 let mut w = self.walk.lock().unwrap();
3773 if exhaustive && !w.big_seen {
3774 w.chunk = NAMED_WALK_CHUNK_MAX;
3775 }
3776 while w.next <= upto {
3777 let pos = w.pos;
3778 if pos >= end {
3779 self.fail();
3781 return None;
3782 }
3783 let have =
3786 |b: &[u8], off: u64, need: u64| pos >= off && pos + need <= off + b.len() as u64;
3787 if !have(&w.buf, w.buf_off, 20.min(end - pos)) {
3788 let len = Self::next_read_len(&mut w, pos, end, 20.min(end - pos));
3789 w.buf = match self.reader.read_at(pos, len) {
3790 Ok(b) => b,
3791 Err(_) => {
3792 self.fail();
3793 return None;
3794 }
3795 };
3796 w.buf_off = pos;
3797 }
3798 let rel = (pos - w.buf_off) as usize;
3801 let Some((ilen, u1)) = w.buf.get(rel..).and_then(read_uvarint) else {
3802 self.fail();
3803 return None;
3804 };
3805 let Some(iri_end) = (u1 as u64)
3806 .checked_add(ilen)
3807 .and_then(|n| pos.checked_add(n))
3808 .filter(|&e| e <= end)
3809 else {
3810 self.fail(); return None;
3812 };
3813 let header_need = (iri_end - pos).saturating_add(10);
3815 if !have(&w.buf, w.buf_off, header_need.min(end - pos)) {
3816 let len = Self::next_read_len(&mut w, pos, end, header_need);
3817 w.buf = match self.reader.read_at(pos, len) {
3818 Ok(b) => b,
3819 Err(_) => {
3820 self.fail();
3821 return None;
3822 }
3823 };
3824 w.buf_off = pos;
3825 }
3826 let rel = (pos - w.buf_off) as usize;
3827 let istart = rel + u1;
3828 let Some(iend) = usize::try_from(ilen)
3829 .ok()
3830 .and_then(|l| istart.checked_add(l))
3831 .filter(|&e| e <= w.buf.len())
3832 else {
3833 self.fail(); return None;
3835 };
3836 let iri = String::from_utf8_lossy(&w.buf[istart..iend]).into_owned();
3837 let Some((clen, u2)) = read_uvarint(&w.buf[iend..]) else {
3838 self.fail();
3839 return None;
3840 };
3841 let cend = match iri_end
3842 .checked_add(u2 as u64)
3843 .and_then(|cstart| cstart.checked_add(clen))
3844 {
3845 Some(e) if e <= end => e,
3846 _ => {
3847 self.fail(); return None;
3849 }
3850 };
3851 if clen > NAMED_GRAPH_RESIDENT_MAX {
3852 w.big_seen = true;
3857 w.chunk = NAMED_WALK_CHUNK;
3858 }
3859 let range = ByteRange {
3860 offset: cend - clen,
3861 len: clen,
3862 };
3863 if let Some(e) = self.entry(w.next) {
3864 let _ = e.meta.set((iri, range));
3865 }
3866 w.pos = cend;
3867 w.next += 1;
3868 }
3869 Some(())
3870 }
3871
3872 fn name_at(&self, i: usize, exhaustive: bool) -> Option<&str> {
3875 self.ensure_meta(i, exhaustive)?;
3876 self.entry(i)?.meta.get().map(|(iri, _)| iri.as_str())
3877 }
3878
3879 fn graph_at(&self, i: usize, exhaustive: bool) -> Option<(&str, &GraphIndex)> {
3882 self.ensure_meta(i, exhaustive)?;
3883 let e = self.entry(i)?;
3884 let (iri, range) = e.meta.get()?;
3885 if e.index.get().is_none() {
3886 let opened = self.open_graph(*range)?;
3887 let _ = e.index.set(Box::new(opened));
3888 }
3889 Some((iri.as_str(), e.index.get()?.as_ref()))
3890 }
3891
3892 fn container_bytes(&self, range: ByteRange) -> Option<Vec<u8>> {
3896 {
3897 let w = self.walk.lock().unwrap();
3898 let buf_end = w.buf_off + w.buf.len() as u64;
3899 if range.offset >= w.buf_off && range.offset + range.len <= buf_end {
3900 let a = (range.offset - w.buf_off) as usize;
3901 return Some(w.buf[a..a + range.len as usize].to_vec());
3902 }
3903 }
3904 match self.reader.read_at(range.offset, range.len) {
3905 Ok(b) if b.len() as u64 == range.len => Some(b),
3907 _ => {
3908 self.fail();
3909 None
3910 }
3911 }
3912 }
3913
3914 fn open_graph(&self, range: ByteRange) -> Option<GraphIndex> {
3919 if range.len <= NAMED_GRAPH_RESIDENT_MAX {
3920 let bytes = self.container_bytes(range)?;
3921 match decode_index_container(&bytes, self.codec, self.perms) {
3922 Ok(g) => {
3923 g.set_size_hint(u64::from(g.triple_count()));
3926 Some(g)
3927 }
3928 Err(_) => {
3929 self.fail();
3930 None
3931 }
3932 }
3933 } else {
3934 match open_index_container_lazy(
3935 &self.reader,
3936 range,
3937 self.codec,
3938 self.has_synopsis,
3939 self.read_concurrency,
3940 self.perms,
3941 self.perms.len(),
3942 ) {
3943 Ok((g, _, _)) => {
3944 let cap = self.tile_cap.load(std::sync::atomic::Ordering::Relaxed);
3948 if cap != u64::MAX {
3949 g.set_cache_cap(cap);
3950 }
3951 Some(g)
3952 }
3953 Err(_) => {
3954 self.fail();
3955 None
3956 }
3957 }
3958 }
3959 }
3960
3961 fn release(&mut self, iri: &str) {
3967 let Some((_, table)) = self.dir.get_mut() else {
3968 return;
3969 };
3970 for slab in table.iter_mut() {
3971 let Some(entries) = slab.get_mut() else {
3972 continue;
3973 };
3974 for e in entries.iter_mut() {
3975 let matches = e
3976 .meta
3977 .get()
3978 .map(|(name, _)| name.as_str() == iri)
3979 .unwrap_or(false);
3980 if matches {
3981 e.index.take();
3982 return;
3983 }
3984 }
3985 }
3986 }
3987
3988 fn for_each_opened(&self, mut f: impl FnMut(&GraphIndex)) {
3990 if let Some((_, table)) = self.dir.get() {
3991 for slab in table.iter().filter_map(|s| s.get()) {
3992 for e in slab.iter() {
3993 if let Some(g) = e.index.get() {
3994 f(g);
3995 }
3996 }
3997 }
3998 }
3999 }
4000}
4001
4002enum NamedGraphsSlot {
4007 Resident(Vec<(String, GraphIndex)>),
4008 Lazy(LazyNamedGraphs),
4009 Quad(crate::quad::QuadNamed),
4012}
4013
4014impl NamedGraphsSlot {
4015 fn directory_bytes(&self) -> u64 {
4019 match self {
4020 NamedGraphsSlot::Resident(v) => v
4021 .iter()
4022 .map(|(iri, g)| iri.capacity() as u64 + g.directory_bytes())
4023 .sum(),
4024 NamedGraphsSlot::Lazy(l) => {
4025 let mut bytes = l.walk.lock().map(|w| w.buf.capacity() as u64).unwrap_or(0);
4026 if let Some((_, table)) = l.dir.get() {
4027 bytes += table.len() as u64
4028 * std::mem::size_of::<std::sync::OnceLock<Box<[NamedEntry]>>>() as u64;
4029 for slab in table.iter().filter_map(|s| s.get()) {
4030 bytes += slab.len() as u64 * std::mem::size_of::<NamedEntry>() as u64;
4031 for e in slab.iter() {
4032 if let Some((iri, _)) = e.meta.get() {
4033 bytes += iri.capacity() as u64;
4034 }
4035 if let Some(g) = e.index.get() {
4036 bytes += g.directory_bytes();
4037 }
4038 }
4039 }
4040 }
4041 bytes
4042 }
4043 NamedGraphsSlot::Quad(q) => q.directory_bytes(),
4044 }
4045 }
4046
4047 fn count(&self) -> usize {
4048 match self {
4049 NamedGraphsSlot::Resident(v) => v.len(),
4050 NamedGraphsSlot::Lazy(l) => l.count(),
4051 NamedGraphsSlot::Quad(q) => q.count(),
4052 }
4053 }
4054
4055 fn name_at<'a>(&'a self, dict: &Dictionary, i: usize, exhaustive: bool) -> Option<&'a str> {
4059 match self {
4060 NamedGraphsSlot::Resident(v) => v.get(i).map(|(iri, _)| iri.as_str()),
4061 NamedGraphsSlot::Lazy(l) => l.name_at(i, exhaustive),
4062 NamedGraphsSlot::Quad(q) => q.name_at(dict, i, exhaustive),
4063 }
4064 }
4065
4066 fn name_owned(&self, dict: &Dictionary, i: usize, exhaustive: bool) -> Option<String> {
4069 match self {
4070 NamedGraphsSlot::Quad(q) => q.name_owned(dict, i, exhaustive),
4071 _ => self.name_at(dict, i, exhaustive).map(str::to_owned),
4072 }
4073 }
4074
4075 fn graph_at<'a>(
4076 &'a self,
4077 dict: &Dictionary,
4078 i: usize,
4079 exhaustive: bool,
4080 ) -> Option<(&'a str, &'a GraphIndex)> {
4081 match self {
4082 NamedGraphsSlot::Resident(v) => v.get(i).map(|(iri, g)| (iri.as_str(), g)),
4083 NamedGraphsSlot::Lazy(l) => l.graph_at(i, exhaustive),
4084 NamedGraphsSlot::Quad(q) => q.graph_at(dict, i, exhaustive),
4085 }
4086 }
4087
4088 fn find<'a>(&'a self, dict: &Dictionary, iri: &str) -> Option<&'a GraphIndex> {
4089 match self {
4090 NamedGraphsSlot::Quad(q) => q.find(dict, iri),
4091 NamedGraphsSlot::Resident(v) => v.iter().find(|(name, _)| name == iri).map(|(_, g)| g),
4092 NamedGraphsSlot::Lazy(l) => {
4093 for i in 0..l.count() {
4098 if l.name_at(i, false)? == iri {
4099 return l.graph_at(i, false).map(|(_, g)| g);
4100 }
4101 }
4102 None
4103 }
4104 }
4105 }
4106
4107 fn load_incomplete(&self) -> bool {
4108 match self {
4109 NamedGraphsSlot::Resident(v) => v.iter().any(|(_, g)| g.load_incomplete()),
4110 NamedGraphsSlot::Lazy(l) => {
4111 if l.failed.load(std::sync::atomic::Ordering::Relaxed) {
4112 return true;
4113 }
4114 let mut bad = false;
4115 l.for_each_opened(|g| bad |= g.load_incomplete());
4116 bad
4117 }
4118 NamedGraphsSlot::Quad(q) => q.load_incomplete(),
4119 }
4120 }
4121
4122 fn reset_load_failures(&self) {
4123 match self {
4124 NamedGraphsSlot::Resident(v) => {
4125 for (_, g) in v {
4126 g.reset_load_failure();
4127 }
4128 }
4129 NamedGraphsSlot::Lazy(l) => {
4130 l.failed.store(false, std::sync::atomic::Ordering::Relaxed);
4131 l.for_each_opened(|g| g.reset_load_failure());
4132 }
4133 NamedGraphsSlot::Quad(q) => q.reset_load_failures(),
4134 }
4135 }
4136
4137 fn resident(
4141 header: &Header,
4142 quad_section: impl FnOnce(u64, u64) -> Result<Vec<u8>, FileError>,
4143 per_graph: impl FnOnce() -> Result<Vec<(String, GraphIndex)>, FileError>,
4144 ) -> Result<Self, FileError> {
4145 check_named_layout(header)?;
4146 if let Some(q) = header.quad_index_section() {
4147 let bytes = quad_section(q.offset, q.length)?;
4148 return Ok(NamedGraphsSlot::Quad(crate::quad::QuadNamed::resident(
4149 bytes,
4150 header.block_codec,
4151 header.perms,
4152 )));
4153 }
4154 Ok(NamedGraphsSlot::Resident(if header.named_graphs_len > 0 {
4155 per_graph()?
4156 } else {
4157 Vec::new()
4158 }))
4159 }
4160}
4161
4162pub struct Rete {
4164 header: Header,
4165 dict: Dictionary,
4166 index: std::sync::Arc<GraphIndex>,
4167 index_section_ranges: [ByteRange; NUM_PERMS],
4168 tile_ranges: [Vec<(u32, u32, ByteRange)>; NUM_PERMS],
4172 pyramid: PyramidSlot,
4173 text_index: TextIndexSlot,
4174 labels: SectionSlot<crate::label_index::LabelIndex>,
4176 schema: SectionSlot<crate::schema_section::SchemaMeta>,
4178 named_graphs: NamedGraphsSlot,
4179 metadata: Vec<u8>,
4184 service_client: Option<Box<dyn crate::service::ServiceClient>>,
4189 service_error: std::sync::Mutex<Option<String>>,
4193 spill_threshold: std::sync::atomic::AtomicU64,
4196 remote_reads: std::sync::atomic::AtomicBool,
4199 v06: Option<std::sync::Arc<crate::v06::open::V06File>>,
4201}
4202
4203impl Rete {
4204 pub fn open(bytes: &[u8]) -> Result<Self, FileError> {
4207 #[cfg(feature = "v06-parity")]
4208 if let Some(v6) = parity_image(bytes) {
4209 return Self::open(&v6);
4210 }
4211 if crate::v06::open::is_v06(bytes) {
4212 let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
4215 std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(bytes)));
4216 let mut rete = Self::open_v06(r, bytes, false)?;
4217 rete.metadata = rete
4218 .v06
4219 .as_ref()
4220 .and_then(|f| f.card().ok().flatten())
4221 .unwrap_or_default();
4222 return Ok(rete);
4223 }
4224 let header = Header::from_bytes(bytes)?;
4225
4226 let region = |off: u64, len: u64| -> Result<&[u8], FileError> {
4230 checked_slice(bytes, off, len)
4231 .ok_or(FileError::Container("section range out of bounds"))
4232 };
4233
4234 let dict = decode_dictionary_container(
4235 region(header.dictionary_offset, header.dictionary_len)?,
4236 header.dict_codec,
4237 header.term_order(),
4238 )?;
4239
4240 check_named_layout(&header)?;
4241 let index_bytes = region(header.root_dir_offset, header.root_dir_len)?;
4242 let (index, _) = decode_root_index_container(index_bytes, &header)?;
4243 let index_section_ranges = decode_index_section_ranges(
4244 index_bytes,
4245 header.root_dir_offset,
4246 header.perms,
4247 header.index_container_sections(),
4248 )?;
4249
4250 let pyramid = PyramidSlot::Resident(if header.pyramid_meta_len > 0 {
4251 Some(
4252 PyramidMeta::decode(region(header.pyramid_meta_offset, header.pyramid_meta_len)?)
4253 .map_err(|_| FileError::Container("malformed pyramid meta"))?,
4254 )
4255 } else {
4256 None
4257 });
4258
4259 let (ti_off, ti_len) = header.text_index_payload();
4262 let text_index = resident_text_index_slot(
4263 if ti_len > 0 {
4264 Some(region(ti_off, ti_len)?)
4265 } else {
4266 None
4267 },
4268 header.block_codec,
4269 )?;
4270 let labels = resident_label_slot(&header, ®ion);
4271 let schema = resident_schema_slot(&header, ®ion);
4272
4273 let named_graphs = NamedGraphsSlot::resident(
4274 &header,
4275 |off, len| Ok(region(off, len)?.to_vec()),
4276 || {
4277 decode_named_graphs(
4278 region(header.named_graphs_offset, header.named_graphs_len)?,
4279 header.block_codec,
4280 header.perms,
4281 )
4282 },
4283 )?;
4284
4285 let metadata = if header.metadata_len > 0 {
4286 region(header.metadata_offset, header.metadata_len)?.to_vec()
4287 } else {
4288 Vec::new()
4289 };
4290
4291 let tile_ranges =
4292 tile_file_ranges(index_bytes, header.root_dir_offset, &index_section_ranges);
4293 Ok(Self {
4294 header,
4295 dict,
4296 index: std::sync::Arc::new(index),
4297 index_section_ranges,
4298 tile_ranges,
4299 pyramid,
4300 text_index,
4301 labels,
4302 schema,
4303 named_graphs,
4304 metadata,
4305 service_client: None,
4306 service_error: std::sync::Mutex::new(None),
4307 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
4308 remote_reads: std::sync::atomic::AtomicBool::new(false),
4309 v06: None,
4310 })
4311 }
4312
4313 pub fn set_service_client(&mut self, client: Box<dyn crate::service::ServiceClient>) {
4318 self.service_client = Some(client);
4319 }
4320
4321 pub(crate) fn service_client(&self) -> Option<&dyn crate::service::ServiceClient> {
4322 self.service_client.as_deref()
4323 }
4324
4325 pub(crate) fn record_service_error(&self, msg: &str) {
4327 let mut e = self.service_error.lock().unwrap();
4328 if e.is_none() {
4329 *e = Some(msg.to_string());
4330 }
4331 }
4332
4333 pub(crate) fn take_service_error(&self) -> Option<String> {
4336 self.service_error.lock().unwrap().take()
4337 }
4338
4339 pub fn header(&self) -> &Header {
4340 &self.header
4341 }
4342
4343 pub fn file_layout(&self) -> Vec<LayoutSegment> {
4349 if let Some(f) = &self.v06 {
4350 return v06_layout(f);
4351 }
4352 let h = &self.header;
4353 let seg = |kind: &'static str, label: String, offset: u64, len: u64| LayoutSegment {
4354 kind,
4355 label,
4356 offset,
4357 len,
4358 };
4359 let mut out = vec![seg(
4365 "header",
4366 format!("header (fixed {} bytes)", crate::header::HEADER_LEN),
4367 0,
4368 crate::header::HEADER_LEN as u64,
4369 )];
4370 if h.metadata_len > 0 {
4371 out.push(seg(
4372 "metadata",
4373 "metadata (dataset card)".into(),
4374 h.metadata_offset,
4375 h.metadata_len,
4376 ));
4377 }
4378 out.push(seg(
4379 "dictionary",
4380 "dictionary (4 front-coded term sections)".into(),
4381 h.dictionary_offset,
4382 h.dictionary_len,
4383 ));
4384 for (si, perm) in crate::index::ALL_PERMS.into_iter().enumerate() {
4385 let sec = self.index_section_ranges[si];
4386 if sec.len == 0 {
4387 continue;
4388 }
4389 let first_tile = self.tile_ranges[si]
4390 .first()
4391 .map(|&(_, _, r)| r.offset)
4392 .unwrap_or(sec.offset + sec.len);
4393 if first_tile > sec.offset {
4394 out.push(seg(
4395 "directory",
4396 format!("{} tile directory", perm.name()),
4397 sec.offset,
4398 first_tile - sec.offset,
4399 ));
4400 }
4401 for (ti, &(min_a, max_a, r)) in self.tile_ranges[si].iter().enumerate() {
4402 out.push(seg(
4403 "tile",
4404 format!("{} tile {ti} (leading ids {min_a}..{max_a})", perm.name()),
4405 r.offset,
4406 r.len,
4407 ));
4408 }
4409 }
4410 if h.pyramid_meta_len > 0 {
4411 out.push(seg(
4412 "pyramid",
4413 "pyramid summary (communities + superedges)".into(),
4414 h.pyramid_meta_offset,
4415 h.pyramid_meta_len,
4416 ));
4417 }
4418 if h.named_graphs_len > 0 {
4419 out.push(seg(
4420 "named-graphs",
4421 format!("named graphs ({})", self.named_graphs.count()),
4422 h.named_graphs_offset,
4423 h.named_graphs_len,
4424 ));
4425 }
4426 if let Some(s) = h.label_index_section() {
4427 let label = match self.labels.get_if_loaded() {
4428 Some(li) => format!("label index ({} entries)", li.entry_count()),
4429 None => "label index".to_string(),
4430 };
4431 out.push(seg("label-index", label, s.offset, s.length));
4432 }
4433 if let Some(s) = h.schema_section() {
4434 out.push(seg(
4435 "schema",
4436 "schema (classes, query_stats, shapes)".into(),
4437 s.offset,
4438 s.length,
4439 ));
4440 }
4441 if let Some(q) = h.quad_index_section() {
4442 out.push(seg(
4443 "quad-index",
4444 format!("quad index ({} named graphs)", self.named_graphs.count()),
4445 q.offset,
4446 q.length,
4447 ));
4448 }
4449 out.sort_by_key(|s| s.offset);
4450 out
4451 }
4452
4453 pub fn metadata(&self) -> Option<&[u8]> {
4458 if self.metadata.is_empty() {
4459 None
4460 } else {
4461 Some(&self.metadata)
4462 }
4463 }
4464
4465 pub fn dictionary(&self) -> &Dictionary {
4466 &self.dict
4467 }
4468
4469 pub fn pyramid(&self) -> Option<&PyramidMeta> {
4471 match &self.pyramid {
4472 PyramidSlot::Resident(p) => p.as_ref(),
4473 PyramidSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
4475 }
4476 }
4477
4478 pub fn pyramid_if_loaded(&self) -> Option<&PyramidMeta> {
4483 match &self.pyramid {
4484 PyramidSlot::Resident(p) => p.as_ref(),
4485 PyramidSlot::Lazy { cell, .. } => cell.get().and_then(|o| o.as_ref()),
4486 }
4487 }
4488
4489 pub fn schema(&self) -> Option<&crate::schema_section::SchemaMeta> {
4495 self.schema.get()
4496 }
4497
4498 pub fn schema_or_pyramid(
4502 &self,
4503 ) -> Option<std::borrow::Cow<'_, crate::schema_section::SchemaMeta>> {
4504 if let Some(s) = self.schema() {
4505 return Some(std::borrow::Cow::Borrowed(s));
4506 }
4507 self.pyramid()
4508 .map(|p| std::borrow::Cow::Owned(crate::schema_section::SchemaMeta::from_pyramid(p)))
4509 }
4510
4511 pub fn predicate_stats(&self) -> &[crate::meta::PredStat] {
4516 if let Some(s) = self.schema().filter(|s| !s.predicate_stats.is_empty()) {
4517 return &s.predicate_stats;
4518 }
4519 self.pyramid_if_loaded()
4520 .map(|p| p.predicate_stats.as_slice())
4521 .unwrap_or(&[])
4522 }
4523
4524 pub(crate) fn planner_stats(&self, fault: bool) -> &[crate::meta::PredStat] {
4529 let schema = if fault {
4530 self.schema.get()
4531 } else {
4532 self.schema.get_if_loaded()
4533 };
4534 if let Some(s) = schema.filter(|s| !s.predicate_stats.is_empty()) {
4535 return &s.predicate_stats;
4536 }
4537 self.pyramid_if_loaded()
4538 .map(|p| p.predicate_stats.as_slice())
4539 .unwrap_or(&[])
4540 }
4541
4542 pub fn char_sets(&self) -> &[crate::meta::CharSet] {
4545 if let Some(s) = self.schema().filter(|s| !s.char_sets.is_empty()) {
4546 return &s.char_sets;
4547 }
4548 self.pyramid_if_loaded()
4549 .map(|p| p.char_sets.as_slice())
4550 .unwrap_or(&[])
4551 }
4552
4553 pub fn label_index(&self) -> &[crate::meta::LabelEntry] {
4559 self.pyramid_if_loaded()
4560 .map(|p| p.label_index.as_slice())
4561 .unwrap_or(&[])
4562 }
4563
4564 pub fn has_label_index(&self) -> bool {
4566 self.header.label_index_section().is_some()
4567 }
4568
4569 pub fn label_index_section(&self) -> Option<&crate::label_index::LabelIndex> {
4571 self.labels.get()
4572 }
4573
4574 pub fn prefix_search(&self, prefix: &str, limit: usize) -> Vec<(String, String)> {
4581 if let Some(li) = self.labels.get() {
4582 return li.prefix_search(prefix, limit);
4583 }
4584 let Some(pyr) = self.pyramid() else {
4585 return Vec::new();
4586 };
4587 resolve_label_hits(&self.dict, pyr.prefix_search(prefix, limit))
4588 }
4589
4590 pub(crate) fn text_index(&self) -> Option<&crate::text_index::TextIndex> {
4593 self.text_index.index()
4594 }
4595
4596 pub fn has_text_index(&self) -> bool {
4599 self.header.text_index_len > 0
4600 }
4601
4602 pub fn text_index_scope(&self) -> Option<crate::text_index::TextIndexScope> {
4606 self.text_index().map(|ti| ti.scope().clone())
4607 }
4608
4609 pub fn text_index_token_table_len(&self) -> Option<u64> {
4624 self.text_index.token_table_len()
4625 }
4626
4627 pub fn text_search(&self, words: &[&str], prefix: Option<&str>, limit: usize) -> Vec<String> {
4637 text_search_in(self.text_index(), &self.dict, words, prefix, limit)
4638 }
4639
4640 pub fn default_index(&self) -> &GraphIndex {
4642 &self.index
4643 }
4644
4645 pub fn dump(&self, graph: Option<&str>) -> Vec<TermTriple> {
4647 let index = match graph {
4648 None => &*self.index,
4649 Some(g) => match self.graph_index(g) {
4650 Some(i) => i,
4651 None => return Vec::new(),
4652 },
4653 };
4654 index
4655 .match_pattern((None, None, None))
4656 .into_iter()
4657 .filter_map(|(s, p, o)| {
4658 Some((
4659 self.dict.subject_term(s)?,
4660 self.dict.predicate_term(p)?,
4661 self.dict.object_term(o)?,
4662 ))
4663 })
4664 .collect()
4665 }
4666
4667 pub fn dump_each<F: FnMut(&str, &str, &str)>(&self, graph: Option<&str>, f: F) {
4683 self.dump_filtered_each(graph, None, None, None, f)
4684 }
4685
4686 pub fn dump_filtered_each<F: FnMut(&str, &str, &str)>(
4757 &self,
4758 graph: Option<&str>,
4759 s: Option<&str>,
4760 p: Option<&str>,
4761 o: Option<&str>,
4762 mut f: F,
4763 ) {
4764 let index = match graph {
4765 None => Some(&*self.index),
4766 Some(g) => self.graph_index(g),
4767 };
4768 let Some(index) = index else { return };
4769 let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
4772 return;
4773 };
4774 let dict = &self.dict;
4775 const WINDOW: usize = 4096;
4820 let cap = dict.cache_cap();
4831 let prefetch_window =
4832 cap == u64::MAX || cap >= (WINDOW as u64) * 2 * (DICT_CHUNK_BUDGET as u64);
4833 const PREFETCH_BATCH: usize = 64;
4840 let prefetch_batch = if cap == u64::MAX {
4841 usize::MAX
4842 } else {
4843 PREFETCH_BATCH
4844 };
4845 let mut ids: Vec<(u32, u32, u32)> = Vec::with_capacity(WINDOW);
4846 let mut nodes: Vec<u32> = Vec::with_capacity(WINDOW * 2);
4847 let mut preds: Vec<u32> = Vec::with_capacity(WINDOW);
4848 let mut resolver = crate::dictionary::WindowResolver::default();
4849 let mut flush = |ids: &mut Vec<(u32, u32, u32)>,
4850 resolver: &mut crate::dictionary::WindowResolver,
4851 f: &mut F| {
4852 if ids.is_empty() {
4853 return;
4854 }
4855 if prefetch_window {
4859 nodes.clear();
4860 preds.clear();
4861 for &(s, p, o) in ids.iter() {
4862 nodes.push(dict.subject_node(s));
4863 nodes.push(dict.object_node(o));
4864 preds.push(p);
4865 }
4866 dict.prefetch_terms_batched(&nodes, &preds, prefetch_batch);
4867 }
4868 dict.resolve_window(ids, resolver, |s, p, o| f(s, p, o));
4869 ids.clear();
4870 };
4871 for t in index.scan_iter(pattern) {
4872 ids.push(t);
4873 if ids.len() == WINDOW {
4874 flush(&mut ids, &mut resolver, &mut f);
4875 }
4876 }
4877 flush(&mut ids, &mut resolver, &mut f);
4878 }
4879
4880 pub fn dump_plan(
4888 &self,
4889 graph: Option<&str>,
4890 s: Option<&str>,
4891 p: Option<&str>,
4892 o: Option<&str>,
4893 ) -> DumpPlan {
4894 let index = match graph {
4895 None => Some(&*self.index),
4896 Some(g) => self.graph_index(g),
4897 };
4898 let scan = index
4899 .zip(self.resolve_query_pattern(s, p, o))
4900 .map(|(ix, pattern)| ix.scan_plan(pattern));
4901 DumpPlan {
4902 scan,
4903 dictionary_bytes: match &self.v06 {
4904 Some(f) => f.section(crate::v06::kind::DICT_DATA).map_or(0, |r| r.len),
4905 None => self.header.dictionary_len,
4906 },
4907 }
4908 }
4909
4910 pub fn dump_iter(&self, graph: Option<&str>) -> impl Iterator<Item = TermTriple> + '_ {
4928 self.query_iter(graph, None, None, None)
4929 }
4930
4931 pub fn query_iter(
4942 &self,
4943 graph: Option<&str>,
4944 s: Option<&str>,
4945 p: Option<&str>,
4946 o: Option<&str>,
4947 ) -> impl Iterator<Item = TermTriple> + '_ {
4948 let index = match graph {
4949 None => Some(&*self.index),
4950 Some(g) => self.graph_index(g),
4951 };
4952 index
4953 .zip(self.resolve_query_pattern(s, p, o))
4954 .into_iter()
4955 .flat_map(|(ix, pattern)| ix.scan_iter(pattern))
4956 .filter_map(move |(s, p, o)| {
4957 Some((
4958 self.dict.subject_term(s)?,
4959 self.dict.predicate_term(p)?,
4960 self.dict.object_term(o)?,
4961 ))
4962 })
4963 }
4964
4965 pub fn query_batch(
4984 &self,
4985 graph: Option<&str>,
4986 s: Option<&str>,
4987 p: Option<&str>,
4988 o: Option<&str>,
4989 cursor: u64,
4990 max_quads: usize,
4991 ) -> (Vec<TermTriple>, u64, bool) {
4992 let index = match graph {
4993 None => &*self.index,
4994 Some(g) => match self.graph_index(g) {
4997 Some(i) => i,
4998 None => return (Vec::new(), cursor, true),
4999 },
5000 };
5001 let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
5004 return (Vec::new(), cursor, true);
5005 };
5006 let (ids, next, done) = index.scan_batch(pattern, cursor, max_quads);
5007 if !ids.is_empty() {
5011 let mut nodes = Vec::with_capacity(ids.len() * 2);
5012 let mut preds = Vec::with_capacity(ids.len());
5013 for &(s, p, o) in &ids {
5014 nodes.push(self.dict.subject_node(s));
5015 nodes.push(self.dict.object_node(o));
5016 preds.push(p);
5017 }
5018 self.dict.prefetch_terms(&nodes, &preds);
5019 }
5020 let triples = ids
5021 .into_iter()
5022 .filter_map(|(s, p, o)| {
5023 Some((
5024 self.dict.subject_term(s)?,
5025 self.dict.predicate_term(p)?,
5026 self.dict.object_term(o)?,
5027 ))
5028 })
5029 .collect();
5030 (triples, next, done)
5031 }
5032
5033 pub fn dump_batch(
5059 &self,
5060 graph: Option<&str>,
5061 cursor: u32,
5062 max_quads: usize,
5063 ) -> (Vec<TermTriple>, u32, bool) {
5064 let index = match graph {
5065 None => self.default_index(),
5066 Some(g) => match self.graph_index(g) {
5069 Some(i) => i,
5070 None => return (Vec::new(), cursor, true),
5071 },
5072 };
5073 let spo = IndexPermutation::Spo.section_index();
5077 let Some((first, last)) = index.leading_bounds(spo) else {
5078 return (Vec::new(), cursor, true); };
5080
5081 let mut sid = cursor.max(first);
5082 let mut probes = max_quads.saturating_mul(4).max(1 << 16);
5083 let mut ids: Vec<(u32, u32, u32)> = Vec::new();
5084 let mut exhausted = false;
5085 while sid <= last && probes > 0 {
5086 probes -= 1;
5087 let hits = index.match_pattern((Some(sid), None, None));
5088 if hits.is_empty() {
5089 match index.next_leading(IndexPermutation::Spo.section_index(), sid) {
5099 Some(next) if next > sid => {
5100 sid = next;
5101 continue;
5102 }
5103 None if !index.load_incomplete() => {
5104 exhausted = true;
5105 break;
5106 }
5107 _ => {}
5108 }
5109 match index.next_tile_start(spo, sid) {
5110 None => {
5111 exhausted = true;
5112 break;
5113 }
5114 Some(next) if next > sid => {
5115 sid = next;
5116 continue;
5117 }
5118 Some(_) => {}
5119 }
5120 } else {
5121 ids.extend(hits);
5122 }
5123 if sid == u32::MAX {
5124 exhausted = true;
5125 break;
5126 }
5127 sid += 1;
5128 if ids.len() >= max_quads {
5129 break;
5130 }
5131 }
5132 let done = exhausted || sid > last;
5133
5134 let dict = self.dictionary();
5141 if !ids.is_empty() {
5142 let mut nodes = Vec::with_capacity(ids.len() * 2);
5143 let mut preds = Vec::with_capacity(ids.len());
5144 for &(s, p, o) in &ids {
5145 nodes.push(dict.subject_node(s));
5146 nodes.push(dict.object_node(o));
5147 preds.push(p);
5148 }
5149 dict.prefetch_terms(&nodes, &preds);
5150 }
5151 let triples = ids
5152 .into_iter()
5153 .filter_map(|(s, p, o)| {
5154 Some((
5155 dict.subject_term(s)?,
5156 dict.predicate_term(p)?,
5157 dict.object_term(o)?,
5158 ))
5159 })
5160 .collect();
5161 (triples, sid, done)
5162 }
5163
5164 pub fn dump_named_quads<E>(
5175 &self,
5176 batch: usize,
5177 visit: &mut dyn FnMut(crate::ingest::RawQuad) -> Result<(), E>,
5178 ) -> Result<bool, E> {
5179 let Some(scan) = self.named_quad_scan((None, None, None)) else {
5180 return Ok(false);
5181 };
5182 let dict = self.dictionary();
5183 let batch = batch.max(1);
5184 let mut ids: Vec<(u32, (u32, u32, u32))> = Vec::with_capacity(batch.min(1 << 20));
5185 let mut graph: Option<(u32, String)> = None;
5187 let mut flush = |ids: &mut Vec<(u32, (u32, u32, u32))>| -> Result<(), E> {
5188 let mut nodes = Vec::with_capacity(ids.len() * 3);
5189 let mut preds = Vec::with_capacity(ids.len());
5190 for &(g, (s, p, o)) in ids.iter() {
5191 nodes.push(dict.subject_node(g));
5192 nodes.push(dict.subject_node(s));
5193 nodes.push(dict.object_node(o));
5194 preds.push(p);
5195 }
5196 dict.prefetch_terms(&nodes, &preds);
5197 for (g, (s, p, o)) in ids.drain(..) {
5198 if graph.as_ref().is_none_or(|(id, _)| *id != g) {
5199 graph = dict.subject_term(g).map(|name| (g, name));
5200 }
5201 let (Some((_, name)), Some(s), Some(p), Some(o)) = (
5202 graph.as_ref(),
5203 dict.subject_term(s),
5204 dict.predicate_term(p),
5205 dict.object_term(o),
5206 ) else {
5207 continue;
5208 };
5209 visit((s, p, o, Some(name.clone())))?;
5210 }
5211 Ok(())
5212 };
5213 for q in scan {
5214 ids.push(q);
5215 if ids.len() >= batch {
5216 flush(&mut ids)?;
5217 }
5218 }
5219 flush(&mut ids)?;
5220 Ok(true)
5221 }
5222
5223 pub fn named_graph_count(&self) -> usize {
5226 self.named_graphs.count()
5227 }
5228
5229 pub fn named_graph_quad_count(&self, i: usize) -> Option<u64> {
5234 match &self.named_graphs {
5235 NamedGraphsSlot::Quad(q) => q.quad_count_at(i),
5236 _ => None,
5237 }
5238 }
5239
5240 pub fn named_graph_transient(&self, i: usize) -> Option<GraphIndex> {
5249 match &self.named_graphs {
5250 NamedGraphsSlot::Quad(q) => q.transient_view(i, TRANSIENT_GRAPH_QUADS),
5251 _ => None,
5252 }
5253 }
5254
5255 pub(crate) fn named_graphs_with_subjects(
5262 &self,
5263 subjects: &std::collections::BTreeSet<u32>,
5264 ) -> Option<Vec<usize>> {
5265 match &self.named_graphs {
5266 NamedGraphsSlot::Quad(q) => q.graphs_with_subjects(subjects),
5267 _ => None,
5268 }
5269 }
5270
5271 pub fn named_quad_count(&self) -> Option<u64> {
5276 match &self.named_graphs {
5277 NamedGraphsSlot::Quad(q) => Some(q.quad_count()),
5278 _ => None,
5279 }
5280 }
5281
5282 pub(crate) fn index_triples(&self, index: &GraphIndex) -> u64 {
5288 index.size_hint().unwrap_or(self.header().quad_count)
5289 }
5290
5291 pub fn named_graph_name_at(&self, i: usize) -> Option<&str> {
5294 self.named_graphs.name_at(&self.dict, i, false)
5295 }
5296
5297 pub fn named_graph_at(&self, i: usize) -> Option<(&str, &GraphIndex)> {
5301 self.named_graphs.graph_at(&self.dict, i, false)
5302 }
5303
5304 pub(crate) fn named_graph_name_owned_demand(
5312 &self,
5313 i: usize,
5314 exhaustive: bool,
5315 ) -> Option<String> {
5316 self.named_graphs.name_owned(&self.dict, i, exhaustive)
5317 }
5318
5319 pub(crate) fn named_graph_node_demand(&self, i: usize, exhaustive: bool) -> Option<u32> {
5323 match &self.named_graphs {
5324 NamedGraphsSlot::Quad(q) => q.node_at(&self.dict, i, exhaustive),
5325 _ => None,
5326 }
5327 }
5328
5329 pub(crate) fn named_graph_ordinal(&self, iri: &str) -> Option<usize> {
5333 match &self.named_graphs {
5334 NamedGraphsSlot::Quad(q) => q.ordinal_of_iri(&self.dict, iri),
5335 _ => None,
5336 }
5337 }
5338
5339 pub(crate) fn named_graph_ordinal_of_node(&self, node: u32) -> Option<usize> {
5343 match &self.named_graphs {
5344 NamedGraphsSlot::Quad(q) => q.ordinal_of_node(&self.dict, node),
5345 _ => None,
5346 }
5347 }
5348
5349 pub(crate) fn named_graph_ordinal_of_id(&self, g: u32) -> Option<usize> {
5352 match &self.named_graphs {
5353 NamedGraphsSlot::Quad(q) => q.ordinal_of_id(g),
5354 _ => None,
5355 }
5356 }
5357
5358 pub(crate) fn named_quad_scan(
5363 &self,
5364 pat: (Option<u32>, Option<u32>, Option<u32>),
5365 ) -> Option<crate::quad::QuadScan> {
5366 match &self.named_graphs {
5367 NamedGraphsSlot::Quad(q) => q.scan(pat),
5368 _ => None,
5369 }
5370 }
5371
5372 pub(crate) fn union_default_index(&self) -> Option<GraphIndex> {
5378 let src = self.quad_named()?.union_src()?;
5379 Some(GraphIndex::union_of(
5380 std::sync::Arc::clone(&self.index),
5381 src,
5382 ))
5383 }
5384
5385 pub(crate) fn quad_named(&self) -> Option<&crate::quad::QuadNamed> {
5387 match &self.named_graphs {
5388 NamedGraphsSlot::Quad(q) => Some(q),
5389 _ => None,
5390 }
5391 }
5392
5393 pub(crate) fn named_graph_at_demand(
5396 &self,
5397 i: usize,
5398 exhaustive: bool,
5399 ) -> Option<(&str, &GraphIndex)> {
5400 self.named_graphs.graph_at(&self.dict, i, exhaustive)
5401 }
5402
5403 pub fn graph_names(&self) -> Vec<&str> {
5408 (0..self.named_graphs.count())
5409 .filter_map(|i| self.named_graphs.name_at(&self.dict, i, false))
5410 .collect()
5411 }
5412
5413 pub fn graph_index(&self, iri: &str) -> Option<&GraphIndex> {
5415 self.named_graphs.find(&self.dict, iri)
5416 }
5417
5418 pub fn match_ids(
5421 &self,
5422 pattern: (Option<u32>, Option<u32>, Option<u32>),
5423 ) -> Vec<(u32, u32, u32)> {
5424 self.index.match_pattern(pattern)
5425 }
5426
5427 pub fn predicate_pairs(&self, predicate: &str) -> Vec<(u32, u32)> {
5430 let pid = match self.dict.predicate_id(predicate) {
5431 Some(p) => p,
5432 None => return Vec::new(),
5433 };
5434 self.index
5435 .match_pattern((None, Some(pid), None))
5436 .into_iter()
5437 .map(|(s, _p, o)| (self.dict.subject_node(s), self.dict.object_node(o)))
5438 .collect()
5439 }
5440
5441 pub fn open_ranged<R: RangeReader>(reader: &R) -> Result<Self, FileError> {
5445 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
5446 if crate::v06::open::is_v06(&head) {
5447 let mut all = head;
5450 let at = all.len() as u64;
5451 if reader.len() > at {
5452 all.extend_from_slice(&reader.read_at(at, reader.len() - at)?);
5453 }
5454 return Self::open(&all);
5455 }
5456 let header = Header::from_bytes(&head)?;
5457
5458 let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
5459 let dict =
5460 decode_dictionary_container(&dict_bytes, header.dict_codec, header.term_order())?;
5461
5462 check_named_layout(&header)?;
5463 let index_bytes = reader.read_at(header.root_dir_offset, header.root_dir_len)?;
5464 let (index, _) = decode_root_index_container(&index_bytes, &header)?;
5465 let index_section_ranges = decode_index_section_ranges(
5466 &index_bytes,
5467 header.root_dir_offset,
5468 header.perms,
5469 header.index_container_sections(),
5470 )?;
5471
5472 let pyramid = PyramidSlot::Resident(if header.pyramid_meta_len > 0 {
5473 let mb = reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?;
5474 Some(
5475 PyramidMeta::decode(&mb)
5476 .map_err(|_| FileError::Container("malformed pyramid meta"))?,
5477 )
5478 } else {
5479 None
5480 });
5481
5482 let (ti_off, ti_len) = header.text_index_payload();
5485 let text_index_bytes = if ti_len > 0 {
5486 Some(reader.read_at(ti_off, ti_len)?)
5487 } else {
5488 None
5489 };
5490 let text_index = resident_text_index_slot(text_index_bytes.as_deref(), header.block_codec)?;
5491 let labels = SectionSlot::Resident(header.label_index_section().and_then(|s| {
5494 let b = reader.read_at(s.offset, s.length).ok()?;
5495 crate::label_index::LabelIndex::from_section(&b, header.block_codec).ok()
5496 }));
5497 let schema = SectionSlot::Resident(read_schema_with_header(reader, &header));
5498
5499 let named_graphs = NamedGraphsSlot::resident(
5500 &header,
5501 |off, len| Ok(reader.read_at(off, len)?),
5502 || {
5503 let nb = reader.read_at(header.named_graphs_offset, header.named_graphs_len)?;
5504 decode_named_graphs(&nb, header.block_codec, header.perms)
5505 },
5506 )?;
5507
5508 let tile_ranges =
5512 tile_file_ranges(&index_bytes, header.root_dir_offset, &index_section_ranges);
5513 Ok(Self {
5514 header,
5515 dict,
5516 index: std::sync::Arc::new(index),
5517 index_section_ranges,
5518 tile_ranges,
5519 pyramid,
5520 text_index,
5521 labels,
5522 schema,
5523 named_graphs,
5524 metadata: Vec::new(),
5525 service_client: None,
5526 service_error: std::sync::Mutex::new(None),
5527 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5528 remote_reads: std::sync::atomic::AtomicBool::new(false),
5529 v06: None,
5530 })
5531 }
5532
5533 fn open_v06(
5539 reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
5540 first: &[u8],
5541 budget: bool,
5542 ) -> Result<Self, FileError> {
5543 let f = std::sync::Arc::new(crate::v06::open::V06File::open(reader.clone(), first)?);
5544 let header = crate::v06::logical_header(&f.h);
5545 let rc = reader.concurrency();
5546 let dict = f.dictionary([true; 4])?;
5547 let index = std::sync::Arc::new(f.index(rc)?);
5548 dict.wire_derived(index.clone());
5549 let (pyramid, text_index, labels, schema) = v06_slots(&f);
5550 let named_graphs = match f.quad(rc) {
5551 Some(q) => NamedGraphsSlot::Quad(q?),
5552 None => NamedGraphsSlot::Resident(Vec::new()),
5553 };
5554 let rete = Self {
5555 header,
5556 dict,
5557 index,
5558 index_section_ranges: [ByteRange::default(); NUM_PERMS],
5559 tile_ranges: Default::default(),
5560 pyramid,
5561 text_index,
5562 labels,
5563 schema,
5564 named_graphs,
5565 metadata: Vec::new(),
5566 service_client: None,
5567 service_error: std::sync::Mutex::new(None),
5568 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5569 remote_reads: std::sync::atomic::AtomicBool::new(false),
5570 v06: Some(f),
5571 };
5572 #[cfg(not(target_arch = "wasm32"))]
5573 if budget {
5574 rete.set_memory_budget(match crate::sysmem::default_query_budget_mb() {
5575 0 => None,
5576 mb => Some(mb << 20),
5577 });
5578 }
5579 #[cfg(target_arch = "wasm32")]
5580 let _ = budget;
5581 Ok(rete)
5582 }
5583
5584 pub fn v06(&self) -> Option<&crate::v06::open::V06File> {
5586 self.v06.as_deref()
5587 }
5588
5589 pub fn open_ranged_lazy<R: RangeReader + Send + Sync + 'static>(
5601 reader: R,
5602 ) -> Result<Self, FileError> {
5603 Self::open_ranged_lazy_with(reader, crate::v06::FIRST_READ)
5604 }
5605
5606 pub fn open_ranged_lazy_with<R: RangeReader + Send + Sync + 'static>(
5611 reader: R,
5612 first_read: u64,
5613 ) -> Result<Self, FileError> {
5614 let remote = !reader.is_local();
5615 let rete = Self::open_ranged_lazy_inner(reader, first_read)?;
5616 rete.remote_reads
5617 .store(remote, std::sync::atomic::Ordering::Relaxed);
5618 Ok(rete)
5619 }
5620
5621 pub fn reads_are_remote(&self) -> bool {
5628 self.remote_reads.load(std::sync::atomic::Ordering::Relaxed)
5629 }
5630
5631 fn open_ranged_lazy_inner<R: RangeReader + Send + Sync + 'static>(
5632 reader: R,
5633 first_read: u64,
5634 ) -> Result<Self, FileError> {
5635 let first_len = first_read.max(HEADER_LEN as u64).min(reader.len());
5638 let first: std::sync::Arc<[u8]> = std::sync::Arc::from(reader.read_at(0, first_len)?);
5639 #[cfg(feature = "v06-parity")]
5640 if !crate::v06::open::is_v06(&first) {
5641 let all = reader.read_at(0, reader.len())?;
5642 if let Some(v6) = parity_image(&all) {
5643 let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
5644 std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(v6)));
5645 let head = r.read_at(0, crate::v06::FIRST_READ.min(r.len()))?;
5646 return Self::open_v06(r, &head, true);
5647 }
5648 }
5649 let reader = crate::reader::PrefixReader::new(reader, first.clone());
5650 if crate::v06::open::is_v06(&first) {
5651 let r: std::sync::Arc<dyn RangeReader + Send + Sync> = std::sync::Arc::new(reader);
5652 return Self::open_v06(r, &first, true);
5653 }
5654 let head = reader.read_at(0, HEADER_LEN as u64)?;
5655 let header = Header::from_bytes(&head)?;
5656 check_named_layout(&header)?;
5657 let reader = std::sync::Arc::new(reader);
5658 let read_concurrency = reader.concurrency();
5661
5662 let open_dict = || ranged_chunked_dictionary(&reader, &header, [true; 4]);
5666
5667 let reader_dyn: std::sync::Arc<dyn RangeReader + Send + Sync> = reader.clone();
5671 let open_index = || {
5672 open_index_container_lazy(
5673 &reader_dyn,
5674 ByteRange {
5675 offset: header.root_dir_offset,
5676 len: header.root_dir_len,
5677 },
5678 header.block_codec,
5679 header.has_tile_synopsis(),
5680 read_concurrency,
5681 header.perms,
5682 header.index_container_sections(),
5683 )
5684 };
5685
5686 let (dict, index) = open_side_by_side(read_concurrency, open_dict, open_index);
5693 let dict = dict?;
5694 let (index, index_section_ranges, tile_ranges) = index?;
5695
5696 let pyramid = ranged_pyramid_slot(&reader, &header);
5700
5701 let text_index = ranged_text_index_slot(&reader, &header);
5706
5707 let labels = ranged_label_slot(&reader, &header);
5711 let schema = ranged_schema_slot(&reader, &header);
5712
5713 let named_graphs = if let Some(q) = header.quad_index_section() {
5720 NamedGraphsSlot::Quad(crate::quad::QuadNamed::new(
5724 reader_dyn,
5725 ByteRange {
5726 offset: q.offset,
5727 len: q.length,
5728 },
5729 header.block_codec,
5730 header.perms,
5731 read_concurrency,
5732 ))
5733 } else if header.named_graphs_len > 0 {
5734 NamedGraphsSlot::Lazy(LazyNamedGraphs::new(
5735 reader_dyn,
5736 ByteRange {
5737 offset: header.named_graphs_offset,
5738 len: header.named_graphs_len,
5739 },
5740 header.block_codec,
5741 header.has_tile_synopsis(),
5742 read_concurrency,
5743 header.perms,
5744 ))
5745 } else {
5746 NamedGraphsSlot::Resident(Vec::new())
5747 };
5748
5749 let rete = Self {
5750 header,
5751 dict,
5752 index: std::sync::Arc::new(index),
5753 index_section_ranges,
5754 tile_ranges,
5755 pyramid,
5756 text_index,
5757 labels,
5758 schema,
5759 named_graphs,
5760 metadata: Vec::new(),
5761 service_client: None,
5762 service_error: std::sync::Mutex::new(None),
5763 spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5764 remote_reads: std::sync::atomic::AtomicBool::new(false),
5765 v06: None,
5766 };
5767 #[cfg(not(target_arch = "wasm32"))]
5779 rete.set_memory_budget(match crate::sysmem::default_query_budget_mb() {
5780 0 => None,
5781 mb => Some(mb << 20),
5782 });
5783 Ok(rete)
5784 }
5785
5786 pub fn index_incomplete(&self) -> bool {
5791 self.index.load_incomplete()
5792 || self.dict.load_incomplete()
5793 || self.named_graphs.load_incomplete()
5794 }
5795
5796 pub fn reset_load_failures(&self) {
5804 self.index.reset_load_failure();
5805 self.dict.reset_load_failure();
5806 self.named_graphs.reset_load_failures();
5807 }
5808
5809 pub fn set_memory_budget(&self, budget: Option<u64>) -> MemoryBudget {
5840 let total = budget.unwrap_or(u64::MAX);
5841 self.set_spill_threshold(budget.map(spill_threshold_for));
5842 let mut split = split_memory_budget(total);
5843 let fixed = if total == u64::MAX {
5844 0
5845 } else {
5846 self.fixed_overhead().total()
5847 };
5848 if split.dict_cache != u64::MAX {
5849 split.dict_cache = split
5850 .dict_cache
5851 .saturating_sub(fixed)
5852 .max(MIN_DICT_CACHE_CAP);
5853 }
5854 self.dict.set_cache_cap(split.dict_cache);
5855 self.index.set_cache_cap(split.tile_cache);
5856 match &self.named_graphs {
5861 NamedGraphsSlot::Lazy(lazy) => lazy.set_tile_cap(split.tile_cache),
5862 NamedGraphsSlot::Quad(q) => q.set_tile_cap(split.tile_cache),
5865 NamedGraphsSlot::Resident(_) => {}
5866 }
5867 if std::env::var("RETE_OPEN_DEBUG").is_ok() {
5868 eprintln!(
5869 "[memory-budget] total={} dict_cache={} tile_cache={} block_cache={} fixed={:.1}MiB",
5870 if total == u64::MAX {
5871 "unlimited".to_string()
5872 } else {
5873 format!("{}MiB", total >> 20)
5874 },
5875 fmt_cap(split.dict_cache),
5876 fmt_cap(split.tile_cache),
5877 fmt_cap(split.block_cache),
5878 fixed as f64 / (1u64 << 20) as f64,
5879 );
5880 }
5881 split
5882 }
5883
5884 pub fn spill_threshold(&self) -> u64 {
5891 self.spill_threshold
5892 .load(std::sync::atomic::Ordering::Relaxed)
5893 }
5894
5895 pub fn set_spill_threshold(&self, bytes: Option<u64>) {
5899 self.spill_threshold.store(
5900 bytes.unwrap_or(u64::MAX),
5901 std::sync::atomic::Ordering::Relaxed,
5902 );
5903 }
5904
5905 pub fn dict_cache_stats(&self) -> crate::chunk_cache::CacheStats {
5909 self.dict.cache_stats()
5910 }
5911
5912 pub fn fixed_overhead(&self) -> FixedOverhead {
5916 FixedOverhead {
5917 dict_directory: self.dict.directory_bytes(),
5918 index_directory: self.index.directory_bytes(),
5919 named_graphs: self.named_graphs.directory_bytes(),
5920 }
5921 }
5922
5923 pub fn index_cache_stats(&self) -> crate::chunk_cache::CacheStats {
5925 self.index.cache_stats()
5926 }
5927
5928 pub fn dict_cache_cap(&self) -> u64 {
5930 self.dict.cache_cap()
5931 }
5932
5933 pub fn release_named_graph(&mut self, iri: &str) {
5946 match &mut self.named_graphs {
5947 NamedGraphsSlot::Lazy(lazy) => lazy.release(iri),
5948 NamedGraphsSlot::Quad(q) => q.release(&self.dict, iri),
5949 NamedGraphsSlot::Resident(_) => {}
5950 }
5951 }
5952
5953 fn resolve_query_pattern(
5954 &self,
5955 s: Option<&str>,
5956 p: Option<&str>,
5957 o: Option<&str>,
5958 ) -> Option<Pattern> {
5959 let sid = match s {
5960 Some(t) => match self.dict.subject_id(t) {
5961 Some(id) => Some(id),
5962 None => return None,
5963 },
5964 None => None,
5965 };
5966 let pid = match p {
5967 Some(t) => match self.dict.predicate_id(t) {
5968 Some(id) => Some(id),
5969 None => return None,
5970 },
5971 None => None,
5972 };
5973 let oid = match o {
5974 Some(t) => match self.dict.object_id(t) {
5975 Some(id) => Some(id),
5976 None => return None,
5977 },
5978 None => None,
5979 };
5980 Some((sid, pid, oid))
5981 }
5982
5983 pub fn query_with_provenance(
5987 &self,
5988 s: Option<&str>,
5989 p: Option<&str>,
5990 o: Option<&str>,
5991 ) -> Vec<TripleProvenance> {
5992 let pattern = match self.resolve_query_pattern(s, p, o) {
5993 Some(pattern) => pattern,
5994 None => return Vec::new(),
5995 };
5996
5997 let index_permutation = GraphIndex::best_permutation_in(self.header.perms, pattern);
5998 let si = index_permutation.section_index();
5999 let (dictionary_range, index_range, index_section_range, pyramid_range) = match &self.v06 {
6000 Some(f) => {
6001 use crate::v06::kind;
6002 let sec = |k| f.section(k).unwrap_or_default();
6003 let perm = f
6004 .root
6005 .unit(crate::v06::tag::INDEX)
6006 .and_then(|u| crate::v06::decode_index_root(u).ok())
6007 .and_then(|v| v.into_iter().find(|p| p.si == si))
6008 .map(|p| ByteRange {
6009 offset: sec(kind::INDEX_DATA).offset + p.data_off,
6010 len: p.data_len,
6011 })
6012 .unwrap_or_default();
6013 (
6014 sec(kind::DICT_DATA),
6015 sec(kind::INDEX_DATA),
6016 perm,
6017 f.section(kind::PYRAMID),
6018 )
6019 }
6020 None => (
6021 ByteRange {
6022 offset: self.header.dictionary_offset,
6023 len: self.header.dictionary_len,
6024 },
6025 ByteRange {
6026 offset: self.header.root_dir_offset,
6027 len: self.header.root_dir_len,
6028 },
6029 self.index_section_ranges[si],
6030 (self.header.pyramid_meta_len > 0).then_some(ByteRange {
6031 offset: self.header.pyramid_meta_offset,
6032 len: self.header.pyramid_meta_len,
6033 }),
6034 ),
6035 };
6036
6037 let tiles = &self.tile_ranges[si];
6038 let v06_tiles = self.v06.is_some().then(|| self.index.tile_sections()[si]);
6039 self.index
6040 .match_pattern(pattern)
6041 .into_iter()
6042 .filter_map(|(s, p, o)| {
6043 let terms = (
6044 self.dict.subject_term(s)?,
6045 self.dict.predicate_term(p)?,
6046 self.dict.object_term(o)?,
6047 );
6048 let a = index_permutation.forward((s, p, o)).0;
6051 let (tile, tile_range) = if let Some(tl) = v06_tiles {
6052 let (ti, tj) = self.index.tile_span(si, Some(a));
6053 match (ti < tj && ti < tl.len()).then(|| tl.range(ti)).flatten() {
6054 Some((offset, len)) => (
6055 Some(format!("{}/{ti}", index_permutation.name())),
6056 Some(ByteRange { offset, len }),
6057 ),
6058 None => (None, None),
6059 }
6060 } else {
6061 let ti = tiles.partition_point(|&(_, max_a, _)| max_a < a);
6062 match tiles.get(ti) {
6063 Some(&(min_a, _, range)) if min_a <= a => (
6064 Some(format!("{}/{ti}", index_permutation.name())),
6065 Some(range),
6066 ),
6067 _ => (None, None),
6068 }
6069 };
6070 Some(TripleProvenance {
6071 terms,
6072 ids: (s, p, o),
6073 graph: None,
6074 matched_pattern: pattern,
6075 index_permutation,
6076 dictionary_range,
6077 index_range,
6078 index_section_range,
6079 pyramid_range,
6080 tile,
6081 tile_range,
6082 })
6083 })
6084 .collect()
6085 }
6086
6087 pub fn query(&self, s: Option<&str>, p: Option<&str>, o: Option<&str>) -> Vec<TermTriple> {
6091 self.query_with_provenance(s, p, o)
6092 .into_iter()
6093 .map(|m| m.terms)
6094 .collect()
6095 }
6096
6097 pub fn query_in_graph(
6105 &self,
6106 graph: Option<&str>,
6107 s: Option<&str>,
6108 p: Option<&str>,
6109 o: Option<&str>,
6110 ) -> Vec<TermTriple> {
6111 let pattern = match self.resolve_query_pattern(s, p, o) {
6112 Some(pattern) => pattern,
6113 None => return Vec::new(),
6114 };
6115 let index = match graph {
6116 None => &*self.index,
6117 Some(g) => match self.graph_index(g) {
6118 Some(i) => i,
6119 None => return Vec::new(),
6120 },
6121 };
6122 self.resolve_matches(index.match_pattern(pattern))
6123 }
6124
6125 pub(crate) fn query_in_index(
6128 &self,
6129 index: &GraphIndex,
6130 s: Option<&str>,
6131 p: Option<&str>,
6132 o: Option<&str>,
6133 ) -> Vec<TermTriple> {
6134 match self.resolve_query_pattern(s, p, o) {
6135 Some(pattern) => self.resolve_matches(index.match_pattern(pattern)),
6136 None => Vec::new(),
6137 }
6138 }
6139
6140 fn resolve_matches(&self, ids: Vec<Triple>) -> Vec<TermTriple> {
6143 if !ids.is_empty() {
6151 let mut nodes = Vec::with_capacity(ids.len() * 2);
6152 let mut preds = Vec::with_capacity(ids.len());
6153 for &(s, p, o) in &ids {
6154 nodes.push(self.dict.subject_node(s));
6155 nodes.push(self.dict.object_node(o));
6156 preds.push(p);
6157 }
6158 self.dict.prefetch_terms(&nodes, &preds);
6159 }
6160 ids.into_iter()
6161 .filter_map(|(s, p, o)| {
6162 Some((
6163 self.dict.subject_term(s)?,
6164 self.dict.predicate_term(p)?,
6165 self.dict.object_term(o)?,
6166 ))
6167 })
6168 .collect()
6169 }
6170
6171 pub fn query_quads(
6176 &self,
6177 s: Option<&str>,
6178 p: Option<&str>,
6179 o: Option<&str>,
6180 ) -> Vec<(TermTriple, Option<String>)> {
6181 let mut out: Vec<(TermTriple, Option<String>)> = self
6182 .query_in_graph(None, s, p, o)
6183 .into_iter()
6184 .map(|t| (t, None))
6185 .collect();
6186 let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
6187 return out;
6188 };
6189 for i in 0..self.named_graphs.count() {
6190 let Some(iri) = self.named_graphs.name_at(&self.dict, i, true) else {
6193 continue;
6194 };
6195 let owned = self.named_graph_transient(i);
6198 let index = match &owned {
6199 Some(view) => view,
6200 None => match self.named_graphs.graph_at(&self.dict, i, true) {
6201 Some((_, g)) => g,
6202 None => continue,
6203 },
6204 };
6205 for triple in self.resolve_matches(index.match_pattern(pattern)) {
6206 out.push((triple, Some(iri.to_string())));
6207 }
6208 }
6209 out
6210 }
6211
6212 pub fn query_ranged<R: RangeReader>(
6219 reader: &R,
6220 s: Option<&str>,
6221 p: Option<&str>,
6222 o: Option<&str>,
6223 ) -> Result<Vec<TermTriple>, FileError> {
6224 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6225 if crate::v06::open::is_v06(&head) {
6226 return Ok(v06_query_ranged(reader, head, s, p, o, true)?.unwrap_or_default());
6227 }
6228 let routed = match route_pattern(reader, head, s, p, o)? {
6229 Some(routed) => routed,
6230 None => return Ok(Vec::new()),
6231 };
6232 let matches = fetch_routed_matches(reader, &routed)?;
6233 Ok(matches
6234 .into_iter()
6235 .filter_map(|(s, p, o)| {
6236 Some((
6237 routed.dict.subject_term(s)?,
6238 routed.dict.predicate_term(p)?,
6239 routed.dict.object_term(o)?,
6240 ))
6241 })
6242 .collect())
6243 }
6244
6245 pub fn route_pattern_ranged<R: RangeReader>(
6249 reader: &R,
6250 s: Option<&str>,
6251 p: Option<&str>,
6252 o: Option<&str>,
6253 ) -> Result<bool, FileError> {
6254 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6255 if crate::v06::open::is_v06(&head) {
6256 return Ok(v06_query_ranged(reader, head, s, p, o, false)?.is_some());
6257 }
6258 Ok(route_pattern(reader, head, s, p, o)?.is_some())
6259 }
6260}
6261
6262fn v06_query_ranged<R: RangeReader>(
6268 reader: &R,
6269 head: Vec<u8>,
6270 s: Option<&str>,
6271 p: Option<&str>,
6272 o: Option<&str>,
6273 fetch: bool,
6274) -> Result<Option<Vec<TermTriple>>, FileError> {
6275 use crate::v06::{kind, open::SparseReader, Header06, Root06, HEADER_LEN};
6276 let len = reader.len();
6277 let h = Header06::from_bytes(&head)?;
6278 if h.flags & crate::v06::F_DERIVED != 0 {
6279 return Err(FileError::Container(
6282 "this file derives IRIs from its index (format 0x08); a one-pattern ranged query cannot read it, use the lazy reader (`rete sparql-url`)",
6283 ));
6284 }
6285 let root_end = (HEADER_LEN as u64 + h.root_len as u64).min(len);
6286 let root_bytes = if root_end <= head.len() as u64 {
6287 head[HEADER_LEN..root_end as usize].to_vec()
6288 } else {
6289 reader.read_at(HEADER_LEN as u64, root_end - HEADER_LEN as u64)?
6290 };
6291 let root = Root06::decode(&root_bytes, h.internal_codec)?;
6292 let dict = root.section(kind::DICT_DATA).and_then(|d| d.abs(&h).ok());
6293 let end = dict
6296 .map_or(h.data_offset, |d| d.offset + d.len)
6297 .max(crate::header::HEADER_LEN as u64)
6298 .min(len);
6299 let mut held = head;
6300 if end > held.len() as u64 {
6301 let at = held.len() as u64;
6302 held.extend_from_slice(&reader.read_at(at, end - at)?);
6303 }
6304 let sparse = SparseReader::fetch(reader, held, &[])?;
6305 let rete = Rete::open_ranged_lazy_with(sparse, 0)?;
6306 let Some(pattern) = resolve_query_pattern(rete.dictionary(), s, p, o) else {
6307 return Ok(None);
6308 };
6309 if !fetch {
6310 return Ok(Some(Vec::new()));
6311 }
6312 let perm = GraphIndex::best_permutation_in(h.perms(), pattern);
6313 let si = perm.section_index();
6314 let index = rete.default_index();
6315 let [pa, _, _] = perm.order_pattern(pattern);
6316 let mut out = Vec::new();
6319 for (start, stop) in index.routed_spans(si, pa) {
6320 let ranges: Vec<(u64, u64)> = (start..stop)
6321 .filter_map(|i| index.tile_range(si, i))
6322 .collect();
6323 if let (Some(&(lo, _)), Some(&(last, last_len))) = (ranges.first(), ranges.last()) {
6324 let body = reader.read_at(lo, last + last_len - lo)?;
6325 for &(off, l) in &ranges {
6326 let at = (off - lo) as usize;
6327 let tile = decompress(h.block_codec, &body[at..at + l as usize])?;
6328 out.extend(GraphIndex::match_serialized_block(&tile, perm, pattern));
6329 }
6330 }
6331 }
6332 out.sort_unstable();
6333 let d = rete.dictionary();
6334 Ok(Some(
6335 out.into_iter()
6336 .filter_map(|(s, p, o)| {
6337 Some((d.subject_term(s)?, d.predicate_term(p)?, d.object_term(o)?))
6338 })
6339 .collect(),
6340 ))
6341}
6342
6343struct RoutedPattern {
6346 dict: Dictionary,
6347 pattern: Pattern,
6348 permutation: IndexPermutation,
6349 header: Header,
6350 section: ByteRange,
6352}
6353
6354fn route_pattern<R: RangeReader>(
6357 reader: &R,
6358 head: Vec<u8>,
6359 s: Option<&str>,
6360 p: Option<&str>,
6361 o: Option<&str>,
6362) -> Result<Option<RoutedPattern>, FileError> {
6363 let header = Header::from_bytes(&head)?;
6364
6365 let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
6366 let dict = decode_dictionary_container(&dict_bytes, header.dict_codec, header.term_order())?;
6367
6368 let Some(pattern) = resolve_query_pattern(&dict, s, p, o) else {
6369 return Ok(None);
6370 };
6371 let permutation = GraphIndex::best_permutation_in(header.perms, pattern);
6372 let section = locate_container_section_ranged(
6373 reader,
6374 header.root_dir_offset,
6375 header.root_dir_len,
6376 header
6377 .perms
6378 .position(permutation)
6379 .ok_or(FileError::Container("routed to an absent permutation"))?,
6380 header.index_container_sections() as u64,
6381 )?;
6382 Ok(Some(RoutedPattern {
6383 dict,
6384 pattern,
6385 permutation,
6386 header,
6387 section,
6388 }))
6389}
6390
6391fn fetch_routed_matches<R: RangeReader>(
6396 reader: &R,
6397 routed: &RoutedPattern,
6398) -> Result<Vec<Triple>, FileError> {
6399 let dir = read_tile_directory_ranged(reader, routed.section)?;
6400 let [pa, _, _] = routed.permutation.order_pattern(routed.pattern);
6401 let codec = routed.header.block_codec;
6402 let mut out = Vec::new();
6403 match pa {
6404 Some(a) => {
6407 for e in dir.iter().filter(|e| e.min_a <= a && a <= e.max_a) {
6408 let len = e.end - e.start;
6409 let bytes = reader.read_at(checked_end(routed.section.offset, e.start)?, len)?;
6410 if bytes.len() as u64 != len {
6411 return Err(FileError::Container("short tile read"));
6412 }
6413 let tile = decompress(codec, &bytes)?;
6414 out.extend(GraphIndex::match_serialized_block(
6415 &tile,
6416 routed.permutation,
6417 routed.pattern,
6418 ));
6419 }
6420 }
6421 None => {
6424 if let (Some(first), Some(last)) = (dir.first(), dir.last()) {
6425 let base = first.start;
6426 let body =
6427 reader.read_at(checked_end(routed.section.offset, base)?, last.end - base)?;
6428 for e in &dir {
6429 let stored = checked_slice(&body, e.start - base, e.end - e.start)
6431 .ok_or(FileError::Container("short tile read"))?;
6432 let tile = decompress(codec, stored)?;
6433 out.extend(GraphIndex::match_serialized_block(
6434 &tile,
6435 routed.permutation,
6436 routed.pattern,
6437 ));
6438 }
6439 }
6440 }
6441 }
6442 out.sort_unstable();
6443 Ok(out)
6444}
6445
6446fn resolve_query_pattern(
6447 dict: &Dictionary,
6448 s: Option<&str>,
6449 p: Option<&str>,
6450 o: Option<&str>,
6451) -> Option<Pattern> {
6452 let sid = match s {
6453 Some(t) => Some(dict.subject_id(t)?),
6454 None => None,
6455 };
6456 let pid = match p {
6457 Some(t) => Some(dict.predicate_id(t)?),
6458 None => None,
6459 };
6460 let oid = match o {
6461 Some(t) => Some(dict.object_id(t)?),
6462 None => None,
6463 };
6464 Some((sid, pid, oid))
6465}
6466
6467fn ranged_chunked_dictionary<R: RangeReader + Send + Sync + 'static>(
6482 reader: &std::sync::Arc<R>,
6483 header: &Header,
6484 want: [bool; 4],
6485) -> Result<Dictionary, FileError> {
6486 let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
6489 let mut dict_sections: Vec<crate::dict::ChunkedSection> = Vec::with_capacity(4);
6490 let mut section_ranges: [std::sync::Arc<Vec<ByteRange>>; 4] = Default::default();
6491 for si in 0..4 {
6492 if !want[si] {
6493 dict_sections.push(
6494 crate::dict::ChunkedSection::from_parts(
6495 crate::dict::SectionMeta {
6496 term_count: 0,
6497 restart_interval: 1,
6498 restart_offsets: Vec::new(),
6499 },
6500 Vec::new(),
6501 None,
6502 cache.clone(),
6503 si as u8,
6504 )
6505 .with_order(header.term_order()),
6506 );
6507 continue;
6508 }
6509 let section = locate_container_section_ranged(
6510 reader.as_ref(),
6511 header.dictionary_offset,
6512 header.dictionary_len,
6513 si,
6514 4,
6515 )?;
6516 let (meta, entries) = read_dict_dir_ranged(reader.as_ref(), section)?;
6517 let ranges: std::sync::Arc<Vec<ByteRange>> = std::sync::Arc::new(
6520 entries
6521 .iter()
6522 .map(|e| ByteRange {
6523 offset: section.offset + e.start,
6524 len: (e.end - e.start),
6525 })
6526 .collect(),
6527 );
6528 section_ranges[si] = ranges.clone();
6529 let chunks: Vec<crate::dict::SectionChunk> = entries
6530 .into_iter()
6531 .map(|e| crate::dict::SectionChunk::new(e.first_run, e.key, e.body_start))
6532 .collect();
6533 let chunk_reader = reader.clone();
6534 let codec = header.dict_codec;
6535 let loader_ranges = ranges.clone();
6536 let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
6537 let range = loader_ranges.get(ci)?;
6538 let bytes = chunk_reader.read_at(range.offset, range.len).ok()?;
6539 decompress(codec, &bytes).ok()
6540 });
6541 let bulk_reader = reader.clone();
6544 let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
6545 let want: Option<Vec<ByteRange>> =
6546 cis.iter().map(|&ci| ranges.get(ci).copied()).collect();
6547 let blobs = read_coalesced(bulk_reader.as_ref(), &want?, DICT_COALESCE_GAP)?;
6548 blobs.iter().map(|b| decompress(codec, b).ok()).collect()
6549 });
6550 dict_sections.push(
6551 crate::dict::ChunkedSection::from_parts(
6552 meta,
6553 chunks,
6554 Some(loader),
6555 cache.clone(),
6556 si as u8,
6557 )
6558 .with_bulk_loader(bulk)
6559 .with_order(header.term_order()),
6560 );
6561 }
6562 let dict_arr: [crate::dict::ChunkedSection; 4] = dict_sections
6563 .try_into()
6564 .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
6565 let multi_reader = reader.clone();
6569 let codec = header.dict_codec;
6570 let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
6571 let want: Option<Vec<ByteRange>> = pairs
6572 .iter()
6573 .map(|&(si, ci)| section_ranges.get(si as usize)?.get(ci).copied())
6574 .collect();
6575 let blobs = read_coalesced(multi_reader.as_ref(), &want?, DICT_COALESCE_GAP)?;
6576 blobs.iter().map(|b| decompress(codec, b).ok()).collect()
6577 });
6578 Ok(Dictionary::from_chunked_sections(dict_arr).with_multi_bulk_loader(multi))
6579}
6580
6581#[cfg(not(target_arch = "wasm32"))]
6586fn open_side_by_side<A, B: Send>(
6587 concurrency: usize,
6588 a: impl FnOnce() -> A,
6589 b: impl FnOnce() -> B + Send,
6590) -> (A, B) {
6591 if concurrency <= 1 {
6592 return (a(), b());
6593 }
6594 std::thread::scope(|scope| {
6595 let hb = scope.spawn(b);
6596 let ra = a();
6597 let rb = hb
6598 .join()
6599 .unwrap_or_else(|panic| std::panic::resume_unwind(panic));
6600 (ra, rb)
6601 })
6602}
6603
6604#[cfg(target_arch = "wasm32")]
6606fn open_side_by_side<A, B>(
6607 _concurrency: usize,
6608 a: impl FnOnce() -> A,
6609 b: impl FnOnce() -> B,
6610) -> (A, B) {
6611 (a(), b())
6612}
6613
6614#[cfg(feature = "v06-parity")]
6617fn parity_image(bytes: &[u8]) -> Option<Vec<u8>> {
6618 if crate::v06::open::is_v06(bytes) || bytes.len() < HEADER_LEN {
6619 return None;
6620 }
6621 crate::v06::write::convert_image(bytes, crate::v06::write::WriteOpts::default()).ok()
6622}
6623
6624fn v06_layout(f: &crate::v06::open::V06File) -> Vec<LayoutSegment> {
6627 let h = &f.h;
6628 let seg = |kind: &'static str, label: String, offset: u64, len: u64| LayoutSegment {
6629 kind,
6630 label,
6631 offset,
6632 len,
6633 };
6634 let mut out = vec![
6635 seg(
6636 "header",
6637 format!("header (fixed {} bytes)", crate::v06::HEADER_LEN),
6638 0,
6639 crate::v06::HEADER_LEN as u64,
6640 ),
6641 seg(
6642 "directory",
6643 "root block (section table, directory roots)".into(),
6644 h.root_offset(),
6645 h.root_len as u64,
6646 ),
6647 ];
6648 let c = h.card_range();
6649 if c.len > 0 {
6650 out.push(seg(
6651 "metadata",
6652 "metadata (dataset card, zstd)".into(),
6653 c.offset,
6654 c.len,
6655 ));
6656 }
6657 let b = h.build_range();
6658 if b.len > 0 {
6659 out.push(seg("build-info", "build record".into(), b.offset, b.len));
6660 }
6661 for s in &f.root.sections {
6662 let Ok(a) = s.abs(h) else { continue };
6663 let kind: &'static str = match s.kind {
6664 crate::v06::kind::SCHEMA => "schema",
6665 crate::v06::kind::DICT_DATA => "dictionary",
6666 crate::v06::kind::INDEX_DATA => "tile",
6667 crate::v06::kind::QUAD_DATA => "named-graphs",
6668 crate::v06::kind::LABEL_DATA => "label-index",
6669 crate::v06::kind::TEXT_DATA => "text-index",
6670 crate::v06::kind::PYRAMID => "pyramid",
6671 _ => "directory",
6672 };
6673 out.push(seg(
6674 kind,
6675 crate::v06::kind::name(s.kind).to_string(),
6676 a.offset,
6677 a.len,
6678 ));
6679 }
6680 out.sort_by_key(|s| s.offset);
6681 out
6682}
6683
6684#[allow(clippy::type_complexity)]
6687fn v06_slots(
6688 f: &std::sync::Arc<crate::v06::open::V06File>,
6689) -> (
6690 PyramidSlot,
6691 TextIndexSlot,
6692 SectionSlot<crate::label_index::LabelIndex>,
6693 SectionSlot<crate::schema_section::SchemaMeta>,
6694) {
6695 use crate::v06::{P_LABELS, P_PYRAMID, P_SCHEMA, P_TEXT};
6696 let pyramid = if f.h.has(P_PYRAMID) {
6697 let a = f.clone();
6698 PyramidSlot::Lazy {
6699 loader: Box::new(move || PyramidMeta::decode(&a.pyramid_bytes()?).ok()),
6700 cell: std::sync::OnceLock::new(),
6701 }
6702 } else {
6703 PyramidSlot::Resident(None)
6704 };
6705 let text_index = if f.h.has(P_TEXT) {
6706 let (a, b) = (f.clone(), f.clone());
6707 TextIndexSlot::Lazy {
6708 loader: Box::new(move || a.text_index()?.ok()),
6709 cell: std::sync::OnceLock::new(),
6710 token_table: Box::new(move || b.token_table_bytes()),
6711 token_table_cell: std::sync::OnceLock::new(),
6712 }
6713 } else {
6714 TextIndexSlot::Resident {
6715 index: None,
6716 token_table_len: None,
6717 }
6718 };
6719 let labels = if f.h.has(P_LABELS) {
6720 let a = f.clone();
6721 SectionSlot::Lazy {
6722 loader: Box::new(move || a.label_index()?.ok()),
6723 cell: std::sync::OnceLock::new(),
6724 }
6725 } else {
6726 SectionSlot::Resident(None)
6727 };
6728 let schema = if f.h.has(P_SCHEMA) {
6729 let a = f.clone();
6730 SectionSlot::Lazy {
6731 loader: Box::new(move || a.schema()),
6732 cell: std::sync::OnceLock::new(),
6733 }
6734 } else {
6735 SectionSlot::Resident(None)
6736 };
6737 (pyramid, text_index, labels, schema)
6738}
6739
6740fn ranged_pyramid_slot<R: RangeReader + Send + Sync + 'static>(
6744 reader: &std::sync::Arc<R>,
6745 header: &Header,
6746) -> PyramidSlot {
6747 if header.pyramid_meta_len == 0 {
6748 return PyramidSlot::Resident(None);
6749 }
6750 let pyr_reader = reader.clone();
6751 let pyr_off = header.pyramid_meta_offset;
6752 let pyr_len = header.pyramid_meta_len;
6753 PyramidSlot::Lazy {
6754 loader: Box::new(move || {
6755 let mb = pyr_reader.read_at(pyr_off, pyr_len).ok()?;
6756 PyramidMeta::decode(&mb).ok()
6757 }),
6758 cell: std::sync::OnceLock::new(),
6759 }
6760}
6761
6762fn ranged_text_index_slot<R: RangeReader + Send + Sync + 'static>(
6767 reader: &std::sync::Arc<R>,
6768 header: &Header,
6769) -> TextIndexSlot {
6770 let (ti_off, ti_len) = header.text_index_payload();
6771 if ti_len == 0 {
6772 return TextIndexSlot::Resident {
6773 index: None,
6774 token_table_len: None,
6775 };
6776 }
6777 let ti_reader = reader.clone();
6778 let probe_reader = reader.clone();
6779 let codec = header.block_codec;
6780 TextIndexSlot::Lazy {
6781 loader: Box::new(move || {
6782 let prefix_len = read_token_table_len(&*ti_reader, ti_off, ti_len)?;
6786 let prefix = ti_reader.read_at(ti_off, prefix_len).ok()?;
6787 let postings_base = crate::text_index::TextIndex::postings_base(&prefix)? as u64;
6788 let postings_abs = ti_off + postings_base;
6789 let pr = ti_reader.clone();
6790 let posting_loader =
6791 Box::new(move |off: u64, len: u64| pr.read_at(postings_abs + off, len).ok());
6792 crate::text_index::TextIndex::from_token_table(&prefix, codec, posting_loader).ok()
6793 }),
6794 cell: std::sync::OnceLock::new(),
6795 token_table: Box::new(move || read_token_table_len(&*probe_reader, ti_off, ti_len)),
6798 token_table_cell: std::sync::OnceLock::new(),
6799 }
6800}
6801
6802fn resolve_label_hits(
6809 dict: &Dictionary,
6810 hits: Vec<&crate::meta::LabelEntry>,
6811) -> Vec<(String, String)> {
6812 let ids: Vec<u32> = hits.iter().map(|e| e.subject).collect();
6813 dict.prefetch_subject_terms(&ids);
6814 hits.iter()
6815 .zip(dict.subject_terms(&ids))
6816 .filter_map(|(e, iri)| Some((e.label.clone(), iri?)))
6817 .collect()
6818}
6819
6820fn text_search_in(
6823 ti: Option<&crate::text_index::TextIndex>,
6824 dict: &Dictionary,
6825 words: &[&str],
6826 prefix: Option<&str>,
6827 limit: usize,
6828) -> Vec<String> {
6829 let Some(ti) = ti else {
6830 return Vec::new();
6831 };
6832 let mut acc: Option<Vec<u32>> = None;
6836 if let Some(p) = prefix {
6837 acc = Some(ti.prefix(&p.to_lowercase()));
6838 }
6839 for w in words {
6840 for tok in crate::text_index::tokenize(w) {
6841 let posting = ti.lookup(&tok);
6842 acc = Some(match acc {
6843 Some(a) => intersect_sorted(&a, &posting),
6844 None => posting,
6845 });
6846 if acc.as_ref().is_some_and(|a| a.is_empty()) {
6847 return Vec::new();
6848 }
6849 }
6850 }
6851 let ids = acc.unwrap_or_default();
6852 let mut out = Vec::with_capacity(if limit > 0 {
6853 limit.min(ids.len())
6854 } else {
6855 ids.len()
6856 });
6857 let page = if limit > 0 {
6861 limit.min(ids.len())
6862 } else {
6863 ids.len()
6864 };
6865 dict.prefetch_subject_terms(&ids[..page]);
6866 for id in ids {
6867 if let Some(iri) = dict.subject_term(id) {
6868 out.push(iri);
6869 if limit > 0 && out.len() >= limit {
6870 break;
6871 }
6872 }
6873 }
6874 out
6875}
6876
6877pub struct SearchView {
6900 header: Header,
6901 dict: SectionSlot<Dictionary>,
6902 pyramid: PyramidSlot,
6903 text_index: TextIndexSlot,
6904 labels: SectionSlot<crate::label_index::LabelIndex>,
6905}
6906
6907impl SearchView {
6908 pub fn open_ranged<R: RangeReader + Send + Sync + 'static>(
6913 reader: R,
6914 ) -> Result<Self, FileError> {
6915 Self::open_ranged_with(reader, crate::v06::FIRST_READ)
6916 }
6917
6918 pub fn open_ranged_with<R: RangeReader + Send + Sync + 'static>(
6921 reader: R,
6922 first_read: u64,
6923 ) -> Result<Self, FileError> {
6924 let first_len = first_read.max(HEADER_LEN as u64).min(reader.len());
6925 let first: std::sync::Arc<[u8]> = std::sync::Arc::from(reader.read_at(0, first_len)?);
6926 let reader = crate::reader::PrefixReader::new(reader, first.clone());
6927 if crate::v06::open::is_v06(&first) {
6928 let rc = reader.concurrency();
6929 let r: std::sync::Arc<dyn RangeReader + Send + Sync> = std::sync::Arc::new(reader);
6930 let f = std::sync::Arc::new(crate::v06::open::V06File::open(r, &first)?);
6931 let header = crate::v06::logical_header(&f.h);
6932 let (pyramid, text_index, labels, _) = v06_slots(&f);
6933 let derived = f.h.flags & crate::v06::F_DERIVED != 0;
6936 let want = [true, true, derived, derived];
6937 let make = move |f: &crate::v06::open::V06File| -> Result<Dictionary, FileError> {
6938 let d = f.dictionary(want)?;
6939 if derived {
6940 d.wire_derived(std::sync::Arc::new(f.index(rc)?));
6941 }
6942 Ok(d)
6943 };
6944 let dict = if f.h.has(crate::v06::P_LABELS) {
6945 let a = f.clone();
6946 SectionSlot::Lazy {
6947 loader: Box::new(move || make(&a).ok()),
6948 cell: std::sync::OnceLock::new(),
6949 }
6950 } else {
6951 SectionSlot::Resident(Some(make(&f)?))
6952 };
6953 return Ok(Self {
6954 header,
6955 dict,
6956 pyramid,
6957 text_index,
6958 labels,
6959 });
6960 }
6961 let head = reader.read_at(0, HEADER_LEN as u64)?;
6962 let header = Header::from_bytes(&head)?;
6963 let reader = std::sync::Arc::new(reader);
6964 let dict = if header.label_index_section().is_some() {
6968 let (r, h) = (reader.clone(), header.clone());
6969 SectionSlot::Lazy {
6970 loader: Box::new(move || {
6971 ranged_chunked_dictionary(&r, &h, [true, true, false, false]).ok()
6972 }),
6973 cell: std::sync::OnceLock::new(),
6974 }
6975 } else {
6976 SectionSlot::Resident(Some(ranged_chunked_dictionary(
6977 &reader,
6978 &header,
6979 [true, true, false, false],
6980 )?))
6981 };
6982 let pyramid = ranged_pyramid_slot(&reader, &header);
6983 let text_index = ranged_text_index_slot(&reader, &header);
6984 let labels = ranged_label_slot(&reader, &header);
6985 Ok(Self {
6986 header,
6987 dict,
6988 pyramid,
6989 text_index,
6990 labels,
6991 })
6992 }
6993
6994 pub fn has_labels(&self) -> bool {
6998 self.header.label_index_section().is_some() || self.header.pyramid_meta_len > 0
6999 }
7000
7001 pub fn has_label_index(&self) -> bool {
7003 self.header.label_index_section().is_some()
7004 }
7005
7006 pub fn header(&self) -> &Header {
7008 &self.header
7009 }
7010
7011 pub fn has_text_index(&self) -> bool {
7013 self.header.text_index_len > 0
7014 }
7015
7016 pub fn has_pyramid(&self) -> bool {
7019 self.header.pyramid_meta_len > 0
7020 }
7021
7022 pub fn text_search(&self, words: &[&str], prefix: Option<&str>, limit: usize) -> Vec<String> {
7025 let Some(dict) = self.dict.get() else {
7026 return Vec::new();
7027 };
7028 text_search_in(self.text_index.index(), dict, words, prefix, limit)
7029 }
7030
7031 pub fn text_index_token_table_len(&self) -> Option<u64> {
7035 self.text_index.token_table_len()
7036 }
7037
7038 pub fn prefix_search(&self, prefix: &str, limit: usize) -> Vec<(String, String)> {
7042 if let Some(li) = self.labels.get() {
7043 return li.prefix_search(prefix, limit);
7044 }
7045 let pyr = match &self.pyramid {
7046 PyramidSlot::Resident(p) => p.as_ref(),
7047 PyramidSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
7048 };
7049 let (Some(pyr), Some(dict)) = (pyr, self.dict.get()) else {
7050 return Vec::new();
7051 };
7052 resolve_label_hits(dict, pyr.prefix_search(prefix, limit))
7053 }
7054}
7055
7056#[must_use]
7061pub struct SummaryView {
7062 pub round: u32,
7063 pub summary: Vec<SuperEdge>,
7064 pub class_hierarchy: Vec<ClassNode>,
7066 pub level_rollups: Vec<LevelRollup>,
7068 pub level_links: Vec<LevelLinks>,
7070 pub descriptors: Vec<CommunityDescriptor>,
7072 pub subclass_cycles: Vec<Vec<String>>,
7074 pub disjoint_pairs: Vec<(String, String)>,
7076 pub equivalent_pairs: Vec<(String, String)>,
7078 dict: Dictionary,
7079}
7080
7081impl SummaryView {
7082 pub fn open_ranged<R: RangeReader>(reader: &R) -> Result<Option<Self>, FileError> {
7084 let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
7085 let header = Header::from_bytes(&head)?;
7086 if header.pyramid_meta_len == 0 {
7087 return Ok(None);
7088 }
7089 if crate::v06::reads(header.version) {
7090 return Self::open_ranged_v06(reader, &header, head).map(Some);
7091 }
7092
7093 let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
7094 let dict =
7095 decode_dictionary_container(&dict_bytes, header.dict_codec, header.term_order())?;
7096
7097 let mb = reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?;
7098 let mut meta =
7099 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7100 if let Some(s) = read_schema_with_header(reader, &header) {
7103 meta.class_hierarchy = s.class_hierarchy;
7104 meta.level_rollups = s.level_rollups;
7105 meta.level_links = s.level_links;
7106 meta.subclass_cycles = s.subclass_cycles;
7107 meta.disjoint_pairs = s.disjoint_pairs;
7108 meta.equivalent_pairs = s.equivalent_pairs;
7109 }
7110
7111 Ok(Some(SummaryView {
7112 round: meta.round,
7113 summary: meta.summary,
7114 class_hierarchy: meta.class_hierarchy,
7115 level_rollups: meta.level_rollups,
7116 level_links: meta.level_links,
7117 descriptors: meta.descriptors,
7118 subclass_cycles: meta.subclass_cycles,
7119 disjoint_pairs: meta.disjoint_pairs,
7120 equivalent_pairs: meta.equivalent_pairs,
7121 dict,
7122 }))
7123 }
7124
7125 fn open_ranged_v06<R: RangeReader>(
7129 reader: &R,
7130 header: &Header,
7131 head: Vec<u8>,
7132 ) -> Result<Self, FileError> {
7133 use crate::v06::{kind, open};
7134 let mut first = head;
7138 let want_first = crate::v06::FIRST_READ.min(reader.len());
7139 if want_first > first.len() as u64 {
7140 let at = first.len() as u64;
7141 first.extend_from_slice(&reader.read_at(at, want_first - at)?);
7142 }
7143 let pre = open::Prefixed {
7144 first: &first,
7145 r: reader,
7146 };
7147 let (h, root) = open::read_front(&pre)?;
7148 let want: Vec<ByteRange> = [
7149 kind::PYRAMID,
7150 kind::SCHEMA,
7151 kind::DICT_LEAVES,
7152 kind::DICT_DATA,
7153 ]
7154 .iter()
7155 .filter_map(|&k| root.section(k).and_then(|s| s.abs(&h).ok()))
7156 .collect();
7157 let sparse = open::SparseReader::fetch(&pre, first.clone(), &want)?;
7158 let mb = open::read_section(&sparse, &h, &root, kind::PYRAMID)?
7159 .ok_or(FileError::Container("0x06 file has no pyramid"))?;
7160 let mut meta =
7161 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7162 if let Some(s) = read_schema_with_header(&sparse, header) {
7163 meta.class_hierarchy = s.class_hierarchy;
7164 meta.level_rollups = s.level_rollups;
7165 meta.level_links = s.level_links;
7166 meta.subclass_cycles = s.subclass_cycles;
7167 meta.disjoint_pairs = s.disjoint_pairs;
7168 meta.equivalent_pairs = s.equivalent_pairs;
7169 }
7170 let f = open::V06File::open(std::sync::Arc::new(sparse), &first)?;
7171 let dict = f.dictionary([true; 4])?;
7172 Ok(SummaryView {
7173 round: meta.round,
7174 summary: meta.summary,
7175 class_hierarchy: meta.class_hierarchy,
7176 level_rollups: meta.level_rollups,
7177 level_links: meta.level_links,
7178 descriptors: meta.descriptors,
7179 subclass_cycles: meta.subclass_cycles,
7180 disjoint_pairs: meta.disjoint_pairs,
7181 equivalent_pairs: meta.equivalent_pairs,
7182 dict,
7183 })
7184 }
7185
7186 pub fn open_schema_ranged<R: RangeReader>(reader: &R) -> Result<Option<Self>, FileError> {
7194 let head = reader.read_at(0, HEADER_LEN as u64)?;
7195 let header = Header::from_bytes(&head)?;
7196 let Some(s) = read_schema_with_header(reader, &header) else {
7197 return Ok(None);
7198 };
7199 let dict = crate::DictionaryBuilder::new().build();
7200 Ok(Some(SummaryView {
7201 round: 0,
7202 summary: Vec::new(),
7203 class_hierarchy: s.class_hierarchy,
7204 level_rollups: s.level_rollups,
7205 level_links: s.level_links,
7206 descriptors: Vec::new(),
7207 subclass_cycles: s.subclass_cycles,
7208 disjoint_pairs: s.disjoint_pairs,
7209 equivalent_pairs: s.equivalent_pairs,
7210 dict,
7211 }))
7212 }
7213
7214 pub fn level_count(&self) -> usize {
7216 self.level_rollups.len()
7217 }
7218
7219 pub fn level_rollup(&self, k: usize) -> Option<&LevelRollup> {
7222 self.level_rollups.get(k)
7223 }
7224
7225 pub fn predicate_term(&self, id: u32) -> Option<String> {
7227 self.dict.predicate_term(id)
7228 }
7229
7230 pub fn predicate_total(&self, predicate: &str) -> u32 {
7233 match self.dict.predicate_id(predicate) {
7234 Some(pid) => self
7235 .summary
7236 .iter()
7237 .filter(|e| e.predicate == pid)
7238 .map(|e| e.count)
7239 .sum(),
7240 None => 0,
7241 }
7242 }
7243
7244 pub fn predicate_totals(&self) -> Vec<(String, u32)> {
7246 let mut by_pred: std::collections::BTreeMap<u32, u32> = std::collections::BTreeMap::new();
7247 for e in &self.summary {
7248 *by_pred.entry(e.predicate).or_default() += e.count;
7249 }
7250 let mut out: Vec<(String, u32)> = by_pred
7251 .into_iter()
7252 .filter_map(|(pid, c)| self.dict.predicate_term(pid).map(|t| (t, c)))
7253 .collect();
7254 out.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
7255 out
7256 }
7257
7258 pub fn community_count(&self) -> usize {
7260 let mut comms = std::collections::BTreeSet::new();
7261 for e in &self.summary {
7262 comms.insert(e.s_comm);
7263 comms.insert(e.o_comm);
7264 }
7265 comms.len()
7266 }
7267
7268 pub fn tbox_coherence(&self) -> Vec<crate::reason::Inconsistency> {
7283 schema_coherence(
7284 &self.class_hierarchy,
7285 &self.subclass_cycles,
7286 &self.disjoint_pairs,
7287 &self.equivalent_pairs,
7288 )
7289 }
7290
7291 pub fn tbox_is_coherent(&self) -> bool {
7294 self.tbox_coherence().is_empty()
7295 }
7296}
7297
7298pub fn schema_coherence(
7304 class_hierarchy: &[ClassNode],
7305 subclass_cycles: &[Vec<String>],
7306 disjoint_pairs: &[(String, String)],
7307 equivalent_pairs: &[(String, String)],
7308) -> Vec<crate::reason::Inconsistency> {
7309 use crate::reason::Inconsistency;
7310 use std::collections::{BTreeMap, BTreeSet, VecDeque};
7311 const MAX_REACH: usize = 100_000;
7312
7313 let mut out: Vec<Inconsistency> = Vec::new();
7314
7315 for cyc in subclass_cycles {
7316 let detail = if cyc.len() == 1 {
7317 format!("{} is rdfs:subClassOf itself (a cycle)", cyc[0])
7318 } else {
7319 format!(
7320 "classes {{{}}} are mutually rdfs:subClassOf (a cycle)",
7321 cyc.join(", ")
7322 )
7323 };
7324 out.push(Inconsistency {
7325 kind: "subclass-cycle",
7326 detail,
7327 });
7328 }
7329
7330 if !disjoint_pairs.is_empty() {
7331 let mut adj: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
7333 for n in class_hierarchy {
7334 let e = adj.entry(n.class.as_str()).or_default();
7335 for p in &n.parents {
7336 e.push(p.as_str());
7337 }
7338 }
7339 for (a, b) in equivalent_pairs {
7340 adj.entry(a.as_str()).or_default().push(b.as_str());
7341 adj.entry(b.as_str()).or_default().push(a.as_str());
7342 }
7343
7344 let mut focuses: BTreeSet<&str> =
7346 class_hierarchy.iter().map(|n| n.class.as_str()).collect();
7347 for (a, b) in disjoint_pairs.iter().chain(equivalent_pairs) {
7348 focuses.insert(a.as_str());
7349 focuses.insert(b.as_str());
7350 }
7351
7352 let mut seen: BTreeSet<&str> = BTreeSet::new();
7353 for &c in &focuses {
7354 let mut reach: BTreeSet<&str> = BTreeSet::new();
7356 let mut q: VecDeque<&str> = VecDeque::new();
7357 reach.insert(c);
7358 q.push_back(c);
7359 while let Some(x) = q.pop_front() {
7360 if reach.len() > MAX_REACH {
7361 break;
7362 }
7363 if let Some(ns) = adj.get(x) {
7364 for &p in ns {
7365 if reach.insert(p) {
7366 q.push_back(p);
7367 }
7368 }
7369 }
7370 }
7371 for (x, y) in disjoint_pairs {
7372 if reach.contains(x.as_str()) && reach.contains(y.as_str()) && seen.insert(c) {
7373 out.push(Inconsistency {
7374 kind: "unsatisfiable-class",
7375 detail: format!(
7376 "{c} is a subclass of both {x} and {y}, which are \
7377 owl:disjointWith — no individual can be a {c}"
7378 ),
7379 });
7380 break;
7381 }
7382 }
7383 }
7384 }
7385
7386 out.sort_by(|a, b| (a.kind, &a.detail).cmp(&(b.kind, &b.detail)));
7387 out
7388}
7389
7390pub fn read_schema_coherence_ranged<R: RangeReader>(
7398 reader: &R,
7399) -> Result<Option<Vec<crate::reason::Inconsistency>>, FileError> {
7400 let head = reader.read_at(0, HEADER_LEN as u64)?;
7401 let header = Header::from_bytes(&head)?;
7402 if let Some(s) = read_schema_with_header(reader, &header) {
7404 return Ok(Some(schema_coherence(
7405 &s.class_hierarchy,
7406 &s.subclass_cycles,
7407 &s.disjoint_pairs,
7408 &s.equivalent_pairs,
7409 )));
7410 }
7411 if header.pyramid_meta_len == 0 {
7412 return Ok(None);
7413 }
7414 if header.schema_meta_len > 0 && (header.schema_meta_len as u64) <= header.pyramid_meta_len {
7418 let off =
7419 header.pyramid_meta_offset + header.pyramid_meta_len - header.schema_meta_len as u64;
7420 let block = reader.read_at(off, header.schema_meta_len as u64)?;
7421 let (hierarchy, cycles, disjoint, equivalent) = crate::meta::decode_schema_block(&block)
7422 .map_err(|_| FileError::Container("malformed schema block"))?;
7423 return Ok(Some(schema_coherence(
7424 &hierarchy,
7425 &cycles,
7426 &disjoint,
7427 &equivalent,
7428 )));
7429 }
7430 let mb = if crate::v06::reads(header.version) {
7432 crate::v06::open::read_pyramid(reader)?.unwrap_or_default()
7433 } else {
7434 reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?
7435 };
7436 let meta =
7437 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7438 Ok(Some(schema_coherence(
7439 &meta.class_hierarchy,
7440 &meta.subclass_cycles,
7441 &meta.disjoint_pairs,
7442 &meta.equivalent_pairs,
7443 )))
7444}
7445
7446#[allow(clippy::type_complexity)]
7454pub fn read_schema_summary_ranged<R: RangeReader>(
7455 reader: &R,
7456) -> Result<Option<(Vec<(String, u64)>, Vec<(String, String, String, u64)>)>, FileError> {
7457 let head = reader.read_at(0, HEADER_LEN as u64)?;
7458 let header = Header::from_bytes(&head)?;
7459 let meta = if let Some(s) = read_schema_with_header(reader, &header) {
7461 PyramidMeta::new(0, Vec::new(), &[]).with_schema(
7462 Vec::new(),
7463 s.level_rollups,
7464 s.level_links,
7465 Vec::new(),
7466 Vec::new(),
7467 Vec::new(),
7468 Vec::new(),
7469 )
7470 } else {
7471 if header.pyramid_meta_len == 0 {
7472 return Ok(None);
7473 }
7474 if header.schema_meta_len > 0 && (header.schema_meta_len as u64) <= header.pyramid_meta_len
7475 {
7476 let off = header.pyramid_meta_offset + header.pyramid_meta_len
7477 - header.schema_meta_len as u64;
7478 let block = reader.read_at(off, header.schema_meta_len as u64)?;
7479 let summary = crate::meta::decode_schema_block_summary(&block)
7480 .map_err(|_| FileError::Container("malformed schema block"))?;
7481 return Ok(Some(summary));
7482 }
7483 let mb = if crate::v06::reads(header.version) {
7485 crate::v06::open::read_pyramid(reader)?.unwrap_or_default()
7486 } else {
7487 reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?
7488 };
7489 PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?
7490 };
7491 if meta.level_rollups.is_empty() && meta.level_links.is_empty() {
7492 return Ok(None);
7493 }
7494 let classes = meta
7495 .level_rollups
7496 .iter()
7497 .max_by_key(|r| r.depth)
7498 .map(|r| r.classes.clone())
7499 .unwrap_or_default();
7500 let relations = meta
7501 .level_links
7502 .iter()
7503 .max_by_key(|l| l.depth)
7504 .map(|l| {
7505 l.links
7506 .iter()
7507 .map(|c| {
7508 (
7509 c.s_class.clone(),
7510 c.predicate.clone(),
7511 c.o_class.clone(),
7512 c.count,
7513 )
7514 })
7515 .collect()
7516 })
7517 .unwrap_or_default();
7518 Ok(Some((classes, relations)))
7519}
7520
7521#[cfg(test)]
7526pub(crate) fn dict_chunk_keys_for_test(image: &[u8]) -> Vec<Vec<Vec<u8>>> {
7527 if crate::v06::open::is_v06(image) {
7528 use crate::v06::{decode_chunk_leaf, decode_dict_root, kind, open, tag};
7530 let r = crate::reader::SliceReader::new(image);
7531 let (h, root) = open::read_front(&r).unwrap();
7532 let roots = decode_dict_root(root.unit(tag::DICT).unwrap(), h.version).unwrap();
7533 let leaves = open::read_section(&r, &h, &root, kind::DICT_LEAVES)
7534 .unwrap()
7535 .unwrap();
7536 return roots
7537 .iter()
7538 .map(|s| {
7539 s.leaves
7540 .iter()
7541 .flat_map(|(l, _, _, _)| {
7542 let stored = &leaves[l.off as usize..(l.off + l.len) as usize];
7543 let raw = decompress(h.internal_codec, stored).unwrap();
7544 decode_chunk_leaf(&raw, l.n).unwrap()
7545 })
7546 .map(|c| c.key)
7547 .collect()
7548 })
7549 .collect();
7550 }
7551 let header = Header::from_bytes(&image[..HEADER_LEN]).unwrap();
7552 let s = header.dictionary_offset as usize;
7553 let e = s + header.dictionary_len as usize;
7554 decode_container(&image[s..e], CODEC_NONE)
7555 .unwrap()
7556 .iter()
7557 .map(|payload| {
7558 let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
7559 entries.into_iter().map(|c| c.key).collect()
7560 })
7561 .collect()
7562}
7563
7564#[cfg(test)]
7565mod tests {
7566 use super::*;
7567 use super::{
7568 write_dataset_v05 as write_dataset,
7569 write_dataset_with_metadata_v05 as write_dataset_with_metadata,
7570 write_file_v05 as write_file,
7571 };
7572 use crate::dictionary::DictionaryBuilder;
7573 use crate::index::GraphIndexBuilder;
7574
7575 #[cfg(not(feature = "compression"))]
7578 #[test]
7579 fn ruzstd_decodes_a_zstd_frame() {
7580 let data: Vec<u8> = (0..50_000u32)
7581 .flat_map(|i| (i % 251).to_le_bytes())
7582 .collect();
7583 let frame = ruzstd::encoding::compress_to_vec(
7584 &data[..],
7585 ruzstd::encoding::CompressionLevel::Fastest,
7586 );
7587 assert!(frame.len() < data.len());
7588 assert_eq!(decompress(CODEC_ZSTD, &frame).unwrap(), data);
7589 }
7590
7591 #[cfg(feature = "compression")]
7596 #[test]
7597 fn decompression_is_bounded() {
7598 let data: Vec<u8> = (0..100_000u32).map(|i| (i % 251) as u8).collect();
7599 let sized = zstd::bulk::compress(&data, 3).unwrap();
7600 assert_eq!(zstd_frame_content_size(&sized), Some(100_000));
7601 assert_eq!(block_raw_len(CODEC_ZSTD, &sized, 1 << 20).unwrap(), 100_000);
7602 assert_eq!(
7603 decompress_bounded(CODEC_ZSTD, &sized, 100_000).unwrap(),
7604 data
7605 );
7606 assert!(decompress_bounded(CODEC_ZSTD, &sized, 99_999).is_err());
7607 let stream = zstd::encode_all(&data[..], 3).unwrap();
7609 assert_eq!(zstd_frame_content_size(&stream), None);
7610 assert_eq!(
7611 block_raw_len(CODEC_ZSTD, &stream, 1 << 20).unwrap(),
7612 100_000
7613 );
7614 assert_eq!(
7615 decompress_bounded(CODEC_ZSTD, &stream, 100_000).unwrap(),
7616 data
7617 );
7618 assert!(decompress_bounded(CODEC_ZSTD, &stream, 50_000).is_err());
7619 let mut liar = sized.clone();
7621 let fhd = liar[4];
7622 let fcs_at = 5 + usize::from(fhd & 0x20 == 0);
7623 liar[fcs_at] = liar[fcs_at].wrapping_sub(1);
7624 assert!(decompress_bounded(CODEC_ZSTD, &liar, 200_000).is_err());
7625 assert!(decompress_bounded(CODEC_NONE, &data, 99_999).is_err());
7627 assert_eq!(
7628 decompress_bounded(CODEC_NONE, &data, 100_000).unwrap(),
7629 data
7630 );
7631 assert_eq!(zstd_frame_content_size(b"RETE"), None);
7632 }
7633
7634 #[test]
7640 fn three_permutation_file_round_trips_every_open_path() {
7641 use crate::index::PermSet;
7642 use crate::reader::SliceReader;
7643
7644 let build = |perms: PermSet| {
7645 let mut db = crate::DictionaryBuilder::new();
7646 let mut triples = Vec::new();
7647 for i in 0..60u32 {
7648 let (s, p, o) = (
7649 format!("<http://ex/s{}>", i % 11),
7650 format!("<http://ex/p{}>", i % 3),
7651 format!("<http://ex/o{}>", i % 7),
7652 );
7653 db.observe(&s, &p, &o);
7654 triples.push((s, p, o));
7655 }
7656 let dict = db.build();
7657 let ids: Vec<(u32, u32, u32)> = triples
7658 .iter()
7659 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
7660 .collect();
7661 let def = GraphIndexBuilder::from_triples(ids.clone())
7662 .with_tile_budget(96)
7663 .with_perms(perms)
7664 .build();
7665 let named = GraphIndexBuilder::from_triples(ids)
7666 .with_tile_budget(96)
7667 .with_perms(perms)
7668 .build();
7669 write_dataset(
7670 &dict,
7671 &def,
7672 &[("<http://ex/g>".to_string(), named)],
7673 true,
7674 &[],
7675 0,
7676 )
7677 };
7678
7679 let six = build(PermSet::ALL);
7680 let three = build(PermSet::CORE);
7681 assert!(three.len() < six.len());
7682 assert_eq!(Header::from_bytes(&three).unwrap().perms, PermSet::CORE);
7683 assert_eq!(Header::from_bytes(&six).unwrap().perms, PermSet::ALL);
7684
7685 let shapes: Vec<(Option<&str>, Option<&str>, Option<&str>)> = vec![
7686 (None, None, None),
7687 (Some("<http://ex/s3>"), None, None),
7688 (None, Some("<http://ex/p1>"), None),
7689 (None, None, Some("<http://ex/o5>")),
7690 (Some("<http://ex/s3>"), Some("<http://ex/p1>"), None),
7691 (Some("<http://ex/s3>"), None, Some("<http://ex/o5>")),
7692 (None, Some("<http://ex/p1>"), Some("<http://ex/o5>")),
7693 (
7694 Some("<http://ex/s3>"),
7695 Some("<http://ex/p1>"),
7696 Some("<http://ex/o5>"),
7697 ),
7698 ];
7699
7700 for (image, tag) in [(&six, "six"), (&three, "three")] {
7701 let resident = Rete::open(image).unwrap();
7702 let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
7703 let ranged = Rete::open_ranged(&SliceReader::new(leaked)).unwrap();
7704 let lazy =
7705 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
7706 for (s, p, o) in &shapes {
7707 let want: Vec<_> = {
7708 let mut v = Rete::open(&six).unwrap().query(*s, *p, *o);
7709 v.sort();
7710 v
7711 };
7712 for (got, path) in [
7713 (resident.query(*s, *p, *o), "resident"),
7714 (ranged.query(*s, *p, *o), "ranged"),
7715 (lazy.query(*s, *p, *o), "lazy"),
7716 ] {
7717 let mut got = got;
7718 got.sort();
7719 assert_eq!(got, want, "{tag}/{path} disagreed on {:?}", (s, p, o));
7720 }
7721 let mut g = resident.query_in_graph(Some("<http://ex/g>"), *s, *p, *o);
7723 g.sort();
7724 assert_eq!(g, want, "{tag}/named disagreed on {:?}", (s, p, o));
7725 }
7726 }
7727
7728 let leaked: &'static [u8] = Box::leak(three.clone().into_boxed_slice());
7732 let reader = SliceReader::new(leaked);
7733 let routed = Rete::query_ranged(&reader, None, None, Some("<http://ex/o5>")).unwrap();
7734 let mut want = Rete::open(&six)
7735 .unwrap()
7736 .query(None, None, Some("<http://ex/o5>"));
7737 want.sort();
7738 let mut got = routed;
7739 got.sort();
7740 assert_eq!(got, want, "routed single-pattern read on a lean file");
7741 }
7742
7743 #[test]
7744 fn read_coalesced_merges_within_gap_and_splits_beyond() {
7745 use crate::reader::{CountingReader, SliceReader};
7746 let bytes = vec![0u8; 4096];
7747 let ranges = [
7749 ByteRange { offset: 0, len: 16 },
7750 ByteRange {
7751 offset: 48,
7752 len: 16,
7753 },
7754 ByteRange {
7755 offset: 1088,
7756 len: 16,
7757 },
7758 ];
7759 let r = CountingReader::new(SliceReader::new(&bytes));
7761 let out = read_coalesced(&r, &ranges, 16).unwrap();
7762 assert_eq!(out.len(), 3);
7763 assert_eq!(r.requests(), 3);
7764 let r = CountingReader::new(SliceReader::new(&bytes));
7766 read_coalesced(&r, &ranges, 64).unwrap();
7767 assert_eq!(r.requests(), 2);
7768 let r = CountingReader::new(SliceReader::new(&bytes));
7770 read_coalesced(&r, &ranges, 4096).unwrap();
7771 assert_eq!(r.requests(), 1);
7772 }
7773
7774 fn build_image() -> Vec<u8> {
7775 let triples = [
7776 ("Alice", "knows", "Bob"),
7777 ("Bob", "knows", "Carol"),
7778 ("Alice", "age", "30"),
7779 ];
7780 let mut db = DictionaryBuilder::new();
7781 for (s, p, o) in triples {
7782 db.observe(s, p, o);
7783 }
7784 let dict = db.build();
7785
7786 let mut ib = GraphIndexBuilder::new();
7787 for (s, p, o) in triples {
7788 ib.push(dict.encode(s, p, o).unwrap());
7789 }
7790 let index = ib.build();
7791
7792 let (meta, levels) = build_pyramid_meta(&dict, &triples_ids(&dict), DEFAULT_TILE_BUDGET);
7793 write_file(&dict, &index, false, &meta, levels)
7794 }
7795
7796 fn triples_ids(dict: &Dictionary) -> Vec<(u32, u32, u32)> {
7797 [
7798 ("Alice", "knows", "Bob"),
7799 ("Bob", "knows", "Carol"),
7800 ("Alice", "age", "30"),
7801 ]
7802 .iter()
7803 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
7804 .collect()
7805 }
7806
7807 #[test]
7808 fn file_round_trips_header_and_counts() {
7809 let bytes = build_image();
7810 let rete = Rete::open(&bytes).unwrap();
7811 assert_eq!(rete.header().quad_count, 3);
7812 assert!(rete.header().term_count >= 5);
7813 let expected_codec = writer_codec();
7814 assert_eq!(rete.header().dict_codec, expected_codec);
7815 assert_eq!(rete.header().block_codec, expected_codec);
7816 assert_eq!(&bytes[bytes.len() - 4..], &MAGIC); }
7818
7819 #[test]
7824 fn multi_tile_file_round_trips_and_routes() {
7825 let triples: Vec<(String, String, String)> = (0..200)
7826 .map(|i| {
7827 (
7828 format!("<http://ex/s/{i}>"),
7829 format!("<http://ex/p/{}>", i % 5),
7830 format!("<http://ex/o/{}>", i % 23),
7831 )
7832 })
7833 .collect();
7834 let mut db = DictionaryBuilder::new();
7835 for (s, p, o) in &triples {
7836 db.observe(s, p, o);
7837 }
7838 let dict = db.build();
7839 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7840 for (s, p, o) in &triples {
7841 ib.push(dict.encode(s, p, o).unwrap());
7842 }
7843 let index = ib.build();
7844 assert!(
7845 index.tile_sections()[0].len() > 3,
7846 "tiny budget must force many tiles"
7847 );
7848 let bytes = write_file(&dict, &index, false, &[], 0);
7849
7850 let rete = Rete::open(&bytes).unwrap();
7851 assert_eq!(rete.header().version, crate::header::LEGACY_FORMAT_VERSION);
7852 assert_eq!(rete.query(None, None, None).len(), 200);
7853 assert_eq!(rete.query(Some("<http://ex/s/7>"), None, None).len(), 1);
7854 assert_eq!(
7855 rete.query(None, Some("<http://ex/p/3>"), None).len(),
7856 40,
7857 "predicate extent spans tiles"
7858 );
7859 assert_eq!(
7860 rete.query(None, None, Some("<http://ex/o/22>")).len(),
7861 8 );
7863
7864 use crate::reader::SliceReader;
7866 let reader = SliceReader::new(&bytes);
7867 let routed = Rete::query_ranged(&reader, Some("<http://ex/s/7>"), None, None).unwrap();
7868 assert_eq!(routed.len(), 1);
7869 let routed = Rete::query_ranged(&reader, None, Some("<http://ex/p/3>"), None).unwrap();
7870 assert_eq!(routed.len(), 40);
7871 let routed = Rete::query_ranged(&reader, None, None, Some("<http://ex/o/22>")).unwrap();
7872 assert_eq!(routed.len(), 8);
7873 }
7874
7875 #[test]
7884 fn tile_directory_offsets_survive_past_4gib() {
7885 let mut dir = Vec::new();
7886 write_uvarint(&mut dir, 2); write_uvarint(&mut dir, 5); write_uvarint(&mut dir, 0); write_uvarint(&mut dir, 3 << 30); write_uvarint(&mut dir, 1); write_uvarint(&mut dir, 0);
7892 write_uvarint(&mut dir, 2 << 30); let total = dir.len() as u64 + (3u64 << 30) + (2u64 << 30) + 64;
7894 let entries = parse_tile_directory(&dir, total).unwrap();
7895 assert_eq!(entries.len(), 2);
7896 assert_eq!(entries[1].start, dir.len() as u64 + (3u64 << 30));
7897 assert!(
7898 entries[1].end > u32::MAX as u64,
7899 "tail tile sits past 4 GiB"
7900 );
7901 assert!(parse_tile_directory(&dir, 1 << 20).is_err());
7903 }
7904
7905 #[test]
7906 fn tile_synopsis_trailer_round_trips() {
7907 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7908 for i in 0..200u32 {
7909 ib.push((i, i % 7, i % 13));
7910 }
7911 let index = ib.build();
7912 let tile_count = index.tile_sections()[0].len();
7913 assert!(tile_count > 3, "tiny budget forces many tiles");
7914
7915 let payload = encode_tiled_section(&index, 0, CODEC_NONE);
7916 let dir = parse_tile_directory(&payload, payload.len() as u64).unwrap();
7917 assert_eq!(dir.len(), tile_count);
7918 let trailer_start = dir.iter().map(|e| e.end).max().unwrap();
7920 assert!(
7921 trailer_start < payload.len() as u64,
7922 "a trailer follows the tiles"
7923 );
7924 for e in &dir {
7925 assert!(
7926 e.end <= payload.len() as u64,
7927 "tiles still located within the payload"
7928 );
7929 }
7930 let syn = parse_tile_synopsis(&payload, trailer_start as usize, dir.len()).unwrap();
7931 for (e, (min_b, max_b, min_c, max_c)) in dir.iter().zip(syn) {
7932 let block = decompress(CODEC_NONE, &payload[e.start as usize..e.end as usize]).unwrap();
7933 let z = *crate::triples::TripleBlock::parse(&block).unwrap().zone();
7934 assert_eq!(
7935 (min_b, max_b, min_c, max_c),
7936 (z.min_b, z.max_b, z.min_c, z.max_c),
7937 "synopsis equals the tile's own zone"
7938 );
7939 }
7940 }
7941
7942 #[test]
7946 fn tile_synopsis_lazy_matches_reference_every_shape() {
7947 use crate::reader::{CountingReader, SliceReader};
7948 let triples: Vec<(String, String, String)> = (0..200u32)
7949 .map(|i| {
7950 (
7951 format!("<http://ex/s/{i:04}>"),
7952 format!("<http://ex/p/{}>", i % 7),
7953 format!("<http://ex/o/{:04}>", i % 13),
7954 )
7955 })
7956 .collect();
7957 let mut db = DictionaryBuilder::new();
7958 for (s, p, o) in &triples {
7959 db.observe(s, p, o);
7960 }
7961 let dict = db.build();
7962 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7963 for (s, p, o) in &triples {
7964 ib.push(dict.encode(s, p, o).unwrap());
7965 }
7966 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
7967
7968 let eager = Rete::open(&bytes).unwrap();
7969 assert!(
7970 eager.header().has_tile_synopsis(),
7971 "new files set the synopsis flag"
7972 );
7973
7974 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
7977 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
7978 let lazy = Rete::open_ranged_lazy_with(reader, HEADER_LEN as u64).unwrap();
7979
7980 let brute = |s: Option<&str>, p: Option<&str>, o: Option<&str>| {
7981 let mut v: Vec<(String, String, String)> = triples
7982 .iter()
7983 .filter(|(a, b, c)| {
7984 s.is_none_or(|x| x == a) && p.is_none_or(|x| x == b) && o.is_none_or(|x| x == c)
7985 })
7986 .cloned()
7987 .collect();
7988 v.sort();
7989 v
7990 };
7991 let sv = [
7993 None,
7994 Some("<http://ex/s/0007>"),
7995 Some("<http://ex/s/0130>"),
7996 Some("<http://ex/s/9999>"),
7997 ];
7998 let pv = [
7999 None,
8000 Some("<http://ex/p/3>"),
8001 Some("<http://ex/p/6>"),
8002 Some("<http://ex/p/999>"),
8003 ];
8004 let ov = [
8005 None,
8006 Some("<http://ex/o/0000>"),
8007 Some("<http://ex/o/0012>"),
8008 Some("<http://ex/o/9999>"),
8009 ];
8010 for &s in &sv {
8011 for &p in &pv {
8012 for &o in &ov {
8013 let mut e = eager.query(s, p, o);
8014 e.sort();
8015 let mut l = lazy.query(s, p, o);
8016 l.sort();
8017 let r = brute(s, p, o);
8018 assert_eq!(e, r, "eager {s:?} {p:?} {o:?}");
8019 assert_eq!(l, r, "lazy {s:?} {p:?} {o:?} — synopsis over-pruned");
8020 }
8021 }
8022 }
8023 assert!(!lazy.index_incomplete(), "no lazy fetch failed");
8024 }
8025
8026 #[test]
8030 fn polyglot_offset_reads_lazily() {
8031 use crate::reader::{
8032 detect_polyglot_base, CountingReader, OffsetReader, SliceReader, POLYGLOT_DIGITS,
8033 POLYGLOT_MARKER,
8034 };
8035
8036 let image = build_image();
8040 let mut shell = Vec::new();
8041 shell.extend_from_slice(b"<!DOCTYPE html><html><head><!--");
8042 shell.extend_from_slice(POLYGLOT_MARKER);
8043 let digits_at = shell.len();
8044 shell.extend_from_slice(&[b'0'; POLYGLOT_DIGITS]); shell.extend_from_slice(b"--></head><body>a web page</body></html>\n");
8046 shell.resize(50_000, b' '); let base = shell.len() as u64;
8048 let digits = format!("{base:0width$}", width = POLYGLOT_DIGITS);
8049 shell[digits_at..digits_at + POLYGLOT_DIGITS].copy_from_slice(digits.as_bytes());
8050
8051 let mut poly = shell;
8052 poly.extend_from_slice(&image);
8053
8054 assert_ne!(
8056 &poly[0..4],
8057 b"RETE",
8058 "polyglot must not start with the magic"
8059 );
8060 let detected = detect_polyglot_base(&poly[..crate::header::HEADER_LEN]).unwrap();
8062 assert_eq!(detected, base);
8063
8064 let leaked: &'static [u8] = Box::leak(poly.into_boxed_slice());
8066 let counting = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8067 let lazy = Rete::open_ranged_lazy_with(
8068 OffsetReader::new(counting.clone(), detected),
8069 HEADER_LEN as u64,
8070 )
8071 .unwrap();
8072
8073 let plain = Rete::open(&image).unwrap();
8075 let mut want = plain.query(None, None, None);
8076 want.sort();
8077 let mut got = lazy.query(None, None, None);
8078 got.sort();
8079 assert_eq!(
8080 got, want,
8081 "polyglot lazy query differs from the plain .rete"
8082 );
8083 assert!(!got.is_empty(), "expected some triples");
8084 assert!(!lazy.index_incomplete(), "a lazy fetch failed");
8085
8086 let read = counting.bytes_read();
8089 eprintln!(
8090 "polyglot lazy read: HTML shell {base} B + .rete image {} B; \
8091 the query touched only {read} B (never the shell).",
8092 image.len()
8093 );
8094 assert!(
8095 read < base,
8096 "read {read} bytes; the HTML shell alone is {base} — the reader \
8097 touched the prefix instead of only the embedded .rete"
8098 );
8099 }
8100
8101 #[cfg(test)]
8104 fn build_text_indexed(triples: &[(String, String, String)]) -> Vec<u8> {
8105 let mut db = DictionaryBuilder::new();
8106 for (s, p, o) in triples {
8107 db.observe(s, p, o);
8108 }
8109 let dict = db.build();
8110 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
8111 let mut id_triples: Vec<(u32, u32, u32)> = Vec::with_capacity(triples.len());
8112 for (s, p, o) in triples {
8113 let t = dict.encode(s, p, o).unwrap();
8114 ib.push(t);
8115 id_triples.push(t);
8116 }
8117 let index = ib.build();
8118 let text_index = compute_text_index(&dict, &id_triples);
8119 assert!(
8120 !text_index.is_empty(),
8121 "literals should produce a text index"
8122 );
8123 write_dataset_with_metadata(&dict, &index, &[], false, &[], 0, &[], &text_index)
8124 }
8125
8126 #[test]
8130 fn text_index_eager_matches_brute_force() {
8131 let triples: Vec<(String, String, String)> = vec![
8132 (
8133 "<http://ex/s0>",
8134 "<http://ex/label>",
8135 "\"alpha glucose phosphate\"",
8136 ),
8137 ("<http://ex/s1>", "<http://ex/label>", "\"beta Glucose\""),
8138 ("<http://ex/s2>", "<http://ex/label>", "\"gamma fructose\""),
8139 (
8140 "<http://ex/s3>",
8141 "<http://ex/note>",
8142 "\"einstein relativity\"",
8143 ),
8144 (
8145 "<http://ex/s4>",
8146 "<http://ex/ref>",
8147 "<http://ex/not-a-literal>",
8148 ),
8149 ]
8150 .into_iter()
8151 .map(|(s, p, o)| (s.to_string(), p.to_string(), o.to_string()))
8152 .collect();
8153 let bytes = build_text_indexed(&triples);
8154 let rete = Rete::open(&bytes).unwrap();
8155 assert!(rete.has_text_index());
8156
8157 let brute = |words: &[&str]| -> Vec<String> {
8159 let mut v: Vec<String> = triples
8160 .iter()
8161 .filter(|(_, _, o)| {
8162 crate::terms::is_literal(o)
8163 && words.iter().all(|w| {
8164 let wl = w.to_lowercase();
8165 crate::terms::literal_lexical(o)
8166 .unwrap()
8167 .split(|c: char| !c.is_alphanumeric())
8168 .any(|t| t.to_lowercase() == wl)
8169 })
8170 })
8171 .map(|(s, _, _)| s.clone())
8172 .collect();
8173 v.sort();
8174 v.dedup();
8175 v
8176 };
8177
8178 let mut got = rete.text_search(&["glucose"], None, 0);
8179 got.sort();
8180 assert_eq!(got, brute(&["glucose"]), "case-insensitive single word");
8181
8182 let mut got = rete.text_search(&["glucose", "phosphate"], None, 0);
8184 got.sort();
8185 assert_eq!(got, brute(&["glucose", "phosphate"]));
8186
8187 assert!(rete.text_search(&["zzznope"], None, 0).is_empty());
8189
8190 let got = rete.text_search(&[], Some("ein"), 0);
8192 assert_eq!(got, vec!["<http://ex/s3>".to_string()]);
8193
8194 let mut db = DictionaryBuilder::new();
8196 for (s, p, o) in &triples {
8197 db.observe(s, p, o);
8198 }
8199 let dict = db.build();
8200 let mut ib = GraphIndexBuilder::new();
8201 for (s, p, o) in &triples {
8202 ib.push(dict.encode(s, p, o).unwrap());
8203 }
8204 let plain = write_dataset(&dict, &ib.build(), &[], false, &[], 0);
8205 let plain_rete = Rete::open(&plain).unwrap();
8206 assert!(!plain_rete.has_text_index());
8207 assert!(plain_rete.text_search(&["glucose"], None, 0).is_empty());
8208 }
8209
8210 #[test]
8214 fn text_index_lazy_faults_only_queried_postings() {
8215 use crate::reader::{CountingReader, SliceReader};
8216 let mut triples: Vec<(String, String, String)> = (0..300u32)
8219 .map(|i| {
8220 (
8221 format!("<http://ex/s/{i:04}>"),
8222 "<http://ex/label>".to_string(),
8223 format!("\"common word number {i}\""),
8224 )
8225 })
8226 .collect();
8227 for i in [3u32, 77, 250] {
8228 triples.push((
8229 format!("<http://ex/s/{i:04}>"),
8230 "<http://ex/tag>".to_string(),
8231 "\"raretoken\"".to_string(),
8232 ));
8233 }
8234 let bytes = build_text_indexed(&triples);
8235 let eager = Rete::open(&bytes).unwrap();
8236 let mut want = eager.text_search(&["raretoken"], None, 0);
8237 want.sort();
8238 assert_eq!(want.len(), 3);
8239
8240 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8241 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8242 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
8243 let before = reader.bytes_read();
8246 let mut got = lazy.text_search(&["raretoken"], None, 0);
8247 got.sort();
8248 assert_eq!(got, want, "lazy search matches eager");
8249 let pulled = reader.bytes_read() - before;
8250 let ti_len = eager.header().text_index_len;
8253 assert!(
8254 pulled < ti_len,
8255 "search pulled {pulled} B but the section is {ti_len} B — faulted too much"
8256 );
8257 assert!(!lazy.index_incomplete());
8258 }
8259
8260 #[test]
8265 fn search_view_matches_full_open_for_far_fewer_bytes() {
8266 use crate::reader::{CountingReader, SliceReader};
8267 let filler = "lorem ipsum dolor sit amet consectetur adipiscing elit ".repeat(20);
8270 let mut triples: Vec<(String, String, String)> = (0..300u32)
8271 .map(|i| {
8272 (
8273 format!("<http://ex/s/{i:04}>"),
8274 "<http://ex/abstract>".to_string(),
8275 format!("\"{filler} number {i}\""),
8276 )
8277 })
8278 .collect();
8279 for i in [3u32, 77, 250] {
8280 triples.push((
8281 format!("<http://ex/s/{i:04}>"),
8282 "<http://ex/tag>".to_string(),
8283 "\"raretoken\"".to_string(),
8284 ));
8285 }
8286 let bytes = build_text_indexed(&triples);
8287 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8288
8289 let full_reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8290 let full = Rete::open_ranged_lazy_with(full_reader.clone(), HEADER_LEN as u64).unwrap();
8291 let mut want = full.text_search(&["raretoken"], None, 0);
8292 want.sort();
8293 assert_eq!(want.len(), 3);
8294
8295 let search_reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8296 let view = SearchView::open_ranged_with(search_reader.clone(), HEADER_LEN as u64).unwrap();
8297 assert!(view.has_text_index());
8298 let mut got = view.text_search(&["raretoken"], None, 0);
8299 got.sort();
8300 assert_eq!(got, want, "search view matches the full open");
8301 assert!(
8302 search_reader.bytes_read() < full_reader.bytes_read(),
8303 "search read {} B, full open read {} B — the narrow open bought nothing",
8304 search_reader.bytes_read(),
8305 full_reader.bytes_read()
8306 );
8307
8308 let mut prefix_hits = view.prefix_search("", 5);
8311 prefix_hits.sort();
8312 assert!(!prefix_hits.is_empty() || !view.has_pyramid());
8313 }
8314
8315 #[test]
8325 fn dict_chunk_directory_probe_costs_about_one_directory() {
8326 use crate::reader::{CountingReader, SliceReader};
8327 let head = "x".repeat(3000);
8341 let triples: Vec<(String, String, String)> = (0..2000u32)
8342 .map(|i| {
8343 let tail = char::from(b'a' + (i % 26) as u8).to_string().repeat(1000);
8344 (
8345 format!("<http://ex/s/{i:04}>"),
8346 "<http://ex/abstract>".to_string(),
8347 format!("\"{head}{i:04}{tail}\""),
8348 )
8349 })
8350 .collect();
8351 let mut db = DictionaryBuilder::new();
8352 for (s, p, o) in &triples {
8353 db.observe(s, p, o);
8354 }
8355 let dict = db.build();
8356 let mut ib = GraphIndexBuilder::new();
8357 for (s, p, o) in &triples {
8358 ib.push(dict.encode(s, p, o).unwrap());
8359 }
8360 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8361
8362 let reader = CountingReader::new(SliceReader::new(&bytes));
8363 let header = Header::from_bytes(&reader.read_at(0, HEADER_LEN as u64).unwrap()).unwrap();
8364 let section = locate_container_section_ranged(
8366 &reader,
8367 header.dictionary_offset,
8368 header.dictionary_len,
8369 2,
8370 4,
8371 )
8372 .unwrap();
8373
8374 let before = reader.bytes_read();
8375 let (_meta, entries) = read_dict_dir_ranged(&reader, section).unwrap();
8376 let spent = reader.bytes_read() - before;
8377
8378 let dir_end = entries[0].start;
8380 let head = reader.read_at(section.offset, 64).unwrap();
8381 let (header_len, n0) = read_uvarint(&head).unwrap();
8382 let dir_start = n0 as u64 + header_len;
8383 let dir_len = dir_end - dir_start;
8384 let dir_total = section.len - dir_start;
8385 assert!(
8386 dir_start + dir_len > 8192,
8387 "directory fits the header prefix ({dir_len} B) — the probe never runs"
8388 );
8389
8390 let mut old = 0u64;
8392 let mut p = 4096u64.min(dir_total).max(1);
8393 loop {
8394 let got = p.min(dir_total);
8395 old += got;
8396 if got >= dir_len || p >= dir_total {
8397 break;
8398 }
8399 p = p.saturating_mul(2).min(dir_total);
8400 }
8401 assert!(
8402 spent <= dir_len * 2,
8403 "probe read {spent} B for a {dir_len} B directory — past the append-only bound"
8404 );
8405 assert!(
8406 spent < old,
8407 "probe read {spent} B; the re-reading loop it replaced spent {old} B"
8408 );
8409 }
8410
8411 fn verbatim_keyed_twin(payload: &[u8], codec: u8) -> Vec<u8> {
8421 let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
8422 let (header_len, n0) = read_uvarint(payload).unwrap();
8423 let mut out = Vec::new();
8424 write_uvarint(&mut out, header_len);
8425 out.extend_from_slice(&payload[n0..n0 + header_len as usize]);
8426 write_uvarint(&mut out, entries.len() as u64);
8427 let mut prev_run = 0usize;
8428 for e in &entries {
8429 let body = decompress(codec, &payload[e.start as usize..e.end as usize]).unwrap();
8430 let first = crate::dict::run_first_term(&body, 0).unwrap();
8431 write_uvarint(&mut out, (e.first_run - prev_run) as u64);
8432 write_uvarint(&mut out, first.len() as u64);
8433 out.extend_from_slice(&first);
8434 write_uvarint(&mut out, e.end - e.start);
8435 prev_run = e.first_run;
8436 }
8437 for e in &entries {
8438 out.extend_from_slice(&payload[e.start as usize..e.end as usize]);
8439 }
8440 out
8441 }
8442
8443 fn chunk_bounds_of(payload: &[u8], codec: u8) -> Vec<(Vec<u8>, Vec<u8>)> {
8445 let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
8446 entries
8447 .iter()
8448 .map(|e| {
8449 let body = decompress(codec, &payload[e.start as usize..e.end as usize]).unwrap();
8450 (
8451 crate::dict::run_first_term(&body, 0).unwrap(),
8452 crate::dict::run_last_term(&body, 0, body.len()).unwrap(),
8453 )
8454 })
8455 .collect()
8456 }
8457
8458 fn adversarial_terms() -> Vec<String> {
8464 let mut terms: Vec<String> = Vec::new();
8465 for i in 0..6_000 {
8466 terms.push(format!("<http://example.org/work/{i:06}>"));
8467 }
8468 let shared = "X".repeat(900);
8469 for i in 0..4_000 {
8470 terms.push(format!("\"{shared}-{i:05}-tail\""));
8471 }
8472 let deep = "Y".repeat(3_000); for i in 0..120 {
8474 terms.push(format!("\"{deep}{i:03}\""));
8475 }
8476 for i in 0..6 {
8477 terms.push(format!("\"BIG-{i:02}-{}\"", "Z".repeat(90_000)));
8479 }
8480 for i in 0..800 {
8481 terms.push(format!(
8482 "\"h\u{e9}llo-{}-\u{1F600}{i:04}\"",
8483 "\u{e9}".repeat(40)
8484 ));
8485 }
8486 for i in 0..2_000 {
8487 terms.push(format!("\"{i:05}-{}\"", "T".repeat(1_800)));
8488 }
8489 terms.sort();
8490 terms.dedup();
8491 terms
8492 }
8493
8494 fn chunked_section_of(terms: &[String]) -> Vec<u8> {
8495 let mut b = crate::dict::DictSectionBuilder::new().with_restart_interval(16);
8496 for t in terms {
8497 b.push(t.clone());
8498 }
8499 b.build()
8500 }
8501
8502 #[test]
8512 fn chunk_directory_keys_are_separators_not_terms() {
8513 let terms = adversarial_terms();
8514 let raw = chunked_section_of(&terms);
8515 let payload = encode_chunked_dict_section(&raw, CODEC_NONE);
8516 let (_meta, entries) = parse_chunked_dict_dir(&payload, payload.len() as u64).unwrap();
8517 let bounds = chunk_bounds_of(&payload, CODEC_NONE);
8518 assert!(
8519 entries.len() > 20,
8520 "need a many-chunk section, got {}",
8521 entries.len()
8522 );
8523
8524 assert!(entries[0].key.is_empty(), "chunk 0 needs no separator");
8525 let mut key_bytes = 0usize;
8526 let mut term_bytes = 0usize;
8527 let mut differ = 0usize;
8528 for (i, e) in entries.iter().enumerate() {
8529 key_bytes += e.key.len();
8530 term_bytes += bounds[i].0.len();
8531 if e.key != bounds[i].0 {
8532 differ += 1;
8533 }
8534 assert!(
8535 e.key <= bounds[i].0,
8536 "chunk {i}: key must be <= its own first term"
8537 );
8538 if i > 0 {
8539 assert!(
8540 e.key.as_slice() > bounds[i - 1].1.as_slice(),
8541 "chunk {i}: key must be > the previous chunk's last term"
8542 );
8543 assert!(
8544 e.key > entries[i - 1].key,
8545 "chunk {i}: keys must ascend for partition_point"
8546 );
8547 assert_eq!(
8549 e.key,
8550 crate::dict::shortest_separator(&bounds[i - 1].1, &bounds[i].0),
8551 "chunk {i}: key is not the SHORTEST separator"
8552 );
8553 }
8554 }
8555 assert!(
8556 differ > 0,
8557 "every key equalled its first term — the writer stores terms again"
8558 );
8559 assert!(
8560 key_bytes * 4 < term_bytes,
8561 "separators bought almost nothing: {key_bytes} B vs {term_bytes} B of first terms"
8562 );
8563 eprintln!(
8564 "chunks={} first-term keys={term_bytes} B separator keys={key_bytes} B ({:.1}x)",
8565 entries.len(),
8566 term_bytes as f64 / key_bytes.max(1) as f64
8567 );
8568 }
8569
8570 #[test]
8581 fn separator_keys_route_identically_to_verbatim_first_terms() {
8582 let terms = adversarial_terms();
8583 let raw = chunked_section_of(&terms);
8584 let sep_payload = encode_chunked_dict_section(&raw, CODEC_NONE);
8585 let ver_payload = verbatim_keyed_twin(&sep_payload, CODEC_NONE);
8586
8587 let (_ms, es) = parse_chunked_dict_dir(&sep_payload, sep_payload.len() as u64).unwrap();
8589 let (_mv, ev) = parse_chunked_dict_dir(&ver_payload, ver_payload.len() as u64).unwrap();
8590 assert_eq!(es.len(), ev.len());
8591 for (a, b) in es.iter().zip(ev.iter()) {
8592 assert_eq!(a.first_run, b.first_run);
8593 assert_eq!(
8594 &sep_payload[a.start as usize..a.end as usize],
8595 &ver_payload[b.start as usize..b.end as usize],
8596 "chunk bodies must be identical"
8597 );
8598 }
8599 assert!(
8600 sep_payload.len() < ver_payload.len(),
8601 "separator payload is not smaller"
8602 );
8603
8604 let sec_s = decode_chunked_dict_section(
8605 &sep_payload,
8606 CODEC_NONE,
8607 crate::chunk_cache::ChunkCache::unlimited_arc(),
8608 0,
8609 )
8610 .unwrap();
8611 let sec_v = decode_chunked_dict_section(
8612 &ver_payload,
8613 CODEC_NONE,
8614 crate::chunk_cache::ChunkCache::unlimited_arc(),
8615 0,
8616 )
8617 .unwrap();
8618
8619 for (i, t) in terms.iter().enumerate() {
8621 let want = Some(i as u32 + 1);
8622 assert_eq!(sec_v.id(t), want, "verbatim id() lost a term");
8623 assert_eq!(sec_s.id(t), want, "separator id() lost a term");
8624 }
8625 for i in 0..terms.len() as u32 {
8627 assert_eq!(sec_s.term(i + 1), sec_v.term(i + 1));
8628 assert_eq!(
8629 sec_s.term(i + 1).as_deref(),
8630 Some(terms[i as usize].as_str())
8631 );
8632 }
8633 let bounds = chunk_bounds_of(&sep_payload, CODEC_NONE);
8635 let mut probes: Vec<String> = Vec::new();
8636 let push = |bytes: &[u8], probes: &mut Vec<String>| {
8637 if let Ok(s) = std::str::from_utf8(bytes) {
8638 probes.push(s.to_string());
8639 }
8640 };
8641 for (first, last) in &bounds {
8642 for base in [first, last] {
8643 push(base, &mut probes); let mut after = base.clone();
8645 after.push(0x01); push(&after, &mut probes);
8647 let mut far = base.clone();
8648 far.push(b'~');
8649 push(&far, &mut probes);
8650 if !base.is_empty() {
8651 push(&base[..base.len() - 1], &mut probes); let mut down = base.clone();
8653 *down.last_mut().unwrap() = down.last().unwrap().wrapping_sub(1);
8654 push(&down, &mut probes);
8655 let mut up = base.clone();
8656 *up.last_mut().unwrap() = up.last().unwrap().wrapping_add(1);
8657 push(&up, &mut probes);
8658 }
8659 }
8660 }
8661 for e in &es {
8663 push(&e.key, &mut probes);
8664 let mut plus = e.key.clone();
8665 plus.push(0);
8666 push(&plus, &mut probes);
8667 }
8668 probes.push(String::new()); probes.push("\u{1}".to_string()); probes.push("\u{10FFFF}\u{10FFFF}".to_string()); probes.push(terms.first().unwrap().clone());
8672 probes.push(terms.last().unwrap().clone());
8673
8674 let mut hits = 0usize;
8675 for p in &probes {
8676 let (a, b) = (sec_v.id(p), sec_s.id(p));
8677 assert_eq!(
8678 a,
8679 b,
8680 "probe diverged: {:?}",
8681 p.get(..p.len().min(48)).unwrap_or(p.as_str())
8682 );
8683 hits += usize::from(a.is_some());
8684 }
8685 assert!(
8686 probes.len() > 500 && hits > 50,
8687 "probe set is too thin: {} probes, {hits} of them present",
8688 probes.len()
8689 );
8690 eprintln!(
8691 "{} boundary probes, {hits} present, all identical",
8692 probes.len()
8693 );
8694 }
8695
8696 mod separator_props {
8702 use super::*;
8703 use proptest::prelude::*;
8704
8705 fn term_set() -> impl Strategy<Value = Vec<String>> {
8712 prop::collection::vec(("[a-c]{1,6}", 0u8..3, 300usize..1200), 60..250).prop_map(
8713 |specs| {
8714 let mut terms: Vec<String> = specs
8715 .into_iter()
8716 .enumerate()
8717 .map(|(i, (core, kind, pad))| {
8718 let block = "z".repeat(pad);
8719 match kind {
8720 0 => format!("<http://ex/{core}/{i:05}>"),
8721 1 => format!("\"{block}{core}{i:05}\""),
8722 _ => format!("\"{core}{i:05}{block}\""),
8723 }
8724 })
8725 .collect();
8726 terms.sort();
8727 terms.dedup();
8728 terms
8729 },
8730 )
8731 }
8732
8733 proptest! {
8734 #[test]
8735 fn prop_separator_keyed_section_answers_like_verbatim(
8736 terms in term_set(),
8737 extra in prop::collection::vec(any::<String>(), 0..8),
8738 ) {
8739 let raw = chunked_section_of(&terms);
8740 let sep = encode_chunked_dict_section(&raw, CODEC_NONE);
8741 let ver = verbatim_keyed_twin(&sep, CODEC_NONE);
8742 prop_assert!(sep.len() <= ver.len());
8743
8744 let (_m, es) = parse_chunked_dict_dir(&sep, sep.len() as u64).unwrap();
8746 let bounds = chunk_bounds_of(&sep, CODEC_NONE);
8747 prop_assert!(es[0].key.is_empty());
8748 for i in 1..es.len() {
8749 prop_assert!(es[i].key.as_slice() > bounds[i - 1].1.as_slice());
8750 prop_assert!(es[i].key <= bounds[i].0);
8751 prop_assert!(es[i].key > es[i - 1].key);
8752 }
8753
8754 let sec_s = decode_chunked_dict_section(
8755 &sep,
8756 CODEC_NONE,
8757 crate::chunk_cache::ChunkCache::unlimited_arc(),
8758 0,
8759 )
8760 .unwrap();
8761 let sec_v = decode_chunked_dict_section(
8762 &ver,
8763 CODEC_NONE,
8764 crate::chunk_cache::ChunkCache::unlimited_arc(),
8765 0,
8766 )
8767 .unwrap();
8768
8769 for (i, t) in terms.iter().enumerate() {
8770 prop_assert_eq!(sec_s.id(t), Some(i as u32 + 1));
8771 prop_assert_eq!(sec_v.id(t), Some(i as u32 + 1));
8772 prop_assert_eq!(sec_s.term(i as u32 + 1), sec_v.term(i as u32 + 1));
8773 }
8774
8775 let mut probes: Vec<String> = extra;
8777 probes.push(String::new());
8778 for (first, last) in &bounds {
8779 for base in [first, last] {
8780 if let Ok(s) = std::str::from_utf8(base) {
8781 probes.push(s.to_string());
8782 probes.push(format!("{s}\u{1}"));
8783 probes.push(s[..s.len() - s.chars().next_back()
8784 .map(char::len_utf8).unwrap_or(0)].to_string());
8785 }
8786 }
8787 }
8788 for e in &es {
8789 if let Ok(s) = std::str::from_utf8(&e.key) {
8790 probes.push(s.to_string());
8791 }
8792 }
8793 for p in &probes {
8794 prop_assert_eq!(sec_s.id(p), sec_v.id(p), "probe diverged: {:?}", p);
8795 }
8796 }
8797 }
8798 }
8799
8800 mod sparql_order_props {
8807 use super::*;
8808 use crate::term_order::TermOrder;
8809 use proptest::prelude::*;
8810
8811 fn term_set() -> impl Strategy<Value = Vec<String>> {
8812 prop::collection::vec(("[a-c]{1,5}", 0u8..7, 200usize..900), 60..220).prop_map(
8813 |specs| {
8814 let mut terms: Vec<String> = specs
8815 .into_iter()
8816 .enumerate()
8817 .map(|(i, (core, kind, pad))| {
8818 let block = "z".repeat(pad);
8819 match kind {
8820 0 => format!("<http://ex/{core}/{}>", i % 40),
8822 1 => format!("<http://ex/{core}/{}{block}_provenance>", i % 40),
8823 2 => format!("_:{core}{i}"),
8824 3 => format!("\"{block} {core}{i:04}\""),
8825 4 => format!("\"{core}{i:04}\"@{}", ["en", "EN", "fr"][i % 3]),
8826 5 => format!("\"{core}\\\"{i:04}{block}\""),
8827 _ => format!("\"{core}{i}\"^^<http://ex/dt>"),
8828 }
8829 })
8830 .collect();
8831 terms.sort_by(|a, b| TermOrder::Sparql.cmp_str(a, b));
8832 terms.dedup();
8833 terms
8834 },
8835 )
8836 }
8837
8838 fn section_of(terms: &[String]) -> Vec<u8> {
8839 let mut b = crate::dict::DictSectionBuilder::new()
8840 .with_restart_interval(16)
8841 .with_order(TermOrder::Sparql);
8842 for t in terms {
8843 b.push(t.clone());
8844 }
8845 b.build()
8846 }
8847
8848 proptest! {
8849 #[test]
8850 fn prop_a_sparql_ordered_section_routes_and_finds(
8851 terms in term_set(),
8852 extra in prop::collection::vec(any::<String>(), 0..8),
8853 ) {
8854 let raw = section_of(&terms);
8855 let payload = encode_chunked_dict_section_ordered(&raw, CODEC_NONE, TermOrder::Sparql);
8856 let (_m, es) = parse_chunked_dict_dir(&payload, payload.len() as u64).unwrap();
8857 let bounds = chunk_bounds_of(&payload, CODEC_NONE);
8858 let o = TermOrder::Sparql;
8859 prop_assert!(es[0].key.is_empty());
8860 for i in 1..es.len() {
8861 prop_assert!(o.cmp(&bounds[i - 1].1, &es[i].key) == std::cmp::Ordering::Less);
8862 prop_assert!(o.cmp(&es[i].key, &bounds[i].0) != std::cmp::Ordering::Greater);
8863 prop_assert!(o.cmp(&es[i - 1].key, &es[i].key) == std::cmp::Ordering::Less);
8864 }
8865 let sec = decode_chunked_dict_section(
8866 &payload,
8867 CODEC_NONE,
8868 crate::chunk_cache::ChunkCache::unlimited_arc(),
8869 0,
8870 )
8871 .unwrap()
8872 .with_order(o);
8873 for (i, t) in terms.iter().enumerate() {
8874 prop_assert_eq!(sec.id(t), Some(i as u32 + 1));
8875 prop_assert_eq!(sec.term(i as u32 + 1), Some(t.clone()));
8876 }
8877 let mut counts = crate::term_order::GroupCounts::default();
8879 terms.iter().for_each(|t| counts.add(t.as_bytes()));
8880 prop_assert_eq!(sec.groups(), Some(counts));
8881 let mut probes: Vec<String> = extra;
8882 probes.push(String::new());
8883 for (first, last) in &bounds {
8884 for base in [first, last] {
8885 if let Ok(s) = std::str::from_utf8(base) {
8886 probes.push(s.to_string());
8887 probes.push(format!("{s}\u{1}"));
8888 }
8889 }
8890 }
8891 for p in &probes {
8892 let want = terms.iter().position(|t| t == p).map(|i| i as u32 + 1);
8893 prop_assert_eq!(sec.id(p), want, "probe: {:?}", p);
8894 let below = terms
8896 .iter()
8897 .filter(|t| o.cmp_str(t, p) == std::cmp::Ordering::Less)
8898 .count() as u32;
8899 prop_assert_eq!(sec.rank(p), Some(below), "rank of {:?}", p);
8900 }
8901 }
8902 }
8903 }
8904
8905 #[test]
8910 fn text_index_is_tamper_evident_and_verifies() {
8911 let triples: Vec<(String, String, String)> = vec![(
8912 "<http://ex/s0>".to_string(),
8913 "<http://ex/label>".to_string(),
8914 "\"alpha glucose phosphate\"".to_string(),
8915 )];
8916 let bytes = build_text_indexed(&triples);
8917 let header = Rete::open(&bytes).unwrap().header().clone();
8918 assert!(header.text_index_len > 0);
8919 assert!(verify(&bytes).unwrap(), "a text-indexed build must verify");
8920
8921 let mut tampered = bytes.clone();
8922 tampered[header.text_index_offset as usize] ^= 0xff;
8923 assert!(
8924 !verify(&tampered).unwrap(),
8925 "tampering with the text index must break verify()"
8926 );
8927 }
8928
8929 #[test]
8933 fn synopsis_cuts_remote_fetch_bytes() {
8934 use crate::header::FLAG_TILE_SYNOPSIS;
8935 use crate::reader::{CountingReader, SliceReader};
8936
8937 let triples: Vec<(String, String, String)> = (0..400u32)
8941 .map(|i| {
8942 (
8943 format!("<http://ex/s/{i:04}>"),
8944 "<http://ex/p>".to_string(),
8945 format!("<http://ex/o/{i:04}>"),
8946 )
8947 })
8948 .collect();
8949 let mut db = DictionaryBuilder::new();
8950 for (s, p, o) in &triples {
8951 db.observe(s, p, o);
8952 }
8953 let dict = db.build();
8954 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
8955 for (s, p, o) in &triples {
8956 ib.push(dict.encode(s, p, o).unwrap());
8957 }
8958 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8959
8960 let q = (Some("<http://ex/s/0395>"), None, Some("<http://ex/o/0005>"));
8964 let query_bytes = |image: &[u8]| -> (u64, usize) {
8968 let leaked: &'static [u8] = Box::leak(image.to_vec().into_boxed_slice());
8969 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8970 let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
8971 let before = reader.bytes_read(); let n = rete.query(q.0, q.1, q.2).len();
8973 assert!(!rete.index_incomplete());
8974 (reader.bytes_read() - before, n)
8975 };
8976
8977 let (on_bytes, on_n) = query_bytes(&bytes);
8978 let mut off = bytes.clone();
8980 off[5] &= !FLAG_TILE_SYNOPSIS;
8981 let (off_bytes, off_n) = query_bytes(&off);
8982
8983 assert_eq!(on_n, 0, "the pair never co-occurs");
8984 assert_eq!(off_n, 0, "same answer without the synopsis");
8985 assert!(
8989 on_bytes < off_bytes,
8990 "synopsis skips the routed tile fetch: {on_bytes} < {off_bytes}"
8991 );
8992 }
8993
8994 #[test]
9000 fn double_bound_object_join_eager_matches_lazy() {
9001 use crate::reader::SliceReader;
9002 let occ = "<http://ex/occ>";
9003 let phys = "<http://ex/physicist>";
9004 let phil = "<http://ex/philosopher>";
9005 let label = "<http://www.w3.org/2000/01/rdf-schema#label>";
9006 let mut triples: Vec<(String, String, String)> = Vec::new();
9008 for i in 0..20u32 {
9009 triples.push((format!("<http://ex/p/{i:02}>"), occ.into(), phys.into()));
9010 if i < 10 {
9011 triples.push((format!("<http://ex/p/{i:02}>"), occ.into(), phil.into()));
9012 }
9013 triples.push((
9014 format!("<http://ex/p/{i:02}>"),
9015 label.into(),
9016 format!("\"Name {i:02}\""),
9017 ));
9018 }
9019 let mut db = DictionaryBuilder::new();
9020 for (s, p, o) in &triples {
9021 db.observe(s, p, o);
9022 }
9023 let dict = db.build();
9024 let mut ib = GraphIndexBuilder::new().with_tile_budget(16);
9025 for (s, p, o) in &triples {
9026 ib.push(dict.encode(s, p, o).unwrap());
9027 }
9028 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9029
9030 let q = "SELECT ?l WHERE { \
9031 ?p <http://ex/occ> <http://ex/physicist> ; \
9032 <http://ex/occ> <http://ex/philosopher> ; \
9033 <http://www.w3.org/2000/01/rdf-schema#label> ?l }";
9034 let run = |rete: &Rete| -> Vec<String> {
9035 let (_, sols) = crate::eval_sparql(rete, q).unwrap();
9036 let mut v: Vec<String> = sols.iter().filter_map(|b| b.get("l").cloned()).collect();
9037 v.sort();
9038 v
9039 };
9040
9041 let eager_rows = run(&Rete::open(&bytes).unwrap());
9042 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9043 let lazy = Rete::open_ranged_lazy_with(
9044 std::sync::Arc::new(SliceReader::new(leaked)),
9045 HEADER_LEN as u64,
9046 )
9047 .unwrap();
9048 let lazy_rows = run(&lazy);
9049 assert!(!lazy.index_incomplete());
9050
9051 assert_eq!(eager_rows.len(), 10, "the 10 physicist∩philosopher labels");
9052 assert_eq!(eager_rows, lazy_rows, "eager and lazy must agree exactly");
9053 }
9054
9055 #[test]
9059 fn multi_chunk_dictionary_round_trips() {
9060 let mut db = DictionaryBuilder::new();
9061 let term = |i: u32| format!("<http://example.org/some/long/prefix/entity/{i:06}>");
9062 for i in 0..6000u32 {
9063 db.observe(&term(i), "<http://ex/p>", &term(i + 1));
9064 }
9065 let dict = db.build();
9066 let mut ib = GraphIndexBuilder::new();
9067 for i in 0..6000u32 {
9068 ib.push(
9069 dict.encode(&term(i), "<http://ex/p>", &term(i + 1))
9070 .unwrap(),
9071 );
9072 }
9073 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9074 let rete = Rete::open(&bytes).unwrap();
9075 let d = rete.dictionary();
9076 assert_eq!(d.term_count(), dict.term_count());
9077 for i in (0..6000).step_by(97).chain([0, 1, 5999, 6000]) {
9078 let t = term(i);
9079 let sid = dict.subject_id(&t);
9080 assert_eq!(d.subject_id(&t), sid, "subject_id({t})");
9081 if let Some(id) = sid {
9082 assert_eq!(d.subject_term(id).as_deref(), Some(t.as_str()));
9083 }
9084 let oid = dict.object_id(&t);
9085 assert_eq!(d.object_id(&t), oid, "object_id({t})");
9086 }
9087 assert_eq!(d.subject_id("<http://example.org/absent>"), None);
9088 assert_eq!(d.predicate_id("<http://ex/p>"), Some(1));
9089 assert_eq!(d.predicate_term(1).as_deref(), Some("<http://ex/p>"));
9090 }
9091
9092 #[test]
9093 #[cfg(feature = "compression")]
9094 fn compression_shrinks_repetitive_data() {
9095 let mut db = DictionaryBuilder::new();
9099 let triples: Vec<(String, String, String)> = (0..500)
9100 .map(|i| {
9101 (
9102 format!("<http://example.org/entity/{i}>"),
9103 "<http://example.org/p/relatedTo>".to_string(),
9104 format!("<http://example.org/entity/{}>", (i + 1) % 500),
9105 )
9106 })
9107 .collect();
9108 for (s, p, o) in &triples {
9109 db.observe(s, p, o);
9110 }
9111 let dict = db.build();
9112 let mut ib = GraphIndexBuilder::new();
9113 for (s, p, o) in &triples {
9114 ib.push(dict.encode(s, p, o).unwrap());
9115 }
9116 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9117
9118 let raw: usize = triples
9119 .iter()
9120 .map(|(s, p, o)| s.len() + p.len() + o.len())
9121 .sum();
9122 assert!(
9123 bytes.len() < raw / 2,
9124 "expected strong compression: file {} vs raw terms {raw}",
9125 bytes.len()
9126 );
9127
9128 let rete = Rete::open(&bytes).unwrap();
9130 let r = rete.query(Some("<http://example.org/entity/0>"), None, None);
9131 assert_eq!(r.len(), 1);
9132 assert_eq!(r[0].2, "<http://example.org/entity/1>");
9133 }
9134
9135 fn big_file_with_pyramid() -> Vec<u8> {
9136 let triples: Vec<(String, String, String)> = (0..300)
9138 .map(|i| {
9139 (
9140 format!("<http://ex/e{i}>"),
9141 "<http://ex/next>".to_string(),
9142 format!("<http://ex/e{}>", (i + 1) % 300),
9143 )
9144 })
9145 .collect();
9146 let mut db = DictionaryBuilder::new();
9147 for (s, p, o) in &triples {
9148 db.observe(s, p, o);
9149 }
9150 let dict = db.build();
9151 let ids: Vec<_> = triples
9152 .iter()
9153 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9154 .collect();
9155 let mut ib = GraphIndexBuilder::new();
9156 for &t in &ids {
9157 ib.push(t);
9158 }
9159 let (meta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
9160 write_file(&dict, &ib.build(), false, &meta, levels)
9161 }
9162
9163 #[test]
9164 fn ranged_open_is_minimal_and_correct() {
9165 use crate::reader::{CountingReader, SliceReader};
9166 let bytes = big_file_with_pyramid();
9167
9168 let full = CountingReader::new(SliceReader::new(&bytes));
9169 let rete = Rete::open_ranged(&full).unwrap();
9170 assert!(full.requests() <= 4, "requests = {}", full.requests());
9172 assert_eq!(
9173 rete.query(Some("<http://ex/e0>"), None, None)[0].2,
9174 "<http://ex/e1>"
9175 );
9176
9177 let summ_reader = CountingReader::new(SliceReader::new(&bytes));
9179 let view = SummaryView::open_ranged(&summ_reader).unwrap().unwrap();
9180 assert!(!view.summary.is_empty());
9181 assert!(
9182 summ_reader.bytes_read() < bytes.len() as u64,
9183 "summary read {} of {} bytes",
9184 summ_reader.bytes_read(),
9185 bytes.len()
9186 );
9187 assert!(summ_reader.bytes_read() < full.bytes_read());
9189 }
9190
9191 #[test]
9192 fn content_hash_is_set_and_verifies() {
9193 let bytes = build_image();
9194 let rete = Rete::open(&bytes).unwrap();
9195 assert_ne!(
9196 rete.header().content_hash,
9197 [0u8; 16],
9198 "hash must be populated"
9199 );
9200 assert!(verify(&bytes).unwrap(), "freshly built file verifies");
9201 assert!(
9202 verify_ranged(&crate::reader::SliceReader::new(&bytes)).unwrap(),
9203 "the streaming verify agrees"
9204 );
9205
9206 assert_eq!(
9208 Rete::open(&build_image()).unwrap().header().content_hash,
9209 rete.header().content_hash
9210 );
9211
9212 let mut tampered = bytes.clone();
9214 let last = tampered.len() - 5; tampered[last] ^= 0xff;
9216 assert!(!verify(&tampered).unwrap());
9217 assert!(!verify_ranged(&crate::reader::SliceReader::new(&tampered)).unwrap());
9218 let short = &bytes[..bytes.len() / 2];
9220 assert!(verify_ranged(&crate::reader::SliceReader::new(short)).is_err());
9221 assert!(verify_ranged(&crate::reader::SliceReader::new(&bytes[..10])).is_err());
9222 }
9223
9224 fn build_with_metadata(meta: &[u8]) -> Vec<u8> {
9226 let triples = [
9227 ("Alice", "knows", "Bob"),
9228 ("Bob", "knows", "Carol"),
9229 ("Alice", "age", "30"),
9230 ];
9231 let mut db = DictionaryBuilder::new();
9232 for (s, p, o) in triples {
9233 db.observe(s, p, o);
9234 }
9235 let dict = db.build();
9236 let ids: Vec<_> = triples
9237 .iter()
9238 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9239 .collect();
9240 let mut ib = GraphIndexBuilder::new();
9241 for &t in &ids {
9242 ib.push(t);
9243 }
9244 let (pmeta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
9245 write_dataset_with_metadata(&dict, &ib.build(), &[], false, &pmeta, levels, meta, &[])
9246 }
9247
9248 #[test]
9249 fn metadata_round_trips_and_shifts_offsets() {
9250 let card = br#"{"title":"My Dataset"}"#;
9251 let bytes = build_with_metadata(card);
9252 let rete = Rete::open(&bytes).unwrap();
9253
9254 assert_eq!(rete.metadata(), Some(card.as_slice()));
9256 let h = rete.header();
9257 assert_eq!(h.metadata_offset, HEADER_LEN as u64);
9258 assert_eq!(h.metadata_len, card.len() as u64);
9259 assert_eq!(h.dictionary_offset, HEADER_LEN as u64 + card.len() as u64);
9261
9262 assert_eq!(
9264 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9265 1
9266 );
9267 assert!(verify(&bytes).unwrap());
9269 }
9270
9271 #[test]
9272 fn empty_metadata_is_byte_identical_to_plain_writer() {
9273 assert_eq!(
9276 build_with_metadata(&[]),
9277 build_image(),
9278 "empty-metadata output must equal the plain writer byte-for-byte"
9279 );
9280 }
9281
9282 #[test]
9287 fn replace_metadata_matches_a_direct_write() {
9288 let first = br#"{"title":"My Dataset","queries":[1,2,3]}"#;
9289 let bytes = build_with_metadata(first);
9290 for target in [
9291 br#"{"title":"My Dataset","queries":[1,2]}"#.as_slice(), br#"{"title":"My Dataset","queries":[1,2,3,4,5,6,7]}"#.as_slice(), b"".as_slice(), first.as_slice(), ] {
9296 let spliced = replace_metadata(&bytes, target).unwrap();
9297 assert_eq!(
9298 spliced,
9299 build_with_metadata(target),
9300 "splicing {} bytes of metadata must equal a direct write",
9301 target.len()
9302 );
9303 assert!(
9304 verify(&spliced).unwrap(),
9305 "the new hash covers the new card"
9306 );
9307 let rete = Rete::open(&spliced).unwrap();
9308 assert_eq!(
9309 rete.metadata(),
9310 (!target.is_empty()).then_some(target),
9311 "the new payload reads back verbatim"
9312 );
9313 assert_eq!(
9315 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9316 1
9317 );
9318 }
9319 }
9320
9321 #[test]
9324 fn replace_metadata_keeps_build_info_adjacent() {
9325 let with_info =
9326 attach_build_info(&build_with_metadata(br#"{"a":1}"#), br#"{"builder":"x"}"#).unwrap();
9327 let spliced = replace_metadata(&with_info, br#"{"a":1,"b":2}"#).unwrap();
9328 let h = Header::from_bytes(&spliced).unwrap();
9329 assert_eq!(h.metadata_offset, HEADER_LEN as u64);
9330 assert_eq!(h.build_info_offset, HEADER_LEN as u64 + h.metadata_len);
9331 assert_eq!(
9332 read_build_info(&spliced).unwrap().as_deref(),
9333 Some(br#"{"builder":"x"}"#.as_slice())
9334 );
9335 assert!(verify(&spliced).unwrap());
9336 assert_eq!(
9338 h.content_hash,
9339 Header::from_bytes(&build_with_metadata(br#"{"a":1,"b":2}"#))
9340 .unwrap()
9341 .content_hash
9342 );
9343 }
9344
9345 #[test]
9346 fn metadata_is_tamper_evident() {
9347 let card = br#"{"title":"x"}"#;
9348 let mut bytes = build_with_metadata(card);
9349 assert!(verify(&bytes).unwrap());
9350 bytes[HEADER_LEN + 2] ^= 0xff;
9352 assert!(
9353 !verify(&bytes).unwrap(),
9354 "tampering with the card must break verify()"
9355 );
9356 }
9357
9358 #[test]
9359 fn ranged_opens_do_not_fetch_metadata() {
9360 use crate::reader::{CountingReader, SliceReader};
9361 let card = vec![0xABu8; 512]; let bytes = build_with_metadata(&card);
9363 let total = bytes.len() as u64;
9364
9365 let r = CountingReader::new(SliceReader::new(&bytes));
9367 let rete = Rete::open_ranged(&r).unwrap();
9368 assert!(
9369 rete.metadata().is_none(),
9370 "open_ranged must not load the card"
9371 );
9372 assert!(r.requests() <= 4, "requests = {}", r.requests());
9373 assert!(
9374 r.bytes_read() <= total - card.len() as u64,
9375 "read {} of {} bytes; the {}-byte card must be skipped",
9376 r.bytes_read(),
9377 total,
9378 card.len()
9379 );
9380
9381 let rs = CountingReader::new(SliceReader::new(&bytes));
9383 let view = SummaryView::open_ranged(&rs).unwrap().unwrap();
9384 assert!(!view.summary.is_empty());
9385 assert!(rs.bytes_read() <= total - card.len() as u64);
9386 }
9387
9388 #[test]
9389 fn metadata_ranged_fetches_only_header_and_card() {
9390 use crate::reader::{CountingReader, SliceReader};
9391 let card = vec![0xCDu8; 384];
9394 let bytes = build_with_metadata(&card);
9395
9396 let r = CountingReader::new(SliceReader::new(&bytes));
9397 let got = read_metadata_ranged(&r).unwrap().unwrap();
9398 assert_eq!(got, card, "the card reads back verbatim");
9399 assert_eq!(r.requests(), 2, "exactly header + metadata ranges");
9400 assert_eq!(
9401 r.bytes_read(),
9402 HEADER_LEN as u64 + card.len() as u64,
9403 "no dictionary/index/pyramid bytes are touched"
9404 );
9405
9406 let plain = build_image();
9408 let rp = CountingReader::new(SliceReader::new(&plain));
9409 assert!(read_metadata_ranged(&rp).unwrap().is_none());
9410 assert_eq!(rp.requests(), 1, "header only for a cardless file");
9411 assert_eq!(rp.bytes_read(), HEADER_LEN as u64);
9412 }
9413
9414 #[test]
9415 fn build_info_attaches_outside_the_hash() {
9416 let card = br#"{"title":"My Dataset"}"#;
9417 let base = build_with_metadata(card);
9418 let base_hash = Header::from_bytes(&base).unwrap().content_hash;
9419
9420 let info = br#"{"built_at":"2026-08-04T00:00:00Z","builder":"rete-cli 0.3.2"}"#;
9421 let with = attach_build_info(&base, info).unwrap();
9422
9423 assert_eq!(read_build_info(&with).unwrap().unwrap(), info);
9425 let h = Header::from_bytes(&with).unwrap();
9426 assert_eq!(h.build_info_offset, HEADER_LEN as u64 + card.len() as u64);
9427 assert_eq!(h.build_info_len, info.len() as u64);
9428 assert_eq!(h.expected_file_len(), Some(with.len() as u64));
9429
9430 assert_eq!(h.content_hash, base_hash);
9434 assert!(verify(&with).unwrap());
9435 let other = attach_build_info(&base, br#"{"built_at":"1999-01-01T00:00:00Z"}"#).unwrap();
9436 assert_eq!(Header::from_bytes(&other).unwrap().content_hash, base_hash);
9437
9438 let rete = Rete::open(&with).unwrap();
9440 assert_eq!(
9441 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9442 1
9443 );
9444 assert_eq!(rete.metadata(), Some(card.as_slice()));
9445
9446 let replaced = attach_build_info(&with, br#"{"builder":"other"}"#).unwrap();
9449 assert_eq!(
9450 read_build_info(&replaced).unwrap().unwrap(),
9451 br#"{"builder":"other"}"#
9452 );
9453 assert!(verify(&replaced).unwrap());
9454 let stripped = attach_build_info(&replaced, &[]).unwrap();
9455 assert_eq!(stripped, base, "stripping build-info restores the image");
9456 }
9457
9458 #[test]
9459 fn build_info_attaches_on_a_metadata_free_file() {
9460 let base = build_image();
9462 let info = b"{\"builder\":\"x\"}";
9463 let with = attach_build_info(&base, info).unwrap();
9464 let h = Header::from_bytes(&with).unwrap();
9465 assert_eq!(h.build_info_offset, HEADER_LEN as u64);
9466 assert_eq!(h.metadata_len, 0);
9467 assert_eq!(read_build_info(&with).unwrap().unwrap(), info);
9468 assert!(verify(&with).unwrap());
9469 assert_eq!(
9470 h.content_hash,
9471 Header::from_bytes(&base).unwrap().content_hash
9472 );
9473 Rete::open(&with).unwrap();
9474 }
9475
9476 #[test]
9483 fn a_plan_rebuilds_exactly_what_the_in_memory_splice_produces() {
9484 let card = br#"{"title":"My Dataset"}"#;
9485 let base = build_with_metadata(card);
9486 let base_hash = Header::from_bytes(&base).unwrap().content_hash;
9487 for start in [
9488 base.clone(),
9489 attach_build_info(&base, b"{\"a\":1}").unwrap(),
9490 ] {
9491 for info in [
9492 br#"{"builder":"rete-cli 0.3.2","query_costs":{}}"#.as_slice(), b"{}".as_slice(), b"".as_slice(), ] {
9496 let plan = plan_build_info(&start, start.len() as u64, info.len() as u64).unwrap();
9497 let mut streamed = Vec::new();
9498 streamed.extend_from_slice(&plan.header);
9499 streamed.extend_from_slice(&start[HEADER_LEN..plan.insert as usize]);
9500 streamed.extend_from_slice(info);
9501 streamed.extend_from_slice(&start[plan.tail_start as usize..]);
9502 assert_eq!(streamed.len() as u64, plan.new_len);
9503 assert_eq!(
9504 streamed,
9505 attach_build_info(&start, info).unwrap(),
9506 "a {}-byte section streamed must equal the same section spliced",
9507 info.len()
9508 );
9509 assert!(verify(&streamed).unwrap());
9511 let h = Header::from_bytes(&streamed).unwrap();
9512 assert_eq!(h.content_hash, base_hash);
9513 assert_eq!(h.expected_file_len(), Some(plan.new_len));
9514 let rete = Rete::open(&streamed).unwrap();
9515 assert_eq!(rete.metadata(), Some(card.as_slice()));
9516 assert_eq!(
9517 rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9518 1
9519 );
9520 }
9521 }
9522 }
9523
9524 #[test]
9527 fn a_plan_refuses_a_header_that_overruns_the_file() {
9528 let base = build_with_metadata(br#"{"title":"x"}"#);
9529 assert!(plan_build_info(&base, 8, 16).is_err());
9530 assert!(plan_build_info(&base[..HEADER_LEN], base.len() as u64, 16).is_ok());
9531 }
9532
9533 #[test]
9534 fn card_and_build_info_ranged_is_one_header_plus_one_range() {
9535 use crate::reader::{CountingReader, SliceReader};
9536 let card = vec![0xCDu8; 384];
9537 let info = vec![0xEFu8; 200];
9538 let bytes = attach_build_info(&build_with_metadata(&card), &info).unwrap();
9539
9540 let r = CountingReader::new(SliceReader::new(&bytes));
9543 let (m, b) = read_card_and_build_info_ranged(&r).unwrap();
9544 assert_eq!(m.unwrap(), card);
9545 assert_eq!(b.unwrap(), info);
9546 assert_eq!(r.requests(), 2, "header + one coalesced range");
9547 assert_eq!(
9548 r.bytes_read(),
9549 HEADER_LEN as u64 + card.len() as u64 + info.len() as u64
9550 );
9551
9552 let plain = build_with_metadata(&card);
9554 let rp = CountingReader::new(SliceReader::new(&plain));
9555 let (m, b) = read_card_and_build_info_ranged(&rp).unwrap();
9556 assert_eq!(m.unwrap(), card);
9557 assert!(b.is_none());
9558 assert_eq!(rp.requests(), 2);
9559 assert_eq!(
9564 r.requests(),
9565 rp.requests(),
9566 "build info must not cost an extra request",
9567 );
9568
9569 let none = build_image();
9571 let rn = CountingReader::new(SliceReader::new(&none));
9572 let (m, b) = read_card_and_build_info_ranged(&rn).unwrap();
9573 assert!(m.is_none() && b.is_none());
9574 assert_eq!(rn.requests(), 1);
9575 }
9576
9577 #[test]
9578 fn schema_summary_groups_by_type() {
9579 let rt = RDF_TYPE;
9580 let bytes = build_from(&[
9581 ("Alice", rt, "Person"),
9582 ("Bob", rt, "Person"),
9583 ("NYC", rt, "City"),
9584 ("Alice", "knows", "Bob"),
9585 ("Alice", "livesIn", "NYC"),
9586 ("Alice", "name", "\"Alice\""),
9587 ]);
9588 let rete = Rete::open(&bytes).unwrap();
9589 let summary = schema_summary(&rete);
9590 assert!(summary.contains(&("Person".into(), "knows".into(), "Person".into(), 1)));
9592 assert!(summary.contains(&("Person".into(), "livesIn".into(), "City".into(), 1)));
9593 assert!(summary.contains(&("Person".into(), "name".into(), "(literal)".into(), 1)));
9594 assert!(!summary.iter().any(|(_, p, _, _)| p == RDF_TYPE));
9596
9597 let classes = schema_classes(&rete);
9599 assert_eq!(
9600 classes,
9601 vec![("Person".into(), 2u32), ("City".into(), 1u32)]
9602 );
9603 }
9604
9605 fn build_from(triples: &[(&str, &str, &str)]) -> Vec<u8> {
9606 let mut db = DictionaryBuilder::new();
9607 for (s, p, o) in triples {
9608 db.observe(s, p, o);
9609 }
9610 let dict = db.build();
9611 let mut ib = GraphIndexBuilder::new();
9612 for (s, p, o) in triples {
9613 ib.push(dict.encode(s, p, o).unwrap());
9614 }
9615 write_file(&dict, &ib.build(), false, &[], 0)
9616 }
9617
9618 fn build_with_pyramid(triples: &[(&str, &str, &str)]) -> Vec<u8> {
9620 let mut db = DictionaryBuilder::new();
9621 for (s, p, o) in triples {
9622 db.observe(s, p, o);
9623 }
9624 let dict = db.build();
9625 let encoded: Vec<_> = triples
9626 .iter()
9627 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9628 .collect();
9629 let mut ib = GraphIndexBuilder::new();
9630 for t in &encoded {
9631 ib.push(*t);
9632 }
9633 let (meta, levels) = build_pyramid_meta(&dict, &encoded, DEFAULT_TILE_BUDGET);
9634 write_dataset(&dict, &ib.build(), &[], false, &meta, levels)
9635 }
9636
9637 #[test]
9638 fn tbox_coherence_flags_unsatisfiable_class_index_free() {
9639 use crate::reader::{CountingReader, SliceReader};
9640 let rt = RDF_TYPE;
9641 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9642 let disj = "<http://www.w3.org/2002/07/owl#disjointWith>";
9643 let bytes = build_with_pyramid(&[
9646 ("<http://ex/C>", sub, "<http://ex/D>"),
9647 ("<http://ex/C>", sub, "<http://ex/E>"),
9648 ("<http://ex/D>", disj, "<http://ex/E>"),
9649 ("<http://ex/x>", rt, "<http://ex/C>"),
9650 ]);
9651
9652 let r = CountingReader::new(SliceReader::new(&bytes));
9653 let view = SummaryView::open_ranged(&r).unwrap().unwrap();
9654 let points = view.tbox_coherence();
9655 assert!(
9656 points
9657 .iter()
9658 .any(|i| i.kind == "unsatisfiable-class" && i.detail.contains("http://ex/C>")),
9659 "expected C unsatisfiable from the schema alone, got {points:?}"
9660 );
9661
9662 let header = Header::from_bytes(&bytes[..HEADER_LEN]).unwrap();
9665 assert!(
9666 r.bytes_read() <= bytes.len() as u64 - header.root_dir_len,
9667 "tbox_coherence must not read the triple index"
9668 );
9669 }
9670
9671 #[test]
9672 fn schema_coherence_reads_only_the_schema_block() {
9673 use crate::reader::{CountingReader, SliceReader};
9674 let rt = RDF_TYPE;
9675 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9676 let disj = "<http://www.w3.org/2002/07/owl#disjointWith>";
9677 let mut triples: Vec<(String, String, String)> = vec![
9681 ("<http://ex/C>".into(), sub.into(), "<http://ex/D>".into()),
9682 ("<http://ex/C>".into(), sub.into(), "<http://ex/E>".into()),
9683 ("<http://ex/D>".into(), disj.into(), "<http://ex/E>".into()),
9684 ];
9685 for i in 0..500 {
9686 let s = format!("<http://ex/x{i}>");
9687 triples.push((s.clone(), rt.into(), "<http://ex/C>".into()));
9688 triples.push((
9689 s,
9690 "<http://ex/label>".into(),
9691 format!("\"unique label {i}\""),
9692 ));
9693 }
9694 let trefs: Vec<(&str, &str, &str)> = triples
9695 .iter()
9696 .map(|(s, p, o)| (s.as_str(), p.as_str(), o.as_str()))
9697 .collect();
9698 let bytes = build_with_pyramid(&trefs);
9699
9700 let header = Header::from_bytes(&bytes[..HEADER_LEN]).unwrap();
9701 assert!(
9702 header.schema_meta_len > 0,
9703 "the writer recorded a schema-block length"
9704 );
9705 assert!(
9706 (header.schema_meta_len as u64) < header.pyramid_meta_len,
9707 "schema block ({}) should be far smaller than the whole pyramid-meta ({})",
9708 header.schema_meta_len,
9709 header.pyramid_meta_len
9710 );
9711
9712 let r = CountingReader::new(SliceReader::new(&bytes));
9713 let points = read_schema_coherence_ranged(&r).unwrap().unwrap();
9714 assert!(points.iter().any(|i| i.kind == "unsatisfiable-class"));
9715 assert!(
9717 r.bytes_read() <= HEADER_LEN as u64 + header.schema_meta_len as u64,
9718 "read {} bytes; expected <= header + schema block ({})",
9719 r.bytes_read(),
9720 HEADER_LEN as u64 + header.schema_meta_len as u64
9721 );
9722 }
9723
9724 #[test]
9725 fn tbox_coherence_clean_schema_is_coherent() {
9726 use crate::reader::SliceReader;
9727 let rt = RDF_TYPE;
9728 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9729 let bytes = build_with_pyramid(&[
9730 ("<http://ex/Dog>", sub, "<http://ex/Animal>"),
9731 ("<http://ex/x>", rt, "<http://ex/Dog>"),
9732 ]);
9733 let view = SummaryView::open_ranged(&SliceReader::new(&bytes))
9734 .unwrap()
9735 .unwrap();
9736 assert!(view.tbox_is_coherent(), "a plain hierarchy is coherent");
9737 }
9738
9739 #[test]
9747 fn readers_before_the_quad_index_refuse_its_files() {
9748 for perms in [PermSet::ALL, PermSet::CORE] {
9749 let quads: Vec<crate::ingest::RawQuad> = (0..50)
9750 .map(|i| {
9751 (
9752 format!("<http://ex/s{i}>"),
9753 "<http://ex/p>".to_string(),
9754 format!("\"{i}\""),
9755 Some(format!("<http://ex/g{}>", i % 7)),
9756 )
9757 })
9758 .collect();
9759 let (bytes, _) = crate::ingest::assemble_dataset_with_layout(
9760 quads,
9761 false,
9762 false,
9763 None,
9764 crate::PyramidAlgo::Louvain,
9765 perms,
9766 crate::ingest::GraphLayout::Quad,
9767 |_, _| Vec::<u8>::new(),
9768 );
9769 let h = Header::from_bytes(&bytes).unwrap();
9770 assert!(h.has_quad_index());
9771 let root =
9772 &bytes[h.root_dir_offset as usize..(h.root_dir_offset + h.root_dir_len) as usize];
9773 assert!(decode_index_container(root, h.block_codec, h.perms).is_err());
9775 assert!(
9776 container_section_payload_ranges(root, h.root_dir_offset, h.perms.len()).is_err()
9777 );
9778 assert_ne!(decode_container(root, CODEC_NONE).unwrap().len(), NUM_PERMS);
9780 assert_ne!(
9781 decode_container(root, CODEC_NONE).unwrap().len(),
9782 h.perms.len()
9783 );
9784 let r = crate::reader::SliceReader::new(&bytes);
9786 for expect in [h.perms.len(), NUM_PERMS] {
9787 assert!(locate_container_section_ranged(
9788 &r,
9789 h.root_dir_offset,
9790 h.root_dir_len,
9791 0,
9792 expect as u64
9793 )
9794 .is_err());
9795 }
9796 assert_eq!(hash_of_sections(&h, &bytes).unwrap(), h.content_hash);
9798 let rete = Rete::open(&bytes).unwrap();
9800 assert_eq!(rete.named_graph_count(), 7);
9801 assert_eq!(rete.dump(Some("<http://ex/g3>")).len(), 7);
9802 }
9803 }
9804
9805 #[test]
9806 fn named_graphs_round_trip() {
9807 let all = [
9809 ("Alice", "knows", "Bob"), ("Bob", "age", "30"), ];
9812 let mut db = DictionaryBuilder::new();
9813 for (s, p, o) in all {
9814 db.observe(s, p, o);
9815 }
9816 let dict = db.build();
9817
9818 let mut def = GraphIndexBuilder::new();
9819 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
9820 let mut g1 = GraphIndexBuilder::new();
9821 g1.push(dict.encode("Bob", "age", "30").unwrap());
9822
9823 let named = vec![("http://ex/g1".to_string(), g1.build())];
9824 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9825
9826 assert!(verify(&bytes).unwrap());
9827 let rete = Rete::open(&bytes).unwrap();
9828 assert_eq!(rete.graph_names(), vec!["http://ex/g1"]);
9829 let gi = rete.graph_index("http://ex/g1").unwrap();
9831 assert_eq!(gi.triple_count(), 1);
9832 assert!(rete.graph_index("http://ex/missing").is_none());
9833
9834 assert_eq!(rete.header().quad_count, 2);
9837
9838 assert_eq!(rete.query(Some("Alice"), None, None).len(), 1);
9840
9841 assert_eq!(
9843 rete.dump(None),
9844 vec![("Alice".into(), "knows".into(), "Bob".into())]
9845 );
9846 assert_eq!(
9847 rete.dump(Some("http://ex/g1")),
9848 vec![("Bob".into(), "age".into(), "30".into())]
9849 );
9850 }
9851
9852 #[test]
9858 fn named_graph_container_opens_tile_lazily_like_the_root() {
9859 let mut db = DictionaryBuilder::new();
9860 let node = |n: u32| format!("<http://ex/n{n}>");
9861 let p = "<http://ex/p>".to_string();
9862 db.observe(&node(0), &p, &node(1));
9863 for i in 0..500u32 {
9864 db.observe(&node(1000 + i), &p, &node(2000 + i));
9865 }
9866 let dict = db.build();
9867 let mut def = GraphIndexBuilder::new();
9868 def.push(dict.encode(&node(0), &p, &node(1)).unwrap());
9869 let mut g = GraphIndexBuilder::new().with_tile_budget(256); for i in 0..500u32 {
9871 g.push(dict.encode(&node(1000 + i), &p, &node(2000 + i)).unwrap());
9872 }
9873 let named = vec![("<http://ex/g>".to_string(), g.build())];
9874 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9875 let rete = Rete::open(&bytes).unwrap();
9876 let header = rete.header();
9877
9878 let soff = header.named_graphs_offset as usize;
9881 let send = soff + header.named_graphs_len as usize;
9882 let sec = &bytes[soff..send];
9883 let (n, mut pos) = read_uvarint(sec).unwrap();
9884 assert_eq!(n, 1);
9885 let (ilen, u1) = read_uvarint(&sec[pos..]).unwrap();
9886 pos += u1 + ilen as usize;
9887 let (clen, u2) = read_uvarint(&sec[pos..]).unwrap();
9888 pos += u2;
9889 let container = ByteRange {
9890 offset: (soff + pos) as u64,
9891 len: clen,
9892 };
9893
9894 use crate::reader::SliceReader;
9895 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9896 let reader: std::sync::Arc<dyn RangeReader + Send + Sync> =
9897 std::sync::Arc::new(SliceReader::new(leaked));
9898 let (lazy_idx, _, _) = open_index_container_lazy(
9899 &reader,
9900 container,
9901 header.block_codec,
9902 header.has_tile_synopsis(),
9903 1,
9904 header.perms,
9905 header.perms.len(),
9906 )
9907 .unwrap();
9908 let mut want = rete
9909 .graph_index("<http://ex/g>")
9910 .unwrap()
9911 .match_pattern((None, None, None));
9912 let mut got = lazy_idx.match_pattern((None, None, None));
9913 want.sort_unstable();
9914 got.sort_unstable();
9915 assert_eq!(got.len(), 500);
9916 assert_eq!(got, want);
9917 assert!(!lazy_idx.load_incomplete());
9918 }
9919
9920 #[test]
9926 fn dump_iter_matches_dump_and_stops_early() {
9927 use crate::reader::{CountingReader, SliceReader};
9928
9929 let triples: Vec<(String, String, String)> = (0..2000u32)
9931 .map(|i| {
9932 (
9933 format!("<http://ex/s/{i:04}>"),
9934 "<http://ex/p>".to_string(),
9935 format!("<http://ex/o/{i:04}>"),
9936 )
9937 })
9938 .collect();
9939 let mut db = DictionaryBuilder::new();
9940 for (s, p, o) in &triples {
9941 db.observe(s, p, o);
9942 }
9943 let dict = db.build();
9944 let mut ib = GraphIndexBuilder::new().with_tile_budget(256);
9945 for (s, p, o) in &triples {
9946 ib.push(dict.encode(s, p, o).unwrap());
9947 }
9948 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9949
9950 let rete = Rete::open(&bytes).unwrap();
9952 assert_eq!(rete.dump_iter(None).collect::<Vec<_>>(), rete.dump(None));
9953 assert_eq!(rete.dump_iter(Some("http://ex/nope")).count(), 0);
9955
9956 let read_bytes = |take: usize| -> u64 {
9958 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9959 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
9960 let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
9961 let before = reader.bytes_read();
9962 let n = rete.dump_iter(None).take(take).count();
9963 assert_eq!(n, take.min(triples.len()));
9964 assert!(!rete.index_incomplete());
9965 reader.bytes_read() - before
9966 };
9967 let first_only = read_bytes(1);
9968 let everything = read_bytes(triples.len());
9969 assert!(
9971 first_only < everything,
9972 "dump_iter is not lazy: one triple read {first_only} B, a full scan {everything} B"
9973 );
9974 #[cfg(feature = "compression")]
9981 assert!(
9982 first_only * 2 < everything,
9983 "dump_iter is not lazy: one triple read {first_only} B of the {everything} B a full scan reads"
9984 );
9985
9986 let fresh = Rete::open(&bytes).unwrap();
9991 let drain = |max_quads: usize| {
9996 let (mut out, mut cursor, mut calls) = (Vec::new(), 0u32, 0);
9997 loop {
9998 let (batch, next, done) = fresh.dump_batch(None, cursor, max_quads);
9999 out.extend(batch);
10000 cursor = next;
10001 calls += 1;
10002 assert!(calls < 10_000, "cursor failed to advance");
10003 if done {
10004 break out;
10005 }
10006 }
10007 };
10008 let eager = fresh.dump(None);
10009 for max in [1usize, 7, 128, 100_000] {
10010 assert_eq!(drain(max), eager, "batch size {max} changed the dump");
10011 }
10012 let (t, _, done) = fresh.dump_batch(Some("http://ex/nope"), 0, 16);
10015 assert!(t.is_empty() && done);
10016 }
10017
10018 #[test]
10026 fn query_batch_matches_query_in_graph_and_stops_early() {
10027 use crate::reader::{CountingReader, SliceReader};
10028
10029 let triples: Vec<(String, String, String)> = (0..2000u32)
10032 .map(|i| {
10033 (
10034 format!("<http://ex/s/{:04}>", i / 2),
10035 format!("<http://ex/p/{}>", i % 2),
10036 format!("<http://ex/o/{i:04}>"),
10037 )
10038 })
10039 .collect();
10040 let mut db = DictionaryBuilder::new();
10041 for (s, p, o) in &triples {
10042 db.observe(s, p, o);
10043 }
10044 let dict = db.build();
10045 let mut ib = GraphIndexBuilder::new().with_tile_budget(256);
10046 for (s, p, o) in &triples {
10047 ib.push(dict.encode(s, p, o).unwrap());
10048 }
10049 let bytes = write_file(&dict, &ib.build(), false, &[], 0);
10050 let rete = Rete::open(&bytes).unwrap();
10051
10052 let drain = |s: Option<&str>, p: Option<&str>, o: Option<&str>, max: usize| {
10053 let (mut out, mut cursor, mut calls) = (Vec::new(), 0u64, 0);
10054 loop {
10055 let (batch, next, done) = rete.query_batch(None, s, p, o, cursor, max);
10056 assert!(!batch.is_empty() || done, "empty non-final batch");
10059 out.extend(batch);
10060 cursor = next;
10061 calls += 1;
10062 assert!(calls < 20_000, "cursor failed to advance");
10063 if done {
10064 break out;
10065 }
10066 }
10067 };
10068 let shapes: [(Option<&str>, Option<&str>, Option<&str>); 6] = [
10069 (None, None, None),
10070 (Some("<http://ex/s/0007>"), None, None),
10071 (None, Some("<http://ex/p/1>"), None),
10072 (None, None, Some("<http://ex/o/0042>")),
10073 (Some("<http://ex/s/0007>"), Some("<http://ex/p/0>"), None),
10074 (
10075 Some("<http://ex/s/0007>"),
10076 Some("<http://ex/p/0>"),
10077 Some("<http://ex/o/0014>"),
10078 ),
10079 ];
10080 for (s, p, o) in shapes {
10081 let eager = rete.query_in_graph(None, s, p, o);
10082 assert!(
10083 !eager.is_empty(),
10084 "fixture shape {s:?} {p:?} {o:?} matches nothing"
10085 );
10086 assert_eq!(
10088 rete.query_iter(None, s, p, o).collect::<Vec<_>>(),
10089 eager,
10090 "query_iter disagrees for {s:?} {p:?} {o:?}"
10091 );
10092 for max in [1usize, 3, 64, 100_000] {
10093 let mut got = drain(s, p, o, max);
10094 let mut want = eager.clone();
10095 got.sort();
10098 want.sort();
10099 assert_eq!(got, want, "batch size {max} changed {s:?} {p:?} {o:?}");
10100 }
10101 if s.is_none() && p.is_none() && o.is_none() {
10103 assert_eq!(drain(s, p, o, 7), eager, "unbound batching reordered rows");
10104 }
10105 }
10106
10107 let (t, _, done) = rete.query_batch(None, Some("<http://ex/nope>"), None, None, 0, 16);
10110 assert!(t.is_empty() && done);
10111 let (t, _, done) = rete.query_batch(Some("http://ex/nope"), None, None, None, 0, 16);
10112 assert!(t.is_empty() && done);
10113 assert_eq!(
10114 rete.query_iter(None, Some("<http://ex/nope>"), None, None)
10115 .count(),
10116 0
10117 );
10118
10119 let read_bytes = |take: usize| -> u64 {
10122 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
10123 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10124 let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10125 let before = reader.bytes_read();
10126 let (mut n, mut cursor) = (0usize, 0u64);
10127 loop {
10128 let (batch, next, done) = rete.query_batch(None, None, None, None, cursor, 1);
10129 n += batch.len();
10130 cursor = next;
10131 if done || n >= take {
10132 break;
10133 }
10134 }
10135 assert!(n >= take.min(triples.len()));
10136 assert!(!rete.index_incomplete());
10137 reader.bytes_read() - before
10138 };
10139 let first_only = read_bytes(1);
10140 let everything = read_bytes(triples.len());
10141 assert!(
10142 first_only < everything,
10143 "query_batch is not lazy: one row read {first_only} B, a full scan {everything} B"
10144 );
10145 #[cfg(feature = "compression")]
10146 assert!(
10147 first_only * 2 < everything,
10148 "query_batch is not lazy: one row read {first_only} B of the {everything} B a full scan reads"
10149 );
10150 }
10151
10152 #[test]
10155 fn query_batch_is_graph_scoped() {
10156 let mut db = DictionaryBuilder::new();
10157 for (s, p, o) in [
10158 ("Alice", "knows", "Bob"),
10159 ("Alice", "knows", "Carol"),
10160 ("Alice", "knows", "Dave"),
10161 ("Erin", "knows", "Frank"),
10162 ("Alice", "knows", "Zoe"),
10163 ] {
10164 db.observe(s, p, o);
10165 }
10166 let dict = db.build();
10167 let mut def = GraphIndexBuilder::new();
10168 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10169 def.push(dict.encode("Alice", "knows", "Carol").unwrap());
10170 let mut g1 = GraphIndexBuilder::new();
10171 g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10172 g1.push(dict.encode("Erin", "knows", "Frank").unwrap());
10173 let mut g2 = GraphIndexBuilder::new();
10174 g2.push(dict.encode("Alice", "knows", "Zoe").unwrap());
10175 let named = vec![
10176 ("http://ex/g1".to_string(), g1.build()),
10177 ("http://ex/g2".to_string(), g2.build()),
10178 ];
10179 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10180 let rete = Rete::open(&bytes).unwrap();
10181 for graph in [None, Some("http://ex/g1"), Some("http://ex/g2")] {
10182 let eager = rete.query_in_graph(graph, None, None, None);
10183 let (mut out, mut cursor) = (Vec::new(), 0u64);
10184 loop {
10185 let (batch, next, done) = rete.query_batch(graph, None, None, None, cursor, 1);
10186 out.extend(batch);
10187 cursor = next;
10188 if done {
10189 break;
10190 }
10191 }
10192 assert_eq!(out, eager, "graph {graph:?} batched differently");
10193 }
10194 }
10195
10196 #[test]
10197 fn query_in_graph_is_graph_scoped() {
10198 let mut db = DictionaryBuilder::new();
10201 for (s, p, o) in [
10202 ("Alice", "knows", "Bob"),
10203 ("Alice", "knows", "Carol"),
10204 ("Alice", "knows", "Dave"),
10205 ] {
10206 db.observe(s, p, o);
10207 }
10208 let dict = db.build();
10209
10210 let mut def = GraphIndexBuilder::new();
10211 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10212 def.push(dict.encode("Alice", "knows", "Carol").unwrap());
10213 let mut g1 = GraphIndexBuilder::new();
10214 g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10215
10216 let named = vec![("http://ex/g1".to_string(), g1.build())];
10217 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10218 let rete = Rete::open(&bytes).unwrap();
10219
10220 let mut def_objs: Vec<String> = rete
10222 .query_in_graph(None, Some("Alice"), Some("knows"), None)
10223 .into_iter()
10224 .map(|(_, _, o)| o)
10225 .collect();
10226 def_objs.sort();
10227 assert_eq!(def_objs, vec!["Bob".to_string(), "Carol".to_string()]);
10228
10229 assert_eq!(
10231 rete.query_in_graph(Some("http://ex/g1"), Some("Alice"), None, None),
10232 vec![("Alice".into(), "knows".into(), "Dave".into())]
10233 );
10234
10235 assert_eq!(rete.query_in_graph(None, None, None, None).len(), 2);
10237 assert_eq!(
10238 rete.query_in_graph(Some("http://ex/g1"), None, None, None)
10239 .len(),
10240 1
10241 );
10242
10243 assert!(rete
10245 .query_in_graph(Some("http://ex/missing"), None, None, None)
10246 .is_empty());
10247 }
10248
10249 #[test]
10262 fn bounded_query_in_graph_does_not_fault_the_whole_dictionary() {
10263 use crate::reader::{CountingReader, SliceReader};
10264
10265 let payload = |i: u32| -> String {
10271 let mut s = String::with_capacity(800);
10272 let mut x = i as u64 ^ 0x9e37_79b9_7f4a_7c15;
10273 while s.len() < 800 {
10274 x = x
10275 .wrapping_mul(6364136223846793005)
10276 .wrapping_add(1442695040888963407);
10277 s.push_str(&format!("{:016x}", x ^ (x >> 29)));
10278 }
10279 s
10280 };
10281 let quads: Vec<(String, String, String, bool)> = (0..2000u32)
10288 .map(|i| {
10289 let named = i % 2 == 0;
10290 let ns = if named { 'a' } else { 'b' };
10291 (
10292 format!("<http://ex/{ns}/s/{i:04}>"),
10293 "<http://ex/abstract>".to_string(),
10294 format!("\"{ns}{} number {i}\"", payload(i)),
10295 named,
10296 )
10297 })
10298 .collect();
10299 let mut db = DictionaryBuilder::new();
10300 for (s, p, o, _) in &quads {
10301 db.observe(s, p, o);
10302 }
10303 let dict = db.build();
10304 let mut def = GraphIndexBuilder::new().with_tile_budget(256);
10305 let mut g1 = GraphIndexBuilder::new().with_tile_budget(256);
10306 for (s, p, o, named) in &quads {
10307 let t = dict.encode(s, p, o).unwrap();
10308 if *named {
10309 g1.push(t);
10310 } else {
10311 def.push(t);
10312 }
10313 }
10314 let named = vec![("http://ex/g1".to_string(), g1.build())];
10315 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10316
10317 let eager = Rete::open(&bytes).unwrap();
10318 let dict_len = eager.header().dictionary_len;
10319
10320 let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
10323 let probe = |f: &dyn Fn(&Rete) -> Vec<TermTriple>| -> (Vec<TermTriple>, u64) {
10324 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10325 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10326 let before = reader.bytes_read();
10327 let out = f(&lazy);
10328 assert!(!lazy.index_incomplete(), "a chunk fetch failed");
10329 (out, reader.bytes_read() - before)
10330 };
10331
10332 let want = eager.query_in_graph(
10334 Some("http://ex/g1"),
10335 Some("<http://ex/a/s/0100>"),
10336 None,
10337 None,
10338 );
10339 assert_eq!(want.len(), 1, "fixture: expected exactly one match");
10340 let (got, pulled) = probe(&|r| {
10341 r.query_in_graph(
10342 Some("http://ex/g1"),
10343 Some("<http://ex/a/s/0100>"),
10344 None,
10345 None,
10346 )
10347 });
10348 assert_eq!(got, want, "lazy query_in_graph disagrees with eager");
10349 assert!(
10350 pulled * 4 < dict_len,
10351 "a 1-row query_in_graph faulted {pulled} B of a {dict_len} B dictionary — \
10352 it is prefetching the whole thing again"
10353 );
10354
10355 let (empty, q_pulled) =
10363 probe(&|r| r.query_in_graph(Some("http://ex/nope"), None, None, None));
10364 assert!(empty.is_empty());
10365 let (_, d_pulled) = probe(&|r| {
10366 let mut n = 0usize;
10367 r.dump_each(Some("http://ex/nope"), |_, _, _| n += 1);
10368 assert_eq!(n, 0, "an absent graph yielded triples");
10369 Vec::new()
10370 });
10371 assert_eq!(
10372 q_pulled, d_pulled,
10373 "query_in_graph and dump_each disagree on what an absent graph costs"
10374 );
10375 assert!(
10376 q_pulled * 4 < dict_len,
10377 "an absent graph faulted {q_pulled} B of a {dict_len} B dictionary"
10378 );
10379
10380 let want: Vec<TermTriple> = eager.dump(Some("http://ex/g1"));
10383 assert_eq!(want.len(), 1000);
10384 let (got, pulled) = probe(&|r| {
10385 let mut out = Vec::new();
10386 r.dump_each(Some("http://ex/g1"), |s, p, o| {
10387 out.push((s.to_string(), p.to_string(), o.to_string()))
10388 });
10389 out
10390 });
10391 assert_eq!(got, want, "lazy dump_each disagrees with eager dump");
10392 assert!(
10393 pulled * 3 < dict_len * 2,
10394 "dumping half the graph faulted {pulled} B of a {dict_len} B dictionary — \
10395 the other graph's chunks are being pulled in too"
10396 );
10397 }
10398
10399 #[cfg(test)]
10404 fn build_multichunk_image() -> Vec<u8> {
10405 use crate::index::PermSet;
10406 let mut db = DictionaryBuilder::new();
10407 let mut triples: Vec<(String, String, String)> = (0..4000u32)
10408 .map(|i| {
10409 (
10410 format!("<http://ex/s/{i:07}>"),
10411 format!("<http://ex/p/{}>", i % 8),
10412 format!("\"object literal number {i:07} with padding text to widen the term\""),
10413 )
10414 })
10415 .collect();
10416 triples.push((
10417 "<http://ex/s/big>".to_string(),
10418 "<http://ex/p/big>".to_string(),
10419 format!("\"{}\"", "Z".repeat(8000)),
10420 ));
10421 for (s, p, o) in &triples {
10422 db.observe(s, p, o);
10423 }
10424 let dict = db.build();
10425 let mut idx = GraphIndexBuilder::new()
10426 .with_tile_budget(64)
10427 .with_perms(PermSet::ALL);
10428 for (s, p, o) in &triples {
10429 idx.push(dict.encode(s, p, o).unwrap());
10430 }
10431 write_dataset(&dict, &idx.build(), &[], true, &[], 0)
10432 }
10433
10434 #[test]
10443 fn tiny_dict_cache_evicts_yet_dump_is_byte_identical() {
10444 use crate::reader::SliceReader;
10445 let leaked: &'static [u8] = Box::leak(build_multichunk_image().into_boxed_slice());
10446 let dump = |cap: u64| -> (
10447 Vec<(String, String, String)>,
10448 crate::chunk_cache::CacheStats,
10449 ) {
10450 let rete =
10451 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10452 rete.dict.set_cache_cap(cap);
10453 let mut out = Vec::new();
10454 rete.dump_filtered_each(None, None, None, None, |s, p, o| {
10455 out.push((s.to_string(), p.to_string(), o.to_string()))
10456 });
10457 assert!(
10458 !rete.index_incomplete(),
10459 "no lazy fault should fail at cap {cap}"
10460 );
10461 let stats = rete.dict_cache_stats();
10462 (out, stats)
10463 };
10464
10465 let (unlimited, us) = dump(u64::MAX);
10466 assert_eq!(unlimited.len(), 4001, "every triple dumped");
10467 assert_eq!(us.evictions, 0, "unlimited cap evicts nothing");
10468
10469 let (tiny, ts) = dump(128 * 1024);
10472 assert_eq!(tiny, unlimited, "128 KiB-cap dump differs from unlimited");
10473 assert!(ts.evictions > 0, "a tiny cap must evict (stats {ts:?})");
10474
10475 let (starved, ss) = dump(1);
10477 assert_eq!(starved, unlimited, "1-byte-cap dump differs from unlimited");
10478 assert!(
10479 ss.evictions > 0,
10480 "1-byte cap evicts every insert (stats {ss:?})"
10481 );
10482 }
10483
10484 #[test]
10495 fn memory_budget_charges_the_fixed_directories() {
10496 use crate::reader::SliceReader;
10497 let leaked: &'static [u8] = Box::leak(build_multichunk_image().into_boxed_slice());
10498 let rete =
10499 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10500
10501 let fixed = rete.fixed_overhead();
10502 assert!(
10505 fixed.dict_directory > 0 && fixed.index_directory > 0,
10506 "{fixed:?}"
10507 );
10508 assert_eq!(fixed.named_graphs, 0, "no named graphs in this file");
10509 assert_eq!(
10510 fixed.total(),
10511 fixed.dict_directory + fixed.index_directory + fixed.named_graphs
10512 );
10513
10514 let total: u64 = 512 << 20;
10517 let applied = rete.set_memory_budget(Some(total));
10518 let plain = split_memory_budget(total);
10519 assert_eq!(applied.dict_cache, plain.dict_cache - fixed.total());
10520 assert_eq!(rete.dict_cache_cap(), applied.dict_cache);
10521 assert_eq!(applied.tile_cache, plain.tile_cache);
10522 assert_eq!(applied.block_cache, plain.block_cache);
10523 assert!(
10524 applied.dict_cache + applied.tile_cache + applied.block_cache + fixed.total() <= total,
10525 "caps + fixed directories exceed the budget: {applied:?} + {fixed:?} > {total}"
10526 );
10527
10528 let applied = rete.set_memory_budget(Some(1 << 20));
10530 assert_eq!(applied.dict_cache, MIN_DICT_CACHE_CAP);
10531
10532 let applied = rete.set_memory_budget(None);
10534 assert_eq!(applied.dict_cache, u64::MAX);
10535 assert_eq!(rete.dict_cache_cap(), u64::MAX);
10536
10537 let rete =
10540 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10541 rete.dict.set_cache_cap(128 * 1024);
10542 let mut rows = 0usize;
10543 rete.dump_filtered_each(None, None, None, None, |_, _, _| rows += 1);
10544 assert_eq!(rows, 4001);
10545 let s = rete.dict_cache_stats();
10546 assert!(s.evictions > 0, "the cap must bind: {s:?}");
10547 assert!(
10550 s.entries <= 1 || s.resident_bytes <= 128 * 1024,
10551 "dictionary cache over its cap after the dump: {s:?}"
10552 );
10553 }
10554
10555 #[test]
10560 fn releasing_a_named_graph_drops_and_reopens_its_index() {
10561 use crate::index::PermSet;
10562 use crate::reader::SliceReader;
10563 let mut db = DictionaryBuilder::new();
10564 let all: Vec<(String, String, String)> = (0..40u32)
10565 .map(|i| {
10566 (
10567 format!("<http://ex/s{}>", i % 7),
10568 format!("<http://ex/p{}>", i % 3),
10569 format!("<http://ex/o{}>", i % 5),
10570 )
10571 })
10572 .collect();
10573 for (s, p, o) in &all {
10574 db.observe(s, p, o);
10575 }
10576 let dict = db.build();
10577 let mk = |take: fn(u32) -> bool| {
10578 let mut g = GraphIndexBuilder::new()
10579 .with_tile_budget(16)
10580 .with_perms(PermSet::ALL);
10581 for (i, (s, p, o)) in all.iter().enumerate() {
10582 if take(i as u32) {
10583 g.push(dict.encode(s, p, o).unwrap());
10584 }
10585 }
10586 g.build()
10587 };
10588 let def = mk(|_| true);
10589 let g1 = mk(|i| i % 2 == 0);
10590 let g2 = mk(|i| i % 3 == 0);
10591 let image = write_dataset(
10592 &dict,
10593 &def,
10594 &[
10595 ("<http://ex/g1>".to_string(), g1),
10596 ("<http://ex/g2>".to_string(), g2),
10597 ],
10598 true,
10599 &[],
10600 0,
10601 );
10602 let leaked: &'static [u8] = Box::leak(image.into_boxed_slice());
10603 let mut rete =
10604 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10605
10606 let opened = |r: &Rete| -> usize {
10607 match &r.named_graphs {
10608 NamedGraphsSlot::Lazy(l) => {
10609 let mut n = 0;
10610 l.for_each_opened(|_| n += 1);
10611 n
10612 }
10613 _ => 0,
10614 }
10615 };
10616
10617 assert_eq!(opened(&rete), 0, "nothing opened before first touch");
10618 assert!(rete.graph_index("<http://ex/g1>").is_some());
10620 assert_eq!(opened(&rete), 1, "g1 opened");
10621 rete.release_named_graph("<http://ex/g1>");
10623 assert_eq!(opened(&rete), 0, "g1 released");
10624 rete.release_named_graph("<http://ex/absent>");
10626 rete.release_named_graph("<http://ex/g1>");
10627 assert_eq!(opened(&rete), 0);
10628 assert!(rete.graph_index("<http://ex/g1>").is_some());
10630 assert_eq!(opened(&rete), 1, "g1 re-opened after release");
10631 }
10632
10633 #[test]
10651 fn filtered_dump_is_exactly_the_unfiltered_dump_filtered() {
10652 use crate::index::PermSet;
10653 use crate::reader::SliceReader;
10654
10655 let terms = |i: u32| {
10656 (
10657 format!("<http://ex/s{}>", i % 11),
10658 format!("<http://ex/p{}>", i % 3),
10659 if i.is_multiple_of(4) {
10663 format!("\"lit {}\"@en", i % 5)
10664 } else {
10665 format!("<http://ex/o{}>", i % 7)
10666 },
10667 )
10668 };
10669 let build = |perms: PermSet| {
10670 let mut db = DictionaryBuilder::new();
10671 let all: Vec<_> = (0..90u32).map(terms).collect();
10672 for (s, p, o) in &all {
10673 db.observe(s, p, o);
10674 }
10675 let dict = db.build();
10676 let mut def = GraphIndexBuilder::new()
10677 .with_tile_budget(16)
10678 .with_perms(perms);
10679 let mut g1 = GraphIndexBuilder::new()
10680 .with_tile_budget(16)
10681 .with_perms(perms);
10682 for (i, (s, p, o)) in all.iter().enumerate() {
10683 let t = dict.encode(s, p, o).unwrap();
10684 if i % 2 == 0 {
10688 g1.push(t);
10689 }
10690 def.push(t);
10691 }
10692 write_dataset(
10693 &dict,
10694 &def.build(),
10695 &[("<http://ex/g1>".to_string(), g1.build())],
10696 true,
10697 &[],
10698 0,
10699 )
10700 };
10701
10702 let shapes: Vec<(Option<&str>, Option<&str>, Option<&str>)> = vec![
10703 (None, None, None),
10704 (Some("<http://ex/s3>"), None, None),
10705 (None, Some("<http://ex/p1>"), None),
10706 (None, None, Some("<http://ex/o5>")),
10707 (None, None, Some("\"lit 0\"@en")),
10708 (Some("<http://ex/s3>"), Some("<http://ex/p0>"), None),
10709 (Some("<http://ex/s4>"), None, Some("<http://ex/o4>")),
10710 (None, Some("<http://ex/p1>"), Some("<http://ex/o5>")),
10711 (
10712 Some("<http://ex/s3>"),
10713 Some("<http://ex/p0>"),
10714 Some("<http://ex/o3>"),
10715 ),
10716 (Some("<http://ex/nope>"), None, None),
10720 (None, Some("<http://ex/nope>"), None),
10721 (None, None, Some("<http://ex/nope>")),
10722 ];
10723
10724 let mut any_pruned = false;
10725 for perms in [PermSet::ALL, PermSet::CORE] {
10726 let image = build(perms);
10727 let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
10728 let opens: Vec<(&str, Rete)> = vec![
10729 ("resident", Rete::open(&image).unwrap()),
10730 (
10731 "ranged",
10732 Rete::open_ranged(&SliceReader::new(leaked)).unwrap(),
10733 ),
10734 (
10735 "lazy",
10736 Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64)
10737 .unwrap(),
10738 ),
10739 ];
10740 for graph in [None, Some("<http://ex/g1>"), Some("<http://ex/absent>")] {
10741 let full: Vec<TermTriple> = {
10743 let mut v = Vec::new();
10744 Rete::open(&image).unwrap().dump_each(graph, |s, p, o| {
10745 v.push((s.to_string(), p.to_string(), o.to_string()))
10746 });
10747 v.sort();
10748 v
10749 };
10750 for &(s, p, o) in &shapes {
10751 let mut want = full.clone();
10752 want.retain(|(ts, tp, to)| {
10753 s.is_none_or(|x| x == ts)
10754 && p.is_none_or(|x| x == tp)
10755 && o.is_none_or(|x| x == to)
10756 });
10757 for (path, rete) in &opens {
10758 let mut got = Vec::new();
10759 rete.dump_filtered_each(graph, s, p, o, |s, p, o| {
10760 got.push((s.to_string(), p.to_string(), o.to_string()))
10761 });
10762 got.sort();
10763 assert_eq!(
10764 got,
10765 want,
10766 "{path}/{perms:?} graph={graph:?} pattern={:?} — filtered dump \
10767 is not the unfiltered dump filtered",
10768 (s, p, o)
10769 );
10770 assert!(!rete.index_incomplete(), "{path}: a fetch failed");
10771 }
10772 let plan = opens[2].1.dump_plan(graph, s, p, o);
10776 if graph == Some("<http://ex/absent>")
10777 || [s, p, o].contains(&Some("<http://ex/nope>"))
10778 {
10779 assert!(
10780 plan.scan.is_none(),
10781 "an unmatchable dump still planned a scan: {:?}",
10782 (graph, s, p, o)
10783 );
10784 } else {
10785 let scan = plan.scan.expect("a matchable dump plans a scan");
10786 assert!(
10787 PermSet::CORE.contains(scan.permutation),
10788 "a dump routed to {:?}, outside PermSet::CORE — a \
10789 --permutations 3 file has no such section",
10790 scan.permutation
10791 );
10792 assert!(scan.tiles_admitted <= scan.tiles_routed);
10793 assert!(scan.tiles_routed <= scan.tiles_total);
10794 assert!(scan.tile_bytes <= scan.section_bytes);
10795 if scan.tiles_admitted < scan.tiles_total {
10796 any_pruned = true;
10797 }
10798 }
10799 }
10800 }
10801 }
10802 assert!(
10803 any_pruned,
10804 "no shape pruned a single tile — the fixture is too small to be \
10805 measuring anything"
10806 );
10807 }
10808
10809 #[test]
10819 fn a_predicate_scoped_dump_fetches_less_than_the_graph() {
10820 use crate::reader::{CountingReader, SliceReader};
10821
10822 let mut db = DictionaryBuilder::new();
10828 let mut all = Vec::new();
10829 for s in 0..200u32 {
10830 for p in 0..20u32 {
10831 let t = (
10832 format!("<http://ex/s/{s:04}>"),
10833 format!("<http://ex/p/{p:02}>"),
10834 format!("\"object {s:04}/{p:02}\""),
10835 );
10836 db.observe(&t.0, &t.1, &t.2);
10837 all.push(t);
10838 }
10839 }
10840 let dict = db.build();
10841 let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
10842 for (s, p, o) in &all {
10843 ib.push(dict.encode(s, p, o).unwrap());
10844 }
10845 let image = write_dataset(&dict, &ib.build(), &[], false, &[], 0);
10846 let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
10847
10848 let probe = |s: Option<&str>, p: Option<&str>, o: Option<&str>| -> (usize, u64) {
10850 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10851 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10852 let before = reader.bytes_read();
10853 let mut n = 0usize;
10854 lazy.dump_filtered_each(None, s, p, o, |_, _, _| n += 1);
10855 assert!(!lazy.index_incomplete());
10856 (n, reader.bytes_read() - before)
10857 };
10858
10859 let (full_rows, full_bytes) = probe(None, None, None);
10860 let (pred_rows, pred_bytes) = probe(None, Some("<http://ex/p/07>"), None);
10861 let (subj_rows, subj_bytes) = probe(Some("<http://ex/s/0021>"), None, None);
10862 assert_eq!(full_rows, 4000);
10863 assert_eq!(pred_rows, 200, "one predicate covers every subject once");
10864 assert_eq!(subj_rows, 20);
10865 assert!(
10867 pred_bytes < full_bytes && subj_bytes < full_bytes,
10868 "a slice fetched {pred_bytes} / {subj_bytes} B where the whole graph fetched \
10869 {full_bytes} B — the scan is not being pruned at all"
10870 );
10871 #[cfg(feature = "compression")]
10882 assert!(
10883 pred_bytes * 4 < full_bytes && subj_bytes * 4 < full_bytes,
10884 "a 1-in-20 predicate slice fetched {pred_bytes} B and one subject \
10885 {subj_bytes} B where the whole graph fetched {full_bytes} B"
10886 );
10887
10888 let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10890 let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10891 let before = reader.bytes_read();
10892 let plan = lazy
10893 .dump_plan(None, None, Some("<http://ex/p/07>"), None)
10894 .scan
10895 .expect("a known predicate plans a scan");
10896 assert!(
10897 reader.bytes_read() - before < 64 * 1024,
10898 "planning fetched {} B — it is supposed to read directories, not tiles",
10899 reader.bytes_read() - before
10900 );
10901 assert!(plan.tiles_admitted < plan.tiles_total);
10902 assert!(
10908 plan.tile_bytes * 5 < plan.section_bytes,
10909 "one predicate of twenty planned {} B of a {} B section",
10910 plan.tile_bytes,
10911 plan.section_bytes
10912 );
10913 }
10914
10915 #[test]
10916 fn query_quads_tags_every_graph() {
10917 let mut db = DictionaryBuilder::new();
10918 for (s, p, o) in [("Alice", "knows", "Bob"), ("Alice", "knows", "Dave")] {
10919 db.observe(s, p, o);
10920 }
10921 let dict = db.build();
10922 let mut def = GraphIndexBuilder::new();
10923 def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10924 let mut g1 = GraphIndexBuilder::new();
10925 g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10926 let named = vec![("http://ex/g1".to_string(), g1.build())];
10927 let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10928 let rete = Rete::open(&bytes).unwrap();
10929
10930 let quads = rete.query_quads(Some("Alice"), Some("knows"), None);
10932 assert_eq!(quads.len(), 2);
10933 assert_eq!(
10934 quads[0],
10935 (("Alice".into(), "knows".into(), "Bob".into()), None)
10936 );
10937 assert_eq!(
10938 quads[1],
10939 (
10940 ("Alice".into(), "knows".into(), "Dave".into()),
10941 Some("http://ex/g1".to_string())
10942 )
10943 );
10944
10945 assert!(rete.query_quads(Some("Nobody"), None, None).is_empty());
10947 }
10948
10949 #[test]
10950 fn pyramid_meta_round_trips_in_file() {
10951 let rete = Rete::open(&build_image()).unwrap();
10952 let pyr = rete.pyramid().expect("file has a pyramid");
10953 let total: u32 = pyr.summary.iter().map(|e| e.count).sum();
10955 assert_eq!(total, 3);
10956 assert!(!pyr.summary.is_empty());
10957 assert!(pyr.tiles.is_empty());
10958 }
10959
10960 #[test]
10961 fn schema_pyramid_round_trips_through_file_index_free() {
10962 use crate::reader::{CountingReader, SliceReader};
10963 let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
10964 let q = |s: &str, p: &str, o: &str| {
10965 (s.to_string(), p.to_string(), o.to_string(), None::<String>)
10966 };
10967 let quads = vec![
10969 q("<a>", RDF_TYPE, "<Astronomer>"),
10970 q("<b>", RDF_TYPE, "<Astronomer>"),
10971 q("<c>", RDF_TYPE, "<Person>"),
10972 q("<Astronomer>", sub, "<Scientist>"),
10973 q("<Scientist>", sub, "<Person>"),
10974 q("<Person>", sub, "<Agent>"),
10975 q("<a>", "<knows>", "<b>"),
10976 q("<b>", "<knows>", "<c>"),
10977 ];
10978 let (bytes, _) =
10979 crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
10980
10981 let rete = Rete::open(&bytes).unwrap();
10983 let pyr = rete.pyramid().expect("pyramid present");
10984 assert!(!pyr.level_rollups.is_empty(), "schema pyramid shipped");
10985 assert!(pyr
10986 .class_hierarchy
10987 .iter()
10988 .any(|n| n.class == "<Agent>" && n.depth == 0));
10989
10990 let r = CountingReader::new(SliceReader::new(&bytes));
10992 let view = SummaryView::open_ranged(&r).unwrap().unwrap();
10993 assert!(view.level_count() >= 2, "multi-level pyramid");
10994 let coarse = view.level_rollup(0).unwrap();
10995 assert!(
10996 coarse.classes.iter().any(|(c, _)| c == "<Agent>"),
10997 "coarsest level rolls up to the root Agent"
10998 );
10999 let h = Header::from_bytes(&bytes).unwrap();
11000 assert!(
11001 r.bytes_read() <= bytes.len() as u64 - h.root_dir_len,
11002 "summary read {} bytes; the {}-byte index section must be skipped",
11003 r.bytes_read(),
11004 h.root_dir_len
11005 );
11006 }
11007
11008 #[test]
11009 fn predicate_totals_from_summary_only() {
11010 use crate::reader::SliceReader;
11011 let bytes = build_image();
11013 let reader = SliceReader::new(&bytes);
11014 let view = SummaryView::open_ranged(&reader).unwrap().unwrap();
11015 assert_eq!(view.predicate_total("knows"), 2);
11016 assert_eq!(view.predicate_total("age"), 1);
11017 assert_eq!(view.predicate_total("missing"), 0);
11018 let totals = view.predicate_totals();
11019 assert_eq!(totals[0], ("knows".to_string(), 2)); }
11021
11022 #[test]
11023 fn query_patterns_resolve_to_terms() {
11024 let rete = Rete::open(&build_image()).unwrap();
11025
11026 assert_eq!(rete.query(None, None, None).len(), 3);
11028
11029 let mut alice = rete.query(Some("Alice"), None, None);
11031 alice.sort();
11032 assert_eq!(
11033 alice,
11034 vec![
11035 ("Alice".into(), "age".into(), "30".into()),
11036 ("Alice".into(), "knows".into(), "Bob".into()),
11037 ]
11038 );
11039
11040 assert_eq!(rete.query(None, Some("knows"), None).len(), 2);
11042
11043 assert_eq!(
11045 rete.query(Some("Bob"), Some("knows"), Some("Carol")),
11046 vec![("Bob".into(), "knows".into(), "Carol".into())]
11047 );
11048 assert!(rete.query(Some("Nobody"), None, None).is_empty());
11049 assert!(rete.query(None, Some("likes"), None).is_empty());
11050 }
11051
11052 #[test]
11053 fn query_provenance_reports_terms_ids_sections_and_index_choice() {
11054 let bytes = build_image();
11055 let rete = Rete::open(&bytes).unwrap();
11056
11057 let mut matches = rete.query_with_provenance(None, Some("knows"), None);
11058 matches.sort_by(|a, b| a.terms.cmp(&b.terms));
11059
11060 assert_eq!(matches.len(), 2);
11061 assert_eq!(
11062 matches[0].terms,
11063 ("Alice".into(), "knows".into(), "Bob".into())
11064 );
11065 assert_eq!(
11066 matches[0].ids,
11067 rete.dictionary().encode("Alice", "knows", "Bob").unwrap()
11068 );
11069 assert_eq!(matches[0].graph.as_deref(), None);
11070 assert_eq!(
11071 matches[0].matched_pattern,
11072 (None, Some(matches[0].ids.1), None)
11073 );
11074 assert_eq!(
11075 matches[0].index_permutation,
11076 crate::index::IndexPermutation::Pos
11077 );
11078
11079 let h = rete.header();
11080 assert_eq!(matches[0].dictionary_range.offset, h.dictionary_offset);
11081 assert_eq!(matches[0].dictionary_range.len, h.dictionary_len);
11082 assert_eq!(matches[0].index_range.offset, h.root_dir_offset);
11083 assert_eq!(matches[0].index_range.len, h.root_dir_len);
11084 assert!(
11085 matches[0].index_section_range.offset > h.root_dir_offset,
11086 "POS is section 1, so its payload starts after the container header and SPO payload"
11087 );
11088 assert!(matches[0].index_section_range.len > 0);
11089 assert!(matches[0].index_section_range.end() <= matches[0].index_range.end());
11090 assert!(matches[0].index_section_range.len < matches[0].index_range.len);
11091 assert_eq!(
11092 matches[0].pyramid_range.as_ref().map(|r| (r.offset, r.len)),
11093 Some((h.pyramid_meta_offset, h.pyramid_meta_len))
11094 );
11095 let tile_range = matches[0].tile_range.expect("tiled file reports a tile");
11098 assert!(matches[0]
11099 .tile
11100 .as_deref()
11101 .unwrap()
11102 .starts_with(matches[0].index_permutation.name()));
11103 assert!(matches[0].index_section_range.offset <= tile_range.offset);
11104 assert!(tile_range.end() <= matches[0].index_section_range.end());
11105 }
11106
11107 fn build_labeled(n: usize) -> Vec<u8> {
11111 const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
11112 const WORDS: &[&str] = &[
11113 "alanine",
11114 "benzene",
11115 "glucose",
11116 "dextrose",
11117 "ethanol",
11118 "formate",
11119 "heptane",
11120 "isoleucine",
11121 ];
11122 let triples: Vec<(String, String, String)> = (0..n)
11123 .flat_map(|i| {
11124 let s = format!("<http://ex/e{i}>");
11125 let w = WORDS[i % WORDS.len()];
11126 [
11127 (s.clone(), LABEL.to_string(), format!("\"{w}-{i:06}\"")),
11128 (
11129 s,
11130 "<http://ex/p>".to_string(),
11131 format!("<http://ex/c{}>", i % 64),
11132 ),
11133 ]
11134 })
11135 .collect();
11136 let mut db = DictionaryBuilder::new();
11137 for (s, p, o) in &triples {
11138 db.observe(s, p, o);
11139 }
11140 let dict = db.build();
11141 let ids: Vec<(u32, u32, u32)> = triples
11142 .iter()
11143 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11144 .collect();
11145 let mut ib = GraphIndexBuilder::new();
11146 for &t in &ids {
11147 ib.push(t);
11148 }
11149 let (meta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
11150 write_file(&dict, &ib.build(), false, &meta, levels)
11151 }
11152
11153 #[test]
11154 fn prefix_search_matches_a_filter_scan() {
11155 let bytes = build_labeled(800);
11158 let rete = Rete::open(&bytes).unwrap();
11159 let idx_subjects: std::collections::BTreeSet<String> = rete
11160 .prefix_search("glucose", 10_000)
11161 .into_iter()
11162 .map(|(_label, subject)| subject)
11163 .collect();
11164 let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11166 FILTER(STRSTARTS(LCASE(?l), \"glucose\")) }";
11167 let crate::QueryOutput::Select(_, rows) = crate::eval_query(&rete, q).unwrap() else {
11168 panic!("expected SELECT");
11169 };
11170 let scan_subjects: std::collections::BTreeSet<String> =
11171 rows.iter().map(|r| r.get("s").cloned().unwrap()).collect();
11172 assert_eq!(idx_subjects, scan_subjects, "index agrees with the scan");
11173 assert_eq!(
11174 idx_subjects.len(),
11175 100,
11176 "800/8 words = 100 glucose-* labels"
11177 );
11178 }
11179
11180 #[test]
11184 fn legacy_label_hits_resolve_in_one_batched_read() {
11185 use std::sync::{Arc, Mutex};
11186 type CallLog = Arc<Mutex<Vec<Vec<(u64, u64)>>>>;
11187 #[derive(Clone)]
11188 struct Calls(Arc<Vec<u8>>, CallLog);
11189 impl crate::RangeReader for Calls {
11190 fn len(&self) -> u64 {
11191 self.0.len() as u64
11192 }
11193 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
11194 self.1.lock().unwrap().push(vec![(offset, len)]);
11195 crate::SliceReader::new(&self.0).read_at(offset, len)
11196 }
11197 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
11198 self.1.lock().unwrap().push(ranges.to_vec());
11199 let r = crate::SliceReader::new(&self.0);
11200 ranges.iter().map(|&(o, l)| r.read_at(o, l)).collect()
11201 }
11202 }
11203 let bytes = build_labeled(20_000);
11204 let h = Header::from_bytes(&bytes).unwrap();
11205 let (d0, dl) = (h.dictionary_offset, h.dictionary_len);
11206 let want = Rete::open(&bytes).unwrap().prefix_search("glucose", 1000);
11207 assert_eq!(want.len(), 1000);
11208 for search_view in [false, true] {
11209 let calls = Calls(Arc::new(bytes.clone()), Arc::new(Mutex::new(Vec::new())));
11210 let got = if search_view {
11211 let view = SearchView::open_ranged_with(calls.clone(), HEADER_LEN as u64).unwrap();
11212 calls.1.lock().unwrap().clear();
11213 view.prefix_search("glucose", 1000)
11214 } else {
11215 let rete = Rete::open_ranged_lazy_with(calls.clone(), HEADER_LEN as u64).unwrap();
11216 calls.1.lock().unwrap().clear();
11217 rete.prefix_search("glucose", 1000)
11218 };
11219 assert_eq!(got, want, "search view = {search_view}");
11220 let dict_reads = calls
11221 .1
11222 .lock()
11223 .unwrap()
11224 .iter()
11225 .filter(|c| c.iter().any(|&(o, l)| o < d0 + dl && d0 < o + l))
11226 .count();
11227 assert_eq!(dict_reads, 1, "search view = {search_view}");
11228 }
11229 }
11230
11231 #[test]
11235 #[ignore]
11236 fn bench_prefix_search_vs_filter_scan() {
11237 use std::time::Instant;
11238 let n = 6000; let bytes = build_labeled(n);
11240 let rete = Rete::open(&bytes).unwrap();
11241 let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11242 FILTER(STRSTARTS(LCASE(?l), \"glucose\")) }";
11243 let reps = 200;
11244 let idx_n = rete.prefix_search("glucose", 100_000).len();
11245 let t = Instant::now();
11246 for _ in 0..reps {
11247 std::hint::black_box(rete.prefix_search("glucose", 100_000));
11248 }
11249 let idx_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11250 let t = Instant::now();
11251 for _ in 0..reps {
11252 let _ = std::hint::black_box(crate::eval_query(&rete, q).unwrap());
11253 }
11254 let scan_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11255 println!(
11256 "label prefix search over {n} labeled subjects ({idx_n} matches): \
11257 index {idx_ms:.4} ms vs FILTER scan {scan_ms:.3} ms ({:.0}× faster)",
11258 scan_ms / idx_ms
11259 );
11260 }
11261
11262 #[test]
11266 #[ignore]
11267 fn bench_text_search_vs_contains_scan() {
11268 use std::time::Instant;
11269 const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
11270 const WORDS: &[&str] = &[
11271 "alanine",
11272 "benzene",
11273 "glucose",
11274 "dextrose",
11275 "ethanol",
11276 "formate",
11277 "heptane",
11278 "isoleucine",
11279 ];
11280 let n = 6000;
11281 let triples: Vec<(String, String, String)> = (0..n)
11282 .map(|i| {
11283 (
11284 format!("<http://ex/e{i}>"),
11285 LABEL.to_string(),
11286 format!("\"{} sample number {i:06}\"", WORDS[i % WORDS.len()]),
11287 )
11288 })
11289 .collect();
11290 let mut db = DictionaryBuilder::new();
11291 for (s, p, o) in &triples {
11292 db.observe(s, p, o);
11293 }
11294 let dict = db.build();
11295 let ids: Vec<(u32, u32, u32)> = triples
11296 .iter()
11297 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11298 .collect();
11299 let mut ib = GraphIndexBuilder::new();
11300 for &t in &ids {
11301 ib.push(t);
11302 }
11303 let ti = compute_text_index(&dict, &ids);
11304 let bytes = write_dataset_with_metadata(&dict, &ib.build(), &[], false, &[], 0, &[], &ti);
11305 let rete = Rete::open(&bytes).unwrap();
11306
11307 let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11308 FILTER(CONTAINS(LCASE(?l), \"glucose\")) }";
11309 let reps = 200;
11310 let idx_n = rete.text_search(&["glucose"], None, 100_000).len();
11311 let t = Instant::now();
11312 for _ in 0..reps {
11313 std::hint::black_box(rete.text_search(&["glucose"], None, 100_000));
11314 }
11315 let idx_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11316 let t = Instant::now();
11317 for _ in 0..reps {
11318 let _ = std::hint::black_box(crate::eval_query(&rete, q).unwrap());
11319 }
11320 let scan_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11321 println!(
11322 "text search over {n} literals ({idx_n} matches): \
11323 index {idx_ms:.4} ms vs FILTER(CONTAINS) scan {scan_ms:.3} ms ({:.0}× faster)",
11324 scan_ms / idx_ms
11325 );
11326 }
11327
11328 #[test]
11332 #[ignore = "operational tool, driven by RETE_DEBUG_* env vars"]
11333 fn debug_bound_po_routing() {
11334 struct FR(std::fs::File);
11335 impl crate::RangeReader for FR {
11336 fn len(&self) -> u64 {
11337 self.0.metadata().map(|m| m.len()).unwrap_or(0)
11338 }
11339 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
11340 use std::os::unix::fs::FileExt;
11341 let mut buf = vec![0u8; len as usize];
11342 self.0.read_exact_at(&mut buf, offset)?;
11343 Ok(buf)
11344 }
11345 }
11346 let path = std::env::var("RETE_DEBUG_FILE").expect("RETE_DEBUG_FILE");
11347 let p_iri = std::env::var("RETE_DEBUG_P").expect("RETE_DEBUG_P");
11348 let o_iri = std::env::var("RETE_DEBUG_O").expect("RETE_DEBUG_O");
11349 let rete = Rete::open_ranged_lazy_with(
11350 std::sync::Arc::new(FR(std::fs::File::open(&path).unwrap())),
11351 HEADER_LEN as u64,
11352 )
11353 .unwrap();
11354 let pid = rete.dict.predicate_id(&p_iri).expect("p resolves");
11355 let oid = rete.dict.object_id(&o_iri).expect("o resolves");
11356 eprintln!("pid={pid} oid={oid}");
11357 let pattern = (None, Some(pid), Some(oid));
11358 let perm = GraphIndex::best_permutation(pattern);
11359 eprintln!("best_permutation = {}", perm.name());
11360 let si = perm.section_index();
11361 let tiles = &rete.index.sections[si];
11362 eprintln!("section {} tiles = {}", perm.name(), tiles.len());
11363 let [pa, pb, pc] = perm.order_pattern(pattern);
11364 eprintln!("permuted pattern pa={pa:?} pb={pb:?} pc={pc:?}");
11365 let (start, end) = rete.index.tile_span(si, pa);
11366 eprintln!("tile_span = [{start}, {end}) -> {} tiles", end - start);
11367 let mut admitted = 0usize;
11368 for (ti, t) in tiles.iter().enumerate().take(end).skip(start) {
11369 if t.syn_admits(pb, pc) {
11370 admitted += 1;
11371 if admitted <= 10 {
11372 let (lo, hi) = t.leading_range();
11373 eprintln!(" admit tile {ti}: a=[{lo},{hi}] syn={:?}", t.syn);
11374 }
11375 }
11376 }
11377 eprintln!("admitted {admitted} tile(s) by synopsis");
11378 let n = rete.index.scan_iter(pattern).count();
11379 eprintln!("scan_iter matches = {n}");
11380 let hi_res = rete.query(None, Some(&p_iri), Some(&o_iri));
11381 eprintln!("high-level query matches = {}", hi_res.len());
11382 }
11383
11384 #[test]
11392 fn layout_header_label_matches_its_span() {
11393 let mut db = DictionaryBuilder::new();
11394 let mut triples = Vec::new();
11395 for i in 0..24u32 {
11396 let (s, p, o) = (
11397 format!("<http://ex/s{}>", i % 5),
11398 format!("<http://ex/p{}>", i % 3),
11399 format!("<http://ex/o{i}>"),
11400 );
11401 db.observe(&s, &p, &o);
11402 triples.push((s, p, o));
11403 }
11404 let dict = db.build();
11405 let ids: Vec<(u32, u32, u32)> = triples
11406 .iter()
11407 .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11408 .collect();
11409 let index = GraphIndexBuilder::from_triples(ids).build();
11410 let image = write_dataset_with_metadata(
11411 &dict,
11412 &index,
11413 &[],
11414 false,
11415 &[],
11416 0,
11417 br#"{"name":"layout-label-fixture"}"#,
11418 &[],
11419 );
11420 let rete = Rete::open(&image).unwrap();
11421
11422 let layout = rete.file_layout();
11423 let header = layout
11424 .iter()
11425 .find(|s| s.kind == "header")
11426 .expect("every layout starts with a header segment");
11427
11428 assert_eq!(
11429 header.offset, 0,
11430 "the header is the first thing in the file"
11431 );
11432 assert_eq!(
11433 header.len, HEADER_LEN as u64,
11434 "the header segment must span exactly HEADER_LEN"
11435 );
11436
11437 let digits: String = header
11440 .label
11441 .chars()
11442 .skip_while(|c| !c.is_ascii_digit())
11443 .take_while(|c| c.is_ascii_digit())
11444 .collect();
11445 assert!(
11446 !digits.is_empty(),
11447 "header label {:?} states no size at all",
11448 header.label
11449 );
11450 assert_eq!(
11451 digits.parse::<u64>().unwrap(),
11452 header.len,
11453 "header label {:?} disagrees with the {}-byte span it describes",
11454 header.label,
11455 header.len
11456 );
11457
11458 assert!(
11461 layout.windows(2).all(|w| w[0].offset <= w[1].offset),
11462 "file_layout must return segments sorted by offset"
11463 );
11464 }
11465}