Skip to main content

rete_core/
file.rs

1//! `.rete` file assembly and reading (SPEC.md §4, §9).
2//!
3//! v0 layout:
4//!
5//! ```text
6//! [0..1024)  header ([`crate::header::HEADER_LEN`])
7//! [dict]     dictionary container: 4 front-coded sections
8//! [index]    permutation container: 6 triple blocks (SPO/POS/OSP/SOP/PSO/OPS)
9//! [pyramid]  summary meta (and, in future, tile directories)
10//! [footer]   trailing magic
11//! ```
12//!
13//! The header points at the dictionary container (`dictionary_offset/len`) and
14//! the permutation container (`root_dir_offset/len`); routed readers can fetch a
15//! single permutation payload from that container.
16
17use crate::dictionary::Dictionary;
18use crate::header::{
19    Header, Section, SectionKind, FLAG_HAS_QUADS, FLAG_HAS_QUOTED_TRIPLES, FLAG_TILE_SYNOPSIS,
20    HEADER_LEN, MAGIC,
21};
22use crate::index::{GraphIndex, IndexPermutation, Pattern, PermSet, NUM_PERMS};
23use crate::meta::{ClassNode, CommunityDescriptor, LevelLinks, LevelRollup, PyramidMeta};
24use crate::pyramid::{build_dendrogram, project_graph, PyramidAlgo};
25use crate::reader::RangeReader;
26use crate::tiling::{choose_round_for_budget, summarize, SuperEdge};
27use crate::triples::Triple;
28use crate::varint::{read_uvarint, write_uvarint};
29
30#[cfg(test)]
31mod hardening_tests;
32
33/// Default per-tile byte budget `T` (SPEC.md §7.1).
34pub const DEFAULT_TILE_BUDGET: usize = 64 * 1024;
35
36/// Build the encoded pyramid-meta section for a graph: cluster, pick a round
37/// sized to `budget`, then emit the **summary** (quotient) graph. Returns
38/// `(encoded_meta, pyramid_levels)`.
39///
40/// Per-community tiles are *not* stored: they would duplicate every triple, and
41/// the exact ranged single-pattern path now routes into one permutation section
42/// without that fourth copy. Physical community-tile directories are the next
43/// storage step (SPEC §7.2).
44pub fn build_pyramid_meta(
45    dict: &Dictionary,
46    triples: &[(u32, u32, u32)],
47    budget: usize,
48) -> (Vec<u8>, u16) {
49    build_pyramid_meta_with(dict, triples, budget, None)
50}
51
52/// Like [`build_pyramid_meta`], but `type_override` forces the schema-pyramid's
53/// type predicate (e.g. `wdt:P31`) instead of auto-detection. Uses the default
54/// [`PyramidAlgo::Louvain`] community algorithm — byte-identical to before.
55pub fn build_pyramid_meta_with(
56    dict: &Dictionary,
57    triples: &[(u32, u32, u32)],
58    budget: usize,
59    type_override: Option<&str>,
60) -> (Vec<u8>, u16) {
61    build_pyramid_meta_algo(dict, triples, budget, type_override, PyramidAlgo::Louvain)
62}
63
64/// Like [`build_pyramid_meta_with`], but selects the community [`PyramidAlgo`].
65/// [`PyramidAlgo::Types`] partitions by `rdf:type` — the deterministic,
66/// parallelizable alternative to Louvain (one linear pass, no modularity) that
67/// still emits the full summary + `query_stats`; it falls back to Louvain when the
68/// graph has no usable typing. Everything downstream of the dendrogram (round
69/// choice, summary, schema pyramid, planner stats) is shared across algorithms.
70pub fn build_pyramid_meta_algo(
71    dict: &Dictionary,
72    triples: &[(u32, u32, u32)],
73    budget: usize,
74    type_override: Option<&str>,
75    algo: PyramidAlgo,
76) -> (Vec<u8>, u16) {
77    build_pyramid_meta_layout(
78        dict,
79        triples,
80        budget,
81        type_override,
82        algo,
83        PyramidLayout::Combined,
84    )
85}
86
87/// What a pyramid-meta section carries besides the community summary.
88#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
89pub enum PyramidLayout {
90    /// Everything, as every build up to 0.3.x wrote it: the summary, the
91    /// per-community descriptors, the schema pyramid, `query_stats`, the
92    /// characteristic sets and the bounded (8,192-entry) label block.
93    #[default]
94    Combined,
95    /// Only what needs the community partition: the summary and the
96    /// per-community descriptors. The schema, `query_stats` and the
97    /// characteristic sets live in the SCHEMA section (kind 9) and the labels
98    /// in the LABEL_INDEX section (kind 8) — what `rete build` writes.
99    CommunityOnly,
100}
101
102/// Like [`build_pyramid_meta_algo`], choosing the [`PyramidLayout`].
103pub fn build_pyramid_meta_layout(
104    dict: &Dictionary,
105    triples: &[(u32, u32, u32)],
106    budget: usize,
107    type_override: Option<&str>,
108    algo: PyramidAlgo,
109    layout: PyramidLayout,
110) -> (Vec<u8>, u16) {
111    // Optional sub-phase timing (set RETE_BUILD_TIMING=1) — the pyramid build is
112    // the dominant cost of a big `rete build`; this shows where inside it.
113    // `Instant::now()` must stay behind the flag: `std::time` is unsupported on
114    // `wasm32-unknown-unknown` and panics ("time not implemented"), so an
115    // unconditional clock read would break every in-browser `build()`.
116    let timing = std::env::var_os("RETE_BUILD_TIMING").is_some();
117    let mut t = timing.then(std::time::Instant::now);
118    let mut lap = |label: &str| {
119        if let Some(t0) = &mut t {
120            eprintln!(
121                "  [pyramid] {label}: {:.0} ms",
122                t0.elapsed().as_secs_f64() * 1000.0
123            );
124            *t0 = std::time::Instant::now();
125        }
126    };
127
128    // The community partition — the only step that differs by algorithm.
129    let louvain = |lap: &mut dyn FnMut(&str)| {
130        let g = project_graph(dict, triples);
131        lap("project_graph");
132        let d = build_dendrogram(&g);
133        lap("build_dendrogram (Louvain)");
134        d
135    };
136    let dend = match algo {
137        PyramidAlgo::Louvain => louvain(&mut lap),
138        PyramidAlgo::Types => {
139            match crate::schema_pyramid::build_type_dendrogram(dict, triples, type_override) {
140                Some(d) => {
141                    lap("build_type_dendrogram");
142                    d
143                }
144                None => {
145                    eprintln!(
146                        "  [pyramid] --pyramid-algo types: no usable rdf:type \
147                         predicate — falling back to louvain"
148                    );
149                    louvain(&mut lap)
150                }
151            }
152        }
153    };
154    let round = choose_round_for_budget(dict, triples, &dend, budget);
155    lap("choose_round_for_budget");
156    let summary = summarize(dict, triples, &dend, round);
157    lap("summarize");
158    // Attach the v2 schema pyramid (the non-exclusive subClassOf DAG + per-level
159    // type rollups + per-level lateral class relations + per-community
160    // descriptors). Empty when the graph has no usable typing, in which case the
161    // encoding stays byte-identical to a v1 pyramid-meta.
162    if layout == PyramidLayout::CommunityOnly {
163        // Descriptors only: the schema pyramid's other fields, the planner
164        // stats, the shapes and the labels have sections of their own.
165        let descriptors = crate::schema_pyramid::build_community_descriptors(
166            dict,
167            triples,
168            &dend,
169            round,
170            type_override,
171        );
172        lap("build_community_descriptors");
173        let meta = PyramidMeta::new(round as u32, summary, &[]).with_schema(
174            Vec::new(),
175            Vec::new(),
176            Vec::new(),
177            descriptors,
178            Vec::new(),
179            Vec::new(),
180            Vec::new(),
181        );
182        let out = (meta.encode(), dend.rounds() as u16);
183        lap("encode");
184        return out;
185    }
186    let sp = crate::schema_pyramid::build_schema_pyramid_with(
187        dict,
188        triples,
189        &dend,
190        round,
191        type_override,
192    );
193    lap("build_schema_pyramid");
194    let predicate_stats = compute_predicate_stats(triples);
195    lap("compute_predicate_stats");
196    let char_sets = compute_char_sets(triples);
197    lap("compute_char_sets");
198    let label_index = compute_label_index(dict, triples);
199    lap("compute_label_index");
200    let meta = PyramidMeta::new(round as u32, summary, &[])
201        .with_schema(
202            sp.class_hierarchy,
203            sp.level_rollups,
204            sp.level_links,
205            sp.descriptors,
206            sp.subclass_cycles,
207            sp.disjoint_pairs,
208            sp.equivalent_pairs,
209        )
210        .with_predicate_stats(predicate_stats)
211        .with_char_sets(char_sets)
212        .with_label_index(label_index);
213    let out = (meta.encode(), dend.rounds() as u16);
214    lap("encode");
215    out
216}
217
218/// The label predicates a [`compute_label_index`] entry can come from — the
219/// common "human-readable name of this subject" terms, angle-bracketed as the
220/// dictionary stores them. Order is irrelevant (we union their ids). The same
221/// list the LABEL_INDEX section's rule L1 uses.
222const LABEL_PREDICATES: &[&str] = crate::label_index::DIRECT_LABEL_PREDICATES;
223
224/// Build the bounded, **legacy** label block of a [`PyramidLayout::Combined`]
225/// pyramid-meta (the LABEL_INDEX section, kind 8, replaces it): the display labels of
226/// the most-connected labeled subjects, sorted by the label's lowercased form.
227/// Ranking keeps autocomplete useful on a huge graph (the prominent entities
228/// survive the bound); a graph with fewer than `MAX_LABELS` labels keeps them
229/// all. Deterministic (degree, then subject id, then label) so builds are
230/// reproducible. O(triples) transient memory, freed before the file is written.
231fn compute_label_index(
232    dict: &Dictionary,
233    triples: &[(u32, u32, u32)],
234) -> Vec<crate::meta::LabelEntry> {
235    use crate::terms::{is_literal, literal_lexical};
236    use std::collections::{HashMap, HashSet};
237    const MAX_LABELS: usize = 8192;
238
239    // Resolve the label predicates that actually occur in this graph.
240    let label_pids: HashSet<u32> = LABEL_PREDICATES
241        .iter()
242        .filter_map(|p| dict.predicate_id(p))
243        .collect();
244    if label_pids.is_empty() {
245        return Vec::new();
246    }
247    // Subject degree (triple count) — the ranking used to bound the index.
248    let mut degree: HashMap<u32, u32> = HashMap::new();
249    for &(s, _p, _o) in triples {
250        *degree.entry(s).or_insert(0) += 1;
251    }
252    // Candidate (subject, label) pairs, deduped on (subject, lowercased label).
253    let mut seen: HashSet<(u32, String)> = HashSet::new();
254    let mut candidates: Vec<(u32, String, u32)> = Vec::new(); // (degree, label, subject)
255    for &(s, p, o) in triples {
256        if !label_pids.contains(&p) {
257            continue;
258        }
259        let Some(term) = dict.object_term(o) else {
260            continue;
261        };
262        if !is_literal(&term) {
263            continue;
264        }
265        let Some(label) = literal_lexical(&term) else {
266            continue;
267        };
268        if label.is_empty() {
269            continue;
270        }
271        if seen.insert((s, label.to_lowercase())) {
272            candidates.push((*degree.get(&s).unwrap_or(&0), label, s));
273        }
274    }
275    // Keep the most-connected entities when over budget: rank by degree desc,
276    // then subject asc, then label asc (deterministic).
277    if candidates.len() > MAX_LABELS {
278        candidates.sort_by(|a, b| {
279            b.0.cmp(&a.0)
280                .then_with(|| a.2.cmp(&b.2))
281                .then_with(|| a.1.cmp(&b.1))
282        });
283        candidates.truncate(MAX_LABELS);
284    }
285    // Final order: by lowercased label (search key), then label, then subject.
286    candidates.sort_by(|a, b| {
287        a.1.to_lowercase()
288            .cmp(&b.1.to_lowercase())
289            .then_with(|| a.1.cmp(&b.1))
290            .then_with(|| a.2.cmp(&b.2))
291    });
292    candidates
293        .into_iter()
294        .map(|(_deg, label, subject)| crate::meta::LabelEntry { label, subject })
295        .collect()
296}
297
298/// Build the default-scope full-text index section (`token → subjects`) over
299/// every literal object of `triples`: tokenize each literal into words and
300/// record the subject that carries it. Empty when the graph has no literals.
301/// This is the pre-0.4.0 index, byte for byte; the builds call
302/// `text_index::build_section` with their scope (`--text-index-graphs`,
303/// `--text-index-include`/`-exclude`). O(literal bytes) transient. The token
304/// table is compressed with `writer_codec` (the reader decompresses with
305/// `block_codec`).
306pub fn compute_text_index(dict: &Dictionary, triples: &[(u32, u32, u32)]) -> Vec<u8> {
307    crate::text_index::build_section(
308        dict,
309        &crate::text_index::TextIndexScope::default(),
310        triples,
311        std::iter::empty(),
312        writer_codec(),
313    )
314}
315
316/// The top entity **shapes** (characteristic sets): group subjects by the exact
317/// set of predicates they carry, keep the most common. Bounded to `MAX_CHAR_SETS`
318/// and sorted deterministically (by subject count, then predicate list) so the
319/// encoding is reproducible. O(triples) transient memory.
320fn compute_char_sets(triples: &[(u32, u32, u32)]) -> Vec<crate::meta::CharSet> {
321    use std::collections::{BTreeSet, HashMap};
322    const MAX_CHAR_SETS: usize = 128;
323    let mut by_subject: HashMap<u32, BTreeSet<u32>> = HashMap::new();
324    for &(s, p, _o) in triples {
325        by_subject.entry(s).or_default().insert(p);
326    }
327    let mut shapes: HashMap<Vec<u32>, u64> = HashMap::new();
328    for set in by_subject.into_values() {
329        *shapes.entry(set.into_iter().collect()).or_insert(0) += 1;
330    }
331    let mut v: Vec<crate::meta::CharSet> = shapes
332        .into_iter()
333        .map(|(predicates, subjects)| crate::meta::CharSet {
334            predicates,
335            subjects,
336        })
337        .collect();
338    v.sort_by(|a, b| {
339        b.subjects
340            .cmp(&a.subjects)
341            .then_with(|| a.predicates.cmp(&b.predicates))
342    });
343    v.truncate(MAX_CHAR_SETS);
344    v
345}
346
347/// Per-predicate cardinality for the cost-based planner, in one pass over the
348/// triples (deduped, so a per-(subject,predicate) count is its distinct-object
349/// count). Returned sorted by predicate id for a reproducible encoding. Holds a
350/// transient `(subject -> count, object -> count)` map per predicate — O(triples)
351/// memory, freed before the file is written.
352fn compute_predicate_stats(triples: &[(u32, u32, u32)]) -> Vec<crate::meta::PredStat> {
353    use std::collections::HashMap;
354    #[allow(clippy::type_complexity)]
355    let mut acc: HashMap<u32, (HashMap<u32, u32>, HashMap<u32, u32>, u64)> = HashMap::new();
356    for &(s, p, o) in triples {
357        let e = acc.entry(p).or_default();
358        *e.0.entry(s).or_insert(0) += 1;
359        *e.1.entry(o).or_insert(0) += 1;
360        e.2 += 1;
361    }
362    let mut stats: Vec<crate::meta::PredStat> = acc
363        .into_iter()
364        .map(|(predicate, (subj, obj, count))| crate::meta::PredStat {
365            predicate,
366            count,
367            distinct_subjects: subj.len() as u64,
368            distinct_objects: obj.len() as u64,
369            max_objects_per_subject: subj.values().copied().max().unwrap_or(0),
370            max_subjects_per_object: obj.values().copied().max().unwrap_or(0),
371        })
372        .collect();
373    stats.sort_by_key(|p| p.predicate);
374    stats
375}
376
377/// No compression.
378pub const CODEC_NONE: u8 = 0;
379/// zstd compression (per section).
380pub const CODEC_ZSTD: u8 = 1;
381
382#[derive(Debug, thiserror::Error)]
383#[non_exhaustive]
384pub enum FileError {
385    #[error("header: {0}")]
386    Header(#[from] crate::header::HeaderError),
387    #[error("malformed container: {0}")]
388    Container(&'static str),
389    #[error("unknown codec: {0}")]
390    UnknownCodec(u8),
391    #[error("decompression failed: {0}")]
392    Decompress(std::io::Error),
393    #[error("io: {0}")]
394    Io(#[from] std::io::Error),
395}
396
397/// The codec the writer uses: zstd when the `compression` feature is on, else
398/// none. Reading honors whatever codec the header records (when supported).
399pub fn writer_codec() -> u8 {
400    if cfg!(feature = "compression") {
401        CODEC_ZSTD
402    } else {
403        CODEC_NONE
404    }
405}
406
407/// Intersection of two ascending-sorted, deduped id lists — the AND of two
408/// posting lists in a multi-word text search. Linear merge, output sorted.
409fn intersect_sorted(a: &[u32], b: &[u32]) -> Vec<u32> {
410    let mut out = Vec::with_capacity(a.len().min(b.len()));
411    let (mut i, mut j) = (0, 0);
412    while i < a.len() && j < b.len() {
413        match a[i].cmp(&b[j]) {
414            std::cmp::Ordering::Less => i += 1,
415            std::cmp::Ordering::Greater => j += 1,
416            std::cmp::Ordering::Equal => {
417                out.push(a[i]);
418                i += 1;
419                j += 1;
420            }
421        }
422    }
423    out
424}
425
426pub(crate) fn compress(codec: u8, bytes: &[u8]) -> Vec<u8> {
427    match codec {
428        // Level and frame shape are writer knobs (`crate::write_opts`); any
429        // combination decodes with every conformant zstd decoder.
430        #[cfg(feature = "compression")]
431        CODEC_ZSTD => {
432            let level = crate::write_opts::zstd_level();
433            match crate::write_opts::frame_shape() {
434                crate::write_opts::FrameShape::Stream => zstd::encode_all(bytes, level),
435                crate::write_opts::FrameShape::Sized => zstd::bulk::compress(bytes, level),
436            }
437            .expect("zstd encode is infallible in-memory")
438        }
439        _ => bytes.to_vec(),
440    }
441}
442
443/// The streaming twin of [`compress`]: the same codec, level and frame shape,
444/// read from `src` (exactly `src_len` bytes) and written to `dst`, for a
445/// payload the memory-bounded build keeps on disk (the TEXT_INDEX token
446/// table). A streaming frame is what `zstd::encode_all` writes, through the
447/// same `Encoder`. A sized frame is told the source size up front
448/// (`set_pledged_src_size`), so it picks the same small-input parameters and
449/// records the same content size as `zstd::bulk::compress`, without holding
450/// the payload. The encoder cuts blocks from its own input buffer, not from
451/// how the input was handed to it, so how `src` hands out its bytes never
452/// changes the output.
453///
454/// It equals [`compress`]'s one-shot output only while the payload fits the
455/// level's window (4 MiB at level 9): past it the streaming encoder's ring
456/// buffer wraps and it finds slightly different matches. A payload that one
457/// writer streams and another holds whole must therefore go through
458/// [`compress_streamed`] on the in-memory side — the token table does.
459/// `text_index_spill` pins both facts.
460#[cfg(not(target_arch = "wasm32"))]
461pub(crate) fn compress_to(
462    codec: u8,
463    src_len: u64,
464    src: &mut dyn std::io::Read,
465    dst: &mut dyn std::io::Write,
466) -> std::io::Result<()> {
467    match codec {
468        #[cfg(feature = "compression")]
469        CODEC_ZSTD => zstd_stream(src_len, src, dst),
470        _ => {
471            let _ = src_len;
472            std::io::copy(src, dst).map(|_| ())
473        }
474    }
475}
476
477/// [`compress`] through the streaming encoder: byte for byte what
478/// [`compress_to`] writes for the same payload, whatever its size. For the
479/// in-memory writer of a payload the memory-bounded build compresses from
480/// disk (the TEXT_INDEX token table), so the two builds write the same bytes
481/// past the zstd window too. Below it this is [`compress`]'s output.
482pub(crate) fn compress_streamed(codec: u8, bytes: &[u8]) -> Vec<u8> {
483    match codec {
484        #[cfg(feature = "compression")]
485        CODEC_ZSTD => {
486            let mut out = Vec::with_capacity(bytes.len() / 4 + 64);
487            zstd_stream(bytes.len() as u64, &mut &bytes[..], &mut out)
488                .expect("zstd encode is infallible in-memory");
489            out
490        }
491        _ => bytes.to_vec(),
492    }
493}
494
495/// The streaming zstd encoder at the writer's level and frame shape.
496#[cfg(feature = "compression")]
497fn zstd_stream(
498    src_len: u64,
499    src: &mut dyn std::io::Read,
500    dst: &mut dyn std::io::Write,
501) -> std::io::Result<()> {
502    let mut enc = zstd::stream::Encoder::new(dst, crate::write_opts::zstd_level())?;
503    if crate::write_opts::frame_shape() == crate::write_opts::FrameShape::Sized {
504        enc.set_pledged_src_size(Some(src_len))?;
505        enc.include_contentsize(true)?;
506    }
507    let copied = std::io::copy(src, &mut enc)?;
508    if copied != src_len {
509        return Err(std::io::Error::other(format!(
510            "zstd stream: {copied} bytes read, {src_len} announced"
511        )));
512    }
513    enc.finish()?;
514    Ok(())
515}
516
517pub(crate) fn decompress(codec: u8, bytes: &[u8]) -> Result<Vec<u8>, FileError> {
518    match codec {
519        CODEC_NONE => Ok(bytes.to_vec()),
520        // Native (the `compression` feature — always on for the CLI, off for
521        // wasm) decodes through the C `zstd` library, the same one
522        // [`compress`] encodes with. It is roughly 2× the pure-Rust decoder
523        // and speeds every read path, not just export; the multi-pass tight
524        // budget of a bounded export (`--memory-budget-mb`) re-decodes the
525        // dictionary, so a faster decoder is the single largest lever there.
526        // A valid zstd frame decodes to the same bytes under any conformant
527        // decoder, so the output is byte-for-byte what `ruzstd` produced.
528        #[cfg(feature = "compression")]
529        CODEC_ZSTD => zstd::decode_all(bytes).map_err(FileError::Decompress),
530        // Pure-Rust decode so a target WITHOUT a C toolchain (wasm, or a
531        // `--no-default-features` build) can still read compressed files.
532        // This is the ONLY decode path wasm ever compiles — the C `zstd`
533        // crate is gated behind `compression`, which wasm never enables.
534        #[cfg(not(feature = "compression"))]
535        CODEC_ZSTD => {
536            use std::io::Read;
537            let mut dec = ruzstd::decoding::StreamingDecoder::new(bytes)
538                .map_err(|e| FileError::Decompress(std::io::Error::other(e.to_string())))?;
539            let mut out = Vec::new();
540            dec.read_to_end(&mut out).map_err(FileError::Decompress)?;
541            Ok(out)
542        }
543        other => Err(FileError::UnknownCodec(other)),
544    }
545}
546
547/// The `Frame_Content_Size` a zstd frame header declares (RFC 8878 §3.1.1.1),
548/// or `None` when the frame does not state it (or `bytes` is not a zstd
549/// frame). Reads the header only.
550pub(crate) fn zstd_frame_content_size(b: &[u8]) -> Option<u64> {
551    if b.len() < 5 || b[..4] != [0x28, 0xB5, 0x2F, 0xFD] {
552        return None;
553    }
554    let fhd = b[4];
555    let fcs_flag = fhd >> 6;
556    let single_segment = fhd & 0x20 != 0;
557    let did_size = [0usize, 1, 2, 4][(fhd & 0x03) as usize];
558    let mut pos = 5 + usize::from(!single_segment) + did_size;
559    let fcs_size = match (fcs_flag, single_segment) {
560        (0, false) => return None,
561        (0, true) => 1,
562        (1, _) => 2,
563        (2, _) => 4,
564        _ => 8,
565    };
566    let f = b.get(pos..pos + fcs_size)?;
567    pos += fcs_size;
568    let _ = pos;
569    let mut v = 0u64;
570    for (i, x) in f.iter().enumerate() {
571        v |= (*x as u64) << (8 * i);
572    }
573    Some(if fcs_size == 2 { v + 256 } else { v })
574}
575
576/// [`decompress`] with the output bounded by `max` bytes: a frame whose header
577/// declares more is refused before any decoding, and a frame that does not
578/// declare its size is decoded at most `max + 1` bytes far and refused past
579/// `max`. Format `0x06` records each section's largest block, so a corrupt or
580/// hostile frame cannot make a reader allocate more than the writer did.
581pub(crate) fn decompress_bounded(codec: u8, bytes: &[u8], max: u64) -> Result<Vec<u8>, FileError> {
582    let too_big = |n: u64| {
583        FileError::Decompress(std::io::Error::other(format!(
584            "a block of {n} bytes exceeds its section's {max}-byte maximum"
585        )))
586    };
587    match codec {
588        CODEC_NONE => {
589            if bytes.len() as u64 > max {
590                return Err(too_big(bytes.len() as u64));
591            }
592            Ok(bytes.to_vec())
593        }
594        CODEC_ZSTD => {
595            let declared = zstd_frame_content_size(bytes);
596            if let Some(n) = declared.filter(|&n| n > max) {
597                return Err(too_big(n));
598            }
599            #[cfg(feature = "compression")]
600            {
601                if let Some(n) = declared {
602                    // One-shot into exactly the declared size: a frame that
603                    // lies about it fails here instead of growing the buffer.
604                    return zstd::bulk::decompress(bytes, n as usize)
605                        .map_err(FileError::Decompress);
606                }
607                use std::io::Read;
608                let mut out = Vec::new();
609                let mut dec = zstd::stream::read::Decoder::with_buffer(bytes)
610                    .map_err(FileError::Decompress)?;
611                dec.window_log_max(27).map_err(FileError::Decompress)?;
612                dec.take(max.saturating_add(1))
613                    .read_to_end(&mut out)
614                    .map_err(FileError::Decompress)?;
615                if out.len() as u64 > max {
616                    return Err(too_big(out.len() as u64));
617                }
618                Ok(out)
619            }
620            #[cfg(not(feature = "compression"))]
621            {
622                use std::io::Read;
623                let dec = ruzstd::decoding::StreamingDecoder::new(bytes)
624                    .map_err(|e| FileError::Decompress(std::io::Error::other(e.to_string())))?;
625                let mut out = Vec::new();
626                dec.take(max.saturating_add(1))
627                    .read_to_end(&mut out)
628                    .map_err(FileError::Decompress)?;
629                if out.len() as u64 > max {
630                    return Err(too_big(out.len() as u64));
631                }
632                Ok(out)
633            }
634        }
635        other => Err(FileError::UnknownCodec(other)),
636    }
637}
638
639/// The decompressed length of one stored block — from its frame header when
640/// it states it, else by decoding it (bounded by `ceiling`). What the `0x06`
641/// writer records as a section's largest block.
642pub(crate) fn block_raw_len(codec: u8, bytes: &[u8], ceiling: u64) -> Result<u64, FileError> {
643    match codec {
644        CODEC_NONE => Ok(bytes.len() as u64),
645        CODEC_ZSTD => match zstd_frame_content_size(bytes) {
646            Some(n) => Ok(n),
647            None => Ok(decompress_bounded(codec, bytes, ceiling)?.len() as u64),
648        },
649        other => Err(FileError::UnknownCodec(other)),
650    }
651}
652
653/// Bytes this close are cheaper fetched as one read than as two round trips:
654/// tiles are laid back-to-back so this only ever bridges a tile already made
655/// resident by an earlier window — keep it tight to avoid re-fetching it.
656const TILE_COALESCE_GAP: u64 = 4096;
657
658/// The dictionary chunks a query's output terms touch are scattered across the
659/// section (terms are sorted, output ids are not), so byte-adjacency is rare.
660/// A wider gap trades a little over-fetch for far fewer round trips — the right
661/// call on a latency-bound remote read, where one skipped 64 KiB chunk is much
662/// cheaper than another request's RTT.
663const DICT_COALESCE_GAP: u64 = 64 * 1024;
664
665/// The default `rete export --memory-budget-mb` when the flag is omitted — the
666/// same number `rete build`/`merge` default to, so one figure governs both the
667/// build and the export of a file. Comfortably above a sub-4-GB-dictionary
668/// file's decompressed dictionary (those export in a single pass at today's
669/// speed), and far under a large VM's RAM so a 50-GB-class file is bounded
670/// rather than OOM. See [`split_memory_budget`].
671pub const DEFAULT_EXPORT_BUDGET_MB: u64 = 4096;
672
673/// The largest default memory budget, in MiB, of a lazily-opened file — the
674/// bound on its dictionary chunk cache, index tile cache and raw block cache,
675/// split as [`split_memory_budget`] describes. The default actually applied is
676/// sized from the memory the process can still use, half of it, between
677/// [`crate::sysmem::MIN_DEFAULT_QUERY_BUDGET_MB`] and this figure (see
678/// [`crate::sysmem::default_query_budget_mb`]; this figure whenever the memory
679/// cannot be read). `rete sparql --memory-budget-mb` (and the other query
680/// commands) default to it too, and so does every client that opens through
681/// [`Rete::open_ranged_lazy`] (Python, R, Java, the relay), so a long-lived
682/// session no longer grows with every tile a query ever touched. The same
683/// figure as [`DEFAULT_EXPORT_BUDGET_MB`].
684pub const DEFAULT_QUERY_BUDGET_MB: u64 = DEFAULT_EXPORT_BUDGET_MB;
685
686/// The [`Rete::spill_threshold`] a memory budget implies: an eighth of it. A
687/// spilling `COUNT(DISTINCT …)` buffer can briefly occupy about twice its
688/// threshold while it sorts, so this keeps it within a quarter of the budget.
689pub fn spill_threshold_for(budget: u64) -> u64 {
690    budget / 8
691}
692
693/// The lowest byte cap the dictionary chunk cache is ever given, whatever the
694/// budget. It only has to guarantee forward progress and hold a handful of
695/// chunks (a single front-coded run can dwarf the 64 KiB chunk budget — a
696/// base64-embedded image, say). Correctness never depends on it: the cache's
697/// `Arc` handout keeps a chunk alive across its one decode even if it is
698/// evicted the instant it is inserted, so a full dump completes at *any* cap.
699/// This floor only keeps a pathologically small `--memory-budget-mb` from
700/// thrashing the whole dictionary chunk-by-chunk.
701const MIN_DICT_CACHE_CAP: u64 = 64 << 20; // 64 MiB
702
703/// Upper bound of the budget share [`split_memory_budget`] holds back for
704/// transient, uncached allocations (see the `reserve` there).
705const TRANSIENT_RESERVE_MAX: u64 = 128 << 20; // 128 MiB
706
707/// How a total `--memory-budget-mb` splits across the export's three bounded
708/// consumers. The three caps sum to the total less a small reserve (≤ 128 MiB)
709/// for the window's buffers and one window's in-flight reads, which no cache
710/// owns. [`Rete::set_memory_budget`] further subtracts the open's fixed
711/// directories ([`Rete::fixed_overhead`]) from the dictionary share, so the
712/// live heap of a dump stays within the total.
713#[derive(Debug, Clone, Copy)]
714pub struct MemoryBudget {
715    /// Raw file-block LRU (the `BlockCacheReader` that backs the ranged reader).
716    pub block_cache: u64,
717    /// Decompressed dictionary chunk bodies (the old ~4.3 GB export floor).
718    pub dict_cache: u64,
719    /// Decoded index tiles of the scanned permutation.
720    pub tile_cache: u64,
721}
722
723/// Split a total byte budget across the block cache, the dictionary chunk cache
724/// and the index tile cache (bounded-export phase 2, plan §5.2). `u64::MAX`
725/// (unlimited, e.g. `--in-memory` or `--memory-budget-mb 0`) passes straight
726/// through to every consumer.
727///
728/// The dictionary is the dominant consumer on scholar files (literal-heavy
729/// object section), and its chunks recur across windows, so it gets the bulk of
730/// the budget; the tile cache is small (a scanned permutation is far smaller
731/// than the dictionary) and the block cache stays modest, capped like every
732/// other ranged reader at [`crate::block_cache::DEFAULT_CACHE_CAP`].
733pub fn split_memory_budget(total: u64) -> MemoryBudget {
734    if total == u64::MAX {
735        return MemoryBudget {
736            block_cache: u64::MAX,
737            dict_cache: u64::MAX,
738            tile_cache: u64::MAX,
739        };
740    }
741    // Raw file blocks: min(256 MiB, total/8). Repeated directory/header reads
742    // must stay cheap, but this cache is not where the floor lived.
743    let block_cache = (total / 8).min(crate::block_cache::DEFAULT_CACHE_CAP);
744    // Scanned-permutation tiles: modest — tens of MB on epfl-class files. Cap
745    // at total/16 (≤ 256 MiB) so the dictionary keeps the rest.
746    let tile_cache = (total / 16).min(256 << 20);
747    // Held back for what no cache owns: the window's id and string buffers,
748    // the coalesced read spans and freshly decoded bodies of one window's
749    // prefetch before they enter the cache, and the program itself. Measured at
750    // ~80-120 MB on epfl and openaire-relation (dev/export-memory), so 1/32 of
751    // the budget, capped at 128 MiB.
752    let reserve = (total / 32).min(TRANSIENT_RESERVE_MAX);
753    // The dictionary chunk cache gets everything else, floored so a tiny budget
754    // still holds a working set of chunks.
755    let dict_cache = total
756        .saturating_sub(block_cache)
757        .saturating_sub(tile_cache)
758        .saturating_sub(reserve)
759        .max(MIN_DICT_CACHE_CAP);
760    MemoryBudget {
761        block_cache,
762        dict_cache,
763        tile_cache,
764    }
765}
766
767/// Heap an open `Rete` holds outside its bounded caches, by structure — see
768/// [`Rete::fixed_overhead`].
769#[derive(Debug, Clone, Copy, Default)]
770pub struct FixedOverhead {
771    /// The four dictionary sections' chunk directories.
772    pub dict_directory: u64,
773    /// The default graph's tile directories (every carried permutation).
774    pub index_directory: u64,
775    /// The named-graph directory plus the open graphs' tile directories.
776    pub named_graphs: u64,
777}
778
779impl FixedOverhead {
780    /// The sum of every part.
781    pub fn total(&self) -> u64 {
782        self.dict_directory + self.index_directory + self.named_graphs
783    }
784}
785
786/// Format a cache byte cap for `RETE_OPEN_DEBUG` (`unlimited` for `u64::MAX`).
787fn fmt_cap(cap: u64) -> String {
788    if cap == u64::MAX {
789        "unlimited".to_string()
790    } else {
791        format!("{}MiB", cap >> 20)
792    }
793}
794
795/// Fetch a set of ascending, disjoint byte ranges, coalescing ranges whose gap
796/// is at most `gap` into one span, then fetching the spans through
797/// [`RangeReader::read_many`] (which a parallelizable reader issues
798/// concurrently). Returns each requested range's bytes in order; `None` if any
799/// read fails.
800pub(crate) fn read_coalesced<R: RangeReader + ?Sized>(
801    reader: &R,
802    ranges: &[ByteRange],
803    gap: u64,
804) -> Option<Vec<Vec<u8>>> {
805    coalesced(reader, ranges, gap, false)
806}
807
808/// [`read_coalesced`] for ranges known to be scattered (a probe window's
809/// tiles): the spans go through [`RangeReader::read_many_scattered`], so the
810/// reader asks for them as they are.
811pub(crate) fn read_coalesced_scattered<R: RangeReader + ?Sized>(
812    reader: &R,
813    ranges: &[ByteRange],
814    gap: u64,
815) -> Option<Vec<Vec<u8>>> {
816    coalesced(reader, ranges, gap, true)
817}
818
819fn coalesced<R: RangeReader + ?Sized>(
820    reader: &R,
821    ranges: &[ByteRange],
822    gap: u64,
823    scattered: bool,
824) -> Option<Vec<Vec<u8>>> {
825    // Build the coalesced spans and remember which span each input range maps
826    // into, so the fetched span blobs can be sliced back apart in order.
827    let mut spans: Vec<(u64, u64)> = Vec::new();
828    let mut span_of: Vec<usize> = Vec::with_capacity(ranges.len());
829    let mut i = 0;
830    while i < ranges.len() {
831        let start = ranges[i].offset;
832        let mut end = ranges[i].offset.checked_add(ranges[i].len)?;
833        let mut j = i + 1;
834        while j < ranges.len() {
835            let r = &ranges[j];
836            if r.offset < end || r.offset - end > gap {
837                break;
838            }
839            end = r.offset.checked_add(r.len)?;
840            j += 1;
841        }
842        let si = spans.len();
843        spans.push((start, end - start));
844        for _ in i..j {
845            span_of.push(si);
846        }
847        i = j;
848    }
849    let blobs = if scattered {
850        reader.read_many_scattered(&spans)
851    } else {
852        reader.read_many(&spans)
853    }
854    .ok()?;
855    if blobs.len() != spans.len() {
856        return None;
857    }
858    let mut out = Vec::with_capacity(ranges.len());
859    for (k, r) in ranges.iter().enumerate() {
860        let (span_start, _) = spans[span_of[k]];
861        let blob = &blobs[span_of[k]];
862        out.push(checked_slice(blob, r.offset - span_start, r.len)?.to_vec());
863    }
864    Some(out)
865}
866
867/// Content hash (first 16 bytes of blake3) over the file payload sections.
868/// Identifies the immutable content independent of the header.
869fn content_hash(parts: &[&[u8]]) -> [u8; 16] {
870    let mut h = blake3::Hasher::new();
871    for p in parts {
872        h.update(p);
873    }
874    let mut out = [0u8; 16];
875    out.copy_from_slice(&h.finalize().as_bytes()[..16]);
876    out
877}
878
879/// A resolved triple as terms.
880pub type TermTriple = (String, String, String);
881
882/// What a filtered dump will cost, before it runs — see
883/// [`Rete::dump_plan`](crate::Rete::dump_plan).
884#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct DumpPlan {
886    /// The routed scan this dump will run. `None` when it provably matches
887    /// nothing without touching the index: an absent graph IRI, or a bound term
888    /// the dictionary does not contain.
889    pub scan: Option<crate::index::ScanPlan>,
890    /// The file's whole dictionary section — the **ceiling** on what resolving
891    /// the matched rows' terms can fault, and on a literal-heavy file the
892    /// dominant term in a dump's real cost.
893    ///
894    /// It is deliberately not an estimate. A dump that matches one row faults
895    /// only the chunks that row's three terms live in, but nothing in the tile
896    /// directories says which chunks those are or how big they are, and a
897    /// preview that guessed would be worse than one that states the bound.
898    pub dictionary_bytes: u64,
899}
900
901/// One labelled byte region of a `.rete` file image (see
902/// [`Rete::file_layout`]). `kind` is a stable machine tag: `header`,
903/// `metadata`, `dictionary`, `directory`, `tile`, `pyramid`, `named-graphs`.
904#[derive(Debug, Clone)]
905pub struct LayoutSegment {
906    pub kind: &'static str,
907    pub label: String,
908    pub offset: u64,
909    pub len: u64,
910}
911
912/// A byte range in the `.rete` file image.
913#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
914pub struct ByteRange {
915    pub offset: u64,
916    pub len: u64,
917}
918
919impl ByteRange {
920    pub fn end(self) -> u64 {
921        self.offset + self.len
922    }
923}
924
925/// Why a triple-pattern result is present in the file.
926#[derive(Debug, Clone, PartialEq, Eq)]
927pub struct TripleProvenance {
928    /// Matched triple resolved to canonical N-Triples tokens.
929    pub terms: TermTriple,
930    /// Matched triple in dictionary ID space.
931    pub ids: Triple,
932    /// Named graph IRI, or `None` for the default graph.
933    pub graph: Option<String>,
934    /// The resolved ID-space pattern that was matched.
935    pub matched_pattern: Pattern,
936    /// Permutation selected to answer the pattern.
937    pub index_permutation: IndexPermutation,
938    /// File byte range containing the dictionary container.
939    pub dictionary_range: ByteRange,
940    /// File byte range containing the permutation index container.
941    pub index_range: ByteRange,
942    /// File byte range containing the selected permutation payload inside the
943    /// index container.
944    pub index_section_range: ByteRange,
945    /// File byte range containing the pyramid metadata, when present.
946    pub pyramid_range: Option<ByteRange>,
947    /// Physical tile identifier, once tile directories are materialized.
948    /// Physical tile identifier (`PERM/index`, e.g. `POS/3`) for tiled (v0.2)
949    /// files; `None` for pre-tiling files.
950    pub tile: Option<String>,
951    /// File byte range of that (compressed) tile — the exact bytes a ranged
952    /// client would fetch to re-derive this match.
953    pub tile_range: Option<ByteRange>,
954}
955
956/// Encode a length-prefixed container of byte sections, each compressed with
957/// `codec` independently (so a range-reading client decompresses only the
958/// sections it fetches). Stored length is the *compressed* length.
959fn encode_container(sections: &[&[u8]], codec: u8) -> Vec<u8> {
960    let mut out = Vec::new();
961    write_uvarint(&mut out, sections.len() as u64);
962    for s in sections {
963        let payload = compress(codec, s);
964        write_uvarint(&mut out, payload.len() as u64);
965        out.extend_from_slice(&payload);
966    }
967    out
968}
969
970/// Decode a container into owned, decompressed sections.
971fn decode_container(bytes: &[u8], codec: u8) -> Result<Vec<Vec<u8>>, FileError> {
972    let (n, mut pos) = read_uvarint(bytes).ok_or(FileError::Container("truncated count"))?;
973    // `n` is untrusted; each section needs ≥1 byte, so cap the pre-allocation at
974    // the buffer length rather than trusting the count (avoids an OOM on a bogus
975    // header pointing at a small region).
976    let n = wire_usize(n, "section count too large")?;
977    let mut out = Vec::with_capacity(n.min(bytes.len()));
978    for _ in 0..n {
979        let (len, used) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
980            .ok_or(FileError::Container("truncated length"))?;
981        pos += used;
982        // Checked: a hostile length near u64::MAX used to overflow `pos + len`
983        // (a panic), and on wasm32 `len as usize` kept the low 32 bits.
984        let end = pos
985            .checked_add(wire_usize(len, "section length too large")?)
986            .filter(|&end| end <= bytes.len())
987            .ok_or(FileError::Container("section overruns buffer"))?;
988        out.push(decompress(codec, &bytes[pos..end])?);
989        pos = end;
990    }
991    Ok(out)
992}
993
994fn checked_end(off: u64, len: u64) -> Result<u64, FileError> {
995    off.checked_add(len)
996        .ok_or(FileError::Container("section range overflows"))
997}
998
999/// A length or count read from the file, as an in-memory `usize` — or a clean
1000/// error. `as usize` truncates on wasm32, a 32-bit target: a length of
1001/// `2^32 + 5` read 5 bytes there and decoding went on as if nothing were wrong.
1002fn wire_usize(v: u64, what: &'static str) -> Result<usize, FileError> {
1003    usize::try_from(v).map_err(|_| FileError::Container(what))
1004}
1005
1006/// The in-memory slice `[off, off + len)` of `bytes`, or `None` when the range
1007/// does not fit this target's `usize` or overruns the buffer.
1008fn checked_slice(bytes: &[u8], off: u64, len: u64) -> Option<&[u8]> {
1009    let start = usize::try_from(off).ok()?;
1010    let end = start.checked_add(usize::try_from(len).ok()?)?;
1011    bytes.get(start..end)
1012}
1013
1014/// Per-chunk budget for dictionary section bodies — same reasoning as
1015/// [`crate::index::INDEX_TILE_BUDGET`]: one fetch, one decompress per touch.
1016const DICT_CHUNK_BUDGET: usize = crate::write_opts::DEFAULT_DICT_CHUNK_BYTES;
1017
1018/// Encode one dictionary section as a chunked payload (format v0.2):
1019/// `[header_len, raw header (term_count/interval/restart table)]
1020///  [num_chunks; per chunk: Δfirst_run, key_len, key, comp_len]
1021///  [individually compressed run-aligned body slices]`.
1022/// The header keeps its original encoding, so restart offsets stay valid in
1023/// the section's coordinate space.
1024///
1025/// `key` is the chunk's **routing separator**, not its first term: the
1026/// shortest string strictly above the previous chunk's last term and at most
1027/// the chunk's own first term (empty for chunk 0). See
1028/// [`crate::dict::SectionChunk::key`] for the invariant and
1029/// [`crate::dict::shortest_separator`] for the choice. Storing the first term
1030/// verbatim — what this wrote before — is the degenerate case of the same
1031/// invariant, so readers of either vintage route both correctly; the
1032/// separator is simply orders of magnitude smaller when the boundary term is a
1033/// long literal.
1034#[cfg(test)]
1035fn encode_chunked_dict_section(raw: &[u8], codec: u8) -> Vec<u8> {
1036    encode_chunked_dict_section_ordered(raw, codec, crate::term_order::TermOrder::Bytewise)
1037}
1038
1039/// [`encode_chunked_dict_section`] for a section whose terms are sorted in
1040/// `order`: the routing keys are separators in that order.
1041fn encode_chunked_dict_section_ordered(
1042    raw: &[u8],
1043    codec: u8,
1044    order: crate::term_order::TermOrder,
1045) -> Vec<u8> {
1046    let meta = crate::dict::parse_meta(raw).unwrap_or(crate::dict::SectionMeta {
1047        term_count: 0,
1048        restart_interval: 1,
1049        restart_offsets: Vec::new(),
1050    });
1051    let body_start = meta
1052        .restart_offsets
1053        .first()
1054        .copied()
1055        .unwrap_or(raw.len() as u64);
1056    let header = &raw[..(body_start.min(raw.len() as u64)) as usize];
1057
1058    // Split runs into chunks by body-byte budget (whole runs only).
1059    let chunk_budget = crate::write_opts::dict_chunk_budget() as u64;
1060    let n_runs = meta.restart_offsets.len();
1061    let mut bounds: Vec<(usize, u64, u64)> = Vec::new(); // (first_run, start, end)
1062    let mut r = 0;
1063    while r < n_runs {
1064        let start = meta.restart_offsets[r];
1065        let mut r2 = r + 1;
1066        while r2 < n_runs && meta.restart_offsets[r2] - start < chunk_budget {
1067            r2 += 1;
1068        }
1069        let end = if r2 < n_runs {
1070            meta.restart_offsets[r2]
1071        } else {
1072            raw.len() as u64
1073        };
1074        bounds.push((r, start, end));
1075        r = r2;
1076    }
1077
1078    // Chunks are independent: compress them across all cores, as the index
1079    // tiles are (`par_iter` keeps the order, so the bytes are unchanged). At a
1080    // high `--zstd-level` this serial loop was most of a build's wall time.
1081    #[cfg(feature = "parallel")]
1082    let compressed: Vec<Vec<u8>> = {
1083        use rayon::prelude::*;
1084        bounds
1085            .par_iter()
1086            .map(|&(_, s, e)| compress(codec, &raw[s as usize..e as usize]))
1087            .collect()
1088    };
1089    #[cfg(not(feature = "parallel"))]
1090    let compressed: Vec<Vec<u8>> = bounds
1091        .iter()
1092        .map(|&(_, s, e)| compress(codec, &raw[s as usize..e as usize]))
1093        .collect();
1094    let mut out = Vec::new();
1095    write_uvarint(&mut out, header.len() as u64);
1096    out.extend_from_slice(header);
1097    write_uvarint(&mut out, bounds.len() as u64);
1098    let mut prev_run = 0usize;
1099    // The previous chunk's last term — the separator's lower bound. Decoding it
1100    // walks at most `restart_interval` front-coded entries of one chunk, which
1101    // is nothing beside the zstd pass over the same 64 KiB.
1102    let mut prev_last: Option<Vec<u8>> = None;
1103    for (i, (&(first_run, start, end), comp)) in bounds.iter().zip(&compressed).enumerate() {
1104        let key = if i == 0 {
1105            // Everything routes to chunk 0; it needs no separator at all.
1106            Vec::new()
1107        } else {
1108            let first = crate::dict::run_first_term(raw, start as usize).unwrap_or_default();
1109            match &prev_last {
1110                // Undecodable predecessor: the verbatim first term is always a
1111                // valid separator, just not a short one.
1112                None => first,
1113                Some(pl) => order.separator(pl, &first),
1114            }
1115        };
1116        if i + 1 < bounds.len() {
1117            // The last run of THIS chunk starts one run before the next chunk's.
1118            let last_run_off = meta.restart_offsets[bounds[i + 1].0 - 1] as usize;
1119            prev_last = crate::dict::run_last_term(raw, last_run_off, end as usize);
1120        }
1121        write_uvarint(&mut out, (first_run - prev_run) as u64);
1122        write_uvarint(&mut out, key.len() as u64);
1123        out.extend_from_slice(&key);
1124        write_uvarint(&mut out, comp.len() as u64);
1125        prev_run = first_run;
1126    }
1127    for comp in &compressed {
1128        out.extend_from_slice(comp);
1129    }
1130    out
1131}
1132
1133/// A parsed chunked-dict-section directory entry: the chunk's run/key/body
1134/// coordinates plus its compressed byte range *within the payload*. `key` is a
1135/// routing separator, not a term — see [`crate::dict::SectionChunk::key`].
1136pub(crate) struct DictChunkEntry {
1137    pub(crate) first_run: usize,
1138    pub(crate) key: Vec<u8>,
1139    body_start: u64,
1140    pub(crate) start: u64,
1141    pub(crate) end: u64,
1142}
1143
1144/// Parse a chunked dictionary section's header + directory (not the chunks).
1145/// `bytes` may be a prefix of the payload; compressed ranges validate against
1146/// `total_len`.
1147fn parse_chunked_dict_dir(
1148    bytes: &[u8],
1149    total_len: u64,
1150) -> Result<(crate::dict::SectionMeta, Vec<DictChunkEntry>), FileError> {
1151    let mut pos = 0usize;
1152    let take = |pos: &mut usize| -> Result<u64, FileError> {
1153        let (v, n) = read_uvarint(bytes.get(*pos..).unwrap_or(&[]))
1154            .ok_or(FileError::Container("truncated dict chunk directory"))?;
1155        *pos += n;
1156        Ok(v)
1157    };
1158    let header_len = wire_usize(take(&mut pos)?, "dict header too large")?;
1159    let header = bytes
1160        .get(pos..pos.saturating_add(header_len))
1161        .ok_or(FileError::Container("truncated dict header"))?;
1162    let meta = crate::dict::parse_meta(header)
1163        .map_err(|_| FileError::Container("malformed dict header"))?;
1164    pos += header_len;
1165
1166    let num_chunks = wire_usize(take(&mut pos)?, "dict chunk count too large")?;
1167    let mut entries = Vec::with_capacity(num_chunks.min(bytes.len()));
1168    let mut lens = Vec::with_capacity(num_chunks.min(bytes.len()));
1169    let mut prev_run = 0usize;
1170    for i in 0..num_chunks {
1171        let drun = wire_usize(take(&mut pos)?, "dict chunk run too large")?;
1172        let klen = wire_usize(take(&mut pos)?, "dict chunk key too large")?;
1173        let key = bytes
1174            .get(pos..pos.saturating_add(klen))
1175            .ok_or(FileError::Container("truncated dict chunk key"))?
1176            .to_vec();
1177        pos += klen;
1178        let clen = take(&mut pos)?;
1179        let first_run = next_chunk_run(i, prev_run, drun)?;
1180        let body_start = meta
1181            .restart_offsets
1182            .get(first_run)
1183            .copied()
1184            .ok_or(FileError::Container("dict chunk run out of range"))?;
1185        entries.push(DictChunkEntry {
1186            first_run,
1187            key,
1188            body_start,
1189            start: 0,
1190            end: 0,
1191        });
1192        lens.push(clen);
1193        prev_run = first_run;
1194    }
1195    let mut start = pos as u64;
1196    for (e, len) in entries.iter_mut().zip(lens) {
1197        let end = start
1198            .checked_add(len)
1199            .filter(|&e| e <= total_len)
1200            .ok_or(FileError::Container("dict chunk overruns section"))?;
1201        e.start = start;
1202        e.end = end;
1203        start = end;
1204    }
1205    Ok((meta, entries))
1206}
1207
1208/// The `i`-th chunk's first run from the previous chunk's and the stored delta.
1209/// Both writers start chunk 0 at run 0 and give every later chunk at least one
1210/// run, so the runs strictly ascend — which is what routing a term id to its
1211/// chunk (a binary search over `first_run`) relies on. A directory that breaks
1212/// that, or whose sum overflows, is corrupt: refuse it instead of routing ids
1213/// to the wrong chunk.
1214fn next_chunk_run(i: usize, prev_run: usize, drun: usize) -> Result<usize, FileError> {
1215    match (i, drun) {
1216        (0, 0) => Ok(0),
1217        (0, _) => Err(FileError::Container(
1218            "first dict chunk does not start at run 0",
1219        )),
1220        (_, 0) => Err(FileError::Container("dict chunk runs do not ascend")),
1221        _ => prev_run
1222            .checked_add(drun)
1223            .ok_or(FileError::Container("dict chunk run overflows")),
1224    }
1225}
1226
1227/// Fetch and parse a remote chunked dict section's header + directory: read a
1228/// small prefix and grow it geometrically until it parses, never fetching past
1229/// the section.
1230/// Read a chunked dictionary section's directory over a range reader WITHOUT
1231/// materializing the section-wide restart table. That table is one offset per
1232/// restart run — a 50 M-term section has ~3 M of them (~24 MiB resident), and
1233/// holding it is an iOS-Safari OOM on a big remote file. We read only the tiny
1234/// header prefix (term_count / interval) and the chunk directory, skipping the
1235/// restart-table bytes entirely; per-run offsets are derived per chunk on fault
1236/// (`SectionChunk::run_offsets`). The returned meta has an empty
1237/// `restart_offsets`, which the chunked lookups read as "derive per chunk".
1238pub(crate) fn read_dict_dir_ranged<R: RangeReader + ?Sized>(
1239    reader: &R,
1240    section: ByteRange,
1241) -> Result<(crate::dict::SectionMeta, Vec<DictChunkEntry>), FileError> {
1242    let total = section.len;
1243    // Initial prefix: the header prefix ([header_len][term_count][interval]) and,
1244    // for a *small* section, the whole chunk directory too — so those still cost
1245    // a single read. A big section has a huge restart table between the header
1246    // and the directory; we detect that (dir_start past the prefix) and range-
1247    // read only the directory below, never fetching the table.
1248    let init = 8192.min(total); // never over-read past the section (a tiny/empty
1249                                // section holds only its header + a stub directory)
1250    let head = reader.read_at(section.offset, init)?;
1251    let (header_len, n0) =
1252        read_uvarint(&head).ok_or(FileError::Container("truncated dict header len"))?;
1253    let hbase = n0; // first byte of the header body
1254    let (term_count, n1) = read_uvarint(head.get(hbase..).unwrap_or(&[]))
1255        .ok_or(FileError::Container("truncated dict term_count"))?;
1256    let (restart_interval, _n2) = read_uvarint(head.get(hbase + n1..).unwrap_or(&[]))
1257        .ok_or(FileError::Container("truncated dict interval"))?;
1258    if restart_interval == 0 {
1259        return Err(FileError::Container("zero restart interval"));
1260    }
1261    // The chunk directory begins right after the header body — i.e. past the
1262    // `header_len` bytes, which include the restart table we never materialize.
1263    let dir_start = (hbase as u64)
1264        .checked_add(header_len)
1265        .filter(|&d| d <= total)
1266        .ok_or(FileError::Container("dict header overruns section"))?;
1267    let dir_total = total - dir_start;
1268    let meta = crate::dict::SectionMeta {
1269        term_count: u32::try_from(term_count)
1270            .map_err(|_| FileError::Container("dict term count too large"))?,
1271        restart_interval: u32::try_from(restart_interval)
1272            .map_err(|_| FileError::Container("dict restart interval too large"))?,
1273        restart_offsets: Vec::new(),
1274    };
1275    // Runs in the section: what a chunk's first run must stay below.
1276    let num_runs = (meta.term_count as usize).div_ceil(meta.restart_interval as usize);
1277    let finish = |mut entries: Vec<DictChunkEntry>| {
1278        for e in &mut entries {
1279            e.start += dir_start; // dir-relative → section-relative
1280            e.end += dir_start;
1281        }
1282        (meta.clone(), entries)
1283    };
1284    // Fast path: the directory already sits in the prefix we read (small section
1285    // — its restart table is tiny, so the ~few KiB over-read is negligible).
1286    // A short prefix is not wasted: those bytes seed the probe below.
1287    let mut have: Vec<u8> = Vec::new();
1288    if dir_start < head.len() as u64 {
1289        match parse_chunk_dir_only(&head[dir_start as usize..], dir_total, num_runs)? {
1290            ChunkDirParse::Done(entries) => return Ok(finish(entries)),
1291            ChunkDirParse::Truncated { .. } => have = head[dir_start as usize..].to_vec(),
1292        }
1293    }
1294    // Big section: range-read the directory on its own, skipping the table.
1295    //
1296    // Nothing records the directory's byte length, so it has to be probed. Two
1297    // things keep that probe near ONE directory's worth of bytes. Each round
1298    // **appends** to the bytes already held instead of re-reading from the
1299    // start — the previous loop re-read the whole prefix every time, so a
1300    // 234 MB directory (epfl-infoscience's object section) cost 537 MB of range
1301    // reads to fetch. And a truncated parse reports how many entries fit in how
1302    // many bytes, which extrapolates the rest instead of blindly doubling. The
1303    // extrapolation may only quadruple a round, so a wild guess on a directory
1304    // with wildly uneven entries (one stored term can be hundreds of KB) can
1305    // never fetch the section body.
1306    let mut want = 4096u64.min(dir_total).max(1);
1307    loop {
1308        let held = have.len() as u64;
1309        if want > held {
1310            let extra = reader.read_at(section.offset + dir_start + held, want - held)?;
1311            if extra.is_empty() {
1312                return Err(FileError::Container("truncated dict chunk directory"));
1313            }
1314            have.extend_from_slice(&extra);
1315        }
1316        let held = have.len() as u64;
1317        match parse_chunk_dir_only(&have, dir_total, num_runs)? {
1318            ChunkDirParse::Done(entries) => return Ok(finish(entries)),
1319            ChunkDirParse::Truncated { .. } if held >= dir_total => {
1320                return Err(FileError::Container("truncated dict chunk directory"));
1321            }
1322            ChunkDirParse::Truncated {
1323                parsed,
1324                used,
1325                total,
1326            } => {
1327                // Extrapolate from a sample big enough to mean something;
1328                // otherwise just double. Undershooting only costs another
1329                // (append-only) round, so the estimate carries a small margin
1330                // rather than a generous one.
1331                let est = if parsed >= 16 && used > 0 {
1332                    let whole = ((used as u64) / (parsed as u64)).saturating_mul(total as u64);
1333                    whole.saturating_add(whole / 8).saturating_add(64)
1334                } else {
1335                    // Too small a sample to extrapolate from: double.
1336                    held.saturating_mul(2)
1337                };
1338                want = est
1339                    .max(held.saturating_add(1))
1340                    .min(held.saturating_mul(4))
1341                    .min(dir_total);
1342            }
1343        }
1344    }
1345}
1346
1347/// What [`parse_chunk_dir_only`] made of a chunk-directory prefix: either the
1348/// whole directory, or how far a truncated one got (the probe's step hint).
1349/// Truncation is not an error here — the caller is deliberately reading a
1350/// prefix and deciding how much more to fetch.
1351enum ChunkDirParse {
1352    Done(Vec<DictChunkEntry>),
1353    Truncated {
1354        /// Entries fully decoded from the prefix.
1355        parsed: usize,
1356        /// Bytes those entries occupied (directory-relative).
1357        used: usize,
1358        /// Chunk count the directory declares.
1359        total: usize,
1360    },
1361}
1362
1363/// Parse just the chunk directory (the bytes after a section header):
1364/// `[num_chunks][per chunk: Δfirst_run, key_len, key, comp_len]`.
1365/// Chunk byte ranges (`start`/`end`) come back relative to the directory's own
1366/// start; `body_start` is 0 (a lite section never uses it — lookups derive run
1367/// offsets per chunk). Bodies aren't needed here, so `dir` may end at the first
1368/// body as long as it covers the whole directory. A `dir` that stops mid-entry
1369/// is reported as [`ChunkDirParse::Truncated`], not an error: only bytes that
1370/// cannot be a directory at any length (a chunk range overrunning the section,
1371/// chunk runs that do not ascend from 0 or pass `num_runs`) fail.
1372fn parse_chunk_dir_only(
1373    dir: &[u8],
1374    dir_total: u64,
1375    num_runs: usize,
1376) -> Result<ChunkDirParse, FileError> {
1377    let mut pos = 0usize;
1378    let take = |pos: &mut usize| -> Option<u64> {
1379        let (v, n) = read_uvarint(dir.get(*pos..).unwrap_or(&[]))?;
1380        *pos += n;
1381        Some(v)
1382    };
1383    let Some(num_chunks) = take(&mut pos) else {
1384        return Ok(ChunkDirParse::Truncated {
1385            parsed: 0,
1386            used: 0,
1387            total: 0,
1388        });
1389    };
1390    let num_chunks = wire_usize(num_chunks, "dict chunk count too large")?;
1391    let mut entries = Vec::with_capacity(num_chunks.min(dir.len()));
1392    let mut lens: Vec<u64> = Vec::with_capacity(num_chunks.min(dir.len()));
1393    let mut prev_run = 0usize;
1394    for i in 0..num_chunks {
1395        let entry_start = pos;
1396        let short = ChunkDirParse::Truncated {
1397            parsed: entries.len(),
1398            used: entry_start,
1399            total: num_chunks,
1400        };
1401        let (Some(drun), Some(klen)) = (take(&mut pos), take(&mut pos)) else {
1402            return Ok(short);
1403        };
1404        let drun = wire_usize(drun, "dict chunk run too large")?;
1405        let klen = wire_usize(klen, "dict chunk key too large")?;
1406        let Some(key) = dir.get(pos..pos.saturating_add(klen)) else {
1407            return Ok(short);
1408        };
1409        let key = key.to_vec();
1410        pos += klen;
1411        let Some(clen) = take(&mut pos) else {
1412            return Ok(short);
1413        };
1414        let first_run = next_chunk_run(i, prev_run, drun)?;
1415        if first_run >= num_runs {
1416            return Err(FileError::Container("dict chunk run out of range"));
1417        }
1418        entries.push(DictChunkEntry {
1419            first_run,
1420            key,
1421            body_start: 0,
1422            start: 0,
1423            end: 0,
1424        });
1425        lens.push(clen);
1426        prev_run = first_run;
1427    }
1428    let mut start = pos as u64;
1429    for (e, len) in entries.iter_mut().zip(lens) {
1430        let end = start
1431            .checked_add(len)
1432            .filter(|&e| e <= dir_total)
1433            .ok_or(FileError::Container("dict chunk overruns section"))?;
1434        e.start = start;
1435        e.end = end;
1436        start = end;
1437    }
1438    Ok(ChunkDirParse::Done(entries))
1439}
1440
1441/// Decode one chunked dictionary section payload into a resident
1442/// [`crate::dict::ChunkedSection`] (chunks decompressed up front — the local
1443/// open path).
1444fn decode_chunked_dict_section(
1445    payload: &[u8],
1446    codec: u8,
1447    cache: std::sync::Arc<crate::chunk_cache::ChunkCache>,
1448    section_index: u8,
1449) -> Result<crate::dict::ChunkedSection, FileError> {
1450    let (meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64)?;
1451    let chunks = entries
1452        .into_iter()
1453        .map(|e| {
1454            let body = decompress(codec, &payload[e.start as usize..e.end as usize])?;
1455            Ok((
1456                crate::dict::SectionChunk::new(e.first_run, e.key, e.body_start),
1457                body,
1458            ))
1459        })
1460        .collect::<Result<Vec<_>, FileError>>()?;
1461    Ok(crate::dict::ChunkedSection::resident(
1462        meta,
1463        chunks,
1464        cache,
1465        section_index,
1466    ))
1467}
1468
1469fn decode_dictionary_container(
1470    bytes: &[u8],
1471    codec: u8,
1472    order: crate::term_order::TermOrder,
1473) -> Result<Dictionary, FileError> {
1474    let dsecs = decode_container(bytes, CODEC_NONE)?;
1475    if dsecs.len() != 4 {
1476        return Err(FileError::Container("expected 4 dictionary sections"));
1477    }
1478    // The four sections share one (unlimited) cache — residency is identical to
1479    // the previous per-chunk `OnceLock`s.
1480    let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
1481    let mut sections = Vec::with_capacity(4);
1482    for (si, sec) in dsecs.iter().enumerate() {
1483        sections.push(
1484            decode_chunked_dict_section(sec, codec, cache.clone(), si as u8)?.with_order(order),
1485        );
1486    }
1487    let arr: [crate::dict::ChunkedSection; 4] = sections
1488        .try_into()
1489        .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
1490    Ok(Dictionary::from_chunked_sections(arr))
1491}
1492
1493/// Encode one permutation's tiled section payload (format v0.2):
1494/// `[num_tiles][per tile: delta(min_a), max_a - min_a, compressed_len][tiles…]`,
1495/// each tile compressed independently with `codec` so a ranged reader can
1496/// fetch and decompress exactly the tiles a query routes to. The directory
1497/// itself is uncompressed (it must be readable before any tile).
1498fn encode_tiled_section(index: &GraphIndex, si: usize, codec: u8) -> Vec<u8> {
1499    let tiles = index.tile_sections()[si];
1500    let n = tiles.len();
1501    // Tile bodies live in the index's chunk cache (keyed by (section, tile));
1502    // fetch each as a shared handle. For a locally-built index every tile is
1503    // resident, so this is a cheap `Arc` clone under an uncontended lock.
1504    // Per-tile compression is the bulk of serialization time on a large graph and
1505    // the tiles are independent, so compress them across all cores. `par_iter`
1506    // preserves order, so the output is byte-identical to the serial map.
1507    #[cfg(feature = "parallel")]
1508    let compressed: Vec<Vec<u8>> = {
1509        use rayon::prelude::*;
1510        (0..n)
1511            .into_par_iter()
1512            .map(|ti| compress(codec, &index.tile_body(si, ti)))
1513            .collect()
1514    };
1515    #[cfg(not(feature = "parallel"))]
1516    let compressed: Vec<Vec<u8>> = (0..n)
1517        .map(|ti| compress(codec, &index.tile_body(si, ti)))
1518        .collect();
1519    let mut out = Vec::new();
1520    write_uvarint(&mut out, tiles.len() as u64);
1521    let mut prev_min = 0u32;
1522    for (tile, comp) in tiles.iter().zip(&compressed) {
1523        let (min_a, max_a) = tile.leading_range();
1524        write_uvarint(&mut out, (min_a - prev_min) as u64);
1525        write_uvarint(&mut out, (max_a - min_a) as u64);
1526        write_uvarint(&mut out, comp.len() as u64);
1527        prev_min = min_a;
1528    }
1529    for comp in &compressed {
1530        out.extend_from_slice(comp);
1531    }
1532    // Tile-synopsis trailer (FLAG_TILE_SYNOPSIS): per tile, the inclusive min/max
1533    // of the two non-leading columns `(min_b, span_b, min_c, span_c)`, derived
1534    // from each tile's zone map. Appended **after** the tile payloads so a reader
1535    // that predates the flag — which locates tiles by length and stops — never
1536    // reads it (backward-compatible). A reader honoring the flag reads it from the
1537    // section tail. On the (impossible for a built tile) parse failure, emit a
1538    // full range so nothing is ever wrongly pruned.
1539    for ti in 0..n {
1540        let body = index.tile_body(si, ti);
1541        let (min_b, max_b, min_c, max_c) = match crate::triples::TripleBlock::parse(&body) {
1542            Ok(b) => {
1543                let z = b.zone();
1544                (z.min_b, z.max_b, z.min_c, z.max_c)
1545            }
1546            Err(_) => (0, u32::MAX, 0, u32::MAX),
1547        };
1548        write_uvarint(&mut out, min_b as u64);
1549        write_uvarint(&mut out, (max_b - min_b) as u64);
1550        write_uvarint(&mut out, min_c as u64);
1551        write_uvarint(&mut out, (max_c - min_c) as u64);
1552    }
1553    out
1554}
1555
1556/// A parsed v0.2 tile-directory entry: leading-id range plus the tile's byte
1557/// range *within the section payload*.
1558pub(crate) struct TileDirEntry {
1559    pub(crate) min_a: u32,
1560    pub(crate) max_a: u32,
1561    pub(crate) start: u64,
1562    pub(crate) end: u64,
1563}
1564
1565/// One tile's synopsis: inclusive min/max of the two non-leading columns.
1566type TileSynopsis = (u32, u32, u32, u32);
1567
1568/// Parse the **tile-synopsis trailer** (when [`FLAG_TILE_SYNOPSIS`] is set): the
1569/// `num_tiles × (min_b, span_b, min_c, span_c)` records that follow the last tile
1570/// payload, starting at `trailer_start` within `payload`. Returns one synopsis per
1571/// tile, in directory order. `payload` may be just the trailer slice (remote) or
1572/// the whole section (local); `trailer_start` is the offset of the trailer within
1573/// it. A short/garbled trailer yields `None` (the caller keeps `None` synopses —
1574/// pruning simply doesn't fire, never a wrong result).
1575fn parse_tile_synopsis(
1576    payload: &[u8],
1577    trailer_start: usize,
1578    num_tiles: usize,
1579) -> Option<Vec<TileSynopsis>> {
1580    let mut pos = trailer_start;
1581    let take = |pos: &mut usize| -> Option<u32> {
1582        let (v, n) = read_uvarint(payload.get(*pos..)?)?;
1583        *pos += n;
1584        u32::try_from(v).ok()
1585    };
1586    let mut out = Vec::with_capacity(num_tiles.min(payload.len()));
1587    for _ in 0..num_tiles {
1588        let min_b = take(&mut pos)?;
1589        let max_b = min_b.checked_add(take(&mut pos)?)?;
1590        let min_c = take(&mut pos)?;
1591        let max_c = min_c.checked_add(take(&mut pos)?)?;
1592        out.push((min_b, max_b, min_c, max_c));
1593    }
1594    Some(out)
1595}
1596
1597/// Parse a tiled section payload's directory (not the tiles). `bytes` may be a
1598/// **prefix** of the payload (a ranged reader fetches the directory before any
1599/// tile); tile byte ranges are validated against `total_len`, the full payload
1600/// length. Every length is untrusted.
1601fn parse_tile_directory(bytes: &[u8], total_len: u64) -> Result<Vec<TileDirEntry>, FileError> {
1602    let mut pos = 0usize;
1603    let take = |pos: &mut usize| -> Result<u64, FileError> {
1604        let (v, n) = read_uvarint(bytes.get(*pos..).unwrap_or(&[]))
1605            .ok_or(FileError::Container("truncated tile directory"))?;
1606        *pos += n;
1607        Ok(v)
1608    };
1609    let num_tiles = wire_usize(take(&mut pos)?, "tile count too large")?;
1610    let mut entries = Vec::with_capacity(num_tiles.min(bytes.len()));
1611    let mut prev_min = 0u32;
1612    let mut lens = Vec::with_capacity(num_tiles.min(bytes.len()));
1613    // Leading ids are u32. A delta or span past that, or a sum that leaves
1614    // the id space, is corrupt: `as u32` and `wrapping_add` used to turn it
1615    // into some other, plausible id range and route lookups to the wrong tile.
1616    let id = |v: u64| u32::try_from(v).map_err(|_| FileError::Container("tile id out of range"));
1617    for _ in 0..num_tiles {
1618        let dmin = id(take(&mut pos)?)?;
1619        let span = id(take(&mut pos)?)?;
1620        let len = take(&mut pos)?;
1621        let min_a = prev_min
1622            .checked_add(dmin)
1623            .ok_or(FileError::Container("tile id out of range"))?;
1624        let max_a = min_a
1625            .checked_add(span)
1626            .ok_or(FileError::Container("tile id out of range"))?;
1627        entries.push(TileDirEntry {
1628            min_a,
1629            max_a,
1630            start: 0,
1631            end: 0,
1632        });
1633        lens.push(len);
1634        prev_min = min_a;
1635    }
1636    let mut start = pos as u64;
1637    for (e, len) in entries.iter_mut().zip(lens) {
1638        let end = start
1639            .checked_add(len)
1640            .filter(|&e| e <= total_len)
1641            .ok_or(FileError::Container("tile overruns section"))?;
1642        e.start = start;
1643        e.end = end;
1644        start = end;
1645    }
1646    Ok(entries)
1647}
1648
1649/// Fetch and parse a remote tiled section's directory: read a small prefix and
1650/// grow it geometrically until the directory parses, never fetching past the
1651/// section. A directory that still fails on the whole section is corrupt.
1652pub(crate) fn read_tile_directory_ranged<R: RangeReader + ?Sized>(
1653    reader: &R,
1654    section: ByteRange,
1655) -> Result<Vec<TileDirEntry>, FileError> {
1656    let total = section.len;
1657    let mut prefetch = 4096u64.min(total);
1658    loop {
1659        let prefix = reader.read_at(section.offset, prefetch)?;
1660        match parse_tile_directory(&prefix, total) {
1661            Ok(dir) => return Ok(dir),
1662            Err(_) if prefetch < total => prefetch = prefetch.saturating_mul(2).min(total),
1663            Err(e) => return Err(e),
1664        }
1665    }
1666}
1667
1668/// Fetch and parse a remote section's **tile-synopsis trailer** (only when the
1669/// header's [`FLAG_TILE_SYNOPSIS`] is set): one targeted range read of the bytes
1670/// past the last tile, parsed into one synopsis per tile (directory order). A
1671/// missing/short/garbled trailer degrades to all-`None` — pruning simply doesn't
1672/// fire, never a wrong result. The directory gives the trailer's start (the last
1673/// tile's end).
1674pub(crate) fn read_tile_synopsis_ranged<R: RangeReader + ?Sized>(
1675    reader: &R,
1676    section: ByteRange,
1677    dir: &[TileDirEntry],
1678) -> Vec<Option<TileSynopsis>> {
1679    let n = dir.len();
1680    let none = vec![None; n];
1681    let trailer_start = dir.iter().map(|e| e.end).max().unwrap_or(0);
1682    let total = section.len;
1683    if n == 0 || trailer_start >= total {
1684        return none; // no trailer bytes present
1685    }
1686    // Four varints per tile, each a u32 (at most 5 bytes): never read more,
1687    // whatever the section claims. The trailer is the section's tail, so a
1688    // hostile section length used to make this one read as large as it said.
1689    let max_trailer = (n as u64).saturating_mul(4 * 5);
1690    let trailer_len = (total - trailer_start).min(max_trailer);
1691    let Some(offset) = section.offset.checked_add(trailer_start) else {
1692        return none;
1693    };
1694    let Ok(bytes) = reader.read_at(offset, trailer_len) else {
1695        return none;
1696    };
1697    match parse_tile_synopsis(&bytes, 0, n) {
1698        Some(v) => v.into_iter().map(Some).collect(),
1699        None => none,
1700    }
1701}
1702
1703/// Per-tile absolute file ranges of each permutation section, for provenance.
1704/// A malformed directory yields an empty section (provenance degrades, queries
1705/// are unaffected).
1706fn tile_file_ranges(
1707    index_bytes: &[u8],
1708    container_offset: u64,
1709    section_ranges: &[ByteRange; NUM_PERMS],
1710) -> [Vec<(u32, u32, ByteRange)>; NUM_PERMS] {
1711    let mut out: [Vec<(u32, u32, ByteRange)>; NUM_PERMS] = Default::default();
1712    for (section, range) in out.iter_mut().zip(section_ranges) {
1713        // A permutation the file does not carry has a zeroed range. Its
1714        // `offset - container_offset` underflows, and before this guard it
1715        // panicked the whole `open_ranged` path on the first lean file.
1716        if range.len == 0 || range.offset < container_offset {
1717            continue;
1718        }
1719        let Some(payload) = checked_slice(index_bytes, range.offset - container_offset, range.len)
1720        else {
1721            continue;
1722        };
1723        if let Ok(dir) = parse_tile_directory(payload, payload.len() as u64) {
1724            *section = dir
1725                .into_iter()
1726                .map(|e| {
1727                    (
1728                        e.min_a,
1729                        e.max_a,
1730                        ByteRange {
1731                            offset: range.offset + e.start,
1732                            len: (e.end - e.start),
1733                        },
1734                    )
1735                })
1736                .collect();
1737        }
1738    }
1739    out
1740}
1741
1742/// Decode a tiled section payload into `(min_a, max_a, uncompressed tile)`
1743/// triples.
1744fn decode_tiled_section(payload: &[u8], codec: u8) -> Result<Vec<(u32, u32, Vec<u8>)>, FileError> {
1745    parse_tile_directory(payload, payload.len() as u64)?
1746        .into_iter()
1747        .map(|e| {
1748            Ok((
1749                e.min_a,
1750                e.max_a,
1751                decompress(codec, &payload[e.start as usize..e.end as usize])?,
1752            ))
1753        })
1754        .collect()
1755}
1756
1757/// Decode the index container: one raw tiled section payload per permutation
1758/// the file carries (`perms`, from the header — [`PermSet::ALL`] for every file
1759/// written before the mask existed), each tile compressed individually.
1760///
1761/// The section count is checked against `perms.len()`, so a file whose header
1762/// and container disagree is rejected rather than silently short-read. A reader
1763/// that does not know about lean files at all reaches the same conclusion by a
1764/// different route: it passes six, the container says three, and this errors —
1765/// which is why a lean file cannot be misread as an empty one.
1766fn decode_index_container(
1767    bytes: &[u8],
1768    codec: u8,
1769    perms: PermSet,
1770) -> Result<GraphIndex, FileError> {
1771    let isecs = decode_container(bytes, CODEC_NONE)?;
1772    if isecs.len() != perms.len() {
1773        return Err(FileError::Container(
1774            "index container section count does not match the header permutation mask",
1775        ));
1776    }
1777    decode_index_sections(isecs, codec, perms)
1778}
1779
1780fn decode_index_sections(
1781    mut isecs: Vec<Vec<u8>>,
1782    codec: u8,
1783    perms: PermSet,
1784) -> Result<GraphIndex, FileError> {
1785    let mut sections: [Vec<(u32, u32, Vec<u8>)>; NUM_PERMS] = Default::default();
1786    for (perm, sec) in perms.iter().zip(isecs.iter_mut()) {
1787        sections[perm.section_index()] = decode_tiled_section(sec, codec)?;
1788    }
1789    Ok(GraphIndex::from_tiles(sections, perms))
1790}
1791
1792/// Decode the file's **root** index container (the default graph): the
1793/// permutation sections, plus — on a quad-index file — the trailing seal, which
1794/// is checked and set aside. Returns the seal's digest of the QUAD_INDEX
1795/// section when there is one.
1796fn decode_root_index_container(
1797    bytes: &[u8],
1798    header: &Header,
1799) -> Result<(GraphIndex, Option<[u8; 16]>), FileError> {
1800    if !header.has_quad_index() {
1801        return Ok((
1802            decode_index_container(bytes, header.block_codec, header.perms)?,
1803            None,
1804        ));
1805    }
1806    let mut isecs = decode_container(bytes, CODEC_NONE)?;
1807    if isecs.len() != header.index_container_sections() {
1808        return Err(FileError::Container(
1809            "index container section count does not match the header permutation mask",
1810        ));
1811    }
1812    let seal = isecs.pop().unwrap_or_default();
1813    let digest = crate::quad::parse_seal(&seal)?;
1814    Ok((
1815        decode_index_sections(isecs, header.block_codec, header.perms)?,
1816        Some(digest),
1817    ))
1818}
1819
1820/// The named-graph layout a header announces must be the one its directory
1821/// holds: the quad flag with a QUAD_INDEX section and no NAMED_GRAPHS section,
1822/// or no flag and no QUAD_INDEX section. Anything else is a corrupt (or a
1823/// hand-edited) header, refused before either layout is read.
1824fn check_named_layout(header: &Header) -> Result<(), FileError> {
1825    let quad = header.quad_index_section().is_some();
1826    if header.has_quad_index() {
1827        if !quad {
1828            return Err(FileError::Container(
1829                "quad-index flag set but the file has no quad-index section",
1830            ));
1831        }
1832        if header.named_graphs_len > 0 {
1833            return Err(FileError::Container(
1834                "a file cannot carry both a quad index and a named-graphs section",
1835            ));
1836        }
1837        crate::quad::check_perms(header.perms)?;
1838    } else if quad {
1839        return Err(FileError::Container(
1840            "quad-index section present but the header does not flag it",
1841        ));
1842    }
1843    Ok(())
1844}
1845
1846fn container_section_payload_ranges(
1847    bytes: &[u8],
1848    container_offset: u64,
1849    expected_sections: usize,
1850) -> Result<Vec<ByteRange>, FileError> {
1851    let (section_count, mut pos) =
1852        read_uvarint(bytes).ok_or(FileError::Container("truncated count"))?;
1853    let section_count = usize::try_from(section_count)
1854        .map_err(|_| FileError::Container("section count too large"))?;
1855    if section_count != expected_sections {
1856        return Err(FileError::Container("unexpected section count"));
1857    }
1858
1859    let mut ranges = Vec::with_capacity(section_count);
1860    for _ in 0..section_count {
1861        let remaining = bytes
1862            .get(pos..)
1863            .ok_or(FileError::Container("truncated length"))?;
1864        let (payload_len, used) =
1865            read_uvarint(remaining).ok_or(FileError::Container("truncated length"))?;
1866        pos = pos
1867            .checked_add(used)
1868            .ok_or(FileError::Container("section range overflows"))?;
1869        let payload_len_usize = usize::try_from(payload_len)
1870            .map_err(|_| FileError::Container("section length too large"))?;
1871        let payload_end = pos
1872            .checked_add(payload_len_usize)
1873            .ok_or(FileError::Container("section range overflows"))?;
1874        if payload_end > bytes.len() {
1875            return Err(FileError::Container("section overruns buffer"));
1876        }
1877        ranges.push(ByteRange {
1878            offset: checked_end(container_offset, pos as u64)?,
1879            len: payload_len,
1880        });
1881        pos = payload_end;
1882    }
1883
1884    Ok(ranges)
1885}
1886
1887/// The permutation payload ranges of an index container holding `sections`
1888/// sections (the permutations, then — on a quad-index root — the seal, which
1889/// is not a permutation and is left out).
1890fn decode_index_section_ranges(
1891    bytes: &[u8],
1892    container_offset: u64,
1893    perms: PermSet,
1894    sections: usize,
1895) -> Result<[ByteRange; NUM_PERMS], FileError> {
1896    let ranges = container_section_payload_ranges(bytes, container_offset, sections)?;
1897    let mut out = [ByteRange { offset: 0, len: 0 }; NUM_PERMS];
1898    for (perm, range) in perms.iter().zip(ranges) {
1899        out[perm.section_index()] = range;
1900    }
1901    Ok(out)
1902}
1903
1904fn read_uvarint_at<R: RangeReader + ?Sized>(
1905    reader: &R,
1906    absolute_offset: u64,
1907    container_end: u64,
1908) -> Result<(u64, u64), FileError> {
1909    if absolute_offset >= container_end {
1910        return Err(FileError::Container("truncated container varint"));
1911    }
1912    let remaining = container_end - absolute_offset;
1913    let probe_len = remaining.min(10);
1914    let bytes = reader.read_at(absolute_offset, probe_len)?;
1915    read_uvarint(&bytes)
1916        .map(|(value, used)| (value, used as u64))
1917        .ok_or(FileError::Container("truncated container varint"))
1918}
1919
1920/// Locate one section's payload byte range inside a remote container, walking
1921/// only the (tiny) varint framing — no payload bytes are fetched.
1922pub(crate) fn locate_container_section_ranged<R: RangeReader + ?Sized>(
1923    reader: &R,
1924    container_offset: u64,
1925    container_len: u64,
1926    section_index: usize,
1927    expected_sections: u64,
1928) -> Result<ByteRange, FileError> {
1929    let container_end = checked_end(container_offset, container_len)?;
1930    let (section_count, used) = read_uvarint_at(reader, container_offset, container_end)?;
1931    if section_count != expected_sections {
1932        return Err(FileError::Container("unexpected container section count"));
1933    }
1934    if section_index >= section_count as usize {
1935        return Err(FileError::Container(
1936            "container section index out of bounds",
1937        ));
1938    }
1939
1940    let mut pos = checked_end(container_offset, used)?;
1941    for i in 0..section_count as usize {
1942        let (payload_len, len_used) = read_uvarint_at(reader, pos, container_end)?;
1943        pos = checked_end(pos, len_used)?;
1944        let payload_end = checked_end(pos, payload_len)?;
1945        if payload_end > container_end {
1946            return Err(FileError::Container("section overruns buffer"));
1947        }
1948        if i == section_index {
1949            return Ok(ByteRange {
1950                offset: pos,
1951                len: payload_len,
1952            });
1953        }
1954        pos = payload_end;
1955    }
1956    Err(FileError::Container("container section not found"))
1957}
1958
1959/// Open one index container (six tiled permutation sections) **lazily** over a
1960/// range reader: fetch each section's tile directory (and synopsis trailer,
1961/// when the file has one) but no tile payloads — tiles fault in on first scan.
1962/// Returns the index plus its section and tile file ranges (provenance).
1963///
1964/// This is the machinery `open_ranged_lazy` always used for the default
1965/// graph's container at `root_dir_offset`; a named graph's container is the
1966/// same format at a different offset, so the lazy named-graphs path opens a
1967/// LARGE graph through this too instead of decoding it resident.
1968#[allow(clippy::type_complexity)]
1969fn open_index_container_lazy(
1970    reader: &std::sync::Arc<dyn RangeReader + Send + Sync>,
1971    container: ByteRange,
1972    block_codec: u8,
1973    has_synopsis: bool,
1974    read_concurrency: usize,
1975    perms: PermSet,
1976    sections: usize,
1977) -> Result<
1978    (
1979        GraphIndex,
1980        [ByteRange; NUM_PERMS],
1981        [Vec<(u32, u32, ByteRange)>; NUM_PERMS],
1982    ),
1983    FileError,
1984> {
1985    let mut index_section_ranges = [ByteRange { offset: 0, len: 0 }; NUM_PERMS];
1986    let mut tile_ranges: [Vec<(u32, u32, ByteRange)>; NUM_PERMS] = Default::default();
1987    #[allow(clippy::type_complexity)]
1988    let mut directories: [Vec<(u32, u32, Option<TileSynopsis>)>; NUM_PERMS] = Default::default();
1989    for (pos, perm) in perms.iter().enumerate() {
1990        let si = perm.section_index();
1991        let section = locate_container_section_ranged(
1992            reader,
1993            container.offset,
1994            container.len,
1995            pos,
1996            sections as u64,
1997        )?;
1998        index_section_ranges[si] = section;
1999        let dir = read_tile_directory_ranged(reader, section)?;
2000        // Tile synopses (one extra small tail read per section) let a routed
2001        // scan prune a tile by a bound secondary component before faulting it.
2002        let syn = if has_synopsis {
2003            read_tile_synopsis_ranged(reader, section, &dir)
2004        } else {
2005            vec![None; dir.len()]
2006        };
2007        directories[si] = dir
2008            .iter()
2009            .zip(syn)
2010            .map(|(e, s)| (e.min_a, e.max_a, s))
2011            .collect();
2012        tile_ranges[si] = dir
2013            .into_iter()
2014            .map(|e| {
2015                (
2016                    e.min_a,
2017                    e.max_a,
2018                    ByteRange {
2019                        offset: section.offset + e.start,
2020                        len: (e.end - e.start),
2021                    },
2022                )
2023            })
2024            .collect();
2025    }
2026
2027    // The loader fetches and decompresses one tile per call; the bulk
2028    // loader serves multi-tile scans by coalescing adjacent tile ranges
2029    // into single range reads (tiles are back-to-back in their section,
2030    // so a full-section scan is typically one request).
2031    //
2032    // Either loader returns `None` (a failed load: the index marks itself
2033    // incomplete and a reset retries) unless the tile arrived whole and its
2034    // zone map parses. Without the length check, a short read of a raw
2035    // (uncompressed) tile decoded as a shorter tile and silently lost rows.
2036    let codec = block_codec;
2037    let loader_ranges = tile_ranges.clone();
2038    let loader_reader = reader.clone();
2039    let loader: crate::index::TileLoader = Box::new(move |si, ti| {
2040        let (_, _, range) = loader_ranges.get(si)?.get(ti)?;
2041        let bytes = loader_reader.read_at(range.offset, range.len).ok()?;
2042        if bytes.len() as u64 != range.len {
2043            return None;
2044        }
2045        checked_tile(codec, &bytes)
2046    });
2047    let bulk_ranges = tile_ranges.clone();
2048    let bulk_reader = reader.clone();
2049    let bulk: crate::index::TileBulkLoader = Box::new(move |si, tis| {
2050        let section = bulk_ranges.get(si)?;
2051        let want: Option<Vec<ByteRange>> = tis
2052            .iter()
2053            .map(|&ti| section.get(ti).map(|&(_, _, r)| r))
2054            .collect();
2055        let blobs = read_coalesced(bulk_reader.as_ref(), &want?, TILE_COALESCE_GAP)?;
2056        blobs.iter().map(|b| checked_tile(codec, b)).collect()
2057    });
2058    let mut index =
2059        GraphIndex::from_remote_directories(directories, perms, loader).with_bulk_loader(bulk);
2060    // Per-tile encoded lengths (from the directory) feed the join planner's
2061    // fatness gates — free here, unavailable later without a fetch.
2062    index.set_tile_lens(std::array::from_fn(|si| {
2063        tile_ranges[si]
2064            .iter()
2065            .map(|&(_, _, r)| r.len.min(u32::MAX as u64) as u32)
2066            .collect()
2067    }));
2068    // The reader's fan-out widens the planner's remote probe budget: a
2069    // desktop/CLI reader overlapping 16 range reads probes far more cheaply
2070    // than a phone's serial sync-XHR path.
2071    index.set_read_concurrency(read_concurrency);
2072    Ok((index, index_section_ranges, tile_ranges))
2073}
2074
2075/// A fetched tile, decompressed, or `None` when it does not decompress or its
2076/// zone map does not parse: a lazy tile load fails on the first scan that
2077/// touches it, instead of a corrupt tile reading as an empty or partial one.
2078fn checked_tile(codec: u8, bytes: &[u8]) -> Option<Vec<u8>> {
2079    let tile = decompress(codec, bytes).ok()?;
2080    crate::triples::TripleBlock::parse(&tile).ok()?;
2081    Some(tile)
2082}
2083
2084/// Serialize a complete `.rete` file image from a dictionary, index, and an
2085/// (optionally empty) encoded pyramid-meta section. `pyramid_levels` records the
2086/// number of dendrogram rounds the pyramid spans (0 if no pyramid).
2087///
2088/// The image is format `0x06`, the one layout every writer produces (the
2089/// legacy `0x05` image is [`write_file_v05`]).
2090pub fn write_file(
2091    dict: &Dictionary,
2092    index: &GraphIndex,
2093    has_quads: bool,
2094    pyramid_meta: &[u8],
2095    pyramid_levels: u16,
2096) -> Vec<u8> {
2097    write_dataset(dict, index, &[], has_quads, pyramid_meta, pyramid_levels)
2098}
2099
2100/// The legacy `0x05` image of [`write_file`]. The layout of the first stable
2101/// generation, kept for the tests that pin its byte layout, for the in-RAM
2102/// builders that hand their `0x05` image to the one `0x06` finish step, and for
2103/// `RETE_WRITE_FORMAT=5`.
2104#[doc(hidden)]
2105pub fn write_file_v05(
2106    dict: &Dictionary,
2107    index: &GraphIndex,
2108    has_quads: bool,
2109    pyramid_meta: &[u8],
2110    pyramid_levels: u16,
2111) -> Vec<u8> {
2112    write_dataset_v05(dict, index, &[], has_quads, pyramid_meta, pyramid_levels)
2113}
2114
2115/// A finished `0x05` image as the `0x06` file the public writers return (the
2116/// image is untouched under `RETE_WRITE_FORMAT=5`).
2117fn into_v06(image: Vec<u8>) -> Vec<u8> {
2118    crate::build_record::finish_in_ram(image, None)
2119        .unwrap_or_else(|e| panic!("writing format 0x06: {e}"))
2120}
2121
2122/// Encode an index container (v0.2): one raw tiled section payload per
2123/// permutation the index carries, in [`crate::index::ALL_PERMS`] order, tiles
2124/// compressed individually with `codec`. A three-permutation index writes three
2125/// sections — not six with three empty, which would be indistinguishable from
2126/// an empty graph to a reader that does not check the header mask.
2127pub(crate) fn encode_index_container(index: &GraphIndex, codec: u8) -> Vec<u8> {
2128    encode_index_container_sealed(index, codec, None)
2129}
2130
2131/// [`encode_index_container`] with an optional trailing **seal** section —
2132/// the root container of a quad-index file (see `quad.rs`). The seal is what
2133/// makes every reader that predates the quad index refuse the file: they all
2134/// check the container's section count before reading a payload.
2135fn encode_index_container_sealed(index: &GraphIndex, codec: u8, seal: Option<&[u8]>) -> Vec<u8> {
2136    let payloads: Vec<Vec<u8>> = index
2137        .perms()
2138        .iter()
2139        .map(|perm| encode_tiled_section(index, perm.section_index(), codec))
2140        .collect();
2141    let mut refs: Vec<&[u8]> = payloads.iter().map(|p| p.as_slice()).collect();
2142    if let Some(seal) = seal {
2143        refs.push(seal);
2144    }
2145    encode_container(&refs, CODEC_NONE)
2146}
2147
2148/// Encode the named-graphs section: each graph as `(iri, permutation container)`.
2149fn encode_named_graphs(named: &[(String, GraphIndex)], codec: u8) -> Vec<u8> {
2150    let mut out = Vec::new();
2151    write_uvarint(&mut out, named.len() as u64);
2152    for (iri, index) in named {
2153        write_uvarint(&mut out, iri.len() as u64);
2154        out.extend_from_slice(iri.as_bytes());
2155        let container = encode_index_container(index, codec);
2156        write_uvarint(&mut out, container.len() as u64);
2157        out.extend_from_slice(&container);
2158    }
2159    out
2160}
2161
2162fn decode_named_graphs(
2163    bytes: &[u8],
2164    codec: u8,
2165    perms: PermSet,
2166) -> Result<Vec<(String, GraphIndex)>, FileError> {
2167    let (n, mut pos) = read_uvarint(bytes).ok_or(FileError::Container("truncated graph count"))?;
2168    // Bounds-checked slice within this (already bounded) section. Lengths read
2169    // below are untrusted, so every range is validated before indexing.
2170    let bound = |start: usize, len: u64| -> Result<usize, FileError> {
2171        usize::try_from(len)
2172            .ok()
2173            .and_then(|len| start.checked_add(len))
2174            .filter(|&e| e <= bytes.len())
2175            .ok_or(FileError::Container("named-graph field overruns buffer"))
2176    };
2177    let n = wire_usize(n, "named-graph count too large")?;
2178    let mut out = Vec::with_capacity(n.min(bytes.len()));
2179    for _ in 0..n {
2180        let (ilen, u1) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
2181            .ok_or(FileError::Container("truncated iri len"))?;
2182        pos += u1;
2183        let iend = bound(pos, ilen)?;
2184        let iri = String::from_utf8_lossy(&bytes[pos..iend]).into_owned();
2185        pos = iend;
2186        let (clen, u2) = read_uvarint(bytes.get(pos..).unwrap_or(&[]))
2187            .ok_or(FileError::Container("truncated container len"))?;
2188        pos += u2;
2189        let cend = bound(pos, clen)?;
2190        let index = decode_index_container(&bytes[pos..cend], codec, perms)?;
2191        // Resident, so the count is a pass over tile headers; the planner
2192        // sizes this graph's patterns by it, not by the file's total.
2193        index.set_size_hint(u64::from(index.triple_count()));
2194        out.push((iri, index));
2195        pos = cend;
2196    }
2197    Ok(out)
2198}
2199
2200/// Serialize a full RDF *dataset*: the default-graph index plus zero or more
2201/// named graphs `(iri, index)`, all sharing one dictionary. Format `0x06`; named
2202/// graphs, given per graph here, are stored as the quad index (`0x06` has no
2203/// other layout).
2204pub fn write_dataset(
2205    dict: &Dictionary,
2206    default_index: &GraphIndex,
2207    named: &[(String, GraphIndex)],
2208    has_quads: bool,
2209    pyramid_meta: &[u8],
2210    pyramid_levels: u16,
2211) -> Vec<u8> {
2212    write_dataset_with_metadata(
2213        dict,
2214        default_index,
2215        named,
2216        has_quads,
2217        pyramid_meta,
2218        pyramid_levels,
2219        &[],
2220        &[],
2221    )
2222}
2223
2224/// The legacy `0x05` image of [`write_dataset`] (see [`write_file_v05`]).
2225#[doc(hidden)]
2226pub fn write_dataset_v05(
2227    dict: &Dictionary,
2228    default_index: &GraphIndex,
2229    named: &[(String, GraphIndex)],
2230    has_quads: bool,
2231    pyramid_meta: &[u8],
2232    pyramid_levels: u16,
2233) -> Vec<u8> {
2234    write_dataset_with_metadata_v05(
2235        dict,
2236        default_index,
2237        named,
2238        has_quads,
2239        pyramid_meta,
2240        pyramid_levels,
2241        &[],
2242        &[],
2243    )
2244}
2245
2246/// Serialize a dictionary to its on-file container bytes (4 front-coded, chunked
2247/// sections). Exposed so a low-RAM build can serialize **and drop** the live
2248/// `Dictionary` before building the permutation index — the index build works on
2249/// id-triples and never needs the dictionary.
2250pub(crate) fn encode_dict_container(dict: &Dictionary, codec: u8) -> Vec<u8> {
2251    let raw_sections = dict.sections();
2252    let dict_payloads: Vec<Vec<u8>> = raw_sections
2253        .iter()
2254        .map(|raw| encode_chunked_dict_section_ordered(raw, codec, dict.order()))
2255        .collect();
2256    encode_container(
2257        &[
2258            dict_payloads[0].as_slice(),
2259            dict_payloads[1].as_slice(),
2260            dict_payloads[2].as_slice(),
2261            dict_payloads[3].as_slice(),
2262        ],
2263        CODEC_NONE,
2264    )
2265}
2266
2267/// Serialize a dataset with an opaque **metadata** payload occupying the file's
2268/// metadata section (the application layer defines its meaning — the CLI stores a
2269/// JSON Dataset Card there). The section sits immediately after the header and
2270/// before the dictionary, so `metadata_offset` stays at `HEADER_LEN` and every
2271/// downstream section shifts by `metadata.len()`. The payload is folded into the
2272/// `content_hash`, so `verify` covers it and it is tamper-evident.
2273///
2274/// Passing an empty `metadata` is byte-identical to [`write_dataset`]: the section
2275/// is omitted (`metadata_len = 0`, `dictionary_offset = HEADER_LEN`) and the hash
2276/// is computed over exactly the same parts (a zero-length hash update is a no-op).
2277#[allow(clippy::too_many_arguments)]
2278pub fn write_dataset_with_metadata(
2279    dict: &Dictionary,
2280    default_index: &GraphIndex,
2281    named: &[(String, GraphIndex)],
2282    has_quads: bool,
2283    pyramid_meta: &[u8],
2284    pyramid_levels: u16,
2285    metadata: &[u8],
2286    text_index: &[u8],
2287) -> Vec<u8> {
2288    into_v06(write_dataset_with_metadata_v05(
2289        dict,
2290        default_index,
2291        named,
2292        has_quads,
2293        pyramid_meta,
2294        pyramid_levels,
2295        metadata,
2296        text_index,
2297    ))
2298}
2299
2300/// The legacy `0x05` image of [`write_dataset_with_metadata`] (see
2301/// [`write_file_v05`]).
2302#[doc(hidden)]
2303#[allow(clippy::too_many_arguments)]
2304pub fn write_dataset_with_metadata_v05(
2305    dict: &Dictionary,
2306    default_index: &GraphIndex,
2307    named: &[(String, GraphIndex)],
2308    has_quads: bool,
2309    pyramid_meta: &[u8],
2310    pyramid_levels: u16,
2311    metadata: &[u8],
2312    text_index: &[u8],
2313) -> Vec<u8> {
2314    let codec = writer_codec();
2315    let dict_container = encode_dict_container(dict, codec);
2316    // This entry point takes a finished section, so it reads the section's own
2317    // scope block to pick the directory kind (6, or 11 for a partial index) and
2318    // the header's named-graphs bit.
2319    let scope = (!text_index.is_empty())
2320        .then(|| crate::text_index::TextIndex::from_section(text_index, codec).ok())
2321        .flatten()
2322        .map(|ti| ti.scope().clone());
2323    let text_index_partial = scope
2324        .as_ref()
2325        .is_some_and(crate::text_index::TextIndexScope::restricts_predicates);
2326    let text_index_named = scope
2327        .as_ref()
2328        .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs);
2329    write_dataset_from_parts(
2330        &dict_container,
2331        dict.term_count() as u64,
2332        default_index,
2333        NamedPart::PerGraph(named),
2334        has_quads,
2335        dict.has_quoted_triples(),
2336        pyramid_meta,
2337        pyramid_levels,
2338        metadata,
2339        text_index,
2340        text_index_partial,
2341        text_index_named,
2342        codec,
2343        dict.order(),
2344    )
2345}
2346
2347/// How the named graphs reach the writer: one index per graph (the
2348/// NAMED_GRAPHS section, kind 5), or an encoded quad index (kind 10).
2349pub(crate) enum NamedPart<'a> {
2350    PerGraph(&'a [(String, GraphIndex)]),
2351    Quad(&'a crate::quad::EncodedQuadIndex),
2352}
2353
2354/// Assemble the final file image from an **already-serialized** dictionary
2355/// container (so the caller can drop the live `Dictionary` before calling this)
2356/// plus the permutation index and optional sections. The byte output is identical
2357/// to serializing the dictionary inline.
2358#[allow(clippy::too_many_arguments)]
2359pub(crate) fn write_dataset_from_parts(
2360    dict_container: &[u8],
2361    term_count: u64,
2362    default_index: &GraphIndex,
2363    named: NamedPart<'_>,
2364    has_quads: bool,
2365    has_quoted_triples: bool,
2366    pyramid_meta: &[u8],
2367    pyramid_levels: u16,
2368    metadata: &[u8],
2369    text_index: &[u8],
2370    text_index_partial: bool,
2371    text_index_named: bool,
2372    codec: u8,
2373    term_order: crate::term_order::TermOrder,
2374) -> Vec<u8> {
2375    // The named graphs: either the per-graph section (kind 5) or the quad index
2376    // (kind 10), whose digest the root container's seal carries. An empty quad
2377    // index is written as no named graphs at all.
2378    let (named_section, quad, named_quads) = match named {
2379        NamedPart::PerGraph(named) => (
2380            if named.is_empty() {
2381                Vec::new()
2382            } else {
2383                encode_named_graphs(named, codec)
2384            },
2385            None,
2386            named
2387                .iter()
2388                .map(|(_, idx)| idx.triple_count() as u64)
2389                .sum::<u64>(),
2390        ),
2391        NamedPart::Quad(q) if q.graph_count == 0 => (Vec::new(), None, 0),
2392        NamedPart::Quad(q) => (Vec::new(), Some(q), q.quad_count),
2393    };
2394    let seal = quad.map(|q| crate::quad::encode_seal(crate::quad::digest(&q.bytes)));
2395    let index_container = encode_index_container_sealed(default_index, codec, seal.as_deref());
2396    // A predicate-restricted index (kind 11) is outside the content hash, like
2397    // kinds 8 and 9, and starts with its own digest instead: a reader older
2398    // than 0.4.0 does not see kind 11, so a hashed one would make its `verify`
2399    // call every such file corrupt. A kind-6 index is hashed, as it always was.
2400    let text_index_partial = text_index_partial && !text_index.is_empty();
2401    let digested;
2402    let text_index: &[u8] = if text_index_partial {
2403        digested = crate::text_index::with_partial_digest(text_index);
2404        &digested
2405    } else {
2406        text_index
2407    };
2408
2409    // The metadata section (if any) sits between the header and the dictionary,
2410    // so the dictionary — and everything after it — shifts forward by its length.
2411    let meta_section_len = metadata.len() as u64;
2412    let dict_offset = HEADER_LEN as u64 + meta_section_len;
2413    let dict_len = dict_container.len() as u64;
2414    let index_offset = dict_offset + dict_len;
2415    let index_len = index_container.len() as u64;
2416    let pyr_offset = index_offset + index_len;
2417    let pyr_len = pyramid_meta.len() as u64;
2418    // Optional full-text index between the pyramid and the named graphs.
2419    let text_offset = pyr_offset + pyr_len;
2420    let text_len = text_index.len() as u64;
2421    let named_offset = text_offset + text_len;
2422    let named_len = named_section.len() as u64;
2423    // The quad index sits where the named-graphs section would (the two are
2424    // exclusive).
2425    let quad_len = quad.map_or(0, |q| q.bytes.len() as u64);
2426
2427    // Hash parts in physical order, with the metadata payload prepended when
2428    // present. Omitting it entirely (rather than hashing an empty slice) keeps the
2429    // no-metadata output's hash byte-identical to the pre-metadata writer.
2430    // `verify()` rebuilds this exact list from the header — any section added
2431    // here must be added there too (and covered by a tamper test).
2432    let mut parts: Vec<&[u8]> = Vec::with_capacity(5);
2433    if meta_section_len > 0 {
2434        parts.push(metadata);
2435    }
2436    parts.push(dict_container);
2437    parts.push(&index_container);
2438    parts.push(pyramid_meta);
2439    if text_len > 0 && !text_index_partial {
2440        parts.push(text_index);
2441    }
2442    if named_len > 0 {
2443        parts.push(&named_section);
2444    }
2445
2446    // Length of the trailing schema-pyramid block (0 if none), so a reader can
2447    // fetch just that block for an index/dictionary/summary-free Tier-0 read.
2448    let schema_meta_len = crate::meta::schema_block_len(pyramid_meta);
2449
2450    // The QUAD_INDEX section is not a hash part of its own: the seal inside the
2451    // index container carries its digest, so the content hash covers it — and a
2452    // reader that predates it computes the same hash over the same parts.
2453    let header = Header {
2454        version: crate::header::LEGACY_FORMAT_VERSION,
2455        flags: FLAG_TILE_SYNOPSIS
2456            | if has_quads { FLAG_HAS_QUADS } else { 0 }
2457            | if quad.is_some() {
2458                crate::header::FLAG_QUAD_INDEX
2459            } else {
2460                0
2461            }
2462            | if has_quoted_triples {
2463                FLAG_HAS_QUOTED_TRIPLES
2464            } else {
2465                0
2466            }
2467            | if text_len > 0 && text_index_named {
2468                crate::header::FLAG_TEXT_INDEX_NAMED_GRAPHS
2469            } else {
2470                0
2471            }
2472            | if term_order == crate::term_order::TermOrder::Sparql {
2473                crate::header::FLAG_SPARQL_TERM_ORDER
2474            } else {
2475                0
2476            },
2477        metadata_offset: HEADER_LEN as u64,
2478        metadata_len: meta_section_len,
2479        dictionary_offset: dict_offset,
2480        dictionary_len: dict_len,
2481        root_dir_offset: index_offset,
2482        root_dir_len: index_len,
2483        pyramid_meta_offset: if pyr_len > 0 { pyr_offset } else { 0 },
2484        pyramid_meta_len: pyr_len,
2485        dict_codec: codec,
2486        block_codec: codec,
2487        pyramid_levels,
2488        id_widths: 0,
2489        // Every container in the file — default graph and named graphs alike —
2490        // is written from an index built with the same set, so the mask is a
2491        // file-level fact and the default index is its source of truth.
2492        perms: default_index.perms(),
2493        quad_count: default_index.triple_count() as u64 + named_quads,
2494        term_count,
2495        content_hash: content_hash(&parts),
2496        named_graphs_offset: if named_len > 0 { named_offset } else { 0 },
2497        named_graphs_len: named_len,
2498        schema_meta_len,
2499        text_index_offset: if text_len > 0 { text_offset } else { 0 },
2500        text_index_len: text_len,
2501        text_index_partial: text_len > 0 && text_index_partial,
2502        build_info_offset: 0,
2503        build_info_len: 0,
2504        extra_sections: if quad_len > 0 {
2505            vec![crate::header::Section {
2506                kind: crate::header::SectionKind::QuadIndex,
2507                flags: 0,
2508                offset: named_offset,
2509                length: quad_len,
2510            }]
2511        } else {
2512            Vec::new()
2513        },
2514    };
2515
2516    let mut out = Vec::with_capacity(
2517        HEADER_LEN
2518            + metadata.len()
2519            + dict_container.len()
2520            + index_container.len()
2521            + pyramid_meta.len()
2522            + text_index.len()
2523            + named_section.len()
2524            + quad_len as usize
2525            + MAGIC.len(),
2526    );
2527    out.extend_from_slice(&header.to_bytes());
2528    if meta_section_len > 0 {
2529        out.extend_from_slice(metadata);
2530    }
2531    out.extend_from_slice(dict_container);
2532    out.extend_from_slice(&index_container);
2533    out.extend_from_slice(pyramid_meta);
2534    if text_len > 0 {
2535        out.extend_from_slice(text_index);
2536    }
2537    if named_len > 0 {
2538        out.extend_from_slice(&named_section);
2539    }
2540    if let Some(q) = quad {
2541        out.extend_from_slice(&q.bytes);
2542    }
2543    out.extend_from_slice(&MAGIC); // footer marker
2544    out
2545}
2546
2547/// `rdf:type` — the predicate that assigns a class to a resource.
2548pub const RDF_TYPE: &str = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
2549
2550/// An **ontology-aware** coarse graph: instead of structural communities, group
2551/// entities by their `rdf:type` class and aggregate relations between classes.
2552/// Returns `(subject_class, predicate, object_class, count)` over the default
2553/// graph. Entities with no type are `(untyped)`; literals are `(literal)`.
2554/// `rdf:type` triples themselves define the classes and are not counted as
2555/// relations. This is the dataset's effective schema with instance volumes.
2556pub fn schema_summary(rete: &Rete) -> Vec<(String, String, String, u32)> {
2557    use std::collections::{BTreeMap, HashMap};
2558    let triples = rete.dump(None);
2559
2560    let mut class_of: HashMap<&str, &str> = HashMap::new();
2561    for (s, p, o) in &triples {
2562        if p == RDF_TYPE {
2563            class_of.insert(s.as_str(), o.as_str());
2564        }
2565    }
2566    let classify = |t: &str| -> String {
2567        if let Some(c) = class_of.get(t) {
2568            (*c).to_string()
2569        } else if t.starts_with('"') {
2570            "(literal)".to_string()
2571        } else {
2572            "(untyped)".to_string()
2573        }
2574    };
2575
2576    let mut counts: BTreeMap<(String, String, String), u32> = BTreeMap::new();
2577    for (s, p, o) in &triples {
2578        if p == RDF_TYPE {
2579            continue; // type assertions define classes, not data relations
2580        }
2581        *counts
2582            .entry((classify(s), p.clone(), classify(o)))
2583            .or_default() += 1;
2584    }
2585    counts
2586        .into_iter()
2587        .map(|((a, p, b), c)| (a, p, b, c))
2588        .collect()
2589}
2590
2591/// Class populations: the number of resources of each `rdf:type` class in the
2592/// default graph, descending by count. The instance-count companion to
2593/// [`schema_summary`].
2594pub fn schema_classes(rete: &Rete) -> Vec<(String, u32)> {
2595    use std::collections::BTreeMap;
2596    let mut counts: BTreeMap<String, u32> = BTreeMap::new();
2597    for (_s, p, o) in rete.dump(None) {
2598        if p == RDF_TYPE {
2599            *counts.entry(o).or_default() += 1;
2600        }
2601    }
2602    let mut out: Vec<(String, u32)> = counts.into_iter().collect();
2603    out.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
2604    out
2605}
2606
2607/// Fetch **only** the metadata section (the opaque Dataset Card blob) via a
2608/// [`RangeReader`]: read the 1 KiB header, then the metadata byte range —
2609/// nothing else. This is the index-free CARD tier of the exploration model: a
2610/// remote/S3 client learns the dataset's self-description in **two small range
2611/// requests**, never touching the dictionary, index, or pyramid. Returns `None`
2612/// when the file carries no metadata.
2613///
2614/// Companion to [`Rete::open_ranged`] (which deliberately *skips* the card to
2615/// keep the query path minimal); this is the explicit "I want the card" path.
2616pub fn read_metadata_ranged<R: RangeReader>(reader: &R) -> Result<Option<Vec<u8>>, FileError> {
2617    let head = reader.read_at(0, HEADER_LEN as u64)?;
2618    if crate::v06::open::is_v06(&head) {
2619        let h = crate::v06::Header06::from_bytes(&head)?;
2620        return Ok(crate::v06::open::read_card_build(reader, &h)?.0);
2621    }
2622    let header = Header::from_bytes(&head)?;
2623    if header.metadata_len == 0 {
2624        return Ok(None);
2625    }
2626    let bytes = reader.read_at(header.metadata_offset, header.metadata_len)?;
2627    Ok(Some(bytes))
2628}
2629
2630/// The card **with its annex** (format `0x07`) merged back in: the fields the
2631/// hot card could not hold (`extra`, `description` over 8 KiB) are fetched from
2632/// the cold annex, a second range request. For a file with no annex (and for
2633/// every older format) this is [`read_metadata_ranged`].
2634pub fn read_card_full_ranged<R: RangeReader>(reader: &R) -> Result<Option<Vec<u8>>, FileError> {
2635    let head = reader.read_at(0, HEADER_LEN as u64)?;
2636    if crate::v06::open::is_v06(&head) {
2637        return Ok(crate::v06::open::read_card_full(reader)?.0);
2638    }
2639    read_metadata_ranged(reader)
2640}
2641
2642/// Fetch the metadata (Dataset Card) **and** build-info sections via a
2643/// [`RangeReader`] in the fewest requests: the 1 KiB header, then — because the
2644/// writer lays the build-info section immediately after the metadata — **one**
2645/// coalesced range covering both. A file with only one of the two costs the
2646/// same two requests; a file with neither costs just the header read. This
2647/// keeps "card + build conditions" within the CARD tier's 1 header + 1 range
2648/// budget instead of adding a third request.
2649///
2650/// Returns `(metadata, build_info)`, each `None` when absent.
2651#[allow(clippy::type_complexity)]
2652pub fn read_card_and_build_info_ranged<R: RangeReader>(
2653    reader: &R,
2654) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
2655    read_card_and_build_info_with_header(reader).map(|(_, m, b)| (m, b))
2656}
2657
2658/// [`read_card_and_build_info_ranged`], also returning the parsed [`Header`] it
2659/// had to read anyway.
2660///
2661/// The header is not a bonus: it carries the content hash and the section
2662/// directory — including whether the file has a TEXT_INDEX section, which the
2663/// card does not state (see
2664/// [`read_text_index_token_table_len_ranged`]). A caller that wants both and
2665/// used the tuple-only form would issue a **second** 1 KiB request for bytes it
2666/// already had, turning the CARD tier's documented "one header + one coalesced
2667/// range" into three requests.
2668#[allow(clippy::type_complexity)]
2669pub fn read_card_and_build_info_with_header<R: RangeReader>(
2670    reader: &R,
2671) -> Result<(Header, Option<Vec<u8>>, Option<Vec<u8>>), FileError> {
2672    let head = reader.read_at(0, HEADER_LEN as u64)?;
2673    let header = Header::from_bytes(&head)?;
2674    if crate::v06::reads(header.version) {
2675        let h = crate::v06::Header06::from_bytes(&head)?;
2676        let (card, build) = crate::v06::open::read_card_build(reader, &h)?;
2677        return Ok((header, card, build));
2678    }
2679    let meta = (header.metadata_offset, header.metadata_len);
2680    let build = (header.build_info_offset, header.build_info_len);
2681    if meta.1 > 0 && build.1 > 0 && build.0 == meta.0 + meta.1 {
2682        // Adjacent (the layout this crate writes): one read spans both.
2683        let both = reader.read_at(meta.0, meta.1 + build.1)?;
2684        let (m, b) = usize::try_from(meta.1)
2685            .ok()
2686            .and_then(|mid| both.split_at_checked(mid))
2687            .ok_or(FileError::Container("short card read"))?;
2688        return Ok((header, Some(m.to_vec()), Some(b.to_vec())));
2689    }
2690    let fetch = |off: u64, len: u64| -> Result<Option<Vec<u8>>, FileError> {
2691        if len == 0 {
2692            return Ok(None);
2693        }
2694        Ok(Some(reader.read_at(off, len)?))
2695    };
2696    Ok((header, fetch(meta.0, meta.1)?, fetch(build.0, build.1)?))
2697}
2698
2699/// The build-info section of a whole file image, or `None` if absent. The
2700/// payload is an opaque application-layer blob (the CLI stores build-conditions
2701/// JSON there); it is **not** covered by the content hash — see
2702/// [`attach_build_info`].
2703pub fn read_build_info(bytes: &[u8]) -> Result<Option<Vec<u8>>, FileError> {
2704    if crate::v06::open::is_v06(bytes) {
2705        let h = crate::v06::Header06::from_bytes(bytes)?;
2706        let r = crate::reader::SliceReader::new(bytes);
2707        return Ok(crate::v06::open::read_card_build(&r, &h)?.1);
2708    }
2709    let header = Header::from_bytes(bytes)?;
2710    if header.build_info_len == 0 {
2711        return Ok(None);
2712    }
2713    let section = checked_slice(bytes, header.build_info_offset, header.build_info_len)
2714        .ok_or(FileError::Container("build-info section overruns buffer"))?;
2715    Ok(Some(section.to_vec()))
2716}
2717
2718/// Everything needed to splice a build-info section into a finished file
2719/// **without holding the file in memory**: the rewritten header, where the
2720/// section goes, and where the bytes that do not move resume.
2721///
2722/// See [`plan_build_info`] for what the fields mean together: the build-info
2723/// section always follows the metadata section, so the layout is
2724/// `[header][metadata][build info][everything else, verbatim]` — or, for a
2725/// card placed after the data, `[header][data…][metadata][build info][footer]`.
2726#[derive(Debug, Clone, PartialEq, Eq)]
2727pub struct BuildInfoPlan {
2728    /// The header bytes to write at offset 0.
2729    pub header: [u8; HEADER_LEN],
2730    /// Offset the new section is written at: the end of the metadata section
2731    /// (the length of the unchanged prefix up to and including the card).
2732    pub insert: u64,
2733    /// Offset in the **old** file where the bytes that do not move resume. The
2734    /// span `insert..tail_start` is the old build-info section, if any, and is
2735    /// the only part of the old file the new one drops.
2736    pub tail_start: u64,
2737    /// Length of the file this plan produces.
2738    pub new_len: u64,
2739}
2740
2741/// Plan the splice of a build-info section of `info_len` bytes into a file
2742/// whose first [`HEADER_LEN`] bytes are `head` and whose total length is
2743/// `file_len`.
2744///
2745/// The section goes immediately after the metadata section — adjacent, so
2746/// [`read_card_and_build_info_ranged`] fetches card + build info in one
2747/// coalesced range — and every later section's offset shifts by the length
2748/// delta. That shift is the whole of the arithmetic, and it lives here so the
2749/// in-memory splice ([`attach_build_info`]) and any streaming rewriter derive
2750/// the same header from the same rule instead of each carrying a copy that can
2751/// drift.
2752///
2753/// The content hash is **deliberately left untouched**: build info records the
2754/// facts that differ between two builds of identical data (timestamp, builder,
2755/// measured costs), and folding them into the hash would break the
2756/// reproducible-hash property. `verify` accordingly ignores this section, on an
2757/// old reader (which sees an unknown kind-7 entry) and a new one alike — so a
2758/// file that gains, loses or changes a build record **keeps its content
2759/// identity**.
2760///
2761/// `info_len` of 0 plans the removal of an existing section.
2762pub fn plan_build_info(
2763    head: &[u8],
2764    file_len: u64,
2765    info_len: u64,
2766) -> Result<BuildInfoPlan, FileError> {
2767    let mut header = Header::from_bytes(head)?;
2768    // The insert point: immediately after the metadata section, wherever it
2769    // lies — at HEADER_LEN as the in-RAM writers place it, or after the data
2770    // where the memory-bounded build and `recard --in-place` put it
2771    // (`crate::recard::plan_card_append`). No card: right after the header.
2772    let insert = if header.metadata_len > 0 {
2773        header
2774            .metadata_offset
2775            .checked_add(header.metadata_len)
2776            .filter(|&end| header.metadata_offset >= HEADER_LEN as u64 && end <= file_len)
2777            .ok_or(FileError::Container(
2778                "the metadata section lies outside the file",
2779            ))?
2780    } else {
2781        HEADER_LEN as u64
2782    };
2783    let old_len = header.build_info_len;
2784    if old_len > 0 && header.build_info_offset != insert {
2785        return Err(FileError::Container(
2786            "existing build-info section is not adjacent to the metadata",
2787        ));
2788    }
2789    let tail_start = insert.saturating_add(old_len);
2790    if tail_start > file_len || insert > file_len {
2791        return Err(FileError::Container("build-info splice out of bounds"));
2792    }
2793
2794    // Shift every section that lives at or after the old tail.
2795    let shift = |off: &mut u64, len: u64| {
2796        if len > 0 && *off >= tail_start {
2797            *off = *off - old_len + info_len;
2798        }
2799    };
2800    shift(&mut header.dictionary_offset, header.dictionary_len);
2801    shift(&mut header.root_dir_offset, header.root_dir_len);
2802    shift(&mut header.pyramid_meta_offset, header.pyramid_meta_len);
2803    shift(&mut header.named_graphs_offset, header.named_graphs_len);
2804    shift(&mut header.text_index_offset, header.text_index_len);
2805    for s in &mut header.extra_sections {
2806        if s.length > 0 && s.offset >= tail_start {
2807            s.offset = s.offset - old_len + info_len;
2808        }
2809    }
2810    if info_len == 0 {
2811        header.build_info_offset = 0;
2812        header.build_info_len = 0;
2813    } else {
2814        header.build_info_offset = insert;
2815        header.build_info_len = info_len;
2816    }
2817
2818    Ok(BuildInfoPlan {
2819        header: header.to_bytes(),
2820        insert,
2821        tail_start,
2822        new_len: file_len - old_len + info_len,
2823    })
2824}
2825
2826/// Attach (or replace) a **build-info** section in a finished file image,
2827/// returning the new image. Layout and hash semantics: [`plan_build_info`].
2828///
2829/// Passing an empty `info` removes an existing section (or returns the image
2830/// unchanged when there is none).
2831pub fn attach_build_info(image: &[u8], info: &[u8]) -> Result<Vec<u8>, FileError> {
2832    if crate::v06::open::is_v06(image) {
2833        // 0x06: the record lives in the card region; the card stays.
2834        return crate::v06::write::replace_card_with(
2835            image,
2836            crate::v06::write::NewCard::Keep,
2837            (!info.is_empty()).then_some(info),
2838        );
2839    }
2840    let plan = plan_build_info(image, image.len() as u64, info.len() as u64)?;
2841    let mut out = Vec::with_capacity(plan.new_len as usize);
2842    out.extend_from_slice(&plan.header);
2843    out.extend_from_slice(&image[HEADER_LEN..plan.insert as usize]);
2844    out.extend_from_slice(info);
2845    out.extend_from_slice(&image[plan.tail_start as usize..]);
2846    debug_assert_eq!(out.len() as u64, plan.new_len);
2847    Ok(out)
2848}
2849
2850/// Replace the **metadata** section (the Dataset Card blob) of a finished file
2851/// image, returning the new image. The section is the first one after the
2852/// header, so every later section's offset shifts by the length delta.
2853///
2854/// Unlike [`attach_build_info`], the metadata payload **is** inside the content
2855/// hash — it is part of what the file says about itself — so the hash is
2856/// recomputed here. The result is byte-identical to what the writer would have
2857/// produced had it been handed this payload in the first place
2858/// (`write_dataset_with_metadata`): the section sits at `HEADER_LEN`, every
2859/// later offset is derived from its length, and the hash folds the same parts
2860/// in the same order.
2861///
2862/// Why splice rather than rebuild: the only caller that needs this is a build
2863/// that has already assembled the file and then learned something about it by
2864/// **running** its own starter queries against it. Re-deriving the whole image
2865/// to correct a metadata blob would mean re-parsing the input; a splice costs
2866/// one memcpy and one rehash of an image that is already resident.
2867///
2868/// Passing an empty payload removes the section.
2869pub fn replace_metadata(image: &[u8], metadata: &[u8]) -> Result<Vec<u8>, FileError> {
2870    if crate::v06::open::is_v06(image) {
2871        // 0x06: the card region is rewritten; every section is kept.
2872        let h = crate::v06::Header06::from_bytes(image)?;
2873        let (_, build) =
2874            crate::v06::open::read_card_build(&crate::reader::SliceReader::new(image), &h)?;
2875        return crate::v06::write::replace_card(
2876            image,
2877            (!metadata.is_empty()).then_some(metadata),
2878            build.as_deref(),
2879        );
2880    }
2881    let mut header = Header::from_bytes(image)?;
2882    // The writers in this crate always place metadata (when present) at
2883    // HEADER_LEN; anything else is a file this function must not reshape.
2884    let insert = HEADER_LEN as u64;
2885    if header.metadata_len > 0 && header.metadata_offset != insert {
2886        return Err(FileError::Container(
2887            "metadata section does not sit immediately after the header",
2888        ));
2889    }
2890    let old_len = header.metadata_len;
2891    let tail_start = (insert + old_len) as usize;
2892    if tail_start > image.len() {
2893        return Err(FileError::Container("metadata splice out of bounds"));
2894    }
2895    let new_len = metadata.len() as u64;
2896
2897    let shift = |off: &mut u64, len: u64| {
2898        if len > 0 && *off >= insert + old_len {
2899            *off = *off - old_len + new_len;
2900        }
2901    };
2902    shift(&mut header.dictionary_offset, header.dictionary_len);
2903    shift(&mut header.root_dir_offset, header.root_dir_len);
2904    shift(&mut header.pyramid_meta_offset, header.pyramid_meta_len);
2905    shift(&mut header.named_graphs_offset, header.named_graphs_len);
2906    shift(&mut header.text_index_offset, header.text_index_len);
2907    shift(&mut header.build_info_offset, header.build_info_len);
2908    for s in &mut header.extra_sections {
2909        if s.length > 0 && s.offset >= insert + old_len {
2910            s.offset = s.offset - old_len + new_len;
2911        }
2912    }
2913    header.metadata_offset = insert;
2914    header.metadata_len = new_len;
2915
2916    let mut out = Vec::with_capacity(image.len() - old_len as usize + metadata.len());
2917    out.extend_from_slice(&header.to_bytes());
2918    out.extend_from_slice(metadata);
2919    out.extend_from_slice(&image[tail_start..]);
2920    // The payload changed, so the hash must: recompute it over the same part
2921    // list `verify` will rebuild from this very header, then re-serialize the
2922    // header carrying it.
2923    header.content_hash = hash_of_sections(&header, &out)?;
2924    out[..HEADER_LEN].copy_from_slice(&header.to_bytes());
2925    Ok(out)
2926}
2927
2928/// The content hash of a file image, folded over exactly the parts the writer
2929/// hashed (see `write_dataset_from_parts`): the metadata payload when present,
2930/// then dict, index, pyramid-meta, and — when present — the text index and the
2931/// named graphs. [`verify`] compares this against the header; [`replace_metadata`]
2932/// writes it.
2933fn hash_of_sections(header: &Header, bytes: &[u8]) -> Result<[u8; 16], FileError> {
2934    let slice = |off: u64, len: u64| -> Result<&[u8], FileError> {
2935        // `off` and `len` come STRAIGHT FROM THE FILE HEADER, so a crafted or
2936        // corrupt .rete can set them near u64::MAX and make `off + len` overflow
2937        // — which panics in debug and wraps in release, BEFORE `.get()` gets the
2938        // chance to reject the range. Found by the weekly fuzz run
2939        // (fuzz_targets/open.rs, "attempt to add with overflow"). Saturating the
2940        // sum turns both cases into the same clean Container error.
2941        // `checked_slice` also refuses an offset past this target's usize
2942        // (wasm32), which `as usize` used to wrap onto other bytes.
2943        checked_slice(bytes, off, len).ok_or(FileError::Container("section overruns buffer"))
2944    };
2945    let parts = content_hash_parts(header)
2946        .into_iter()
2947        .map(|(off, len)| slice(off, len))
2948        .collect::<Result<Vec<&[u8]>, FileError>>()?;
2949    Ok(content_hash(&parts))
2950}
2951
2952/// The byte ranges the content hash folds, **in the order it folds them**: the
2953/// metadata payload when present, then dictionary, index, pyramid-meta, and —
2954/// when present — a kind-6 text index and the named graphs. Build info, the
2955/// self-digested kinds (8 LABEL_INDEX, 9 SCHEMA, 11 predicate-restricted text
2956/// index) and any section kind this reader does not know are deliberately
2957/// outside it. The quad index (kind 10) is not a range here either: its digest
2958/// is the seal at the end of the index container, so it is hashed by reference.
2959///
2960/// The one list [`verify`], [`verify_ranged`] and [`replace_metadata`] hash
2961/// over, and what a streaming rewriter (`rete recard`) feeds a hasher while it
2962/// copies — so a section a future format adds to the hash is added here, once,
2963/// and every one of them follows.
2964pub fn content_hash_parts(header: &Header) -> Vec<(u64, u64)> {
2965    let mut parts: Vec<(u64, u64)> = Vec::with_capacity(6);
2966    if header.metadata_len > 0 {
2967        parts.push((header.metadata_offset, header.metadata_len));
2968    }
2969    parts.push((header.dictionary_offset, header.dictionary_len));
2970    parts.push((header.root_dir_offset, header.root_dir_len));
2971    // A zero-length pyramid folds nothing either way; it is listed only when
2972    // present so the ranges a rewriter walks are all real sections.
2973    if header.pyramid_meta_len > 0 {
2974        parts.push((header.pyramid_meta_offset, header.pyramid_meta_len));
2975    }
2976    // A kind-11 (predicate-restricted) text index is self-digested instead.
2977    if header.text_index_len > 0 && !header.text_index_partial {
2978        parts.push((header.text_index_offset, header.text_index_len));
2979    }
2980    if header.named_graphs_len > 0 {
2981        parts.push((header.named_graphs_offset, header.named_graphs_len));
2982    }
2983    parts
2984}
2985
2986/// `verify` for a `0x06` file over any reader: what failed (empty = all good).
2987/// `only` restricts the section checks to one kind (`verify --section`).
2988pub fn verify_ranged_v06(
2989    reader: &dyn crate::reader::RangeReader,
2990    head: &[u8],
2991    only: Option<u8>,
2992) -> Result<Vec<String>, FileError> {
2993    const SLICE: u64 = 8 << 20;
2994    let h = crate::v06::Header06::from_bytes(head)?;
2995    if h.file_len != reader.len() {
2996        return Ok(vec!["file length".to_string()]);
2997    }
2998    let mut bad = Vec::new();
2999    let root_bytes = reader.read_at(crate::v06::HEADER_LEN as u64, h.root_len as u64)?;
3000    if crate::v06::digest16(&root_bytes) != h.root_digest {
3001        bad.push("root".to_string());
3002        return Ok(bad);
3003    }
3004    let root = crate::v06::Root06::decode(&root_bytes, h.internal_codec)?;
3005    if crate::v06::content_hash(&h) != h.content_hash {
3006        bad.push("content hash".to_string());
3007    }
3008    let card = h.card_range();
3009    if only.is_none() && card.len > 0 {
3010        let stored = reader.read_at(card.offset, card.len)?;
3011        if crate::v06::digest16(&stored) != h.card_digest {
3012            bad.push("card".to_string());
3013        }
3014    }
3015    for s in &root.sections {
3016        if only.is_some_and(|k| k != s.kind) {
3017            continue;
3018        }
3019        let a = match s.abs(&h) {
3020            Ok(a) if a.offset + a.len <= h.file_len => a,
3021            _ => {
3022                bad.push(crate::v06::kind::name(s.kind).to_string());
3023                continue;
3024            }
3025        };
3026        let mut hs = blake3::Hasher::new();
3027        let mut at = a.offset;
3028        while at < a.offset + a.len {
3029            let n = (a.offset + a.len - at).min(SLICE);
3030            hs.update(&reader.read_at(at, n)?);
3031            at += n;
3032        }
3033        if hs.finalize().as_bytes()[..16] != s.digest {
3034            bad.push(crate::v06::kind::name(s.kind).to_string());
3035        }
3036    }
3037    if only.is_none() {
3038        let n = crate::v06::FOOTER.len() as u64;
3039        if reader.read_at(h.file_len - n, n)? != crate::v06::footer(h.version) {
3040            bad.push("footer".to_string());
3041        }
3042    }
3043    Ok(bad)
3044}
3045
3046/// Recompute the content hash from a file image and check it against the header
3047/// — detects corruption or truncation of the payload sections. The
3048/// LABEL_INDEX, SCHEMA and predicate-restricted TEXT_INDEX sections (kinds 8,
3049/// 9, 11) are outside that hash — an older reader's `verify` must keep passing
3050/// on a file that carries them — so each is checked against its own leading
3051/// digest instead. The QUAD_INDEX section (kind 10) is hashed by reference: its
3052/// digest is the seal inside the (hashed) index container, checked here.
3053pub fn verify(bytes: &[u8]) -> Result<bool, FileError> {
3054    if crate::v06::open::is_v06(bytes) {
3055        let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
3056            std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(bytes)));
3057        return crate::v06::open::V06File::open(r, bytes)?.verify(None);
3058    }
3059    let header = Header::from_bytes(bytes)?;
3060    if hash_of_sections(&header, bytes)? != header.content_hash {
3061        return Ok(false);
3062    }
3063    // The QUAD_INDEX section is covered through the seal's digest (inside the
3064    // hashed index container), so it is checked against that.
3065    if header.has_quad_index() || header.quad_index_section().is_some() {
3066        check_named_layout(&header)?;
3067        let slice = |off: u64, len: u64| -> Result<&[u8], FileError> {
3068            checked_slice(bytes, off, len).ok_or(FileError::Container("section overruns buffer"))
3069        };
3070        let root = slice(header.root_dir_offset, header.root_dir_len)?;
3071        let ranges = container_section_payload_ranges(
3072            root,
3073            header.root_dir_offset,
3074            header.index_container_sections(),
3075        )?;
3076        let seal = ranges
3077            .last()
3078            .ok_or(FileError::Container("missing quad-index seal"))?;
3079        let want = crate::quad::parse_seal(slice(seal.offset, seal.len)?)?;
3080        let q = header
3081            .quad_index_section()
3082            .ok_or(FileError::Container("missing quad-index section"))?;
3083        if crate::quad::digest(slice(q.offset, q.length)?) != want {
3084            return Ok(false);
3085        }
3086    }
3087    for s in self_digested_sections(&header) {
3088        let section = checked_slice(bytes, s.offset, s.length)
3089            .ok_or(FileError::Container("section overruns buffer"))?;
3090        if !crate::label_index::digest_ok(section) {
3091            return Ok(false);
3092        }
3093    }
3094    Ok(true)
3095}
3096
3097/// The sections that carry their own leading 16-byte digest rather than being
3098/// folded into the header content hash: LABEL_INDEX (8), SCHEMA (9) and a
3099/// predicate-restricted TEXT_INDEX (11).
3100fn self_digested_sections(header: &Header) -> Vec<Section> {
3101    [
3102        header.label_index_section(),
3103        header.schema_section(),
3104        header
3105            .section(crate::header::SectionKind::TextIndexPartial)
3106            .filter(|s| s.length > 0),
3107    ]
3108    .into_iter()
3109    .flatten()
3110    .collect()
3111}
3112
3113/// Append LABEL_INDEX / SCHEMA sections (kinds 8, 9) to a finished file image,
3114/// just before its 4-byte footer, and return the new image.
3115///
3116/// They sit outside the content hash, so nothing before them moves and the
3117/// hash is unchanged: only the header's directory gains entries. An empty
3118/// payload is skipped. Each kind may be appended once; a second append of a
3119/// kind the file already carries is an error, as is any other kind.
3120pub fn append_sections(
3121    image: &[u8],
3122    sections: &[(SectionKind, &[u8])],
3123) -> Result<Vec<u8>, FileError> {
3124    let lens: Vec<(SectionKind, u64)> =
3125        sections.iter().map(|(k, b)| (*k, b.len() as u64)).collect();
3126    let (header, body_end) = plan_append(image, image.len() as u64, &lens)?;
3127    let extra: usize = sections.iter().map(|(_, b)| b.len()).sum();
3128    let mut out = Vec::with_capacity(image.len() + extra);
3129    out.extend_from_slice(&header.to_bytes());
3130    out.extend_from_slice(&image[HEADER_LEN..body_end as usize]);
3131    for (_, b) in sections {
3132        out.extend_from_slice(b);
3133    }
3134    out.extend_from_slice(&MAGIC);
3135    Ok(out)
3136}
3137
3138/// The header an append produces, and where the old footer starts.
3139fn plan_append(
3140    head: &[u8],
3141    file_len: u64,
3142    sections: &[(SectionKind, u64)],
3143) -> Result<(Header, u64), FileError> {
3144    let mut header = Header::from_bytes(head)?;
3145    if header.expected_file_len() != Some(file_len) {
3146        return Err(FileError::Container(
3147            "cannot append sections: the file length does not match its header",
3148        ));
3149    }
3150    let body_end = file_len - MAGIC.len() as u64;
3151    let mut at = body_end;
3152    for &(kind, len) in sections {
3153        if !matches!(kind, SectionKind::LabelIndex | SectionKind::Schema) {
3154            return Err(FileError::Container(
3155                "only LABEL_INDEX and SCHEMA sections can be appended",
3156            ));
3157        }
3158        if len == 0 {
3159            continue;
3160        }
3161        if header.section(kind).is_some_and(|s| s.length > 0) {
3162            return Err(FileError::Container(
3163                "the file already carries that section",
3164            ));
3165        }
3166        header = header.with_section(kind, at, len);
3167        at += len;
3168    }
3169    Ok((header, body_end))
3170}
3171
3172/// Open a file's dictionary over a [`RangeReader`] with only its chunk
3173/// directories resident — for the memory-bounded build, which resolves the
3174/// SCHEMA section's few class and predicate ids through the file it just
3175/// wrote.
3176#[cfg(not(target_arch = "wasm32"))]
3177pub(crate) fn open_dictionary_lazy<R: RangeReader + Send + Sync + 'static>(
3178    reader: R,
3179) -> Result<Dictionary, FileError> {
3180    let head = reader.read_at(0, HEADER_LEN as u64)?;
3181    let header = Header::from_bytes(&head)?;
3182    ranged_chunked_dictionary(&std::sync::Arc::new(reader), &header, [true; 4])
3183}
3184
3185/// [`verify`] over a [`RangeReader`](crate::reader::RangeReader), streaming:
3186/// the hashed sections are read in bounded slices and folded into one BLAKE3
3187/// state, so peak memory is one slice, not the file. The same sections, in
3188/// the same order, as `hash_of_sections`; on a 49 GB file the in-memory form
3189/// needs 49 GB.
3190pub fn verify_ranged(reader: &dyn crate::reader::RangeReader) -> Result<bool, FileError> {
3191    /// Read size per step: large enough that a local or remote read is
3192    /// throughput-bound, small enough to be negligible next to any budget.
3193    const SLICE: u64 = 8 << 20;
3194    let len = reader.len();
3195    if len < HEADER_LEN as u64 {
3196        return Err(FileError::Container("file shorter than its header"));
3197    }
3198    let head = reader.read_at(0, HEADER_LEN as u64)?;
3199    if crate::v06::open::is_v06(&head) {
3200        return verify_ranged_v06(reader, &head, None).map(|bad| bad.is_empty());
3201    }
3202    let header = Header::from_bytes(&head)?;
3203    let mut h = blake3::Hasher::new();
3204    for (off, n) in content_hash_parts(&header) {
3205        // Offsets come straight from the header: reject an overrun (or an
3206        // overflowing sum) before reading, as `hash_of_sections` does.
3207        if off.checked_add(n).is_none_or(|end| end > len) {
3208            return Err(FileError::Container("section overruns buffer"));
3209        }
3210        let mut at = off;
3211        while at < off + n {
3212            let take = (off + n - at).min(SLICE);
3213            h.update(&reader.read_at(at, take)?);
3214            at += take;
3215        }
3216    }
3217    let mut out = [0u8; 16];
3218    out.copy_from_slice(&h.finalize().as_bytes()[..16]);
3219    if out != header.content_hash {
3220        return Ok(false);
3221    }
3222    // The QUAD_INDEX section, against the digest in the index container's seal
3223    // (see `verify`), streamed the same way.
3224    if header.has_quad_index() || header.quad_index_section().is_some() {
3225        check_named_layout(&header)?;
3226        let sections = header.index_container_sections();
3227        let seal = locate_container_section_ranged(
3228            reader,
3229            header.root_dir_offset,
3230            header.root_dir_len,
3231            sections - 1,
3232            sections as u64,
3233        )?;
3234        let want = crate::quad::parse_seal(&reader.read_at(seal.offset, seal.len)?)?;
3235        let q = header
3236            .quad_index_section()
3237            .ok_or(FileError::Container("missing quad-index section"))?;
3238        if q.offset.checked_add(q.length).is_none_or(|end| end > len) {
3239            return Err(FileError::Container("section overruns buffer"));
3240        }
3241        let mut h = blake3::Hasher::new();
3242        let mut at = q.offset;
3243        while at < q.offset + q.length {
3244            let take = (q.offset + q.length - at).min(SLICE);
3245            h.update(&reader.read_at(at, take)?);
3246            at += take;
3247        }
3248        let mut got = [0u8; 16];
3249        got.copy_from_slice(&h.finalize().as_bytes()[..16]);
3250        if got != want {
3251            return Ok(false);
3252        }
3253    }
3254    // The self-digested sections (kinds 8, 9, 11), streamed the same way.
3255    for s in self_digested_sections(&header) {
3256        if s.length < 16 || s.offset.checked_add(s.length).is_none_or(|end| end > len) {
3257            return Err(FileError::Container("section overruns buffer"));
3258        }
3259        let want = reader.read_at(s.offset, 16)?;
3260        let mut h = blake3::Hasher::new();
3261        let mut at = s.offset + 16;
3262        while at < s.offset + s.length {
3263            let take = (s.offset + s.length - at).min(SLICE);
3264            h.update(&reader.read_at(at, take)?);
3265            at += take;
3266        }
3267        if h.finalize().as_bytes()[..16] != want[..] {
3268            return Ok(false);
3269        }
3270    }
3271    Ok(true)
3272}
3273
3274/// Faults the pyramid meta in on first access. `None` = the fetch failed.
3275type PyramidLoader = Box<dyn Fn() -> Option<PyramidMeta> + Send + Sync>;
3276
3277/// The pyramid-meta section, held either resident (eager opens) or deferred
3278/// (the lazy remote open). SPARQL never touches the pyramid, but on a Wikidata
3279/// file it can be tens of MB (114k communities, millions of superedges), so a
3280/// remote SPARQL query must not pay to fetch it — it faults in only when a
3281/// community/pyramid query actually calls [`Rete::pyramid`].
3282enum PyramidSlot {
3283    Resident(Option<PyramidMeta>),
3284    Lazy {
3285        loader: PyramidLoader,
3286        cell: std::sync::OnceLock<Option<PyramidMeta>>,
3287    },
3288}
3289
3290/// A section held resident (eager opens) or faulted in on first use (lazy
3291/// opens) — the LABEL_INDEX and SCHEMA sections. `None` inside means the file
3292/// has no such section, or it could not be read or parsed: both are optional
3293/// and derived, so a bad one degrades to "absent" (and `verify` reports it).
3294pub(crate) enum SectionSlot<T> {
3295    Resident(Option<T>),
3296    Lazy {
3297        loader: Box<dyn Fn() -> Option<T> + Send + Sync>,
3298        cell: std::sync::OnceLock<Option<T>>,
3299    },
3300}
3301
3302impl<T> SectionSlot<T> {
3303    /// The section, faulting it in on the lazy path.
3304    pub(crate) fn get(&self) -> Option<&T> {
3305        match self {
3306            SectionSlot::Resident(v) => v.as_ref(),
3307            SectionSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
3308        }
3309    }
3310
3311    /// The section only if it is resident or already faulted — never a read.
3312    pub(crate) fn get_if_loaded(&self) -> Option<&T> {
3313        match self {
3314            SectionSlot::Resident(v) => v.as_ref(),
3315            SectionSlot::Lazy { cell, .. } => cell.get().and_then(|o| o.as_ref()),
3316        }
3317    }
3318}
3319
3320/// The LABEL_INDEX of a file image held in memory.
3321fn resident_label_slot<'a>(
3322    header: &Header,
3323    region: &dyn Fn(u64, u64) -> Result<&'a [u8], FileError>,
3324) -> SectionSlot<crate::label_index::LabelIndex> {
3325    SectionSlot::Resident(header.label_index_section().and_then(|s| {
3326        let bytes = region(s.offset, s.length).ok()?;
3327        crate::label_index::LabelIndex::from_section(bytes, header.block_codec).ok()
3328    }))
3329}
3330
3331/// The SCHEMA section of a file image held in memory.
3332fn resident_schema_slot<'a>(
3333    header: &Header,
3334    region: &dyn Fn(u64, u64) -> Result<&'a [u8], FileError>,
3335) -> SectionSlot<crate::schema_section::SchemaMeta> {
3336    SectionSlot::Resident(header.schema_section().and_then(|s| {
3337        crate::schema_section::SchemaMeta::decode(region(s.offset, s.length).ok()?).ok()
3338    }))
3339}
3340
3341/// The deferred LABEL_INDEX of a ranged open: the first search reads the
3342/// section head (preamble + root, ≤ [`crate::label_index::HEAD_READ`] bytes),
3343/// then one directory node per level and the data blocks it needs.
3344fn ranged_label_slot<R: RangeReader + Send + Sync + 'static>(
3345    reader: &std::sync::Arc<R>,
3346    header: &Header,
3347) -> SectionSlot<crate::label_index::LabelIndex> {
3348    let Some(s) = header.label_index_section() else {
3349        return SectionSlot::Resident(None);
3350    };
3351    let r = reader.clone();
3352    let codec = header.block_codec;
3353    SectionSlot::Lazy {
3354        loader: Box::new(move || {
3355            let head = r
3356                .read_at(s.offset, crate::label_index::HEAD_READ.min(s.length))
3357                .ok()?;
3358            let rr = r.clone();
3359            let loader = Box::new(move |off: u64, len: u64| rr.read_at(s.offset + off, len).ok());
3360            crate::label_index::LabelIndex::from_head(&head, s.length, codec, loader).ok()
3361        }),
3362        cell: std::sync::OnceLock::new(),
3363    }
3364}
3365
3366/// The deferred SCHEMA section of a ranged open: one range read, on first use.
3367fn ranged_schema_slot<R: RangeReader + Send + Sync + 'static>(
3368    reader: &std::sync::Arc<R>,
3369    header: &Header,
3370) -> SectionSlot<crate::schema_section::SchemaMeta> {
3371    let Some(s) = header.schema_section() else {
3372        return SectionSlot::Resident(None);
3373    };
3374    let r = reader.clone();
3375    SectionSlot::Lazy {
3376        loader: Box::new(move || {
3377            let bytes = r.read_at(s.offset, s.length).ok()?;
3378            crate::schema_section::SchemaMeta::decode(&bytes).ok()
3379        }),
3380        cell: std::sync::OnceLock::new(),
3381    }
3382}
3383
3384/// Read and decode a file's SCHEMA section over a [`RangeReader`] — the
3385/// header and one range. `Ok(None)` when the file has none (or it is
3386/// malformed: the section is optional and derived).
3387pub fn read_schema_ranged<R: RangeReader + ?Sized>(
3388    reader: &R,
3389) -> Result<Option<crate::schema_section::SchemaMeta>, FileError> {
3390    let head = reader.read_at(0, HEADER_LEN as u64)?;
3391    let header = Header::from_bytes(&head)?;
3392    Ok(read_schema_with_header(reader, &header))
3393}
3394
3395fn read_schema_with_header<R: RangeReader + ?Sized>(
3396    reader: &R,
3397    header: &Header,
3398) -> Option<crate::schema_section::SchemaMeta> {
3399    if crate::v06::reads(header.version) {
3400        let b = crate::v06::open::read_schema_bytes(reader).ok()??;
3401        return crate::schema_section::SchemaMeta::decode(&b).ok();
3402    }
3403    let s = header.schema_section()?;
3404    let bytes = reader.read_at(s.offset, s.length).ok()?;
3405    crate::schema_section::SchemaMeta::decode(&bytes).ok()
3406}
3407
3408/// Faults the text index in on first search. `None` = the file has none, or the
3409/// fetch/parse failed.
3410type TextIndexLoader = Box<dyn Fn() -> Option<crate::text_index::TextIndex> + Send + Sync>;
3411
3412/// Answers the token table's byte length from the section's first ≤10 bytes —
3413/// the length varint alone, never the table it measures.
3414type TokenTableProbe = Box<dyn Fn() -> Option<u64> + Send + Sync>;
3415
3416/// The TEXT_INDEX section, held either resident (eager opens decode the whole
3417/// thing) or deferred (the lazy remote open keeps only a loader that fetches the
3418/// token table on first search, then faults posting lists one at a time). SPARQL
3419/// never touches it, so the lazy remote path keeps its small range budget.
3420///
3421/// Both variants can also answer how long the section's leading **token table**
3422/// is without faulting it — the figure a caller needs to state what a first
3423/// search will cost, since `header.text_index_len` counts the postings blob too
3424/// and overstates it by several times on a literal-heavy graph.
3425enum TextIndexSlot {
3426    Resident {
3427        index: Option<crate::text_index::TextIndex>,
3428        /// Measured from the section bytes this open already held; `None` when
3429        /// the file carries no text index.
3430        token_table_len: Option<u64>,
3431    },
3432    Lazy {
3433        loader: TextIndexLoader,
3434        cell: std::sync::OnceLock<Option<crate::text_index::TextIndex>>,
3435        /// One ≤10-byte range read, memoized in `token_table_cell`.
3436        token_table: TokenTableProbe,
3437        token_table_cell: std::sync::OnceLock<Option<u64>>,
3438    },
3439}
3440
3441impl TextIndexSlot {
3442    /// The parsed index, faulting it in (token table first) on the lazy path.
3443    fn index(&self) -> Option<&crate::text_index::TextIndex> {
3444        match self {
3445            TextIndexSlot::Resident { index, .. } => index.as_ref(),
3446            TextIndexSlot::Lazy { loader, cell, .. } => cell.get_or_init(loader).as_ref(),
3447        }
3448    }
3449
3450    /// Byte length of the section's leading token table — see
3451    /// [`Rete::text_index_token_table_len`].
3452    fn token_table_len(&self) -> Option<u64> {
3453        match self {
3454            TextIndexSlot::Resident {
3455                token_table_len, ..
3456            } => *token_table_len,
3457            TextIndexSlot::Lazy {
3458                token_table,
3459                token_table_cell,
3460                ..
3461            } => *token_table_cell.get_or_init(token_table),
3462        }
3463    }
3464}
3465
3466/// Decode a whole TEXT_INDEX section held in memory, measuring its token table
3467/// on the way past. `section` is `None` when the file carries no text index.
3468fn resident_text_index_slot(section: Option<&[u8]>, codec: u8) -> Result<TextIndexSlot, FileError> {
3469    let Some(section) = section else {
3470        return Ok(TextIndexSlot::Resident {
3471            index: None,
3472            token_table_len: None,
3473        });
3474    };
3475    Ok(TextIndexSlot::Resident {
3476        index: Some(
3477            crate::text_index::TextIndex::from_section(section, codec)
3478                .map_err(|_| FileError::Container("malformed text index"))?,
3479        ),
3480        token_table_len: crate::text_index::TextIndex::postings_base(section).map(|b| b as u64),
3481    })
3482}
3483
3484/// Byte length of the TEXT_INDEX section's leading **token table**, from a
3485/// header a caller has already parsed — the prefix range a first
3486/// [`Rete::text_search`] fetches, and the honest figure to quote as its cost
3487/// (`header.text_index_len` counts the postings blob too and overstates it
3488/// several-fold). `None` when the file carries no text index, or its first
3489/// bytes could not be read.
3490///
3491/// Costs **one range read of ≤10 bytes** — the section's leading length varint
3492/// and nothing else, never the table it measures. This is the free-standing
3493/// companion to [`Rete::text_index_token_table_len`], for readers that hold a
3494/// [`Header`] and a [`RangeReader`] but never open the file (the CARD tier:
3495/// `rete card`, `rete card-url`).
3496pub fn read_text_index_token_table_len_ranged<R: RangeReader + ?Sized>(
3497    reader: &R,
3498    header: &Header,
3499) -> Option<u64> {
3500    if crate::v06::reads(header.version) {
3501        return crate::v06::open::read_token_table_bytes(reader);
3502    }
3503    let (ti_off, ti_len) = header.text_index_payload();
3504    if ti_len == 0 {
3505        return None;
3506    }
3507    read_token_table_len(reader, ti_off, ti_len)
3508}
3509
3510/// Byte length of the TEXT_INDEX section's leading token table — its length
3511/// varint plus the compressed table, i.e. the prefix range a first search
3512/// fetches. Reads the section's first ≤10 bytes and nothing else.
3513fn read_token_table_len<R: RangeReader + ?Sized>(reader: &R, off: u64, len: u64) -> Option<u64> {
3514    let head = reader.read_at(off, 10u64.min(len)).ok()?;
3515    let (ttlen, n) = crate::varint::read_uvarint(&head)?;
3516    Some((n as u64 + ttlen).min(len))
3517}
3518
3519/// A named graph's container decodes RESIDENT (one range read, one decode) up
3520/// to this size on the lazy path; a larger container opens as a remote-lazy
3521/// tile index instead, so one selective `GRAPH <g>` query over a multi-GB
3522/// graph faults only the tile directories plus the tiles it touches.
3523const NAMED_GRAPH_RESIDENT_MAX: u64 = 1 << 20; // 1 MiB
3524
3525/// Largest quad-index graph (in quads) a many-graph walk evaluates through an
3526/// owned, dropped view ([`Rete::named_graph_transient`]) rather than a memoised
3527/// one. A view is ~1–2 KB of descriptors whatever the graph's size, so keeping
3528/// one per graph is what grows with the graph count; past this size a graph is
3529/// worth memoising, and its rows are no longer worth materializing per graph.
3530pub const TRANSIENT_GRAPH_QUADS: u64 = 1024;
3531
3532/// Entries per lazily-allocated named-graph slab. The outer table is sized by
3533/// the section's leading count varint, which is UNTRUSTED — slabbing keeps a
3534/// hostile count from ballooning the allocation (the outer table costs a few
3535/// bytes per claimed slab; entry storage only materializes when touched).
3536const NAMED_SLAB: usize = 1024;
3537
3538/// First read of the named-graph walk, and the floor its ramp resets to. One
3539/// chunk covers a few dozen small-graph records, so a `LIMIT`ed `GRAPH ?g`
3540/// pays exactly one small read — the targeted-query win this size protects.
3541const NAMED_WALK_CHUNK: u64 = 64 * 1024;
3542
3543/// Ceiling of the walk's geometric read ramp — the read size an exhaustive
3544/// walk converges to (and starts at, when the demand is known exhaustive).
3545///
3546/// Sized from measurement, not taste. Against the nkod reference file
3547/// (67.9 MB, 31,974 named graphs, 65.4 MB section) the fixed 64 KiB walk cost
3548/// 262 requests / ~16.5 s in Chromium where the old eager whole-section fetch
3549/// cost ~8.5 s — i.e. ~31 ms of per-request overhead on ~8 s of transfer
3550/// (~7.7 MB/s). At 8 MiB per read a chunk transfers in ~1 s, putting the
3551/// request overhead near 3% of total time; halving the cap to 4 MiB doubles
3552/// that overhead, doubling the cap to 16 MiB buys back under 2% while doubling
3553/// both the walk buffer a wasm32 session must hold resident and the worst-case
3554/// over-read past the point a query stops. 8 MiB is the knee.
3555const NAMED_WALK_CHUNK_MAX: u64 = 8 * 1024 * 1024;
3556
3557/// One named graph, walked lazily out of the NAMED_GRAPHS section.
3558#[derive(Default)]
3559struct NamedEntry {
3560    /// `(iri, absolute byte range of the graph's index container)` — filled
3561    /// when the sequential directory walk reaches this slot.
3562    meta: std::sync::OnceLock<(String, ByteRange)>,
3563    /// The graph's permutation index, opened on first access. Boxed: most
3564    /// entries of a many-graph dataset are never touched by a given query.
3565    index: std::sync::OnceLock<Box<GraphIndex>>,
3566}
3567
3568/// Sequential position of the lazy named-graphs walk. The section has no
3569/// random-access directory — each entry's offset is the previous entry's end —
3570/// so entries are parsed strictly in order and memoised.
3571#[derive(Default)]
3572struct NamedWalk {
3573    /// Next entry index to parse.
3574    next: usize,
3575    /// Absolute file offset of that entry's header; `0` = the leading count
3576    /// varint has not been consumed yet (offset 0 is the file header, never
3577    /// inside a section).
3578    pos: u64,
3579    /// Read buffer carried ACROSS walk calls: `buf` holds the section bytes at
3580    /// `[buf_off, buf_off + buf.len())`. Entry-by-entry iteration (the
3581    /// `GRAPH ?g` path advances one entry per call) parses ~30 small-graph
3582    /// headers per 64 KiB chunk instead of re-reading a chunk per entry.
3583    /// [`LazyNamedGraphs::open_graph`] also serves whole small containers out
3584    /// of it, so a bulk chunk's ride-along payloads are never re-fetched.
3585    buf: Vec<u8>,
3586    buf_off: u64,
3587    /// Current read size of the geometric ramp (`0` = not started; treated as
3588    /// [`NAMED_WALK_CHUNK`]). Doubles on every buffer refill up to
3589    /// [`NAMED_WALK_CHUNK_MAX`], so a walk that keeps going converges to bulk
3590    /// reads after fetching at most as many bytes as it already used (the
3591    /// doubling bound: wasted read-ahead ≤ useful bytes). Reset to the floor
3592    /// whenever an oversized container is parsed — see `big_seen`.
3593    chunk: u64,
3594    /// A container larger than [`NAMED_GRAPH_RESIDENT_MAX`] has been parsed.
3595    /// Such graphs open tile-lazily (their payload bytes are NOT wanted by the
3596    /// walk), so ride-along prefetch stops paying off: the ramp resets and the
3597    /// exhaustive-demand hint stops seeding full-size reads. Without this, a
3598    /// header walk over a file of multi-MB containers would fetch the payloads
3599    /// it exists to hop over.
3600    big_seen: bool,
3601}
3602
3603/// The NAMED_GRAPHS section opened lazily over a [`RangeReader`]: nothing is
3604/// fetched at open. The section is a count varint followed by
3605/// `iri_len | iri | container_len | container` records, so the directory is
3606/// walked forward on demand — header bytes only, skipping container payloads —
3607/// and each touched graph's index is decoded (or opened tile-lazily) on first
3608/// access. A `LIMIT`ed `GRAPH ?g` query therefore reads a prefix of the
3609/// section; only a query that genuinely touches every graph (e.g. a full
3610/// `COUNT` over `GRAPH ?g`) walks it all — which is exactly what the eager
3611/// open used to fetch unconditionally.
3612///
3613/// **Failure contract** (same as lazy tiles/chunks): a failed read or a
3614/// malformed record sets a sticky flag and the accessor returns `None`;
3615/// nothing failed is ever memoised, so a later evaluation retries.
3616struct LazyNamedGraphs {
3617    reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
3618    /// Absolute byte range of the NAMED_GRAPHS section.
3619    section: ByteRange,
3620    codec: u8,
3621    has_synopsis: bool,
3622    read_concurrency: usize,
3623    /// The file's permutation set — every graph container in a file carries
3624    /// the same one, so it comes from the header rather than per graph.
3625    perms: PermSet,
3626    /// `(count, slab table)` — set once the leading count varint is read.
3627    /// Slabs allocate on first touch; boxed slices never move, so `&` handed
3628    /// out to entries stay valid for `&self`'s lifetime with no unsafe.
3629    #[allow(clippy::type_complexity)]
3630    dir: std::sync::OnceLock<(usize, Box<[std::sync::OnceLock<Box<[NamedEntry]>>]>)>,
3631    walk: std::sync::Mutex<NamedWalk>,
3632    failed: std::sync::atomic::AtomicBool,
3633    /// Tile-cache byte cap applied to each LARGE (lazily-opened, loader-bearing)
3634    /// named graph as it is opened (bounded-export phase 2). `u64::MAX` =
3635    /// unlimited. Small resident graphs (no loader, ≤ 1 MiB) are never capped —
3636    /// an evicted tile could not be reloaded — but they are bounded by size and
3637    /// released per slot, so they need no cap.
3638    tile_cap: std::sync::atomic::AtomicU64,
3639}
3640
3641impl LazyNamedGraphs {
3642    fn new(
3643        reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
3644        section: ByteRange,
3645        codec: u8,
3646        has_synopsis: bool,
3647        read_concurrency: usize,
3648        perms: PermSet,
3649    ) -> Self {
3650        LazyNamedGraphs {
3651            reader,
3652            section,
3653            codec,
3654            has_synopsis,
3655            read_concurrency,
3656            perms,
3657            dir: std::sync::OnceLock::new(),
3658            walk: std::sync::Mutex::new(NamedWalk::default()),
3659            failed: std::sync::atomic::AtomicBool::new(false),
3660            tile_cap: std::sync::atomic::AtomicU64::new(u64::MAX),
3661        }
3662    }
3663
3664    /// Set the tile-cache cap for large named graphs: records it for graphs
3665    /// opened later, and re-caps every large graph already opened. Small
3666    /// resident (loaderless) graphs are left unlimited.
3667    fn set_tile_cap(&self, cap: u64) {
3668        self.tile_cap
3669            .store(cap, std::sync::atomic::Ordering::Relaxed);
3670        self.for_each_opened(|g| {
3671            if g.is_lazy() {
3672                g.set_cache_cap(cap);
3673            }
3674        });
3675    }
3676
3677    /// Record a fetch/parse failure (checked by `index_incomplete`).
3678    fn fail(&self) {
3679        self.failed
3680            .store(true, std::sync::atomic::Ordering::Relaxed);
3681    }
3682
3683    /// The directory scaffold, reading the leading count varint on first call.
3684    #[allow(clippy::type_complexity)]
3685    fn directory(&self) -> Option<&(usize, Box<[std::sync::OnceLock<Box<[NamedEntry]>>]>)> {
3686        if let Some(d) = self.dir.get() {
3687            return Some(d);
3688        }
3689        let end = self.section.offset.checked_add(self.section.len)?;
3690        let (n, used) = match read_uvarint_at(&self.reader, self.section.offset, end) {
3691            Ok(v) => v,
3692            Err(_) => {
3693                self.fail();
3694                return None;
3695            }
3696        };
3697        // Each record needs at least 2 bytes of framing (two length varints),
3698        // so a count beyond len/2 is malformed — refuse before sizing by it.
3699        if n > self.section.len / 2 {
3700            self.fail();
3701            return None;
3702        }
3703        let Ok(n) = usize::try_from(n) else {
3704            self.fail();
3705            return None;
3706        };
3707        let slabs = n.div_ceil(NAMED_SLAB);
3708        let table: Box<[std::sync::OnceLock<Box<[NamedEntry]>>]> =
3709            (0..slabs).map(|_| std::sync::OnceLock::new()).collect();
3710        let _ = self.dir.set((n, table));
3711        {
3712            // Idempotent under an init race: both racers computed the same
3713            // `used` from the same bytes.
3714            let mut w = self.walk.lock().unwrap();
3715            if w.pos == 0 {
3716                w.pos = self.section.offset + used;
3717            }
3718        }
3719        self.dir.get()
3720    }
3721
3722    fn count(&self) -> usize {
3723        self.directory().map(|(n, _)| *n).unwrap_or(0)
3724    }
3725
3726    /// The `i`-th entry's cell (allocating its slab on first touch).
3727    fn entry(&self, i: usize) -> Option<&NamedEntry> {
3728        let (n, table) = self.directory()?;
3729        if i >= *n {
3730            return None;
3731        }
3732        let slab = table[i / NAMED_SLAB].get_or_init(|| {
3733            let len = NAMED_SLAB.min(n - (i / NAMED_SLAB) * NAMED_SLAB);
3734            (0..len).map(|_| NamedEntry::default()).collect()
3735        });
3736        slab.get(i % NAMED_SLAB)
3737    }
3738
3739    /// The next walk read's length, advancing the geometric ramp: at least
3740    /// `need` (a long IRI's header), at most the section remainder, and
3741    /// otherwise the current ramp value — which doubles per refill up to
3742    /// [`NAMED_WALK_CHUNK_MAX`]. The doubling keeps a stopping walk's wasted
3743    /// read-ahead bounded by the bytes it already used, while a walk that
3744    /// keeps going converges to bulk reads: the runtime safety net for
3745    /// exhaustive-looking queries the demand analysis could not prove.
3746    fn next_read_len(w: &mut NamedWalk, pos: u64, end: u64, need: u64) -> u64 {
3747        let cur = w.chunk.max(NAMED_WALK_CHUNK);
3748        w.chunk = (cur * 2).min(NAMED_WALK_CHUNK_MAX);
3749        cur.max(need).min(end - pos)
3750    }
3751
3752    /// Walk entry headers forward until `entries[upto].meta` is set. Reads the
3753    /// section in chunks and hops over container payloads, so the walk fetches
3754    /// O(headers) — not O(section) — where containers are large; small
3755    /// containers ride along in the same chunk and cost nothing extra.
3756    ///
3757    /// `exhaustive` is the demand hint from the query engine: the caller has
3758    /// PROVEN it will visit and open every named graph (an unrestricted
3759    /// `GRAPH ?g` under a consumer that drains the pipeline), so the walk
3760    /// starts at full-size reads instead of ramping up to them — the whole
3761    /// section arrives in `section/8 MiB` requests instead of hundreds. The
3762    /// hint is per-call, never memoised: a later targeted query on the same
3763    /// handle walks small again. It also defers to the layout: once an
3764    /// oversized (tile-lazy) container is seen, ride-along bytes stop being
3765    /// useful and the hint no longer seeds bulk reads.
3766    fn ensure_meta(&self, upto: usize, exhaustive: bool) -> Option<()> {
3767        let target = self.entry(upto)?;
3768        if target.meta.get().is_some() {
3769            return Some(());
3770        }
3771        let end = self.section.offset.checked_add(self.section.len)?;
3772        let mut w = self.walk.lock().unwrap();
3773        if exhaustive && !w.big_seen {
3774            w.chunk = NAMED_WALK_CHUNK_MAX;
3775        }
3776        while w.next <= upto {
3777            let pos = w.pos;
3778            if pos >= end {
3779                // The count varint promised more entries than the bytes hold.
3780                self.fail();
3781                return None;
3782            }
3783            // Make sure the whole header (two varints + the IRI, ≤ 20+iri_len
3784            // bytes) is buffered; refetch a larger window when the IRI is long.
3785            let have =
3786                |b: &[u8], off: u64, need: u64| pos >= off && pos + need <= off + b.len() as u64;
3787            if !have(&w.buf, w.buf_off, 20.min(end - pos)) {
3788                let len = Self::next_read_len(&mut w, pos, end, 20.min(end - pos));
3789                w.buf = match self.reader.read_at(pos, len) {
3790                    Ok(b) => b,
3791                    Err(_) => {
3792                        self.fail();
3793                        return None;
3794                    }
3795                };
3796                w.buf_off = pos;
3797            }
3798            // Every slice of `buf` below is checked: a reader may return fewer
3799            // bytes than asked, and the lengths are untrusted.
3800            let rel = (pos - w.buf_off) as usize;
3801            let Some((ilen, u1)) = w.buf.get(rel..).and_then(read_uvarint) else {
3802                self.fail();
3803                return None;
3804            };
3805            let Some(iri_end) = (u1 as u64)
3806                .checked_add(ilen)
3807                .and_then(|n| pos.checked_add(n))
3808                .filter(|&e| e <= end)
3809            else {
3810                self.fail(); // IRI overruns the section: malformed
3811                return None;
3812            };
3813            // iri_len + iri + container_len varint
3814            let header_need = (iri_end - pos).saturating_add(10);
3815            if !have(&w.buf, w.buf_off, header_need.min(end - pos)) {
3816                let len = Self::next_read_len(&mut w, pos, end, header_need);
3817                w.buf = match self.reader.read_at(pos, len) {
3818                    Ok(b) => b,
3819                    Err(_) => {
3820                        self.fail();
3821                        return None;
3822                    }
3823                };
3824                w.buf_off = pos;
3825            }
3826            let rel = (pos - w.buf_off) as usize;
3827            let istart = rel + u1;
3828            let Some(iend) = usize::try_from(ilen)
3829                .ok()
3830                .and_then(|l| istart.checked_add(l))
3831                .filter(|&e| e <= w.buf.len())
3832            else {
3833                self.fail(); // short read: the header is not all here
3834                return None;
3835            };
3836            let iri = String::from_utf8_lossy(&w.buf[istart..iend]).into_owned();
3837            let Some((clen, u2)) = read_uvarint(&w.buf[iend..]) else {
3838                self.fail();
3839                return None;
3840            };
3841            let cend = match iri_end
3842                .checked_add(u2 as u64)
3843                .and_then(|cstart| cstart.checked_add(clen))
3844            {
3845                Some(e) if e <= end => e,
3846                _ => {
3847                    self.fail(); // container overruns the section: malformed
3848                    return None;
3849                }
3850            };
3851            if clen > NAMED_GRAPH_RESIDENT_MAX {
3852                // This graph opens tile-lazily: the walk exists to HOP its
3853                // payload, so big read-ahead here is pure over-fetch. Drop the
3854                // ramp back to the floor and stop the exhaustive hint from
3855                // re-seeding it (headers stay cheap on payload-heavy files).
3856                w.big_seen = true;
3857                w.chunk = NAMED_WALK_CHUNK;
3858            }
3859            let range = ByteRange {
3860                offset: cend - clen,
3861                len: clen,
3862            };
3863            if let Some(e) = self.entry(w.next) {
3864                let _ = e.meta.set((iri, range));
3865            }
3866            w.pos = cend;
3867            w.next += 1;
3868        }
3869        Some(())
3870    }
3871
3872    /// The `i`-th graph's IRI — a header walk, never an index decode.
3873    /// `exhaustive`: see [`ensure_meta`](Self::ensure_meta).
3874    fn name_at(&self, i: usize, exhaustive: bool) -> Option<&str> {
3875        self.ensure_meta(i, exhaustive)?;
3876        self.entry(i)?.meta.get().map(|(iri, _)| iri.as_str())
3877    }
3878
3879    /// The `i`-th graph as `(iri, index)`, opening the index on first access.
3880    /// `exhaustive`: see [`ensure_meta`](Self::ensure_meta).
3881    fn graph_at(&self, i: usize, exhaustive: bool) -> Option<(&str, &GraphIndex)> {
3882        self.ensure_meta(i, exhaustive)?;
3883        let e = self.entry(i)?;
3884        let (iri, range) = e.meta.get()?;
3885        if e.index.get().is_none() {
3886            let opened = self.open_graph(*range)?;
3887            let _ = e.index.set(Box::new(opened));
3888        }
3889        Some((iri.as_str(), e.index.get()?.as_ref()))
3890    }
3891
3892    /// A small container's bytes: out of the walk buffer when the last chunk
3893    /// already carried them (bulk chunks always do — that is their point),
3894    /// else one targeted range read. Never called for tile-lazy containers.
3895    fn container_bytes(&self, range: ByteRange) -> Option<Vec<u8>> {
3896        {
3897            let w = self.walk.lock().unwrap();
3898            let buf_end = w.buf_off + w.buf.len() as u64;
3899            if range.offset >= w.buf_off && range.offset + range.len <= buf_end {
3900                let a = (range.offset - w.buf_off) as usize;
3901                return Some(w.buf[a..a + range.len as usize].to_vec());
3902            }
3903        }
3904        match self.reader.read_at(range.offset, range.len) {
3905            // A short read is a failed read, not a shorter container.
3906            Ok(b) if b.len() as u64 == range.len => Some(b),
3907            _ => {
3908                self.fail();
3909                None
3910            }
3911        }
3912    }
3913
3914    /// Open one graph's index container: small ones resident (one read),
3915    /// large ones as a remote-lazy tile index — the same machinery as the
3916    /// default graph, so a selective query over a huge named graph fetches
3917    /// only the tile directories plus the tiles it touches.
3918    fn open_graph(&self, range: ByteRange) -> Option<GraphIndex> {
3919        if range.len <= NAMED_GRAPH_RESIDENT_MAX {
3920            let bytes = self.container_bytes(range)?;
3921            match decode_index_container(&bytes, self.codec, self.perms) {
3922                Ok(g) => {
3923                    // Resident, so counting is a pass over tile headers: the
3924                    // planner sizes this graph's patterns by it.
3925                    g.set_size_hint(u64::from(g.triple_count()));
3926                    Some(g)
3927                }
3928                Err(_) => {
3929                    self.fail();
3930                    None
3931                }
3932            }
3933        } else {
3934            match open_index_container_lazy(
3935                &self.reader,
3936                range,
3937                self.codec,
3938                self.has_synopsis,
3939                self.read_concurrency,
3940                self.perms,
3941                self.perms.len(),
3942            ) {
3943                Ok((g, _, _)) => {
3944                    // A large graph faults its tiles through a loader, so bound
3945                    // its tile residency to the budget's share (an evicted tile
3946                    // simply re-faults). No-op when unlimited.
3947                    let cap = self.tile_cap.load(std::sync::atomic::Ordering::Relaxed);
3948                    if cap != u64::MAX {
3949                        g.set_cache_cap(cap);
3950                    }
3951                    Some(g)
3952                }
3953                Err(_) => {
3954                    self.fail();
3955                    None
3956                }
3957            }
3958        }
3959    }
3960
3961    /// Drop the decoded index of the graph named `iri`, if it has been opened,
3962    /// so its tiles and per-graph tile cache are freed. Takes `&mut self`:
3963    /// `OnceLock::take` needs unique access, which the export loop has between
3964    /// slots. Leaves the entry's `meta` (iri + range) so the graph re-opens
3965    /// lazily if touched again. A no-op if the graph is absent or never opened.
3966    fn release(&mut self, iri: &str) {
3967        let Some((_, table)) = self.dir.get_mut() else {
3968            return;
3969        };
3970        for slab in table.iter_mut() {
3971            let Some(entries) = slab.get_mut() else {
3972                continue;
3973            };
3974            for e in entries.iter_mut() {
3975                let matches = e
3976                    .meta
3977                    .get()
3978                    .map(|(name, _)| name.as_str() == iri)
3979                    .unwrap_or(false);
3980                if matches {
3981                    e.index.take();
3982                    return;
3983                }
3984            }
3985        }
3986    }
3987
3988    /// Visit every already-opened graph index (never triggers a fetch).
3989    fn for_each_opened(&self, mut f: impl FnMut(&GraphIndex)) {
3990        if let Some((_, table)) = self.dir.get() {
3991            for slab in table.iter().filter_map(|s| s.get()) {
3992                for e in slab.iter() {
3993                    if let Some(g) = e.index.get() {
3994                        f(g);
3995                    }
3996                }
3997            }
3998        }
3999    }
4000}
4001
4002/// The NAMED_GRAPHS section, held resident (in-memory and eager ranged opens —
4003/// unchanged) or walked lazily (the lazy ranged open, where fetching and
4004/// decoding every graph's index up front defeated remote laziness: 67 MB
4005/// fetched and ~32k indexes built before the first query on a many-graph file).
4006enum NamedGraphsSlot {
4007    Resident(Vec<(String, GraphIndex)>),
4008    Lazy(LazyNamedGraphs),
4009    /// The quad index (kind 10), on every open path: graphs are views over
4010    /// shared tiles, found through the dictionary and the sorted directory.
4011    Quad(crate::quad::QuadNamed),
4012}
4013
4014impl NamedGraphsSlot {
4015    /// Heap bytes the named-graph directory holds outside any cache: every
4016    /// walked entry's IRI and range, the walk buffer, and the directories of
4017    /// the graph indexes currently open (see [`GraphIndex::directory_bytes`]).
4018    fn directory_bytes(&self) -> u64 {
4019        match self {
4020            NamedGraphsSlot::Resident(v) => v
4021                .iter()
4022                .map(|(iri, g)| iri.capacity() as u64 + g.directory_bytes())
4023                .sum(),
4024            NamedGraphsSlot::Lazy(l) => {
4025                let mut bytes = l.walk.lock().map(|w| w.buf.capacity() as u64).unwrap_or(0);
4026                if let Some((_, table)) = l.dir.get() {
4027                    bytes += table.len() as u64
4028                        * std::mem::size_of::<std::sync::OnceLock<Box<[NamedEntry]>>>() as u64;
4029                    for slab in table.iter().filter_map(|s| s.get()) {
4030                        bytes += slab.len() as u64 * std::mem::size_of::<NamedEntry>() as u64;
4031                        for e in slab.iter() {
4032                            if let Some((iri, _)) = e.meta.get() {
4033                                bytes += iri.capacity() as u64;
4034                            }
4035                            if let Some(g) = e.index.get() {
4036                                bytes += g.directory_bytes();
4037                            }
4038                        }
4039                    }
4040                }
4041                bytes
4042            }
4043            NamedGraphsSlot::Quad(q) => q.directory_bytes(),
4044        }
4045    }
4046
4047    fn count(&self) -> usize {
4048        match self {
4049            NamedGraphsSlot::Resident(v) => v.len(),
4050            NamedGraphsSlot::Lazy(l) => l.count(),
4051            NamedGraphsSlot::Quad(q) => q.count(),
4052        }
4053    }
4054
4055    /// `exhaustive` is the walk-demand hint (resident opens ignore it): see
4056    /// [`LazyNamedGraphs::ensure_meta`]. `dict` names the graphs of a quad
4057    /// index (their names are dictionary terms there).
4058    fn name_at<'a>(&'a self, dict: &Dictionary, i: usize, exhaustive: bool) -> Option<&'a str> {
4059        match self {
4060            NamedGraphsSlot::Resident(v) => v.get(i).map(|(iri, _)| iri.as_str()),
4061            NamedGraphsSlot::Lazy(l) => l.name_at(i, exhaustive),
4062            NamedGraphsSlot::Quad(q) => q.name_at(dict, i, exhaustive),
4063        }
4064    }
4065
4066    /// [`name_at`](Self::name_at) without keeping the name: a quad-index walk
4067    /// over many graphs must not memoise one string per graph it passes.
4068    fn name_owned(&self, dict: &Dictionary, i: usize, exhaustive: bool) -> Option<String> {
4069        match self {
4070            NamedGraphsSlot::Quad(q) => q.name_owned(dict, i, exhaustive),
4071            _ => self.name_at(dict, i, exhaustive).map(str::to_owned),
4072        }
4073    }
4074
4075    fn graph_at<'a>(
4076        &'a self,
4077        dict: &Dictionary,
4078        i: usize,
4079        exhaustive: bool,
4080    ) -> Option<(&'a str, &'a GraphIndex)> {
4081        match self {
4082            NamedGraphsSlot::Resident(v) => v.get(i).map(|(iri, g)| (iri.as_str(), g)),
4083            NamedGraphsSlot::Lazy(l) => l.graph_at(i, exhaustive),
4084            NamedGraphsSlot::Quad(q) => q.graph_at(dict, i, exhaustive),
4085        }
4086    }
4087
4088    fn find<'a>(&'a self, dict: &Dictionary, iri: &str) -> Option<&'a GraphIndex> {
4089        match self {
4090            NamedGraphsSlot::Quad(q) => q.find(dict, iri),
4091            NamedGraphsSlot::Resident(v) => v.iter().find(|(name, _)| name == iri).map(|(_, g)| g),
4092            NamedGraphsSlot::Lazy(l) => {
4093                // Walk headers only until the IRI matches; decode just that
4094                // graph's index. An absent IRI costs a full header walk — the
4095                // layout has no by-name directory. A point lookup is targeted
4096                // demand: never hint the walk exhaustive.
4097                for i in 0..l.count() {
4098                    if l.name_at(i, false)? == iri {
4099                        return l.graph_at(i, false).map(|(_, g)| g);
4100                    }
4101                }
4102                None
4103            }
4104        }
4105    }
4106
4107    fn load_incomplete(&self) -> bool {
4108        match self {
4109            NamedGraphsSlot::Resident(v) => v.iter().any(|(_, g)| g.load_incomplete()),
4110            NamedGraphsSlot::Lazy(l) => {
4111                if l.failed.load(std::sync::atomic::Ordering::Relaxed) {
4112                    return true;
4113                }
4114                let mut bad = false;
4115                l.for_each_opened(|g| bad |= g.load_incomplete());
4116                bad
4117            }
4118            NamedGraphsSlot::Quad(q) => q.load_incomplete(),
4119        }
4120    }
4121
4122    fn reset_load_failures(&self) {
4123        match self {
4124            NamedGraphsSlot::Resident(v) => {
4125                for (_, g) in v {
4126                    g.reset_load_failure();
4127                }
4128            }
4129            NamedGraphsSlot::Lazy(l) => {
4130                l.failed.store(false, std::sync::atomic::Ordering::Relaxed);
4131                l.for_each_opened(|g| g.reset_load_failure());
4132            }
4133            NamedGraphsSlot::Quad(q) => q.reset_load_failures(),
4134        }
4135    }
4136
4137    /// The named-graph slot of a file opened with all its bytes in hand
4138    /// (`Rete::open`, `open_ranged`): the per-graph section decoded resident,
4139    /// or the quad index read through an owned copy of its section.
4140    fn resident(
4141        header: &Header,
4142        quad_section: impl FnOnce(u64, u64) -> Result<Vec<u8>, FileError>,
4143        per_graph: impl FnOnce() -> Result<Vec<(String, GraphIndex)>, FileError>,
4144    ) -> Result<Self, FileError> {
4145        check_named_layout(header)?;
4146        if let Some(q) = header.quad_index_section() {
4147            let bytes = quad_section(q.offset, q.length)?;
4148            return Ok(NamedGraphsSlot::Quad(crate::quad::QuadNamed::resident(
4149                bytes,
4150                header.block_codec,
4151                header.perms,
4152            )));
4153        }
4154        Ok(NamedGraphsSlot::Resident(if header.named_graphs_len > 0 {
4155            per_graph()?
4156        } else {
4157            Vec::new()
4158        }))
4159    }
4160}
4161
4162/// A read-only, in-memory view over a `.rete` file image.
4163pub struct Rete {
4164    header: Header,
4165    dict: Dictionary,
4166    index: std::sync::Arc<GraphIndex>,
4167    index_section_ranges: [ByteRange; NUM_PERMS],
4168    /// Per-permutation tile directories as absolute file ranges
4169    /// (`(min_a, max_a, compressed-tile range)`), for provenance. Empty for
4170    /// pre-tiling (v0.1) files.
4171    tile_ranges: [Vec<(u32, u32, ByteRange)>; NUM_PERMS],
4172    pyramid: PyramidSlot,
4173    text_index: TextIndexSlot,
4174    /// The LABEL_INDEX section (kind 8), if any.
4175    labels: SectionSlot<crate::label_index::LabelIndex>,
4176    /// The SCHEMA section (kind 9), if any.
4177    schema: SectionSlot<crate::schema_section::SchemaMeta>,
4178    named_graphs: NamedGraphsSlot,
4179    /// Raw bytes of the metadata section (empty if the file has none). The
4180    /// application layer decodes this (the CLI stores a JSON Dataset Card here).
4181    /// Only [`Rete::open`] populates it; [`Rete::open_ranged`] leaves it empty to
4182    /// preserve its minimal-fetch budget.
4183    metadata: Vec<u8>,
4184    /// Executes remote `SERVICE` blocks (SPARQL 1.1 federated query) — attached
4185    /// by the host via [`Rete::set_service_client`]; `None` means a non-SILENT
4186    /// `SERVICE` fails the query. Like the range readers, the engine never does
4187    /// I/O itself.
4188    service_client: Option<Box<dyn crate::service::ServiceClient>>,
4189    /// First failed non-SILENT `SERVICE` call of the current query. The row
4190    /// pipeline is infallible (the same contract as lazy tile fetches), so the
4191    /// failure is recorded here and taken by the top-level eval entry points.
4192    service_error: std::sync::Mutex<Option<String>>,
4193    /// Bytes a query's `COUNT(DISTINCT …)` set may hold before it spills to
4194    /// disk — see [`Rete::spill_threshold`].
4195    spill_threshold: std::sync::atomic::AtomicU64,
4196    /// The file is read lazily through a reader whose reads are round trips
4197    /// (not [`RangeReader::is_local`]): see [`Rete::reads_are_remote`].
4198    remote_reads: std::sync::atomic::AtomicBool,
4199    /// The open `0x06` file (header, root, reader), for a `0x06` open.
4200    v06: Option<std::sync::Arc<crate::v06::open::V06File>>,
4201}
4202
4203impl Rete {
4204    /// Parse a full file image (v0 loads everything; a range-reading client
4205    /// will fetch only the sections it needs — same container format).
4206    pub fn open(bytes: &[u8]) -> Result<Self, FileError> {
4207        #[cfg(feature = "v06-parity")]
4208        if let Some(v6) = parity_image(bytes) {
4209            return Self::open(&v6);
4210        }
4211        if crate::v06::open::is_v06(bytes) {
4212            // A 0x06 image opens through the lazy machinery over the bytes in
4213            // memory: the same structures, faulted from RAM.
4214            let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
4215                std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(bytes)));
4216            let mut rete = Self::open_v06(r, bytes, false)?;
4217            rete.metadata = rete
4218                .v06
4219                .as_ref()
4220                .and_then(|f| f.card().ok().flatten())
4221                .unwrap_or_default();
4222            return Ok(rete);
4223        }
4224        let header = Header::from_bytes(bytes)?;
4225
4226        // Header offsets/lengths are untrusted (a `.rete` may be fetched truncated
4227        // or corrupt from an arbitrary URL). Slice through a checked helper so a
4228        // bad region yields an error instead of panicking on an OOB index.
4229        let region = |off: u64, len: u64| -> Result<&[u8], FileError> {
4230            checked_slice(bytes, off, len)
4231                .ok_or(FileError::Container("section range out of bounds"))
4232        };
4233
4234        let dict = decode_dictionary_container(
4235            region(header.dictionary_offset, header.dictionary_len)?,
4236            header.dict_codec,
4237            header.term_order(),
4238        )?;
4239
4240        check_named_layout(&header)?;
4241        let index_bytes = region(header.root_dir_offset, header.root_dir_len)?;
4242        let (index, _) = decode_root_index_container(index_bytes, &header)?;
4243        let index_section_ranges = decode_index_section_ranges(
4244            index_bytes,
4245            header.root_dir_offset,
4246            header.perms,
4247            header.index_container_sections(),
4248        )?;
4249
4250        let pyramid = PyramidSlot::Resident(if header.pyramid_meta_len > 0 {
4251            Some(
4252                PyramidMeta::decode(region(header.pyramid_meta_offset, header.pyramid_meta_len)?)
4253                    .map_err(|_| FileError::Container("malformed pyramid meta"))?,
4254            )
4255        } else {
4256            None
4257        });
4258
4259        // The TEXT_INDEX section (opt-in `--text-index`); decode the whole thing
4260        // resident on a full-image open.
4261        let (ti_off, ti_len) = header.text_index_payload();
4262        let text_index = resident_text_index_slot(
4263            if ti_len > 0 {
4264                Some(region(ti_off, ti_len)?)
4265            } else {
4266                None
4267            },
4268            header.block_codec,
4269        )?;
4270        let labels = resident_label_slot(&header, &region);
4271        let schema = resident_schema_slot(&header, &region);
4272
4273        let named_graphs = NamedGraphsSlot::resident(
4274            &header,
4275            |off, len| Ok(region(off, len)?.to_vec()),
4276            || {
4277                decode_named_graphs(
4278                    region(header.named_graphs_offset, header.named_graphs_len)?,
4279                    header.block_codec,
4280                    header.perms,
4281                )
4282            },
4283        )?;
4284
4285        let metadata = if header.metadata_len > 0 {
4286            region(header.metadata_offset, header.metadata_len)?.to_vec()
4287        } else {
4288            Vec::new()
4289        };
4290
4291        let tile_ranges =
4292            tile_file_ranges(index_bytes, header.root_dir_offset, &index_section_ranges);
4293        Ok(Self {
4294            header,
4295            dict,
4296            index: std::sync::Arc::new(index),
4297            index_section_ranges,
4298            tile_ranges,
4299            pyramid,
4300            text_index,
4301            labels,
4302            schema,
4303            named_graphs,
4304            metadata,
4305            service_client: None,
4306            service_error: std::sync::Mutex::new(None),
4307            spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
4308            remote_reads: std::sync::atomic::AtomicBool::new(false),
4309            v06: None,
4310        })
4311    }
4312
4313    /// Attach the client that executes `SERVICE <endpoint> { … }` blocks
4314    /// (SPARQL 1.1 federated query) against remote SPARQL endpoints. Without
4315    /// one, a non-SILENT `SERVICE` fails the query with a clear error and a
4316    /// `SERVICE SILENT` degrades to one empty solution, per the spec.
4317    pub fn set_service_client(&mut self, client: Box<dyn crate::service::ServiceClient>) {
4318        self.service_client = Some(client);
4319    }
4320
4321    pub(crate) fn service_client(&self) -> Option<&dyn crate::service::ServiceClient> {
4322        self.service_client.as_deref()
4323    }
4324
4325    /// Record a failed non-SILENT `SERVICE` call (first error wins).
4326    pub(crate) fn record_service_error(&self, msg: &str) {
4327        let mut e = self.service_error.lock().unwrap();
4328        if e.is_none() {
4329            *e = Some(msg.to_string());
4330        }
4331    }
4332
4333    /// Take (and clear) the pending `SERVICE` failure — called by every
4334    /// top-level eval entry so it can never leak into a later query.
4335    pub(crate) fn take_service_error(&self) -> Option<String> {
4336        self.service_error.lock().unwrap().take()
4337    }
4338
4339    pub fn header(&self) -> &Header {
4340        &self.header
4341    }
4342
4343    /// The file's byte layout, for visualization: header, metadata,
4344    /// dictionary, each index permutation's tile directory and individual
4345    /// tiles, pyramid summary, and named graphs — sorted by offset. Bytes not
4346    /// covered by any segment are container framing (section directories and
4347    /// length fields).
4348    pub fn file_layout(&self) -> Vec<LayoutSegment> {
4349        if let Some(f) = &self.v06 {
4350            return v06_layout(f);
4351        }
4352        let h = &self.header;
4353        let seg = |kind: &'static str, label: String, offset: u64, len: u64| LayoutSegment {
4354            kind,
4355            label,
4356            offset,
4357            len,
4358        };
4359        // The size in the label is DERIVED from the constant that defines the
4360        // span, never typed out: this line read "fixed 128 bytes" — the
4361        // pre-v0.3 header size — while emitting a 1024-byte span, and it is
4362        // what the file explorer shows a reader inspecting a real file.
4363        // `layout_header_label_matches_its_span` pins the two together.
4364        let mut out = vec![seg(
4365            "header",
4366            format!("header (fixed {} bytes)", crate::header::HEADER_LEN),
4367            0,
4368            crate::header::HEADER_LEN as u64,
4369        )];
4370        if h.metadata_len > 0 {
4371            out.push(seg(
4372                "metadata",
4373                "metadata (dataset card)".into(),
4374                h.metadata_offset,
4375                h.metadata_len,
4376            ));
4377        }
4378        out.push(seg(
4379            "dictionary",
4380            "dictionary (4 front-coded term sections)".into(),
4381            h.dictionary_offset,
4382            h.dictionary_len,
4383        ));
4384        for (si, perm) in crate::index::ALL_PERMS.into_iter().enumerate() {
4385            let sec = self.index_section_ranges[si];
4386            if sec.len == 0 {
4387                continue;
4388            }
4389            let first_tile = self.tile_ranges[si]
4390                .first()
4391                .map(|&(_, _, r)| r.offset)
4392                .unwrap_or(sec.offset + sec.len);
4393            if first_tile > sec.offset {
4394                out.push(seg(
4395                    "directory",
4396                    format!("{} tile directory", perm.name()),
4397                    sec.offset,
4398                    first_tile - sec.offset,
4399                ));
4400            }
4401            for (ti, &(min_a, max_a, r)) in self.tile_ranges[si].iter().enumerate() {
4402                out.push(seg(
4403                    "tile",
4404                    format!("{} tile {ti} (leading ids {min_a}..{max_a})", perm.name()),
4405                    r.offset,
4406                    r.len,
4407                ));
4408            }
4409        }
4410        if h.pyramid_meta_len > 0 {
4411            out.push(seg(
4412                "pyramid",
4413                "pyramid summary (communities + superedges)".into(),
4414                h.pyramid_meta_offset,
4415                h.pyramid_meta_len,
4416            ));
4417        }
4418        if h.named_graphs_len > 0 {
4419            out.push(seg(
4420                "named-graphs",
4421                format!("named graphs ({})", self.named_graphs.count()),
4422                h.named_graphs_offset,
4423                h.named_graphs_len,
4424            ));
4425        }
4426        if let Some(s) = h.label_index_section() {
4427            let label = match self.labels.get_if_loaded() {
4428                Some(li) => format!("label index ({} entries)", li.entry_count()),
4429                None => "label index".to_string(),
4430            };
4431            out.push(seg("label-index", label, s.offset, s.length));
4432        }
4433        if let Some(s) = h.schema_section() {
4434            out.push(seg(
4435                "schema",
4436                "schema (classes, query_stats, shapes)".into(),
4437                s.offset,
4438                s.length,
4439            ));
4440        }
4441        if let Some(q) = h.quad_index_section() {
4442            out.push(seg(
4443                "quad-index",
4444                format!("quad index ({} named graphs)", self.named_graphs.count()),
4445                q.offset,
4446                q.length,
4447            ));
4448        }
4449        out.sort_by_key(|s| s.offset);
4450        out
4451    }
4452
4453    /// Raw bytes of the file's metadata section, or `None` if it has none. The
4454    /// CLI stores a JSON Dataset Card here; `rete-core` treats it as opaque.
4455    /// Populated by [`Rete::open`] only — an [`Rete::open_ranged`] view returns
4456    /// `None` here (the card is not fetched on the minimal query path).
4457    pub fn metadata(&self) -> Option<&[u8]> {
4458        if self.metadata.is_empty() {
4459            None
4460        } else {
4461            Some(&self.metadata)
4462        }
4463    }
4464
4465    pub fn dictionary(&self) -> &Dictionary {
4466        &self.dict
4467    }
4468
4469    /// The pyramid metadata (summary graph + tiles), if the file has a pyramid.
4470    pub fn pyramid(&self) -> Option<&PyramidMeta> {
4471        match &self.pyramid {
4472            PyramidSlot::Resident(p) => p.as_ref(),
4473            // Faults the (possibly large) pyramid section on first access only.
4474            PyramidSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
4475        }
4476    }
4477
4478    /// The pyramid metadata **only if already resident or previously faulted** —
4479    /// never triggers a lazy range read. The query planner uses this for
4480    /// cardinality estimation so it is free for an in-memory file and never adds
4481    /// a fetch on the lazy remote path (which defers the pyramid by design).
4482    pub fn pyramid_if_loaded(&self) -> Option<&PyramidMeta> {
4483        match &self.pyramid {
4484            PyramidSlot::Resident(p) => p.as_ref(),
4485            PyramidSlot::Lazy { cell, .. } => cell.get().and_then(|o| o.as_ref()),
4486        }
4487    }
4488
4489    /// The SCHEMA section (kind 9): class hierarchy, level rollups and links,
4490    /// coherence axioms, `query_stats`, characteristic sets. Faults it in (one
4491    /// small range read) on the lazy path. `None` when the file has none — a
4492    /// file built before it existed keeps those fields in its pyramid-meta
4493    /// ([`schema_or_pyramid`](Self::schema_or_pyramid) reads either).
4494    pub fn schema(&self) -> Option<&crate::schema_section::SchemaMeta> {
4495        self.schema.get()
4496    }
4497
4498    /// The schema fields from the SCHEMA section when the file has one, else
4499    /// from the pyramid-meta (faulting either on the lazy path). `None` when
4500    /// the file carries neither.
4501    pub fn schema_or_pyramid(
4502        &self,
4503    ) -> Option<std::borrow::Cow<'_, crate::schema_section::SchemaMeta>> {
4504        if let Some(s) = self.schema() {
4505            return Some(std::borrow::Cow::Borrowed(s));
4506        }
4507        self.pyramid()
4508            .map(|p| std::borrow::Cow::Owned(crate::schema_section::SchemaMeta::from_pyramid(p)))
4509    }
4510
4511    /// Per-predicate planner statistics (`query_stats`): from the SCHEMA
4512    /// section (faulting it on the lazy path — it is small), else from the
4513    /// pyramid-meta when that is resident. Empty when the file has neither.
4514    /// See [`crate::meta::PredStat`].
4515    pub fn predicate_stats(&self) -> &[crate::meta::PredStat] {
4516        if let Some(s) = self.schema().filter(|s| !s.predicate_stats.is_empty()) {
4517            return &s.predicate_stats;
4518        }
4519        self.pyramid_if_loaded()
4520            .map(|p| p.predicate_stats.as_slice())
4521            .unwrap_or(&[])
4522    }
4523
4524    /// The statistics the planner may use **without a read it was not going to
4525    /// make**: the SCHEMA section if it is resident or already faulted — or,
4526    /// with `fault`, after faulting it (one small range, once per session) —
4527    /// else the resident pyramid-meta's block.
4528    pub(crate) fn planner_stats(&self, fault: bool) -> &[crate::meta::PredStat] {
4529        let schema = if fault {
4530            self.schema.get()
4531        } else {
4532            self.schema.get_if_loaded()
4533        };
4534        if let Some(s) = schema.filter(|s| !s.predicate_stats.is_empty()) {
4535            return &s.predicate_stats;
4536        }
4537        self.pyramid_if_loaded()
4538            .map(|p| p.predicate_stats.as_slice())
4539            .unwrap_or(&[])
4540    }
4541
4542    /// The entity shapes (characteristic sets): from the SCHEMA section, else
4543    /// from a resident pyramid-meta. See [`crate::meta::CharSet`].
4544    pub fn char_sets(&self) -> &[crate::meta::CharSet] {
4545        if let Some(s) = self.schema().filter(|s| !s.char_sets.is_empty()) {
4546            return &s.char_sets;
4547        }
4548        self.pyramid_if_loaded()
4549            .map(|p| p.char_sets.as_slice())
4550            .unwrap_or(&[])
4551    }
4552
4553    /// The **legacy** label block of a pyramid-meta (≤ 8,192 entries) — empty
4554    /// when the file has none or the pyramid isn't resident. Files built with
4555    /// `--label-index` carry a LABEL_INDEX section instead; see
4556    /// [`label_index_section`](Self::label_index_section). See
4557    /// [`crate::meta::LabelEntry`].
4558    pub fn label_index(&self) -> &[crate::meta::LabelEntry] {
4559        self.pyramid_if_loaded()
4560            .map(|p| p.label_index.as_slice())
4561            .unwrap_or(&[])
4562    }
4563
4564    /// Whether the file carries a LABEL_INDEX section (kind 8). Header only.
4565    pub fn has_label_index(&self) -> bool {
4566        self.header.label_index_section().is_some()
4567    }
4568
4569    /// The LABEL_INDEX section, faulting its head in on the lazy path.
4570    pub fn label_index_section(&self) -> Option<&crate::label_index::LabelIndex> {
4571        self.labels.get()
4572    }
4573
4574    /// Prefix-search the labels: the subjects whose label starts with `prefix`
4575    /// (case-insensitive), as `(label, subject term)`, capped at `limit`.
4576    /// Reads the LABEL_INDEX section when the file has one (every labelled
4577    /// subject, no dictionary read); otherwise the legacy pyramid-meta block,
4578    /// which **faults the pyramid** on the lazy path. Empty when the file
4579    /// carries neither.
4580    pub fn prefix_search(&self, prefix: &str, limit: usize) -> Vec<(String, String)> {
4581        if let Some(li) = self.labels.get() {
4582            return li.prefix_search(prefix, limit);
4583        }
4584        let Some(pyr) = self.pyramid() else {
4585            return Vec::new();
4586        };
4587        resolve_label_hits(&self.dict, pyr.prefix_search(prefix, limit))
4588    }
4589
4590    /// The full-text index (TEXT_INDEX section), faulting it in on first access
4591    /// on the lazy remote path. `None` when the file carries no text index.
4592    pub(crate) fn text_index(&self) -> Option<&crate::text_index::TextIndex> {
4593        self.text_index.index()
4594    }
4595
4596    /// Whether this file carries a full-text (TEXT_INDEX) section, i.e. it was
4597    /// built with `--text-index`. Cheap — reads the header, never faults.
4598    pub fn has_text_index(&self) -> bool {
4599        self.header.text_index_len > 0
4600    }
4601
4602    /// What the full-text index covers (graphs, predicates) — `None` when the
4603    /// file has none. Faults the token table on the lazy remote path, as a
4604    /// search does; the scope block lives in it.
4605    pub fn text_index_scope(&self) -> Option<crate::text_index::TextIndexScope> {
4606        self.text_index().map(|ti| ti.scope().clone())
4607    }
4608
4609    /// Byte length of the TEXT_INDEX section's leading **token table** — the
4610    /// length varint plus the compressed table, which is exactly what a first
4611    /// [`text_search`](Self::text_search) faults. `None` when the file carries
4612    /// no text index (or its head could not be read).
4613    ///
4614    /// This is the number to quote as the cost of a first search, NOT
4615    /// `header().text_index_len`: that counts the postings blob too, which is
4616    /// the bulk of the section and is only ever fetched one posting list at a
4617    /// time. On the published `epfl-infoscience.rete` the section is 195 MB and
4618    /// its token table 29 MB — a 6.5× difference.
4619    ///
4620    /// Costs nothing on a resident open (measured while the bytes were in hand);
4621    /// a lazy/ranged open pays one ≤10-byte range read, memoized, and still
4622    /// never faults the table itself.
4623    pub fn text_index_token_table_len(&self) -> Option<u64> {
4624        self.text_index.token_table_len()
4625    }
4626
4627    /// Full-text search over the literals: subject IRIs that carry **every** word
4628    /// in `words` (whole-word, case-insensitive — AND semantics), optionally also
4629    /// requiring a word that **starts with** `prefix` (token-prefix). Results are
4630    /// ordered by subject id and capped at `limit` (0 = uncapped). Empty when the
4631    /// file has no text index or nothing matches.
4632    ///
4633    /// Like [`prefix_search`](Self::prefix_search), this **faults** the index on
4634    /// the lazy remote path — a search is an explicit read, and only the queried
4635    /// posting lists are fetched, not the whole index.
4636    pub fn text_search(&self, words: &[&str], prefix: Option<&str>, limit: usize) -> Vec<String> {
4637        text_search_in(self.text_index(), &self.dict, words, prefix, limit)
4638    }
4639
4640    /// The default-graph permutation index.
4641    pub fn default_index(&self) -> &GraphIndex {
4642        &self.index
4643    }
4644
4645    /// Resolve every triple of a graph (`None` = default graph) back to terms.
4646    pub fn dump(&self, graph: Option<&str>) -> Vec<TermTriple> {
4647        let index = match graph {
4648            None => &*self.index,
4649            Some(g) => match self.graph_index(g) {
4650                Some(i) => i,
4651                None => return Vec::new(),
4652            },
4653        };
4654        index
4655            .match_pattern((None, None, None))
4656            .into_iter()
4657            .filter_map(|(s, p, o)| {
4658                Some((
4659                    self.dict.subject_term(s)?,
4660                    self.dict.predicate_term(p)?,
4661                    self.dict.object_term(o)?,
4662                ))
4663            })
4664            .collect()
4665    }
4666
4667    /// Stream every triple of a graph (`None` = default) to `f`, resolving terms
4668    /// a batch at a time — no full `Vec` materialization, so it is safe on graphs far
4669    /// larger than RAM. `rete export` uses this to serialize 100M+ triple files
4670    /// that `dump()` (which collects every term into a `Vec<String>`) would OOM on.
4671    ///
4672    /// On a lazy/ranged open the dictionary is **not** prefetched whole: each
4673    /// batch faults only the chunks its own terms live in. Dumping ONE named
4674    /// graph therefore costs that graph's terms, not the file's. Dumping
4675    /// *everything* still reads every dictionary byte and leaves every chunk
4676    /// resident — that is inherent, and it, not the prefetch, is what sets the
4677    /// peak of a full export. See the comment in the body for both measurements.
4678    ///
4679    /// To dump a *slice* — one predicate, one subject, one object — call
4680    /// [`dump_filtered_each`](Self::dump_filtered_each), which is this method
4681    /// with a triple pattern and prunes tiles instead of filtering rows.
4682    pub fn dump_each<F: FnMut(&str, &str, &str)>(&self, graph: Option<&str>, f: F) {
4683        self.dump_filtered_each(graph, None, None, None, f)
4684    }
4685
4686    /// [`dump_each`](Self::dump_each) restricted to a triple **pattern** — the
4687    /// filtered dump. Any of `s` / `p` / `o` may be bound (canonical N-Triples
4688    /// term tokens); all three unbound is exactly `dump_each`.
4689    ///
4690    /// # Why this is cheap, and how much
4691    ///
4692    /// It is the same access path a routed *query* takes, not a full scan with a
4693    /// predicate test bolted on: [`GraphIndex::scan_iter`] picks the permutation
4694    /// with the longest bound prefix, binary-searches that section's tile
4695    /// directory down to the span a bound leading id can live in, and then drops
4696    /// every remaining tile whose recorded synopsis proves it cannot hold a
4697    /// bound secondary component — **all from directories, without fetching a
4698    /// tile**. So a filtered dump pays for the slice, which is the whole point
4699    /// of issue #117.
4700    ///
4701    /// Measured on `cordis.rete` (801 MB, a 417 MB dictionary, 26.4M quads,
4702    /// six named graphs), lazily opened; "before" is the only way to get a
4703    /// predicate slice out of a dump previously — dump the graph and throw away
4704    /// what does not match, in the consumer:
4705    ///
4706    /// ```text
4707    ///   graph=results  p=s66#doi     -> 337,811 rows
4708    ///     before  375.8 MB · 1797 req · 2105.8 MB RSS · 12.8 s · first row 409 ms
4709    ///     after    16.0 MB ·  105 req ·  182.6 MB RSS ·  0.41 s · first row  14 ms
4710    ///   graph=results  p=s66#isbn    ->  40,761 rows
4711    ///     before  375.8 MB · 1797 req · 2105.7 MB RSS · 12.8 s
4712    ///     after    15.5 MB ·  104 req ·  155.2 MB RSS ·  0.23 s
4713    /// ```
4714    ///
4715    /// # Where it does NOT help
4716    ///
4717    /// The index is pruned; the **dictionary is not**. Term resolution still
4718    /// faults whichever chunks the surviving rows' terms live in, and on a file
4719    /// whose payload *is* the literals those chunks are most of the file. Same
4720    /// file, a predicate whose objects are the long abstracts:
4721    ///
4722    /// ```text
4723    ///   graph=projects p=s66#abstract ->  80,206 rows
4724    ///     before  260.7 MB · 1142 req · 1496.9 MB RSS · 5.97 s
4725    ///     after   213.4 MB · 1606 req · 1144.4 MB RSS · 2.91 s   (1.2x, not 23x)
4726    /// ```
4727    ///
4728    /// An *unfiltered* dump is unchanged by construction — it is the floor, and
4729    /// its peak is the resident dictionary, not this. Two ceilings sit above
4730    /// both: a faulted dictionary chunk is a `OnceCell` nothing ever evicts, and
4731    /// on a literal-heavy file the chunk directory alone can be a third of the
4732    /// file before any dump work at all (#198).
4733    ///
4734    /// # Permutations
4735    ///
4736    /// Every one of the eight bound/unbound shapes routes inside
4737    /// [`PermSet::CORE`] — `{SPO, POS, OSP}`, the minimum a legal file carries
4738    /// (see `perm_routing_never_leaves_core`). A file built with
4739    /// `--permutations 3` therefore prunes *identically* to a six-permutation
4740    /// one: same tiles, same rows, no fallback path to get wrong.
4741    ///
4742    /// A bound term the dictionary does not know, or an absent graph IRI, yields
4743    /// nothing without touching the index.
4744    ///
4745    /// # Order
4746    ///
4747    /// Rows arrive in the **routed permutation's** order, which is the price of
4748    /// routing at all. Unfiltered (and subject-bound) that is SPO — canonical
4749    /// `(s, p, o)`, unchanged from [`dump_each`](Self::dump_each). A bound
4750    /// predicate routes to POS and streams `(p, o, s)`; a bound object routes to
4751    /// OSP. The *set* is identical to the unfiltered dump filtered; the order is
4752    /// not, so a consumer that needs canonical order must sort. N-Quads, the
4753    /// format `rete export` writes, is order-independent.
4754    ///
4755    /// [`PermSet::CORE`]: crate::index::PermSet::CORE
4756    pub fn dump_filtered_each<F: FnMut(&str, &str, &str)>(
4757        &self,
4758        graph: Option<&str>,
4759        s: Option<&str>,
4760        p: Option<&str>,
4761        o: Option<&str>,
4762        mut f: F,
4763    ) {
4764        let index = match graph {
4765            None => Some(&*self.index),
4766            Some(g) => self.graph_index(g),
4767        };
4768        let Some(index) = index else { return };
4769        // A bound term the dictionary does not know can match nothing, and the
4770        // index is never touched — same contract as `query_iter`/`query_batch`.
4771        let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
4772            return;
4773        };
4774        let dict = &self.dict;
4775        // Resolve in fixed-size batches with ONE coalesced dictionary fault per
4776        // batch — the same `prefetch_terms` call `dump_batch` makes, for the
4777        // same reason.
4778        //
4779        // This used to be `Dictionary::prefetch_all` up front, on the argument
4780        // that a full dump reaches every term anyway so the bytes are owed
4781        // either way. That argument only holds for a dump of EVERYTHING, and
4782        // this takes a graph AND a pattern. Measured on `cordis.rete` (801 MB,
4783        // a 417 MB dictionary, six named graphs), lazily opened:
4784        //
4785        //   dump_each(Some(fundingschemes))  417 MB read, 2510 MB peak RSS
4786        //                              →     173 MB read,  619 MB peak RSS
4787        //   dump_each(None) — the EMPTY default graph, which is the first thing
4788        //   `rete export` calls on a quads file:
4789        //                                    415 MB read, 2508 MB peak RSS
4790        //                              →       0 MB read,   95 MB peak RSS
4791        //
4792        // The 2.5 GB comes from asking for every chunk at once: the bulk loader
4793        // coalesces adjacent chunks into one span, so the whole dictionary
4794        // section is materialized three times over before the first triple —
4795        // the fetched span, the per-chunk copies sliced out of it, and the
4796        // decompressed bodies.
4797        //
4798        // Be clear about what this does NOT buy. A dump of the whole default
4799        // graph still faults every chunk, and chunks stay resident once
4800        // faulted, so `rete export` of a single-graph file is unchanged:
4801        // `figshare.rete` (222 MB, a 126 MB dictionary) peaks at 874 MB lazily
4802        // either way. The resident dictionary is the floor there, not this.
4803        //
4804        // Within each window, term resolution is CHUNK-ORDERED, not row-ordered
4805        // (`Dictionary::resolve_window`, bounded-export phase 1): the window's
4806        // ids are grouped by the `(section, chunk)` they live in and each chunk
4807        // is decoded once and its runs walked once, filling every slot that
4808        // chunk owns, then the rows are emitted in the original scan order. Two
4809        // things follow. First, a window's objects — which scatter across the
4810        // object-only section for one subject — touch each chunk a single time
4811        // instead of re-faulting it per row. Second, the per-id re-walk from a
4812        // run's start that the old row-order path did for *every* id collapses
4813        // to one monotonic forward pass per run. The output is byte-for-byte
4814        // identical (same bytes per id, same emission order, same
4815        // all-three-resolved emit rule); the cap is still unlimited, so this is
4816        // structural preparation for the real bound (phase 2), plus a small
4817        // free speedup. The coalesced bulk fault (`prefetch_terms`) is
4818        // unchanged, so the read pattern and peak residency are as before.
4819        const WINDOW: usize = 4096;
4820        // The whole-window coalesced prefetch (`prefetch_terms`) materializes
4821        // every distinct chunk the window touches at once — up to ~2·WINDOW
4822        // chunks for scattered objects. Under a GENEROUS budget that transient
4823        // fits and the coalesced reads are the fast path (small/normal files,
4824        // `--in-memory`, unlimited). Under a TIGHT budget it would blow the cap
4825        // and thrash (insert-then-evict every chunk, then re-fault it in
4826        // `resolve_window`), so skip it: `resolve_window` is chunk-ordered and
4827        // holds one chunk `Arc` at a time (plan §3.5), keeping the window's
4828        // working set to a single chunk regardless of how the objects scatter.
4829        // Output is byte-identical either way — prefetch only warms the cache.
4830        let cap = dict.cache_cap();
4831        let prefetch_window =
4832            cap == u64::MAX || cap >= (WINDOW as u64) * 2 * (DICT_CHUNK_BUDGET as u64);
4833        // Under a finite budget the prefetch also goes in batches of
4834        // PREFETCH_BATCH chunks: one bulk read holds everything it fetched
4835        // (span, slices, decoded bodies) until it is all cached, which for a
4836        // scattered window was thousands of chunks — up to ~1 GB on
4837        // openaire-relation — outside the cache cap. Unlimited keeps one read
4838        // per section (the latency-optimal shape for a remote file).
4839        const PREFETCH_BATCH: usize = 64;
4840        let prefetch_batch = if cap == u64::MAX {
4841            usize::MAX
4842        } else {
4843            PREFETCH_BATCH
4844        };
4845        let mut ids: Vec<(u32, u32, u32)> = Vec::with_capacity(WINDOW);
4846        let mut nodes: Vec<u32> = Vec::with_capacity(WINDOW * 2);
4847        let mut preds: Vec<u32> = Vec::with_capacity(WINDOW);
4848        let mut resolver = crate::dictionary::WindowResolver::default();
4849        let mut flush = |ids: &mut Vec<(u32, u32, u32)>,
4850                         resolver: &mut crate::dictionary::WindowResolver,
4851                         f: &mut F| {
4852            if ids.is_empty() {
4853                return;
4854            }
4855            // Coalesced bulk fault of exactly the chunks this window needs —
4856            // only under a generous budget (see `prefetch_window` above); a
4857            // tight budget faults chunk-by-chunk inside `resolve_window`.
4858            if prefetch_window {
4859                nodes.clear();
4860                preds.clear();
4861                for &(s, p, o) in ids.iter() {
4862                    nodes.push(dict.subject_node(s));
4863                    nodes.push(dict.object_node(o));
4864                    preds.push(p);
4865                }
4866                dict.prefetch_terms_batched(&nodes, &preds, prefetch_batch);
4867            }
4868            dict.resolve_window(ids, resolver, |s, p, o| f(s, p, o));
4869            ids.clear();
4870        };
4871        for t in index.scan_iter(pattern) {
4872            ids.push(t);
4873            if ids.len() == WINDOW {
4874                flush(&mut ids, &mut resolver, &mut f);
4875            }
4876        }
4877        flush(&mut ids, &mut resolver, &mut f);
4878    }
4879
4880    /// What a [`dump_filtered_each`](Self::dump_filtered_each) over this graph
4881    /// and pattern will fetch, **before** it starts — the dump twin of
4882    /// `rete cost`'s query preview.
4883    ///
4884    /// Costed from the tile directories, so it fetches no tile. It does read the
4885    /// dictionary far enough to resolve the bound terms (that is a real,
4886    /// unavoidable cost of the dump too) and to learn whether the graph exists.
4887    pub fn dump_plan(
4888        &self,
4889        graph: Option<&str>,
4890        s: Option<&str>,
4891        p: Option<&str>,
4892        o: Option<&str>,
4893    ) -> DumpPlan {
4894        let index = match graph {
4895            None => Some(&*self.index),
4896            Some(g) => self.graph_index(g),
4897        };
4898        let scan = index
4899            .zip(self.resolve_query_pattern(s, p, o))
4900            .map(|(ix, pattern)| ix.scan_plan(pattern));
4901        DumpPlan {
4902            scan,
4903            dictionary_bytes: match &self.v06 {
4904                Some(f) => f.section(crate::v06::kind::DICT_DATA).map_or(0, |r| r.len),
4905                None => self.header.dictionary_len,
4906            },
4907        }
4908    }
4909
4910    /// The **pull** twin of [`Rete::dump_each`](Self::dump_each): every triple of a
4911    /// graph (`None` = default) as a lazy iterator of resolved terms.
4912    ///
4913    /// Same constant-memory scan — one triple decoded and resolved per `next()`,
4914    /// never a `Vec` of the whole graph — but the caller drives it, so it can
4915    /// stop early or be **suspended and resumed** across a foreign-function
4916    /// boundary. That is what the wasm/JS client's batched quad cursor needs:
4917    /// a callback cannot be paused mid-scan to hand control back to JavaScript,
4918    /// an iterator can.
4919    ///
4920    /// The dictionary is NOT prefetched: terms fault in as they are resolved, so
4921    /// stopping after a handful of triples costs a handful of dictionary reads
4922    /// rather than the whole dictionary — which on a lazy remote open of a large
4923    /// file would be gigabytes before the first triple, defeating the point of a
4924    /// pull API. [`dump_each`](Self::dump_each) does not prefetch it whole
4925    /// either; it batches the same per-term faults. Peak memory is
4926    /// O(faulted dictionary chunks + faulted index tiles), *not* O(triples).
4927    pub fn dump_iter(&self, graph: Option<&str>) -> impl Iterator<Item = TermTriple> + '_ {
4928        self.query_iter(graph, None, None, None)
4929    }
4930
4931    /// [`dump_iter`](Self::dump_iter) generalized to a triple **pattern**: the
4932    /// lazy pull form of [`query_in_graph`](Self::query_in_graph). A bound term
4933    /// unknown to the dictionary, or an unknown graph IRI, yields nothing.
4934    ///
4935    /// Same constant-memory guarantee — one triple decoded and resolved per
4936    /// `next()`, never a `Vec` of every match — and the same absence of a
4937    /// dictionary prefetch, so stopping after a handful of matches costs a
4938    /// handful of dictionary reads. `query_in_graph` is the eager twin; prefer
4939    /// this one wherever the consumer can stop early (`ASK`, `LIMIT`, an RDF4J
4940    /// `getStatements` that is about to be sliced).
4941    pub fn query_iter(
4942        &self,
4943        graph: Option<&str>,
4944        s: Option<&str>,
4945        p: Option<&str>,
4946        o: Option<&str>,
4947    ) -> impl Iterator<Item = TermTriple> + '_ {
4948        let index = match graph {
4949            None => Some(&*self.index),
4950            Some(g) => self.graph_index(g),
4951        };
4952        index
4953            .zip(self.resolve_query_pattern(s, p, o))
4954            .into_iter()
4955            .flat_map(|(ix, pattern)| ix.scan_iter(pattern))
4956            .filter_map(move |(s, p, o)| {
4957                Some((
4958                    self.dict.subject_term(s)?,
4959                    self.dict.predicate_term(p)?,
4960                    self.dict.object_term(o)?,
4961                ))
4962            })
4963    }
4964
4965    /// One bounded, **resumable** slice of a pattern's matches inside one graph
4966    /// (`None` = default): `(triples, next_cursor, done)`. Start at `cursor = 0`
4967    /// and feed the returned cursor back until `done`.
4968    ///
4969    /// This is [`query_in_graph`](Self::query_in_graph) made streamable across a
4970    /// boundary that cannot hold a Rust borrow — the wasm/JVM one. The engine
4971    /// keeps no iterator alive between calls; the entire resume state is the
4972    /// opaque `u64` (see [`GraphIndex::scan_batch`]), so a client's cursor
4973    /// survives being suspended, and a cursor that is abandoned leaks nothing
4974    /// but a `u64`.
4975    ///
4976    /// The dictionary is faulted **per batch**, not whole: taking ten matches
4977    /// off the front of a 9.8-billion-quad file reads ten matches' worth of
4978    /// terms. That is the difference between `LIMIT 1` answering in bounded time
4979    /// and it reading a 30 GB dictionary first.
4980    ///
4981    /// `max_quads` is a floor (batches end on an a-group boundary), and every
4982    /// call either returns at least one row or reports `done`.
4983    pub fn query_batch(
4984        &self,
4985        graph: Option<&str>,
4986        s: Option<&str>,
4987        p: Option<&str>,
4988        o: Option<&str>,
4989        cursor: u64,
4990        max_quads: usize,
4991    ) -> (Vec<TermTriple>, u64, bool) {
4992        let index = match graph {
4993            None => &*self.index,
4994            // An unknown graph IRI is an empty scan, not an error — same as
4995            // `query_in_graph` and `dump_batch`.
4996            Some(g) => match self.graph_index(g) {
4997                Some(i) => i,
4998                None => return (Vec::new(), cursor, true),
4999            },
5000        };
5001        // A bound term the dictionary does not know can match nothing, and the
5002        // index is never touched.
5003        let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
5004            return (Vec::new(), cursor, true);
5005        };
5006        let (ids, next, done) = index.scan_batch(pattern, cursor, max_quads);
5007        // One coalesced dictionary fault for THIS batch — the same call
5008        // `query_in_graph` and `dump_batch` make, and the reason a partial walk
5009        // costs in proportion to what it actually read.
5010        if !ids.is_empty() {
5011            let mut nodes = Vec::with_capacity(ids.len() * 2);
5012            let mut preds = Vec::with_capacity(ids.len());
5013            for &(s, p, o) in &ids {
5014                nodes.push(self.dict.subject_node(s));
5015                nodes.push(self.dict.object_node(o));
5016                preds.push(p);
5017            }
5018            self.dict.prefetch_terms(&nodes, &preds);
5019        }
5020        let triples = ids
5021            .into_iter()
5022            .filter_map(|(s, p, o)| {
5023                Some((
5024                    self.dict.subject_term(s)?,
5025                    self.dict.predicate_term(p)?,
5026                    self.dict.object_term(o)?,
5027                ))
5028            })
5029            .collect();
5030        (triples, next, done)
5031    }
5032
5033    /// Resolve one bounded slice of a graph's triples. Returns
5034    /// `(triples, next_cursor, done)`; start with `cursor = 0` and feed the
5035    /// returned cursor back until `done`.
5036    ///
5037    /// This is the PULL half of the dump, and it exists because [`Rete::dump_each`] is
5038    /// push-based: it drives the scan itself and hands each triple to a
5039    /// callback. A client that wants to pull (a Python generator, a JS async
5040    /// iterator) has to hold the scan's stack between calls — which means a
5041    /// thread (unavailable on Pyodide and in wasm) or a self-referential struct
5042    /// holding an iterator that borrows the `Rete` stored next to it, whose
5043    /// soundness rests on drop order plus an unstated promise that this crate's
5044    /// lazily-faulted caches stay write-once. Neither is worth it for a dump.
5045    ///
5046    /// Instead the scan is made RESUMABLE: SPO tiles are ordered by subject id,
5047    /// so "every triple of subject `sid`, for ascending `sid`" visits exactly the
5048    /// tiles a full scan visits, in the same order, and the whole resume state
5049    /// collapses to one `u32`. No thread, no `unsafe`, no borrow held across
5050    /// calls — and unlike `(offset, limit)` it is O(n) overall rather than
5051    /// O(n²/limit), because nothing is ever re-scanned.
5052    ///
5053    /// `max_quads` is the FLOOR of a batch, not a hard cut: a batch always ends
5054    /// on a subject boundary, so no subject is split across two calls. Work per
5055    /// call is bounded by a probe budget too — a sparse named graph whose few
5056    /// subjects are spread over a huge id space returns early with `done = false`
5057    /// rather than grinding through absent ids, and the caller just asks again.
5058    pub fn dump_batch(
5059        &self,
5060        graph: Option<&str>,
5061        cursor: u32,
5062        max_quads: usize,
5063    ) -> (Vec<TermTriple>, u32, bool) {
5064        let index = match graph {
5065            None => self.default_index(),
5066            // An unknown graph IRI is an empty dump, not an error — same as
5067            // `dump_each`.
5068            Some(g) => match self.graph_index(g) {
5069                Some(i) => i,
5070                None => return (Vec::new(), cursor, true),
5071            },
5072        };
5073        // Tile leading ranges live in the section directory, so both the span and
5074        // the gap jumps below are free on a lazy/remote open — no tile is fetched
5075        // to decide where to look.
5076        let spo = IndexPermutation::Spo.section_index();
5077        let Some((first, last)) = index.leading_bounds(spo) else {
5078            return (Vec::new(), cursor, true); // empty graph
5079        };
5080
5081        let mut sid = cursor.max(first);
5082        let mut probes = max_quads.saturating_mul(4).max(1 << 16);
5083        let mut ids: Vec<(u32, u32, u32)> = Vec::new();
5084        let mut exhausted = false;
5085        while sid <= last && probes > 0 {
5086            probes -= 1;
5087            let hits = index.match_pattern((Some(sid), None, None));
5088            if hits.is_empty() {
5089                // No triples at this id: jump to the next subject the index
5090                // holds (the tile's group directory knows it), instead of
5091                // stepping through absent ids one at a time. That is what keeps
5092                // a quad-index view — whose tile shared with other graphs has
5093                // the conservative range `0..=u32::MAX` — from probing four
5094                // billion ids. Only when that lookup cannot answer (a failed
5095                // fetch, already recorded) does it fall back to the directory:
5096                // if the id falls in a hole BETWEEN tiles, jump to the next
5097                // tile's first id.
5098                match index.next_leading(IndexPermutation::Spo.section_index(), sid) {
5099                    Some(next) if next > sid => {
5100                        sid = next;
5101                        continue;
5102                    }
5103                    None if !index.load_incomplete() => {
5104                        exhausted = true;
5105                        break;
5106                    }
5107                    _ => {}
5108                }
5109                match index.next_tile_start(spo, sid) {
5110                    None => {
5111                        exhausted = true;
5112                        break;
5113                    }
5114                    Some(next) if next > sid => {
5115                        sid = next;
5116                        continue;
5117                    }
5118                    Some(_) => {}
5119                }
5120            } else {
5121                ids.extend(hits);
5122            }
5123            if sid == u32::MAX {
5124                exhausted = true;
5125                break;
5126            }
5127            sid += 1;
5128            if ids.len() >= max_quads {
5129                break;
5130            }
5131        }
5132        let done = exhausted || sid > last;
5133
5134        // One coalesced dictionary fault for this batch, then resolve.
5135        // Deliberately NOT `Dictionary::prefetch_all` (what `dump_each` uses):
5136        // on a lazy open that pulls the ENTIRE dictionary — gigabytes on a big
5137        // file — before the first quad, so taking five quads off the front would
5138        // cost as much as a full dump. Per-batch prefetch keeps a partial walk
5139        // proportional to what it actually read.
5140        let dict = self.dictionary();
5141        if !ids.is_empty() {
5142            let mut nodes = Vec::with_capacity(ids.len() * 2);
5143            let mut preds = Vec::with_capacity(ids.len());
5144            for &(s, p, o) in &ids {
5145                nodes.push(dict.subject_node(s));
5146                nodes.push(dict.object_node(o));
5147                preds.push(p);
5148            }
5149            dict.prefetch_terms(&nodes, &preds);
5150        }
5151        let triples = ids
5152            .into_iter()
5153            .filter_map(|(s, p, o)| {
5154                Some((
5155                    dict.subject_term(s)?,
5156                    dict.predicate_term(p)?,
5157                    dict.object_term(o)?,
5158                ))
5159            })
5160            .collect();
5161        (triples, sid, done)
5162    }
5163
5164    /// Every quad of the **named graphs** of a quad-index file, as
5165    /// `(s, p, o, Some(graph))`, in graph order, handed to `visit` in batches
5166    /// of at least `batch` quads (the terms of a batch are faulted in one
5167    /// coalesced read). One sequential pass over the graph-leading tiles: no
5168    /// view per graph and no list of the graphs' names, so it walks 43 M graphs
5169    /// in the memory of a batch (`graph_names` memoises every name, which over
5170    /// that many graphs is gigabytes). `Ok(false)`, and nothing visited, when
5171    /// the file has no quad index to scan (a per-graph file: walk
5172    /// [`graph_names`](Self::graph_names) and [`dump_batch`](Self::dump_batch)
5173    /// instead). A failed read is recorded ([`index_incomplete`](Self::index_incomplete)).
5174    pub fn dump_named_quads<E>(
5175        &self,
5176        batch: usize,
5177        visit: &mut dyn FnMut(crate::ingest::RawQuad) -> Result<(), E>,
5178    ) -> Result<bool, E> {
5179        let Some(scan) = self.named_quad_scan((None, None, None)) else {
5180            return Ok(false);
5181        };
5182        let dict = self.dictionary();
5183        let batch = batch.max(1);
5184        let mut ids: Vec<(u32, (u32, u32, u32))> = Vec::with_capacity(batch.min(1 << 20));
5185        // The graph names of one batch decode once each.
5186        let mut graph: Option<(u32, String)> = None;
5187        let mut flush = |ids: &mut Vec<(u32, (u32, u32, u32))>| -> Result<(), E> {
5188            let mut nodes = Vec::with_capacity(ids.len() * 3);
5189            let mut preds = Vec::with_capacity(ids.len());
5190            for &(g, (s, p, o)) in ids.iter() {
5191                nodes.push(dict.subject_node(g));
5192                nodes.push(dict.subject_node(s));
5193                nodes.push(dict.object_node(o));
5194                preds.push(p);
5195            }
5196            dict.prefetch_terms(&nodes, &preds);
5197            for (g, (s, p, o)) in ids.drain(..) {
5198                if graph.as_ref().is_none_or(|(id, _)| *id != g) {
5199                    graph = dict.subject_term(g).map(|name| (g, name));
5200                }
5201                let (Some((_, name)), Some(s), Some(p), Some(o)) = (
5202                    graph.as_ref(),
5203                    dict.subject_term(s),
5204                    dict.predicate_term(p),
5205                    dict.object_term(o),
5206                ) else {
5207                    continue;
5208                };
5209                visit((s, p, o, Some(name.clone())))?;
5210            }
5211            Ok(())
5212        };
5213        for q in scan {
5214            ids.push(q);
5215            if ids.len() >= batch {
5216                flush(&mut ids)?;
5217            }
5218        }
5219        flush(&mut ids)?;
5220        Ok(true)
5221    }
5222
5223    /// How many named graphs this dataset has. On a lazy ranged open this
5224    /// reads only the section's leading count varint — never the graphs.
5225    pub fn named_graph_count(&self) -> usize {
5226        self.named_graphs.count()
5227    }
5228
5229    /// The `i`-th named graph's quad count, when the file records it: a
5230    /// quad-index file keeps it in its graph directory, so this reads one
5231    /// directory block and never the graph's index. `None` for a per-graph
5232    /// (NAMED_GRAPHS section) file, or past the last graph.
5233    pub fn named_graph_quad_count(&self, i: usize) -> Option<u64> {
5234        match &self.named_graphs {
5235            NamedGraphsSlot::Quad(q) => q.quad_count_at(i),
5236            _ => None,
5237        }
5238    }
5239
5240    /// The `i`-th named graph as an **owned** index that is not memoised,
5241    /// when the file can make one cheaply: a quad-index file's graph of at
5242    /// most [`TRANSIENT_GRAPH_QUADS`] quads. A walk over every graph (`GRAPH
5243    /// ?g`, the union default graph, `query_quads`) evaluates each such graph
5244    /// and drops it, instead of keeping one index per graph for the life of
5245    /// the open — 1.5 M of them on the OpenCitations sample. `None` for a
5246    /// larger graph, a per-graph file, or a failed read: use
5247    /// [`named_graph_at`](Self::named_graph_at) then.
5248    pub fn named_graph_transient(&self, i: usize) -> Option<GraphIndex> {
5249        match &self.named_graphs {
5250            NamedGraphsSlot::Quad(q) => q.transient_view(i, TRANSIENT_GRAPH_QUADS),
5251            _ => None,
5252        }
5253    }
5254
5255    /// The ordinals (ascending, as [`named_graph_at`](Self::named_graph_at)
5256    /// counts them) of the named graphs holding at least one quad whose
5257    /// subject is one of `subjects` (subject ids), for a quad-index file: one
5258    /// sequential pass over its graph-leading SPO tiles, never a view per
5259    /// graph. `None` for a per-graph file, which has no such pass, or on a
5260    /// failed read (recorded, see [`index_incomplete`](Self::index_incomplete)).
5261    pub(crate) fn named_graphs_with_subjects(
5262        &self,
5263        subjects: &std::collections::BTreeSet<u32>,
5264    ) -> Option<Vec<usize>> {
5265        match &self.named_graphs {
5266            NamedGraphsSlot::Quad(q) => q.graphs_with_subjects(subjects),
5267            _ => None,
5268        }
5269    }
5270
5271    /// Deduplicated quads across every named graph, when the file records it
5272    /// (a quad-index file, from its preamble: one small read). `None` for a
5273    /// per-graph file — there it is the header's `quad_count` minus the
5274    /// default graph's triples.
5275    pub fn named_quad_count(&self) -> Option<u64> {
5276        match &self.named_graphs {
5277            NamedGraphsSlot::Quad(q) => Some(q.quad_count()),
5278            _ => None,
5279        }
5280    }
5281
5282    /// How many triples `index` holds, for sizing a pattern's tile estimate:
5283    /// a named graph's own count when it is known without a tile read (a
5284    /// quad-index graph's directory count, a resident per-graph index), else
5285    /// the header's total. The default graph keeps the total: its size is not
5286    /// recorded in the header, and the planner's thresholds were tuned on it.
5287    pub(crate) fn index_triples(&self, index: &GraphIndex) -> u64 {
5288        index.size_hint().unwrap_or(self.header().quad_count)
5289    }
5290
5291    /// The `i`-th named graph's IRI (stored order). On a lazy ranged open this
5292    /// walks entry HEADERS up to `i` — it never decodes a graph's index.
5293    pub fn named_graph_name_at(&self, i: usize) -> Option<&str> {
5294        self.named_graphs.name_at(&self.dict, i, false)
5295    }
5296
5297    /// The `i`-th named graph as `(iri, index)`. On a lazy ranged open the
5298    /// index is fetched and decoded on first access and memoised; check
5299    /// [`index_incomplete`](Self::index_incomplete) after evaluating.
5300    pub fn named_graph_at(&self, i: usize) -> Option<(&str, &GraphIndex)> {
5301        self.named_graphs.graph_at(&self.dict, i, false)
5302    }
5303
5304    /// The `i`-th graph's name as an owned string that the file does **not**
5305    /// keep: what a walk over millions of graphs uses, so the names it has
5306    /// passed are not all held at once. `exhaustive = true` tells a lazy walk
5307    /// the caller will visit every graph, so it may read the section in bulk
5308    /// chunks; purely a fetch strategy, results are identical either way. Only
5309    /// the SPARQL evaluator sets it, from shapes where full consumption is
5310    /// provable.
5311    pub(crate) fn named_graph_name_owned_demand(
5312        &self,
5313        i: usize,
5314        exhaustive: bool,
5315    ) -> Option<String> {
5316        self.named_graphs.name_owned(&self.dict, i, exhaustive)
5317    }
5318
5319    /// The dictionary node that names the quad-index graph at ordinal `i`, as
5320    /// a query row binds a graph variable: no term is decoded. `None` for
5321    /// another layout (whose graphs are named by strings).
5322    pub(crate) fn named_graph_node_demand(&self, i: usize, exhaustive: bool) -> Option<u32> {
5323        match &self.named_graphs {
5324            NamedGraphsSlot::Quad(q) => q.node_at(&self.dict, i, exhaustive),
5325            _ => None,
5326        }
5327    }
5328
5329    /// The ordinal of the quad-index graph named `iri`: a dictionary lookup and
5330    /// a directory binary search. `None` for another file layout, or no such
5331    /// graph.
5332    pub(crate) fn named_graph_ordinal(&self, iri: &str) -> Option<usize> {
5333        match &self.named_graphs {
5334            NamedGraphsSlot::Quad(q) => q.ordinal_of_iri(&self.dict, iri),
5335            _ => None,
5336        }
5337    }
5338
5339    /// The ordinal of the quad-index graph named by the dictionary node `node`,
5340    /// as a query row holds it: a directory search, no term decoded. `None`
5341    /// for another layout, or when the node names no graph.
5342    pub(crate) fn named_graph_ordinal_of_node(&self, node: u32) -> Option<usize> {
5343        match &self.named_graphs {
5344            NamedGraphsSlot::Quad(q) => q.ordinal_of_node(&self.dict, node),
5345            _ => None,
5346        }
5347    }
5348
5349    /// The ordinal of the quad-index graph whose subject id is `g` (as
5350    /// [`named_quad_scan`](Self::named_quad_scan) yields it).
5351    pub(crate) fn named_graph_ordinal_of_id(&self, g: u32) -> Option<usize> {
5352        match &self.named_graphs {
5353            NamedGraphsSlot::Quad(q) => q.ordinal_of_id(g),
5354            _ => None,
5355        }
5356    }
5357
5358    /// A streaming scan of the quads matching `pat` (dictionary ids: subject,
5359    /// predicate, object) over every named graph of a quad-index file, as
5360    /// `(graph subject id, (s, p, o))`; see [`crate::quad::QuadScan`]. `None`
5361    /// for another layout, or when the section cannot be read.
5362    pub(crate) fn named_quad_scan(
5363        &self,
5364        pat: (Option<u32>, Option<u32>, Option<u32>),
5365    ) -> Option<crate::quad::QuadScan> {
5366        match &self.named_graphs {
5367            NamedGraphsSlot::Quad(q) => q.scan(pat),
5368            _ => None,
5369        }
5370    }
5371
5372    /// The union default graph as an index that streams it from the quad
5373    /// index (see [`crate::union_scan`]): the file's default graph and every
5374    /// named graph, each triple once, nothing materialized. `None` when the
5375    /// file has no quad index with the three graph-last orders (a per-graph
5376    /// layout, or a file written before `0x07`): the caller merges instead.
5377    pub(crate) fn union_default_index(&self) -> Option<GraphIndex> {
5378        let src = self.quad_named()?.union_src()?;
5379        Some(GraphIndex::union_of(
5380            std::sync::Arc::clone(&self.index),
5381            src,
5382        ))
5383    }
5384
5385    /// The quad index, when the file has one.
5386    pub(crate) fn quad_named(&self) -> Option<&crate::quad::QuadNamed> {
5387        match &self.named_graphs {
5388            NamedGraphsSlot::Quad(q) => Some(q),
5389            _ => None,
5390        }
5391    }
5392
5393    /// [`named_graph_at`](Self::named_graph_at) under a declared demand: see
5394    /// [`named_graph_name_owned_demand`](Self::named_graph_name_owned_demand).
5395    pub(crate) fn named_graph_at_demand(
5396        &self,
5397        i: usize,
5398        exhaustive: bool,
5399    ) -> Option<(&str, &GraphIndex)> {
5400        self.named_graphs.graph_at(&self.dict, i, exhaustive)
5401    }
5402
5403    /// IRIs of the named graphs in this dataset (the default graph is unnamed).
5404    /// On a lazy ranged open this walks the whole directory (headers only) —
5405    /// prefer [`named_graph_count`](Self::named_graph_count) when the number
5406    /// is all that's needed.
5407    pub fn graph_names(&self) -> Vec<&str> {
5408        (0..self.named_graphs.count())
5409            .filter_map(|i| self.named_graphs.name_at(&self.dict, i, false))
5410            .collect()
5411    }
5412
5413    /// The permutation index of a named graph, or `None` if absent.
5414    pub fn graph_index(&self, iri: &str) -> Option<&GraphIndex> {
5415        self.named_graphs.find(&self.dict, iri)
5416    }
5417
5418    /// Match a triple pattern in dictionary-ID space (subject/predicate/object
5419    /// IDs), returning integer triples — the fast path used by the BGP engine.
5420    pub fn match_ids(
5421        &self,
5422        pattern: (Option<u32>, Option<u32>, Option<u32>),
5423    ) -> Vec<(u32, u32, u32)> {
5424        self.index.match_pattern(pattern)
5425    }
5426
5427    /// All `(subject_node, object_node)` pairs for a predicate, as unified node
5428    /// IDs — no term resolution. The fast path for graph traversal.
5429    pub fn predicate_pairs(&self, predicate: &str) -> Vec<(u32, u32)> {
5430        let pid = match self.dict.predicate_id(predicate) {
5431            Some(p) => p,
5432            None => return Vec::new(),
5433        };
5434        self.index
5435            .match_pattern((None, Some(pid), None))
5436            .into_iter()
5437            .map(|(s, _p, o)| (self.dict.subject_node(s), self.dict.object_node(o)))
5438            .collect()
5439    }
5440
5441    /// Open via a [`RangeReader`], fetching only the header and the named
5442    /// section ranges — never a linear scan of the whole resource. A full query
5443    /// open touches at most 4 ranges (header, dictionary, index, pyramid-meta).
5444    pub fn open_ranged<R: RangeReader>(reader: &R) -> Result<Self, FileError> {
5445        let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
5446        if crate::v06::open::is_v06(&head) {
5447            // The eager path reads the whole image — the rest of it, in one
5448            // more request — and a 0x06 one then opens from memory.
5449            let mut all = head;
5450            let at = all.len() as u64;
5451            if reader.len() > at {
5452                all.extend_from_slice(&reader.read_at(at, reader.len() - at)?);
5453            }
5454            return Self::open(&all);
5455        }
5456        let header = Header::from_bytes(&head)?;
5457
5458        let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
5459        let dict =
5460            decode_dictionary_container(&dict_bytes, header.dict_codec, header.term_order())?;
5461
5462        check_named_layout(&header)?;
5463        let index_bytes = reader.read_at(header.root_dir_offset, header.root_dir_len)?;
5464        let (index, _) = decode_root_index_container(&index_bytes, &header)?;
5465        let index_section_ranges = decode_index_section_ranges(
5466            &index_bytes,
5467            header.root_dir_offset,
5468            header.perms,
5469            header.index_container_sections(),
5470        )?;
5471
5472        let pyramid = PyramidSlot::Resident(if header.pyramid_meta_len > 0 {
5473            let mb = reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?;
5474            Some(
5475                PyramidMeta::decode(&mb)
5476                    .map_err(|_| FileError::Container("malformed pyramid meta"))?,
5477            )
5478        } else {
5479            None
5480        });
5481
5482        // Fetch the whole TEXT_INDEX section resident (this opener does one range
5483        // read per section; the lazy opener below is the one that defers it).
5484        let (ti_off, ti_len) = header.text_index_payload();
5485        let text_index_bytes = if ti_len > 0 {
5486            Some(reader.read_at(ti_off, ti_len)?)
5487        } else {
5488            None
5489        };
5490        let text_index = resident_text_index_slot(text_index_bytes.as_deref(), header.block_codec)?;
5491        // LABEL_INDEX and SCHEMA: one range read each, held resident like the
5492        // other sections of this opener.
5493        let labels = SectionSlot::Resident(header.label_index_section().and_then(|s| {
5494            let b = reader.read_at(s.offset, s.length).ok()?;
5495            crate::label_index::LabelIndex::from_section(&b, header.block_codec).ok()
5496        }));
5497        let schema = SectionSlot::Resident(read_schema_with_header(reader, &header));
5498
5499        let named_graphs = NamedGraphsSlot::resident(
5500            &header,
5501            |off, len| Ok(reader.read_at(off, len)?),
5502            || {
5503                let nb = reader.read_at(header.named_graphs_offset, header.named_graphs_len)?;
5504                decode_named_graphs(&nb, header.block_codec, header.perms)
5505            },
5506        )?;
5507
5508        // The metadata section (Dataset Card) is deliberately NOT fetched here:
5509        // a ranged query open keeps to its small range budget. Use `Rete::open`
5510        // (or a dedicated card fetch) when the card is actually needed.
5511        let tile_ranges =
5512            tile_file_ranges(&index_bytes, header.root_dir_offset, &index_section_ranges);
5513        Ok(Self {
5514            header,
5515            dict,
5516            index: std::sync::Arc::new(index),
5517            index_section_ranges,
5518            tile_ranges,
5519            pyramid,
5520            text_index,
5521            labels,
5522            schema,
5523            named_graphs,
5524            metadata: Vec::new(),
5525            service_client: None,
5526            service_error: std::sync::Mutex::new(None),
5527            spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5528            remote_reads: std::sync::atomic::AtomicBool::new(false),
5529            v06: None,
5530        })
5531    }
5532
5533    /// Open a `0x06` file whose first bytes are `first`: the header and root
5534    /// come from them; the dictionary and index are two-level directories whose
5535    /// leaves fault on demand; text, labels, schema, pyramid and named graphs
5536    /// are deferred. `budget` applies the default memory budget (the lazy
5537    /// remote path) as [`open_ranged_lazy`](Self::open_ranged_lazy) does.
5538    fn open_v06(
5539        reader: std::sync::Arc<dyn RangeReader + Send + Sync>,
5540        first: &[u8],
5541        budget: bool,
5542    ) -> Result<Self, FileError> {
5543        let f = std::sync::Arc::new(crate::v06::open::V06File::open(reader.clone(), first)?);
5544        let header = crate::v06::logical_header(&f.h);
5545        let rc = reader.concurrency();
5546        let dict = f.dictionary([true; 4])?;
5547        let index = std::sync::Arc::new(f.index(rc)?);
5548        dict.wire_derived(index.clone());
5549        let (pyramid, text_index, labels, schema) = v06_slots(&f);
5550        let named_graphs = match f.quad(rc) {
5551            Some(q) => NamedGraphsSlot::Quad(q?),
5552            None => NamedGraphsSlot::Resident(Vec::new()),
5553        };
5554        let rete = Self {
5555            header,
5556            dict,
5557            index,
5558            index_section_ranges: [ByteRange::default(); NUM_PERMS],
5559            tile_ranges: Default::default(),
5560            pyramid,
5561            text_index,
5562            labels,
5563            schema,
5564            named_graphs,
5565            metadata: Vec::new(),
5566            service_client: None,
5567            service_error: std::sync::Mutex::new(None),
5568            spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5569            remote_reads: std::sync::atomic::AtomicBool::new(false),
5570            v06: Some(f),
5571        };
5572        #[cfg(not(target_arch = "wasm32"))]
5573        if budget {
5574            rete.set_memory_budget(match crate::sysmem::default_query_budget_mb() {
5575                0 => None,
5576                mb => Some(mb << 20),
5577            });
5578        }
5579        #[cfg(target_arch = "wasm32")]
5580        let _ = budget;
5581        Ok(rete)
5582    }
5583
5584    /// The open `0x06` file (header, root, reader), `None` for a `0x05` open.
5585    pub fn v06(&self) -> Option<&crate::v06::open::V06File> {
5586        self.v06.as_deref()
5587    }
5588
5589    /// Open via an **owned** [`RangeReader`] with lazy tile faulting (tiled
5590    /// v0.2 files): fetches the header, dictionary, pyramid meta, named graphs,
5591    /// and each permutation's tile **directory** — but no default-graph tile
5592    /// payloads. Tiles fault in (one range request each) the first time a scan
5593    /// touches them, so a selective SPARQL query fetches O(touched tiles)
5594    /// bytes instead of the whole index.
5595    ///
5596    /// **Failure contract:** scans are infallible by design, so a failed tile
5597    /// fetch yields an empty tile and sets a sticky flag — after evaluating,
5598    /// callers MUST check [`index_incomplete`](Self::index_incomplete) and
5599    /// surface an error instead of the (possibly partial) results.
5600    pub fn open_ranged_lazy<R: RangeReader + Send + Sync + 'static>(
5601        reader: R,
5602    ) -> Result<Self, FileError> {
5603        Self::open_ranged_lazy_with(reader, crate::v06::FIRST_READ)
5604    }
5605
5606    /// [`open_ranged_lazy`](Self::open_ranged_lazy) with an explicit first-read
5607    /// size (at least the 1 KiB a `0x05` header needs). The default is
5608    /// [`crate::v06::FIRST_READ`]; a smaller one keeps a byte-accounting test
5609    /// of a tiny `0x05` file from reading the whole file up front.
5610    pub fn open_ranged_lazy_with<R: RangeReader + Send + Sync + 'static>(
5611        reader: R,
5612        first_read: u64,
5613    ) -> Result<Self, FileError> {
5614        let remote = !reader.is_local();
5615        let rete = Self::open_ranged_lazy_inner(reader, first_read)?;
5616        rete.remote_reads
5617            .store(remote, std::sync::atomic::Ordering::Relaxed);
5618        Ok(rete)
5619    }
5620
5621    /// Does a read of this file cost a round trip? True for a lazy open over a
5622    /// reader that is not [`RangeReader::is_local`] (HTTP, a host callback);
5623    /// false for a file held in memory or read lazily from a local disk. Only
5624    /// a strategy hint: a grouped query is cut into slices for several threads
5625    /// when reads are local, and keeps its one scan with its coalesced
5626    /// read-ahead when each read is a request.
5627    pub fn reads_are_remote(&self) -> bool {
5628        self.remote_reads.load(std::sync::atomic::Ordering::Relaxed)
5629    }
5630
5631    fn open_ranged_lazy_inner<R: RangeReader + Send + Sync + 'static>(
5632        reader: R,
5633        first_read: u64,
5634    ) -> Result<Self, FileError> {
5635        // One first read — 64 KiB, or the whole file when smaller — decides the
5636        // format, and every later read inside it is served from memory.
5637        let first_len = first_read.max(HEADER_LEN as u64).min(reader.len());
5638        let first: std::sync::Arc<[u8]> = std::sync::Arc::from(reader.read_at(0, first_len)?);
5639        #[cfg(feature = "v06-parity")]
5640        if !crate::v06::open::is_v06(&first) {
5641            let all = reader.read_at(0, reader.len())?;
5642            if let Some(v6) = parity_image(&all) {
5643                let r: std::sync::Arc<dyn RangeReader + Send + Sync> =
5644                    std::sync::Arc::new(crate::reader::BytesReader(std::sync::Arc::from(v6)));
5645                let head = r.read_at(0, crate::v06::FIRST_READ.min(r.len()))?;
5646                return Self::open_v06(r, &head, true);
5647            }
5648        }
5649        let reader = crate::reader::PrefixReader::new(reader, first.clone());
5650        if crate::v06::open::is_v06(&first) {
5651            let r: std::sync::Arc<dyn RangeReader + Send + Sync> = std::sync::Arc::new(reader);
5652            return Self::open_v06(r, &first, true);
5653        }
5654        let head = reader.read_at(0, HEADER_LEN as u64)?;
5655        let header = Header::from_bytes(&head)?;
5656        check_named_layout(&header)?;
5657        let reader = std::sync::Arc::new(reader);
5658        // Captured before the loader closures take the Arc: the reader's
5659        // concurrent-range fan-out, stamped onto the index for the planner.
5660        let read_concurrency = reader.concurrency();
5661
5662        // Lazily-chunked dictionary: locate the four sections, fetch each
5663        // section's header + restart table + chunk directory (small), and
5664        // fault the chunk bodies in on first term lookup.
5665        let open_dict = || ranged_chunked_dictionary(&reader, &header, [true; 4]);
5666
5667        // Locate the six index section payloads (container framing only) and
5668        // fetch just their tile directories — shared with the per-named-graph
5669        // lazy opener, which opens a large graph's container the same way.
5670        let reader_dyn: std::sync::Arc<dyn RangeReader + Send + Sync> = reader.clone();
5671        let open_index = || {
5672            open_index_container_lazy(
5673                &reader_dyn,
5674                ByteRange {
5675                    offset: header.root_dir_offset,
5676                    len: header.root_dir_len,
5677                },
5678                header.block_codec,
5679                header.has_tile_synopsis(),
5680                read_concurrency,
5681                header.perms,
5682                header.index_container_sections(),
5683            )
5684        };
5685
5686        // Each of the two is a chain of small dependent reads (a container's
5687        // sections are found by walking their length prefixes), ~8 round trips
5688        // apiece on a real file, and the header already says where both
5689        // containers start. So a reader that can keep several reads in flight
5690        // walks them at the same time: the open costs the longer chain, not
5691        // the sum. Same reads, same result; only their overlap changes.
5692        let (dict, index) = open_side_by_side(read_concurrency, open_dict, open_index);
5693        let dict = dict?;
5694        let (index, index_section_ranges, tile_ranges) = index?;
5695
5696        // The pyramid meta is large on real graphs (tens of MB) and SPARQL never
5697        // reads it, so defer its fetch: it faults in only if `pyramid()` is
5698        // called (community / pyramid_tree / inspect queries).
5699        let pyramid = ranged_pyramid_slot(&reader, &header);
5700
5701        // The TEXT_INDEX section is also deferred: a text search faults the token
5702        // table on first call (the leading varint then its compressed bytes), then
5703        // fetches individual posting lists by `(offset, len)` — never the whole
5704        // postings blob. A SPARQL query, which never searches, pays nothing.
5705        let text_index = ranged_text_index_slot(&reader, &header);
5706
5707        // LABEL_INDEX: the first prefix search reads its head and walks down.
5708        // SCHEMA: read once, on first use — a stats question, or the planner
5709        // ordering a join that needs a measured selectivity.
5710        let labels = ranged_label_slot(&reader, &header);
5711        let schema = ranged_schema_slot(&reader, &header);
5712
5713        // Named graphs are the last eagerly-fetched section standing on this
5714        // path — and on a many-graph file they dwarf everything else (67 MB
5715        // fetched and ~32k graph indexes decoded before the first query, on a
5716        // file whose queries then touch a handful of graphs). Defer them like
5717        // the pyramid and text index: nothing is read at open; a query walks
5718        // the directory and decodes only the graphs it touches.
5719        let named_graphs = if let Some(q) = header.quad_index_section() {
5720            // The quad index: nothing read at open either; the preamble on the
5721            // first named-graph touch, then only the routing parts and tiles
5722            // a query uses.
5723            NamedGraphsSlot::Quad(crate::quad::QuadNamed::new(
5724                reader_dyn,
5725                ByteRange {
5726                    offset: q.offset,
5727                    len: q.length,
5728                },
5729                header.block_codec,
5730                header.perms,
5731                read_concurrency,
5732            ))
5733        } else if header.named_graphs_len > 0 {
5734            NamedGraphsSlot::Lazy(LazyNamedGraphs::new(
5735                reader_dyn,
5736                ByteRange {
5737                    offset: header.named_graphs_offset,
5738                    len: header.named_graphs_len,
5739                },
5740                header.block_codec,
5741                header.has_tile_synopsis(),
5742                read_concurrency,
5743                header.perms,
5744            ))
5745        } else {
5746            NamedGraphsSlot::Resident(Vec::new())
5747        };
5748
5749        let rete = Self {
5750            header,
5751            dict,
5752            index: std::sync::Arc::new(index),
5753            index_section_ranges,
5754            tile_ranges,
5755            pyramid,
5756            text_index,
5757            labels,
5758            schema,
5759            named_graphs,
5760            metadata: Vec::new(),
5761            service_client: None,
5762            service_error: std::sync::Mutex::new(None),
5763            spill_threshold: std::sync::atomic::AtomicU64::new(u64::MAX),
5764            remote_reads: std::sync::atomic::AtomicBool::new(false),
5765            v06: None,
5766        };
5767        // A lazy open is the path for files too big to read whole, and the
5768        // one a long-lived session (relay, Python/R/Java clients, `rete serve`)
5769        // keeps resident across queries. Unbounded, its dictionary and tile
5770        // caches grow with everything any query ever touched: a GROUP BY over
5771        // a 9.5-billion-triple file ran a 16 GB container out of memory. Cap
5772        // them by default; a caller that wants another figure (or unlimited)
5773        // calls `set_memory_budget` again. The default is sized from the
5774        // memory this process can still use (`RETE_QUERY_BUDGET_MB` overrides
5775        // it; see `crate::sysmem`): a fixed 4096 MiB did not fit a 4 GiB
5776        // container. Not on wasm32, whose 4 GiB address space makes the default
5777        // meaningless and whose callers bound their own range budgets.
5778        #[cfg(not(target_arch = "wasm32"))]
5779        rete.set_memory_budget(match crate::sysmem::default_query_budget_mb() {
5780            0 => None,
5781            mb => Some(mb << 20),
5782        });
5783        Ok(rete)
5784    }
5785
5786    /// Did any lazy fetch (index tile or dictionary chunk) fail since this
5787    /// `Rete` was opened? When true, query results may be silently incomplete —
5788    /// callers using [`Rete::open_ranged_lazy`] must check this after
5789    /// evaluating and turn it into an error.
5790    pub fn index_incomplete(&self) -> bool {
5791        self.index.load_incomplete()
5792            || self.dict.load_incomplete()
5793            || self.named_graphs.load_incomplete()
5794    }
5795
5796    /// Forget recorded lazy-fetch failures — the start-of-evaluation reset for
5797    /// a RESIDENT session (a browser worker holding one `Rete` across many
5798    /// queries): it makes [`index_incomplete`](Self::index_incomplete) a
5799    /// per-query verdict instead of a per-open one, so a single transient
5800    /// network failure no longer fails every subsequent query on the session.
5801    /// Sound because failed tiles/chunks are never cached — the next
5802    /// evaluation simply retries the fetch.
5803    pub fn reset_load_failures(&self) {
5804        self.index.reset_load_failure();
5805        self.dict.reset_load_failure();
5806        self.named_graphs.reset_load_failures();
5807    }
5808
5809    /// Bound peak resident memory to roughly `budget` bytes by capping the
5810    /// evictable dictionary chunk cache and the default-graph index tile cache
5811    /// (bounded-export phase 2). `None` or `u64::MAX` means unlimited — nothing
5812    /// is evicted and residency is identical to a plain open, which is what
5813    /// `--in-memory` and every query/wasm path use.
5814    ///
5815    /// Sound at any cap: the dictionary and the default-graph index both fault
5816    /// their chunks/tiles through a loader, so an evicted body is simply
5817    /// re-faulted on its next touch; the export dump reads each needed body
5818    /// while holding it (the cache `Arc` handout invariant), so eviction never
5819    /// drops bytes in use. Named-graph indexes are **not** capped here — a small
5820    /// resident graph has no loader and could never reload an evicted tile — so
5821    /// they stay unlimited and are released whole per slot instead (see
5822    /// [`release_named_graph`](Self::release_named_graph)).
5823    ///
5824    /// The open's **fixed** heap — chunk and tile directories, the named-graph
5825    /// directory walked so far ([`fixed_overhead`](Self::fixed_overhead)) — is
5826    /// paid whatever the caches hold, so it comes out of the dictionary share:
5827    /// the caps plus the fixed heap stay within `budget`. It grows with the
5828    /// file's chunk/tile/graph count (~18 MB of directories on the 14.75 GiB
5829    /// openaire-relation, 88 MB with its tile directories; 224 MB on a pre-#198
5830    /// epfl-infoscience whose chunk keys are whole boundary terms). Calling
5831    /// this again after a walk of the named-graph directory (as `rete export`
5832    /// does once it has listed the graph slots) re-subtracts the larger figure.
5833    /// The dictionary share never drops below its 64 MiB floor, which is the
5834    /// only way the total can be exceeded: a file whose directories alone
5835    /// approach the budget.
5836    ///
5837    /// Returns the split actually applied (dictionary share net of the fixed
5838    /// heap), for `RETE_OPEN_DEBUG` logging.
5839    pub fn set_memory_budget(&self, budget: Option<u64>) -> MemoryBudget {
5840        let total = budget.unwrap_or(u64::MAX);
5841        self.set_spill_threshold(budget.map(spill_threshold_for));
5842        let mut split = split_memory_budget(total);
5843        let fixed = if total == u64::MAX {
5844            0
5845        } else {
5846            self.fixed_overhead().total()
5847        };
5848        if split.dict_cache != u64::MAX {
5849            split.dict_cache = split
5850                .dict_cache
5851                .saturating_sub(fixed)
5852                .max(MIN_DICT_CACHE_CAP);
5853        }
5854        self.dict.set_cache_cap(split.dict_cache);
5855        self.index.set_cache_cap(split.tile_cache);
5856        // Large named graphs (opened lazily, loader-bearing) get the same tile
5857        // cap; small resident ones stay unlimited (bounded by size + per-slot
5858        // release). Combined with `release_named_graph`, a many-graph dump holds
5859        // one graph's tiles at a time, each capped.
5860        match &self.named_graphs {
5861            NamedGraphsSlot::Lazy(lazy) => lazy.set_tile_cap(split.tile_cache),
5862            // Every quad-index view, and the tiles they share, re-fault
5863            // through loaders — on every open path — so they take the cap too.
5864            NamedGraphsSlot::Quad(q) => q.set_tile_cap(split.tile_cache),
5865            NamedGraphsSlot::Resident(_) => {}
5866        }
5867        if std::env::var("RETE_OPEN_DEBUG").is_ok() {
5868            eprintln!(
5869                "[memory-budget] total={} dict_cache={} tile_cache={} block_cache={} fixed={:.1}MiB",
5870                if total == u64::MAX {
5871                    "unlimited".to_string()
5872                } else {
5873                    format!("{}MiB", total >> 20)
5874                },
5875                fmt_cap(split.dict_cache),
5876                fmt_cap(split.tile_cache),
5877                fmt_cap(split.block_cache),
5878                fixed as f64 / (1u64 << 20) as f64,
5879            );
5880        }
5881        split
5882    }
5883
5884    /// How many bytes a query's `COUNT(DISTINCT …)` keeps in memory before it
5885    /// writes sorted runs to the temporary directory (`TMPDIR`) and counts by
5886    /// merging them; the count is exact either way. `u64::MAX` (the default of
5887    /// an eager open) never spills. [`set_memory_budget`](Self::set_memory_budget)
5888    /// sets it to an eighth of the budget, so a lazy open spills past 512 MiB
5889    /// by default.
5890    pub fn spill_threshold(&self) -> u64 {
5891        self.spill_threshold
5892            .load(std::sync::atomic::Ordering::Relaxed)
5893    }
5894
5895    /// Set [`spill_threshold`](Self::spill_threshold) without touching the
5896    /// caches — for an eager (resident) open, whose caches cannot be capped.
5897    /// `None` = never spill.
5898    pub fn set_spill_threshold(&self, bytes: Option<u64>) {
5899        self.spill_threshold.store(
5900            bytes.unwrap_or(u64::MAX),
5901            std::sync::atomic::Ordering::Relaxed,
5902        );
5903    }
5904
5905    /// The dictionary chunk cache's observability counters (decoded chunks and
5906    /// bytes, hits/misses, evictions, resident bytes) — for the bounded-export
5907    /// work-accounting tests and `RETE_OPEN_DEBUG`.
5908    pub fn dict_cache_stats(&self) -> crate::chunk_cache::CacheStats {
5909        self.dict.cache_stats()
5910    }
5911
5912    /// Heap bytes this open holds outside the bounded caches — the dictionary
5913    /// chunk directories, the index tile directories and the named-graph
5914    /// directory — for `RETE_OPEN_DEBUG` / `RETE_MEM_TRACE` accounting.
5915    pub fn fixed_overhead(&self) -> FixedOverhead {
5916        FixedOverhead {
5917            dict_directory: self.dict.directory_bytes(),
5918            index_directory: self.index.directory_bytes(),
5919            named_graphs: self.named_graphs.directory_bytes(),
5920        }
5921    }
5922
5923    /// The default-graph index tile cache's observability counters.
5924    pub fn index_cache_stats(&self) -> crate::chunk_cache::CacheStats {
5925        self.index.cache_stats()
5926    }
5927
5928    /// The current dictionary chunk-cache byte cap (`u64::MAX` = unlimited).
5929    pub fn dict_cache_cap(&self) -> u64 {
5930        self.dict.cache_cap()
5931    }
5932
5933    /// Release a named graph's decoded index after its slot has been exported,
5934    /// dropping its `GraphIndex` (and, with it, that graph's decoded tiles and
5935    /// its own tile cache) so a many-graph dump does not accumulate one resident
5936    /// index per graph — the next linear floor on many-graph files once the
5937    /// dictionary is bounded (plan §6). A no-op for the default graph, an
5938    /// unknown IRI, a resident (in-memory/eager) open, or a graph never opened.
5939    ///
5940    /// Takes `&mut self`: the export loop finishes one slot's dump (which only
5941    /// borrows `&self`) before releasing, so no outstanding borrow of the index
5942    /// exists. Correct because a released index is simply re-opened (headers +
5943    /// tile directory) on a later access — the same lazy contract as a chunk
5944    /// that was never faulted.
5945    pub fn release_named_graph(&mut self, iri: &str) {
5946        match &mut self.named_graphs {
5947            NamedGraphsSlot::Lazy(lazy) => lazy.release(iri),
5948            NamedGraphsSlot::Quad(q) => q.release(&self.dict, iri),
5949            NamedGraphsSlot::Resident(_) => {}
5950        }
5951    }
5952
5953    fn resolve_query_pattern(
5954        &self,
5955        s: Option<&str>,
5956        p: Option<&str>,
5957        o: Option<&str>,
5958    ) -> Option<Pattern> {
5959        let sid = match s {
5960            Some(t) => match self.dict.subject_id(t) {
5961                Some(id) => Some(id),
5962                None => return None,
5963            },
5964            None => None,
5965        };
5966        let pid = match p {
5967            Some(t) => match self.dict.predicate_id(t) {
5968                Some(id) => Some(id),
5969                None => return None,
5970            },
5971            None => None,
5972        };
5973        let oid = match o {
5974            Some(t) => match self.dict.object_id(t) {
5975                Some(id) => Some(id),
5976                None => return None,
5977            },
5978            None => None,
5979        };
5980        Some((sid, pid, oid))
5981    }
5982
5983    /// Evaluate a triple pattern and include the file/index provenance for every
5984    /// matched result. A bound term that is unknown to the dictionary yields no
5985    /// matches.
5986    pub fn query_with_provenance(
5987        &self,
5988        s: Option<&str>,
5989        p: Option<&str>,
5990        o: Option<&str>,
5991    ) -> Vec<TripleProvenance> {
5992        let pattern = match self.resolve_query_pattern(s, p, o) {
5993            Some(pattern) => pattern,
5994            None => return Vec::new(),
5995        };
5996
5997        let index_permutation = GraphIndex::best_permutation_in(self.header.perms, pattern);
5998        let si = index_permutation.section_index();
5999        let (dictionary_range, index_range, index_section_range, pyramid_range) = match &self.v06 {
6000            Some(f) => {
6001                use crate::v06::kind;
6002                let sec = |k| f.section(k).unwrap_or_default();
6003                let perm = f
6004                    .root
6005                    .unit(crate::v06::tag::INDEX)
6006                    .and_then(|u| crate::v06::decode_index_root(u).ok())
6007                    .and_then(|v| v.into_iter().find(|p| p.si == si))
6008                    .map(|p| ByteRange {
6009                        offset: sec(kind::INDEX_DATA).offset + p.data_off,
6010                        len: p.data_len,
6011                    })
6012                    .unwrap_or_default();
6013                (
6014                    sec(kind::DICT_DATA),
6015                    sec(kind::INDEX_DATA),
6016                    perm,
6017                    f.section(kind::PYRAMID),
6018                )
6019            }
6020            None => (
6021                ByteRange {
6022                    offset: self.header.dictionary_offset,
6023                    len: self.header.dictionary_len,
6024                },
6025                ByteRange {
6026                    offset: self.header.root_dir_offset,
6027                    len: self.header.root_dir_len,
6028                },
6029                self.index_section_ranges[si],
6030                (self.header.pyramid_meta_len > 0).then_some(ByteRange {
6031                    offset: self.header.pyramid_meta_offset,
6032                    len: self.header.pyramid_meta_len,
6033                }),
6034            ),
6035        };
6036
6037        let tiles = &self.tile_ranges[si];
6038        let v06_tiles = self.v06.is_some().then(|| self.index.tile_sections()[si]);
6039        self.index
6040            .match_pattern(pattern)
6041            .into_iter()
6042            .filter_map(|(s, p, o)| {
6043                let terms = (
6044                    self.dict.subject_term(s)?,
6045                    self.dict.predicate_term(p)?,
6046                    self.dict.object_term(o)?,
6047                );
6048                // The physical tile holding this match: the one whose
6049                // leading-id range covers the match's permuted leading id.
6050                let a = index_permutation.forward((s, p, o)).0;
6051                let (tile, tile_range) = if let Some(tl) = v06_tiles {
6052                    let (ti, tj) = self.index.tile_span(si, Some(a));
6053                    match (ti < tj && ti < tl.len()).then(|| tl.range(ti)).flatten() {
6054                        Some((offset, len)) => (
6055                            Some(format!("{}/{ti}", index_permutation.name())),
6056                            Some(ByteRange { offset, len }),
6057                        ),
6058                        None => (None, None),
6059                    }
6060                } else {
6061                    let ti = tiles.partition_point(|&(_, max_a, _)| max_a < a);
6062                    match tiles.get(ti) {
6063                        Some(&(min_a, _, range)) if min_a <= a => (
6064                            Some(format!("{}/{ti}", index_permutation.name())),
6065                            Some(range),
6066                        ),
6067                        _ => (None, None),
6068                    }
6069                };
6070                Some(TripleProvenance {
6071                    terms,
6072                    ids: (s, p, o),
6073                    graph: None,
6074                    matched_pattern: pattern,
6075                    index_permutation,
6076                    dictionary_range,
6077                    index_range,
6078                    index_section_range,
6079                    pyramid_range,
6080                    tile,
6081                    tile_range,
6082                })
6083            })
6084            .collect()
6085    }
6086
6087    /// Evaluate a triple pattern given as optional term strings, returning
6088    /// matching triples resolved back to terms. A bound term that is unknown to
6089    /// the dictionary yields no matches.
6090    pub fn query(&self, s: Option<&str>, p: Option<&str>, o: Option<&str>) -> Vec<TermTriple> {
6091        self.query_with_provenance(s, p, o)
6092            .into_iter()
6093            .map(|m| m.terms)
6094            .collect()
6095    }
6096
6097    /// Match a triple pattern **within a single graph** — `None` is the default
6098    /// graph, `Some(iri)` a named graph — resolving matches to canonical terms.
6099    /// This is [`Rete::query`] (default-graph only) generalized to any graph: the
6100    /// graph-scoped primitive a quad-aware consumer (e.g. an RDF4J `Sail`'s
6101    /// `getStatements`) needs. An unknown graph IRI, or a bound term absent from
6102    /// the shared dictionary, yields an empty result. All graphs share one
6103    /// dictionary, so the pattern resolves once against that ID space.
6104    pub fn query_in_graph(
6105        &self,
6106        graph: Option<&str>,
6107        s: Option<&str>,
6108        p: Option<&str>,
6109        o: Option<&str>,
6110    ) -> Vec<TermTriple> {
6111        let pattern = match self.resolve_query_pattern(s, p, o) {
6112            Some(pattern) => pattern,
6113            None => return Vec::new(),
6114        };
6115        let index = match graph {
6116            None => &*self.index,
6117            Some(g) => match self.graph_index(g) {
6118                Some(i) => i,
6119                None => return Vec::new(),
6120            },
6121        };
6122        self.resolve_matches(index.match_pattern(pattern))
6123    }
6124
6125    /// [`query_in_graph`](Self::query_in_graph) over any index of this file,
6126    /// such as the streamed union default graph.
6127    pub(crate) fn query_in_index(
6128        &self,
6129        index: &GraphIndex,
6130        s: Option<&str>,
6131        p: Option<&str>,
6132        o: Option<&str>,
6133    ) -> Vec<TermTriple> {
6134        match self.resolve_query_pattern(s, p, o) {
6135            Some(pattern) => self.resolve_matches(index.match_pattern(pattern)),
6136            None => Vec::new(),
6137        }
6138    }
6139
6140    /// Resolve matched id-triples to terms, with one coalesced dictionary
6141    /// fault for exactly their terms.
6142    fn resolve_matches(&self, ids: Vec<Triple>) -> Vec<TermTriple> {
6143        // One coalesced dictionary fault for THIS pattern's output, not the
6144        // whole dictionary. `prefetch_all` used to sit here, which made a
6145        // bounded `(s, p, o, g)` lookup — the primitive an RDF4J `Sail`'s
6146        // `getStatements` calls, once per named graph via `query_quads` — pay
6147        // for every term in the file: on `cordis.rete` (801 MB, a 417 MB
6148        // dictionary) a 15-row answer read 416 MB and peaked at 2.5 GB RSS.
6149        // Same call `dump_batch` makes, for the same reason.
6150        if !ids.is_empty() {
6151            let mut nodes = Vec::with_capacity(ids.len() * 2);
6152            let mut preds = Vec::with_capacity(ids.len());
6153            for &(s, p, o) in &ids {
6154                nodes.push(self.dict.subject_node(s));
6155                nodes.push(self.dict.object_node(o));
6156                preds.push(p);
6157            }
6158            self.dict.prefetch_terms(&nodes, &preds);
6159        }
6160        ids.into_iter()
6161            .filter_map(|(s, p, o)| {
6162                Some((
6163                    self.dict.subject_term(s)?,
6164                    self.dict.predicate_term(p)?,
6165                    self.dict.object_term(o)?,
6166                ))
6167            })
6168            .collect()
6169    }
6170
6171    /// Match a triple pattern across the default graph **and every named graph**,
6172    /// tagging each match with its graph (`None` = default). The quad-level
6173    /// companion to [`Rete::query`]; default-graph matches come first, then each
6174    /// named graph in stored order.
6175    pub fn query_quads(
6176        &self,
6177        s: Option<&str>,
6178        p: Option<&str>,
6179        o: Option<&str>,
6180    ) -> Vec<(TermTriple, Option<String>)> {
6181        let mut out: Vec<(TermTriple, Option<String>)> = self
6182            .query_in_graph(None, s, p, o)
6183            .into_iter()
6184            .map(|t| (t, None))
6185            .collect();
6186        let Some(pattern) = self.resolve_query_pattern(s, p, o) else {
6187            return out;
6188        };
6189        for i in 0..self.named_graphs.count() {
6190            // A quad dump visits every graph by construction — exhaustive
6191            // demand, so a lazy walk may read the section in bulk chunks.
6192            let Some(iri) = self.named_graphs.name_at(&self.dict, i, true) else {
6193                continue;
6194            };
6195            // A small quad-index graph through an owned view, dropped here:
6196            // the walk holds one view at a time, not one per graph.
6197            let owned = self.named_graph_transient(i);
6198            let index = match &owned {
6199                Some(view) => view,
6200                None => match self.named_graphs.graph_at(&self.dict, i, true) {
6201                    Some((_, g)) => g,
6202                    None => continue,
6203                },
6204            };
6205            for triple in self.resolve_matches(index.match_pattern(pattern)) {
6206                out.push((triple, Some(iri.to_string())));
6207            }
6208        }
6209        out
6210    }
6211
6212    /// Evaluate one triple pattern through a [`RangeReader`] by fetching only
6213    /// the header, the dictionary, and — for a tiled (v0.2) file — the
6214    /// selected permutation section's tile **directory** plus the tile(s) the
6215    /// bound leading id routes to; an unbound leading id fetches the section's
6216    /// tile body in one request. v0.1 files fetch the whole selected section.
6217    /// Unknown bound terms return an empty result before touching the index.
6218    pub fn query_ranged<R: RangeReader>(
6219        reader: &R,
6220        s: Option<&str>,
6221        p: Option<&str>,
6222        o: Option<&str>,
6223    ) -> Result<Vec<TermTriple>, FileError> {
6224        let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6225        if crate::v06::open::is_v06(&head) {
6226            return Ok(v06_query_ranged(reader, head, s, p, o, true)?.unwrap_or_default());
6227        }
6228        let routed = match route_pattern(reader, head, s, p, o)? {
6229            Some(routed) => routed,
6230            None => return Ok(Vec::new()),
6231        };
6232        let matches = fetch_routed_matches(reader, &routed)?;
6233        Ok(matches
6234            .into_iter()
6235            .filter_map(|(s, p, o)| {
6236                Some((
6237                    routed.dict.subject_term(s)?,
6238                    routed.dict.predicate_term(p)?,
6239                    routed.dict.object_term(o)?,
6240                ))
6241            })
6242            .collect())
6243    }
6244
6245    /// Route one triple pattern to its permutation section without fetching
6246    /// any payload bytes. Returns `false` when a bound term is unknown and the
6247    /// index was skipped.
6248    pub fn route_pattern_ranged<R: RangeReader>(
6249        reader: &R,
6250        s: Option<&str>,
6251        p: Option<&str>,
6252        o: Option<&str>,
6253    ) -> Result<bool, FileError> {
6254        let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
6255        if crate::v06::open::is_v06(&head) {
6256            return Ok(v06_query_ranged(reader, head, s, p, o, false)?.is_some());
6257        }
6258        Ok(route_pattern(reader, head, s, p, o)?.is_some())
6259    }
6260}
6261
6262/// [`Rete::query_ranged`] / [`Rete::route_pattern_ranged`] on a `0x06` file
6263/// through a borrowed reader: the front and the dictionary data — contiguous,
6264/// what the `0x05` path reads as header + dictionary — in one more request
6265/// after `head`, the pattern resolved there, then (with `fetch`) the routed
6266/// tiles in one span. `None` when a bound term is not in the dictionary.
6267fn v06_query_ranged<R: RangeReader>(
6268    reader: &R,
6269    head: Vec<u8>,
6270    s: Option<&str>,
6271    p: Option<&str>,
6272    o: Option<&str>,
6273    fetch: bool,
6274) -> Result<Option<Vec<TermTriple>>, FileError> {
6275    use crate::v06::{kind, open::SparseReader, Header06, Root06, HEADER_LEN};
6276    let len = reader.len();
6277    let h = Header06::from_bytes(&head)?;
6278    if h.flags & crate::v06::F_DERIVED != 0 {
6279        // A derived IRI is rebuilt from the index, which this one-request path
6280        // does not hold: answering without it would drop rows, so it refuses.
6281        return Err(FileError::Container(
6282            "this file derives IRIs from its index (format 0x08); a one-pattern ranged query cannot read it, use the lazy reader (`rete sparql-url`)",
6283        ));
6284    }
6285    let root_end = (HEADER_LEN as u64 + h.root_len as u64).min(len);
6286    let root_bytes = if root_end <= head.len() as u64 {
6287        head[HEADER_LEN..root_end as usize].to_vec()
6288    } else {
6289        reader.read_at(HEADER_LEN as u64, root_end - HEADER_LEN as u64)?
6290    };
6291    let root = Root06::decode(&root_bytes, h.internal_codec)?;
6292    let dict = root.section(kind::DICT_DATA).and_then(|d| d.abs(&h).ok());
6293    // The open below reads only the 1 KiB it must (not the 64 KiB first
6294    // read): what it needs past that is the front and the dictionary, here.
6295    let end = dict
6296        .map_or(h.data_offset, |d| d.offset + d.len)
6297        .max(crate::header::HEADER_LEN as u64)
6298        .min(len);
6299    let mut held = head;
6300    if end > held.len() as u64 {
6301        let at = held.len() as u64;
6302        held.extend_from_slice(&reader.read_at(at, end - at)?);
6303    }
6304    let sparse = SparseReader::fetch(reader, held, &[])?;
6305    let rete = Rete::open_ranged_lazy_with(sparse, 0)?;
6306    let Some(pattern) = resolve_query_pattern(rete.dictionary(), s, p, o) else {
6307        return Ok(None);
6308    };
6309    if !fetch {
6310        return Ok(Some(Vec::new()));
6311    }
6312    let perm = GraphIndex::best_permutation_in(h.perms(), pattern);
6313    let si = perm.section_index();
6314    let index = rete.default_index();
6315    let [pa, _, _] = perm.order_pattern(pattern);
6316    // One span of tiles, contiguous in the file: one request. A clustered
6317    // section has a span per class run that can hold a bound subject.
6318    let mut out = Vec::new();
6319    for (start, stop) in index.routed_spans(si, pa) {
6320        let ranges: Vec<(u64, u64)> = (start..stop)
6321            .filter_map(|i| index.tile_range(si, i))
6322            .collect();
6323        if let (Some(&(lo, _)), Some(&(last, last_len))) = (ranges.first(), ranges.last()) {
6324            let body = reader.read_at(lo, last + last_len - lo)?;
6325            for &(off, l) in &ranges {
6326                let at = (off - lo) as usize;
6327                let tile = decompress(h.block_codec, &body[at..at + l as usize])?;
6328                out.extend(GraphIndex::match_serialized_block(&tile, perm, pattern));
6329            }
6330        }
6331    }
6332    out.sort_unstable();
6333    let d = rete.dictionary();
6334    Ok(Some(
6335        out.into_iter()
6336            .filter_map(|(s, p, o)| {
6337                Some((d.subject_term(s)?, d.predicate_term(p)?, d.object_term(o)?))
6338            })
6339            .collect(),
6340    ))
6341}
6342
6343/// A pattern routed to its permutation section: everything needed to fetch
6344/// matches, with no payload bytes read yet.
6345struct RoutedPattern {
6346    dict: Dictionary,
6347    pattern: Pattern,
6348    permutation: IndexPermutation,
6349    header: Header,
6350    /// Absolute byte range of the selected section's payload.
6351    section: ByteRange,
6352}
6353
6354/// Resolve a pattern against the remote dictionary and locate its permutation
6355/// section (header + dictionary + container framing only).
6356fn route_pattern<R: RangeReader>(
6357    reader: &R,
6358    head: Vec<u8>,
6359    s: Option<&str>,
6360    p: Option<&str>,
6361    o: Option<&str>,
6362) -> Result<Option<RoutedPattern>, FileError> {
6363    let header = Header::from_bytes(&head)?;
6364
6365    let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
6366    let dict = decode_dictionary_container(&dict_bytes, header.dict_codec, header.term_order())?;
6367
6368    let Some(pattern) = resolve_query_pattern(&dict, s, p, o) else {
6369        return Ok(None);
6370    };
6371    let permutation = GraphIndex::best_permutation_in(header.perms, pattern);
6372    let section = locate_container_section_ranged(
6373        reader,
6374        header.root_dir_offset,
6375        header.root_dir_len,
6376        header
6377            .perms
6378            .position(permutation)
6379            .ok_or(FileError::Container("routed to an absent permutation"))?,
6380        header.index_container_sections() as u64,
6381    )?;
6382    Ok(Some(RoutedPattern {
6383        dict,
6384        pattern,
6385        permutation,
6386        header,
6387        section,
6388    }))
6389}
6390
6391/// Fetch and scan a routed pattern's matches: read the tile directory, then only
6392/// the matching tile byte ranges (the run of covering tiles for a bound leading
6393/// id — one for ordinary groups, several for a split mega-group — the
6394/// O(matching bytes) promise).
6395fn fetch_routed_matches<R: RangeReader>(
6396    reader: &R,
6397    routed: &RoutedPattern,
6398) -> Result<Vec<Triple>, FileError> {
6399    let dir = read_tile_directory_ranged(reader, routed.section)?;
6400    let [pa, _, _] = routed.permutation.order_pattern(routed.pattern);
6401    let codec = routed.header.block_codec;
6402    let mut out = Vec::new();
6403    match pa {
6404        // Bound leading id: the run of covering tiles (several for a split
6405        // mega-group; one otherwise).
6406        Some(a) => {
6407            for e in dir.iter().filter(|e| e.min_a <= a && a <= e.max_a) {
6408                let len = e.end - e.start;
6409                let bytes = reader.read_at(checked_end(routed.section.offset, e.start)?, len)?;
6410                if bytes.len() as u64 != len {
6411                    return Err(FileError::Container("short tile read"));
6412                }
6413                let tile = decompress(codec, &bytes)?;
6414                out.extend(GraphIndex::match_serialized_block(
6415                    &tile,
6416                    routed.permutation,
6417                    routed.pattern,
6418                ));
6419            }
6420        }
6421        // Unbound leading id: every tile matters — fetch the contiguous tile
6422        // body in one request and slice it.
6423        None => {
6424            if let (Some(first), Some(last)) = (dir.first(), dir.last()) {
6425                let base = first.start;
6426                let body =
6427                    reader.read_at(checked_end(routed.section.offset, base)?, last.end - base)?;
6428                for e in &dir {
6429                    // Checked: a reader may return fewer bytes than asked.
6430                    let stored = checked_slice(&body, e.start - base, e.end - e.start)
6431                        .ok_or(FileError::Container("short tile read"))?;
6432                    let tile = decompress(codec, stored)?;
6433                    out.extend(GraphIndex::match_serialized_block(
6434                        &tile,
6435                        routed.permutation,
6436                        routed.pattern,
6437                    ));
6438                }
6439            }
6440        }
6441    }
6442    out.sort_unstable();
6443    Ok(out)
6444}
6445
6446fn resolve_query_pattern(
6447    dict: &Dictionary,
6448    s: Option<&str>,
6449    p: Option<&str>,
6450    o: Option<&str>,
6451) -> Option<Pattern> {
6452    let sid = match s {
6453        Some(t) => Some(dict.subject_id(t)?),
6454        None => None,
6455    };
6456    let pid = match p {
6457        Some(t) => Some(dict.predicate_id(t)?),
6458        None => None,
6459    };
6460    let oid = match o {
6461        Some(t) => Some(dict.object_id(t)?),
6462        None => None,
6463    };
6464    Some((sid, pid, oid))
6465}
6466
6467/// The lazily-chunked dictionary of a ranged open: per section, fetch the
6468/// header + chunk directory and fault the chunk bodies in on first term lookup.
6469///
6470/// `want` selects which of the four sections (0 shared, 1 subject-only,
6471/// 2 object-only, 3 predicates) are really read; a skipped one becomes an empty
6472/// section whose terms resolve to `None`. On a file written before #198 a
6473/// **directory is not small**: it carried each chunk's first term verbatim, so
6474/// the object-only section of a dataset that stores abstracts can run to
6475/// hundreds of megabytes, and fetching it is most of what a remote open costs.
6476/// Current builds key it by the shortest separator instead (a few bytes per
6477/// chunk), but that is write-side, so every already-published file still pays.
6478/// A reader that only ever resolves subjects (see [`SearchView`]) skips
6479/// sections 2 and 3 and pays none of it. [`Rete::open_ranged_lazy`] wants all
6480/// four.
6481fn ranged_chunked_dictionary<R: RangeReader + Send + Sync + 'static>(
6482    reader: &std::sync::Arc<R>,
6483    header: &Header,
6484    want: [bool; 4],
6485) -> Result<Dictionary, FileError> {
6486    // One (unlimited) cache shared across the four sections, keyed by
6487    // (section_index, chunk_index).
6488    let cache = crate::chunk_cache::ChunkCache::unlimited_arc();
6489    let mut dict_sections: Vec<crate::dict::ChunkedSection> = Vec::with_capacity(4);
6490    let mut section_ranges: [std::sync::Arc<Vec<ByteRange>>; 4] = Default::default();
6491    for si in 0..4 {
6492        if !want[si] {
6493            dict_sections.push(
6494                crate::dict::ChunkedSection::from_parts(
6495                    crate::dict::SectionMeta {
6496                        term_count: 0,
6497                        restart_interval: 1,
6498                        restart_offsets: Vec::new(),
6499                    },
6500                    Vec::new(),
6501                    None,
6502                    cache.clone(),
6503                    si as u8,
6504                )
6505                .with_order(header.term_order()),
6506            );
6507            continue;
6508        }
6509        let section = locate_container_section_ranged(
6510            reader.as_ref(),
6511            header.dictionary_offset,
6512            header.dictionary_len,
6513            si,
6514            4,
6515        )?;
6516        let (meta, entries) = read_dict_dir_ranged(reader.as_ref(), section)?;
6517        // One copy of the chunk ranges, shared by this section's loaders and
6518        // the dictionary's cross-section batch loader.
6519        let ranges: std::sync::Arc<Vec<ByteRange>> = std::sync::Arc::new(
6520            entries
6521                .iter()
6522                .map(|e| ByteRange {
6523                    offset: section.offset + e.start,
6524                    len: (e.end - e.start),
6525                })
6526                .collect(),
6527        );
6528        section_ranges[si] = ranges.clone();
6529        let chunks: Vec<crate::dict::SectionChunk> = entries
6530            .into_iter()
6531            .map(|e| crate::dict::SectionChunk::new(e.first_run, e.key, e.body_start))
6532            .collect();
6533        let chunk_reader = reader.clone();
6534        let codec = header.dict_codec;
6535        let loader_ranges = ranges.clone();
6536        let loader: crate::dict::ChunkLoader = Box::new(move |ci| {
6537            let range = loader_ranges.get(ci)?;
6538            let bytes = chunk_reader.read_at(range.offset, range.len).ok()?;
6539            decompress(codec, &bytes).ok()
6540        });
6541        // Full-section sweeps (export/dump) batch their chunk fetches:
6542        // adjacent chunk ranges coalesce into a handful of range reads.
6543        let bulk_reader = reader.clone();
6544        let bulk: crate::dict::ChunkBulkLoader = Box::new(move |cis| {
6545            let want: Option<Vec<ByteRange>> =
6546                cis.iter().map(|&ci| ranges.get(ci).copied()).collect();
6547            let blobs = read_coalesced(bulk_reader.as_ref(), &want?, DICT_COALESCE_GAP)?;
6548            blobs.iter().map(|b| decompress(codec, b).ok()).collect()
6549        });
6550        dict_sections.push(
6551            crate::dict::ChunkedSection::from_parts(
6552                meta,
6553                chunks,
6554                Some(loader),
6555                cache.clone(),
6556                si as u8,
6557            )
6558            .with_bulk_loader(bulk)
6559            .with_order(header.term_order()),
6560        );
6561    }
6562    let dict_arr: [crate::dict::ChunkedSection; 4] = dict_sections
6563        .try_into()
6564        .map_err(|_| FileError::Container("expected 4 dictionary sections"))?;
6565    // Decoding a result touches several sections at once; this fetches all
6566    // their missing chunks in one batch (pairs arrive section by section in
6567    // ascending chunk order, so ascending in the file, as coalescing wants).
6568    let multi_reader = reader.clone();
6569    let codec = header.dict_codec;
6570    let multi: crate::dict::MultiChunkBulkLoader = Box::new(move |pairs| {
6571        let want: Option<Vec<ByteRange>> = pairs
6572            .iter()
6573            .map(|&(si, ci)| section_ranges.get(si as usize)?.get(ci).copied())
6574            .collect();
6575        let blobs = read_coalesced(multi_reader.as_ref(), &want?, DICT_COALESCE_GAP)?;
6576        blobs.iter().map(|b| decompress(codec, b).ok()).collect()
6577    });
6578    Ok(Dictionary::from_chunked_sections(dict_arr).with_multi_bulk_loader(multi))
6579}
6580
6581/// Run two independent steps of an open: at the same time when the reader can
6582/// keep several reads in flight (`concurrency > 1`) on a target with threads,
6583/// else one after the other. Either way both run to completion and their
6584/// results come back as they would inline; a panic in either propagates.
6585#[cfg(not(target_arch = "wasm32"))]
6586fn open_side_by_side<A, B: Send>(
6587    concurrency: usize,
6588    a: impl FnOnce() -> A,
6589    b: impl FnOnce() -> B + Send,
6590) -> (A, B) {
6591    if concurrency <= 1 {
6592        return (a(), b());
6593    }
6594    std::thread::scope(|scope| {
6595        let hb = scope.spawn(b);
6596        let ra = a();
6597        let rb = hb
6598            .join()
6599            .unwrap_or_else(|panic| std::panic::resume_unwind(panic));
6600        (ra, rb)
6601    })
6602}
6603
6604/// wasm32 has no threads here: always one after the other.
6605#[cfg(target_arch = "wasm32")]
6606fn open_side_by_side<A, B>(
6607    _concurrency: usize,
6608    a: impl FnOnce() -> A,
6609    b: impl FnOnce() -> B,
6610) -> (A, B) {
6611    (a(), b())
6612}
6613
6614/// The `v06-parity` test hook: a 0x05 image converted to 0x06 (`None` for a
6615/// 0x06 image, or one the converter refuses, such as per-graph named graphs).
6616#[cfg(feature = "v06-parity")]
6617fn parity_image(bytes: &[u8]) -> Option<Vec<u8>> {
6618    if crate::v06::open::is_v06(bytes) || bytes.len() < HEADER_LEN {
6619        return None;
6620    }
6621    crate::v06::write::convert_image(bytes, crate::v06::write::WriteOpts::default()).ok()
6622}
6623
6624/// [`Rete::file_layout`] for a `0x06` file: the header, the root, the card
6625/// region, then every front and data section, in file order.
6626fn v06_layout(f: &crate::v06::open::V06File) -> Vec<LayoutSegment> {
6627    let h = &f.h;
6628    let seg = |kind: &'static str, label: String, offset: u64, len: u64| LayoutSegment {
6629        kind,
6630        label,
6631        offset,
6632        len,
6633    };
6634    let mut out = vec![
6635        seg(
6636            "header",
6637            format!("header (fixed {} bytes)", crate::v06::HEADER_LEN),
6638            0,
6639            crate::v06::HEADER_LEN as u64,
6640        ),
6641        seg(
6642            "directory",
6643            "root block (section table, directory roots)".into(),
6644            h.root_offset(),
6645            h.root_len as u64,
6646        ),
6647    ];
6648    let c = h.card_range();
6649    if c.len > 0 {
6650        out.push(seg(
6651            "metadata",
6652            "metadata (dataset card, zstd)".into(),
6653            c.offset,
6654            c.len,
6655        ));
6656    }
6657    let b = h.build_range();
6658    if b.len > 0 {
6659        out.push(seg("build-info", "build record".into(), b.offset, b.len));
6660    }
6661    for s in &f.root.sections {
6662        let Ok(a) = s.abs(h) else { continue };
6663        let kind: &'static str = match s.kind {
6664            crate::v06::kind::SCHEMA => "schema",
6665            crate::v06::kind::DICT_DATA => "dictionary",
6666            crate::v06::kind::INDEX_DATA => "tile",
6667            crate::v06::kind::QUAD_DATA => "named-graphs",
6668            crate::v06::kind::LABEL_DATA => "label-index",
6669            crate::v06::kind::TEXT_DATA => "text-index",
6670            crate::v06::kind::PYRAMID => "pyramid",
6671            _ => "directory",
6672        };
6673        out.push(seg(
6674            kind,
6675            crate::v06::kind::name(s.kind).to_string(),
6676            a.offset,
6677            a.len,
6678        ));
6679    }
6680    out.sort_by_key(|s| s.offset);
6681    out
6682}
6683
6684/// The deferred slots of a `0x06` open: pyramid, text index, label index and
6685/// schema, each faulted on first use.
6686#[allow(clippy::type_complexity)]
6687fn v06_slots(
6688    f: &std::sync::Arc<crate::v06::open::V06File>,
6689) -> (
6690    PyramidSlot,
6691    TextIndexSlot,
6692    SectionSlot<crate::label_index::LabelIndex>,
6693    SectionSlot<crate::schema_section::SchemaMeta>,
6694) {
6695    use crate::v06::{P_LABELS, P_PYRAMID, P_SCHEMA, P_TEXT};
6696    let pyramid = if f.h.has(P_PYRAMID) {
6697        let a = f.clone();
6698        PyramidSlot::Lazy {
6699            loader: Box::new(move || PyramidMeta::decode(&a.pyramid_bytes()?).ok()),
6700            cell: std::sync::OnceLock::new(),
6701        }
6702    } else {
6703        PyramidSlot::Resident(None)
6704    };
6705    let text_index = if f.h.has(P_TEXT) {
6706        let (a, b) = (f.clone(), f.clone());
6707        TextIndexSlot::Lazy {
6708            loader: Box::new(move || a.text_index()?.ok()),
6709            cell: std::sync::OnceLock::new(),
6710            token_table: Box::new(move || b.token_table_bytes()),
6711            token_table_cell: std::sync::OnceLock::new(),
6712        }
6713    } else {
6714        TextIndexSlot::Resident {
6715            index: None,
6716            token_table_len: None,
6717        }
6718    };
6719    let labels = if f.h.has(P_LABELS) {
6720        let a = f.clone();
6721        SectionSlot::Lazy {
6722            loader: Box::new(move || a.label_index()?.ok()),
6723            cell: std::sync::OnceLock::new(),
6724        }
6725    } else {
6726        SectionSlot::Resident(None)
6727    };
6728    let schema = if f.h.has(P_SCHEMA) {
6729        let a = f.clone();
6730        SectionSlot::Lazy {
6731            loader: Box::new(move || a.schema()),
6732            cell: std::sync::OnceLock::new(),
6733        }
6734    } else {
6735        SectionSlot::Resident(None)
6736    };
6737    (pyramid, text_index, labels, schema)
6738}
6739
6740/// The pyramid meta is large on real graphs (tens of MB) and SPARQL never reads
6741/// it, so a ranged open defers its fetch: it faults in only if `pyramid()` is
6742/// called (community / pyramid_tree / label-prefix queries).
6743fn ranged_pyramid_slot<R: RangeReader + Send + Sync + 'static>(
6744    reader: &std::sync::Arc<R>,
6745    header: &Header,
6746) -> PyramidSlot {
6747    if header.pyramid_meta_len == 0 {
6748        return PyramidSlot::Resident(None);
6749    }
6750    let pyr_reader = reader.clone();
6751    let pyr_off = header.pyramid_meta_offset;
6752    let pyr_len = header.pyramid_meta_len;
6753    PyramidSlot::Lazy {
6754        loader: Box::new(move || {
6755            let mb = pyr_reader.read_at(pyr_off, pyr_len).ok()?;
6756            PyramidMeta::decode(&mb).ok()
6757        }),
6758        cell: std::sync::OnceLock::new(),
6759    }
6760}
6761
6762/// The deferred TEXT_INDEX of a ranged open: a text search faults the token
6763/// table on first call (the leading varint then its compressed bytes), then
6764/// fetches individual posting lists by `(offset, len)` — never the whole
6765/// postings blob. A caller that never searches pays nothing.
6766fn ranged_text_index_slot<R: RangeReader + Send + Sync + 'static>(
6767    reader: &std::sync::Arc<R>,
6768    header: &Header,
6769) -> TextIndexSlot {
6770    let (ti_off, ti_len) = header.text_index_payload();
6771    if ti_len == 0 {
6772        return TextIndexSlot::Resident {
6773            index: None,
6774            token_table_len: None,
6775        };
6776    }
6777    let ti_reader = reader.clone();
6778    let probe_reader = reader.clone();
6779    let codec = header.block_codec;
6780    TextIndexSlot::Lazy {
6781        loader: Box::new(move || {
6782            // The section opens with `varint token_table_len`; read enough to
6783            // decode it (a uvarint is ≤ 10 bytes), then fetch the varint + the
6784            // compressed token table as one prefix range.
6785            let prefix_len = read_token_table_len(&*ti_reader, ti_off, ti_len)?;
6786            let prefix = ti_reader.read_at(ti_off, prefix_len).ok()?;
6787            let postings_base = crate::text_index::TextIndex::postings_base(&prefix)? as u64;
6788            let postings_abs = ti_off + postings_base;
6789            let pr = ti_reader.clone();
6790            let posting_loader =
6791                Box::new(move |off: u64, len: u64| pr.read_at(postings_abs + off, len).ok());
6792            crate::text_index::TextIndex::from_token_table(&prefix, codec, posting_loader).ok()
6793        }),
6794        cell: std::sync::OnceLock::new(),
6795        // The same leading varint the loader starts from, asked on its own so a
6796        // caller can *state* the cost of a first search without paying it.
6797        token_table: Box::new(move || read_token_table_len(&*probe_reader, ti_off, ti_len)),
6798        token_table_cell: std::sync::OnceLock::new(),
6799    }
6800}
6801
6802/// The `(label, subject term)` pairs of a legacy (pyramid-block) label search.
6803/// The hits' subjects are scattered over the dictionary, so their chunks are
6804/// faulted in one batch first (one round trip on a remote file, as
6805/// [`text_search_in`] does for full-text hits), then decoded in one
6806/// chunk-ordered pass. A subject that does not resolve drops its hit, as
6807/// before; the order is the search's.
6808fn resolve_label_hits(
6809    dict: &Dictionary,
6810    hits: Vec<&crate::meta::LabelEntry>,
6811) -> Vec<(String, String)> {
6812    let ids: Vec<u32> = hits.iter().map(|e| e.subject).collect();
6813    dict.prefetch_subject_terms(&ids);
6814    hits.iter()
6815        .zip(dict.subject_terms(&ids))
6816        .filter_map(|(e, iri)| Some((e.label.clone(), iri?)))
6817        .collect()
6818}
6819
6820/// Full-text search over `ti`, resolving the matched subject ids through `dict`.
6821/// The engine behind [`Rete::text_search`] and [`SearchView::text_search`].
6822fn text_search_in(
6823    ti: Option<&crate::text_index::TextIndex>,
6824    dict: &Dictionary,
6825    words: &[&str],
6826    prefix: Option<&str>,
6827    limit: usize,
6828) -> Vec<String> {
6829    let Some(ti) = ti else {
6830        return Vec::new();
6831    };
6832    // Each query word is tokenized exactly as at build time (so "Glucose"
6833    // matches the stored "glucose"); a word that splits into several tokens
6834    // requires all of them. AND across every required token + the prefix.
6835    let mut acc: Option<Vec<u32>> = None;
6836    if let Some(p) = prefix {
6837        acc = Some(ti.prefix(&p.to_lowercase()));
6838    }
6839    for w in words {
6840        for tok in crate::text_index::tokenize(w) {
6841            let posting = ti.lookup(&tok);
6842            acc = Some(match acc {
6843                Some(a) => intersect_sorted(&a, &posting),
6844                None => posting,
6845            });
6846            if acc.as_ref().is_some_and(|a| a.is_empty()) {
6847                return Vec::new();
6848            }
6849        }
6850    }
6851    let ids = acc.unwrap_or_default();
6852    let mut out = Vec::with_capacity(if limit > 0 {
6853        limit.min(ids.len())
6854    } else {
6855        ids.len()
6856    });
6857    // The hits' subject IRIs are scattered over many dictionary chunks: fault
6858    // the ones a page can return in one batch (on a remote file, one round
6859    // trip instead of one per chunk). Only warms the cache.
6860    let page = if limit > 0 {
6861        limit.min(ids.len())
6862    } else {
6863        ids.len()
6864    };
6865    dict.prefetch_subject_terms(&ids[..page]);
6866    for id in ids {
6867        if let Some(iri) = dict.subject_term(id) {
6868            out.push(iri);
6869            if limit > 0 && out.len() >= limit {
6870                break;
6871            }
6872        }
6873    }
6874    out
6875}
6876
6877/// A `.rete` opened for **search only**: the dictionary's chunk directories, and
6878/// the TEXT_INDEX / pyramid deferred behind their lazy slots. Nothing else.
6879///
6880/// What makes this cheap is skipping dictionary sections 2 and 3 (object-only
6881/// and predicates) — NOT skipping the index. Measured on the published
6882/// `epfl-infoscience.rete` (1.64 GB), a full [`Rete::open_ranged_lazy`] reads
6883/// 536,947,344 B, of which the six permutation tile directories are **49,940 B
6884/// in 40 reads** and the dictionary is 536,896,380 B in 36 reads: the
6885/// object-only chunk directory alone is 234,400,728 B, because that file stores
6886/// every chunk's first term verbatim. Files built since #198 key it by the
6887/// shortest separator (~600 KB on this graph); the published one still does
6888/// not, and only a rebuild changes that. This view opens sections 0 and 1
6889/// only — both search modes return subject IRIs — and costs **21,554 B in 9
6890/// reads** on the same file. It also skips the index container and defers the
6891/// TEXT_INDEX and pyramid, so it pays only for what it reads: the token table
6892/// on the first search, then one range per posting list and per dictionary
6893/// chunk holding a matched subject. This is what backs `rete search-url`.
6894///
6895/// On a file with a LABEL_INDEX section (kind 8) the dictionary is not opened
6896/// at all until a full-text search needs it: the section stores each subject's
6897/// term, so a prefix search costs the header, the section head, one directory
6898/// node per level and one data block.
6899pub struct SearchView {
6900    header: Header,
6901    dict: SectionSlot<Dictionary>,
6902    pyramid: PyramidSlot,
6903    text_index: TextIndexSlot,
6904    labels: SectionSlot<crate::label_index::LabelIndex>,
6905}
6906
6907impl SearchView {
6908    /// Open for search over any [`RangeReader`] (HTTP or local): header +
6909    /// dictionary chunk directories only (none at all on a file with a
6910    /// LABEL_INDEX, until a text search needs them), with the text index,
6911    /// label index and pyramid faulting in on first use.
6912    pub fn open_ranged<R: RangeReader + Send + Sync + 'static>(
6913        reader: R,
6914    ) -> Result<Self, FileError> {
6915        Self::open_ranged_with(reader, crate::v06::FIRST_READ)
6916    }
6917
6918    /// [`open_ranged`](Self::open_ranged) with an explicit first-read size
6919    /// (see [`Rete::open_ranged_lazy_with`]).
6920    pub fn open_ranged_with<R: RangeReader + Send + Sync + 'static>(
6921        reader: R,
6922        first_read: u64,
6923    ) -> Result<Self, FileError> {
6924        let first_len = first_read.max(HEADER_LEN as u64).min(reader.len());
6925        let first: std::sync::Arc<[u8]> = std::sync::Arc::from(reader.read_at(0, first_len)?);
6926        let reader = crate::reader::PrefixReader::new(reader, first.clone());
6927        if crate::v06::open::is_v06(&first) {
6928            let rc = reader.concurrency();
6929            let r: std::sync::Arc<dyn RangeReader + Send + Sync> = std::sync::Arc::new(reader);
6930            let f = std::sync::Arc::new(crate::v06::open::V06File::open(r, &first)?);
6931            let header = crate::v06::logical_header(&f.h);
6932            let (pyramid, text_index, labels, _) = v06_slots(&f);
6933            // A derived IRI is rebuilt from the index and the object dictionary,
6934            // so a file that has some opens both.
6935            let derived = f.h.flags & crate::v06::F_DERIVED != 0;
6936            let want = [true, true, derived, derived];
6937            let make = move |f: &crate::v06::open::V06File| -> Result<Dictionary, FileError> {
6938                let d = f.dictionary(want)?;
6939                if derived {
6940                    d.wire_derived(std::sync::Arc::new(f.index(rc)?));
6941                }
6942                Ok(d)
6943            };
6944            let dict = if f.h.has(crate::v06::P_LABELS) {
6945                let a = f.clone();
6946                SectionSlot::Lazy {
6947                    loader: Box::new(move || make(&a).ok()),
6948                    cell: std::sync::OnceLock::new(),
6949                }
6950            } else {
6951                SectionSlot::Resident(Some(make(&f)?))
6952            };
6953            return Ok(Self {
6954                header,
6955                dict,
6956                pyramid,
6957                text_index,
6958                labels,
6959            });
6960        }
6961        let head = reader.read_at(0, HEADER_LEN as u64)?;
6962        let header = Header::from_bytes(&head)?;
6963        let reader = std::sync::Arc::new(reader);
6964        // Sections 0 (shared) and 1 (subject-only) are the only ones a search
6965        // resolves — both modes return subject IRIs. Skipping 2 and 3 is what
6966        // makes this open cheap where a full one is not.
6967        let dict = if header.label_index_section().is_some() {
6968            let (r, h) = (reader.clone(), header.clone());
6969            SectionSlot::Lazy {
6970                loader: Box::new(move || {
6971                    ranged_chunked_dictionary(&r, &h, [true, true, false, false]).ok()
6972                }),
6973                cell: std::sync::OnceLock::new(),
6974            }
6975        } else {
6976            SectionSlot::Resident(Some(ranged_chunked_dictionary(
6977                &reader,
6978                &header,
6979                [true, true, false, false],
6980            )?))
6981        };
6982        let pyramid = ranged_pyramid_slot(&reader, &header);
6983        let text_index = ranged_text_index_slot(&reader, &header);
6984        let labels = ranged_label_slot(&reader, &header);
6985        Ok(Self {
6986            header,
6987            dict,
6988            pyramid,
6989            text_index,
6990            labels,
6991        })
6992    }
6993
6994    /// Whether the file can answer a prefix search: it carries a LABEL_INDEX
6995    /// section or a pyramid (whose meta may hold the legacy label block). A
6996    /// header read, no fault.
6997    pub fn has_labels(&self) -> bool {
6998        self.header.label_index_section().is_some() || self.header.pyramid_meta_len > 0
6999    }
7000
7001    /// Whether the file carries a LABEL_INDEX section (kind 8).
7002    pub fn has_label_index(&self) -> bool {
7003        self.header.label_index_section().is_some()
7004    }
7005
7006    /// The file header (already fetched by [`open_ranged`](Self::open_ranged)).
7007    pub fn header(&self) -> &Header {
7008        &self.header
7009    }
7010
7011    /// Whether the file carries a TEXT_INDEX section — a header read, no fault.
7012    pub fn has_text_index(&self) -> bool {
7013        self.header.text_index_len > 0
7014    }
7015
7016    /// Whether the file carries a pyramid (where the label index lives) — a
7017    /// header read, no fault.
7018    pub fn has_pyramid(&self) -> bool {
7019        self.header.pyramid_meta_len > 0
7020    }
7021
7022    /// Full-text search — see [`Rete::text_search`] for the semantics. Faults
7023    /// the token table on the first call, then only the queried postings.
7024    pub fn text_search(&self, words: &[&str], prefix: Option<&str>, limit: usize) -> Vec<String> {
7025        let Some(dict) = self.dict.get() else {
7026            return Vec::new();
7027        };
7028        text_search_in(self.text_index.index(), dict, words, prefix, limit)
7029    }
7030
7031    /// What that first search will cost — see
7032    /// [`Rete::text_index_token_table_len`]. One ≤10-byte range read, so a
7033    /// caller can quote the figure before deciding to search.
7034    pub fn text_index_token_table_len(&self) -> Option<u64> {
7035        self.text_index.token_table_len()
7036    }
7037
7038    /// Label-prefix search — see [`Rete::prefix_search`]. Reads the
7039    /// LABEL_INDEX section when the file has one; otherwise faults the pyramid
7040    /// (which carries the legacy label block), the pricier path.
7041    pub fn prefix_search(&self, prefix: &str, limit: usize) -> Vec<(String, String)> {
7042        if let Some(li) = self.labels.get() {
7043            return li.prefix_search(prefix, limit);
7044        }
7045        let pyr = match &self.pyramid {
7046            PyramidSlot::Resident(p) => p.as_ref(),
7047            PyramidSlot::Lazy { loader, cell } => cell.get_or_init(loader).as_ref(),
7048        };
7049        let (Some(pyr), Some(dict)) = (pyr, self.dict.get()) else {
7050            return Vec::new();
7051        };
7052        resolve_label_hits(dict, pyr.prefix_search(prefix, limit))
7053    }
7054}
7055
7056/// A lightweight, overview-only view of a file: the pyramid summary graph plus
7057/// just enough dictionary to label predicates. Fetched via ranges *without*
7058/// touching the (large) triple index — the "load the coarse graph first" path
7059/// from SPEC.md §7.2.
7060#[must_use]
7061pub struct SummaryView {
7062    pub round: u32,
7063    pub summary: Vec<SuperEdge>,
7064    /// The shipped `subClassOf` hierarchy (v2 schema pyramid; empty on v1 files).
7065    pub class_hierarchy: Vec<ClassNode>,
7066    /// Per-level type rollups — the leveled legend, read index-free.
7067    pub level_rollups: Vec<LevelRollup>,
7068    /// Per-level lateral class-relation graph (the non-`is-a` connections).
7069    pub level_links: Vec<LevelLinks>,
7070    /// Per-community descriptors (Phase 4 progressive refinement; may be empty).
7071    pub descriptors: Vec<CommunityDescriptor>,
7072    /// `subClassOf` cycles (v2.1; empty on older files).
7073    pub subclass_cycles: Vec<Vec<String>>,
7074    /// `owl:disjointWith` class pairs (v2.1; empty on older files).
7075    pub disjoint_pairs: Vec<(String, String)>,
7076    /// `owl:equivalentClass` class pairs (v2.1; empty on older files).
7077    pub equivalent_pairs: Vec<(String, String)>,
7078    dict: Dictionary,
7079}
7080
7081impl SummaryView {
7082    /// Read header → dictionary → pyramid-meta only (skips the index container).
7083    pub fn open_ranged<R: RangeReader>(reader: &R) -> Result<Option<Self>, FileError> {
7084        let head = reader.read_at(0, (HEADER_LEN as u64).min(reader.len()))?;
7085        let header = Header::from_bytes(&head)?;
7086        if header.pyramid_meta_len == 0 {
7087            return Ok(None);
7088        }
7089        if crate::v06::reads(header.version) {
7090            return Self::open_ranged_v06(reader, &header, head).map(Some);
7091        }
7092
7093        let dict_bytes = reader.read_at(header.dictionary_offset, header.dictionary_len)?;
7094        let dict =
7095            decode_dictionary_container(&dict_bytes, header.dict_codec, header.term_order())?;
7096
7097        let mb = reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?;
7098        let mut meta =
7099            PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7100        // A file whose schema moved to its own section (kind 9) keeps only the
7101        // community material here: take the schema fields from that section.
7102        if let Some(s) = read_schema_with_header(reader, &header) {
7103            meta.class_hierarchy = s.class_hierarchy;
7104            meta.level_rollups = s.level_rollups;
7105            meta.level_links = s.level_links;
7106            meta.subclass_cycles = s.subclass_cycles;
7107            meta.disjoint_pairs = s.disjoint_pairs;
7108            meta.equivalent_pairs = s.equivalent_pairs;
7109        }
7110
7111        Ok(Some(SummaryView {
7112            round: meta.round,
7113            summary: meta.summary,
7114            class_hierarchy: meta.class_hierarchy,
7115            level_rollups: meta.level_rollups,
7116            level_links: meta.level_links,
7117            descriptors: meta.descriptors,
7118            subclass_cycles: meta.subclass_cycles,
7119            disjoint_pairs: meta.disjoint_pairs,
7120            equivalent_pairs: meta.equivalent_pairs,
7121            dict,
7122        }))
7123    }
7124
7125    /// [`open_ranged`](Self::open_ranged) for a `0x06` file: the pyramid
7126    /// section, the schema, and the dictionary (its leaves and chunks read
7127    /// whole, as the `0x05` path reads the whole dictionary container).
7128    fn open_ranged_v06<R: RangeReader>(
7129        reader: &R,
7130        header: &Header,
7131        head: Vec<u8>,
7132    ) -> Result<Self, FileError> {
7133        use crate::v06::{kind, open};
7134        // One first read, then every range the view needs past it — the
7135        // pyramid, the schema, the dictionary's leaves and chunks — in one
7136        // batch.
7137        let mut first = head;
7138        let want_first = crate::v06::FIRST_READ.min(reader.len());
7139        if want_first > first.len() as u64 {
7140            let at = first.len() as u64;
7141            first.extend_from_slice(&reader.read_at(at, want_first - at)?);
7142        }
7143        // A root longer than the first read (a large dictionary directory) is
7144        // read too: the sparse reader below holds only what is fetched here,
7145        // and opening the file through it needs the whole root.
7146        let root_end = (crate::v06::HEADER_LEN as u64)
7147            .saturating_add(crate::v06::Header06::from_bytes(&first)?.root_len as u64)
7148            .min(reader.len());
7149        if root_end > first.len() as u64 {
7150            let at = first.len() as u64;
7151            first.extend_from_slice(&reader.read_at(at, root_end - at)?);
7152        }
7153        let pre = open::Prefixed {
7154            first: &first,
7155            r: reader,
7156        };
7157        let (h, root) = open::read_front(&pre)?;
7158        let want: Vec<ByteRange> = [
7159            kind::PYRAMID,
7160            kind::SCHEMA,
7161            kind::DICT_LEAVES,
7162            kind::DICT_DATA,
7163        ]
7164        .iter()
7165        .filter_map(|&k| root.section(k).and_then(|s| s.abs(&h).ok()))
7166        .collect();
7167        let sparse = open::SparseReader::fetch(&pre, first.clone(), &want)?;
7168        let mb = open::read_section(&sparse, &h, &root, kind::PYRAMID)?
7169            .ok_or(FileError::Container("0x06 file has no pyramid"))?;
7170        let mut meta =
7171            PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7172        if let Some(s) = read_schema_with_header(&sparse, header) {
7173            meta.class_hierarchy = s.class_hierarchy;
7174            meta.level_rollups = s.level_rollups;
7175            meta.level_links = s.level_links;
7176            meta.subclass_cycles = s.subclass_cycles;
7177            meta.disjoint_pairs = s.disjoint_pairs;
7178            meta.equivalent_pairs = s.equivalent_pairs;
7179        }
7180        let f = open::V06File::open(std::sync::Arc::new(sparse), &first)?;
7181        let dict = f.dictionary([true; 4])?;
7182        Ok(SummaryView {
7183            round: meta.round,
7184            summary: meta.summary,
7185            class_hierarchy: meta.class_hierarchy,
7186            level_rollups: meta.level_rollups,
7187            level_links: meta.level_links,
7188            descriptors: meta.descriptors,
7189            subclass_cycles: meta.subclass_cycles,
7190            disjoint_pairs: meta.disjoint_pairs,
7191            equivalent_pairs: meta.equivalent_pairs,
7192            dict,
7193        })
7194    }
7195
7196    /// A schema-only view of a file **without a pyramid**, from its SCHEMA
7197    /// section (kind 9): the hierarchy, levels and axioms, with an empty
7198    /// community summary. Reads the header and the SCHEMA section only — the
7199    /// section carries its own class and predicate strings, so no dictionary.
7200    /// `Ok(None)` when the file has no SCHEMA section. Callers that need the
7201    /// summary (`cost`, federation source selection, progressive queries) use
7202    /// [`open_ranged`](Self::open_ranged), which keeps requiring a pyramid.
7203    pub fn open_schema_ranged<R: RangeReader>(reader: &R) -> Result<Option<Self>, FileError> {
7204        let head = reader.read_at(0, HEADER_LEN as u64)?;
7205        let header = Header::from_bytes(&head)?;
7206        let Some(s) = read_schema_with_header(reader, &header) else {
7207            return Ok(None);
7208        };
7209        let dict = crate::DictionaryBuilder::new().build();
7210        Ok(Some(SummaryView {
7211            round: 0,
7212            summary: Vec::new(),
7213            class_hierarchy: s.class_hierarchy,
7214            level_rollups: s.level_rollups,
7215            level_links: s.level_links,
7216            descriptors: Vec::new(),
7217            subclass_cycles: s.subclass_cycles,
7218            disjoint_pairs: s.disjoint_pairs,
7219            equivalent_pairs: s.equivalent_pairs,
7220            dict,
7221        }))
7222    }
7223
7224    /// Number of semantic-zoom levels in the schema pyramid (0 if none shipped).
7225    pub fn level_count(&self) -> usize {
7226        self.level_rollups.len()
7227    }
7228
7229    /// The type rollup at semantic level `k` (0 = coarsest/most abstract), or
7230    /// `None` if `k` is out of range. Index-free — answered from the pyramid-meta.
7231    pub fn level_rollup(&self, k: usize) -> Option<&LevelRollup> {
7232        self.level_rollups.get(k)
7233    }
7234
7235    /// Resolve a predicate ID in the summary to its term.
7236    pub fn predicate_term(&self, id: u32) -> Option<String> {
7237        self.dict.predicate_term(id)
7238    }
7239
7240    /// Exact number of triples using `predicate`, summed from the summary's
7241    /// superedge counts — answered without ever reading the triple index.
7242    pub fn predicate_total(&self, predicate: &str) -> u32 {
7243        match self.dict.predicate_id(predicate) {
7244            Some(pid) => self
7245                .summary
7246                .iter()
7247                .filter(|e| e.predicate == pid)
7248                .map(|e| e.count)
7249                .sum(),
7250            None => 0,
7251        }
7252    }
7253
7254    /// All predicates with their exact triple totals, descending by count.
7255    pub fn predicate_totals(&self) -> Vec<(String, u32)> {
7256        let mut by_pred: std::collections::BTreeMap<u32, u32> = std::collections::BTreeMap::new();
7257        for e in &self.summary {
7258            *by_pred.entry(e.predicate).or_default() += e.count;
7259        }
7260        let mut out: Vec<(String, u32)> = by_pred
7261            .into_iter()
7262            .filter_map(|(pid, c)| self.dict.predicate_term(pid).map(|t| (t, c)))
7263            .collect();
7264        out.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
7265        out
7266    }
7267
7268    /// Number of communities the summary spans (distinct supernode endpoints).
7269    pub fn community_count(&self) -> usize {
7270        let mut comms = std::collections::BTreeSet::new();
7271        for e in &self.summary {
7272            comms.insert(e.s_comm);
7273            comms.insert(e.o_comm);
7274        }
7275        comms.len()
7276    }
7277
7278    /// **Index-free T-Box coherence (Tier-0).** Detect schema-level incoherent
7279    /// points purely from the shipped schema pyramid — no triple index, no
7280    /// instance data, O(ontology) regardless of graph size:
7281    /// - `subclass-cycle`: a set of classes that are mutually `rdfs:subClassOf`.
7282    /// - `unsatisfiable-class`: a class whose ancestor closure (over all parents,
7283    ///   folded through `owl:equivalentClass`) contains both ends of an
7284    ///   `owl:disjointWith` pair, so no individual can ever be one.
7285    ///
7286    /// Soundness is bounded by what the pyramid ships: the `subClassOf` hierarchy
7287    /// is capped (`MAX_HIERARCHY` in `schema_pyramid`), so on a very large ontology
7288    /// a pruned ancestor can hide an unsatisfiable class (a false *coherent*, never
7289    /// a false *incoherent*). Instance-level clashes (a node typed into disjoint
7290    /// classes, functional-property clashes) are NOT visible here — they need the
7291    /// A-Box (Tier-1/Tier-2 `reason`).
7292    pub fn tbox_coherence(&self) -> Vec<crate::reason::Inconsistency> {
7293        schema_coherence(
7294            &self.class_hierarchy,
7295            &self.subclass_cycles,
7296            &self.disjoint_pairs,
7297            &self.equivalent_pairs,
7298        )
7299    }
7300
7301    /// True when [`tbox_coherence`](Self::tbox_coherence) finds no schema-level
7302    /// incoherent point.
7303    pub fn tbox_is_coherent(&self) -> bool {
7304        self.tbox_coherence().is_empty()
7305    }
7306}
7307
7308/// Compute T-Box coherence points from the schema-pyramid fields alone — no
7309/// dictionary, no index, no instance data. Shared by [`SummaryView::tbox_coherence`]
7310/// and the dictionary-free [`read_schema_coherence_ranged`]. Emits `subclass-cycle`
7311/// and `unsatisfiable-class` (a class whose ancestor closure — over all parents,
7312/// folded through `owl:equivalentClass` — contains both ends of a disjoint pair).
7313pub fn schema_coherence(
7314    class_hierarchy: &[ClassNode],
7315    subclass_cycles: &[Vec<String>],
7316    disjoint_pairs: &[(String, String)],
7317    equivalent_pairs: &[(String, String)],
7318) -> Vec<crate::reason::Inconsistency> {
7319    use crate::reason::Inconsistency;
7320    use std::collections::{BTreeMap, BTreeSet, VecDeque};
7321    const MAX_REACH: usize = 100_000;
7322
7323    let mut out: Vec<Inconsistency> = Vec::new();
7324
7325    for cyc in subclass_cycles {
7326        let detail = if cyc.len() == 1 {
7327            format!("{} is rdfs:subClassOf itself (a cycle)", cyc[0])
7328        } else {
7329            format!(
7330                "classes {{{}}} are mutually rdfs:subClassOf (a cycle)",
7331                cyc.join(", ")
7332            )
7333        };
7334        out.push(Inconsistency {
7335            kind: "subclass-cycle",
7336            detail,
7337        });
7338    }
7339
7340    if !disjoint_pairs.is_empty() {
7341        // Upward adjacency: subClassOf parents + bidirectional equivalence.
7342        let mut adj: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
7343        for n in class_hierarchy {
7344            let e = adj.entry(n.class.as_str()).or_default();
7345            for p in &n.parents {
7346                e.push(p.as_str());
7347            }
7348        }
7349        for (a, b) in equivalent_pairs {
7350            adj.entry(a.as_str()).or_default().push(b.as_str());
7351            adj.entry(b.as_str()).or_default().push(a.as_str());
7352        }
7353
7354        // Candidate focus classes: every class named anywhere in the schema.
7355        let mut focuses: BTreeSet<&str> =
7356            class_hierarchy.iter().map(|n| n.class.as_str()).collect();
7357        for (a, b) in disjoint_pairs.iter().chain(equivalent_pairs) {
7358            focuses.insert(a.as_str());
7359            focuses.insert(b.as_str());
7360        }
7361
7362        let mut seen: BTreeSet<&str> = BTreeSet::new();
7363        for &c in &focuses {
7364            // reach(c) = {c} ∪ ancestors (capped BFS over `adj`).
7365            let mut reach: BTreeSet<&str> = BTreeSet::new();
7366            let mut q: VecDeque<&str> = VecDeque::new();
7367            reach.insert(c);
7368            q.push_back(c);
7369            while let Some(x) = q.pop_front() {
7370                if reach.len() > MAX_REACH {
7371                    break;
7372                }
7373                if let Some(ns) = adj.get(x) {
7374                    for &p in ns {
7375                        if reach.insert(p) {
7376                            q.push_back(p);
7377                        }
7378                    }
7379                }
7380            }
7381            for (x, y) in disjoint_pairs {
7382                if reach.contains(x.as_str()) && reach.contains(y.as_str()) && seen.insert(c) {
7383                    out.push(Inconsistency {
7384                        kind: "unsatisfiable-class",
7385                        detail: format!(
7386                            "{c} is a subclass of both {x} and {y}, which are \
7387                             owl:disjointWith — no individual can be a {c}"
7388                        ),
7389                    });
7390                    break;
7391                }
7392            }
7393        }
7394    }
7395
7396    out.sort_by(|a, b| (a.kind, &a.detail).cmp(&(b.kind, &b.detail)));
7397    out
7398}
7399
7400/// **Dictionary-free Tier-0 coherence read.** Fetch only the header and the
7401/// pyramid-meta range (2 small range reads) and run [`schema_coherence`] over the
7402/// schema pyramid — never touching the **dictionary** (which a literal-heavy file
7403/// makes large) or the triple index. `Ok(None)` if the file ships no pyramid.
7404///
7405/// This is what makes the Tier-0 check cheap on big graphs: the schema pyramid
7406/// carries its own class-string table, so coherence needs none of the dictionary.
7407pub fn read_schema_coherence_ranged<R: RangeReader>(
7408    reader: &R,
7409) -> Result<Option<Vec<crate::reason::Inconsistency>>, FileError> {
7410    let head = reader.read_at(0, HEADER_LEN as u64)?;
7411    let header = Header::from_bytes(&head)?;
7412    // The SCHEMA section (kind 9) first: one range, and no pyramid needed.
7413    if let Some(s) = read_schema_with_header(reader, &header) {
7414        return Ok(Some(schema_coherence(
7415            &s.class_hierarchy,
7416            &s.subclass_cycles,
7417            &s.disjoint_pairs,
7418            &s.equivalent_pairs,
7419        )));
7420    }
7421    if header.pyramid_meta_len == 0 {
7422        return Ok(None);
7423    }
7424    // Fast path: the header records the trailing schema block's length, so read ONLY
7425    // that block (at the end of pyramid-meta) — never the community summary, the
7426    // dictionary, or the index. This is what makes Tier-0 flat at any graph size.
7427    if header.schema_meta_len > 0 && (header.schema_meta_len as u64) <= header.pyramid_meta_len {
7428        let off =
7429            header.pyramid_meta_offset + header.pyramid_meta_len - header.schema_meta_len as u64;
7430        let block = reader.read_at(off, header.schema_meta_len as u64)?;
7431        let (hierarchy, cycles, disjoint, equivalent) = crate::meta::decode_schema_block(&block)
7432            .map_err(|_| FileError::Container("malformed schema block"))?;
7433        return Ok(Some(schema_coherence(
7434            &hierarchy,
7435            &cycles,
7436            &disjoint,
7437            &equivalent,
7438        )));
7439    }
7440    // Fallback (pre-v0.2.1 files with no header field): decode the whole pyramid-meta.
7441    let mb = if crate::v06::reads(header.version) {
7442        crate::v06::open::read_pyramid(reader)?.unwrap_or_default()
7443    } else {
7444        reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?
7445    };
7446    let meta =
7447        PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?;
7448    Ok(Some(schema_coherence(
7449        &meta.class_hierarchy,
7450        &meta.subclass_cycles,
7451        &meta.disjoint_pairs,
7452        &meta.equivalent_pairs,
7453    )))
7454}
7455
7456/// The **schema summary** (per-class histogram + class relations at the finest
7457/// level) read over a [`RangeReader`] from the schema pyramid alone — the
7458/// index-free, range-readable source for a Schema view of a remote graph. Returns
7459/// `(classes, relations)` with `classes = [(class_iri, count)]` and `relations =
7460/// [(s_class, predicate, o_class, count)]`; `None` when the file has no schema
7461/// pyramid. Like [`read_schema_coherence_ranged`], it reads only the trailing
7462/// schema block, so it stays flat at any graph size.
7463#[allow(clippy::type_complexity)]
7464pub fn read_schema_summary_ranged<R: RangeReader>(
7465    reader: &R,
7466) -> Result<Option<(Vec<(String, u64)>, Vec<(String, String, String, u64)>)>, FileError> {
7467    let head = reader.read_at(0, HEADER_LEN as u64)?;
7468    let header = Header::from_bytes(&head)?;
7469    // The SCHEMA section (kind 9) first: one range, and no pyramid needed.
7470    let meta = if let Some(s) = read_schema_with_header(reader, &header) {
7471        PyramidMeta::new(0, Vec::new(), &[]).with_schema(
7472            Vec::new(),
7473            s.level_rollups,
7474            s.level_links,
7475            Vec::new(),
7476            Vec::new(),
7477            Vec::new(),
7478            Vec::new(),
7479        )
7480    } else {
7481        if header.pyramid_meta_len == 0 {
7482            return Ok(None);
7483        }
7484        if header.schema_meta_len > 0 && (header.schema_meta_len as u64) <= header.pyramid_meta_len
7485        {
7486            let off = header.pyramid_meta_offset + header.pyramid_meta_len
7487                - header.schema_meta_len as u64;
7488            let block = reader.read_at(off, header.schema_meta_len as u64)?;
7489            let summary = crate::meta::decode_schema_block_summary(&block)
7490                .map_err(|_| FileError::Container("malformed schema block"))?;
7491            return Ok(Some(summary));
7492        }
7493        // Fallback (pre-v0.2.1 files): decode the whole pyramid-meta, pull finest levels.
7494        let mb = if crate::v06::reads(header.version) {
7495            crate::v06::open::read_pyramid(reader)?.unwrap_or_default()
7496        } else {
7497            reader.read_at(header.pyramid_meta_offset, header.pyramid_meta_len)?
7498        };
7499        PyramidMeta::decode(&mb).map_err(|_| FileError::Container("malformed pyramid meta"))?
7500    };
7501    if meta.level_rollups.is_empty() && meta.level_links.is_empty() {
7502        return Ok(None);
7503    }
7504    let classes = meta
7505        .level_rollups
7506        .iter()
7507        .max_by_key(|r| r.depth)
7508        .map(|r| r.classes.clone())
7509        .unwrap_or_default();
7510    let relations = meta
7511        .level_links
7512        .iter()
7513        .max_by_key(|l| l.depth)
7514        .map(|l| {
7515            l.links
7516                .iter()
7517                .map(|c| {
7518                    (
7519                        c.s_class.clone(),
7520                        c.predicate.clone(),
7521                        c.o_class.clone(),
7522                        c.count,
7523                    )
7524                })
7525                .collect()
7526        })
7527        .unwrap_or_default();
7528    Ok(Some((classes, relations)))
7529}
7530
7531/// Every dictionary section's chunk **routing keys**, exactly as stored, from a
7532/// finished file image. Test-only: it exists so a test in another module of
7533/// this crate (`extbuild`) can assert on the directory the writers produce
7534/// without any of this becoming public surface.
7535#[cfg(test)]
7536pub(crate) fn dict_chunk_keys_for_test(image: &[u8]) -> Vec<Vec<Vec<u8>>> {
7537    if crate::v06::open::is_v06(image) {
7538        // 0x06: the routing keys sit in the leaf directories of the dictionary.
7539        use crate::v06::{decode_chunk_leaf, decode_dict_root, kind, open, tag};
7540        let r = crate::reader::SliceReader::new(image);
7541        let (h, root) = open::read_front(&r).unwrap();
7542        let roots = decode_dict_root(root.unit(tag::DICT).unwrap(), h.version).unwrap();
7543        let leaves = open::read_section(&r, &h, &root, kind::DICT_LEAVES)
7544            .unwrap()
7545            .unwrap();
7546        return roots
7547            .iter()
7548            .map(|s| {
7549                s.leaves
7550                    .iter()
7551                    .flat_map(|(l, _, _, _)| {
7552                        let stored = &leaves[l.off as usize..(l.off + l.len) as usize];
7553                        let raw = decompress(h.internal_codec, stored).unwrap();
7554                        decode_chunk_leaf(&raw, l.n).unwrap()
7555                    })
7556                    .map(|c| c.key)
7557                    .collect()
7558            })
7559            .collect();
7560    }
7561    let header = Header::from_bytes(&image[..HEADER_LEN]).unwrap();
7562    let s = header.dictionary_offset as usize;
7563    let e = s + header.dictionary_len as usize;
7564    decode_container(&image[s..e], CODEC_NONE)
7565        .unwrap()
7566        .iter()
7567        .map(|payload| {
7568            let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
7569            entries.into_iter().map(|c| c.key).collect()
7570        })
7571        .collect()
7572}
7573
7574#[cfg(test)]
7575mod tests {
7576    use super::*;
7577    use super::{
7578        write_dataset_v05 as write_dataset,
7579        write_dataset_with_metadata_v05 as write_dataset_with_metadata,
7580        write_file_v05 as write_file,
7581    };
7582    use crate::dictionary::DictionaryBuilder;
7583    use crate::index::GraphIndexBuilder;
7584
7585    /// The pure-Rust decode path (wasm, `--no-default-features`) reads a zstd
7586    /// frame back byte for byte. Guards the `ruzstd` API across upgrades.
7587    #[cfg(not(feature = "compression"))]
7588    #[test]
7589    fn ruzstd_decodes_a_zstd_frame() {
7590        let data: Vec<u8> = (0..50_000u32)
7591            .flat_map(|i| (i % 251).to_le_bytes())
7592            .collect();
7593        let frame = ruzstd::encoding::compress_to_vec(
7594            &data[..],
7595            ruzstd::encoding::CompressionLevel::Fastest,
7596        );
7597        assert!(frame.len() < data.len());
7598        assert_eq!(decompress(CODEC_ZSTD, &frame).unwrap(), data);
7599    }
7600
7601    /// Bounded decompression: a frame whose header declares more than the
7602    /// bound is refused before decoding; one that states no size is decoded at
7603    /// most one byte past the bound; one that lies about its size fails; and
7604    /// the size a header states is read without decoding.
7605    #[cfg(feature = "compression")]
7606    #[test]
7607    fn decompression_is_bounded() {
7608        let data: Vec<u8> = (0..100_000u32).map(|i| (i % 251) as u8).collect();
7609        let sized = zstd::bulk::compress(&data, 3).unwrap();
7610        assert_eq!(zstd_frame_content_size(&sized), Some(100_000));
7611        assert_eq!(block_raw_len(CODEC_ZSTD, &sized, 1 << 20).unwrap(), 100_000);
7612        assert_eq!(
7613            decompress_bounded(CODEC_ZSTD, &sized, 100_000).unwrap(),
7614            data
7615        );
7616        assert!(decompress_bounded(CODEC_ZSTD, &sized, 99_999).is_err());
7617        // A streamed frame states no size: bounded by decoding.
7618        let stream = zstd::encode_all(&data[..], 3).unwrap();
7619        assert_eq!(zstd_frame_content_size(&stream), None);
7620        assert_eq!(
7621            block_raw_len(CODEC_ZSTD, &stream, 1 << 20).unwrap(),
7622            100_000
7623        );
7624        assert_eq!(
7625            decompress_bounded(CODEC_ZSTD, &stream, 100_000).unwrap(),
7626            data
7627        );
7628        assert!(decompress_bounded(CODEC_ZSTD, &stream, 50_000).is_err());
7629        // A header that under-states its content fails instead of growing.
7630        let mut liar = sized.clone();
7631        let fhd = liar[4];
7632        let fcs_at = 5 + usize::from(fhd & 0x20 == 0);
7633        liar[fcs_at] = liar[fcs_at].wrapping_sub(1);
7634        assert!(decompress_bounded(CODEC_ZSTD, &liar, 200_000).is_err());
7635        // Uncompressed blocks: the length is the size.
7636        assert!(decompress_bounded(CODEC_NONE, &data, 99_999).is_err());
7637        assert_eq!(
7638            decompress_bounded(CODEC_NONE, &data, 100_000).unwrap(),
7639            data
7640        );
7641        assert_eq!(zstd_frame_content_size(b"RETE"), None);
7642    }
7643
7644    /// A three-permutation file must survive every opener — resident, ranged,
7645    /// ranged-lazy — and answer every pattern shape with the rows the
7646    /// six-permutation twin gives, including through a named graph's own
7647    /// container. The mask is a file-level fact, so getting it to the named
7648    /// graphs' decoder is a separate thing that can be forgotten.
7649    #[test]
7650    fn three_permutation_file_round_trips_every_open_path() {
7651        use crate::index::PermSet;
7652        use crate::reader::SliceReader;
7653
7654        let build = |perms: PermSet| {
7655            let mut db = crate::DictionaryBuilder::new();
7656            let mut triples = Vec::new();
7657            for i in 0..60u32 {
7658                let (s, p, o) = (
7659                    format!("<http://ex/s{}>", i % 11),
7660                    format!("<http://ex/p{}>", i % 3),
7661                    format!("<http://ex/o{}>", i % 7),
7662                );
7663                db.observe(&s, &p, &o);
7664                triples.push((s, p, o));
7665            }
7666            let dict = db.build();
7667            let ids: Vec<(u32, u32, u32)> = triples
7668                .iter()
7669                .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
7670                .collect();
7671            let def = GraphIndexBuilder::from_triples(ids.clone())
7672                .with_tile_budget(96)
7673                .with_perms(perms)
7674                .build();
7675            let named = GraphIndexBuilder::from_triples(ids)
7676                .with_tile_budget(96)
7677                .with_perms(perms)
7678                .build();
7679            write_dataset(
7680                &dict,
7681                &def,
7682                &[("<http://ex/g>".to_string(), named)],
7683                true,
7684                &[],
7685                0,
7686            )
7687        };
7688
7689        let six = build(PermSet::ALL);
7690        let three = build(PermSet::CORE);
7691        assert!(three.len() < six.len());
7692        assert_eq!(Header::from_bytes(&three).unwrap().perms, PermSet::CORE);
7693        assert_eq!(Header::from_bytes(&six).unwrap().perms, PermSet::ALL);
7694
7695        let shapes: Vec<(Option<&str>, Option<&str>, Option<&str>)> = vec![
7696            (None, None, None),
7697            (Some("<http://ex/s3>"), None, None),
7698            (None, Some("<http://ex/p1>"), None),
7699            (None, None, Some("<http://ex/o5>")),
7700            (Some("<http://ex/s3>"), Some("<http://ex/p1>"), None),
7701            (Some("<http://ex/s3>"), None, Some("<http://ex/o5>")),
7702            (None, Some("<http://ex/p1>"), Some("<http://ex/o5>")),
7703            (
7704                Some("<http://ex/s3>"),
7705                Some("<http://ex/p1>"),
7706                Some("<http://ex/o5>"),
7707            ),
7708        ];
7709
7710        for (image, tag) in [(&six, "six"), (&three, "three")] {
7711            let resident = Rete::open(image).unwrap();
7712            let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
7713            let ranged = Rete::open_ranged(&SliceReader::new(leaked)).unwrap();
7714            let lazy =
7715                Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
7716            for (s, p, o) in &shapes {
7717                let want: Vec<_> = {
7718                    let mut v = Rete::open(&six).unwrap().query(*s, *p, *o);
7719                    v.sort();
7720                    v
7721                };
7722                for (got, path) in [
7723                    (resident.query(*s, *p, *o), "resident"),
7724                    (ranged.query(*s, *p, *o), "ranged"),
7725                    (lazy.query(*s, *p, *o), "lazy"),
7726                ] {
7727                    let mut got = got;
7728                    got.sort();
7729                    assert_eq!(got, want, "{tag}/{path} disagreed on {:?}", (s, p, o));
7730                }
7731                // The named graph's container carries the same mask.
7732                let mut g = resident.query_in_graph(Some("<http://ex/g>"), *s, *p, *o);
7733                g.sort();
7734                assert_eq!(g, want, "{tag}/named disagreed on {:?}", (s, p, o));
7735            }
7736        }
7737
7738        // And the single-pattern ROUTED read (the `query-url` path) locates its
7739        // section by container position, not by the format's fixed six-wide
7740        // slot — the one place the two indexes differ.
7741        let leaked: &'static [u8] = Box::leak(three.clone().into_boxed_slice());
7742        let reader = SliceReader::new(leaked);
7743        let routed = Rete::query_ranged(&reader, None, None, Some("<http://ex/o5>")).unwrap();
7744        let mut want = Rete::open(&six)
7745            .unwrap()
7746            .query(None, None, Some("<http://ex/o5>"));
7747        want.sort();
7748        let mut got = routed;
7749        got.sort();
7750        assert_eq!(got, want, "routed single-pattern read on a lean file");
7751    }
7752
7753    #[test]
7754    fn read_coalesced_merges_within_gap_and_splits_beyond() {
7755        use crate::reader::{CountingReader, SliceReader};
7756        let bytes = vec![0u8; 4096];
7757        // Three 16-byte ranges: A..B gap = 32, B..C gap = 1024.
7758        let ranges = [
7759            ByteRange { offset: 0, len: 16 },
7760            ByteRange {
7761                offset: 48,
7762                len: 16,
7763            },
7764            ByteRange {
7765                offset: 1088,
7766                len: 16,
7767            },
7768        ];
7769        // Tight gap (16): nothing merges → one read per range.
7770        let r = CountingReader::new(SliceReader::new(&bytes));
7771        let out = read_coalesced(&r, &ranges, 16).unwrap();
7772        assert_eq!(out.len(), 3);
7773        assert_eq!(r.requests(), 3);
7774        // Gap 64 merges A+B (gap 32) but not C (gap 1024) → two reads.
7775        let r = CountingReader::new(SliceReader::new(&bytes));
7776        read_coalesced(&r, &ranges, 64).unwrap();
7777        assert_eq!(r.requests(), 2);
7778        // Gap 4096 merges all three into one read, over-fetching the gaps.
7779        let r = CountingReader::new(SliceReader::new(&bytes));
7780        read_coalesced(&r, &ranges, 4096).unwrap();
7781        assert_eq!(r.requests(), 1);
7782    }
7783
7784    fn build_image() -> Vec<u8> {
7785        let triples = [
7786            ("Alice", "knows", "Bob"),
7787            ("Bob", "knows", "Carol"),
7788            ("Alice", "age", "30"),
7789        ];
7790        let mut db = DictionaryBuilder::new();
7791        for (s, p, o) in triples {
7792            db.observe(s, p, o);
7793        }
7794        let dict = db.build();
7795
7796        let mut ib = GraphIndexBuilder::new();
7797        for (s, p, o) in triples {
7798            ib.push(dict.encode(s, p, o).unwrap());
7799        }
7800        let index = ib.build();
7801
7802        let (meta, levels) = build_pyramid_meta(&dict, &triples_ids(&dict), DEFAULT_TILE_BUDGET);
7803        write_file(&dict, &index, false, &meta, levels)
7804    }
7805
7806    fn triples_ids(dict: &Dictionary) -> Vec<(u32, u32, u32)> {
7807        [
7808            ("Alice", "knows", "Bob"),
7809            ("Bob", "knows", "Carol"),
7810            ("Alice", "age", "30"),
7811        ]
7812        .iter()
7813        .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
7814        .collect()
7815    }
7816
7817    #[test]
7818    fn file_round_trips_header_and_counts() {
7819        let bytes = build_image();
7820        let rete = Rete::open(&bytes).unwrap();
7821        assert_eq!(rete.header().quad_count, 3);
7822        assert!(rete.header().term_count >= 5);
7823        let expected_codec = writer_codec();
7824        assert_eq!(rete.header().dict_codec, expected_codec);
7825        assert_eq!(rete.header().block_codec, expected_codec);
7826        assert_eq!(&bytes[bytes.len() - 4..], &MAGIC); // footer marker
7827    }
7828
7829    /// A file whose index was built with a tiny tile budget (forcing many
7830    /// tiles per permutation) must round-trip through write/open and answer
7831    /// every query shape identically — through both the in-memory and the
7832    /// routed ranged read paths.
7833    #[test]
7834    fn multi_tile_file_round_trips_and_routes() {
7835        let triples: Vec<(String, String, String)> = (0..200)
7836            .map(|i| {
7837                (
7838                    format!("<http://ex/s/{i}>"),
7839                    format!("<http://ex/p/{}>", i % 5),
7840                    format!("<http://ex/o/{}>", i % 23),
7841                )
7842            })
7843            .collect();
7844        let mut db = DictionaryBuilder::new();
7845        for (s, p, o) in &triples {
7846            db.observe(s, p, o);
7847        }
7848        let dict = db.build();
7849        let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7850        for (s, p, o) in &triples {
7851            ib.push(dict.encode(s, p, o).unwrap());
7852        }
7853        let index = ib.build();
7854        assert!(
7855            index.tile_sections()[0].len() > 3,
7856            "tiny budget must force many tiles"
7857        );
7858        let bytes = write_file(&dict, &index, false, &[], 0);
7859
7860        let rete = Rete::open(&bytes).unwrap();
7861        assert_eq!(rete.header().version, crate::header::LEGACY_FORMAT_VERSION);
7862        assert_eq!(rete.query(None, None, None).len(), 200);
7863        assert_eq!(rete.query(Some("<http://ex/s/7>"), None, None).len(), 1);
7864        assert_eq!(
7865            rete.query(None, Some("<http://ex/p/3>"), None).len(),
7866            40,
7867            "predicate extent spans tiles"
7868        );
7869        assert_eq!(
7870            rete.query(None, None, Some("<http://ex/o/22>")).len(),
7871            8 // 22, 45, 68, ... < 200
7872        );
7873
7874        // Routed ranged read must agree (and only decompress matching tiles).
7875        use crate::reader::SliceReader;
7876        let reader = SliceReader::new(&bytes);
7877        let routed = Rete::query_ranged(&reader, Some("<http://ex/s/7>"), None, None).unwrap();
7878        assert_eq!(routed.len(), 1);
7879        let routed = Rete::query_ranged(&reader, None, Some("<http://ex/p/3>"), None).unwrap();
7880        assert_eq!(routed.len(), 40);
7881        let routed = Rete::query_ranged(&reader, None, None, Some("<http://ex/o/22>")).unwrap();
7882        assert_eq!(routed.len(), 8);
7883    }
7884
7885    /// The tile-synopsis trailer round-trips through encode/parse, and each parsed
7886    /// synopsis is **exactly** the tile block's own b/c zone — so the directory
7887    /// can never prune a tile the tile itself would have matched.
7888    /// Section-internal byte offsets are u64: a directory whose tiles sit past
7889    /// 4 GiB must parse with exact offsets on EVERY platform. On wasm32 (32-bit
7890    /// usize) the old parse truncated a >4 GiB section length and rejected the
7891    /// tail ("dict chunk overruns section" on the first >4 GiB dictionary —
7892    /// crossref's 5.2 GB g.obj — the playground regression this guards).
7893    #[test]
7894    fn tile_directory_offsets_survive_past_4gib() {
7895        let mut dir = Vec::new();
7896        write_uvarint(&mut dir, 2); // two tiles
7897        write_uvarint(&mut dir, 5); // tile 1: Δmin_a
7898        write_uvarint(&mut dir, 0); //         span
7899        write_uvarint(&mut dir, 3 << 30); //   len = 3 GiB
7900        write_uvarint(&mut dir, 1); // tile 2: Δmin_a
7901        write_uvarint(&mut dir, 0);
7902        write_uvarint(&mut dir, 2 << 30); //   len = 2 GiB
7903        let total = dir.len() as u64 + (3u64 << 30) + (2u64 << 30) + 64;
7904        let entries = parse_tile_directory(&dir, total).unwrap();
7905        assert_eq!(entries.len(), 2);
7906        assert_eq!(entries[1].start, dir.len() as u64 + (3u64 << 30));
7907        assert!(
7908            entries[1].end > u32::MAX as u64,
7909            "tail tile sits past 4 GiB"
7910        );
7911        // a total smaller than the tiles must still reject the directory
7912        assert!(parse_tile_directory(&dir, 1 << 20).is_err());
7913    }
7914
7915    #[test]
7916    fn tile_synopsis_trailer_round_trips() {
7917        let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7918        for i in 0..200u32 {
7919            ib.push((i, i % 7, i % 13));
7920        }
7921        let index = ib.build();
7922        let tile_count = index.tile_sections()[0].len();
7923        assert!(tile_count > 3, "tiny budget forces many tiles");
7924
7925        let payload = encode_tiled_section(&index, 0, CODEC_NONE);
7926        let dir = parse_tile_directory(&payload, payload.len() as u64).unwrap();
7927        assert_eq!(dir.len(), tile_count);
7928        // The trailer sits past the last tile; the old directory parse stops there.
7929        let trailer_start = dir.iter().map(|e| e.end).max().unwrap();
7930        assert!(
7931            trailer_start < payload.len() as u64,
7932            "a trailer follows the tiles"
7933        );
7934        for e in &dir {
7935            assert!(
7936                e.end <= payload.len() as u64,
7937                "tiles still located within the payload"
7938            );
7939        }
7940        let syn = parse_tile_synopsis(&payload, trailer_start as usize, dir.len()).unwrap();
7941        for (e, (min_b, max_b, min_c, max_c)) in dir.iter().zip(syn) {
7942            let block = decompress(CODEC_NONE, &payload[e.start as usize..e.end as usize]).unwrap();
7943            let z = *crate::triples::TripleBlock::parse(&block).unwrap().zone();
7944            assert_eq!(
7945                (min_b, max_b, min_c, max_c),
7946                (z.min_b, z.max_b, z.min_c, z.max_c),
7947                "synopsis equals the tile's own zone"
7948            );
7949        }
7950    }
7951
7952    /// End-to-end safety: a synopsis-carrying file, opened **lazily** (range
7953    /// reads), must return exactly the brute-force answer for every pattern shape
7954    /// — the synopsis prune may never drop a real match.
7955    #[test]
7956    fn tile_synopsis_lazy_matches_reference_every_shape() {
7957        use crate::reader::{CountingReader, SliceReader};
7958        let triples: Vec<(String, String, String)> = (0..200u32)
7959            .map(|i| {
7960                (
7961                    format!("<http://ex/s/{i:04}>"),
7962                    format!("<http://ex/p/{}>", i % 7),
7963                    format!("<http://ex/o/{:04}>", i % 13),
7964                )
7965            })
7966            .collect();
7967        let mut db = DictionaryBuilder::new();
7968        for (s, p, o) in &triples {
7969            db.observe(s, p, o);
7970        }
7971        let dict = db.build();
7972        let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
7973        for (s, p, o) in &triples {
7974            ib.push(dict.encode(s, p, o).unwrap());
7975        }
7976        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
7977
7978        let eager = Rete::open(&bytes).unwrap();
7979        assert!(
7980            eager.header().has_tile_synopsis(),
7981            "new files set the synopsis flag"
7982        );
7983
7984        // `open_ranged_lazy` needs a `'static` reader; leak the image (the test
7985        // process exits straight after).
7986        let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
7987        let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
7988        let lazy = Rete::open_ranged_lazy_with(reader, HEADER_LEN as u64).unwrap();
7989
7990        let brute = |s: Option<&str>, p: Option<&str>, o: Option<&str>| {
7991            let mut v: Vec<(String, String, String)> = triples
7992                .iter()
7993                .filter(|(a, b, c)| {
7994                    s.is_none_or(|x| x == a) && p.is_none_or(|x| x == b) && o.is_none_or(|x| x == c)
7995                })
7996                .cloned()
7997                .collect();
7998            v.sort();
7999            v
8000        };
8001        // Existing + absent terms in every position (and unbound) — 4×4×4 shapes.
8002        let sv = [
8003            None,
8004            Some("<http://ex/s/0007>"),
8005            Some("<http://ex/s/0130>"),
8006            Some("<http://ex/s/9999>"),
8007        ];
8008        let pv = [
8009            None,
8010            Some("<http://ex/p/3>"),
8011            Some("<http://ex/p/6>"),
8012            Some("<http://ex/p/999>"),
8013        ];
8014        let ov = [
8015            None,
8016            Some("<http://ex/o/0000>"),
8017            Some("<http://ex/o/0012>"),
8018            Some("<http://ex/o/9999>"),
8019        ];
8020        for &s in &sv {
8021            for &p in &pv {
8022                for &o in &ov {
8023                    let mut e = eager.query(s, p, o);
8024                    e.sort();
8025                    let mut l = lazy.query(s, p, o);
8026                    l.sort();
8027                    let r = brute(s, p, o);
8028                    assert_eq!(e, r, "eager {s:?} {p:?} {o:?}");
8029                    assert_eq!(l, r, "lazy {s:?} {p:?} {o:?} — synopsis over-pruned");
8030                }
8031            }
8032        }
8033        assert!(!lazy.index_incomplete(), "no lazy fetch failed");
8034    }
8035
8036    /// Byte-0 / polyglot experiment: a real `.rete` embedded behind a large HTML
8037    /// shell (so byte 0 is `<`, not `RETE`) still opens and queries LAZILY through
8038    /// an `OffsetReader`, touching only the graph's bytes and never the shell.
8039    #[test]
8040    fn polyglot_offset_reads_lazily() {
8041        use crate::reader::{
8042            detect_polyglot_base, CountingReader, OffsetReader, SliceReader, POLYGLOT_DIGITS,
8043            POLYGLOT_MARKER,
8044        };
8045
8046        // A real .rete image, hidden behind a 50 KB HTML shell that carries the
8047        // polyglot base-offset marker in its first bytes (a browser-ignored
8048        // comment). The .rete begins right after the shell.
8049        let image = build_image();
8050        let mut shell = Vec::new();
8051        shell.extend_from_slice(b"<!DOCTYPE html><html><head><!--");
8052        shell.extend_from_slice(POLYGLOT_MARKER);
8053        let digits_at = shell.len();
8054        shell.extend_from_slice(&[b'0'; POLYGLOT_DIGITS]); // patched below
8055        shell.extend_from_slice(b"--></head><body>a web page</body></html>\n");
8056        shell.resize(50_000, b' '); // >> the .rete, to prove the shell is untouched
8057        let base = shell.len() as u64;
8058        let digits = format!("{base:0width$}", width = POLYGLOT_DIGITS);
8059        shell[digits_at..digits_at + POLYGLOT_DIGITS].copy_from_slice(digits.as_bytes());
8060
8061        let mut poly = shell;
8062        poly.extend_from_slice(&image);
8063
8064        // byte 0 is a web page, not a .rete...
8065        assert_ne!(
8066            &poly[0..4],
8067            b"RETE",
8068            "polyglot must not start with the magic"
8069        );
8070        // ...but the marker in the first header window points at the embedded .rete.
8071        let detected = detect_polyglot_base(&poly[..crate::header::HEADER_LEN]).unwrap();
8072        assert_eq!(detected, base);
8073
8074        // Open it LAZILY through the offset shim and query it.
8075        let leaked: &'static [u8] = Box::leak(poly.into_boxed_slice());
8076        let counting = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8077        let lazy = Rete::open_ranged_lazy_with(
8078            OffsetReader::new(counting.clone(), detected),
8079            HEADER_LEN as u64,
8080        )
8081        .unwrap();
8082
8083        // The polyglot must answer exactly like the plain .rete it embeds.
8084        let plain = Rete::open(&image).unwrap();
8085        let mut want = plain.query(None, None, None);
8086        want.sort();
8087        let mut got = lazy.query(None, None, None);
8088        got.sort();
8089        assert_eq!(
8090            got, want,
8091            "polyglot lazy query differs from the plain .rete"
8092        );
8093        assert!(!got.is_empty(), "expected some triples");
8094        assert!(!lazy.index_incomplete(), "a lazy fetch failed");
8095
8096        // It was LAZY: it read fewer bytes than the HTML shell alone, i.e. it
8097        // never touched the prefix — only the embedded .rete's own bytes.
8098        let read = counting.bytes_read();
8099        eprintln!(
8100            "polyglot lazy read: HTML shell {base} B + .rete image {} B; \
8101             the query touched only {read} B (never the shell).",
8102            image.len()
8103        );
8104        assert!(
8105            read < base,
8106            "read {read} bytes; the HTML shell alone is {base} — the reader \
8107             touched the prefix instead of only the embedded .rete"
8108        );
8109    }
8110
8111    /// Build a small file whose objects are string literals, **with** a text
8112    /// index, and return `(image, triples)`. Shared by the text-index tests.
8113    #[cfg(test)]
8114    fn build_text_indexed(triples: &[(String, String, String)]) -> Vec<u8> {
8115        let mut db = DictionaryBuilder::new();
8116        for (s, p, o) in triples {
8117            db.observe(s, p, o);
8118        }
8119        let dict = db.build();
8120        let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
8121        let mut id_triples: Vec<(u32, u32, u32)> = Vec::with_capacity(triples.len());
8122        for (s, p, o) in triples {
8123            let t = dict.encode(s, p, o).unwrap();
8124            ib.push(t);
8125            id_triples.push(t);
8126        }
8127        let index = ib.build();
8128        let text_index = compute_text_index(&dict, &id_triples);
8129        assert!(
8130            !text_index.is_empty(),
8131            "literals should produce a text index"
8132        );
8133        write_dataset_with_metadata(&dict, &index, &[], false, &[], 0, &[], &text_index)
8134    }
8135
8136    /// A `--text-index` build round-trips: `text_search` returns exactly the
8137    /// subjects whose literals contain the queried word(s), with AND across words
8138    /// and token-prefix — matching a brute-force scan of the literals.
8139    #[test]
8140    fn text_index_eager_matches_brute_force() {
8141        let triples: Vec<(String, String, String)> = vec![
8142            (
8143                "<http://ex/s0>",
8144                "<http://ex/label>",
8145                "\"alpha glucose phosphate\"",
8146            ),
8147            ("<http://ex/s1>", "<http://ex/label>", "\"beta Glucose\""),
8148            ("<http://ex/s2>", "<http://ex/label>", "\"gamma fructose\""),
8149            (
8150                "<http://ex/s3>",
8151                "<http://ex/note>",
8152                "\"einstein relativity\"",
8153            ),
8154            (
8155                "<http://ex/s4>",
8156                "<http://ex/ref>",
8157                "<http://ex/not-a-literal>",
8158            ),
8159        ]
8160        .into_iter()
8161        .map(|(s, p, o)| (s.to_string(), p.to_string(), o.to_string()))
8162        .collect();
8163        let bytes = build_text_indexed(&triples);
8164        let rete = Rete::open(&bytes).unwrap();
8165        assert!(rete.has_text_index());
8166
8167        // Brute-force reference: subjects whose literal objects contain all words.
8168        let brute = |words: &[&str]| -> Vec<String> {
8169            let mut v: Vec<String> = triples
8170                .iter()
8171                .filter(|(_, _, o)| {
8172                    crate::terms::is_literal(o)
8173                        && words.iter().all(|w| {
8174                            let wl = w.to_lowercase();
8175                            crate::terms::literal_lexical(o)
8176                                .unwrap()
8177                                .split(|c: char| !c.is_alphanumeric())
8178                                .any(|t| t.to_lowercase() == wl)
8179                        })
8180                })
8181                .map(|(s, _, _)| s.clone())
8182                .collect();
8183            v.sort();
8184            v.dedup();
8185            v
8186        };
8187
8188        let mut got = rete.text_search(&["glucose"], None, 0);
8189        got.sort();
8190        assert_eq!(got, brute(&["glucose"]), "case-insensitive single word");
8191
8192        // AND across two words: only s0 has both.
8193        let mut got = rete.text_search(&["glucose", "phosphate"], None, 0);
8194        got.sort();
8195        assert_eq!(got, brute(&["glucose", "phosphate"]));
8196
8197        // A word nobody has → empty.
8198        assert!(rete.text_search(&["zzznope"], None, 0).is_empty());
8199
8200        // Token-prefix: "ein…" matches "einstein".
8201        let got = rete.text_search(&[], Some("ein"), 0);
8202        assert_eq!(got, vec!["<http://ex/s3>".to_string()]);
8203
8204        // No text index → empty, has_text_index() false.
8205        let mut db = DictionaryBuilder::new();
8206        for (s, p, o) in &triples {
8207            db.observe(s, p, o);
8208        }
8209        let dict = db.build();
8210        let mut ib = GraphIndexBuilder::new();
8211        for (s, p, o) in &triples {
8212            ib.push(dict.encode(s, p, o).unwrap());
8213        }
8214        let plain = write_dataset(&dict, &ib.build(), &[], false, &[], 0);
8215        let plain_rete = Rete::open(&plain).unwrap();
8216        assert!(!plain_rete.has_text_index());
8217        assert!(plain_rete.text_search(&["glucose"], None, 0).is_empty());
8218    }
8219
8220    /// The lazy/remote path returns the same subjects as the eager path **and**
8221    /// faults only the token table + the queried posting list — never the whole
8222    /// postings blob. A `CountingReader` proves the byte budget stays small.
8223    #[test]
8224    fn text_index_lazy_faults_only_queried_postings() {
8225        use crate::reader::{CountingReader, SliceReader};
8226        // Many subjects so the postings blob is large relative to one posting:
8227        // every subject carries "common", but only a few carry "rare".
8228        let mut triples: Vec<(String, String, String)> = (0..300u32)
8229            .map(|i| {
8230                (
8231                    format!("<http://ex/s/{i:04}>"),
8232                    "<http://ex/label>".to_string(),
8233                    format!("\"common word number {i}\""),
8234                )
8235            })
8236            .collect();
8237        for i in [3u32, 77, 250] {
8238            triples.push((
8239                format!("<http://ex/s/{i:04}>"),
8240                "<http://ex/tag>".to_string(),
8241                "\"raretoken\"".to_string(),
8242            ));
8243        }
8244        let bytes = build_text_indexed(&triples);
8245        let eager = Rete::open(&bytes).unwrap();
8246        let mut want = eager.text_search(&["raretoken"], None, 0);
8247        want.sort();
8248        assert_eq!(want.len(), 3);
8249
8250        let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8251        let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8252        let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
8253        // Bytes pulled by the open itself (dict dirs, index dirs, named graphs) —
8254        // the text index is deferred and not touched yet.
8255        let before = reader.bytes_read();
8256        let mut got = lazy.text_search(&["raretoken"], None, 0);
8257        got.sort();
8258        assert_eq!(got, want, "lazy search matches eager");
8259        let pulled = reader.bytes_read() - before;
8260        // The search faulted the token table + the one "raretoken" posting; it must
8261        // be far less than the whole text-index section (300 "common" postings).
8262        let ti_len = eager.header().text_index_len;
8263        assert!(
8264            pulled < ti_len,
8265            "search pulled {pulled} B but the section is {ti_len} B — faulted too much"
8266        );
8267        assert!(!lazy.index_incomplete());
8268    }
8269
8270    /// `SearchView` answers exactly what a full open answers, having read
8271    /// strictly less: it skips the permutation tile directories and — the part
8272    /// that actually costs on a literal-heavy graph — the object-only
8273    /// dictionary's chunk directory.
8274    #[test]
8275    fn search_view_matches_full_open_for_far_fewer_bytes() {
8276        use crate::reader::{CountingReader, SliceReader};
8277        // Long object literals so the object-only dictionary (and its chunk
8278        // directory) dominates the file, as on a graph that stores abstracts.
8279        let filler = "lorem ipsum dolor sit amet consectetur adipiscing elit ".repeat(20);
8280        let mut triples: Vec<(String, String, String)> = (0..300u32)
8281            .map(|i| {
8282                (
8283                    format!("<http://ex/s/{i:04}>"),
8284                    "<http://ex/abstract>".to_string(),
8285                    format!("\"{filler} number {i}\""),
8286                )
8287            })
8288            .collect();
8289        for i in [3u32, 77, 250] {
8290            triples.push((
8291                format!("<http://ex/s/{i:04}>"),
8292                "<http://ex/tag>".to_string(),
8293                "\"raretoken\"".to_string(),
8294            ));
8295        }
8296        let bytes = build_text_indexed(&triples);
8297        let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
8298
8299        let full_reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8300        let full = Rete::open_ranged_lazy_with(full_reader.clone(), HEADER_LEN as u64).unwrap();
8301        let mut want = full.text_search(&["raretoken"], None, 0);
8302        want.sort();
8303        assert_eq!(want.len(), 3);
8304
8305        let search_reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8306        let view = SearchView::open_ranged_with(search_reader.clone(), HEADER_LEN as u64).unwrap();
8307        assert!(view.has_text_index());
8308        let mut got = view.text_search(&["raretoken"], None, 0);
8309        got.sort();
8310        assert_eq!(got, want, "search view matches the full open");
8311        assert!(
8312            search_reader.bytes_read() < full_reader.bytes_read(),
8313            "search read {} B, full open read {} B — the narrow open bought nothing",
8314            search_reader.bytes_read(),
8315            full_reader.bytes_read()
8316        );
8317
8318        // A file without a text index says so from the header alone, and the
8319        // label-prefix mode still works through the same view.
8320        let mut prefix_hits = view.prefix_search("", 5);
8321        prefix_hits.sort();
8322        assert!(!prefix_hits.is_empty() || !view.has_pyramid());
8323    }
8324
8325    /// Nothing records a dictionary chunk directory's byte length, so a ranged
8326    /// open probes for it. That probe must cost about ONE directory, not several
8327    /// — it used to re-read the whole prefix on every round, so the sum was
8328    /// ~2x the last (already over-sized) read: 537 MB of range reads to fetch
8329    /// epfl-infoscience's 234 MB object-only directory. Here the same probe runs
8330    /// against a file whose object literals are long enough that its directory
8331    /// is well past the 8 KiB header prefix, and must (a) stay inside the
8332    /// append-only bound of 2x the directory and (b) beat what the old
8333    /// re-reading loop would have spent.
8334    #[test]
8335    fn dict_chunk_directory_probe_costs_about_one_directory() {
8336        use crate::reader::{CountingReader, SliceReader};
8337        // Long literals that share a 3 KiB prefix with their neighbour, each
8338        // ending in a 1 KiB tail of its own: the WKT-polygon / JSON-blob shape
8339        // (#198's weak case). The directory stores the SHORTEST SEPARATOR per
8340        // chunk, and a separator has to reproduce the whole shared prefix — so
8341        // these keys are still ~3 KiB and the directory still runs well past
8342        // the 8 KiB header prefix, which is what gives the probe something to
8343        // measure.
8344        //
8345        // This fixture used to put the varying bytes FIRST (`"{i:04} xxxx…"`).
8346        // That shape is the one separators annihilate: its keys collapse to ~4
8347        // bytes, the whole directory to 192 B, it fits the header prefix, and
8348        // the probe never runs — the assertion below then fails naming itself.
8349        // Keep the shared prefix.
8350        let head = "x".repeat(3000);
8351        let triples: Vec<(String, String, String)> = (0..2000u32)
8352            .map(|i| {
8353                let tail = char::from(b'a' + (i % 26) as u8).to_string().repeat(1000);
8354                (
8355                    format!("<http://ex/s/{i:04}>"),
8356                    "<http://ex/abstract>".to_string(),
8357                    format!("\"{head}{i:04}{tail}\""),
8358                )
8359            })
8360            .collect();
8361        let mut db = DictionaryBuilder::new();
8362        for (s, p, o) in &triples {
8363            db.observe(s, p, o);
8364        }
8365        let dict = db.build();
8366        let mut ib = GraphIndexBuilder::new();
8367        for (s, p, o) in &triples {
8368            ib.push(dict.encode(s, p, o).unwrap());
8369        }
8370        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8371
8372        let reader = CountingReader::new(SliceReader::new(&bytes));
8373        let header = Header::from_bytes(&reader.read_at(0, HEADER_LEN as u64).unwrap()).unwrap();
8374        // Section 2 is object-only: the one carrying the long literals.
8375        let section = locate_container_section_ranged(
8376            &reader,
8377            header.dictionary_offset,
8378            header.dictionary_len,
8379            2,
8380            4,
8381        )
8382        .unwrap();
8383
8384        let before = reader.bytes_read();
8385        let (_meta, entries) = read_dict_dir_ranged(&reader, section).unwrap();
8386        let spent = reader.bytes_read() - before;
8387
8388        // The first chunk body starts where the directory ends.
8389        let dir_end = entries[0].start;
8390        let head = reader.read_at(section.offset, 64).unwrap();
8391        let (header_len, n0) = read_uvarint(&head).unwrap();
8392        let dir_start = n0 as u64 + header_len;
8393        let dir_len = dir_end - dir_start;
8394        let dir_total = section.len - dir_start;
8395        assert!(
8396            dir_start + dir_len > 8192,
8397            "directory fits the header prefix ({dir_len} B) — the probe never runs"
8398        );
8399
8400        // What the previous loop spent: every round re-read from the start.
8401        let mut old = 0u64;
8402        let mut p = 4096u64.min(dir_total).max(1);
8403        loop {
8404            let got = p.min(dir_total);
8405            old += got;
8406            if got >= dir_len || p >= dir_total {
8407                break;
8408            }
8409            p = p.saturating_mul(2).min(dir_total);
8410        }
8411        assert!(
8412            spent <= dir_len * 2,
8413            "probe read {spent} B for a {dir_len} B directory — past the append-only bound"
8414        );
8415        assert!(
8416            spent < old,
8417            "probe read {spent} B; the re-reading loop it replaced spent {old} B"
8418        );
8419    }
8420
8421    // ---------------------------------------------------------------------
8422    // Separator routing keys (#198 option (d))
8423    // ---------------------------------------------------------------------
8424
8425    /// A verbatim-keyed twin of a chunked dictionary section: same header, same
8426    /// chunking, byte-identical compressed bodies — only every directory key
8427    /// replaced by the chunk's actual first term. That is exactly what
8428    /// `encode_chunked_dict_section` wrote before separator keys, so it is the
8429    /// control for every comparison below (and for the proptest).
8430    fn verbatim_keyed_twin(payload: &[u8], codec: u8) -> Vec<u8> {
8431        let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
8432        let (header_len, n0) = read_uvarint(payload).unwrap();
8433        let mut out = Vec::new();
8434        write_uvarint(&mut out, header_len);
8435        out.extend_from_slice(&payload[n0..n0 + header_len as usize]);
8436        write_uvarint(&mut out, entries.len() as u64);
8437        let mut prev_run = 0usize;
8438        for e in &entries {
8439            let body = decompress(codec, &payload[e.start as usize..e.end as usize]).unwrap();
8440            let first = crate::dict::run_first_term(&body, 0).unwrap();
8441            write_uvarint(&mut out, (e.first_run - prev_run) as u64);
8442            write_uvarint(&mut out, first.len() as u64);
8443            out.extend_from_slice(&first);
8444            write_uvarint(&mut out, e.end - e.start);
8445            prev_run = e.first_run;
8446        }
8447        for e in &entries {
8448            out.extend_from_slice(&payload[e.start as usize..e.end as usize]);
8449        }
8450        out
8451    }
8452
8453    /// Each chunk's (first term, last term), decoded from the bodies.
8454    fn chunk_bounds_of(payload: &[u8], codec: u8) -> Vec<(Vec<u8>, Vec<u8>)> {
8455        let (_meta, entries) = parse_chunked_dict_dir(payload, payload.len() as u64).unwrap();
8456        entries
8457            .iter()
8458            .map(|e| {
8459                let body = decompress(codec, &payload[e.start as usize..e.end as usize]).unwrap();
8460                (
8461                    crate::dict::run_first_term(&body, 0).unwrap(),
8462                    crate::dict::run_last_term(&body, 0, body.len()).unwrap(),
8463                )
8464            })
8465            .collect()
8466    }
8467
8468    /// A term set built to hit every shape the corpus has: plain IRIs, literals
8469    /// sharing a long prefix with their neighbour (separators are LONG there —
8470    /// the WKT/`ohm-full` weak case), literals that diverge in their first bytes
8471    /// (separators are tiny — the `epfl-infoscience` case), literals larger than
8472    /// `DICT_CHUNK_BUDGET` on their own, and multibyte UTF-8.
8473    fn adversarial_terms() -> Vec<String> {
8474        let mut terms: Vec<String> = Vec::new();
8475        for i in 0..6_000 {
8476            terms.push(format!("<http://example.org/work/{i:06}>"));
8477        }
8478        let shared = "X".repeat(900);
8479        for i in 0..4_000 {
8480            terms.push(format!("\"{shared}-{i:05}-tail\""));
8481        }
8482        let deep = "Y".repeat(3_000); // shared prefix far past any short key
8483        for i in 0..120 {
8484            terms.push(format!("\"{deep}{i:03}\""));
8485        }
8486        for i in 0..6 {
8487            // one literal larger than the 64 KiB budget: a chunk of one run
8488            terms.push(format!("\"BIG-{i:02}-{}\"", "Z".repeat(90_000)));
8489        }
8490        for i in 0..800 {
8491            terms.push(format!(
8492                "\"h\u{e9}llo-{}-\u{1F600}{i:04}\"",
8493                "\u{e9}".repeat(40)
8494            ));
8495        }
8496        for i in 0..2_000 {
8497            terms.push(format!("\"{i:05}-{}\"", "T".repeat(1_800)));
8498        }
8499        terms.sort();
8500        terms.dedup();
8501        terms
8502    }
8503
8504    fn chunked_section_of(terms: &[String]) -> Vec<u8> {
8505        let mut b = crate::dict::DictSectionBuilder::new().with_restart_interval(16);
8506        for t in terms {
8507            b.push(t.clone());
8508        }
8509        b.build()
8510    }
8511
8512    /// The invariant, asserted against the real chunk contents: the stored key
8513    /// of chunk `i` is strictly above chunk `i-1`'s LAST term and at most chunk
8514    /// `i`'s own first term, chunk 0's key is empty, and the keys ascend — the
8515    /// three facts `ChunkedSection::id`'s `partition_point` relies on.
8516    ///
8517    /// Plus the anti-regression the rename exists for: on a section like this
8518    /// the key must NOT be the first term. A writer that quietly went back to
8519    /// storing terms would pass every other test in this file — dumps and
8520    /// `term(id)` route by `first_run` and never read the key.
8521    #[test]
8522    fn chunk_directory_keys_are_separators_not_terms() {
8523        let terms = adversarial_terms();
8524        let raw = chunked_section_of(&terms);
8525        let payload = encode_chunked_dict_section(&raw, CODEC_NONE);
8526        let (_meta, entries) = parse_chunked_dict_dir(&payload, payload.len() as u64).unwrap();
8527        let bounds = chunk_bounds_of(&payload, CODEC_NONE);
8528        assert!(
8529            entries.len() > 20,
8530            "need a many-chunk section, got {}",
8531            entries.len()
8532        );
8533
8534        assert!(entries[0].key.is_empty(), "chunk 0 needs no separator");
8535        let mut key_bytes = 0usize;
8536        let mut term_bytes = 0usize;
8537        let mut differ = 0usize;
8538        for (i, e) in entries.iter().enumerate() {
8539            key_bytes += e.key.len();
8540            term_bytes += bounds[i].0.len();
8541            if e.key != bounds[i].0 {
8542                differ += 1;
8543            }
8544            assert!(
8545                e.key <= bounds[i].0,
8546                "chunk {i}: key must be <= its own first term"
8547            );
8548            if i > 0 {
8549                assert!(
8550                    e.key.as_slice() > bounds[i - 1].1.as_slice(),
8551                    "chunk {i}: key must be > the previous chunk's last term"
8552                );
8553                assert!(
8554                    e.key > entries[i - 1].key,
8555                    "chunk {i}: keys must ascend for partition_point"
8556                );
8557                // shortest: one byte past the divergence from the predecessor
8558                assert_eq!(
8559                    e.key,
8560                    crate::dict::shortest_separator(&bounds[i - 1].1, &bounds[i].0),
8561                    "chunk {i}: key is not the SHORTEST separator"
8562                );
8563            }
8564        }
8565        assert!(
8566            differ > 0,
8567            "every key equalled its first term — the writer stores terms again"
8568        );
8569        assert!(
8570            key_bytes * 4 < term_bytes,
8571            "separators bought almost nothing: {key_bytes} B vs {term_bytes} B of first terms"
8572        );
8573        eprintln!(
8574            "chunks={} first-term keys={term_bytes} B  separator keys={key_bytes} B ({:.1}x)",
8575            entries.len(),
8576            term_bytes as f64 / key_bytes.max(1) as f64
8577        );
8578    }
8579
8580    /// A separator-keyed section answers **identically** to the verbatim-keyed
8581    /// one built over the same bodies, under the same unmodified reader: every
8582    /// id, every term, and every boundary probe — the first and last term of
8583    /// each chunk, one byte either side of each, strings that fall strictly
8584    /// *between* two chunks, truncations, the empty string, and probes below
8585    /// and above the whole section.
8586    ///
8587    /// Probing `id(term)` is the entire point. A broken key (a truncation, a
8588    /// hash) leaves `term(id)`, `dump` and `export` byte-perfect — they route by
8589    /// `first_run` — and only lookups lie.
8590    #[test]
8591    fn separator_keys_route_identically_to_verbatim_first_terms() {
8592        let terms = adversarial_terms();
8593        let raw = chunked_section_of(&terms);
8594        let sep_payload = encode_chunked_dict_section(&raw, CODEC_NONE);
8595        let ver_payload = verbatim_keyed_twin(&sep_payload, CODEC_NONE);
8596
8597        // Same chunking, byte-identical bodies — only the keys differ.
8598        let (_ms, es) = parse_chunked_dict_dir(&sep_payload, sep_payload.len() as u64).unwrap();
8599        let (_mv, ev) = parse_chunked_dict_dir(&ver_payload, ver_payload.len() as u64).unwrap();
8600        assert_eq!(es.len(), ev.len());
8601        for (a, b) in es.iter().zip(ev.iter()) {
8602            assert_eq!(a.first_run, b.first_run);
8603            assert_eq!(
8604                &sep_payload[a.start as usize..a.end as usize],
8605                &ver_payload[b.start as usize..b.end as usize],
8606                "chunk bodies must be identical"
8607            );
8608        }
8609        assert!(
8610            sep_payload.len() < ver_payload.len(),
8611            "separator payload is not smaller"
8612        );
8613
8614        let sec_s = decode_chunked_dict_section(
8615            &sep_payload,
8616            CODEC_NONE,
8617            crate::chunk_cache::ChunkCache::unlimited_arc(),
8618            0,
8619        )
8620        .unwrap();
8621        let sec_v = decode_chunked_dict_section(
8622            &ver_payload,
8623            CODEC_NONE,
8624            crate::chunk_cache::ChunkCache::unlimited_arc(),
8625            0,
8626        )
8627        .unwrap();
8628
8629        // 1. every present term → the same, correct id
8630        for (i, t) in terms.iter().enumerate() {
8631            let want = Some(i as u32 + 1);
8632            assert_eq!(sec_v.id(t), want, "verbatim id() lost a term");
8633            assert_eq!(sec_s.id(t), want, "separator id() lost a term");
8634        }
8635        // 2. every id → the same term
8636        for i in 0..terms.len() as u32 {
8637            assert_eq!(sec_s.term(i + 1), sec_v.term(i + 1));
8638            assert_eq!(
8639                sec_s.term(i + 1).as_deref(),
8640                Some(terms[i as usize].as_str())
8641            );
8642        }
8643        // 3. boundary probes, from the real chunk bounds
8644        let bounds = chunk_bounds_of(&sep_payload, CODEC_NONE);
8645        let mut probes: Vec<String> = Vec::new();
8646        let push = |bytes: &[u8], probes: &mut Vec<String>| {
8647            if let Ok(s) = std::str::from_utf8(bytes) {
8648                probes.push(s.to_string());
8649            }
8650        };
8651        for (first, last) in &bounds {
8652            for base in [first, last] {
8653                push(base, &mut probes); // present
8654                let mut after = base.clone();
8655                after.push(0x01); // immediately after — between chunks when base = last
8656                push(&after, &mut probes);
8657                let mut far = base.clone();
8658                far.push(b'~');
8659                push(&far, &mut probes);
8660                if !base.is_empty() {
8661                    push(&base[..base.len() - 1], &mut probes); // truncated
8662                    let mut down = base.clone();
8663                    *down.last_mut().unwrap() = down.last().unwrap().wrapping_sub(1);
8664                    push(&down, &mut probes);
8665                    let mut up = base.clone();
8666                    *up.last_mut().unwrap() = up.last().unwrap().wrapping_add(1);
8667                    push(&up, &mut probes);
8668                }
8669            }
8670        }
8671        // 4. the stored separators themselves, and the ends of the world
8672        for e in &es {
8673            push(&e.key, &mut probes);
8674            let mut plus = e.key.clone();
8675            plus.push(0);
8676            push(&plus, &mut probes);
8677        }
8678        probes.push(String::new()); // the empty key, i.e. chunk 0's
8679        probes.push("\u{1}".to_string()); // below every term
8680        probes.push("\u{10FFFF}\u{10FFFF}".to_string()); // above every term
8681        probes.push(terms.first().unwrap().clone());
8682        probes.push(terms.last().unwrap().clone());
8683
8684        let mut hits = 0usize;
8685        for p in &probes {
8686            let (a, b) = (sec_v.id(p), sec_s.id(p));
8687            assert_eq!(
8688                a,
8689                b,
8690                "probe diverged: {:?}",
8691                p.get(..p.len().min(48)).unwrap_or(p.as_str())
8692            );
8693            hits += usize::from(a.is_some());
8694        }
8695        assert!(
8696            probes.len() > 500 && hits > 50,
8697            "probe set is too thin: {} probes, {hits} of them present",
8698            probes.len()
8699        );
8700        eprintln!(
8701            "{} boundary probes, {hits} present, all identical",
8702            probes.len()
8703        );
8704    }
8705
8706    /// The same claim as a **property**: for arbitrary term sets and arbitrary
8707    /// probes, a separator-keyed section and the verbatim-keyed twin built from
8708    /// the same input are indistinguishable through the reader. The fixed test
8709    /// above picks the shapes I thought of; this one shrinks a counterexample
8710    /// out of the ones I did not.
8711    mod separator_props {
8712        use super::*;
8713        use proptest::prelude::*;
8714
8715        /// Sorted, deduplicated term sets mixing the three shapes that decide a
8716        /// separator's length: short IRIs, literals whose payload comes BEFORE
8717        /// the discriminating bytes (long shared prefixes ⇒ long separators),
8718        /// and literals that diverge immediately (⇒ one-byte separators). Sizes
8719        /// straddle `DICT_CHUNK_BUDGET`, so single- and many-chunk sections both
8720        /// occur.
8721        fn term_set() -> impl Strategy<Value = Vec<String>> {
8722            prop::collection::vec(("[a-c]{1,6}", 0u8..3, 300usize..1200), 60..250).prop_map(
8723                |specs| {
8724                    let mut terms: Vec<String> = specs
8725                        .into_iter()
8726                        .enumerate()
8727                        .map(|(i, (core, kind, pad))| {
8728                            let block = "z".repeat(pad);
8729                            match kind {
8730                                0 => format!("<http://ex/{core}/{i:05}>"),
8731                                1 => format!("\"{block}{core}{i:05}\""),
8732                                _ => format!("\"{core}{i:05}{block}\""),
8733                            }
8734                        })
8735                        .collect();
8736                    terms.sort();
8737                    terms.dedup();
8738                    terms
8739                },
8740            )
8741        }
8742
8743        proptest! {
8744            #[test]
8745            fn prop_separator_keyed_section_answers_like_verbatim(
8746                terms in term_set(),
8747                extra in prop::collection::vec(any::<String>(), 0..8),
8748            ) {
8749                let raw = chunked_section_of(&terms);
8750                let sep = encode_chunked_dict_section(&raw, CODEC_NONE);
8751                let ver = verbatim_keyed_twin(&sep, CODEC_NONE);
8752                prop_assert!(sep.len() <= ver.len());
8753
8754                // The separator invariant, against the real chunk contents.
8755                let (_m, es) = parse_chunked_dict_dir(&sep, sep.len() as u64).unwrap();
8756                let bounds = chunk_bounds_of(&sep, CODEC_NONE);
8757                prop_assert!(es[0].key.is_empty());
8758                for i in 1..es.len() {
8759                    prop_assert!(es[i].key.as_slice() > bounds[i - 1].1.as_slice());
8760                    prop_assert!(es[i].key <= bounds[i].0);
8761                    prop_assert!(es[i].key > es[i - 1].key);
8762                }
8763
8764                let sec_s = decode_chunked_dict_section(
8765                    &sep,
8766                    CODEC_NONE,
8767                    crate::chunk_cache::ChunkCache::unlimited_arc(),
8768                    0,
8769                )
8770                .unwrap();
8771                let sec_v = decode_chunked_dict_section(
8772                    &ver,
8773                    CODEC_NONE,
8774                    crate::chunk_cache::ChunkCache::unlimited_arc(),
8775                    0,
8776                )
8777                .unwrap();
8778
8779                for (i, t) in terms.iter().enumerate() {
8780                    prop_assert_eq!(sec_s.id(t), Some(i as u32 + 1));
8781                    prop_assert_eq!(sec_v.id(t), Some(i as u32 + 1));
8782                    prop_assert_eq!(sec_s.term(i as u32 + 1), sec_v.term(i as u32 + 1));
8783                }
8784
8785                // Probes: arbitrary strings, plus mutations of every boundary.
8786                let mut probes: Vec<String> = extra;
8787                probes.push(String::new());
8788                for (first, last) in &bounds {
8789                    for base in [first, last] {
8790                        if let Ok(s) = std::str::from_utf8(base) {
8791                            probes.push(s.to_string());
8792                            probes.push(format!("{s}\u{1}"));
8793                            probes.push(s[..s.len() - s.chars().next_back()
8794                                .map(char::len_utf8).unwrap_or(0)].to_string());
8795                        }
8796                    }
8797                }
8798                for e in &es {
8799                    if let Ok(s) = std::str::from_utf8(&e.key) {
8800                        probes.push(s.to_string());
8801                    }
8802                }
8803                for p in &probes {
8804                    prop_assert_eq!(sec_s.id(p), sec_v.id(p), "probe diverged: {:?}", p);
8805                }
8806            }
8807        }
8808    }
8809
8810    /// The same property for a SPARQL-ordered section (format `0x08`): terms of
8811    /// every group (IRIs that are prefixes of each other, blank nodes, text
8812    /// literals with spaces, tags and escapes, typed literals), big enough to
8813    /// make several chunks. Every separator lies between its neighbours **in
8814    /// that order**, every term is found at its rank, and arbitrary and
8815    /// mutated probes are found exactly when a linear search finds them.
8816    mod sparql_order_props {
8817        use super::*;
8818        use crate::term_order::TermOrder;
8819        use proptest::prelude::*;
8820
8821        fn term_set() -> impl Strategy<Value = Vec<String>> {
8822            prop::collection::vec(("[a-c]{1,5}", 0u8..7, 200usize..900), 60..220).prop_map(
8823                |specs| {
8824                    let mut terms: Vec<String> = specs
8825                        .into_iter()
8826                        .enumerate()
8827                        .map(|(i, (core, kind, pad))| {
8828                            let block = "z".repeat(pad);
8829                            match kind {
8830                                // an IRI and its extension: Ng8 / Ng8130_provenance
8831                                0 => format!("<http://ex/{core}/{}>", i % 40),
8832                                1 => format!("<http://ex/{core}/{}{block}_provenance>", i % 40),
8833                                2 => format!("_:{core}{i}"),
8834                                3 => format!("\"{block} {core}{i:04}\""),
8835                                4 => format!("\"{core}{i:04}\"@{}", ["en", "EN", "fr"][i % 3]),
8836                                5 => format!("\"{core}\\\"{i:04}{block}\""),
8837                                _ => format!("\"{core}{i}\"^^<http://ex/dt>"),
8838                            }
8839                        })
8840                        .collect();
8841                    terms.sort_by(|a, b| TermOrder::Sparql.cmp_str(a, b));
8842                    terms.dedup();
8843                    terms
8844                },
8845            )
8846        }
8847
8848        fn section_of(terms: &[String]) -> Vec<u8> {
8849            let mut b = crate::dict::DictSectionBuilder::new()
8850                .with_restart_interval(16)
8851                .with_order(TermOrder::Sparql);
8852            for t in terms {
8853                b.push(t.clone());
8854            }
8855            b.build()
8856        }
8857
8858        proptest! {
8859            #[test]
8860            fn prop_a_sparql_ordered_section_routes_and_finds(
8861                terms in term_set(),
8862                extra in prop::collection::vec(any::<String>(), 0..8),
8863            ) {
8864                let raw = section_of(&terms);
8865                let payload = encode_chunked_dict_section_ordered(&raw, CODEC_NONE, TermOrder::Sparql);
8866                let (_m, es) = parse_chunked_dict_dir(&payload, payload.len() as u64).unwrap();
8867                let bounds = chunk_bounds_of(&payload, CODEC_NONE);
8868                let o = TermOrder::Sparql;
8869                prop_assert!(es[0].key.is_empty());
8870                for i in 1..es.len() {
8871                    prop_assert!(o.cmp(&bounds[i - 1].1, &es[i].key) == std::cmp::Ordering::Less);
8872                    prop_assert!(o.cmp(&es[i].key, &bounds[i].0) != std::cmp::Ordering::Greater);
8873                    prop_assert!(o.cmp(&es[i - 1].key, &es[i].key) == std::cmp::Ordering::Less);
8874                }
8875                let sec = decode_chunked_dict_section(
8876                    &payload,
8877                    CODEC_NONE,
8878                    crate::chunk_cache::ChunkCache::unlimited_arc(),
8879                    0,
8880                )
8881                .unwrap()
8882                .with_order(o);
8883                for (i, t) in terms.iter().enumerate() {
8884                    prop_assert_eq!(sec.id(t), Some(i as u32 + 1));
8885                    prop_assert_eq!(sec.term(i as u32 + 1), Some(t.clone()));
8886                }
8887                // the counts of the groups are the terms'
8888                let mut counts = crate::term_order::GroupCounts::default();
8889                terms.iter().for_each(|t| counts.add(t.as_bytes()));
8890                prop_assert_eq!(sec.groups(), Some(counts));
8891                let mut probes: Vec<String> = extra;
8892                probes.push(String::new());
8893                for (first, last) in &bounds {
8894                    for base in [first, last] {
8895                        if let Ok(s) = std::str::from_utf8(base) {
8896                            probes.push(s.to_string());
8897                            probes.push(format!("{s}\u{1}"));
8898                        }
8899                    }
8900                }
8901                for p in &probes {
8902                    let want = terms.iter().position(|t| t == p).map(|i| i as u32 + 1);
8903                    prop_assert_eq!(sec.id(p), want, "probe: {:?}", p);
8904                    // the rank is the number of terms below the probe
8905                    let below = terms
8906                        .iter()
8907                        .filter(|t| o.cmp_str(t, p) == std::cmp::Ordering::Less)
8908                        .count() as u32;
8909                    prop_assert_eq!(sec.rank(p), Some(below), "rank of {:?}", p);
8910                }
8911            }
8912        }
8913    }
8914
8915    /// The TEXT_INDEX section is inside the content hash: a freshly built
8916    /// text-indexed file must pass `verify()`, and flipping a byte inside the
8917    /// section must break it. (Regression: `verify()` once rebuilt the hash
8918    /// without the text index, so every `--text-index` file failed as corrupt.)
8919    #[test]
8920    fn text_index_is_tamper_evident_and_verifies() {
8921        let triples: Vec<(String, String, String)> = vec![(
8922            "<http://ex/s0>".to_string(),
8923            "<http://ex/label>".to_string(),
8924            "\"alpha glucose phosphate\"".to_string(),
8925        )];
8926        let bytes = build_text_indexed(&triples);
8927        let header = Rete::open(&bytes).unwrap().header().clone();
8928        assert!(header.text_index_len > 0);
8929        assert!(verify(&bytes).unwrap(), "a text-indexed build must verify");
8930
8931        let mut tampered = bytes.clone();
8932        tampered[header.text_index_offset as usize] ^= 0xff;
8933        assert!(
8934            !verify(&tampered).unwrap(),
8935            "tampering with the text index must break verify()"
8936        );
8937    }
8938
8939    /// End-to-end win: on a remote (range-read) file, a lookup whose routed tile
8940    /// is ruled out by a bound secondary fetches **fewer bytes** with the synopsis
8941    /// than without it — and the answer is identical (empty) either way.
8942    #[test]
8943    fn synopsis_cuts_remote_fetch_bytes() {
8944        use crate::header::FLAG_TILE_SYNOPSIS;
8945        use crate::reader::{CountingReader, SliceReader};
8946
8947        // Zero-padded terms ⇒ dictionary ids are monotonic in i; subject s_i pairs
8948        // only with object o_i, so an OSP tile (routed by object) holds a
8949        // contiguous subject range — a subject from a far tile is provably absent.
8950        let triples: Vec<(String, String, String)> = (0..400u32)
8951            .map(|i| {
8952                (
8953                    format!("<http://ex/s/{i:04}>"),
8954                    "<http://ex/p>".to_string(),
8955                    format!("<http://ex/o/{i:04}>"),
8956                )
8957            })
8958            .collect();
8959        let mut db = DictionaryBuilder::new();
8960        for (s, p, o) in &triples {
8961            db.observe(s, p, o);
8962        }
8963        let dict = db.build();
8964        let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
8965        for (s, p, o) in &triples {
8966            ib.push(dict.encode(s, p, o).unwrap());
8967        }
8968        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
8969
8970        // (s_0395, ?, o_0005): routes OSP by the (early) object, secondary = the
8971        // (late) subject — outside that tile's subject range, so the synopsis
8972        // prunes the one routed tile.
8973        let q = (Some("<http://ex/s/0395>"), None, Some("<http://ex/o/0005>"));
8974        // Measure the bytes the QUERY pulls (after open) — isolating the per-query
8975        // saving from the one-time synopsis trailer reads done at open, which a
8976        // persistent remote session amortizes over many queries.
8977        let query_bytes = |image: &[u8]| -> (u64, usize) {
8978            let leaked: &'static [u8] = Box::leak(image.to_vec().into_boxed_slice());
8979            let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
8980            let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
8981            let before = reader.bytes_read(); // after open (incl. trailer reads)
8982            let n = rete.query(q.0, q.1, q.2).len();
8983            assert!(!rete.index_incomplete());
8984            (reader.bytes_read() - before, n)
8985        };
8986
8987        let (on_bytes, on_n) = query_bytes(&bytes);
8988        // Same file with the synopsis flag cleared = an older reader's behavior.
8989        let mut off = bytes.clone();
8990        off[5] &= !FLAG_TILE_SYNOPSIS;
8991        let (off_bytes, off_n) = query_bytes(&off);
8992
8993        assert_eq!(on_n, 0, "the pair never co-occurs");
8994        assert_eq!(off_n, 0, "same answer without the synopsis");
8995        // Both pay the same dictionary-resolution bytes; the difference is the one
8996        // routed index tile that the synopsis skips (and the no-synopsis path
8997        // fetches only to have its zone map reject it).
8998        assert!(
8999            on_bytes < off_bytes,
9000            "synopsis skips the routed tile fetch: {on_bytes} < {off_bytes}"
9001        );
9002    }
9003
9004    /// A double-bound-object intersection (`?p P o1 ; P o2 ; label ?l`) — the
9005    /// shape whose REMOTE join strategy changed (scan + hash-join instead of
9006    /// probing each prefix row) — must return the SAME rows opened eagerly (in
9007    /// memory) and lazily (remote-style, `is_remote()` true). Strategy is a
9008    /// performance choice; the result multiset is invariant.
9009    #[test]
9010    fn double_bound_object_join_eager_matches_lazy() {
9011        use crate::reader::SliceReader;
9012        let occ = "<http://ex/occ>";
9013        let phys = "<http://ex/physicist>";
9014        let phil = "<http://ex/philosopher>";
9015        let label = "<http://www.w3.org/2000/01/rdf-schema#label>";
9016        // p00..p19 are physicists; p00..p09 are also philosophers (the answer).
9017        let mut triples: Vec<(String, String, String)> = Vec::new();
9018        for i in 0..20u32 {
9019            triples.push((format!("<http://ex/p/{i:02}>"), occ.into(), phys.into()));
9020            if i < 10 {
9021                triples.push((format!("<http://ex/p/{i:02}>"), occ.into(), phil.into()));
9022            }
9023            triples.push((
9024                format!("<http://ex/p/{i:02}>"),
9025                label.into(),
9026                format!("\"Name {i:02}\""),
9027            ));
9028        }
9029        let mut db = DictionaryBuilder::new();
9030        for (s, p, o) in &triples {
9031            db.observe(s, p, o);
9032        }
9033        let dict = db.build();
9034        let mut ib = GraphIndexBuilder::new().with_tile_budget(16);
9035        for (s, p, o) in &triples {
9036            ib.push(dict.encode(s, p, o).unwrap());
9037        }
9038        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9039
9040        let q = "SELECT ?l WHERE { \
9041            ?p <http://ex/occ> <http://ex/physicist> ; \
9042               <http://ex/occ> <http://ex/philosopher> ; \
9043               <http://www.w3.org/2000/01/rdf-schema#label> ?l }";
9044        let run = |rete: &Rete| -> Vec<String> {
9045            let (_, sols) = crate::eval_sparql(rete, q).unwrap();
9046            let mut v: Vec<String> = sols.iter().filter_map(|b| b.get("l").cloned()).collect();
9047            v.sort();
9048            v
9049        };
9050
9051        let eager_rows = run(&Rete::open(&bytes).unwrap());
9052        let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9053        let lazy = Rete::open_ranged_lazy_with(
9054            std::sync::Arc::new(SliceReader::new(leaked)),
9055            HEADER_LEN as u64,
9056        )
9057        .unwrap();
9058        let lazy_rows = run(&lazy);
9059        assert!(!lazy.index_incomplete());
9060
9061        assert_eq!(eager_rows.len(), 10, "the 10 physicist∩philosopher labels");
9062        assert_eq!(eager_rows, lazy_rows, "eager and lazy must agree exactly");
9063    }
9064
9065    /// A dictionary big enough to split into multiple chunks per section must
9066    /// round-trip every id↔term mapping through the chunked (v0.2) encoding —
9067    /// including terms at chunk boundaries and absent near-misses.
9068    #[test]
9069    fn multi_chunk_dictionary_round_trips() {
9070        let mut db = DictionaryBuilder::new();
9071        let term = |i: u32| format!("<http://example.org/some/long/prefix/entity/{i:06}>");
9072        for i in 0..6000u32 {
9073            db.observe(&term(i), "<http://ex/p>", &term(i + 1));
9074        }
9075        let dict = db.build();
9076        let mut ib = GraphIndexBuilder::new();
9077        for i in 0..6000u32 {
9078            ib.push(
9079                dict.encode(&term(i), "<http://ex/p>", &term(i + 1))
9080                    .unwrap(),
9081            );
9082        }
9083        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9084        let rete = Rete::open(&bytes).unwrap();
9085        let d = rete.dictionary();
9086        assert_eq!(d.term_count(), dict.term_count());
9087        for i in (0..6000).step_by(97).chain([0, 1, 5999, 6000]) {
9088            let t = term(i);
9089            let sid = dict.subject_id(&t);
9090            assert_eq!(d.subject_id(&t), sid, "subject_id({t})");
9091            if let Some(id) = sid {
9092                assert_eq!(d.subject_term(id).as_deref(), Some(t.as_str()));
9093            }
9094            let oid = dict.object_id(&t);
9095            assert_eq!(d.object_id(&t), oid, "object_id({t})");
9096        }
9097        assert_eq!(d.subject_id("<http://example.org/absent>"), None);
9098        assert_eq!(d.predicate_id("<http://ex/p>"), Some(1));
9099        assert_eq!(d.predicate_term(1).as_deref(), Some("<http://ex/p>"));
9100    }
9101
9102    #[test]
9103    #[cfg(feature = "compression")]
9104    fn compression_shrinks_repetitive_data() {
9105        // Many triples sharing IRI prefixes — exactly what front-coding + zstd
9106        // should crush. The compressed file must be far smaller than the raw
9107        // term bytes, and still query correctly.
9108        let mut db = DictionaryBuilder::new();
9109        let triples: Vec<(String, String, String)> = (0..500)
9110            .map(|i| {
9111                (
9112                    format!("<http://example.org/entity/{i}>"),
9113                    "<http://example.org/p/relatedTo>".to_string(),
9114                    format!("<http://example.org/entity/{}>", (i + 1) % 500),
9115                )
9116            })
9117            .collect();
9118        for (s, p, o) in &triples {
9119            db.observe(s, p, o);
9120        }
9121        let dict = db.build();
9122        let mut ib = GraphIndexBuilder::new();
9123        for (s, p, o) in &triples {
9124            ib.push(dict.encode(s, p, o).unwrap());
9125        }
9126        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9127
9128        let raw: usize = triples
9129            .iter()
9130            .map(|(s, p, o)| s.len() + p.len() + o.len())
9131            .sum();
9132        assert!(
9133            bytes.len() < raw / 2,
9134            "expected strong compression: file {} vs raw terms {raw}",
9135            bytes.len()
9136        );
9137
9138        // Still queryable after compression.
9139        let rete = Rete::open(&bytes).unwrap();
9140        let r = rete.query(Some("<http://example.org/entity/0>"), None, None);
9141        assert_eq!(r.len(), 1);
9142        assert_eq!(r[0].2, "<http://example.org/entity/1>");
9143    }
9144
9145    fn big_file_with_pyramid() -> Vec<u8> {
9146        // A ring of 300 entities -> index dwarfs dict+meta.
9147        let triples: Vec<(String, String, String)> = (0..300)
9148            .map(|i| {
9149                (
9150                    format!("<http://ex/e{i}>"),
9151                    "<http://ex/next>".to_string(),
9152                    format!("<http://ex/e{}>", (i + 1) % 300),
9153                )
9154            })
9155            .collect();
9156        let mut db = DictionaryBuilder::new();
9157        for (s, p, o) in &triples {
9158            db.observe(s, p, o);
9159        }
9160        let dict = db.build();
9161        let ids: Vec<_> = triples
9162            .iter()
9163            .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9164            .collect();
9165        let mut ib = GraphIndexBuilder::new();
9166        for &t in &ids {
9167            ib.push(t);
9168        }
9169        let (meta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
9170        write_file(&dict, &ib.build(), false, &meta, levels)
9171    }
9172
9173    #[test]
9174    fn ranged_open_is_minimal_and_correct() {
9175        use crate::reader::{CountingReader, SliceReader};
9176        let bytes = big_file_with_pyramid();
9177
9178        let full = CountingReader::new(SliceReader::new(&bytes));
9179        let rete = Rete::open_ranged(&full).unwrap();
9180        // Full open touches at most 4 ranges (header, dict, index, meta).
9181        assert!(full.requests() <= 4, "requests = {}", full.requests());
9182        assert_eq!(
9183            rete.query(Some("<http://ex/e0>"), None, None)[0].2,
9184            "<http://ex/e1>"
9185        );
9186
9187        // Summary-only open skips the index → strictly fewer bytes than the file.
9188        let summ_reader = CountingReader::new(SliceReader::new(&bytes));
9189        let view = SummaryView::open_ranged(&summ_reader).unwrap().unwrap();
9190        assert!(!view.summary.is_empty());
9191        assert!(
9192            summ_reader.bytes_read() < bytes.len() as u64,
9193            "summary read {} of {} bytes",
9194            summ_reader.bytes_read(),
9195            bytes.len()
9196        );
9197        // And fewer than a full open, since it never fetched the index.
9198        assert!(summ_reader.bytes_read() < full.bytes_read());
9199    }
9200
9201    #[test]
9202    fn content_hash_is_set_and_verifies() {
9203        let bytes = build_image();
9204        let rete = Rete::open(&bytes).unwrap();
9205        assert_ne!(
9206            rete.header().content_hash,
9207            [0u8; 16],
9208            "hash must be populated"
9209        );
9210        assert!(verify(&bytes).unwrap(), "freshly built file verifies");
9211        assert!(
9212            verify_ranged(&crate::reader::SliceReader::new(&bytes)).unwrap(),
9213            "the streaming verify agrees"
9214        );
9215
9216        // Same data builds an identical hash (deterministic).
9217        assert_eq!(
9218            Rete::open(&build_image()).unwrap().header().content_hash,
9219            rete.header().content_hash
9220        );
9221
9222        // Corrupting a payload byte breaks verification.
9223        let mut tampered = bytes.clone();
9224        let last = tampered.len() - 5; // inside payload, before footer magic
9225        tampered[last] ^= 0xff;
9226        assert!(!verify(&tampered).unwrap());
9227        assert!(!verify_ranged(&crate::reader::SliceReader::new(&tampered)).unwrap());
9228        // A truncated file is an error, never a pass.
9229        let short = &bytes[..bytes.len() / 2];
9230        assert!(verify_ranged(&crate::reader::SliceReader::new(short)).is_err());
9231        assert!(verify_ranged(&crate::reader::SliceReader::new(&bytes[..10])).is_err());
9232    }
9233
9234    /// Build the standard 3-triple image with an opaque metadata payload.
9235    fn build_with_metadata(meta: &[u8]) -> Vec<u8> {
9236        let triples = [
9237            ("Alice", "knows", "Bob"),
9238            ("Bob", "knows", "Carol"),
9239            ("Alice", "age", "30"),
9240        ];
9241        let mut db = DictionaryBuilder::new();
9242        for (s, p, o) in triples {
9243            db.observe(s, p, o);
9244        }
9245        let dict = db.build();
9246        let ids: Vec<_> = triples
9247            .iter()
9248            .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9249            .collect();
9250        let mut ib = GraphIndexBuilder::new();
9251        for &t in &ids {
9252            ib.push(t);
9253        }
9254        let (pmeta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
9255        write_dataset_with_metadata(&dict, &ib.build(), &[], false, &pmeta, levels, meta, &[])
9256    }
9257
9258    #[test]
9259    fn metadata_round_trips_and_shifts_offsets() {
9260        let card = br#"{"title":"My Dataset"}"#;
9261        let bytes = build_with_metadata(card);
9262        let rete = Rete::open(&bytes).unwrap();
9263
9264        // The opaque payload reads back verbatim.
9265        assert_eq!(rete.metadata(), Some(card.as_slice()));
9266        let h = rete.header();
9267        assert_eq!(h.metadata_offset, HEADER_LEN as u64);
9268        assert_eq!(h.metadata_len, card.len() as u64);
9269        // The dictionary (and everything after it) shifted forward by the card.
9270        assert_eq!(h.dictionary_offset, HEADER_LEN as u64 + card.len() as u64);
9271
9272        // The index still decodes correctly at its shifted offset.
9273        assert_eq!(
9274            rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9275            1
9276        );
9277        // The card is inside the content hash, so the file still verifies.
9278        assert!(verify(&bytes).unwrap());
9279    }
9280
9281    #[test]
9282    fn empty_metadata_is_byte_identical_to_plain_writer() {
9283        // The `&[]` path must produce exactly the bytes of the metadata-free
9284        // writer for identical inputs — old files and outputs are unchanged.
9285        assert_eq!(
9286            build_with_metadata(&[]),
9287            build_image(),
9288            "empty-metadata output must equal the plain writer byte-for-byte"
9289        );
9290    }
9291
9292    /// The contract `replace_metadata` has to meet to be usable at the end of a
9293    /// build: swapping the card must land on **exactly** the bytes the writer
9294    /// would have produced with that card in the first place — shorter, longer,
9295    /// and gone — so a spliced file and a rebuilt one are the same file.
9296    #[test]
9297    fn replace_metadata_matches_a_direct_write() {
9298        let first = br#"{"title":"My Dataset","queries":[1,2,3]}"#;
9299        let bytes = build_with_metadata(first);
9300        for target in [
9301            br#"{"title":"My Dataset","queries":[1,2]}"#.as_slice(), // shorter
9302            br#"{"title":"My Dataset","queries":[1,2,3,4,5,6,7]}"#.as_slice(), // longer
9303            b"".as_slice(),                                          // removed
9304            first.as_slice(),                                        // unchanged
9305        ] {
9306            let spliced = replace_metadata(&bytes, target).unwrap();
9307            assert_eq!(
9308                spliced,
9309                build_with_metadata(target),
9310                "splicing {} bytes of metadata must equal a direct write",
9311                target.len()
9312            );
9313            assert!(
9314                verify(&spliced).unwrap(),
9315                "the new hash covers the new card"
9316            );
9317            let rete = Rete::open(&spliced).unwrap();
9318            assert_eq!(
9319                rete.metadata(),
9320                (!target.is_empty()).then_some(target),
9321                "the new payload reads back verbatim"
9322            );
9323            // The data is untouched by the surgery.
9324            assert_eq!(
9325                rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9326                1
9327            );
9328        }
9329    }
9330
9331    /// A build-info section survives the splice, still adjacent to the card, and
9332    /// still outside the content hash.
9333    #[test]
9334    fn replace_metadata_keeps_build_info_adjacent() {
9335        let with_info =
9336            attach_build_info(&build_with_metadata(br#"{"a":1}"#), br#"{"builder":"x"}"#).unwrap();
9337        let spliced = replace_metadata(&with_info, br#"{"a":1,"b":2}"#).unwrap();
9338        let h = Header::from_bytes(&spliced).unwrap();
9339        assert_eq!(h.metadata_offset, HEADER_LEN as u64);
9340        assert_eq!(h.build_info_offset, HEADER_LEN as u64 + h.metadata_len);
9341        assert_eq!(
9342            read_build_info(&spliced).unwrap().as_deref(),
9343            Some(br#"{"builder":"x"}"#.as_slice())
9344        );
9345        assert!(verify(&spliced).unwrap());
9346        // Same card, same content hash, whatever the build-info says.
9347        assert_eq!(
9348            h.content_hash,
9349            Header::from_bytes(&build_with_metadata(br#"{"a":1,"b":2}"#))
9350                .unwrap()
9351                .content_hash
9352        );
9353    }
9354
9355    #[test]
9356    fn metadata_is_tamper_evident() {
9357        let card = br#"{"title":"x"}"#;
9358        let mut bytes = build_with_metadata(card);
9359        assert!(verify(&bytes).unwrap());
9360        // The card occupies [HEADER_LEN .. HEADER_LEN+card_len); flip a byte in it.
9361        bytes[HEADER_LEN + 2] ^= 0xff;
9362        assert!(
9363            !verify(&bytes).unwrap(),
9364            "tampering with the card must break verify()"
9365        );
9366    }
9367
9368    #[test]
9369    fn ranged_opens_do_not_fetch_metadata() {
9370        use crate::reader::{CountingReader, SliceReader};
9371        let card = vec![0xABu8; 512]; // distinctive and sizable
9372        let bytes = build_with_metadata(&card);
9373        let total = bytes.len() as u64;
9374
9375        // A full ranged open never loads the card and never reads its byte range.
9376        let r = CountingReader::new(SliceReader::new(&bytes));
9377        let rete = Rete::open_ranged(&r).unwrap();
9378        assert!(
9379            rete.metadata().is_none(),
9380            "open_ranged must not load the card"
9381        );
9382        assert!(r.requests() <= 4, "requests = {}", r.requests());
9383        assert!(
9384            r.bytes_read() <= total - card.len() as u64,
9385            "read {} of {} bytes; the {}-byte card must be skipped",
9386            r.bytes_read(),
9387            total,
9388            card.len()
9389        );
9390
9391        // Summary-only open likewise ignores the card and still summarizes.
9392        let rs = CountingReader::new(SliceReader::new(&bytes));
9393        let view = SummaryView::open_ranged(&rs).unwrap().unwrap();
9394        assert!(!view.summary.is_empty());
9395        assert!(rs.bytes_read() <= total - card.len() as u64);
9396    }
9397
9398    #[test]
9399    fn metadata_ranged_fetches_only_header_and_card() {
9400        use crate::reader::{CountingReader, SliceReader};
9401        // The CARD tier: fetch the self-description over a RangeReader touching
9402        // only the header + metadata range — never the dictionary/index/pyramid.
9403        let card = vec![0xCDu8; 384];
9404        let bytes = build_with_metadata(&card);
9405
9406        let r = CountingReader::new(SliceReader::new(&bytes));
9407        let got = read_metadata_ranged(&r).unwrap().unwrap();
9408        assert_eq!(got, card, "the card reads back verbatim");
9409        assert_eq!(r.requests(), 2, "exactly header + metadata ranges");
9410        assert_eq!(
9411            r.bytes_read(),
9412            HEADER_LEN as u64 + card.len() as u64,
9413            "no dictionary/index/pyramid bytes are touched"
9414        );
9415
9416        // A cardless file resolves to None after a single header read.
9417        let plain = build_image();
9418        let rp = CountingReader::new(SliceReader::new(&plain));
9419        assert!(read_metadata_ranged(&rp).unwrap().is_none());
9420        assert_eq!(rp.requests(), 1, "header only for a cardless file");
9421        assert_eq!(rp.bytes_read(), HEADER_LEN as u64);
9422    }
9423
9424    #[test]
9425    fn build_info_attaches_outside_the_hash() {
9426        let card = br#"{"title":"My Dataset"}"#;
9427        let base = build_with_metadata(card);
9428        let base_hash = Header::from_bytes(&base).unwrap().content_hash;
9429
9430        let info = br#"{"built_at":"2026-08-04T00:00:00Z","builder":"rete-cli 0.3.2"}"#;
9431        let with = attach_build_info(&base, info).unwrap();
9432
9433        // The section reads back verbatim, adjacent to the metadata.
9434        assert_eq!(read_build_info(&with).unwrap().unwrap(), info);
9435        let h = Header::from_bytes(&with).unwrap();
9436        assert_eq!(h.build_info_offset, HEADER_LEN as u64 + card.len() as u64);
9437        assert_eq!(h.build_info_len, info.len() as u64);
9438        assert_eq!(h.expected_file_len(), Some(with.len() as u64));
9439
9440        // OUTSIDE the hash: the content hash is unchanged and the file still
9441        // verifies — two builds of identical data stay hash-equal even though
9442        // their build-info differs.
9443        assert_eq!(h.content_hash, base_hash);
9444        assert!(verify(&with).unwrap());
9445        let other = attach_build_info(&base, br#"{"built_at":"1999-01-01T00:00:00Z"}"#).unwrap();
9446        assert_eq!(Header::from_bytes(&other).unwrap().content_hash, base_hash);
9447
9448        // The graph still opens and answers at its shifted offsets.
9449        let rete = Rete::open(&with).unwrap();
9450        assert_eq!(
9451            rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9452            1
9453        );
9454        assert_eq!(rete.metadata(), Some(card.as_slice()));
9455
9456        // Replacing an existing section works and removing it restores the
9457        // original image byte-for-byte (the strip-equality determinism proof).
9458        let replaced = attach_build_info(&with, br#"{"builder":"other"}"#).unwrap();
9459        assert_eq!(
9460            read_build_info(&replaced).unwrap().unwrap(),
9461            br#"{"builder":"other"}"#
9462        );
9463        assert!(verify(&replaced).unwrap());
9464        let stripped = attach_build_info(&replaced, &[]).unwrap();
9465        assert_eq!(stripped, base, "stripping build-info restores the image");
9466    }
9467
9468    #[test]
9469    fn build_info_attaches_on_a_metadata_free_file() {
9470        // No card: the section slots in right after the header.
9471        let base = build_image();
9472        let info = b"{\"builder\":\"x\"}";
9473        let with = attach_build_info(&base, info).unwrap();
9474        let h = Header::from_bytes(&with).unwrap();
9475        assert_eq!(h.build_info_offset, HEADER_LEN as u64);
9476        assert_eq!(h.metadata_len, 0);
9477        assert_eq!(read_build_info(&with).unwrap().unwrap(), info);
9478        assert!(verify(&with).unwrap());
9479        assert_eq!(
9480            h.content_hash,
9481            Header::from_bytes(&base).unwrap().content_hash
9482        );
9483        Rete::open(&with).unwrap();
9484    }
9485
9486    /// The contract a **streaming** rewriter needs: a plan is enough to rebuild
9487    /// the file from three spans — the header it hands back, the untouched
9488    /// `[HEADER_LEN, insert)` prefix, the new section, and the untouched
9489    /// `[tail_start, len)` tail — and the result is byte-identical to the
9490    /// in-memory splice. That is what lets a 17 GB file gain a build record
9491    /// with a 4 MiB buffer instead of 34 GB of RAM.
9492    #[test]
9493    fn a_plan_rebuilds_exactly_what_the_in_memory_splice_produces() {
9494        let card = br#"{"title":"My Dataset"}"#;
9495        let base = build_with_metadata(card);
9496        let base_hash = Header::from_bytes(&base).unwrap().content_hash;
9497        for start in [
9498            base.clone(),
9499            attach_build_info(&base, b"{\"a\":1}").unwrap(),
9500        ] {
9501            for info in [
9502                br#"{"builder":"rete-cli 0.3.2","query_costs":{}}"#.as_slice(), // longer
9503                b"{}".as_slice(),                                               // shorter
9504                b"".as_slice(),                                                 // removed
9505            ] {
9506                let plan = plan_build_info(&start, start.len() as u64, info.len() as u64).unwrap();
9507                let mut streamed = Vec::new();
9508                streamed.extend_from_slice(&plan.header);
9509                streamed.extend_from_slice(&start[HEADER_LEN..plan.insert as usize]);
9510                streamed.extend_from_slice(info);
9511                streamed.extend_from_slice(&start[plan.tail_start as usize..]);
9512                assert_eq!(streamed.len() as u64, plan.new_len);
9513                assert_eq!(
9514                    streamed,
9515                    attach_build_info(&start, info).unwrap(),
9516                    "a {}-byte section streamed must equal the same section spliced",
9517                    info.len()
9518                );
9519                // The point of the exercise: the file's identity is untouched.
9520                assert!(verify(&streamed).unwrap());
9521                let h = Header::from_bytes(&streamed).unwrap();
9522                assert_eq!(h.content_hash, base_hash);
9523                assert_eq!(h.expected_file_len(), Some(plan.new_len));
9524                let rete = Rete::open(&streamed).unwrap();
9525                assert_eq!(rete.metadata(), Some(card.as_slice()));
9526                assert_eq!(
9527                    rete.query(Some("Bob"), Some("knows"), Some("Carol")).len(),
9528                    1
9529                );
9530            }
9531        }
9532    }
9533
9534    /// A plan is refused rather than trusted when the header's own numbers put
9535    /// the section outside the file — the header is attacker-controlled input.
9536    #[test]
9537    fn a_plan_refuses_a_header_that_overruns_the_file() {
9538        let base = build_with_metadata(br#"{"title":"x"}"#);
9539        assert!(plan_build_info(&base, 8, 16).is_err());
9540        assert!(plan_build_info(&base[..HEADER_LEN], base.len() as u64, 16).is_ok());
9541    }
9542
9543    #[test]
9544    fn card_and_build_info_ranged_is_one_header_plus_one_range() {
9545        use crate::reader::{CountingReader, SliceReader};
9546        let card = vec![0xCDu8; 384];
9547        let info = vec![0xEFu8; 200];
9548        let bytes = attach_build_info(&build_with_metadata(&card), &info).unwrap();
9549
9550        // Both sections in ONE coalesced range after the header — the CARD
9551        // tier's 1 header + 1 range budget holds with build info included.
9552        let r = CountingReader::new(SliceReader::new(&bytes));
9553        let (m, b) = read_card_and_build_info_ranged(&r).unwrap();
9554        assert_eq!(m.unwrap(), card);
9555        assert_eq!(b.unwrap(), info);
9556        assert_eq!(r.requests(), 2, "header + one coalesced range");
9557        assert_eq!(
9558            r.bytes_read(),
9559            HEADER_LEN as u64 + card.len() as u64 + info.len() as u64
9560        );
9561
9562        // Card only: same two requests, no build info.
9563        let plain = build_with_metadata(&card);
9564        let rp = CountingReader::new(SliceReader::new(&plain));
9565        let (m, b) = read_card_and_build_info_ranged(&rp).unwrap();
9566        assert_eq!(m.unwrap(), card);
9567        assert!(b.is_none());
9568        assert_eq!(rp.requests(), 2);
9569        // Stated as the comparison every client depends on: carrying a build
9570        // record costs a reader NOTHING in requests. Any client that shows the
9571        // build record — the CLI, the browser card modal — can therefore do it
9572        // in the card's own budget, and does not need a second call.
9573        assert_eq!(
9574            r.requests(),
9575            rp.requests(),
9576            "build info must not cost an extra request",
9577        );
9578
9579        // Neither: the header read alone answers.
9580        let none = build_image();
9581        let rn = CountingReader::new(SliceReader::new(&none));
9582        let (m, b) = read_card_and_build_info_ranged(&rn).unwrap();
9583        assert!(m.is_none() && b.is_none());
9584        assert_eq!(rn.requests(), 1);
9585    }
9586
9587    #[test]
9588    fn schema_summary_groups_by_type() {
9589        let rt = RDF_TYPE;
9590        let bytes = build_from(&[
9591            ("Alice", rt, "Person"),
9592            ("Bob", rt, "Person"),
9593            ("NYC", rt, "City"),
9594            ("Alice", "knows", "Bob"),
9595            ("Alice", "livesIn", "NYC"),
9596            ("Alice", "name", "\"Alice\""),
9597        ]);
9598        let rete = Rete::open(&bytes).unwrap();
9599        let summary = schema_summary(&rete);
9600        // Expect class-level relations, rdf:type excluded.
9601        assert!(summary.contains(&("Person".into(), "knows".into(), "Person".into(), 1)));
9602        assert!(summary.contains(&("Person".into(), "livesIn".into(), "City".into(), 1)));
9603        assert!(summary.contains(&("Person".into(), "name".into(), "(literal)".into(), 1)));
9604        // No rdf:type relations in the summary.
9605        assert!(!summary.iter().any(|(_, p, _, _)| p == RDF_TYPE));
9606
9607        // Class populations: 2 People, 1 City, sorted by count desc.
9608        let classes = schema_classes(&rete);
9609        assert_eq!(
9610            classes,
9611            vec![("Person".into(), 2u32), ("City".into(), 1u32)]
9612        );
9613    }
9614
9615    fn build_from(triples: &[(&str, &str, &str)]) -> Vec<u8> {
9616        let mut db = DictionaryBuilder::new();
9617        for (s, p, o) in triples {
9618            db.observe(s, p, o);
9619        }
9620        let dict = db.build();
9621        let mut ib = GraphIndexBuilder::new();
9622        for (s, p, o) in triples {
9623            ib.push(dict.encode(s, p, o).unwrap());
9624        }
9625        write_file(&dict, &ib.build(), false, &[], 0)
9626    }
9627
9628    /// Build a file WITH a pyramid (so the schema pyramid + coherence axioms ship).
9629    fn build_with_pyramid(triples: &[(&str, &str, &str)]) -> Vec<u8> {
9630        let mut db = DictionaryBuilder::new();
9631        for (s, p, o) in triples {
9632            db.observe(s, p, o);
9633        }
9634        let dict = db.build();
9635        let encoded: Vec<_> = triples
9636            .iter()
9637            .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
9638            .collect();
9639        let mut ib = GraphIndexBuilder::new();
9640        for t in &encoded {
9641            ib.push(*t);
9642        }
9643        let (meta, levels) = build_pyramid_meta(&dict, &encoded, DEFAULT_TILE_BUDGET);
9644        write_dataset(&dict, &ib.build(), &[], false, &meta, levels)
9645    }
9646
9647    #[test]
9648    fn tbox_coherence_flags_unsatisfiable_class_index_free() {
9649        use crate::reader::{CountingReader, SliceReader};
9650        let rt = RDF_TYPE;
9651        let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9652        let disj = "<http://www.w3.org/2002/07/owl#disjointWith>";
9653        // C ⊑ D, C ⊑ E, D disjointWith E ⇒ C unsatisfiable — schema-only, with one
9654        // instance present just so the schema pyramid gets built.
9655        let bytes = build_with_pyramid(&[
9656            ("<http://ex/C>", sub, "<http://ex/D>"),
9657            ("<http://ex/C>", sub, "<http://ex/E>"),
9658            ("<http://ex/D>", disj, "<http://ex/E>"),
9659            ("<http://ex/x>", rt, "<http://ex/C>"),
9660        ]);
9661
9662        let r = CountingReader::new(SliceReader::new(&bytes));
9663        let view = SummaryView::open_ranged(&r).unwrap().unwrap();
9664        let points = view.tbox_coherence();
9665        assert!(
9666            points
9667                .iter()
9668                .any(|i| i.kind == "unsatisfiable-class" && i.detail.contains("http://ex/C>")),
9669            "expected C unsatisfiable from the schema alone, got {points:?}"
9670        );
9671
9672        // Proven index-free: bytes read never reach the (root_dir) index section,
9673        // mirroring `schema_pyramid_round_trips_through_file_index_free`.
9674        let header = Header::from_bytes(&bytes[..HEADER_LEN]).unwrap();
9675        assert!(
9676            r.bytes_read() <= bytes.len() as u64 - header.root_dir_len,
9677            "tbox_coherence must not read the triple index"
9678        );
9679    }
9680
9681    #[test]
9682    fn schema_coherence_reads_only_the_schema_block() {
9683        use crate::reader::{CountingReader, SliceReader};
9684        let rt = RDF_TYPE;
9685        let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9686        let disj = "<http://www.w3.org/2002/07/owl#disjointWith>";
9687        // 500 instances with unique literals → a sizable dictionary + community
9688        // summary, so a whole-pyramid-meta read would be large; the schema block
9689        // (bounded by the tiny ontology) stays small.
9690        let mut triples: Vec<(String, String, String)> = vec![
9691            ("<http://ex/C>".into(), sub.into(), "<http://ex/D>".into()),
9692            ("<http://ex/C>".into(), sub.into(), "<http://ex/E>".into()),
9693            ("<http://ex/D>".into(), disj.into(), "<http://ex/E>".into()),
9694        ];
9695        for i in 0..500 {
9696            let s = format!("<http://ex/x{i}>");
9697            triples.push((s.clone(), rt.into(), "<http://ex/C>".into()));
9698            triples.push((
9699                s,
9700                "<http://ex/label>".into(),
9701                format!("\"unique label {i}\""),
9702            ));
9703        }
9704        let trefs: Vec<(&str, &str, &str)> = triples
9705            .iter()
9706            .map(|(s, p, o)| (s.as_str(), p.as_str(), o.as_str()))
9707            .collect();
9708        let bytes = build_with_pyramid(&trefs);
9709
9710        let header = Header::from_bytes(&bytes[..HEADER_LEN]).unwrap();
9711        assert!(
9712            header.schema_meta_len > 0,
9713            "the writer recorded a schema-block length"
9714        );
9715        assert!(
9716            (header.schema_meta_len as u64) < header.pyramid_meta_len,
9717            "schema block ({}) should be far smaller than the whole pyramid-meta ({})",
9718            header.schema_meta_len,
9719            header.pyramid_meta_len
9720        );
9721
9722        let r = CountingReader::new(SliceReader::new(&bytes));
9723        let points = read_schema_coherence_ranged(&r).unwrap().unwrap();
9724        assert!(points.iter().any(|i| i.kind == "unsatisfiable-class"));
9725        // It read only the header + the schema block — not the summary or dictionary.
9726        assert!(
9727            r.bytes_read() <= HEADER_LEN as u64 + header.schema_meta_len as u64,
9728            "read {} bytes; expected <= header + schema block ({})",
9729            r.bytes_read(),
9730            HEADER_LEN as u64 + header.schema_meta_len as u64
9731        );
9732    }
9733
9734    #[test]
9735    fn tbox_coherence_clean_schema_is_coherent() {
9736        use crate::reader::SliceReader;
9737        let rt = RDF_TYPE;
9738        let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
9739        let bytes = build_with_pyramid(&[
9740            ("<http://ex/Dog>", sub, "<http://ex/Animal>"),
9741            ("<http://ex/x>", rt, "<http://ex/Dog>"),
9742        ]);
9743        let view = SummaryView::open_ranged(&SliceReader::new(&bytes))
9744            .unwrap()
9745            .unwrap();
9746        assert!(view.tbox_is_coherent(), "a plain hierarchy is coherent");
9747    }
9748
9749    /// A quad-index file must be **refused** by every reader that predates it —
9750    /// an old reader ignores flag bit 3 and the unknown kind 10, so it would
9751    /// otherwise open the file as one without named graphs and answer `GRAPH`
9752    /// queries with nothing. The seal makes each of their open paths fail on the
9753    /// index container's section count, before any payload is read; their
9754    /// `verify` still passes, because the part list it hashes is unchanged.
9755    /// (Checked against real 0.3.x binaries too: QUAD-INDEX-DESIGN.md §9.)
9756    #[test]
9757    fn readers_before_the_quad_index_refuse_its_files() {
9758        for perms in [PermSet::ALL, PermSet::CORE] {
9759            let quads: Vec<crate::ingest::RawQuad> = (0..50)
9760                .map(|i| {
9761                    (
9762                        format!("<http://ex/s{i}>"),
9763                        "<http://ex/p>".to_string(),
9764                        format!("\"{i}\""),
9765                        Some(format!("<http://ex/g{}>", i % 7)),
9766                    )
9767                })
9768                .collect();
9769            let (bytes, _) = crate::ingest::assemble_dataset_with_layout(
9770                quads,
9771                false,
9772                false,
9773                None,
9774                crate::PyramidAlgo::Louvain,
9775                perms,
9776                crate::ingest::GraphLayout::Quad,
9777                |_, _| Vec::<u8>::new(),
9778            );
9779            let h = Header::from_bytes(&bytes).unwrap();
9780            assert!(h.has_quad_index());
9781            let root =
9782                &bytes[h.root_dir_offset as usize..(h.root_dir_offset + h.root_dir_len) as usize];
9783            // 0.3.4–0.3.7, resident (`Rete::open`, `open_ranged`): count != mask.
9784            assert!(decode_index_container(root, h.block_codec, h.perms).is_err());
9785            assert!(
9786                container_section_payload_ranges(root, h.root_dir_offset, h.perms.len()).is_err()
9787            );
9788            // 0.3.0–0.3.3 expected exactly six sections, whatever the mask.
9789            assert_ne!(decode_container(root, CODEC_NONE).unwrap().len(), NUM_PERMS);
9790            assert_ne!(
9791                decode_container(root, CODEC_NONE).unwrap().len(),
9792                h.perms.len()
9793            );
9794            // Every lazy / remote open (`open_ranged_lazy`, `query_ranged`).
9795            let r = crate::reader::SliceReader::new(&bytes);
9796            for expect in [h.perms.len(), NUM_PERMS] {
9797                assert!(locate_container_section_ranged(
9798                    &r,
9799                    h.root_dir_offset,
9800                    h.root_dir_len,
9801                    0,
9802                    expect as u64
9803                )
9804                .is_err());
9805            }
9806            // Their `verify` hashes the same parts the writer hashed.
9807            assert_eq!(hash_of_sections(&h, &bytes).unwrap(), h.content_hash);
9808            // And a current reader reads the file.
9809            let rete = Rete::open(&bytes).unwrap();
9810            assert_eq!(rete.named_graph_count(), 7);
9811            assert_eq!(rete.dump(Some("<http://ex/g3>")).len(), 7);
9812        }
9813    }
9814
9815    #[test]
9816    fn named_graphs_round_trip() {
9817        // One shared dictionary; default graph + a named graph "g1".
9818        let all = [
9819            ("Alice", "knows", "Bob"), // default
9820            ("Bob", "age", "30"),      // named g1
9821        ];
9822        let mut db = DictionaryBuilder::new();
9823        for (s, p, o) in all {
9824            db.observe(s, p, o);
9825        }
9826        let dict = db.build();
9827
9828        let mut def = GraphIndexBuilder::new();
9829        def.push(dict.encode("Alice", "knows", "Bob").unwrap());
9830        let mut g1 = GraphIndexBuilder::new();
9831        g1.push(dict.encode("Bob", "age", "30").unwrap());
9832
9833        let named = vec![("http://ex/g1".to_string(), g1.build())];
9834        let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9835
9836        assert!(verify(&bytes).unwrap());
9837        let rete = Rete::open(&bytes).unwrap();
9838        assert_eq!(rete.graph_names(), vec!["http://ex/g1"]);
9839        // The named graph contains Bob age 30, not the default-graph triple.
9840        let gi = rete.graph_index("http://ex/g1").unwrap();
9841        assert_eq!(gi.triple_count(), 1);
9842        assert!(rete.graph_index("http://ex/missing").is_none());
9843
9844        // quad_count counts ALL quads — default graph + named graphs (1 + 1),
9845        // not just the default index (which would report 1).
9846        assert_eq!(rete.header().quad_count, 2);
9847
9848        // Default-graph query path is unchanged.
9849        assert_eq!(rete.query(Some("Alice"), None, None).len(), 1);
9850
9851        // dump() round-trips each graph back to terms.
9852        assert_eq!(
9853            rete.dump(None),
9854            vec![("Alice".into(), "knows".into(), "Bob".into())]
9855        );
9856        assert_eq!(
9857            rete.dump(Some("http://ex/g1")),
9858            vec![("Bob".into(), "age".into(), "30".into())]
9859        );
9860    }
9861
9862    /// A named graph whose container exceeds [`NAMED_GRAPH_RESIDENT_MAX`]
9863    /// opens through [`open_index_container_lazy`] — the default graph's own
9864    /// machinery, at the container's offset. Drive that helper directly over a
9865    /// named graph's container range (the threshold itself is too large to
9866    /// cross cheaply in a unit test) and require identical matches.
9867    #[test]
9868    fn named_graph_container_opens_tile_lazily_like_the_root() {
9869        let mut db = DictionaryBuilder::new();
9870        let node = |n: u32| format!("<http://ex/n{n}>");
9871        let p = "<http://ex/p>".to_string();
9872        db.observe(&node(0), &p, &node(1));
9873        for i in 0..500u32 {
9874            db.observe(&node(1000 + i), &p, &node(2000 + i));
9875        }
9876        let dict = db.build();
9877        let mut def = GraphIndexBuilder::new();
9878        def.push(dict.encode(&node(0), &p, &node(1)).unwrap());
9879        let mut g = GraphIndexBuilder::new().with_tile_budget(256); // many tiles
9880        for i in 0..500u32 {
9881            g.push(dict.encode(&node(1000 + i), &p, &node(2000 + i)).unwrap());
9882        }
9883        let named = vec![("<http://ex/g>".to_string(), g.build())];
9884        let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
9885        let rete = Rete::open(&bytes).unwrap();
9886        let header = rete.header();
9887
9888        // The single entry's container range: count varint, iri_len varint,
9889        // iri, container_len varint, then the container.
9890        let soff = header.named_graphs_offset as usize;
9891        let send = soff + header.named_graphs_len as usize;
9892        let sec = &bytes[soff..send];
9893        let (n, mut pos) = read_uvarint(sec).unwrap();
9894        assert_eq!(n, 1);
9895        let (ilen, u1) = read_uvarint(&sec[pos..]).unwrap();
9896        pos += u1 + ilen as usize;
9897        let (clen, u2) = read_uvarint(&sec[pos..]).unwrap();
9898        pos += u2;
9899        let container = ByteRange {
9900            offset: (soff + pos) as u64,
9901            len: clen,
9902        };
9903
9904        use crate::reader::SliceReader;
9905        let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9906        let reader: std::sync::Arc<dyn RangeReader + Send + Sync> =
9907            std::sync::Arc::new(SliceReader::new(leaked));
9908        let (lazy_idx, _, _) = open_index_container_lazy(
9909            &reader,
9910            container,
9911            header.block_codec,
9912            header.has_tile_synopsis(),
9913            1,
9914            header.perms,
9915            header.perms.len(),
9916        )
9917        .unwrap();
9918        let mut want = rete
9919            .graph_index("<http://ex/g>")
9920            .unwrap()
9921            .match_pattern((None, None, None));
9922        let mut got = lazy_idx.match_pattern((None, None, None));
9923        want.sort_unstable();
9924        got.sort_unstable();
9925        assert_eq!(got.len(), 500);
9926        assert_eq!(got, want);
9927        assert!(!lazy_idx.load_incomplete());
9928    }
9929
9930    /// `dump_iter` must agree with `dump` exactly — and must be *lazy*: taking
9931    /// the first triple of a lazily range-read file may not drag the whole
9932    /// index across the reader, or the JS client's streaming quad cursor (which
9933    /// is this iterator, suspended between wasm calls) would be a materializing
9934    /// dump wearing a cursor's clothes.
9935    #[test]
9936    fn dump_iter_matches_dump_and_stops_early() {
9937        use crate::reader::{CountingReader, SliceReader};
9938
9939        // Enough triples over tiny tiles to force many tiles per permutation.
9940        let triples: Vec<(String, String, String)> = (0..2000u32)
9941            .map(|i| {
9942                (
9943                    format!("<http://ex/s/{i:04}>"),
9944                    "<http://ex/p>".to_string(),
9945                    format!("<http://ex/o/{i:04}>"),
9946                )
9947            })
9948            .collect();
9949        let mut db = DictionaryBuilder::new();
9950        for (s, p, o) in &triples {
9951            db.observe(s, p, o);
9952        }
9953        let dict = db.build();
9954        let mut ib = GraphIndexBuilder::new().with_tile_budget(256);
9955        for (s, p, o) in &triples {
9956            ib.push(dict.encode(s, p, o).unwrap());
9957        }
9958        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
9959
9960        // Same triples, same order, as the eager dump.
9961        let rete = Rete::open(&bytes).unwrap();
9962        assert_eq!(rete.dump_iter(None).collect::<Vec<_>>(), rete.dump(None));
9963        // A missing named graph yields nothing rather than the default graph.
9964        assert_eq!(rete.dump_iter(Some("http://ex/nope")).count(), 0);
9965
9966        // Lazily: taking one triple must cost far fewer bytes than draining.
9967        let read_bytes = |take: usize| -> u64 {
9968            let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
9969            let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
9970            let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
9971            let before = reader.bytes_read();
9972            let n = rete.dump_iter(None).take(take).count();
9973            assert_eq!(n, take.min(triples.len()));
9974            assert!(!rete.index_incomplete());
9975            reader.bytes_read() - before
9976        };
9977        let first_only = read_bytes(1);
9978        let everything = read_bytes(triples.len());
9979        // Always true: stopping after one triple cannot cost a full scan.
9980        assert!(
9981            first_only < everything,
9982            "dump_iter is not lazy: one triple read {first_only} B, a full scan {everything} B"
9983        );
9984        // The STRONG claim — under half — only holds once the payload dwarfs the
9985        // fixed open cost. This fixture is ~32 KB, and with `compression` off the
9986        // dictionary is essentially the whole file, so opening it at all already
9987        // pays most of `everything` (21 KB of 33 KB) and the ratio says nothing
9988        // about laziness. Asserting it unconditionally is how this test failed on
9989        // `--no-default-features` while passing everywhere else.
9990        #[cfg(feature = "compression")]
9991        assert!(
9992            first_only * 2 < everything,
9993            "dump_iter is not lazy: one triple read {first_only} B of the {everything} B a full scan reads"
9994        );
9995
9996        // A FRESH open: these assertions fault tiles and the dictionary in,
9997        // and the laziness measurement above counts bytes on a COLD reader —
9998        // sharing one `rete` makes first_only and everything both ~0, so the
9999        // comparison silently stops testing anything.
10000        let fresh = Rete::open(&bytes).unwrap();
10001        // `dump_batch` is the resumable form the language clients pull through.
10002        // Driven to exhaustion it must reproduce the eager dump EXACTLY — same
10003        // triples, same order — for any batch size, including one so small that
10004        // every call ends on a subject boundary before reaching it.
10005        let drain = |max_quads: usize| {
10006            let (mut out, mut cursor, mut calls) = (Vec::new(), 0u32, 0);
10007            loop {
10008                let (batch, next, done) = fresh.dump_batch(None, cursor, max_quads);
10009                out.extend(batch);
10010                cursor = next;
10011                calls += 1;
10012                assert!(calls < 10_000, "cursor failed to advance");
10013                if done {
10014                    break out;
10015                }
10016            }
10017        };
10018        let eager = fresh.dump(None);
10019        for max in [1usize, 7, 128, 100_000] {
10020            assert_eq!(drain(max), eager, "batch size {max} changed the dump");
10021        }
10022        // An unknown graph is an empty, already-finished dump — not an error and
10023        // not the default graph leaking through.
10024        let (t, _, done) = fresh.dump_batch(Some("http://ex/nope"), 0, 16);
10025        assert!(t.is_empty() && done);
10026    }
10027
10028    /// `query_batch` is `dump_batch` generalized to a PATTERN — the primitive the
10029    /// Java client's RDF4J cursor pulls through, one bounded batch per wasm call.
10030    /// Three things have to hold or the cursor is a materializing scan wearing a
10031    /// cursor's clothes: driven to exhaustion it must reproduce the eager
10032    /// `query_in_graph` for every pattern shape and every batch size; taking one
10033    /// row must cost far fewer bytes than draining; and every call must either
10034    /// yield a row or say `done`, so the caller can never spin.
10035    #[test]
10036    fn query_batch_matches_query_in_graph_and_stops_early() {
10037        use crate::reader::{CountingReader, SliceReader};
10038
10039        // Enough triples over tiny tiles that a scan spans many tiles, with two
10040        // predicates and a repeated subject so bound patterns have real fanout.
10041        let triples: Vec<(String, String, String)> = (0..2000u32)
10042            .map(|i| {
10043                (
10044                    format!("<http://ex/s/{:04}>", i / 2),
10045                    format!("<http://ex/p/{}>", i % 2),
10046                    format!("<http://ex/o/{i:04}>"),
10047                )
10048            })
10049            .collect();
10050        let mut db = DictionaryBuilder::new();
10051        for (s, p, o) in &triples {
10052            db.observe(s, p, o);
10053        }
10054        let dict = db.build();
10055        let mut ib = GraphIndexBuilder::new().with_tile_budget(256);
10056        for (s, p, o) in &triples {
10057            ib.push(dict.encode(s, p, o).unwrap());
10058        }
10059        let bytes = write_file(&dict, &ib.build(), false, &[], 0);
10060        let rete = Rete::open(&bytes).unwrap();
10061
10062        let drain = |s: Option<&str>, p: Option<&str>, o: Option<&str>, max: usize| {
10063            let (mut out, mut cursor, mut calls) = (Vec::new(), 0u64, 0);
10064            loop {
10065                let (batch, next, done) = rete.query_batch(None, s, p, o, cursor, max);
10066                // The contract the client relies on: rows, or done. Never
10067                // "nothing yet, call again" — that is an unbounded spin.
10068                assert!(!batch.is_empty() || done, "empty non-final batch");
10069                out.extend(batch);
10070                cursor = next;
10071                calls += 1;
10072                assert!(calls < 20_000, "cursor failed to advance");
10073                if done {
10074                    break out;
10075                }
10076            }
10077        };
10078        let shapes: [(Option<&str>, Option<&str>, Option<&str>); 6] = [
10079            (None, None, None),
10080            (Some("<http://ex/s/0007>"), None, None),
10081            (None, Some("<http://ex/p/1>"), None),
10082            (None, None, Some("<http://ex/o/0042>")),
10083            (Some("<http://ex/s/0007>"), Some("<http://ex/p/0>"), None),
10084            (
10085                Some("<http://ex/s/0007>"),
10086                Some("<http://ex/p/0>"),
10087                Some("<http://ex/o/0014>"),
10088            ),
10089        ];
10090        for (s, p, o) in shapes {
10091            let eager = rete.query_in_graph(None, s, p, o);
10092            assert!(
10093                !eager.is_empty(),
10094                "fixture shape {s:?} {p:?} {o:?} matches nothing"
10095            );
10096            // The lazy pull form agrees exactly, order included.
10097            assert_eq!(
10098                rete.query_iter(None, s, p, o).collect::<Vec<_>>(),
10099                eager,
10100                "query_iter disagrees for {s:?} {p:?} {o:?}"
10101            );
10102            for max in [1usize, 3, 64, 100_000] {
10103                let mut got = drain(s, p, o, max);
10104                let mut want = eager.clone();
10105                // A batch is re-sorted canonically per batch, not globally, so
10106                // a bound pattern's batches can arrive in permutation order.
10107                got.sort();
10108                want.sort();
10109                assert_eq!(got, want, "batch size {max} changed {s:?} {p:?} {o:?}");
10110            }
10111            // The fully unbound scan routes to SPO, so batching is order-exact.
10112            if s.is_none() && p.is_none() && o.is_none() {
10113                assert_eq!(drain(s, p, o, 7), eager, "unbound batching reordered rows");
10114            }
10115        }
10116
10117        // A bound term the dictionary has never seen, and an unknown graph, are
10118        // both finished-and-empty rather than errors or a leak of everything.
10119        let (t, _, done) = rete.query_batch(None, Some("<http://ex/nope>"), None, None, 0, 16);
10120        assert!(t.is_empty() && done);
10121        let (t, _, done) = rete.query_batch(Some("http://ex/nope"), None, None, None, 0, 16);
10122        assert!(t.is_empty() && done);
10123        assert_eq!(
10124            rete.query_iter(None, Some("<http://ex/nope>"), None, None)
10125                .count(),
10126            0
10127        );
10128
10129        // Lazily: one row must not drag the whole scan across the reader. This
10130        // is the property that makes `LIMIT 1` bounded on a 48 GiB file.
10131        let read_bytes = |take: usize| -> u64 {
10132            let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
10133            let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10134            let rete = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10135            let before = reader.bytes_read();
10136            let (mut n, mut cursor) = (0usize, 0u64);
10137            loop {
10138                let (batch, next, done) = rete.query_batch(None, None, None, None, cursor, 1);
10139                n += batch.len();
10140                cursor = next;
10141                if done || n >= take {
10142                    break;
10143                }
10144            }
10145            assert!(n >= take.min(triples.len()));
10146            assert!(!rete.index_incomplete());
10147            reader.bytes_read() - before
10148        };
10149        let first_only = read_bytes(1);
10150        let everything = read_bytes(triples.len());
10151        assert!(
10152            first_only < everything,
10153            "query_batch is not lazy: one row read {first_only} B, a full scan {everything} B"
10154        );
10155        #[cfg(feature = "compression")]
10156        assert!(
10157            first_only * 2 < everything,
10158            "query_batch is not lazy: one row read {first_only} B of the {everything} B a full scan reads"
10159        );
10160    }
10161
10162    /// A batched cursor over a QUADS file must stay inside the graph it was
10163    /// given: the default graph and each named graph resume independently.
10164    #[test]
10165    fn query_batch_is_graph_scoped() {
10166        let mut db = DictionaryBuilder::new();
10167        for (s, p, o) in [
10168            ("Alice", "knows", "Bob"),
10169            ("Alice", "knows", "Carol"),
10170            ("Alice", "knows", "Dave"),
10171            ("Erin", "knows", "Frank"),
10172            ("Alice", "knows", "Zoe"),
10173        ] {
10174            db.observe(s, p, o);
10175        }
10176        let dict = db.build();
10177        let mut def = GraphIndexBuilder::new();
10178        def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10179        def.push(dict.encode("Alice", "knows", "Carol").unwrap());
10180        let mut g1 = GraphIndexBuilder::new();
10181        g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10182        g1.push(dict.encode("Erin", "knows", "Frank").unwrap());
10183        let mut g2 = GraphIndexBuilder::new();
10184        g2.push(dict.encode("Alice", "knows", "Zoe").unwrap());
10185        let named = vec![
10186            ("http://ex/g1".to_string(), g1.build()),
10187            ("http://ex/g2".to_string(), g2.build()),
10188        ];
10189        let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10190        let rete = Rete::open(&bytes).unwrap();
10191        for graph in [None, Some("http://ex/g1"), Some("http://ex/g2")] {
10192            let eager = rete.query_in_graph(graph, None, None, None);
10193            let (mut out, mut cursor) = (Vec::new(), 0u64);
10194            loop {
10195                let (batch, next, done) = rete.query_batch(graph, None, None, None, cursor, 1);
10196                out.extend(batch);
10197                cursor = next;
10198                if done {
10199                    break;
10200                }
10201            }
10202            assert_eq!(out, eager, "graph {graph:?} batched differently");
10203        }
10204    }
10205
10206    #[test]
10207    fn query_in_graph_is_graph_scoped() {
10208        // Default graph: Alice knows Bob, Alice knows Carol.
10209        // Named g1: Alice knows Dave (same predicate, different graph).
10210        let mut db = DictionaryBuilder::new();
10211        for (s, p, o) in [
10212            ("Alice", "knows", "Bob"),
10213            ("Alice", "knows", "Carol"),
10214            ("Alice", "knows", "Dave"),
10215        ] {
10216            db.observe(s, p, o);
10217        }
10218        let dict = db.build();
10219
10220        let mut def = GraphIndexBuilder::new();
10221        def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10222        def.push(dict.encode("Alice", "knows", "Carol").unwrap());
10223        let mut g1 = GraphIndexBuilder::new();
10224        g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10225
10226        let named = vec![("http://ex/g1".to_string(), g1.build())];
10227        let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10228        let rete = Rete::open(&bytes).unwrap();
10229
10230        // Default graph only: the two default-graph objects, not Dave.
10231        let mut def_objs: Vec<String> = rete
10232            .query_in_graph(None, Some("Alice"), Some("knows"), None)
10233            .into_iter()
10234            .map(|(_, _, o)| o)
10235            .collect();
10236        def_objs.sort();
10237        assert_eq!(def_objs, vec!["Bob".to_string(), "Carol".to_string()]);
10238
10239        // Named graph only: just Dave.
10240        assert_eq!(
10241            rete.query_in_graph(Some("http://ex/g1"), Some("Alice"), None, None),
10242            vec![("Alice".into(), "knows".into(), "Dave".into())]
10243        );
10244
10245        // A wildcard-everything scan is scoped to its graph.
10246        assert_eq!(rete.query_in_graph(None, None, None, None).len(), 2);
10247        assert_eq!(
10248            rete.query_in_graph(Some("http://ex/g1"), None, None, None)
10249                .len(),
10250            1
10251        );
10252
10253        // An unknown graph IRI is empty, not an error.
10254        assert!(rete
10255            .query_in_graph(Some("http://ex/missing"), None, None, None)
10256            .is_empty());
10257    }
10258
10259    /// A **bounded** `query_in_graph` — and a dump of one named graph — must not
10260    /// fault the whole dictionary on a lazily range-read file.
10261    ///
10262    /// Both used to call `Dictionary::prefetch_all`, so the graph-scoped pattern
10263    /// primitive an RDF4J `Sail` calls once per named graph paid for every term
10264    /// in the file to answer a handful of rows. Measured on `cordis.rete`
10265    /// (801 MB, a 417 MB dictionary) before the fix: a 15-row answer read
10266    /// 416 MB and peaked at 2.5 GB RSS. The fixture below is the same shape in
10267    /// miniature — long literals so the dictionary dominates the file — and the
10268    /// assertion is the invariant that regressed: bytes read must stay well
10269    /// under the dictionary section. Lazy answers are compared against eager
10270    /// throughout: cheaper is only interesting if it is also correct.
10271    #[test]
10272    fn bounded_query_in_graph_does_not_fault_the_whole_dictionary() {
10273        use crate::reader::{CountingReader, SliceReader};
10274
10275        // Long, distinct, poorly-compressible object literals: the dictionary is
10276        // the payload here, exactly as on a graph that stores abstracts or
10277        // embedded media. Repeating one filler string would not do — front
10278        // coding plus zstd shrink 2000 near-identical literals to ~6 KB, which
10279        // is smaller than the index and makes the measurement meaningless.
10280        let payload = |i: u32| -> String {
10281            let mut s = String::with_capacity(800);
10282            let mut x = i as u64 ^ 0x9e37_79b9_7f4a_7c15;
10283            while s.len() < 800 {
10284                x = x
10285                    .wrapping_mul(6364136223846793005)
10286                    .wrapping_add(1442695040888963407);
10287                s.push_str(&format!("{:016x}", x ^ (x >> 29)));
10288            }
10289            s
10290        };
10291        // Half the quads go to the named graph, and the two halves are kept
10292        // lexically apart (`a…` / `b…`) so each lands in its own dictionary
10293        // chunks — the real layout on a quads file whose graphs use different
10294        // IRI namespaces, e.g. `cordis.rete`. Interleaving them instead would
10295        // put a term of both graphs in every chunk, and then no per-graph
10296        // access pattern could avoid faulting the whole dictionary.
10297        let quads: Vec<(String, String, String, bool)> = (0..2000u32)
10298            .map(|i| {
10299                let named = i % 2 == 0;
10300                let ns = if named { 'a' } else { 'b' };
10301                (
10302                    format!("<http://ex/{ns}/s/{i:04}>"),
10303                    "<http://ex/abstract>".to_string(),
10304                    format!("\"{ns}{} number {i}\"", payload(i)),
10305                    named,
10306                )
10307            })
10308            .collect();
10309        let mut db = DictionaryBuilder::new();
10310        for (s, p, o, _) in &quads {
10311            db.observe(s, p, o);
10312        }
10313        let dict = db.build();
10314        let mut def = GraphIndexBuilder::new().with_tile_budget(256);
10315        let mut g1 = GraphIndexBuilder::new().with_tile_budget(256);
10316        for (s, p, o, named) in &quads {
10317            let t = dict.encode(s, p, o).unwrap();
10318            if *named {
10319                g1.push(t);
10320            } else {
10321                def.push(t);
10322            }
10323        }
10324        let named = vec![("http://ex/g1".to_string(), g1.build())];
10325        let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10326
10327        let eager = Rete::open(&bytes).unwrap();
10328        let dict_len = eager.header().dictionary_len;
10329
10330        // `probe` runs one closure against a COLD lazy open and reports the
10331        // bytes it faulted beyond the open itself.
10332        let leaked: &'static [u8] = Box::leak(bytes.clone().into_boxed_slice());
10333        let probe = |f: &dyn Fn(&Rete) -> Vec<TermTriple>| -> (Vec<TermTriple>, u64) {
10334            let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10335            let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10336            let before = reader.bytes_read();
10337            let out = f(&lazy);
10338            assert!(!lazy.index_incomplete(), "a chunk fetch failed");
10339            (out, reader.bytes_read() - before)
10340        };
10341
10342        // 1. One bound subject in the named graph.
10343        let want = eager.query_in_graph(
10344            Some("http://ex/g1"),
10345            Some("<http://ex/a/s/0100>"),
10346            None,
10347            None,
10348        );
10349        assert_eq!(want.len(), 1, "fixture: expected exactly one match");
10350        let (got, pulled) = probe(&|r| {
10351            r.query_in_graph(
10352                Some("http://ex/g1"),
10353                Some("<http://ex/a/s/0100>"),
10354                None,
10355                None,
10356            )
10357        });
10358        assert_eq!(got, want, "lazy query_in_graph disagrees with eager");
10359        assert!(
10360            pulled * 4 < dict_len,
10361            "a 1-row query_in_graph faulted {pulled} B of a {dict_len} B dictionary — \
10362             it is prefetching the whole thing again"
10363        );
10364
10365        // 2. A graph with NOTHING to resolve must not fault the dictionary.
10366        //    Both shapes below used to pull every chunk before returning an
10367        //    empty answer, and `rete export` hits the second one first on a
10368        //    quads file whose default graph is empty — on `cordis.rete` that
10369        //    was 415 MB read and 2.5 GB of peak RSS to emit zero triples. What
10370        //    is left is the named-graph directory walk it takes to learn the
10371        //    graph is absent at all; that is index, not dictionary.
10372        let (empty, q_pulled) =
10373            probe(&|r| r.query_in_graph(Some("http://ex/nope"), None, None, None));
10374        assert!(empty.is_empty());
10375        let (_, d_pulled) = probe(&|r| {
10376            let mut n = 0usize;
10377            r.dump_each(Some("http://ex/nope"), |_, _, _| n += 1);
10378            assert_eq!(n, 0, "an absent graph yielded triples");
10379            Vec::new()
10380        });
10381        assert_eq!(
10382            q_pulled, d_pulled,
10383            "query_in_graph and dump_each disagree on what an absent graph costs"
10384        );
10385        assert!(
10386            q_pulled * 4 < dict_len,
10387            "an absent graph faulted {q_pulled} B of a {dict_len} B dictionary"
10388        );
10389
10390        // 3. Dumping ONE named graph costs that graph, not the whole file: the
10391        //    default graph's half of the dictionary must stay unfaulted.
10392        let want: Vec<TermTriple> = eager.dump(Some("http://ex/g1"));
10393        assert_eq!(want.len(), 1000);
10394        let (got, pulled) = probe(&|r| {
10395            let mut out = Vec::new();
10396            r.dump_each(Some("http://ex/g1"), |s, p, o| {
10397                out.push((s.to_string(), p.to_string(), o.to_string()))
10398            });
10399            out
10400        });
10401        assert_eq!(got, want, "lazy dump_each disagrees with eager dump");
10402        assert!(
10403            pulled * 3 < dict_len * 2,
10404            "dumping half the graph faulted {pulled} B of a {dict_len} B dictionary — \
10405             the other graph's chunks are being pulled in too"
10406        );
10407    }
10408
10409    /// Build a multi-chunk single-graph ranged file image for the eviction
10410    /// tests: 4000 distinct ~70-byte object literals overflow the 64 KiB
10411    /// dictionary chunk budget into several chunks, and one multi-KiB literal
10412    /// forces a large run/chunk boundary — the shape a tiny budget must survive.
10413    #[cfg(test)]
10414    fn build_multichunk_image() -> Vec<u8> {
10415        use crate::index::PermSet;
10416        let mut db = DictionaryBuilder::new();
10417        let mut triples: Vec<(String, String, String)> = (0..4000u32)
10418            .map(|i| {
10419                (
10420                    format!("<http://ex/s/{i:07}>"),
10421                    format!("<http://ex/p/{}>", i % 8),
10422                    format!("\"object literal number {i:07} with padding text to widen the term\""),
10423                )
10424            })
10425            .collect();
10426        triples.push((
10427            "<http://ex/s/big>".to_string(),
10428            "<http://ex/p/big>".to_string(),
10429            format!("\"{}\"", "Z".repeat(8000)),
10430        ));
10431        for (s, p, o) in &triples {
10432            db.observe(s, p, o);
10433        }
10434        let dict = db.build();
10435        let mut idx = GraphIndexBuilder::new()
10436            .with_tile_budget(64)
10437            .with_perms(PermSet::ALL);
10438        for (s, p, o) in &triples {
10439            idx.push(dict.encode(s, p, o).unwrap());
10440        }
10441        write_dataset(&dict, &idx.build(), &[], true, &[], 0)
10442    }
10443
10444    /// **Bounded-export phase 2: eviction is byte-transparent, and progress is
10445    /// guaranteed at any cap.** A cap far below the decompressed dictionary
10446    /// forces the chunk cache to evict during a full dump, yet the dump is
10447    /// byte-for-byte the unlimited-cap dump — the `Arc` handout keeps every
10448    /// chunk alive across its one decode, so eviction never drops bytes in use.
10449    /// A 1-byte cap (smaller than any single chunk, so every insert evicts
10450    /// itself immediately) still completes with identical output: it is the
10451    /// handout invariant, not residency, that makes a window make progress.
10452    #[test]
10453    fn tiny_dict_cache_evicts_yet_dump_is_byte_identical() {
10454        use crate::reader::SliceReader;
10455        let leaked: &'static [u8] = Box::leak(build_multichunk_image().into_boxed_slice());
10456        let dump = |cap: u64| -> (
10457            Vec<(String, String, String)>,
10458            crate::chunk_cache::CacheStats,
10459        ) {
10460            let rete =
10461                Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10462            rete.dict.set_cache_cap(cap);
10463            let mut out = Vec::new();
10464            rete.dump_filtered_each(None, None, None, None, |s, p, o| {
10465                out.push((s.to_string(), p.to_string(), o.to_string()))
10466            });
10467            assert!(
10468                !rete.index_incomplete(),
10469                "no lazy fault should fail at cap {cap}"
10470            );
10471            let stats = rete.dict_cache_stats();
10472            (out, stats)
10473        };
10474
10475        let (unlimited, us) = dump(u64::MAX);
10476        assert_eq!(unlimited.len(), 4001, "every triple dumped");
10477        assert_eq!(us.evictions, 0, "unlimited cap evicts nothing");
10478
10479        // A cap far below the decompressed dictionary: eviction happens, output
10480        // unchanged.
10481        let (tiny, ts) = dump(128 * 1024);
10482        assert_eq!(tiny, unlimited, "128 KiB-cap dump differs from unlimited");
10483        assert!(ts.evictions > 0, "a tiny cap must evict (stats {ts:?})");
10484
10485        // Progress guarantee: a 1-byte cap still completes, byte-identical.
10486        let (starved, ss) = dump(1);
10487        assert_eq!(starved, unlimited, "1-byte-cap dump differs from unlimited");
10488        assert!(
10489            ss.evictions > 0,
10490            "1-byte cap evicts every insert (stats {ss:?})"
10491        );
10492    }
10493
10494    /// **The budget pays for the open's fixed directories.** A ranged open holds
10495    /// one descriptor per dictionary chunk and per index tile for its whole
10496    /// life, outside every cache — 88 MB on the 14.75 GiB openaire-relation,
10497    /// 224 MB on epfl-infoscience. `set_memory_budget` used to give the caches
10498    /// the whole budget on top of that; it now measures the directories
10499    /// ([`Rete::fixed_overhead`]) and takes them out of the dictionary share, so
10500    /// caps + directories stay within the total. This checks the arithmetic on
10501    /// a many-chunk file, the 64 MiB floor, the unlimited pass-through, and that
10502    /// a dump under the capped budget keeps the dictionary cache within its
10503    /// (net) share.
10504    #[test]
10505    fn memory_budget_charges_the_fixed_directories() {
10506        use crate::reader::SliceReader;
10507        let leaked: &'static [u8] = Box::leak(build_multichunk_image().into_boxed_slice());
10508        let rete =
10509            Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10510
10511        let fixed = rete.fixed_overhead();
10512        // Every chunk and every tile has a descriptor; a multi-chunk dictionary
10513        // and a 64-triple tile budget make both directories non-trivial.
10514        assert!(
10515            fixed.dict_directory > 0 && fixed.index_directory > 0,
10516            "{fixed:?}"
10517        );
10518        assert_eq!(fixed.named_graphs, 0, "no named graphs in this file");
10519        assert_eq!(
10520            fixed.total(),
10521            fixed.dict_directory + fixed.index_directory + fixed.named_graphs
10522        );
10523
10524        // A budget well above the floor: the dictionary share is the plain
10525        // split less the directories, exactly.
10526        let total: u64 = 512 << 20;
10527        let applied = rete.set_memory_budget(Some(total));
10528        let plain = split_memory_budget(total);
10529        assert_eq!(applied.dict_cache, plain.dict_cache - fixed.total());
10530        assert_eq!(rete.dict_cache_cap(), applied.dict_cache);
10531        assert_eq!(applied.tile_cache, plain.tile_cache);
10532        assert_eq!(applied.block_cache, plain.block_cache);
10533        assert!(
10534            applied.dict_cache + applied.tile_cache + applied.block_cache + fixed.total() <= total,
10535            "caps + fixed directories exceed the budget: {applied:?} + {fixed:?} > {total}"
10536        );
10537
10538        // A budget whose share would fall under the floor keeps the floor.
10539        let applied = rete.set_memory_budget(Some(1 << 20));
10540        assert_eq!(applied.dict_cache, MIN_DICT_CACHE_CAP);
10541
10542        // Unlimited stays unlimited: nothing is subtracted from u64::MAX.
10543        let applied = rete.set_memory_budget(None);
10544        assert_eq!(applied.dict_cache, u64::MAX);
10545        assert_eq!(rete.dict_cache_cap(), u64::MAX);
10546
10547        // Under a cap smaller than the decompressed dictionary, a full dump
10548        // keeps the cache within it.
10549        let rete =
10550            Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10551        rete.dict.set_cache_cap(128 * 1024);
10552        let mut rows = 0usize;
10553        rete.dump_filtered_each(None, None, None, None, |_, _, _| rows += 1);
10554        assert_eq!(rows, 4001);
10555        let s = rete.dict_cache_stats();
10556        assert!(s.evictions > 0, "the cap must bind: {s:?}");
10557        // One oversized chunk may sit alone above the cap (the handout
10558        // invariant); otherwise residency is within it.
10559        assert!(
10560            s.entries <= 1 || s.resident_bytes <= 128 * 1024,
10561            "dictionary cache over its cap after the dump: {s:?}"
10562        );
10563    }
10564
10565    /// **Bounded-export phase 2: per-graph index release.** After a named
10566    /// graph's slot is exported, [`Rete::release_named_graph`] drops its decoded
10567    /// index (and its tiles) so a many-graph dump holds one graph's index at a
10568    /// time, not all of them. The graph re-opens lazily if touched again.
10569    #[test]
10570    fn releasing_a_named_graph_drops_and_reopens_its_index() {
10571        use crate::index::PermSet;
10572        use crate::reader::SliceReader;
10573        let mut db = DictionaryBuilder::new();
10574        let all: Vec<(String, String, String)> = (0..40u32)
10575            .map(|i| {
10576                (
10577                    format!("<http://ex/s{}>", i % 7),
10578                    format!("<http://ex/p{}>", i % 3),
10579                    format!("<http://ex/o{}>", i % 5),
10580                )
10581            })
10582            .collect();
10583        for (s, p, o) in &all {
10584            db.observe(s, p, o);
10585        }
10586        let dict = db.build();
10587        let mk = |take: fn(u32) -> bool| {
10588            let mut g = GraphIndexBuilder::new()
10589                .with_tile_budget(16)
10590                .with_perms(PermSet::ALL);
10591            for (i, (s, p, o)) in all.iter().enumerate() {
10592                if take(i as u32) {
10593                    g.push(dict.encode(s, p, o).unwrap());
10594                }
10595            }
10596            g.build()
10597        };
10598        let def = mk(|_| true);
10599        let g1 = mk(|i| i % 2 == 0);
10600        let g2 = mk(|i| i % 3 == 0);
10601        let image = write_dataset(
10602            &dict,
10603            &def,
10604            &[
10605                ("<http://ex/g1>".to_string(), g1),
10606                ("<http://ex/g2>".to_string(), g2),
10607            ],
10608            true,
10609            &[],
10610            0,
10611        );
10612        let leaked: &'static [u8] = Box::leak(image.into_boxed_slice());
10613        let mut rete =
10614            Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64).unwrap();
10615
10616        let opened = |r: &Rete| -> usize {
10617            match &r.named_graphs {
10618                NamedGraphsSlot::Lazy(l) => {
10619                    let mut n = 0;
10620                    l.for_each_opened(|_| n += 1);
10621                    n
10622                }
10623                _ => 0,
10624            }
10625        };
10626
10627        assert_eq!(opened(&rete), 0, "nothing opened before first touch");
10628        // Touching g1 opens its index.
10629        assert!(rete.graph_index("<http://ex/g1>").is_some());
10630        assert_eq!(opened(&rete), 1, "g1 opened");
10631        // Release it: the index is dropped.
10632        rete.release_named_graph("<http://ex/g1>");
10633        assert_eq!(opened(&rete), 0, "g1 released");
10634        // Releasing an unknown / already-released graph is a no-op.
10635        rete.release_named_graph("<http://ex/absent>");
10636        rete.release_named_graph("<http://ex/g1>");
10637        assert_eq!(opened(&rete), 0);
10638        // The graph re-opens lazily on the next access.
10639        assert!(rete.graph_index("<http://ex/g1>").is_some());
10640        assert_eq!(opened(&rete), 1, "g1 re-opened after release");
10641    }
10642
10643    /// **The filtered-dump correctness invariant** (#117 point 1): for every
10644    /// graph and every one of the eight bound/unbound shapes,
10645    /// `dump_filtered_each` must return *exactly* the quads `dump_each` returns,
10646    /// filtered — no more, and above all no fewer.
10647    ///
10648    /// "No fewer" is the whole risk. The speedup comes from *not fetching* tiles
10649    /// a synopsis says cannot match, and a pruning bug does not crash, it
10650    /// silently returns a shorter answer. So the oracle here is the unfiltered
10651    /// dump with a plain Rust `retain` over it — the thing the filter is meant
10652    /// to be a cheaper spelling of — and it is checked on every open path, since
10653    /// the eager path prunes with a different code path than the lazy one.
10654    ///
10655    /// It runs on a `PermSet::CORE` (`--permutations 3`) file as well as a
10656    /// six-permutation one. Every shape routes inside CORE — `perm_routing_never_leaves_core`
10657    /// proves the routing table does — so the two must agree row for row, and if
10658    /// a future shape ever routed outside CORE this test fails on the three-perm
10659    /// build rather than silently returning nothing from an absent section.
10660    #[test]
10661    fn filtered_dump_is_exactly_the_unfiltered_dump_filtered() {
10662        use crate::index::PermSet;
10663        use crate::reader::SliceReader;
10664
10665        let terms = |i: u32| {
10666            (
10667                format!("<http://ex/s{}>", i % 11),
10668                format!("<http://ex/p{}>", i % 3),
10669                // Mix IRIs and literals in the object column: a literal lives in
10670                // a different dictionary section than a node, and the object
10671                // filter has to resolve through the right one.
10672                if i.is_multiple_of(4) {
10673                    format!("\"lit {}\"@en", i % 5)
10674                } else {
10675                    format!("<http://ex/o{}>", i % 7)
10676                },
10677            )
10678        };
10679        let build = |perms: PermSet| {
10680            let mut db = DictionaryBuilder::new();
10681            let all: Vec<_> = (0..90u32).map(terms).collect();
10682            for (s, p, o) in &all {
10683                db.observe(s, p, o);
10684            }
10685            let dict = db.build();
10686            let mut def = GraphIndexBuilder::new()
10687                .with_tile_budget(16)
10688                .with_perms(perms);
10689            let mut g1 = GraphIndexBuilder::new()
10690                .with_tile_budget(16)
10691                .with_perms(perms);
10692            for (i, (s, p, o)) in all.iter().enumerate() {
10693                let t = dict.encode(s, p, o).unwrap();
10694                // Deliberately overlapping, not disjoint: every second triple is
10695                // in both graphs, so a filter that leaked across graphs would
10696                // still look plausible on counts alone.
10697                if i % 2 == 0 {
10698                    g1.push(t);
10699                }
10700                def.push(t);
10701            }
10702            write_dataset(
10703                &dict,
10704                &def.build(),
10705                &[("<http://ex/g1>".to_string(), g1.build())],
10706                true,
10707                &[],
10708                0,
10709            )
10710        };
10711
10712        let shapes: Vec<(Option<&str>, Option<&str>, Option<&str>)> = vec![
10713            (None, None, None),
10714            (Some("<http://ex/s3>"), None, None),
10715            (None, Some("<http://ex/p1>"), None),
10716            (None, None, Some("<http://ex/o5>")),
10717            (None, None, Some("\"lit 0\"@en")),
10718            (Some("<http://ex/s3>"), Some("<http://ex/p0>"), None),
10719            (Some("<http://ex/s4>"), None, Some("<http://ex/o4>")),
10720            (None, Some("<http://ex/p1>"), Some("<http://ex/o5>")),
10721            (
10722                Some("<http://ex/s3>"),
10723                Some("<http://ex/p0>"),
10724                Some("<http://ex/o3>"),
10725            ),
10726            // Terms the dictionary has never seen, in each position: these must
10727            // return nothing, not everything (an unresolvable bound term must
10728            // not silently degrade to "unbound").
10729            (Some("<http://ex/nope>"), None, None),
10730            (None, Some("<http://ex/nope>"), None),
10731            (None, None, Some("<http://ex/nope>")),
10732        ];
10733
10734        let mut any_pruned = false;
10735        for perms in [PermSet::ALL, PermSet::CORE] {
10736            let image = build(perms);
10737            let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
10738            let opens: Vec<(&str, Rete)> = vec![
10739                ("resident", Rete::open(&image).unwrap()),
10740                (
10741                    "ranged",
10742                    Rete::open_ranged(&SliceReader::new(leaked)).unwrap(),
10743                ),
10744                (
10745                    "lazy",
10746                    Rete::open_ranged_lazy_with(SliceReader::new(leaked), HEADER_LEN as u64)
10747                        .unwrap(),
10748                ),
10749            ];
10750            for graph in [None, Some("<http://ex/g1>"), Some("<http://ex/absent>")] {
10751                // The oracle: the unfiltered dump, filtered in Rust.
10752                let full: Vec<TermTriple> = {
10753                    let mut v = Vec::new();
10754                    Rete::open(&image).unwrap().dump_each(graph, |s, p, o| {
10755                        v.push((s.to_string(), p.to_string(), o.to_string()))
10756                    });
10757                    v.sort();
10758                    v
10759                };
10760                for &(s, p, o) in &shapes {
10761                    let mut want = full.clone();
10762                    want.retain(|(ts, tp, to)| {
10763                        s.is_none_or(|x| x == ts)
10764                            && p.is_none_or(|x| x == tp)
10765                            && o.is_none_or(|x| x == to)
10766                    });
10767                    for (path, rete) in &opens {
10768                        let mut got = Vec::new();
10769                        rete.dump_filtered_each(graph, s, p, o, |s, p, o| {
10770                            got.push((s.to_string(), p.to_string(), o.to_string()))
10771                        });
10772                        got.sort();
10773                        assert_eq!(
10774                            got,
10775                            want,
10776                            "{path}/{perms:?} graph={graph:?} pattern={:?} — filtered dump \
10777                             is not the unfiltered dump filtered",
10778                            (s, p, o)
10779                        );
10780                        assert!(!rete.index_incomplete(), "{path}: a fetch failed");
10781                    }
10782                    // …and the plan must agree with what the dump actually did:
10783                    // an empty answer means either no scan at all, or a scan
10784                    // whose admitted tiles simply held nothing.
10785                    let plan = opens[2].1.dump_plan(graph, s, p, o);
10786                    if graph == Some("<http://ex/absent>")
10787                        || [s, p, o].contains(&Some("<http://ex/nope>"))
10788                    {
10789                        assert!(
10790                            plan.scan.is_none(),
10791                            "an unmatchable dump still planned a scan: {:?}",
10792                            (graph, s, p, o)
10793                        );
10794                    } else {
10795                        let scan = plan.scan.expect("a matchable dump plans a scan");
10796                        assert!(
10797                            PermSet::CORE.contains(scan.permutation),
10798                            "a dump routed to {:?}, outside PermSet::CORE — a \
10799                             --permutations 3 file has no such section",
10800                            scan.permutation
10801                        );
10802                        assert!(scan.tiles_admitted <= scan.tiles_routed);
10803                        assert!(scan.tiles_routed <= scan.tiles_total);
10804                        assert!(scan.tile_bytes <= scan.section_bytes);
10805                        if scan.tiles_admitted < scan.tiles_total {
10806                            any_pruned = true;
10807                        }
10808                    }
10809                }
10810            }
10811        }
10812        assert!(
10813            any_pruned,
10814            "no shape pruned a single tile — the fixture is too small to be \
10815             measuring anything"
10816        );
10817    }
10818
10819    /// The point of the filter: a predicate-scoped dump of a lazily range-read
10820    /// file must **fetch less** than the same graph's full dump, not merely
10821    /// return less.
10822    ///
10823    /// The fixture is the shape the win shows on — many tiles, and a predicate
10824    /// whose triples live in a minority of them — so the assertion can be about
10825    /// bytes rather than about the code path being taken on faith. Measured on
10826    /// the real thing (`cordis.rete`, 801 MB): one predicate of one named graph
10827    /// read 16.0 MB where the graph's full dump read 375.8 MB.
10828    #[test]
10829    fn a_predicate_scoped_dump_fetches_less_than_the_graph() {
10830        use crate::reader::{CountingReader, SliceReader};
10831
10832        // 200 subjects × 20 predicates, tiled small so there are many tiles, and
10833        // one distinct object per cell so that a predicate slice needs a
10834        // twentieth of the object dictionary rather than all of it — otherwise
10835        // the dictionary, which this change does not prune, swamps the index
10836        // saving the test is about.
10837        let mut db = DictionaryBuilder::new();
10838        let mut all = Vec::new();
10839        for s in 0..200u32 {
10840            for p in 0..20u32 {
10841                let t = (
10842                    format!("<http://ex/s/{s:04}>"),
10843                    format!("<http://ex/p/{p:02}>"),
10844                    format!("\"object {s:04}/{p:02}\""),
10845                );
10846                db.observe(&t.0, &t.1, &t.2);
10847                all.push(t);
10848            }
10849        }
10850        let dict = db.build();
10851        let mut ib = GraphIndexBuilder::new().with_tile_budget(64);
10852        for (s, p, o) in &all {
10853            ib.push(dict.encode(s, p, o).unwrap());
10854        }
10855        let image = write_dataset(&dict, &ib.build(), &[], false, &[], 0);
10856        let leaked: &'static [u8] = Box::leak(image.clone().into_boxed_slice());
10857
10858        // Bytes faulted past the open, for one dump.
10859        let probe = |s: Option<&str>, p: Option<&str>, o: Option<&str>| -> (usize, u64) {
10860            let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10861            let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10862            let before = reader.bytes_read();
10863            let mut n = 0usize;
10864            lazy.dump_filtered_each(None, s, p, o, |_, _, _| n += 1);
10865            assert!(!lazy.index_incomplete());
10866            (n, reader.bytes_read() - before)
10867        };
10868
10869        let (full_rows, full_bytes) = probe(None, None, None);
10870        let (pred_rows, pred_bytes) = probe(None, Some("<http://ex/p/07>"), None);
10871        let (subj_rows, subj_bytes) = probe(Some("<http://ex/s/0021>"), None, None);
10872        assert_eq!(full_rows, 4000);
10873        assert_eq!(pred_rows, 200, "one predicate covers every subject once");
10874        assert_eq!(subj_rows, 20);
10875        // True in any build: a slice fetches strictly less than the graph.
10876        assert!(
10877            pred_bytes < full_bytes && subj_bytes < full_bytes,
10878            "a slice fetched {pred_bytes} / {subj_bytes} B where the whole graph fetched \
10879             {full_bytes} B — the scan is not being pruned at all"
10880        );
10881        // The SIZE of the win is a property of the FILE, not of this change: it is
10882        // the index share of what a dump reads, and the dictionary — which no
10883        // filter prunes — is the rest. With the default `compression` feature the
10884        // dictionary chunk bodies are zstd-compressed and the slice costs about a
10885        // tenth of the graph (measured here: 1,565 B for the predicate and 864 B
10886        // for the subject, against 15,714 B). Built `--no-default-features` those
10887        // same chunks are stored RAW, so resolving 200 rows' terms dominates and
10888        // the identical, identically-pruned scan costs 22,225 B of 40,165 — 55%.
10889        // Asserting the strong ratio unconditionally would be asserting that zstd
10890        // is enabled; the index-side prune is checked below and holds either way.
10891        #[cfg(feature = "compression")]
10892        assert!(
10893            pred_bytes * 4 < full_bytes && subj_bytes * 4 < full_bytes,
10894            "a 1-in-20 predicate slice fetched {pred_bytes} B and one subject \
10895             {subj_bytes} B where the whole graph fetched {full_bytes} B"
10896        );
10897
10898        // And the plan predicted the index side of it without fetching a tile.
10899        let reader = std::sync::Arc::new(CountingReader::new(SliceReader::new(leaked)));
10900        let lazy = Rete::open_ranged_lazy_with(reader.clone(), HEADER_LEN as u64).unwrap();
10901        let before = reader.bytes_read();
10902        let plan = lazy
10903            .dump_plan(None, None, Some("<http://ex/p/07>"), None)
10904            .scan
10905            .expect("a known predicate plans a scan");
10906        assert!(
10907            reader.bytes_read() - before < 64 * 1024,
10908            "planning fetched {} B — it is supposed to read directories, not tiles",
10909            reader.bytes_read() - before
10910        );
10911        assert!(plan.tiles_admitted < plan.tiles_total);
10912        // The index side on its own, with the dictionary's share taken out: one
10913        // predicate of twenty routes to a handful of POS tiles. (Measured here:
10914        // the whole dump faults 15,714 B, the predicate slice 1,565 B and one
10915        // subject 864 B — a 10x and an 18x, most of what is left being the
10916        // dictionary this change does not prune.)
10917        assert!(
10918            plan.tile_bytes * 5 < plan.section_bytes,
10919            "one predicate of twenty planned {} B of a {} B section",
10920            plan.tile_bytes,
10921            plan.section_bytes
10922        );
10923    }
10924
10925    #[test]
10926    fn query_quads_tags_every_graph() {
10927        let mut db = DictionaryBuilder::new();
10928        for (s, p, o) in [("Alice", "knows", "Bob"), ("Alice", "knows", "Dave")] {
10929            db.observe(s, p, o);
10930        }
10931        let dict = db.build();
10932        let mut def = GraphIndexBuilder::new();
10933        def.push(dict.encode("Alice", "knows", "Bob").unwrap());
10934        let mut g1 = GraphIndexBuilder::new();
10935        g1.push(dict.encode("Alice", "knows", "Dave").unwrap());
10936        let named = vec![("http://ex/g1".to_string(), g1.build())];
10937        let bytes = write_dataset(&dict, &def.build(), &named, true, &[], 0);
10938        let rete = Rete::open(&bytes).unwrap();
10939
10940        // `Alice knows ?` spans both graphs; each match carries its graph tag.
10941        let quads = rete.query_quads(Some("Alice"), Some("knows"), None);
10942        assert_eq!(quads.len(), 2);
10943        assert_eq!(
10944            quads[0],
10945            (("Alice".into(), "knows".into(), "Bob".into()), None)
10946        );
10947        assert_eq!(
10948            quads[1],
10949            (
10950                ("Alice".into(), "knows".into(), "Dave".into()),
10951                Some("http://ex/g1".to_string())
10952            )
10953        );
10954
10955        // A bound term absent from the dictionary yields nothing, in any graph.
10956        assert!(rete.query_quads(Some("Nobody"), None, None).is_empty());
10957    }
10958
10959    #[test]
10960    fn pyramid_meta_round_trips_in_file() {
10961        let rete = Rete::open(&build_image()).unwrap();
10962        let pyr = rete.pyramid().expect("file has a pyramid");
10963        // Summary covers all 3 triples by count; tiles are not stored in v0.
10964        let total: u32 = pyr.summary.iter().map(|e| e.count).sum();
10965        assert_eq!(total, 3);
10966        assert!(!pyr.summary.is_empty());
10967        assert!(pyr.tiles.is_empty());
10968    }
10969
10970    #[test]
10971    fn schema_pyramid_round_trips_through_file_index_free() {
10972        use crate::reader::{CountingReader, SliceReader};
10973        let sub = "<http://www.w3.org/2000/01/rdf-schema#subClassOf>";
10974        let q = |s: &str, p: &str, o: &str| {
10975            (s.to_string(), p.to_string(), o.to_string(), None::<String>)
10976        };
10977        // Astronomer ⊑ Scientist ⊑ Person ⊑ Agent, instances at the leaves.
10978        let quads = vec![
10979            q("<a>", RDF_TYPE, "<Astronomer>"),
10980            q("<b>", RDF_TYPE, "<Astronomer>"),
10981            q("<c>", RDF_TYPE, "<Person>"),
10982            q("<Astronomer>", sub, "<Scientist>"),
10983            q("<Scientist>", sub, "<Person>"),
10984            q("<Person>", sub, "<Agent>"),
10985            q("<a>", "<knows>", "<b>"),
10986            q("<b>", "<knows>", "<c>"),
10987        ];
10988        let (bytes, _) =
10989            crate::ingest::assemble_dataset_with_opts(quads, true, false, None, |_, _| Vec::new());
10990
10991        // The v2 schema pyramid round-trips through the built file.
10992        let rete = Rete::open(&bytes).unwrap();
10993        let pyr = rete.pyramid().expect("pyramid present");
10994        assert!(!pyr.level_rollups.is_empty(), "schema pyramid shipped");
10995        assert!(pyr
10996            .class_hierarchy
10997            .iter()
10998            .any(|n| n.class == "<Agent>" && n.depth == 0));
10999
11000        // It reads index-free: a SummaryView open never touches the index section.
11001        let r = CountingReader::new(SliceReader::new(&bytes));
11002        let view = SummaryView::open_ranged(&r).unwrap().unwrap();
11003        assert!(view.level_count() >= 2, "multi-level pyramid");
11004        let coarse = view.level_rollup(0).unwrap();
11005        assert!(
11006            coarse.classes.iter().any(|(c, _)| c == "<Agent>"),
11007            "coarsest level rolls up to the root Agent"
11008        );
11009        let h = Header::from_bytes(&bytes).unwrap();
11010        assert!(
11011            r.bytes_read() <= bytes.len() as u64 - h.root_dir_len,
11012            "summary read {} bytes; the {}-byte index section must be skipped",
11013            r.bytes_read(),
11014            h.root_dir_len
11015        );
11016    }
11017
11018    #[test]
11019    fn predicate_totals_from_summary_only() {
11020        use crate::reader::SliceReader;
11021        // build_image: 2 `knows` triples + 1 `age` triple.
11022        let bytes = build_image();
11023        let reader = SliceReader::new(&bytes);
11024        let view = SummaryView::open_ranged(&reader).unwrap().unwrap();
11025        assert_eq!(view.predicate_total("knows"), 2);
11026        assert_eq!(view.predicate_total("age"), 1);
11027        assert_eq!(view.predicate_total("missing"), 0);
11028        let totals = view.predicate_totals();
11029        assert_eq!(totals[0], ("knows".to_string(), 2)); // sorted by count desc
11030    }
11031
11032    #[test]
11033    fn query_patterns_resolve_to_terms() {
11034        let rete = Rete::open(&build_image()).unwrap();
11035
11036        // All triples.
11037        assert_eq!(rete.query(None, None, None).len(), 3);
11038
11039        // Subject bound.
11040        let mut alice = rete.query(Some("Alice"), None, None);
11041        alice.sort();
11042        assert_eq!(
11043            alice,
11044            vec![
11045                ("Alice".into(), "age".into(), "30".into()),
11046                ("Alice".into(), "knows".into(), "Bob".into()),
11047            ]
11048        );
11049
11050        // Predicate bound.
11051        assert_eq!(rete.query(None, Some("knows"), None).len(), 2);
11052
11053        // Full triple, present and absent.
11054        assert_eq!(
11055            rete.query(Some("Bob"), Some("knows"), Some("Carol")),
11056            vec![("Bob".into(), "knows".into(), "Carol".into())]
11057        );
11058        assert!(rete.query(Some("Nobody"), None, None).is_empty());
11059        assert!(rete.query(None, Some("likes"), None).is_empty());
11060    }
11061
11062    #[test]
11063    fn query_provenance_reports_terms_ids_sections_and_index_choice() {
11064        let bytes = build_image();
11065        let rete = Rete::open(&bytes).unwrap();
11066
11067        let mut matches = rete.query_with_provenance(None, Some("knows"), None);
11068        matches.sort_by(|a, b| a.terms.cmp(&b.terms));
11069
11070        assert_eq!(matches.len(), 2);
11071        assert_eq!(
11072            matches[0].terms,
11073            ("Alice".into(), "knows".into(), "Bob".into())
11074        );
11075        assert_eq!(
11076            matches[0].ids,
11077            rete.dictionary().encode("Alice", "knows", "Bob").unwrap()
11078        );
11079        assert_eq!(matches[0].graph.as_deref(), None);
11080        assert_eq!(
11081            matches[0].matched_pattern,
11082            (None, Some(matches[0].ids.1), None)
11083        );
11084        assert_eq!(
11085            matches[0].index_permutation,
11086            crate::index::IndexPermutation::Pos
11087        );
11088
11089        let h = rete.header();
11090        assert_eq!(matches[0].dictionary_range.offset, h.dictionary_offset);
11091        assert_eq!(matches[0].dictionary_range.len, h.dictionary_len);
11092        assert_eq!(matches[0].index_range.offset, h.root_dir_offset);
11093        assert_eq!(matches[0].index_range.len, h.root_dir_len);
11094        assert!(
11095            matches[0].index_section_range.offset > h.root_dir_offset,
11096            "POS is section 1, so its payload starts after the container header and SPO payload"
11097        );
11098        assert!(matches[0].index_section_range.len > 0);
11099        assert!(matches[0].index_section_range.end() <= matches[0].index_range.end());
11100        assert!(matches[0].index_section_range.len < matches[0].index_range.len);
11101        assert_eq!(
11102            matches[0].pyramid_range.as_ref().map(|r| (r.offset, r.len)),
11103            Some((h.pyramid_meta_offset, h.pyramid_meta_len))
11104        );
11105        // Tiled (v0.2) files report the physical tile holding the match; its
11106        // compressed byte range nests inside the selected section payload.
11107        let tile_range = matches[0].tile_range.expect("tiled file reports a tile");
11108        assert!(matches[0]
11109            .tile
11110            .as_deref()
11111            .unwrap()
11112            .starts_with(matches[0].index_permutation.name()));
11113        assert!(matches[0].index_section_range.offset <= tile_range.offset);
11114        assert!(tile_range.end() <= matches[0].index_section_range.end());
11115    }
11116
11117    /// Build an in-memory `.rete` with `n` labeled subjects: each carries an
11118    /// `rdfs:label` literal drawn from `WORDS` (a word prefix selects ~1/|WORDS|
11119    /// of them) plus one extra edge so the subject has a degree to rank by.
11120    fn build_labeled(n: usize) -> Vec<u8> {
11121        const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
11122        const WORDS: &[&str] = &[
11123            "alanine",
11124            "benzene",
11125            "glucose",
11126            "dextrose",
11127            "ethanol",
11128            "formate",
11129            "heptane",
11130            "isoleucine",
11131        ];
11132        let triples: Vec<(String, String, String)> = (0..n)
11133            .flat_map(|i| {
11134                let s = format!("<http://ex/e{i}>");
11135                let w = WORDS[i % WORDS.len()];
11136                [
11137                    (s.clone(), LABEL.to_string(), format!("\"{w}-{i:06}\"")),
11138                    (
11139                        s,
11140                        "<http://ex/p>".to_string(),
11141                        format!("<http://ex/c{}>", i % 64),
11142                    ),
11143                ]
11144            })
11145            .collect();
11146        let mut db = DictionaryBuilder::new();
11147        for (s, p, o) in &triples {
11148            db.observe(s, p, o);
11149        }
11150        let dict = db.build();
11151        let ids: Vec<(u32, u32, u32)> = triples
11152            .iter()
11153            .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11154            .collect();
11155        let mut ib = GraphIndexBuilder::new();
11156        for &t in &ids {
11157            ib.push(t);
11158        }
11159        let (meta, levels) = build_pyramid_meta(&dict, &ids, DEFAULT_TILE_BUDGET);
11160        write_file(&dict, &ib.build(), false, &meta, levels)
11161    }
11162
11163    #[test]
11164    fn prefix_search_matches_a_filter_scan() {
11165        // 800 < the 8192 label-index cap, so the index is COMPLETE — every label
11166        // is present and the two paths must return the exact same subject set.
11167        let bytes = build_labeled(800);
11168        let rete = Rete::open(&bytes).unwrap();
11169        let idx_subjects: std::collections::BTreeSet<String> = rete
11170            .prefix_search("glucose", 10_000)
11171            .into_iter()
11172            .map(|(_label, subject)| subject)
11173            .collect();
11174        // The same selection via a SPARQL FILTER scan over every label literal.
11175        let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11176                 FILTER(STRSTARTS(LCASE(?l), \"glucose\")) }";
11177        let crate::QueryOutput::Select(_, rows) = crate::eval_query(&rete, q).unwrap() else {
11178            panic!("expected SELECT");
11179        };
11180        let scan_subjects: std::collections::BTreeSet<String> =
11181            rows.iter().map(|r| r.get("s").cloned().unwrap()).collect();
11182        assert_eq!(idx_subjects, scan_subjects, "index agrees with the scan");
11183        assert_eq!(
11184            idx_subjects.len(),
11185            100,
11186            "800/8 words = 100 glucose-* labels"
11187        );
11188    }
11189
11190    /// A legacy (pyramid-block) label search resolves its hits' subjects in one
11191    /// batched dictionary read on a lazy open — the label-search twin of the
11192    /// full-text hits' batch — instead of one dependent read per chunk.
11193    #[test]
11194    fn legacy_label_hits_resolve_in_one_batched_read() {
11195        use std::sync::{Arc, Mutex};
11196        type CallLog = Arc<Mutex<Vec<Vec<(u64, u64)>>>>;
11197        #[derive(Clone)]
11198        struct Calls(Arc<Vec<u8>>, CallLog);
11199        impl crate::RangeReader for Calls {
11200            fn len(&self) -> u64 {
11201                self.0.len() as u64
11202            }
11203            fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
11204                self.1.lock().unwrap().push(vec![(offset, len)]);
11205                crate::SliceReader::new(&self.0).read_at(offset, len)
11206            }
11207            fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
11208                self.1.lock().unwrap().push(ranges.to_vec());
11209                let r = crate::SliceReader::new(&self.0);
11210                ranges.iter().map(|&(o, l)| r.read_at(o, l)).collect()
11211            }
11212        }
11213        let bytes = build_labeled(20_000);
11214        let h = Header::from_bytes(&bytes).unwrap();
11215        let (d0, dl) = (h.dictionary_offset, h.dictionary_len);
11216        let want = Rete::open(&bytes).unwrap().prefix_search("glucose", 1000);
11217        assert_eq!(want.len(), 1000);
11218        for search_view in [false, true] {
11219            let calls = Calls(Arc::new(bytes.clone()), Arc::new(Mutex::new(Vec::new())));
11220            let got = if search_view {
11221                let view = SearchView::open_ranged_with(calls.clone(), HEADER_LEN as u64).unwrap();
11222                calls.1.lock().unwrap().clear();
11223                view.prefix_search("glucose", 1000)
11224            } else {
11225                let rete = Rete::open_ranged_lazy_with(calls.clone(), HEADER_LEN as u64).unwrap();
11226                calls.1.lock().unwrap().clear();
11227                rete.prefix_search("glucose", 1000)
11228            };
11229            assert_eq!(got, want, "search view = {search_view}");
11230            let dict_reads = calls
11231                .1
11232                .lock()
11233                .unwrap()
11234                .iter()
11235                .filter(|c| c.iter().any(|&(o, l)| o < d0 + dl && d0 < o + l))
11236                .count();
11237            assert_eq!(dict_reads, 1, "search view = {search_view}");
11238        }
11239    }
11240
11241    /// Latency: the binary-search label index vs the FILTER scan it replaces.
11242    /// Ignored by default (timing-sensitive); run with
11243    /// `cargo test -p rete-core -- --ignored --nocapture bench_prefix_search`.
11244    #[test]
11245    #[ignore]
11246    fn bench_prefix_search_vs_filter_scan() {
11247        use std::time::Instant;
11248        let n = 6000; // < the 8192 cap, so both paths return identical sets
11249        let bytes = build_labeled(n);
11250        let rete = Rete::open(&bytes).unwrap();
11251        let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11252                 FILTER(STRSTARTS(LCASE(?l), \"glucose\")) }";
11253        let reps = 200;
11254        let idx_n = rete.prefix_search("glucose", 100_000).len();
11255        let t = Instant::now();
11256        for _ in 0..reps {
11257            std::hint::black_box(rete.prefix_search("glucose", 100_000));
11258        }
11259        let idx_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11260        let t = Instant::now();
11261        for _ in 0..reps {
11262            let _ = std::hint::black_box(crate::eval_query(&rete, q).unwrap());
11263        }
11264        let scan_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11265        println!(
11266            "label prefix search over {n} labeled subjects ({idx_n} matches): \
11267             index {idx_ms:.4} ms vs FILTER scan {scan_ms:.3} ms ({:.0}× faster)",
11268            scan_ms / idx_ms
11269        );
11270    }
11271
11272    /// Latency: the TEXT_INDEX word search vs the `FILTER(CONTAINS(?l, …))` scan
11273    /// it replaces. Ignored by default (timing-sensitive); run with
11274    /// `cargo test -p rete-core -- --ignored --nocapture bench_text_search`.
11275    #[test]
11276    #[ignore]
11277    fn bench_text_search_vs_contains_scan() {
11278        use std::time::Instant;
11279        const LABEL: &str = "<http://www.w3.org/2000/01/rdf-schema#label>";
11280        const WORDS: &[&str] = &[
11281            "alanine",
11282            "benzene",
11283            "glucose",
11284            "dextrose",
11285            "ethanol",
11286            "formate",
11287            "heptane",
11288            "isoleucine",
11289        ];
11290        let n = 6000;
11291        let triples: Vec<(String, String, String)> = (0..n)
11292            .map(|i| {
11293                (
11294                    format!("<http://ex/e{i}>"),
11295                    LABEL.to_string(),
11296                    format!("\"{} sample number {i:06}\"", WORDS[i % WORDS.len()]),
11297                )
11298            })
11299            .collect();
11300        let mut db = DictionaryBuilder::new();
11301        for (s, p, o) in &triples {
11302            db.observe(s, p, o);
11303        }
11304        let dict = db.build();
11305        let ids: Vec<(u32, u32, u32)> = triples
11306            .iter()
11307            .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11308            .collect();
11309        let mut ib = GraphIndexBuilder::new();
11310        for &t in &ids {
11311            ib.push(t);
11312        }
11313        let ti = compute_text_index(&dict, &ids);
11314        let bytes = write_dataset_with_metadata(&dict, &ib.build(), &[], false, &[], 0, &[], &ti);
11315        let rete = Rete::open(&bytes).unwrap();
11316
11317        let q = "SELECT ?s WHERE { ?s <http://www.w3.org/2000/01/rdf-schema#label> ?l \
11318                 FILTER(CONTAINS(LCASE(?l), \"glucose\")) }";
11319        let reps = 200;
11320        let idx_n = rete.text_search(&["glucose"], None, 100_000).len();
11321        let t = Instant::now();
11322        for _ in 0..reps {
11323            std::hint::black_box(rete.text_search(&["glucose"], None, 100_000));
11324        }
11325        let idx_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11326        let t = Instant::now();
11327        for _ in 0..reps {
11328            let _ = std::hint::black_box(crate::eval_query(&rete, q).unwrap());
11329        }
11330        let scan_ms = t.elapsed().as_secs_f64() * 1000.0 / reps as f64;
11331        println!(
11332            "text search over {n} literals ({idx_n} matches): \
11333             index {idx_ms:.4} ms vs FILTER(CONTAINS) scan {scan_ms:.3} ms ({:.0}× faster)",
11334            scan_ms / idx_ms
11335        );
11336    }
11337
11338    /// Operational debugging harness for a REAL on-disk file (ignored; driven
11339    /// by env vars): step-by-step dump of a bound (p, o) POS routing.
11340    ///   RETE_DEBUG_FILE=<path.rete> RETE_DEBUG_P=<iri> RETE_DEBUG_O=<iri>
11341    #[test]
11342    #[ignore = "operational tool, driven by RETE_DEBUG_* env vars"]
11343    fn debug_bound_po_routing() {
11344        struct FR(std::fs::File);
11345        impl crate::RangeReader for FR {
11346            fn len(&self) -> u64 {
11347                self.0.metadata().map(|m| m.len()).unwrap_or(0)
11348            }
11349            fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
11350                use std::os::unix::fs::FileExt;
11351                let mut buf = vec![0u8; len as usize];
11352                self.0.read_exact_at(&mut buf, offset)?;
11353                Ok(buf)
11354            }
11355        }
11356        let path = std::env::var("RETE_DEBUG_FILE").expect("RETE_DEBUG_FILE");
11357        let p_iri = std::env::var("RETE_DEBUG_P").expect("RETE_DEBUG_P");
11358        let o_iri = std::env::var("RETE_DEBUG_O").expect("RETE_DEBUG_O");
11359        let rete = Rete::open_ranged_lazy_with(
11360            std::sync::Arc::new(FR(std::fs::File::open(&path).unwrap())),
11361            HEADER_LEN as u64,
11362        )
11363        .unwrap();
11364        let pid = rete.dict.predicate_id(&p_iri).expect("p resolves");
11365        let oid = rete.dict.object_id(&o_iri).expect("o resolves");
11366        eprintln!("pid={pid} oid={oid}");
11367        let pattern = (None, Some(pid), Some(oid));
11368        let perm = GraphIndex::best_permutation(pattern);
11369        eprintln!("best_permutation = {}", perm.name());
11370        let si = perm.section_index();
11371        let tiles = &rete.index.sections[si];
11372        eprintln!("section {} tiles = {}", perm.name(), tiles.len());
11373        let [pa, pb, pc] = perm.order_pattern(pattern);
11374        eprintln!("permuted pattern pa={pa:?} pb={pb:?} pc={pc:?}");
11375        let (start, end) = rete.index.tile_span(si, pa);
11376        eprintln!("tile_span = [{start}, {end}) -> {} tiles", end - start);
11377        let mut admitted = 0usize;
11378        for (ti, t) in tiles.iter().enumerate().take(end).skip(start) {
11379            if t.syn_admits(pb, pc) {
11380                admitted += 1;
11381                if admitted <= 10 {
11382                    let (lo, hi) = t.leading_range();
11383                    eprintln!("  admit tile {ti}: a=[{lo},{hi}] syn={:?}", t.syn);
11384                }
11385            }
11386        }
11387        eprintln!("admitted {admitted} tile(s) by synopsis");
11388        let n = rete.index.scan_iter(pattern).count();
11389        eprintln!("scan_iter matches = {n}");
11390        let hi_res = rete.query(None, Some(&p_iri), Some(&o_iri));
11391        eprintln!("high-level query matches = {}", hi_res.len());
11392    }
11393
11394    /// The header segment's LABEL and the span it describes must agree. They
11395    /// did not from v0.3 (which grew the header to 1 KiB) until this test: the
11396    /// label still said "fixed 128 bytes", the pre-v0.3 size, while the segment
11397    /// covered `HEADER_LEN` = 1024 — and that label is exactly what the file
11398    /// explorer shows someone inspecting a real file. Reading the size back out
11399    /// of the string is the point: it fails if anyone retypes the number
11400    /// instead of deriving it from `HEADER_LEN`.
11401    #[test]
11402    fn layout_header_label_matches_its_span() {
11403        let mut db = DictionaryBuilder::new();
11404        let mut triples = Vec::new();
11405        for i in 0..24u32 {
11406            let (s, p, o) = (
11407                format!("<http://ex/s{}>", i % 5),
11408                format!("<http://ex/p{}>", i % 3),
11409                format!("<http://ex/o{i}>"),
11410            );
11411            db.observe(&s, &p, &o);
11412            triples.push((s, p, o));
11413        }
11414        let dict = db.build();
11415        let ids: Vec<(u32, u32, u32)> = triples
11416            .iter()
11417            .map(|(s, p, o)| dict.encode(s, p, o).unwrap())
11418            .collect();
11419        let index = GraphIndexBuilder::from_triples(ids).build();
11420        let image = write_dataset_with_metadata(
11421            &dict,
11422            &index,
11423            &[],
11424            false,
11425            &[],
11426            0,
11427            br#"{"name":"layout-label-fixture"}"#,
11428            &[],
11429        );
11430        let rete = Rete::open(&image).unwrap();
11431
11432        let layout = rete.file_layout();
11433        let header = layout
11434            .iter()
11435            .find(|s| s.kind == "header")
11436            .expect("every layout starts with a header segment");
11437
11438        assert_eq!(
11439            header.offset, 0,
11440            "the header is the first thing in the file"
11441        );
11442        assert_eq!(
11443            header.len, HEADER_LEN as u64,
11444            "the header segment must span exactly HEADER_LEN"
11445        );
11446
11447        // Pull the number back out of the human-readable label and hold it to
11448        // the span. A hard-coded literal that drifts from HEADER_LEN dies here.
11449        let digits: String = header
11450            .label
11451            .chars()
11452            .skip_while(|c| !c.is_ascii_digit())
11453            .take_while(|c| c.is_ascii_digit())
11454            .collect();
11455        assert!(
11456            !digits.is_empty(),
11457            "header label {:?} states no size at all",
11458            header.label
11459        );
11460        assert_eq!(
11461            digits.parse::<u64>().unwrap(),
11462            header.len,
11463            "header label {:?} disagrees with the {}-byte span it describes",
11464            header.label,
11465            header.len
11466        );
11467
11468        // The layout must also stay sorted and start at byte zero, which is
11469        // what makes "the first segment is the header" meaningful.
11470        assert!(
11471            layout.windows(2).all(|w| w[0].offset <= w[1].offset),
11472            "file_layout must return segments sorted by offset"
11473        );
11474    }
11475}