Skip to main content

rete_core/
ingest.rs

1//! Ingestion: parse RDF text (N-Triples / N-Quads / Turtle) and assemble a
2//! complete `.rete` file image. Shared by the CLI's `build`/`validate` commands
3//! and the wasm bindings (the playground's in-browser builder).
4//!
5//! The N-Triples/N-Quads reader is line-based and keeps terms as their exact
6//! canonical token strings (`<iri>`, `_:bnode`, `"lit"`, `"lit"^^<dt>`,
7//! `"lit"@lang`) so they double as dictionary keys and so a `query` can match
8//! by the same string. This is not a full RDF 1.1 parser — it covers the
9//! canonical N-Triples surface (and is deliberately tolerant of IRIs a strict
10//! parser would reject), enough for v0 ingestion. Turtle goes through `oxttl`.
11//!
12//! That tolerance is *measured*, not silent: pass an [`IriAudit`] to any of the
13//! `*_audited` entry points and the parse counts every IRI outside the
14//! N-Triples `IRIREF` grammar / RFC 3987 (see [`crate::iri`]) — or, with
15//! `strict`, refuses the input on the first one. `rete build` always audits.
16//!
17//! Assembly degrades with the build features: without the `compression`
18//! feature (e.g. on wasm) sections are written with the `NONE` codec — larger
19//! files, byte-compatible readers.
20
21use crate::{Dictionary, DictionaryBuilder, GraphIndexBuilder, PyramidAlgo, DEFAULT_TILE_BUDGET};
22
23mod jelly;
24mod jsonld;
25
26pub use jsonld::{ContextLoader, JsonLdOptions};
27
28/// A parsed triple as three canonical term tokens.
29pub type RawTriple = (String, String, String);
30
31/// A parsed quad: the triple plus an optional graph term (`None` = default graph).
32pub type RawQuad = (String, String, String, Option<String>);
33
34/// Which **surface** a `<< … >>` in Turtle/TriG input is read as — the input
35/// half of `rete export --quoted-triple-syntax`, same vocabulary, same two
36/// values.
37///
38/// The flag exists because the two standards give one piece of syntax two
39/// meanings, and no amount of looking at the bytes can tell them apart:
40///
41/// | written            | under [`RdfStar`](Self::RdfStar) | under [`Rdf12`](Self::Rdf12)          |
42/// |--------------------|----------------------------------|---------------------------------------|
43/// | `<< s p o >>`      | a **quoted triple**, one term    | a **reifier**: `_:r rdf:reifies <<( s p o )>>`, plus a blank node standing where it was |
44/// | `<<( s p o )>>`    | a syntax error                   | a **triple term**, object position only |
45/// | `{\| … \|}`         | a syntax error                   | an annotation on the statement before it |
46///
47/// One file, two readings, two different graphs — see the round-trip matrix in
48/// `crates/rete-cli/tests/quoted_triple_surfaces.rs`, which asserts both.
49///
50/// `<<( s p o )>>` is unambiguous: it is RDF 1.2's and nothing else's. That
51/// asymmetry is why [`RdfStar`](Self::RdfStar) is the **default**. Reading an
52/// RDF-star file as RDF 1.2 silently yields a different graph; reading an
53/// RDF 1.2 file as RDF-star is a hard parse error that names the flag. Only one
54/// of the two mistakes is survivable, so the default is the one that makes the
55/// other mistake loud.
56///
57/// **N-Triples and N-Quads ignore this entirely.** Their reader is rete's own
58/// (`take_term`), it has accepted both `<< s p o >>` and `<<( s p o )>>` since
59/// #262, and RDF 1.2 N-Triples has no reifier syntax for `<< … >>` to be — so
60/// there is no ambiguity to resolve and nothing to choose. The same goes for
61/// RDF/XML.
62///
63/// Whichever surface reads the file, what gets **stored** is the same canonical
64/// token `<<s p o>>`. rete's term model is a superset of both: a triple term is
65/// a term, and RDF 1.2 reification is ordinary RDF — a blank node, a predicate
66/// and a term — which rete already stored before this flag existed.
67#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Hash)]
68pub enum QuotedTripleSurface {
69    /// `<< s p o >>` is a **quoted triple** (RDF-star), in subject or object
70    /// position — rete's own storage token. The default, and byte-for-byte the
71    /// behaviour every rete release before this flag had.
72    #[default]
73    RdfStar,
74    /// `<<( s p o )>>` is a **triple term** and `<< s p o >>` is a **reifier**
75    /// (RDF 1.2), with `{| … |}` annotations read too.
76    Rdf12,
77}
78
79impl QuotedTripleSurface {
80    /// Parse the `--quoted-triple-syntax` value, `None` for anything else.
81    /// Shared by `rete build`, `rete validate` and `rete export` so the flag
82    /// cannot come to mean two things in two directions.
83    pub fn parse(s: &str) -> Option<Self> {
84        match s {
85            crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF12 => Some(Self::Rdf12),
86            crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF_STAR => Some(Self::RdfStar),
87            _ => None,
88        }
89    }
90
91    /// The flag spelling of this surface — the inverse of [`Self::parse`].
92    pub const fn as_str(self) -> &'static str {
93        match self {
94            Self::Rdf12 => crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF12,
95            Self::RdfStar => crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF_STAR,
96        }
97    }
98}
99
100/// The one-line hint appended to a Turtle/TriG parse error when the input looks
101/// like RDF 1.2 and was read as RDF-star — i.e. the exact mistake
102/// [`QuotedTripleSurface`]'s default makes, made loud.
103const RDF12_SURFACE_HINT: &str = "\nhint: this input contains `<<(`, the RDF 1.2 triple-term \
104     syntax — which is what `rete export` writes by default. Reading it takes \
105     `--quoted-triple-syntax rdf12`; the default, `rdf-star`, reads `<< s p o >>` as a quoted \
106     triple instead.";
107
108/// Does `text` look like it carries RDF 1.2 triple terms? A deliberately dumb
109/// substring test: `<<(` cannot begin anything else in Turtle, and this is only
110/// ever consulted to *explain a parse that already failed*, never to decide how
111/// to read a file. A `<<(` inside a string literal would make it a false
112/// positive on some other error's hint, which costs one confusing sentence and
113/// nothing else.
114fn looks_like_rdf12(text: &str) -> bool {
115    text.contains("<<(")
116}
117
118/// Why an ingest failed. Parser errors from `oxttl`/`oxrdfxml` and I/O errors
119/// are flattened to their `Display` string rather than wrapped: the variants
120/// then carry no foreign types, which keeps the error usable from the wasm
121/// bindings and stable across upstream parser upgrades.
122#[derive(Debug, thiserror::Error)]
123#[non_exhaustive]
124pub enum IngestError {
125    /// A malformed N-Triples/N-Quads line: its 1-based line number and a short
126    /// reason (`"bad subject"`, `"missing trailing '.'"`, …).
127    #[error("line {0}: {1}")]
128    Line(usize, &'static str),
129    /// The Turtle parser rejected the input.
130    #[error("turtle: {0}")]
131    Turtle(String),
132    /// The TriG parser rejected the input.
133    #[error("trig: {0}")]
134    TriG(String),
135    /// The RDF/XML parser rejected the input.
136    #[error("rdf/xml: {0}")]
137    RdfXml(String),
138    /// The requested format is not one this crate can parse.
139    #[error(
140        "unknown input format: {0} (expected nt, nq, ttl, trig, rdf/xml, jelly, jsonld, or yamlld)"
141    )]
142    UnknownFormat(String),
143    /// A Jelly file that is malformed, uses a feature this reader refuses, or
144    /// breaks a rule of the specification.
145    #[error("jelly: {0}")]
146    Jelly(String),
147    /// A JSON-LD or YAML-LD document that is malformed or that the JSON-LD
148    /// processor refuses. Also what a build without the `jsonld` feature, or a
149    /// streaming reader (JSON-LD is expanded whole), answers.
150    #[error("{0}")]
151    JsonLd(String),
152    /// A JSON-LD or YAML-LD document names a remote `@context` (the IRI) and no
153    /// [`ContextLoader`] was given, so it cannot be expanded.
154    #[error(
155        "json-ld: the document needs the remote context {0}, and remote contexts are not loaded"
156    )]
157    JsonLdRemoteContext(String),
158    /// [`QuotedTripleSurface::Rdf12`] was asked for on a Turtle/TriG input, but
159    /// this build of `rete-core` has the `rdf12-turtle` feature off, so the
160    /// RDF 1.2 reader was not compiled in. Refused by name rather than read as
161    /// RDF-star, which would be the wrong graph told in silence.
162    #[error(
163        "this build cannot read the RDF 1.2 Turtle/TriG surface: rete-core was compiled without \
164         the `rdf12-turtle` feature (N-Triples/N-Quads still accept `<<( s p o )>>`)"
165    )]
166    Rdf12TurtleUnavailable,
167    /// The input could not be read (streaming builds surface the path here).
168    #[error("io: {0}")]
169    Io(String),
170    /// An IRI outside the N-Triples/N-Quads `IRIREF` grammar / RFC 3987, refused
171    /// because the caller asked for a strict [`IriAudit`] (`rete build
172    /// --strict`). Without `strict` the same finding is only counted.
173    /// `location` is `line N` for the line-based syntaxes, the format name for
174    /// the others (oxttl reports its own positions).
175    #[error("{location}: invalid IRI {token} — {reason}")]
176    InvalidIri {
177        /// Where it was seen: `line 42`, or the format name.
178        location: String,
179        /// The offending term token, verbatim.
180        token: String,
181        /// Which rule it breaks ([`crate::iri::IriDefect::reason`]).
182        reason: &'static str,
183    },
184}
185
186/// Policy **and** tally for the invalid-IRI check that runs during ingest.
187///
188/// rete's reader is deliberately tolerant of IRIs a strict parser rejects (see
189/// [`crate::iri`]), which is what lets it hold every published dataset — and
190/// also what let three of the `scholar/` exports produce N-Quads Oxigraph
191/// refuses to load. Passing an audit makes the tolerance *visible*: the build
192/// still succeeds and stores exactly what it was given, but it can now say how
193/// much of what it stored is not valid RDF. `strict` turns the same finding into
194/// a refusal, on the first offending statement.
195#[derive(Debug, Default)]
196pub struct IriAudit {
197    /// Fail the parse on the first invalid IRI instead of counting it.
198    pub strict: bool,
199    /// What the parse saw. Constant memory — see [`crate::iri::IriReport`].
200    pub report: crate::iri::IriReport,
201}
202
203impl IriAudit {
204    /// Count invalid IRIs; never fail (`rete build`'s default).
205    pub fn counting() -> Self {
206        Self::default()
207    }
208
209    /// Refuse the input at the first invalid IRI (`rete build --strict`).
210    pub fn refusing() -> Self {
211        Self {
212            strict: true,
213            report: crate::iri::IriReport::default(),
214        }
215    }
216}
217
218/// Run one statement past the audit: record any invalid IRI, and in `strict`
219/// mode turn it into an error naming the offending token. `location` is only
220/// called on the error path, so the common case allocates nothing.
221fn audit_quad<F: FnOnce() -> String>(
222    audit: Option<&mut IriAudit>,
223    s: &str,
224    p: &str,
225    o: &str,
226    g: Option<&str>,
227    location: F,
228) -> Result<(), IngestError> {
229    let Some(a) = audit else { return Ok(()) };
230    let Some(defect) = a.report.observe_quad(s, p, o, g) else {
231        return Ok(());
232    };
233    if !a.strict {
234        return Ok(());
235    }
236    let token = [Some(s), Some(p), Some(o), g]
237        .into_iter()
238        .flatten()
239        .find(|t| crate::iri::term_defect(t).is_some())
240        .unwrap_or("<?>")
241        .to_string();
242    Err(IngestError::InvalidIri {
243        location: location(),
244        token,
245        reason: defect.reason(),
246    })
247}
248
249/// Estimate the statement count of an N-Triples/N-Quads text from its newline
250/// count, so the output `Vec` can be pre-sized — a big build otherwise pays
251/// repeated `Vec` doublings, each briefly holding ~2× the (large) spine. An
252/// over-estimate by a few blank/comment lines is harmless (it's a capacity hint).
253fn estimate_statements(input: &str) -> usize {
254    bytecount_newlines(input).max(1)
255}
256
257/// Count `\n` bytes — one linear pass, far cheaper than the parse it sizes.
258fn bytecount_newlines(input: &str) -> usize {
259    input.as_bytes().iter().filter(|&&b| b == b'\n').count()
260}
261
262/// Parse one N-Quads line into a quad, or `None` for a blank/comment line.
263/// Shared by the whole-text [`parse_quads`] and the streaming [`parse_reader`].
264fn parse_nq_line(
265    raw: &str,
266    lineno: usize,
267    audit: Option<&mut IriAudit>,
268) -> Result<Option<RawQuad>, IngestError> {
269    let line = raw.trim();
270    if line.is_empty() || line.starts_with('#') {
271        return Ok(None);
272    }
273    let stripped = line
274        .strip_suffix('.')
275        .ok_or(IngestError::Line(lineno, "missing trailing '.'"))?
276        .trim_end();
277    let (s, rest) = take_term(stripped).ok_or(IngestError::Line(lineno, "bad subject"))?;
278    let (p, rest) =
279        take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad predicate"))?;
280    let (o, rest) = take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad object"))?;
281    let rest = rest.trim();
282    let graph = if rest.is_empty() {
283        None
284    } else {
285        let (g, tail) = take_term(rest).ok_or(IngestError::Line(lineno, "bad graph"))?;
286        if !tail.trim().is_empty() {
287            return Err(IngestError::Line(lineno, "trailing content after graph"));
288        }
289        Some(g)
290    };
291    audit_quad(audit, &s, &p, &o, graph.as_deref(), || {
292        format!("line {lineno}")
293    })?;
294    Ok(Some((s, p, o, graph)))
295}
296
297/// Parse one N-Triples line into a triple, or `None` for a blank/comment line.
298fn parse_nt_line(
299    raw: &str,
300    lineno: usize,
301    audit: Option<&mut IriAudit>,
302) -> Result<Option<RawTriple>, IngestError> {
303    let line = raw.trim();
304    if line.is_empty() || line.starts_with('#') {
305        return Ok(None);
306    }
307    let stripped = line
308        .strip_suffix('.')
309        .ok_or(IngestError::Line(lineno, "missing trailing '.'"))?
310        .trim_end();
311    let (s, rest) = take_term(stripped).ok_or(IngestError::Line(lineno, "bad subject"))?;
312    let (p, rest) =
313        take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad predicate"))?;
314    let (o, rest) = take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad object"))?;
315    if !rest.trim().is_empty() {
316        return Err(IngestError::Line(lineno, "trailing content after object"));
317    }
318    audit_quad(audit, &s, &p, &o, None, || format!("line {lineno}"))?;
319    Ok(Some((s, p, o)))
320}
321
322/// Parse N-Quads text: `subject predicate object [graph] .` per line.
323pub fn parse_quads(input: &str) -> Result<Vec<RawQuad>, IngestError> {
324    parse_quads_audited(input, None)
325}
326
327/// [`parse_quads`] with an optional [`IriAudit`].
328pub fn parse_quads_audited(
329    input: &str,
330    mut audit: Option<&mut IriAudit>,
331) -> Result<Vec<RawQuad>, IngestError> {
332    let mut out = Vec::with_capacity(estimate_statements(input));
333    for (i, raw) in input.lines().enumerate() {
334        if let Some(q) = parse_nq_line(raw, i + 1, audit.as_deref_mut())? {
335            out.push(q);
336        }
337    }
338    Ok(out)
339}
340
341/// Parse N-Triples text into raw term-token triples, skipping blank/comment lines.
342pub fn parse(input: &str) -> Result<Vec<RawTriple>, IngestError> {
343    parse_audited(input, None)
344}
345
346/// [`parse`] with an optional [`IriAudit`].
347pub fn parse_audited(
348    input: &str,
349    mut audit: Option<&mut IriAudit>,
350) -> Result<Vec<RawTriple>, IngestError> {
351    let mut out = Vec::with_capacity(estimate_statements(input));
352    for (i, raw) in input.lines().enumerate() {
353        if let Some(t) = parse_nt_line(raw, i + 1, audit.as_deref_mut())? {
354            out.push(t);
355        }
356    }
357    Ok(out)
358}
359
360/// **Stream-parse** N-Triples (`"nt"`) or N-Quads (`"nq"`) from a reader, one
361/// line at a time, so the whole input text is **never resident** — the big-build
362/// memory win over reading the file into a `String` first (each line String is
363/// transient, freed every iteration). `cap` pre-sizes the output `Vec` (e.g.
364/// `file_len / 64`) to avoid reallocation doublings of the large spine. Turtle is
365/// not streamable here (oxttl needs the whole input); callers use the text path
366/// for `"ttl"`.
367pub fn parse_reader<R: std::io::BufRead>(
368    reader: R,
369    format: &str,
370    cap: usize,
371) -> Result<Vec<RawQuad>, IngestError> {
372    parse_reader_audited(reader, format, cap, None)
373}
374
375/// [`parse_reader`] with an optional [`IriAudit`].
376pub fn parse_reader_audited<R: std::io::BufRead>(
377    reader: R,
378    format: &str,
379    cap: usize,
380    audit: Option<&mut IriAudit>,
381) -> Result<Vec<RawQuad>, IngestError> {
382    parse_reader_audited_surface(reader, format, cap, audit, QuotedTripleSurface::default())
383}
384
385/// [`parse_reader_audited`] reading `<< … >>` in the given
386/// [`QuotedTripleSurface`] (Turtle/TriG only — see the enum).
387pub fn parse_reader_audited_surface<R: std::io::BufRead>(
388    reader: R,
389    format: &str,
390    cap: usize,
391    audit: Option<&mut IriAudit>,
392    surface: QuotedTripleSurface,
393) -> Result<Vec<RawQuad>, IngestError> {
394    let mut out = Vec::with_capacity(cap);
395    stream_reader_audited_surface(reader, format, audit, surface, &mut |q| out.push(q))?;
396    Ok(out)
397}
398
399/// **Stream** N-Triples (`"nt"`) / N-Quads (`"nq"`) from a reader, invoking `f`
400/// once per parsed quad — like [`parse_reader`] but **without collecting** into a
401/// `Vec`. Each quad's term Strings are owned by `f` (and dropped when it returns
402/// if it doesn't retain them), so a caller that only needs to *observe* every
403/// term — e.g. the two-pass [`assemble_dataset_streaming`] building its
404/// dictionary — never materializes the whole quad multiset. The big-graph,
405/// low-RAM ingest primitive. Blank/comment lines are skipped; a parse error stops
406/// the stream and is returned.
407///
408/// Every format streams: the line-based ones a line at a time, Turtle / TriG /
409/// RDF-XML through their `oxttl`/`oxrdfxml` reader parsers, which pull from the
410/// reader incrementally. Nothing here holds the input text — so a 60 GB Turtle
411/// file is as ingestible as a 60 GB `.nt` one, which is what lets the external
412/// (memory-bounded) build accept them.
413pub fn stream_reader<R: std::io::BufRead>(
414    reader: R,
415    format: &str,
416    f: &mut dyn FnMut(RawQuad),
417) -> Result<(), IngestError> {
418    stream_reader_audited(reader, format, None, f)
419}
420
421/// [`stream_reader`] with an optional [`IriAudit`].
422///
423/// Every syntax is audited, not only the line-based ones. `oxttl`/`oxrdfxml`
424/// validate IRIs themselves and reject an invalid one before it reaches us, so
425/// in practice the Turtle/TriG/RDF-XML tallies stay at zero — but auditing them
426/// too means the count is a property of the *file*, not of which reader happened
427/// to produce it, and a future parser change cannot silently reopen the hole.
428pub fn stream_reader_audited<R: std::io::BufRead>(
429    reader: R,
430    format: &str,
431    audit: Option<&mut IriAudit>,
432    f: &mut dyn FnMut(RawQuad),
433) -> Result<(), IngestError> {
434    stream_reader_audited_surface(reader, format, audit, QuotedTripleSurface::default(), f)
435}
436
437/// [`stream_reader_audited`] reading `<< … >>` in the given
438/// [`QuotedTripleSurface`].
439///
440/// The surface reaches only the `"ttl"` and `"trig"` arms: the line-based
441/// readers are rete's own and take both spellings unconditionally, and RDF/XML
442/// has no syntax for either. Under [`QuotedTripleSurface::Rdf12`] Turtle/TriG go
443/// through `oxttl` 0.2 instead of 0.1, and the RDF 1.2 triple terms it hands
444/// back (`<<( s p o )>>`) are folded to rete's stored token on the way out, so
445/// the dictionary a file lands in does not depend on which reader read it.
446///
447/// **This entry point does not carry the RDF 1.2 hint** that the text path adds
448/// to a failed RDF-star parse: it is handed a reader, not a string, and sniffing
449/// the input would mean buffering the 60 GB file this function exists to avoid
450/// buffering. `rete build --memory-budget-mb file.ttl` therefore reports the raw
451/// parser error; every other route reports the hint.
452pub fn stream_reader_audited_surface<R: std::io::BufRead>(
453    reader: R,
454    format: &str,
455    mut audit: Option<&mut IriAudit>,
456    surface: QuotedTripleSurface,
457    f: &mut dyn FnMut(RawQuad),
458) -> Result<(), IngestError> {
459    if surface == QuotedTripleSurface::Rdf12 && matches!(format, "ttl" | "trig") {
460        return rdf12::stream(reader, format, audit, f);
461    }
462    match format {
463        "nt" | "nq" => {
464            for (i, line) in reader.lines().enumerate() {
465                let line = line.map_err(|e| IngestError::Io(e.to_string()))?;
466                if format == "nq" {
467                    if let Some(q) = parse_nq_line(&line, i + 1, audit.as_deref_mut())? {
468                        f(q);
469                    }
470                } else if let Some((s, p, o)) = parse_nt_line(&line, i + 1, audit.as_deref_mut())? {
471                    f((s, p, o, None));
472                }
473            }
474            Ok(())
475        }
476        "ttl" => turtle_star(reader, audit, f),
477        "trig" => trig_star(reader, audit, f),
478        "jelly" => jelly::stream(reader, audit, f),
479        "jsonld" | "yamlld" => Err(IngestError::JsonLd(format!(
480            "{format}: a JSON-LD document is expanded whole, so it cannot be streamed from a \
481             reader; build without --memory-budget-mb, or convert it to N-Quads first"
482        ))),
483        "rdfxml" => {
484            for r in oxrdfxml::RdfXmlParser::new().for_reader(reader) {
485                let t = r.map_err(|e| IngestError::RdfXml(e.to_string()))?;
486                let q = (
487                    t.subject.to_string(),
488                    t.predicate.to_string(),
489                    t.object.to_string(),
490                    None,
491                );
492                audit_quad(audit.as_deref_mut(), &q.0, &q.1, &q.2, None, || {
493                    "rdf/xml".into()
494                })?;
495                f(q);
496            }
497            Ok(())
498        }
499        other => Err(IngestError::UnknownFormat(other.to_string())),
500    }
501}
502
503/// Stream **RDF-star** Turtle: `with_quoted_triples` accepts `<< s p o >>` in
504/// subject/object position, and oxrdf 0.2's `Term::Triple` Displays as the
505/// canonical `<<…>>` token rete's own N-Triples-star tokenizer emits — so no
506/// translation is needed on the way out.
507///
508/// Its own function, and called directly from both the reader path and the text
509/// path, so that the wasm engine — which reaches only the text path and has no
510/// RDF 1.2 reader compiled in — links this and nothing else.
511fn turtle_star<R: std::io::BufRead>(
512    reader: R,
513    mut audit: Option<&mut IriAudit>,
514    f: &mut dyn FnMut(RawQuad),
515) -> Result<(), IngestError> {
516    for r in oxttl::TurtleParser::new()
517        .with_quoted_triples()
518        .for_reader(reader)
519    {
520        let t = r.map_err(|e| IngestError::Turtle(e.to_string()))?;
521        let q = (
522            t.subject.to_string(),
523            t.predicate.to_string(),
524            t.object.to_string(),
525            None,
526        );
527        audit_quad(audit.as_deref_mut(), &q.0, &q.1, &q.2, None, || {
528            "turtle".into()
529        })?;
530        f(q);
531    }
532    Ok(())
533}
534
535/// [`turtle_star`] for TriG — Turtle plus named-graph blocks (`<g> { … }`), the
536/// shape most large RDF dumps ship in.
537fn trig_star<R: std::io::BufRead>(
538    reader: R,
539    mut audit: Option<&mut IriAudit>,
540    f: &mut dyn FnMut(RawQuad),
541) -> Result<(), IngestError> {
542    for r in oxttl::TriGParser::new()
543        .with_quoted_triples()
544        .for_reader(reader)
545    {
546        let q = r.map_err(|e| IngestError::TriG(e.to_string()))?;
547        let q = (
548            q.subject.to_string(),
549            q.predicate.to_string(),
550            q.object.to_string(),
551            graph_token(&q.graph_name),
552        );
553        audit_quad(
554            audit.as_deref_mut(),
555            &q.0,
556            &q.1,
557            &q.2,
558            q.3.as_deref(),
559            || "trig".into(),
560        )?;
561        f(q);
562    }
563    Ok(())
564}
565
566/// The canonical graph token for a parsed quad — `None` for the default graph,
567/// otherwise the same `<iri>` / `_:b` spelling the N-Quads reader produces, so
568/// TriG and N-Quads inputs land identical dictionary keys. `GraphName`'s own
569/// `Display` renders the default graph as `DEFAULT`, which must never become a
570/// term, hence the explicit match.
571fn graph_token(g: &oxrdf::GraphName) -> Option<String> {
572    match g {
573        oxrdf::GraphName::DefaultGraph => None,
574        other => Some(other.to_string()),
575    }
576}
577
578/// The **RDF 1.2** Turtle/TriG reader — `oxttl` 0.2 with its `rdf-12` feature,
579/// living beside the `oxttl` 0.1 reader above rather than replacing it.
580///
581/// Everything RDF 1.2 adds over RDF 1.1 arrives through this one module:
582/// `<<( s p o )>>` triple terms, `<< s p o >>` reifiers (which expand to
583/// `_:r rdf:reifies <<( s p o )>>` plus the statement that mentioned them),
584/// `{| … |}` annotations, and `"…"@lang--dir` directional literals. None of it
585/// needs a storage change, which is the point: a reifier is a blank node, an
586/// IRI and a term, and rete has stored those since v0.
587///
588/// The one translation is at the term boundary. `oxrdf` 0.3 renders a triple
589/// term as `<<( s p o )>>`; rete's dictionary key is `<<s p o>>`. [`take_term`]
590/// already reads both — that is what made RDF 1.2 N-Quads ingestible in #262 —
591/// so the fold is a re-scan of a token oxttl has already validated.
592#[cfg(feature = "rdf12-turtle")]
593mod rdf12 {
594    use super::{audit_quad, take_term, IngestError, IriAudit, RawQuad};
595
596    /// Fold `oxrdf` 0.3's `<<( s p o )>>` rendering of a triple term into rete's
597    /// stored `<<s p o>>`, recursively (a nested triple term folds with it).
598    /// Every other token — the overwhelming majority, and all of them in a file
599    /// with no triple terms — is returned untouched, not even copied.
600    pub(super) fn fold_triple_term(t: String) -> String {
601        if !t.starts_with("<<(") {
602            return t;
603        }
604        match take_term(&t) {
605            Some((token, rest)) if rest.trim().is_empty() => token,
606            // oxttl only ever hands back a well-formed term, so this is
607            // unreachable; returning the original beats mangling it silently.
608            _ => t,
609        }
610    }
611
612    /// `super::graph_token` for `oxrdf` 0.3's `GraphName` — the same rule, a
613    /// different crate version, which is exactly why it cannot be shared.
614    fn graph_token(g: &oxrdf12::GraphName) -> Option<String> {
615        match g {
616            oxrdf12::GraphName::DefaultGraph => None,
617            other => Some(other.to_string()),
618        }
619    }
620
621    /// Stream one RDF 1.2 Turtle (`"ttl"`) or TriG (`"trig"`) input, auditing
622    /// every quad the way the RDF-star reader does.
623    pub(super) fn stream<R: std::io::BufRead>(
624        reader: R,
625        format: &str,
626        mut audit: Option<&mut IriAudit>,
627        f: &mut dyn FnMut(RawQuad),
628    ) -> Result<(), IngestError> {
629        if format == "ttl" {
630            for r in oxttl12::TurtleParser::new().for_reader(reader) {
631                let t = r.map_err(|e| IngestError::Turtle(e.to_string()))?;
632                let q = (
633                    t.subject.to_string(),
634                    t.predicate.to_string(),
635                    fold_triple_term(t.object.to_string()),
636                    None,
637                );
638                audit_quad(audit.as_deref_mut(), &q.0, &q.1, &q.2, None, || {
639                    "turtle".into()
640                })?;
641                f(q);
642            }
643        } else {
644            for r in oxttl12::TriGParser::new().for_reader(reader) {
645                let q = r.map_err(|e| IngestError::TriG(e.to_string()))?;
646                let q = (
647                    q.subject.to_string(),
648                    q.predicate.to_string(),
649                    fold_triple_term(q.object.to_string()),
650                    graph_token(&q.graph_name),
651                );
652                audit_quad(
653                    audit.as_deref_mut(),
654                    &q.0,
655                    &q.1,
656                    &q.2,
657                    q.3.as_deref(),
658                    || "trig".into(),
659                )?;
660                f(q);
661            }
662        }
663        Ok(())
664    }
665}
666
667/// The stand-in for [`rdf12`] in a build with `rdf12-turtle` off: it refuses by
668/// name. Reading the file as RDF-star instead would answer a question nobody
669/// asked, with a different graph.
670#[cfg(not(feature = "rdf12-turtle"))]
671mod rdf12 {
672    use super::{IngestError, IriAudit, RawQuad};
673
674    pub(super) fn stream<R: std::io::BufRead>(
675        _reader: R,
676        _format: &str,
677        _audit: Option<&mut IriAudit>,
678        _f: &mut dyn FnMut(RawQuad),
679    ) -> Result<(), IngestError> {
680        Err(IngestError::Rdf12TurtleUnavailable)
681    }
682}
683
684/// Parse Turtle into canonical N-Triples-token triples via oxttl, reading
685/// `<< s p o >>` as an RDF-star quoted triple. [`parse_turtle_surface`] chooses.
686pub fn parse_turtle(text: &str) -> Result<Vec<RawTriple>, IngestError> {
687    parse_turtle_surface(text, QuotedTripleSurface::default())
688}
689
690/// [`parse_turtle`] in the given [`QuotedTripleSurface`].
691pub fn parse_turtle_surface(
692    text: &str,
693    surface: QuotedTripleSurface,
694) -> Result<Vec<RawTriple>, IngestError> {
695    Ok(parse_text_surface(text, "ttl", surface)?
696        .into_iter()
697        .map(|(s, p, o, _)| (s, p, o))
698        .collect())
699}
700
701/// Parse TriG into canonical quads via oxttl — Turtle plus named-graph blocks
702/// (`<g> { … }`), the shape most large RDF dumps ship in. Reads `<< s p o >>` as
703/// an RDF-star quoted triple; [`parse_trig_surface`] chooses.
704pub fn parse_trig(text: &str) -> Result<Vec<RawQuad>, IngestError> {
705    parse_trig_surface(text, QuotedTripleSurface::default())
706}
707
708/// [`parse_trig`] in the given [`QuotedTripleSurface`].
709pub fn parse_trig_surface(
710    text: &str,
711    surface: QuotedTripleSurface,
712) -> Result<Vec<RawQuad>, IngestError> {
713    parse_text_surface(text, "trig", surface)
714}
715
716/// The shared Turtle/TriG text path: stream the string through whichever reader
717/// `surface` selects, and — this is the part only the text path can do, because
718/// only it holds the input — say so when an RDF-star parse fails on what is
719/// plainly RDF 1.2.
720///
721/// It does not audit: the Turtle/TriG text callers audit over the finished quads
722/// (see [`parse_statements_audited_surface`]), and auditing here too would count
723/// every IRI twice.
724///
725/// The RDF-star arm calls `oxttl` 0.1 **directly** rather than routing through
726/// [`stream_reader_audited_surface`]. That is not style: this is the only ingest
727/// path the wasm engine reaches, and going through the generic reader would
728/// monomorphize it over `&[u8]` and pull the whole `match format` — every
729/// syntax's reader — into a module that previously linked one Turtle parser.
730/// Measured at +17 KB of wasm for code no browser can run, since the RDF 1.2
731/// reader is not compiled there at all. Keeping the arm direct keeps the
732/// browser artifacts what they were.
733fn parse_text_surface(
734    text: &str,
735    format: &str,
736    surface: QuotedTripleSurface,
737) -> Result<Vec<RawQuad>, IngestError> {
738    let mut out = Vec::new();
739    let r = if surface == QuotedTripleSurface::Rdf12 {
740        rdf12::stream(text.as_bytes(), format, None, &mut |q| out.push(q))
741    } else if format == "ttl" {
742        turtle_star(text.as_bytes(), None, &mut |q| out.push(q))
743    } else {
744        trig_star(text.as_bytes(), None, &mut |q| out.push(q))
745    };
746    match r {
747        Ok(()) => Ok(out),
748        // The default surface met `<<(`: that is an RDF 1.2 file being read as
749        // RDF-star, the one mistake this flag exists to prevent, and the parse
750        // error alone ("unexpected character") would send the reader hunting for
751        // a typo that is not there.
752        Err(IngestError::Turtle(m))
753            if surface == QuotedTripleSurface::RdfStar && looks_like_rdf12(text) =>
754        {
755            Err(IngestError::Turtle(format!("{m}{RDF12_SURFACE_HINT}")))
756        }
757        Err(IngestError::TriG(m))
758            if surface == QuotedTripleSurface::RdfStar && looks_like_rdf12(text) =>
759        {
760            Err(IngestError::TriG(format!("{m}{RDF12_SURFACE_HINT}")))
761        }
762        Err(e) => Err(e),
763    }
764}
765
766/// Parse RDF/XML into canonical N-Triples-token triples via oxrdfxml. This is how
767/// most OWL ontologies ship (`.rdf`/`.owl`/`.xml` with an `rdf:RDF` root) — so rete
768/// ingests them directly, no external conversion. (OWL/XML — the non-RDF functional
769/// XML serialization — is a different language; convert it with owlready2 first.)
770pub fn parse_rdfxml(text: &str) -> Result<Vec<RawTriple>, IngestError> {
771    let mut out = Vec::new();
772    for r in oxrdfxml::RdfXmlParser::new().for_reader(text.as_bytes()) {
773        let t = r.map_err(|e| IngestError::RdfXml(e.to_string()))?;
774        out.push((
775            t.subject.to_string(),
776            t.predicate.to_string(),
777            t.object.to_string(),
778        ));
779    }
780    Ok(out)
781}
782
783/// Parse one text input by format name (`"nt"`, `"nq"`, `"ttl"`, `"trig"`, or
784/// `"rdfxml"`) into quads (triples land in the default graph).
785pub fn parse_statements(text: &str, format: &str) -> Result<Vec<RawQuad>, IngestError> {
786    parse_statements_audited(text, format, None)
787}
788
789/// [`parse_statements`] with an optional [`IriAudit`]. The line-based syntaxes
790/// audit as they parse (so `strict` fails on the offending line); the others are
791/// audited over the parsed quads, which is where `oxttl` has already had its say.
792pub fn parse_statements_audited(
793    text: &str,
794    format: &str,
795    audit: Option<&mut IriAudit>,
796) -> Result<Vec<RawQuad>, IngestError> {
797    parse_statements_audited_surface(text, format, audit, QuotedTripleSurface::default())
798}
799
800/// [`parse_statements_audited`] reading `<< … >>` in the given
801/// [`QuotedTripleSurface`]. This is the entry point `rete build` and
802/// `rete validate` use for everything but a streamed line-based input, and the
803/// one that turns "RDF 1.2 file, RDF-star reader" into an error that names the
804/// flag instead of a bare syntax complaint.
805pub fn parse_statements_audited_surface(
806    text: &str,
807    format: &str,
808    mut audit: Option<&mut IriAudit>,
809    surface: QuotedTripleSurface,
810) -> Result<Vec<RawQuad>, IngestError> {
811    let quads = match format {
812        "nq" => return parse_quads_audited(text, audit),
813        "nt" => {
814            return Ok(parse_audited(text, audit)?
815                .into_iter()
816                .map(|(s, p, o)| (s, p, o, None))
817                .collect())
818        }
819        // Straight to the quad path for both: going via `parse_turtle_surface`
820        // would build a `Vec<RawQuad>`, rebuild it as a `Vec<RawTriple>` to
821        // satisfy that function's signature, and rebuild it as quads again —
822        // three copies of every term String on the biggest input rete takes.
823        "ttl" | "trig" => parse_text_surface(text, format, surface)?,
824        "rdfxml" => parse_rdfxml(text)?
825            .into_iter()
826            .map(|(s, p, o)| (s, p, o, None))
827            .collect(),
828        // Inline contexts only here; `parse_jsonld_audited` takes a loader.
829        "jsonld" | "yamlld" => {
830            return parse_jsonld_audited(text, format == "yamlld", &JsonLdOptions::default(), audit)
831        }
832        "jelly" => {
833            return Err(IngestError::Jelly(
834                "Jelly is a binary format, so it cannot come in as text; \
835                 use parse_jelly_bytes or stream_reader"
836                    .into(),
837            ))
838        }
839        other => return Err(IngestError::UnknownFormat(other.to_string())),
840    };
841    for (s, p, o, g) in &quads {
842        audit_quad(audit.as_deref_mut(), s, p, o, g.as_deref(), || {
843            format.to_string()
844        })?;
845    }
846    Ok(quads)
847}
848
849/// Parse one JSON-LD (`yaml == false`) or YAML-LD document into quads. Named
850/// graphs stay named. Remote contexts come from `options`; with none, a document
851/// that needs one fails with [`IngestError::JsonLdRemoteContext`].
852pub fn parse_jsonld_audited(
853    text: &str,
854    yaml: bool,
855    options: &JsonLdOptions,
856    audit: Option<&mut IriAudit>,
857) -> Result<Vec<RawQuad>, IngestError> {
858    let mut out = Vec::new();
859    jsonld::parse(text, yaml, options, audit, &mut |q| out.push(q))?;
860    Ok(out)
861}
862
863/// Parse a whole Jelly file held in memory (`"jelly"` is binary, so it has no
864/// text entry point). The streaming path is [`stream_reader`] with `"jelly"`;
865/// this is for callers that already hold the bytes, such as the browser engine.
866pub fn parse_jelly_bytes(bytes: &[u8]) -> Result<Vec<RawQuad>, IngestError> {
867    parse_jelly_bytes_audited(bytes, None)
868}
869
870/// [`parse_jelly_bytes`] with an optional [`IriAudit`].
871pub fn parse_jelly_bytes_audited(
872    bytes: &[u8],
873    audit: Option<&mut IriAudit>,
874) -> Result<Vec<RawQuad>, IngestError> {
875    let mut out = Vec::new();
876    jelly::stream(bytes, audit, &mut |q| out.push(q))?;
877    Ok(out)
878}
879
880/// Take one term from the front of `s`, returning `(term, remainder)`.
881pub(crate) fn take_term(s: &str) -> Option<(String, &str)> {
882    let bytes = s.as_bytes();
883    let first = *bytes.first()?;
884    match first {
885        // A quoted triple / triple term, in EITHER surface — the inner terms are
886        // themselves terms (so this recurses; nesting works). `<<` starts with `<`
887        // and an IRI scan would stop at the first inner `>`, so it must be handled
888        // before the plain-IRI case.
889        //   * RDF-star:  `<< subject predicate object >>`
890        //   * RDF 1.2:   `<<( subject predicate object )>>`  (triple term)
891        // Both re-emit the SAME canonical token `<<s p o>>`, so a file written in
892        // either surface — and a query written either way — dedupe and match. This
893        // makes RDF 1.2 N-Triples interoperable with the RDF-star we already store,
894        // with no format change and no dependency swap.
895        b'<' if bytes.get(1) == Some(&b'<') => {
896            // Distinguish `<<(` (RDF 1.2) from `<<` (RDF-star) by the char after `<<`.
897            let (inner, rdf12) = match s[2..].strip_prefix('(') {
898                Some(after) => (after.trim_start(), true),
899                None => (s[2..].trim_start(), false),
900            };
901            let (subj, r) = take_term(inner)?;
902            let (pred, r) = take_term(r.trim_start())?;
903            let (obj, r) = take_term(r.trim_start())?;
904            let r = r.trim_start();
905            let rest = if rdf12 {
906                r.strip_prefix(")>>")?
907            } else {
908                r.strip_prefix(">>")?
909            };
910            // Canonical surface = oxrdf's `Triple` Display: `<<s p o>>` (tight
911            // brackets, single spaces between components) — the same token for both
912            // input surfaces, so N-Triples-star, Turtle-star, and RDF 1.2 triple
913            // terms all resolve to one dictionary entry.
914            Some((format!("<<{subj} {pred} {obj}>>"), rest))
915        }
916        b'<' => {
917            // IRI ref: up to the closing '>'.
918            let end = s.find('>')?;
919            Some((s[..=end].to_string(), &s[end + 1..]))
920        }
921        b'_' => {
922            // Blank node: up to whitespace.
923            let end = s.find(char::is_whitespace).unwrap_or(s.len());
924            Some((s[..end].to_string(), &s[end..]))
925        }
926        b'"' => {
927            // Literal: closing unescaped quote, then optional ^^<dt> or @lang.
928            let mut i = 1;
929            let b = s.as_bytes();
930            while i < b.len() {
931                match b[i] {
932                    b'\\' => i += 2, // skip escaped char
933                    b'"' => break,
934                    _ => i += 1,
935                }
936            }
937            if i >= b.len() {
938                return None; // unterminated
939            }
940            let mut end = i + 1; // past closing quote
941            if s[end..].starts_with("^^<") {
942                let close = s[end..].find('>')? + end;
943                // RDF 1.1: `"x"^^xsd:string` IS `"x"`; keep the one canonical
944                // spelling, so the two never become two dictionary terms and
945                // a plain query constant matches either input.
946                if &s[end + 3..close] == "http://www.w3.org/2001/XMLSchema#string" {
947                    return Some((s[..end].to_string(), &s[close + 1..]));
948                }
949                end = close + 1;
950            } else if s[end..].starts_with('@') {
951                // Language tag: '@' then BCP-47 subtags `[a-zA-Z0-9-]+`. Stop at
952                // the first char that can't be part of a tag — normally the
953                // whitespace before the predicate, but ALSO the `>>` that closes
954                // a quoted triple when this literal is its object (`"x"@en>>`),
955                // where there is no separating whitespace.
956                let mut j = end + 1; // past '@'
957                while j < b.len() && (b[j].is_ascii_alphanumeric() || b[j] == b'-') {
958                    j += 1;
959                }
960                end = j;
961            }
962            Some((s[..end].to_string(), &s[end..]))
963        }
964        _ => None,
965    }
966}
967
968/// Split a canonical quoted-triple token `<<s p o>>` (RDF-star) into its three
969/// component term tokens, or `None` if `t` is not a quoted triple. Reuses
970/// [`take_term`] for term-boundary scanning, so nested quoting parses correctly.
971/// The inverse of the `<<…>>` construction; used by the SUBJECT/PREDICATE/OBJECT
972/// SPARQL-star builtins.
973pub(crate) fn quoted_triple_parts(t: &str) -> Option<(String, String, String)> {
974    let inner = t.strip_prefix("<<")?.strip_suffix(">>")?.trim();
975    let (s, r) = take_term(inner)?;
976    let (p, r) = take_term(r.trim_start())?;
977    let (o, r) = take_term(r.trim_start())?;
978    if !r.trim().is_empty() {
979        return None;
980    }
981    Some((s, p, o))
982}
983
984/// Counts describing an assembled file, for status lines and UIs.
985///
986/// In the [`BuildStats`] a metadata callback receives, `statements` and
987/// `default_triples` are the counts **ingested** — the input multiset, before
988/// the index deduplicates it. In the `BuildStats` an assemble function
989/// *returns*, they are the counts actually **written** (see
990/// [`FinalCounts`]), so a status line reports what the file holds. Use
991/// [`FinalCounts`] rather than these when a payload must agree with the header.
992#[derive(Debug, Clone, Copy)]
993pub struct BuildStats {
994    /// Total statements, across the default graph and every named one.
995    pub statements: usize,
996    /// Statements that landed in the default graph.
997    pub default_triples: usize,
998    /// How many named graphs the input mentioned; one index is written per graph.
999    pub named_graphs: usize,
1000    /// Distinct terms in the shared dictionary.
1001    pub terms: usize,
1002    /// Levels in the community pyramid, or `0` when the build skipped it.
1003    pub pyramid_levels: u16,
1004    /// Whether the file got a label index. The record says so even when the
1005    /// caller (a binding) set no option for it.
1006    pub label_index: bool,
1007}
1008
1009/// How a build stores the **named graphs** (the default graph is unaffected).
1010#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1011pub enum GraphLayout {
1012    /// One permutation index per graph, in the NAMED_GRAPHS section (kind 5):
1013    /// the graph IRI as text, then a full container per graph. What every rete
1014    /// before 0.4 writes and reads.
1015    #[default]
1016    PerGraph,
1017    /// The **quad index** (kind 10, SPEC.md §6.5): every named-graph quad
1018    /// stored once, the graph as a fourth column of graph-leading permutations
1019    /// in shared tiles, behind a sorted graph directory; graph names become
1020    /// dictionary terms. Far smaller and randomly addressable when graphs are
1021    /// many and small. Readers before 0.4.0 refuse these files (loudly — see
1022    /// `docs/compatibility.md`). Needs the 3- or 6-permutation set.
1023    Quad,
1024}
1025
1026/// One pass's named-graph quads, in the layout being built.
1027enum NamedQuads {
1028    PerGraph(std::collections::BTreeMap<String, Vec<(u32, u32, u32)>>),
1029    Quad {
1030        /// `(graph subject id, s, p, o)`.
1031        quads: Vec<[u32; 4]>,
1032        /// Distinct graphs seen (a bitset over subject ids).
1033        seen: Vec<u64>,
1034        graphs: usize,
1035        /// The last graph name and its id: N-Quads dumps keep a graph's
1036        /// statements together, so most quads skip the dictionary lookup.
1037        last: Option<(String, u32)>,
1038    },
1039}
1040
1041impl NamedQuads {
1042    fn new(layout: GraphLayout, dict: &Dictionary) -> Self {
1043        match layout {
1044            GraphLayout::PerGraph => NamedQuads::PerGraph(Default::default()),
1045            GraphLayout::Quad => NamedQuads::Quad {
1046                quads: Vec::new(),
1047                seen: vec![
1048                    0u64;
1049                    (dict.shared_count() as usize + dict.subject_only_count() as usize) / 64
1050                        + 2
1051                ],
1052                graphs: 0,
1053                last: None,
1054            },
1055        }
1056    }
1057
1058    fn push(&mut self, dict: &Dictionary, graph: String, t: (u32, u32, u32)) {
1059        match self {
1060            NamedQuads::PerGraph(m) => m.entry(graph).or_default().push(t),
1061            NamedQuads::Quad {
1062                quads,
1063                seen,
1064                graphs,
1065                last,
1066            } => {
1067                let g = match last {
1068                    Some((name, id)) if *name == graph => *id,
1069                    _ => {
1070                        let id = dict.subject_id(&graph).expect("observed graph name");
1071                        *last = Some((graph, id));
1072                        id
1073                    }
1074                };
1075                let (w, b) = (g as usize / 64, g % 64);
1076                if let Some(word) = seen.get_mut(w) {
1077                    if *word & (1 << b) == 0 {
1078                        *word |= 1 << b;
1079                        *graphs += 1;
1080                    }
1081                }
1082                quads.push([g, t.0, t.1, t.2]);
1083            }
1084        }
1085    }
1086
1087    fn graph_count(&self) -> usize {
1088        match self {
1089            NamedQuads::PerGraph(m) => m.len(),
1090            NamedQuads::Quad { graphs, .. } => *graphs,
1091        }
1092    }
1093
1094    fn quad_count(&self) -> usize {
1095        match self {
1096            NamedQuads::PerGraph(m) => m.values().map(Vec::len).sum(),
1097            NamedQuads::Quad { quads, .. } => quads.len(),
1098        }
1099    }
1100}
1101
1102/// The **deduplicated** counts of the file being written — what the header
1103/// records, and therefore what any embedded metadata has to report.
1104///
1105/// The input multiset is not the file: every permutation index sorts and
1106/// deduplicates, so a harvest that pages with overlapping windows writes fewer
1107/// quads than it ingested. These counts are known only once the indexes exist,
1108/// which is why a payload that carries them is derived in two stages (see
1109/// [`IntoMetadata`]).
1110#[derive(Debug, Clone, Copy)]
1111pub struct FinalCounts {
1112    /// Unique triples in the default graph.
1113    pub default_triples: u64,
1114    /// Unique quads across the default graph and every named graph — exactly
1115    /// the header's `quad_count`.
1116    pub quads: u64,
1117}
1118
1119/// What a build's metadata callback may return.
1120///
1121/// A `Vec<u8>` is the payload verbatim — the common case (no metadata, or a
1122/// payload that carries no counts). [`DeferredMetadata`] is a payload that
1123/// *does* carry counts: the callback derives everything it needs while the
1124/// source quads are still resident, and returns a finalizer that the writer
1125/// calls with the [`FinalCounts`], which only exist once the indexes have
1126/// deduplicated the input. Without that second stage a card can only report the
1127/// ingested count, which over-reports any input containing duplicates.
1128pub trait IntoMetadata {
1129    /// Produce the metadata section payload for a file with these counts.
1130    fn into_metadata(self, counts: FinalCounts) -> Vec<u8>;
1131}
1132
1133impl IntoMetadata for Vec<u8> {
1134    fn into_metadata(self, _counts: FinalCounts) -> Vec<u8> {
1135        self
1136    }
1137}
1138
1139/// A metadata payload whose final form needs the written (deduplicated) counts.
1140/// See [`IntoMetadata`].
1141pub struct DeferredMetadata(Box<dyn FnOnce(FinalCounts) -> Vec<u8>>);
1142
1143impl DeferredMetadata {
1144    /// Defer `f` until the indexes are built and the true counts are known.
1145    pub fn new(f: impl FnOnce(FinalCounts) -> Vec<u8> + 'static) -> Self {
1146        DeferredMetadata(Box::new(f))
1147    }
1148
1149    /// No metadata section — byte-identical to a metadata-free build. Lets one
1150    /// `match` arm opt out while the other defers, without a type mismatch.
1151    pub fn none() -> Self {
1152        DeferredMetadata::new(|_| Vec::new())
1153    }
1154}
1155
1156impl IntoMetadata for DeferredMetadata {
1157    fn into_metadata(self, counts: FinalCounts) -> Vec<u8> {
1158        (self.0)(counts)
1159    }
1160}
1161
1162/// [`assemble_dataset`] for the writers that finish as format `0x06` (the
1163/// bindings and the browser): named graphs go in the quad index, the only
1164/// layout `0x06` has. Everything else is `assemble_dataset`'s.
1165pub fn assemble_dataset_quad(quads: Vec<RawQuad>, metadata: &[u8]) -> (Vec<u8>, BuildStats) {
1166    let blob = metadata.to_vec();
1167    assemble_dataset_with_quad(quads, move |_, _| blob)
1168}
1169
1170/// [`assemble_dataset_with`] with the named graphs in the quad index.
1171pub fn assemble_dataset_with_quad<M: IntoMetadata>(
1172    quads: Vec<RawQuad>,
1173    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1174) -> (Vec<u8>, BuildStats) {
1175    assemble_dataset_with_opts_algo_quad(quads, true, false, None, PyramidAlgo::Louvain, metadata)
1176}
1177
1178/// [`assemble_dataset_with_opts_algo`] for the `0x06` writers, with **the layout
1179/// `rete build --label-index` writes**: the named graphs in the quad index, the
1180/// pyramid block carrying only the community material, the SCHEMA section and
1181/// the label index. The browser engine and the Python, R and Java clients all
1182/// assemble through this, so one input and one set of options is one content
1183/// hash whichever front end built it (the browser and Java, having no zstd
1184/// encoder, write their sections uncompressed and so differ in bytes). The label
1185/// index is on because label prefix search is what a graph built in memory is
1186/// opened to do (the playground, the MCP server), and the legacy label block it
1187/// replaces is gone from the community-only pyramid block.
1188pub fn assemble_dataset_with_opts_algo_quad<M: IntoMetadata>(
1189    quads: Vec<RawQuad>,
1190    with_pyramid: bool,
1191    with_text_index: impl crate::text_index::IntoTextIndex,
1192    type_override: Option<&str>,
1193    algo: PyramidAlgo,
1194    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1195) -> (Vec<u8>, BuildStats) {
1196    let text_index = crate::text_index::IntoTextIndex::into_text_index(with_text_index);
1197    let opts = AssembleOptions {
1198        with_pyramid,
1199        with_text_index: text_index.is_some(),
1200        text_index_scope: text_index.as_ref(),
1201        type_override,
1202        algo,
1203        graph_layout: GraphLayout::Quad,
1204        label_index: true,
1205        ..AssembleOptions::sectioned()
1206    };
1207    assemble_dataset_opts(quads, &opts, metadata)
1208}
1209
1210/// Assemble a complete `.rete` file image from parsed quads: one shared
1211/// dictionary, the default-graph index, one index per named graph, and the
1212/// community pyramid. `metadata` is the opaque metadata-section payload (the
1213/// CLI puts a JSON Dataset Card there); pass `&[]` for none — that is
1214/// byte-identical to a metadata-free build.
1215pub fn assemble_dataset(quads: Vec<RawQuad>, metadata: &[u8]) -> (Vec<u8>, BuildStats) {
1216    let blob = metadata.to_vec();
1217    assemble_dataset_with(quads, move |_, _| blob)
1218}
1219
1220/// Like [`assemble_dataset`], but the metadata payload is derived from the
1221/// [`BuildStats`] and the source quads while they are still resident — for
1222/// metadata that describes the graph (the Dataset Card). Returning an empty
1223/// `Vec` is byte-identical to a metadata-free build; return a
1224/// [`DeferredMetadata`] for a payload that must also carry the file's
1225/// deduplicated counts, which exist only once the indexes are built.
1226pub fn assemble_dataset_with<M: IntoMetadata>(
1227    quads: Vec<RawQuad>,
1228    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1229) -> (Vec<u8>, BuildStats) {
1230    assemble_dataset_with_opts(quads, true, false, None, metadata)
1231}
1232
1233/// Like [`assemble_dataset_with`], but `with_pyramid = false` skips the Louvain
1234/// community pyramid entirely — no pyramid section is written (header length 0).
1235/// SPARQL / SHACL / triple / reachability queries don't use the pyramid, so a
1236/// pyramid-less file is fully queryable and markedly smaller (the pyramid is the
1237/// largest section on highly-connected graphs). Only the community / summary /
1238/// progressive paths need it.
1239pub fn assemble_dataset_with_opts<M: IntoMetadata>(
1240    quads: Vec<RawQuad>,
1241    with_pyramid: bool,
1242    with_text_index: impl crate::text_index::IntoTextIndex,
1243    type_override: Option<&str>,
1244    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1245) -> (Vec<u8>, BuildStats) {
1246    assemble_dataset_with_opts_algo(
1247        quads,
1248        with_pyramid,
1249        with_text_index,
1250        type_override,
1251        PyramidAlgo::Louvain,
1252        metadata,
1253    )
1254}
1255
1256/// Like [`assemble_dataset_with_opts`], but selects the community [`PyramidAlgo`]
1257/// (the in-memory build path for `rete build --pyramid-algo …`).
1258#[allow(clippy::too_many_arguments)]
1259pub fn assemble_dataset_with_opts_algo<M: IntoMetadata>(
1260    quads: Vec<RawQuad>,
1261    with_pyramid: bool,
1262    with_text_index: impl crate::text_index::IntoTextIndex,
1263    type_override: Option<&str>,
1264    algo: PyramidAlgo,
1265    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1266) -> (Vec<u8>, BuildStats) {
1267    assemble_dataset_with_perms(
1268        quads,
1269        with_pyramid,
1270        with_text_index,
1271        type_override,
1272        algo,
1273        crate::index::PermSet::ALL,
1274        metadata,
1275    )
1276}
1277
1278/// Like [`assemble_dataset_with_opts_algo`], but writes only the permutations in
1279/// `perms` (`rete build --permutations 3`). Every other byte of the file is
1280/// unchanged; [`crate::index::PermSet::ALL`] reproduces the default build
1281/// exactly, down to the header byte.
1282#[allow(clippy::too_many_arguments)]
1283pub fn assemble_dataset_with_perms<M: IntoMetadata>(
1284    quads: Vec<RawQuad>,
1285    with_pyramid: bool,
1286    with_text_index: impl crate::text_index::IntoTextIndex,
1287    type_override: Option<&str>,
1288    algo: PyramidAlgo,
1289    perms: crate::index::PermSet,
1290    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1291) -> (Vec<u8>, BuildStats) {
1292    assemble_dataset_with_layout(
1293        quads,
1294        with_pyramid,
1295        with_text_index,
1296        type_override,
1297        algo,
1298        perms,
1299        GraphLayout::PerGraph,
1300        metadata,
1301    )
1302}
1303
1304/// Like [`assemble_dataset_with_perms`], storing the named graphs in `graphs`
1305/// layout. [`GraphLayout::PerGraph`] reproduces it exactly.
1306///
1307/// # Panics
1308///
1309/// With [`GraphLayout::Quad`] and a permutation set other than
1310/// [`PermSet::CORE`](crate::index::PermSet::CORE) or
1311/// [`PermSet::ALL`](crate::index::PermSet::ALL), when the input has named graphs.
1312#[allow(clippy::too_many_arguments)]
1313pub fn assemble_dataset_with_layout<M: IntoMetadata>(
1314    quads: Vec<RawQuad>,
1315    with_pyramid: bool,
1316    with_text_index: impl crate::text_index::IntoTextIndex,
1317    type_override: Option<&str>,
1318    algo: PyramidAlgo,
1319    perms: crate::index::PermSet,
1320    graphs: GraphLayout,
1321    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1322) -> (Vec<u8>, BuildStats) {
1323    let text_index = crate::text_index::IntoTextIndex::into_text_index(with_text_index);
1324    let opts = AssembleOptions {
1325        with_pyramid,
1326        with_text_index: text_index.is_some(),
1327        text_index_scope: text_index.as_ref(),
1328        type_override,
1329        algo,
1330        perms,
1331        graph_layout: graphs,
1332        ..AssembleOptions::legacy()
1333    };
1334    assemble_dataset_opts(quads, &opts, metadata)
1335}
1336
1337/// Everything an in-memory build decides besides its input. The fields the
1338/// older `assemble_*` entry points take positionally, plus the sections added
1339/// after the generation-1 freeze.
1340#[derive(Debug, Clone, Copy)]
1341pub struct AssembleOptions<'a> {
1342    /// Build the community pyramid (kind 4).
1343    pub with_pyramid: bool,
1344    /// Build the full-text index (kind 6, or kind 11 when its scope leaves
1345    /// out some predicates).
1346    pub with_text_index: bool,
1347    /// The scope of that index (`--text-index-graphs`, `--text-index-include`,
1348    /// `--text-index-exclude`); `None` is the default scope, the only one
1349    /// before 0.4.0: the default graph, every predicate.
1350    pub text_index_scope: Option<&'a crate::text_index::TextIndexScope>,
1351    /// The type predicate for the schema (e.g. `wdt:P31`); `rdf:type` if `None`.
1352    pub type_override: Option<&'a str>,
1353    /// The community algorithm of the pyramid.
1354    pub algo: PyramidAlgo,
1355    /// Which index permutations to write.
1356    pub perms: crate::index::PermSet,
1357    /// How the named graphs are stored: one index per graph (kind 5) or the
1358    /// quad index (kind 10, `rete build --named-graphs quad`).
1359    pub graph_layout: GraphLayout,
1360    /// What the pyramid-meta carries besides the summary.
1361    pub pyramid_layout: crate::file::PyramidLayout,
1362    /// Write the LABEL_INDEX section (kind 8, `rete build --label-index`).
1363    pub label_index: bool,
1364    /// Write the SCHEMA section (kind 9; `rete build` writes it unless
1365    /// `--no-schema`).
1366    pub schema: bool,
1367}
1368
1369impl AssembleOptions<'_> {
1370    /// What every `assemble_*` entry point wrote before kinds 8 and 9 existed:
1371    /// a Louvain pyramid with the combined pyramid-meta, no text index, all six
1372    /// permutations, one index per named graph, no LABEL_INDEX or SCHEMA
1373    /// section.
1374    pub fn legacy() -> Self {
1375        AssembleOptions {
1376            with_pyramid: true,
1377            with_text_index: false,
1378            text_index_scope: None,
1379            type_override: None,
1380            algo: PyramidAlgo::Louvain,
1381            perms: crate::index::PermSet::ALL,
1382            graph_layout: GraphLayout::PerGraph,
1383            pyramid_layout: crate::file::PyramidLayout::Combined,
1384            label_index: false,
1385            schema: false,
1386        }
1387    }
1388
1389    /// What `rete build` writes: the pyramid-meta carries only the community
1390    /// material, the schema has its own section, and the label index is
1391    /// opt-in.
1392    pub fn sectioned() -> Self {
1393        AssembleOptions {
1394            pyramid_layout: crate::file::PyramidLayout::CommunityOnly,
1395            schema: true,
1396            ..Self::legacy()
1397        }
1398    }
1399}
1400
1401/// Assemble a file image from parsed quads under explicit [`AssembleOptions`].
1402///
1403/// # Panics
1404///
1405/// With [`GraphLayout::Quad`] and a permutation set other than
1406/// [`PermSet::CORE`](crate::index::PermSet::CORE) or
1407/// [`PermSet::ALL`](crate::index::PermSet::ALL), when the input has named graphs.
1408pub fn assemble_dataset_opts<M: IntoMetadata>(
1409    quads: Vec<RawQuad>,
1410    opts: &AssembleOptions<'_>,
1411    metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1412) -> (Vec<u8>, BuildStats) {
1413    use crate::build_progress::{self as progress, Stat, Ticker, Unit};
1414    use crate::build_trace::Phase;
1415    let graphs = opts.graph_layout;
1416    let n = Some(quads.len() as u64);
1417    progress::begin(Phase::Dictionary, n, Unit::Statements);
1418    let mut ticker = Ticker::new();
1419    let mut db = DictionaryBuilder::new();
1420    for (s, p, o, g) in &quads {
1421        db.observe(s, p, o);
1422        if let (GraphLayout::Quad, Some(g)) = (graphs, g) {
1423            db.observe_graph(g);
1424        }
1425        ticker.tick(1);
1426    }
1427    ticker.flush();
1428    let dict = db.build();
1429    crate::build_trace::lap(Phase::Dictionary);
1430    progress::set_stat(Stat::Terms, dict.term_count().into());
1431
1432    progress::begin(Phase::Encode, n, Unit::Statements);
1433    let mut default_triples = Vec::new();
1434    let mut named = NamedQuads::new(graphs, &dict);
1435    for (s, p, o, g) in &quads {
1436        let t = dict.encode(s, p, o).expect("observed term");
1437        match g {
1438            None => default_triples.push(t),
1439            Some(graph) => named.push(&dict, graph.clone(), t),
1440        }
1441        ticker.tick(1);
1442    }
1443    drop(ticker);
1444    crate::build_trace::lap(Phase::Encode);
1445
1446    // Derive the metadata blob (the Dataset Card) from the raw quads NOW, while
1447    // they are resident — then DROP them before the memory-heavy pyramid + index
1448    // phases. On a big build the string quads are the largest working set (every
1449    // term an owned String, heavily duplicated) and are fully redundant with the
1450    // dictionary + id-triples once encoded, so freeing them here cuts peak RAM by
1451    // their whole size. `pyramid_levels` is not known yet (0 in the callback);
1452    // it is filled into the returned `stats` by `finish_assembly`, and no metadata
1453    // callback depends on it (the card derives from the quads + term/graph counts).
1454    let stats = BuildStats {
1455        statements: quads.len(),
1456        default_triples: default_triples.len(),
1457        named_graphs: named.graph_count(),
1458        terms: dict.term_count() as usize,
1459        pyramid_levels: 0,
1460        label_index: opts.label_index,
1461    };
1462    let pending = metadata(&stats, &quads);
1463    crate::build_trace::lap(crate::build_trace::Phase::CardDerive);
1464    drop(quads);
1465
1466    finish_assembly(dict, default_triples, named, opts, pending, stats)
1467}
1468
1469/// **Two-pass, low-RAM** assembly: build a `.rete` by **streaming** the input(s)
1470/// twice instead of holding every parsed quad in memory. `stream` is invoked
1471/// **twice** and MUST replay the exact same quads in the same order each time —
1472/// pass 1 observes every term into the dictionary; pass 2 encodes them to
1473/// id-triples. The raw string quads (every term an owned String, heavily
1474/// duplicated — by far the largest working set on a big graph) are **never
1475/// collected**, so peak RAM is bounded by the dictionary + id-triples + index
1476/// rather than the string-quad multiset. Output is **byte-identical** to
1477/// [`assemble_dataset_with_opts`] on the same quads (same dictionary, same
1478/// id-triples in file order, same downstream pipeline).
1479///
1480/// The metadata callback derives the Dataset Card from the built dictionary +
1481/// default-graph id-triples (resolving terms through the dictionary), since the
1482/// raw quads were never retained. `stream` propagates parse/IO errors.
1483pub fn assemble_dataset_streaming<S, M: IntoMetadata>(
1484    stream: S,
1485    with_pyramid: bool,
1486    with_text_index: impl crate::text_index::IntoTextIndex,
1487    type_override: Option<&str>,
1488    metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1489) -> Result<(Vec<u8>, BuildStats), IngestError>
1490where
1491    S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1492{
1493    assemble_dataset_streaming_algo(
1494        stream,
1495        with_pyramid,
1496        with_text_index,
1497        type_override,
1498        PyramidAlgo::Louvain,
1499        metadata,
1500    )
1501}
1502
1503/// Like [`assemble_dataset_streaming`], but selects the community [`PyramidAlgo`]
1504/// (the streaming, low-RAM build path for `rete build --pyramid-algo …`).
1505#[allow(clippy::too_many_arguments)]
1506pub fn assemble_dataset_streaming_algo<S, M: IntoMetadata>(
1507    stream: S,
1508    with_pyramid: bool,
1509    with_text_index: impl crate::text_index::IntoTextIndex,
1510    type_override: Option<&str>,
1511    algo: PyramidAlgo,
1512    metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1513) -> Result<(Vec<u8>, BuildStats), IngestError>
1514where
1515    S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1516{
1517    assemble_dataset_streaming_with_perms(
1518        stream,
1519        with_pyramid,
1520        with_text_index,
1521        type_override,
1522        algo,
1523        crate::index::PermSet::ALL,
1524        metadata,
1525    )
1526}
1527
1528/// Like [`assemble_dataset_streaming_algo`], but writes only the permutations in
1529/// `perms` — the two-pass low-RAM twin of [`assemble_dataset_with_perms`].
1530#[allow(clippy::too_many_arguments)]
1531pub fn assemble_dataset_streaming_with_perms<S, M: IntoMetadata>(
1532    stream: S,
1533    with_pyramid: bool,
1534    with_text_index: impl crate::text_index::IntoTextIndex,
1535    type_override: Option<&str>,
1536    algo: PyramidAlgo,
1537    perms: crate::index::PermSet,
1538    metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1539) -> Result<(Vec<u8>, BuildStats), IngestError>
1540where
1541    S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1542{
1543    assemble_dataset_streaming_with_layout(
1544        stream,
1545        with_pyramid,
1546        with_text_index,
1547        type_override,
1548        algo,
1549        perms,
1550        GraphLayout::PerGraph,
1551        metadata,
1552    )
1553}
1554
1555/// Like [`assemble_dataset_streaming_with_perms`], storing the named graphs in
1556/// `graphs` layout — the two-pass low-RAM twin of
1557/// [`assemble_dataset_with_layout`]. [`GraphLayout::Quad`] with a permutation
1558/// set other than the 3- or 6-permutation one is an error.
1559#[allow(clippy::too_many_arguments)]
1560pub fn assemble_dataset_streaming_with_layout<S, M: IntoMetadata>(
1561    stream: S,
1562    with_pyramid: bool,
1563    with_text_index: impl crate::text_index::IntoTextIndex,
1564    type_override: Option<&str>,
1565    algo: PyramidAlgo,
1566    perms: crate::index::PermSet,
1567    graphs: GraphLayout,
1568    metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1569) -> Result<(Vec<u8>, BuildStats), IngestError>
1570where
1571    S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1572{
1573    let text_index = crate::text_index::IntoTextIndex::into_text_index(with_text_index);
1574    let opts = AssembleOptions {
1575        with_pyramid,
1576        with_text_index: text_index.is_some(),
1577        text_index_scope: text_index.as_ref(),
1578        type_override,
1579        algo,
1580        perms,
1581        graph_layout: graphs,
1582        ..AssembleOptions::legacy()
1583    };
1584    assemble_dataset_streaming_opts(stream, &opts, metadata)
1585}
1586
1587/// The two-pass low-RAM twin of [`assemble_dataset_opts`]. A quad-index layout
1588/// with a permutation set other than the 3- or 6-permutation one is an error.
1589pub fn assemble_dataset_streaming_opts<S, M: IntoMetadata>(
1590    mut stream: S,
1591    opts: &AssembleOptions<'_>,
1592    metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1593) -> Result<(Vec<u8>, BuildStats), IngestError>
1594where
1595    S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1596{
1597    let graphs = opts.graph_layout;
1598    if graphs == GraphLayout::Quad
1599        && crate::build_record::writes_legacy()
1600        && crate::quad::check_perms(opts.perms).is_err()
1601    {
1602        return Err(IngestError::Io(
1603            "the quad index needs the 3- or 6-permutation set".into(),
1604        ));
1605    }
1606    // Pass 1: observe every term (the dictionary dedups; the string quads are
1607    // freed line by line and never collected). With the quad index, graph
1608    // names are terms too. The stream sets each pass's progress total (the
1609    // input bytes it will read).
1610    use crate::build_progress::{self as progress, Stat, Unit};
1611    use crate::build_trace::Phase;
1612    progress::begin(Phase::ParseDictionary, None, Unit::None);
1613    let mut db = DictionaryBuilder::new();
1614    stream(&mut |(s, p, o, g)| {
1615        db.observe(&s, &p, &o);
1616        if let (GraphLayout::Quad, Some(g)) = (graphs, g) {
1617            db.observe_graph(&g);
1618        }
1619    })?;
1620    let dict = db.build();
1621    crate::build_trace::lap(Phase::ParseDictionary);
1622    progress::set_stat(Stat::Terms, dict.term_count().into());
1623
1624    // Pass 2: encode each quad to an id-triple, bucketing named graphs.
1625    progress::begin(Phase::ParseEncode, None, Unit::None);
1626    let mut default_triples: Vec<(u32, u32, u32)> = Vec::new();
1627    let mut named = NamedQuads::new(graphs, &dict);
1628    stream(&mut |(s, p, o, g)| {
1629        let t = dict.encode(&s, &p, &o).expect("observed term");
1630        match g {
1631            None => default_triples.push(t),
1632            Some(graph) => named.push(&dict, graph, t),
1633        }
1634    })?;
1635    crate::build_trace::lap(crate::build_trace::Phase::ParseEncode);
1636
1637    let statements = default_triples.len() + named.quad_count();
1638    let stats = BuildStats {
1639        statements,
1640        default_triples: default_triples.len(),
1641        named_graphs: named.graph_count(),
1642        terms: dict.term_count() as usize,
1643        pyramid_levels: 0,
1644        label_index: opts.label_index,
1645    };
1646    let pending = metadata(&stats, &dict, &default_triples);
1647    crate::build_trace::lap(crate::build_trace::Phase::CardDerive);
1648    Ok(finish_assembly(
1649        dict,
1650        default_triples,
1651        named,
1652        opts,
1653        pending,
1654        stats,
1655    ))
1656}
1657
1658/// The shared tail of every build path: from a finished dictionary + encoded
1659/// id-triples (default graph + named), build the community pyramid, the optional
1660/// full-text index, the permutation indexes, and serialize the file image. Takes
1661/// the id-triples **by value** so they are freed as the permutations consume them.
1662fn finish_assembly<M: IntoMetadata>(
1663    dict: Dictionary,
1664    mut default_triples: Vec<(u32, u32, u32)>,
1665    named: NamedQuads,
1666    opts: &AssembleOptions<'_>,
1667    pending: M,
1668    mut stats: BuildStats,
1669) -> (Vec<u8>, BuildStats) {
1670    use crate::build_progress::{self as progress, Unit};
1671    use crate::build_trace::Phase;
1672    let has_named = named.graph_count() > 0;
1673    let perms = opts.perms;
1674
1675    // The pyramid and the full-text index are built FIRST, while the dictionary and
1676    // the default id-triples are both still resident — both need the two together.
1677    // The pyramid sees the triples in input order (Louvain is order-sensitive).
1678    let (meta, levels) = if opts.with_pyramid {
1679        progress::begin(Phase::Pyramid, None, Unit::None);
1680        crate::file::build_pyramid_meta_layout(
1681            &dict,
1682            &default_triples,
1683            DEFAULT_TILE_BUDGET,
1684            opts.type_override,
1685            opts.algo,
1686            opts.pyramid_layout,
1687        )
1688    } else {
1689        (Vec::new(), 0)
1690    };
1691    stats.pyramid_levels = levels;
1692    if opts.with_pyramid {
1693        crate::build_trace::lap(crate::build_trace::Phase::Pyramid);
1694    }
1695
1696    // The full-text index over the literals of the requested scope: the
1697    // default graph's (the only scope before 0.4.0), optionally every named
1698    // graph's too, optionally only some predicates'.
1699    let with_text_index = opts
1700        .with_text_index
1701        .then(|| opts.text_index_scope.cloned().unwrap_or_default());
1702    // The named graphs' triples, for an index whose scope covers them: per
1703    // graph, or (in the quad layout) the quads' `(s, p, o)`, copied only when
1704    // the scope needs them; the index does not care which graph a word came
1705    // from.
1706    let quad_spo: Vec<(u32, u32, u32)>;
1707    let named_triples: Vec<&[(u32, u32, u32)]> = match &named {
1708        NamedQuads::PerGraph(m) => m.values().map(Vec::as_slice).collect(),
1709        NamedQuads::Quad { quads, .. } => {
1710            quad_spo = if with_text_index
1711                .as_ref()
1712                .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs)
1713            {
1714                quads.iter().map(|q| (q[1], q[2], q[3])).collect()
1715            } else {
1716                Vec::new()
1717            };
1718            vec![quad_spo.as_slice()]
1719        }
1720    };
1721    if with_text_index.is_some() {
1722        progress::begin(Phase::TextIndex, None, Unit::None);
1723    }
1724    let text_index = match &with_text_index {
1725        Some(scope) => crate::text_index::build_section(
1726            &dict,
1727            scope,
1728            &default_triples,
1729            named_triples.into_iter(),
1730            crate::file::writer_codec(),
1731        ),
1732        None => Vec::new(),
1733    };
1734    if with_text_index.is_some() {
1735        crate::build_trace::lap(crate::build_trace::Phase::TextIndex);
1736    }
1737    let text_index_partial = with_text_index
1738        .as_ref()
1739        .is_some_and(crate::text_index::TextIndexScope::restricts_predicates);
1740    let text_index_named = with_text_index
1741        .as_ref()
1742        .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs);
1743
1744    // The LABEL_INDEX and SCHEMA sections are computed over the DEDUPLICATED
1745    // default graph in SPO order (and the schema's second pass in POS order),
1746    // the order every index build re-sorts into anyway — which is what lets
1747    // the memory-bounded build, reading its sorted permutation streams, write
1748    // the very same bytes. Both need the dictionary, so they run before it is
1749    // dropped. Without either section the triples stay in input order.
1750    let mut label_section = Vec::new();
1751    let mut schema_section = Vec::new();
1752    if opts.label_index || opts.schema {
1753        progress::begin(Phase::LabelsSchema, None, Unit::None);
1754        crate::schema_section::sort_dedup_spo(&mut default_triples);
1755        if opts.label_index {
1756            let entries = crate::label_index::entries_from_sorted_triples(&dict, &default_triples);
1757            label_section =
1758                crate::label_index::encode_section(&entries, crate::file::writer_codec());
1759        }
1760        if opts.schema {
1761            schema_section = crate::schema_section::compute_in_memory(
1762                &dict,
1763                &mut default_triples,
1764                opts.type_override,
1765            )
1766            .encode();
1767        }
1768        crate::build_trace::lap(crate::build_trace::Phase::LabelsSchema);
1769    }
1770
1771    // From here the dictionary is only needed for its own serialized bytes. Encode
1772    // it, capture the header term count, then DROP it before building the
1773    // permutation indexes — which work purely on id-triples and never touch the
1774    // dictionary. On a large graph this frees the single biggest resident structure
1775    // (the dictionary) right when the index sort needs the headroom. The output is
1776    // byte-for-byte identical to serializing the dictionary inline.
1777    let codec = crate::file::writer_codec();
1778    progress::begin(Phase::DictionarySection, None, Unit::None);
1779    let dict_container = crate::file::encode_dict_container(&dict, codec);
1780    let term_count = dict.term_count() as u64;
1781    let has_quoted_triples = dict.has_quoted_triples();
1782    let term_order = dict.order();
1783    drop(dict);
1784    crate::build_trace::lap(crate::build_trace::Phase::DictionarySection);
1785
1786    // Build each graph's six permutations one-at-a-time on a large graph (a single
1787    // permuted copy resident at a time, each sort still parallel) instead of all
1788    // six concurrently; below the threshold the faster all-permutations-parallel
1789    // build is used (its 6x transient copy is negligible there). `build_seq` is
1790    // byte-identical to `build`.
1791    // Every graph's triples, once per permutation (the quad index's quads
1792    // once, as it finishes).
1793    let (per_graph, quad_rows): (u64, u64) = match &named {
1794        NamedQuads::PerGraph(m) => (m.values().map(|t| t.len() as u64).sum(), 0),
1795        NamedQuads::Quad { quads, .. } => (0, quads.len() as u64),
1796    };
1797    let rows = (default_triples.len() as u64 + per_graph) * perms.len() as u64 + quad_rows;
1798    progress::begin(Phase::IndexPermutations, Some(rows), Unit::Records);
1799    let build_index = |triples: Vec<(u32, u32, u32)>| -> crate::GraphIndex {
1800        let n = triples.len();
1801        let b = GraphIndexBuilder::from_triples(triples)
1802            .with_perms(perms)
1803            .with_progress();
1804        if n > LOWMEM_TRIPLE_THRESHOLD {
1805            b.build_seq()
1806        } else {
1807            b.build()
1808        }
1809    };
1810    let def = build_index(default_triples);
1811    // The named graphs: one index per graph, or one quad index for all of them
1812    // (no per-graph index is ever built on that path).
1813    let (named_indexes, quad_index): (Vec<(String, crate::GraphIndex)>, _) = match named {
1814        NamedQuads::PerGraph(named) => (
1815            named
1816                .into_iter()
1817                .map(|(g, ts)| (g, build_index(ts)))
1818                .collect(),
1819            None,
1820        ),
1821        NamedQuads::Quad { quads, .. } => {
1822            let n = quads.len() as u64;
1823            // `0x07` derives every other order from GSPO, so the in-RAM image
1824            // carries that one; the legacy `0x05` output (a test hook) keeps
1825            // the file's whole permutation set.
1826            let q = if crate::build_record::writes_legacy() {
1827                crate::quad::encode_quad_index(quads, perms, codec)
1828            } else {
1829                crate::quad::encode_quad_gspo(quads, codec)
1830            }
1831            .expect("the quad index needs the 3- or 6-permutation set");
1832            progress::advance(n);
1833            (Vec::new(), Some(q))
1834        }
1835    };
1836    let named_quads = match &quad_index {
1837        Some(q) => q.quad_count,
1838        None => named_indexes
1839            .iter()
1840            .map(|(_, ix)| ix.triple_count() as u64)
1841            .sum::<u64>(),
1842    };
1843    crate::build_trace::lap(crate::build_trace::Phase::IndexPermutations);
1844
1845    // Only now is the DEDUPLICATED size of the file known: every index sorts and
1846    // dedups, so an input containing duplicate statements (a harvest that pages
1847    // with overlapping windows, a merge of overlapping dumps) writes fewer quads
1848    // than it ingested. `write_dataset_from_parts` stamps exactly this sum into
1849    // the header, so a metadata payload finalized with it cannot disagree with
1850    // the header — which is what an embedded Dataset Card used to do.
1851    let counts = FinalCounts {
1852        default_triples: def.triple_count() as u64,
1853        quads: def.triple_count() as u64 + named_quads,
1854    };
1855    let blob = pending.into_metadata(counts);
1856    progress::begin(Phase::Serialize, None, Unit::None);
1857    // Report what was WRITTEN, not what was read: the returned stats drive the
1858    // CLI's "wrote …: N quads" line, which described the file, not the input.
1859    stats.statements = counts.quads as usize;
1860    stats.default_triples = counts.default_triples as usize;
1861
1862    let bytes = crate::file::write_dataset_from_parts(
1863        &dict_container,
1864        term_count,
1865        &def,
1866        match &quad_index {
1867            Some(q) => crate::file::NamedPart::Quad(q),
1868            None => crate::file::NamedPart::PerGraph(&named_indexes),
1869        },
1870        has_named,
1871        has_quoted_triples,
1872        &meta,
1873        levels,
1874        &blob,
1875        &text_index,
1876        text_index_partial,
1877        text_index_named,
1878        codec,
1879        term_order,
1880    );
1881    // Kinds 8 and 9 go after everything else: they are outside the content
1882    // hash, so appending them moves nothing and changes no hash.
1883    let bytes = if label_section.is_empty() && schema_section.is_empty() {
1884        bytes
1885    } else {
1886        crate::file::append_sections(
1887            &bytes,
1888            &[
1889                (crate::header::SectionKind::LabelIndex, &label_section),
1890                (crate::header::SectionKind::Schema, &schema_section),
1891            ],
1892        )
1893        .expect("appending to an image this function just wrote")
1894    };
1895    crate::build_trace::lap(crate::build_trace::Phase::Serialize);
1896    (bytes, stats)
1897}
1898
1899/// Above this default-graph triple count, the index is built one permutation at a
1900/// time (low peak RAM) instead of all six in parallel. Chosen so typical builds
1901/// keep the faster parallel path while multi-hundred-million-triple graphs stay
1902/// within a bounded memory budget.
1903const LOWMEM_TRIPLE_THRESHOLD: usize = 30_000_000;
1904
1905#[cfg(test)]
1906mod tests {
1907    use super::*;
1908    use crate::Rete;
1909
1910    #[test]
1911    fn parses_iris_bnodes_literals() {
1912        let input = r#"
1913            # a comment
1914            <http://ex/Alice> <http://ex/knows> <http://ex/Bob> .
1915            <http://ex/Alice> <http://ex/age> "30"^^<http://www.w3.org/2001/XMLSchema#integer> .
1916            <http://ex/Bob> <http://ex/label> "Bob"@en .
1917            _:b0 <http://ex/p> "plain" .
1918        "#;
1919        let t = parse(input).unwrap();
1920        assert_eq!(t.len(), 4);
1921        assert_eq!(t[0].0, "<http://ex/Alice>");
1922        assert_eq!(t[0].2, "<http://ex/Bob>");
1923        assert_eq!(t[1].2, "\"30\"^^<http://www.w3.org/2001/XMLSchema#integer>");
1924        assert_eq!(t[2].2, "\"Bob\"@en");
1925        assert_eq!(t[3].0, "_:b0");
1926        assert_eq!(t[3].2, "\"plain\"");
1927    }
1928
1929    #[test]
1930    fn literal_with_spaces_and_escaped_quote() {
1931        let input = r#"<http://ex/s> <http://ex/p> "a \"quoted\" phrase here" ."#;
1932        let t = parse(input).unwrap();
1933        assert_eq!(t.len(), 1);
1934        assert_eq!(t[0].2, r#""a \"quoted\" phrase here""#);
1935    }
1936
1937    /// RDF-star: a quoted triple whose object is a **language-tagged literal**
1938    /// sits directly against the closing `>>` with no separating whitespace
1939    /// (`"name"@fr>>`). The language-tag scan must stop at `>` — a regression
1940    /// guard for the greedy scan-to-whitespace that swallowed `@fr>>` as the tag.
1941    #[test]
1942    fn quoted_triple_langtagged_object() {
1943        let s = r#"<<<http://ex/sp> <http://ex/name> "Hirondelle rustique"@fr>>"#;
1944        let (tok, rest) = take_term(s).unwrap();
1945        assert_eq!(
1946            tok,
1947            r#"<<<http://ex/sp> <http://ex/name> "Hirondelle rustique"@fr>>"#
1948        );
1949        assert_eq!(rest, "");
1950        // and it round-trips through a full annotation line
1951        let line = r#"<<<http://ex/sp> <http://ex/name> "Oreneta vulgar"@ca>> <http://purl.org/dc/terms/source> "Catalogue of Life" ."#;
1952        let t = parse(line).unwrap();
1953        assert_eq!(t.len(), 1);
1954        assert_eq!(
1955            t[0].0,
1956            r#"<<<http://ex/sp> <http://ex/name> "Oreneta vulgar"@ca>>"#
1957        );
1958        assert_eq!(t[0].2, r#""Catalogue of Life""#);
1959        // a plain lang-tagged object still terminates at whitespace
1960        assert_eq!(take_term(r#""x"@pt-BR ."#).unwrap().0, r#""x"@pt-BR"#);
1961    }
1962
1963    /// RDF 1.2 triple-term surface `<<( s p o )>>` parses to the SAME canonical
1964    /// token as the RDF-star `<< s p o >>`, so the two are interoperable (a query
1965    /// written either way matches data ingested either way).
1966    #[test]
1967    fn rdf12_triple_term_same_token_as_rdf_star() {
1968        let rdf12 = r#"<http://ex/r> <http://ex/p> <<( <http://ex/s> <http://ex/q> "o" )>> ."#;
1969        let star = r#"<http://ex/r> <http://ex/p> << <http://ex/s> <http://ex/q> "o" >> ."#;
1970        let a = parse(rdf12).unwrap();
1971        let b = parse(star).unwrap();
1972        assert_eq!(a.len(), 1);
1973        assert_eq!(a[0].2, r#"<<<http://ex/s> <http://ex/q> "o">>"#);
1974        assert_eq!(
1975            a[0].2, b[0].2,
1976            "RDF 1.2 and RDF-star must dedupe to one token"
1977        );
1978        // take_term consumes exactly the triple term (nothing left dangling).
1979        let (tok, rest) =
1980            take_term(r#"<<( <http://ex/s> <http://ex/q> <http://ex/o> )>>"#).unwrap();
1981        assert_eq!(tok, "<<<http://ex/s> <http://ex/q> <http://ex/o>>>");
1982        assert_eq!(rest, "");
1983    }
1984
1985    #[test]
1986    fn rejects_missing_dot() {
1987        assert!(parse("<a> <b> <c>").is_err());
1988    }
1989
1990    #[test]
1991    fn parses_quads_with_and_without_graph() {
1992        let input = "<http://ex/a> <http://ex/p> <http://ex/b> .\n\
1993                     <http://ex/a> <http://ex/p> <http://ex/c> <http://ex/g> .";
1994        let q = parse_quads(input).unwrap();
1995        assert_eq!(q.len(), 2);
1996        assert_eq!(q[0].3, None); // default graph
1997        assert_eq!(q[1].3, Some("<http://ex/g>".to_string()));
1998    }
1999
2000    #[test]
2001    fn parse_reader_matches_text_parse() {
2002        // The streaming reader must produce exactly the same quads as parsing the
2003        // whole text — including blank/comment skipping, a named graph, and CRLF.
2004        let nt = "<http://ex/a> <http://ex/p> <http://ex/b> .\r\n\
2005                  # comment\n\
2006                  \n\
2007                  _:b0 <http://ex/q> \"lit\"@en .\n";
2008        let via_text = parse_statements(nt, "nt").unwrap();
2009        let via_reader = parse_reader(std::io::Cursor::new(nt), "nt", 0).unwrap();
2010        assert_eq!(via_text, via_reader);
2011
2012        let nq = "<http://ex/a> <http://ex/p> <http://ex/b> .\n\
2013                  <http://ex/a> <http://ex/p> <http://ex/c> <http://ex/g> .\n";
2014        assert_eq!(
2015            parse_quads(nq).unwrap(),
2016            parse_reader(std::io::Cursor::new(nq), "nq", 0).unwrap()
2017        );
2018        // Turtle and TriG stream too — that is what lets the external build take a
2019        // multi-gigabyte `.ttl.gz` without expanding it to N-Triples first. Reading
2020        // a structured syntax from a reader must agree with parsing its whole text.
2021        let ttl = "@prefix ex: <http://ex/> .\nex:A ex:knows ex:B , ex:C .\n";
2022        assert_eq!(
2023            parse_statements(ttl, "ttl").unwrap(),
2024            parse_reader(std::io::Cursor::new(ttl), "ttl", 0).unwrap()
2025        );
2026        let trig = "@prefix ex: <http://ex/> .\nex:g { ex:A ex:knows ex:B . }\n";
2027        let streamed = parse_reader(std::io::Cursor::new(trig), "trig", 0).unwrap();
2028        assert_eq!(parse_statements(trig, "trig").unwrap(), streamed);
2029        assert_eq!(streamed[0].3.as_deref(), Some("<http://ex/g>"));
2030        // An unknown format is still a hard error, not a silent guess.
2031        assert!(parse_reader(std::io::Cursor::new(nt), "jsonld", 0).is_err());
2032    }
2033
2034    #[test]
2035    fn parse_statements_dispatches_by_format() {
2036        let nt = "<http://ex/a> <http://ex/p> <http://ex/b> .";
2037        assert_eq!(parse_statements(nt, "nt").unwrap().len(), 1);
2038        let ttl = "@prefix ex: <http://ex/> .\nex:A ex:knows ex:B , ex:C .";
2039        assert_eq!(parse_statements(ttl, "ttl").unwrap().len(), 2);
2040        // N-Triples is a subset of both Turtle and TriG, so the same text parses
2041        // under all three — and lands in the default graph under each.
2042        assert_eq!(parse_statements(nt, "trig").unwrap().len(), 1);
2043        assert!(parse_statements(nt, "trig").unwrap()[0].3.is_none());
2044        assert!(parse_statements(nt, "jsonld").is_err());
2045    }
2046
2047    /// A TriG graph block carries its graph name through as the same canonical
2048    /// token N-Quads would produce — so the two syntaxes agree on dictionary keys
2049    /// and a `.trig` shard federates with a `.nq` one.
2050    #[test]
2051    fn trig_graph_names_match_nquads() {
2052        let trig = "@prefix ex: <http://ex/> .\nex:g { ex:a ex:p ex:b . }\nex:a ex:p ex:c .\n";
2053        let nq = "<http://ex/a> <http://ex/p> <http://ex/b> <http://ex/g> .\n\
2054                  <http://ex/a> <http://ex/p> <http://ex/c> .\n";
2055        let mut from_trig = parse_statements(trig, "trig").unwrap();
2056        let mut from_nq = parse_statements(nq, "nq").unwrap();
2057        from_trig.sort();
2058        from_nq.sort();
2059        assert_eq!(from_trig, from_nq);
2060    }
2061
2062    /// RDF/XML (how most OWL ontologies ship) parses to the same canonical tokens,
2063    /// including the abbreviated typed-node syntax and `rdf:resource` references.
2064    #[test]
2065    fn parses_rdfxml_owl() {
2066        let xml = r#"<?xml version="1.0"?>
2067            <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
2068                     xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
2069                     xmlns:owl="http://www.w3.org/2002/07/owl#">
2070              <owl:Class rdf:about="http://ex/Dog">
2071                <rdfs:subClassOf rdf:resource="http://ex/Animal"/>
2072                <rdfs:label>Dog</rdfs:label>
2073              </owl:Class>
2074            </rdf:RDF>"#;
2075        let triples = parse_rdfxml(xml).unwrap();
2076        // rdf:type owl:Class, rdfs:subClassOf, rdfs:label = 3 triples.
2077        assert_eq!(triples.len(), 3);
2078        assert!(triples.iter().any(|(s, p, o)| s == "<http://ex/Dog>"
2079            && p == "<http://www.w3.org/2000/01/rdf-schema#subClassOf>"
2080            && o == "<http://ex/Animal>"));
2081        // Same data through the format dispatcher (triples → default graph).
2082        assert_eq!(parse_statements(xml, "rdfxml").unwrap().len(), 3);
2083        // Malformed XML is a clear RdfXml error, not a silent empty parse.
2084        assert!(matches!(
2085            parse_statements("<rdf:RDF><not closed", "rdfxml"),
2086            Err(IngestError::RdfXml(_))
2087        ));
2088    }
2089
2090    /// Regression coverage for RUSTSEC-2026-0195: namespace declarations from
2091    /// an untrusted RDF/XML document must not make parsing panic or exhaust the
2092    /// bounded test process.
2093    #[test]
2094    fn rdfxml_namespace_fanout_is_bounded() {
2095        let declarations = (0..2_000)
2096            .map(|i| format!(r#" xmlns:p{i}="http://example.test/{i}/""#))
2097            .collect::<String>();
2098        let xml = format!(
2099            r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"{declarations}/>"#
2100        );
2101        let result = parse_statements(&xml, "rdfxml");
2102        assert!(result.is_ok() || result.unwrap_err().to_string().contains("namespace"));
2103    }
2104
2105    /// Regression coverage for RUSTSEC-2026-0194: duplicate-name checking must
2106    /// complete for a large valid start tag without panicking or exhausting the
2107    /// bounded test process.
2108    #[test]
2109    fn rdfxml_attribute_fanout_completes() {
2110        let attributes = (0..2_000)
2111            .map(|i| format!(r#" p:a{i}="{i}""#))
2112            .collect::<String>();
2113        let xml = format!(
2114            r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:p="http://example.test/"><rdf:Description rdf:about="http://example.test/s"{attributes}/></rdf:RDF>"#
2115        );
2116        assert!(parse_statements(&xml, "rdfxml").is_ok());
2117    }
2118
2119    /// Text in, queryable file image out — the whole in-memory build path the
2120    /// wasm `build()` binding uses, including a named graph.
2121    #[test]
2122    fn assemble_dataset_round_trips() {
2123        let text = "<http://ex/a> <http://ex/knows> <http://ex/b> .\n\
2124                    <http://ex/b> <http://ex/knows> <http://ex/c> .\n\
2125                    <http://ex/a> <http://ex/age> \"30\"^^<http://www.w3.org/2001/XMLSchema#integer> .\n\
2126                    <http://ex/a> <http://ex/p> <http://ex/d> <http://ex/g1> .";
2127        let quads = parse_statements(text, "nq").unwrap();
2128        let (bytes, stats) = assemble_dataset(quads, &[]);
2129        assert_eq!(stats.statements, 4);
2130        assert_eq!(stats.default_triples, 3);
2131        assert_eq!(stats.named_graphs, 1);
2132        assert!(stats.terms >= 7);
2133
2134        let rete = Rete::open(&bytes).unwrap();
2135        assert_eq!(
2136            rete.query(None, Some("<http://ex/knows>"), None).len(),
2137            2,
2138            "default-graph pattern query"
2139        );
2140        assert_eq!(rete.graph_names(), vec!["<http://ex/g1>"]);
2141        let out = crate::eval_query(
2142            &rete,
2143            "SELECT ?x WHERE { ?x <http://ex/knows> ?y . ?y <http://ex/knows> ?z }",
2144        )
2145        .unwrap();
2146        match out {
2147            crate::QueryOutput::Select(_, rows) => assert_eq!(rows.len(), 1),
2148            other => panic!("expected select result, got {other:?}"),
2149        }
2150    }
2151
2152    /// The exact minimal input the playground Build tab uses: two triples, no
2153    /// literals, no rdf:type, no named graph — the smallest graph that still
2154    /// builds a community pyramid. (The wasm `build()` panic this guards was a
2155    /// `std::time::Instant::now()` in the pyramid timing path; native has a clock
2156    /// so this passes here, while the playground harness exercises the wasm path.)
2157    #[test]
2158    fn assemble_minimal_typeless_graph() {
2159        let text = "<http://ex/A> <http://ex/knows> <http://ex/B> .\n\
2160                    <http://ex/B> <http://ex/knows> <http://ex/C> .\n";
2161        let quads = parse_statements(text, "nt").unwrap();
2162        let (bytes, stats) = assemble_dataset(quads, &[]);
2163        assert_eq!(stats.default_triples, 2);
2164        let rete = Rete::open(&bytes).unwrap();
2165        assert_eq!(rete.query(None, Some("<http://ex/knows>"), None).len(), 2);
2166    }
2167}