rete_core/ingest.rs
1//! Ingestion: parse RDF text (N-Triples / N-Quads / Turtle) and assemble a
2//! complete `.rete` file image. Shared by the CLI's `build`/`validate` commands
3//! and the wasm bindings (the playground's in-browser builder).
4//!
5//! The N-Triples/N-Quads reader is line-based and keeps terms as their exact
6//! canonical token strings (`<iri>`, `_:bnode`, `"lit"`, `"lit"^^<dt>`,
7//! `"lit"@lang`) so they double as dictionary keys and so a `query` can match
8//! by the same string. This is not a full RDF 1.1 parser — it covers the
9//! canonical N-Triples surface (and is deliberately tolerant of IRIs a strict
10//! parser would reject), enough for v0 ingestion. Turtle goes through `oxttl`.
11//!
12//! That tolerance is *measured*, not silent: pass an [`IriAudit`] to any of the
13//! `*_audited` entry points and the parse counts every IRI outside the
14//! N-Triples `IRIREF` grammar / RFC 3987 (see [`crate::iri`]) — or, with
15//! `strict`, refuses the input on the first one. `rete build` always audits.
16//!
17//! Assembly degrades with the build features: without the `compression`
18//! feature (e.g. on wasm) sections are written with the `NONE` codec — larger
19//! files, byte-compatible readers.
20
21use crate::{Dictionary, DictionaryBuilder, GraphIndexBuilder, PyramidAlgo, DEFAULT_TILE_BUDGET};
22
23mod jelly;
24mod jsonld;
25
26pub use jsonld::{ContextLoader, JsonLdOptions};
27
28/// A parsed triple as three canonical term tokens.
29pub type RawTriple = (String, String, String);
30
31/// A parsed quad: the triple plus an optional graph term (`None` = default graph).
32pub type RawQuad = (String, String, String, Option<String>);
33
34/// Which **surface** a `<< … >>` in Turtle/TriG input is read as — the input
35/// half of `rete export --quoted-triple-syntax`, same vocabulary, same two
36/// values.
37///
38/// The flag exists because the two standards give one piece of syntax two
39/// meanings, and no amount of looking at the bytes can tell them apart:
40///
41/// | written | under [`RdfStar`](Self::RdfStar) | under [`Rdf12`](Self::Rdf12) |
42/// |--------------------|----------------------------------|---------------------------------------|
43/// | `<< s p o >>` | a **quoted triple**, one term | a **reifier**: `_:r rdf:reifies <<( s p o )>>`, plus a blank node standing where it was |
44/// | `<<( s p o )>>` | a syntax error | a **triple term**, object position only |
45/// | `{\| … \|}` | a syntax error | an annotation on the statement before it |
46///
47/// One file, two readings, two different graphs — see the round-trip matrix in
48/// `crates/rete-cli/tests/quoted_triple_surfaces.rs`, which asserts both.
49///
50/// `<<( s p o )>>` is unambiguous: it is RDF 1.2's and nothing else's. That
51/// asymmetry is why [`RdfStar`](Self::RdfStar) is the **default**. Reading an
52/// RDF-star file as RDF 1.2 silently yields a different graph; reading an
53/// RDF 1.2 file as RDF-star is a hard parse error that names the flag. Only one
54/// of the two mistakes is survivable, so the default is the one that makes the
55/// other mistake loud.
56///
57/// **N-Triples and N-Quads ignore this entirely.** Their reader is rete's own
58/// (`take_term`), it has accepted both `<< s p o >>` and `<<( s p o )>>` since
59/// #262, and RDF 1.2 N-Triples has no reifier syntax for `<< … >>` to be — so
60/// there is no ambiguity to resolve and nothing to choose. The same goes for
61/// RDF/XML.
62///
63/// Whichever surface reads the file, what gets **stored** is the same canonical
64/// token `<<s p o>>`. rete's term model is a superset of both: a triple term is
65/// a term, and RDF 1.2 reification is ordinary RDF — a blank node, a predicate
66/// and a term — which rete already stored before this flag existed.
67#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Hash)]
68pub enum QuotedTripleSurface {
69 /// `<< s p o >>` is a **quoted triple** (RDF-star), in subject or object
70 /// position — rete's own storage token. The default, and byte-for-byte the
71 /// behaviour every rete release before this flag had.
72 #[default]
73 RdfStar,
74 /// `<<( s p o )>>` is a **triple term** and `<< s p o >>` is a **reifier**
75 /// (RDF 1.2), with `{| … |}` annotations read too.
76 Rdf12,
77}
78
79impl QuotedTripleSurface {
80 /// Parse the `--quoted-triple-syntax` value, `None` for anything else.
81 /// Shared by `rete build`, `rete validate` and `rete export` so the flag
82 /// cannot come to mean two things in two directions.
83 pub fn parse(s: &str) -> Option<Self> {
84 match s {
85 crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF12 => Some(Self::Rdf12),
86 crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF_STAR => Some(Self::RdfStar),
87 _ => None,
88 }
89 }
90
91 /// The flag spelling of this surface — the inverse of [`Self::parse`].
92 pub const fn as_str(self) -> &'static str {
93 match self {
94 Self::Rdf12 => crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF12,
95 Self::RdfStar => crate::card_derive::QUOTED_TRIPLE_SURFACE_RDF_STAR,
96 }
97 }
98}
99
100/// The one-line hint appended to a Turtle/TriG parse error when the input looks
101/// like RDF 1.2 and was read as RDF-star — i.e. the exact mistake
102/// [`QuotedTripleSurface`]'s default makes, made loud.
103const RDF12_SURFACE_HINT: &str = "\nhint: this input contains `<<(`, the RDF 1.2 triple-term \
104 syntax — which is what `rete export` writes by default. Reading it takes \
105 `--quoted-triple-syntax rdf12`; the default, `rdf-star`, reads `<< s p o >>` as a quoted \
106 triple instead.";
107
108/// Does `text` look like it carries RDF 1.2 triple terms? A deliberately dumb
109/// substring test: `<<(` cannot begin anything else in Turtle, and this is only
110/// ever consulted to *explain a parse that already failed*, never to decide how
111/// to read a file. A `<<(` inside a string literal would make it a false
112/// positive on some other error's hint, which costs one confusing sentence and
113/// nothing else.
114fn looks_like_rdf12(text: &str) -> bool {
115 text.contains("<<(")
116}
117
118/// Why an ingest failed. Parser errors from `oxttl`/`oxrdfxml` and I/O errors
119/// are flattened to their `Display` string rather than wrapped: the variants
120/// then carry no foreign types, which keeps the error usable from the wasm
121/// bindings and stable across upstream parser upgrades.
122#[derive(Debug, thiserror::Error)]
123#[non_exhaustive]
124pub enum IngestError {
125 /// A malformed N-Triples/N-Quads line: its 1-based line number and a short
126 /// reason (`"bad subject"`, `"missing trailing '.'"`, …).
127 #[error("line {0}: {1}")]
128 Line(usize, &'static str),
129 /// The Turtle parser rejected the input.
130 #[error("turtle: {0}")]
131 Turtle(String),
132 /// The TriG parser rejected the input.
133 #[error("trig: {0}")]
134 TriG(String),
135 /// The RDF/XML parser rejected the input.
136 #[error("rdf/xml: {0}")]
137 RdfXml(String),
138 /// The requested format is not one this crate can parse.
139 #[error(
140 "unknown input format: {0} (expected nt, nq, ttl, trig, rdf/xml, jelly, jsonld, or yamlld)"
141 )]
142 UnknownFormat(String),
143 /// A Jelly file that is malformed, uses a feature this reader refuses, or
144 /// breaks a rule of the specification.
145 #[error("jelly: {0}")]
146 Jelly(String),
147 /// A JSON-LD or YAML-LD document that is malformed or that the JSON-LD
148 /// processor refuses. Also what a build without the `jsonld` feature, or a
149 /// streaming reader (JSON-LD is expanded whole), answers.
150 #[error("{0}")]
151 JsonLd(String),
152 /// A JSON-LD or YAML-LD document names a remote `@context` (the IRI) and no
153 /// [`ContextLoader`] was given, so it cannot be expanded.
154 #[error(
155 "json-ld: the document needs the remote context {0}, and remote contexts are not loaded"
156 )]
157 JsonLdRemoteContext(String),
158 /// [`QuotedTripleSurface::Rdf12`] was asked for on a Turtle/TriG input, but
159 /// this build of `rete-core` has the `rdf12-turtle` feature off, so the
160 /// RDF 1.2 reader was not compiled in. Refused by name rather than read as
161 /// RDF-star, which would be the wrong graph told in silence.
162 #[error(
163 "this build cannot read the RDF 1.2 Turtle/TriG surface: rete-core was compiled without \
164 the `rdf12-turtle` feature (N-Triples/N-Quads still accept `<<( s p o )>>`)"
165 )]
166 Rdf12TurtleUnavailable,
167 /// The input could not be read (streaming builds surface the path here).
168 #[error("io: {0}")]
169 Io(String),
170 /// An IRI outside the N-Triples/N-Quads `IRIREF` grammar / RFC 3987, refused
171 /// because the caller asked for a strict [`IriAudit`] (`rete build
172 /// --strict`). Without `strict` the same finding is only counted.
173 /// `location` is `line N` for the line-based syntaxes, the format name for
174 /// the others (oxttl reports its own positions).
175 #[error("{location}: invalid IRI {token} — {reason}")]
176 InvalidIri {
177 /// Where it was seen: `line 42`, or the format name.
178 location: String,
179 /// The offending term token, verbatim.
180 token: String,
181 /// Which rule it breaks ([`crate::iri::IriDefect::reason`]).
182 reason: &'static str,
183 },
184}
185
186/// Policy **and** tally for the invalid-IRI check that runs during ingest.
187///
188/// rete's reader is deliberately tolerant of IRIs a strict parser rejects (see
189/// [`crate::iri`]), which is what lets it hold every published dataset — and
190/// also what let three of the `scholar/` exports produce N-Quads Oxigraph
191/// refuses to load. Passing an audit makes the tolerance *visible*: the build
192/// still succeeds and stores exactly what it was given, but it can now say how
193/// much of what it stored is not valid RDF. `strict` turns the same finding into
194/// a refusal, on the first offending statement.
195#[derive(Debug, Default)]
196pub struct IriAudit {
197 /// Fail the parse on the first invalid IRI instead of counting it.
198 pub strict: bool,
199 /// What the parse saw. Constant memory — see [`crate::iri::IriReport`].
200 pub report: crate::iri::IriReport,
201}
202
203impl IriAudit {
204 /// Count invalid IRIs; never fail (`rete build`'s default).
205 pub fn counting() -> Self {
206 Self::default()
207 }
208
209 /// Refuse the input at the first invalid IRI (`rete build --strict`).
210 pub fn refusing() -> Self {
211 Self {
212 strict: true,
213 report: crate::iri::IriReport::default(),
214 }
215 }
216}
217
218/// Run one statement past the audit: record any invalid IRI, and in `strict`
219/// mode turn it into an error naming the offending token. `location` is only
220/// called on the error path, so the common case allocates nothing.
221fn audit_quad<F: FnOnce() -> String>(
222 audit: Option<&mut IriAudit>,
223 s: &str,
224 p: &str,
225 o: &str,
226 g: Option<&str>,
227 location: F,
228) -> Result<(), IngestError> {
229 let Some(a) = audit else { return Ok(()) };
230 let Some(defect) = a.report.observe_quad(s, p, o, g) else {
231 return Ok(());
232 };
233 if !a.strict {
234 return Ok(());
235 }
236 let token = [Some(s), Some(p), Some(o), g]
237 .into_iter()
238 .flatten()
239 .find(|t| crate::iri::term_defect(t).is_some())
240 .unwrap_or("<?>")
241 .to_string();
242 Err(IngestError::InvalidIri {
243 location: location(),
244 token,
245 reason: defect.reason(),
246 })
247}
248
249/// Estimate the statement count of an N-Triples/N-Quads text from its newline
250/// count, so the output `Vec` can be pre-sized — a big build otherwise pays
251/// repeated `Vec` doublings, each briefly holding ~2× the (large) spine. An
252/// over-estimate by a few blank/comment lines is harmless (it's a capacity hint).
253fn estimate_statements(input: &str) -> usize {
254 bytecount_newlines(input).max(1)
255}
256
257/// Count `\n` bytes — one linear pass, far cheaper than the parse it sizes.
258fn bytecount_newlines(input: &str) -> usize {
259 input.as_bytes().iter().filter(|&&b| b == b'\n').count()
260}
261
262/// Parse one N-Quads line into a quad, or `None` for a blank/comment line.
263/// Shared by the whole-text [`parse_quads`] and the streaming [`parse_reader`].
264fn parse_nq_line(
265 raw: &str,
266 lineno: usize,
267 audit: Option<&mut IriAudit>,
268) -> Result<Option<RawQuad>, IngestError> {
269 let line = raw.trim();
270 if line.is_empty() || line.starts_with('#') {
271 return Ok(None);
272 }
273 let stripped = line
274 .strip_suffix('.')
275 .ok_or(IngestError::Line(lineno, "missing trailing '.'"))?
276 .trim_end();
277 let (s, rest) = take_term(stripped).ok_or(IngestError::Line(lineno, "bad subject"))?;
278 let (p, rest) =
279 take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad predicate"))?;
280 let (o, rest) = take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad object"))?;
281 let rest = rest.trim();
282 let graph = if rest.is_empty() {
283 None
284 } else {
285 let (g, tail) = take_term(rest).ok_or(IngestError::Line(lineno, "bad graph"))?;
286 if !tail.trim().is_empty() {
287 return Err(IngestError::Line(lineno, "trailing content after graph"));
288 }
289 Some(g)
290 };
291 audit_quad(audit, &s, &p, &o, graph.as_deref(), || {
292 format!("line {lineno}")
293 })?;
294 Ok(Some((s, p, o, graph)))
295}
296
297/// Parse one N-Triples line into a triple, or `None` for a blank/comment line.
298fn parse_nt_line(
299 raw: &str,
300 lineno: usize,
301 audit: Option<&mut IriAudit>,
302) -> Result<Option<RawTriple>, IngestError> {
303 let line = raw.trim();
304 if line.is_empty() || line.starts_with('#') {
305 return Ok(None);
306 }
307 let stripped = line
308 .strip_suffix('.')
309 .ok_or(IngestError::Line(lineno, "missing trailing '.'"))?
310 .trim_end();
311 let (s, rest) = take_term(stripped).ok_or(IngestError::Line(lineno, "bad subject"))?;
312 let (p, rest) =
313 take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad predicate"))?;
314 let (o, rest) = take_term(rest.trim_start()).ok_or(IngestError::Line(lineno, "bad object"))?;
315 if !rest.trim().is_empty() {
316 return Err(IngestError::Line(lineno, "trailing content after object"));
317 }
318 audit_quad(audit, &s, &p, &o, None, || format!("line {lineno}"))?;
319 Ok(Some((s, p, o)))
320}
321
322/// Parse N-Quads text: `subject predicate object [graph] .` per line.
323pub fn parse_quads(input: &str) -> Result<Vec<RawQuad>, IngestError> {
324 parse_quads_audited(input, None)
325}
326
327/// [`parse_quads`] with an optional [`IriAudit`].
328pub fn parse_quads_audited(
329 input: &str,
330 mut audit: Option<&mut IriAudit>,
331) -> Result<Vec<RawQuad>, IngestError> {
332 let mut out = Vec::with_capacity(estimate_statements(input));
333 for (i, raw) in input.lines().enumerate() {
334 if let Some(q) = parse_nq_line(raw, i + 1, audit.as_deref_mut())? {
335 out.push(q);
336 }
337 }
338 Ok(out)
339}
340
341/// Parse N-Triples text into raw term-token triples, skipping blank/comment lines.
342pub fn parse(input: &str) -> Result<Vec<RawTriple>, IngestError> {
343 parse_audited(input, None)
344}
345
346/// [`parse`] with an optional [`IriAudit`].
347pub fn parse_audited(
348 input: &str,
349 mut audit: Option<&mut IriAudit>,
350) -> Result<Vec<RawTriple>, IngestError> {
351 let mut out = Vec::with_capacity(estimate_statements(input));
352 for (i, raw) in input.lines().enumerate() {
353 if let Some(t) = parse_nt_line(raw, i + 1, audit.as_deref_mut())? {
354 out.push(t);
355 }
356 }
357 Ok(out)
358}
359
360/// **Stream-parse** N-Triples (`"nt"`) or N-Quads (`"nq"`) from a reader, one
361/// line at a time, so the whole input text is **never resident** — the big-build
362/// memory win over reading the file into a `String` first (each line String is
363/// transient, freed every iteration). `cap` pre-sizes the output `Vec` (e.g.
364/// `file_len / 64`) to avoid reallocation doublings of the large spine. Turtle is
365/// not streamable here (oxttl needs the whole input); callers use the text path
366/// for `"ttl"`.
367pub fn parse_reader<R: std::io::BufRead>(
368 reader: R,
369 format: &str,
370 cap: usize,
371) -> Result<Vec<RawQuad>, IngestError> {
372 parse_reader_audited(reader, format, cap, None)
373}
374
375/// [`parse_reader`] with an optional [`IriAudit`].
376pub fn parse_reader_audited<R: std::io::BufRead>(
377 reader: R,
378 format: &str,
379 cap: usize,
380 audit: Option<&mut IriAudit>,
381) -> Result<Vec<RawQuad>, IngestError> {
382 parse_reader_audited_surface(reader, format, cap, audit, QuotedTripleSurface::default())
383}
384
385/// [`parse_reader_audited`] reading `<< … >>` in the given
386/// [`QuotedTripleSurface`] (Turtle/TriG only — see the enum).
387pub fn parse_reader_audited_surface<R: std::io::BufRead>(
388 reader: R,
389 format: &str,
390 cap: usize,
391 audit: Option<&mut IriAudit>,
392 surface: QuotedTripleSurface,
393) -> Result<Vec<RawQuad>, IngestError> {
394 let mut out = Vec::with_capacity(cap);
395 stream_reader_audited_surface(reader, format, audit, surface, &mut |q| out.push(q))?;
396 Ok(out)
397}
398
399/// **Stream** N-Triples (`"nt"`) / N-Quads (`"nq"`) from a reader, invoking `f`
400/// once per parsed quad — like [`parse_reader`] but **without collecting** into a
401/// `Vec`. Each quad's term Strings are owned by `f` (and dropped when it returns
402/// if it doesn't retain them), so a caller that only needs to *observe* every
403/// term — e.g. the two-pass [`assemble_dataset_streaming`] building its
404/// dictionary — never materializes the whole quad multiset. The big-graph,
405/// low-RAM ingest primitive. Blank/comment lines are skipped; a parse error stops
406/// the stream and is returned.
407///
408/// Every format streams: the line-based ones a line at a time, Turtle / TriG /
409/// RDF-XML through their `oxttl`/`oxrdfxml` reader parsers, which pull from the
410/// reader incrementally. Nothing here holds the input text — so a 60 GB Turtle
411/// file is as ingestible as a 60 GB `.nt` one, which is what lets the external
412/// (memory-bounded) build accept them.
413pub fn stream_reader<R: std::io::BufRead>(
414 reader: R,
415 format: &str,
416 f: &mut dyn FnMut(RawQuad),
417) -> Result<(), IngestError> {
418 stream_reader_audited(reader, format, None, f)
419}
420
421/// [`stream_reader`] with an optional [`IriAudit`].
422///
423/// Every syntax is audited, not only the line-based ones. `oxttl`/`oxrdfxml`
424/// validate IRIs themselves and reject an invalid one before it reaches us, so
425/// in practice the Turtle/TriG/RDF-XML tallies stay at zero — but auditing them
426/// too means the count is a property of the *file*, not of which reader happened
427/// to produce it, and a future parser change cannot silently reopen the hole.
428pub fn stream_reader_audited<R: std::io::BufRead>(
429 reader: R,
430 format: &str,
431 audit: Option<&mut IriAudit>,
432 f: &mut dyn FnMut(RawQuad),
433) -> Result<(), IngestError> {
434 stream_reader_audited_surface(reader, format, audit, QuotedTripleSurface::default(), f)
435}
436
437/// [`stream_reader_audited`] reading `<< … >>` in the given
438/// [`QuotedTripleSurface`].
439///
440/// The surface reaches only the `"ttl"` and `"trig"` arms: the line-based
441/// readers are rete's own and take both spellings unconditionally, and RDF/XML
442/// has no syntax for either. Under [`QuotedTripleSurface::Rdf12`] Turtle/TriG go
443/// through `oxttl` 0.2 instead of 0.1, and the RDF 1.2 triple terms it hands
444/// back (`<<( s p o )>>`) are folded to rete's stored token on the way out, so
445/// the dictionary a file lands in does not depend on which reader read it.
446///
447/// **This entry point does not carry the RDF 1.2 hint** that the text path adds
448/// to a failed RDF-star parse: it is handed a reader, not a string, and sniffing
449/// the input would mean buffering the 60 GB file this function exists to avoid
450/// buffering. `rete build --memory-budget-mb file.ttl` therefore reports the raw
451/// parser error; every other route reports the hint.
452pub fn stream_reader_audited_surface<R: std::io::BufRead>(
453 reader: R,
454 format: &str,
455 mut audit: Option<&mut IriAudit>,
456 surface: QuotedTripleSurface,
457 f: &mut dyn FnMut(RawQuad),
458) -> Result<(), IngestError> {
459 if surface == QuotedTripleSurface::Rdf12 && matches!(format, "ttl" | "trig") {
460 return rdf12::stream(reader, format, audit, f);
461 }
462 match format {
463 "nt" | "nq" => {
464 for (i, line) in reader.lines().enumerate() {
465 let line = line.map_err(|e| IngestError::Io(e.to_string()))?;
466 if format == "nq" {
467 if let Some(q) = parse_nq_line(&line, i + 1, audit.as_deref_mut())? {
468 f(q);
469 }
470 } else if let Some((s, p, o)) = parse_nt_line(&line, i + 1, audit.as_deref_mut())? {
471 f((s, p, o, None));
472 }
473 }
474 Ok(())
475 }
476 "ttl" => turtle_star(reader, audit, f),
477 "trig" => trig_star(reader, audit, f),
478 "jelly" => jelly::stream(reader, audit, f),
479 "jsonld" | "yamlld" => Err(IngestError::JsonLd(format!(
480 "{format}: a JSON-LD document is expanded whole, so it cannot be streamed from a \
481 reader; build without --memory-budget-mb, or convert it to N-Quads first"
482 ))),
483 "rdfxml" => {
484 for r in oxrdfxml::RdfXmlParser::new().for_reader(reader) {
485 let t = r.map_err(|e| IngestError::RdfXml(e.to_string()))?;
486 let q = (
487 t.subject.to_string(),
488 t.predicate.to_string(),
489 t.object.to_string(),
490 None,
491 );
492 audit_quad(audit.as_deref_mut(), &q.0, &q.1, &q.2, None, || {
493 "rdf/xml".into()
494 })?;
495 f(q);
496 }
497 Ok(())
498 }
499 other => Err(IngestError::UnknownFormat(other.to_string())),
500 }
501}
502
503/// Stream **RDF-star** Turtle: `with_quoted_triples` accepts `<< s p o >>` in
504/// subject/object position, and oxrdf 0.2's `Term::Triple` Displays as the
505/// canonical `<<…>>` token rete's own N-Triples-star tokenizer emits — so no
506/// translation is needed on the way out.
507///
508/// Its own function, and called directly from both the reader path and the text
509/// path, so that the wasm engine — which reaches only the text path and has no
510/// RDF 1.2 reader compiled in — links this and nothing else.
511fn turtle_star<R: std::io::BufRead>(
512 reader: R,
513 mut audit: Option<&mut IriAudit>,
514 f: &mut dyn FnMut(RawQuad),
515) -> Result<(), IngestError> {
516 for r in oxttl::TurtleParser::new()
517 .with_quoted_triples()
518 .for_reader(reader)
519 {
520 let t = r.map_err(|e| IngestError::Turtle(e.to_string()))?;
521 let q = (
522 t.subject.to_string(),
523 t.predicate.to_string(),
524 t.object.to_string(),
525 None,
526 );
527 audit_quad(audit.as_deref_mut(), &q.0, &q.1, &q.2, None, || {
528 "turtle".into()
529 })?;
530 f(q);
531 }
532 Ok(())
533}
534
535/// [`turtle_star`] for TriG — Turtle plus named-graph blocks (`<g> { … }`), the
536/// shape most large RDF dumps ship in.
537fn trig_star<R: std::io::BufRead>(
538 reader: R,
539 mut audit: Option<&mut IriAudit>,
540 f: &mut dyn FnMut(RawQuad),
541) -> Result<(), IngestError> {
542 for r in oxttl::TriGParser::new()
543 .with_quoted_triples()
544 .for_reader(reader)
545 {
546 let q = r.map_err(|e| IngestError::TriG(e.to_string()))?;
547 let q = (
548 q.subject.to_string(),
549 q.predicate.to_string(),
550 q.object.to_string(),
551 graph_token(&q.graph_name),
552 );
553 audit_quad(
554 audit.as_deref_mut(),
555 &q.0,
556 &q.1,
557 &q.2,
558 q.3.as_deref(),
559 || "trig".into(),
560 )?;
561 f(q);
562 }
563 Ok(())
564}
565
566/// The canonical graph token for a parsed quad — `None` for the default graph,
567/// otherwise the same `<iri>` / `_:b` spelling the N-Quads reader produces, so
568/// TriG and N-Quads inputs land identical dictionary keys. `GraphName`'s own
569/// `Display` renders the default graph as `DEFAULT`, which must never become a
570/// term, hence the explicit match.
571fn graph_token(g: &oxrdf::GraphName) -> Option<String> {
572 match g {
573 oxrdf::GraphName::DefaultGraph => None,
574 other => Some(other.to_string()),
575 }
576}
577
578/// The **RDF 1.2** Turtle/TriG reader — `oxttl` 0.2 with its `rdf-12` feature,
579/// living beside the `oxttl` 0.1 reader above rather than replacing it.
580///
581/// Everything RDF 1.2 adds over RDF 1.1 arrives through this one module:
582/// `<<( s p o )>>` triple terms, `<< s p o >>` reifiers (which expand to
583/// `_:r rdf:reifies <<( s p o )>>` plus the statement that mentioned them),
584/// `{| … |}` annotations, and `"…"@lang--dir` directional literals. None of it
585/// needs a storage change, which is the point: a reifier is a blank node, an
586/// IRI and a term, and rete has stored those since v0.
587///
588/// The one translation is at the term boundary. `oxrdf` 0.3 renders a triple
589/// term as `<<( s p o )>>`; rete's dictionary key is `<<s p o>>`. [`take_term`]
590/// already reads both — that is what made RDF 1.2 N-Quads ingestible in #262 —
591/// so the fold is a re-scan of a token oxttl has already validated.
592#[cfg(feature = "rdf12-turtle")]
593mod rdf12 {
594 use super::{audit_quad, take_term, IngestError, IriAudit, RawQuad};
595
596 /// Fold `oxrdf` 0.3's `<<( s p o )>>` rendering of a triple term into rete's
597 /// stored `<<s p o>>`, recursively (a nested triple term folds with it).
598 /// Every other token — the overwhelming majority, and all of them in a file
599 /// with no triple terms — is returned untouched, not even copied.
600 pub(super) fn fold_triple_term(t: String) -> String {
601 if !t.starts_with("<<(") {
602 return t;
603 }
604 match take_term(&t) {
605 Some((token, rest)) if rest.trim().is_empty() => token,
606 // oxttl only ever hands back a well-formed term, so this is
607 // unreachable; returning the original beats mangling it silently.
608 _ => t,
609 }
610 }
611
612 /// `super::graph_token` for `oxrdf` 0.3's `GraphName` — the same rule, a
613 /// different crate version, which is exactly why it cannot be shared.
614 fn graph_token(g: &oxrdf12::GraphName) -> Option<String> {
615 match g {
616 oxrdf12::GraphName::DefaultGraph => None,
617 other => Some(other.to_string()),
618 }
619 }
620
621 /// Stream one RDF 1.2 Turtle (`"ttl"`) or TriG (`"trig"`) input, auditing
622 /// every quad the way the RDF-star reader does.
623 pub(super) fn stream<R: std::io::BufRead>(
624 reader: R,
625 format: &str,
626 mut audit: Option<&mut IriAudit>,
627 f: &mut dyn FnMut(RawQuad),
628 ) -> Result<(), IngestError> {
629 if format == "ttl" {
630 for r in oxttl12::TurtleParser::new().for_reader(reader) {
631 let t = r.map_err(|e| IngestError::Turtle(e.to_string()))?;
632 let q = (
633 t.subject.to_string(),
634 t.predicate.to_string(),
635 fold_triple_term(t.object.to_string()),
636 None,
637 );
638 audit_quad(audit.as_deref_mut(), &q.0, &q.1, &q.2, None, || {
639 "turtle".into()
640 })?;
641 f(q);
642 }
643 } else {
644 for r in oxttl12::TriGParser::new().for_reader(reader) {
645 let q = r.map_err(|e| IngestError::TriG(e.to_string()))?;
646 let q = (
647 q.subject.to_string(),
648 q.predicate.to_string(),
649 fold_triple_term(q.object.to_string()),
650 graph_token(&q.graph_name),
651 );
652 audit_quad(
653 audit.as_deref_mut(),
654 &q.0,
655 &q.1,
656 &q.2,
657 q.3.as_deref(),
658 || "trig".into(),
659 )?;
660 f(q);
661 }
662 }
663 Ok(())
664 }
665}
666
667/// The stand-in for [`rdf12`] in a build with `rdf12-turtle` off: it refuses by
668/// name. Reading the file as RDF-star instead would answer a question nobody
669/// asked, with a different graph.
670#[cfg(not(feature = "rdf12-turtle"))]
671mod rdf12 {
672 use super::{IngestError, IriAudit, RawQuad};
673
674 pub(super) fn stream<R: std::io::BufRead>(
675 _reader: R,
676 _format: &str,
677 _audit: Option<&mut IriAudit>,
678 _f: &mut dyn FnMut(RawQuad),
679 ) -> Result<(), IngestError> {
680 Err(IngestError::Rdf12TurtleUnavailable)
681 }
682}
683
684/// Parse Turtle into canonical N-Triples-token triples via oxttl, reading
685/// `<< s p o >>` as an RDF-star quoted triple. [`parse_turtle_surface`] chooses.
686pub fn parse_turtle(text: &str) -> Result<Vec<RawTriple>, IngestError> {
687 parse_turtle_surface(text, QuotedTripleSurface::default())
688}
689
690/// [`parse_turtle`] in the given [`QuotedTripleSurface`].
691pub fn parse_turtle_surface(
692 text: &str,
693 surface: QuotedTripleSurface,
694) -> Result<Vec<RawTriple>, IngestError> {
695 Ok(parse_text_surface(text, "ttl", surface)?
696 .into_iter()
697 .map(|(s, p, o, _)| (s, p, o))
698 .collect())
699}
700
701/// Parse TriG into canonical quads via oxttl — Turtle plus named-graph blocks
702/// (`<g> { … }`), the shape most large RDF dumps ship in. Reads `<< s p o >>` as
703/// an RDF-star quoted triple; [`parse_trig_surface`] chooses.
704pub fn parse_trig(text: &str) -> Result<Vec<RawQuad>, IngestError> {
705 parse_trig_surface(text, QuotedTripleSurface::default())
706}
707
708/// [`parse_trig`] in the given [`QuotedTripleSurface`].
709pub fn parse_trig_surface(
710 text: &str,
711 surface: QuotedTripleSurface,
712) -> Result<Vec<RawQuad>, IngestError> {
713 parse_text_surface(text, "trig", surface)
714}
715
716/// The shared Turtle/TriG text path: stream the string through whichever reader
717/// `surface` selects, and — this is the part only the text path can do, because
718/// only it holds the input — say so when an RDF-star parse fails on what is
719/// plainly RDF 1.2.
720///
721/// It does not audit: the Turtle/TriG text callers audit over the finished quads
722/// (see [`parse_statements_audited_surface`]), and auditing here too would count
723/// every IRI twice.
724///
725/// The RDF-star arm calls `oxttl` 0.1 **directly** rather than routing through
726/// [`stream_reader_audited_surface`]. That is not style: this is the only ingest
727/// path the wasm engine reaches, and going through the generic reader would
728/// monomorphize it over `&[u8]` and pull the whole `match format` — every
729/// syntax's reader — into a module that previously linked one Turtle parser.
730/// Measured at +17 KB of wasm for code no browser can run, since the RDF 1.2
731/// reader is not compiled there at all. Keeping the arm direct keeps the
732/// browser artifacts what they were.
733fn parse_text_surface(
734 text: &str,
735 format: &str,
736 surface: QuotedTripleSurface,
737) -> Result<Vec<RawQuad>, IngestError> {
738 let mut out = Vec::new();
739 let r = if surface == QuotedTripleSurface::Rdf12 {
740 rdf12::stream(text.as_bytes(), format, None, &mut |q| out.push(q))
741 } else if format == "ttl" {
742 turtle_star(text.as_bytes(), None, &mut |q| out.push(q))
743 } else {
744 trig_star(text.as_bytes(), None, &mut |q| out.push(q))
745 };
746 match r {
747 Ok(()) => Ok(out),
748 // The default surface met `<<(`: that is an RDF 1.2 file being read as
749 // RDF-star, the one mistake this flag exists to prevent, and the parse
750 // error alone ("unexpected character") would send the reader hunting for
751 // a typo that is not there.
752 Err(IngestError::Turtle(m))
753 if surface == QuotedTripleSurface::RdfStar && looks_like_rdf12(text) =>
754 {
755 Err(IngestError::Turtle(format!("{m}{RDF12_SURFACE_HINT}")))
756 }
757 Err(IngestError::TriG(m))
758 if surface == QuotedTripleSurface::RdfStar && looks_like_rdf12(text) =>
759 {
760 Err(IngestError::TriG(format!("{m}{RDF12_SURFACE_HINT}")))
761 }
762 Err(e) => Err(e),
763 }
764}
765
766/// Parse RDF/XML into canonical N-Triples-token triples via oxrdfxml. This is how
767/// most OWL ontologies ship (`.rdf`/`.owl`/`.xml` with an `rdf:RDF` root) — so rete
768/// ingests them directly, no external conversion. (OWL/XML — the non-RDF functional
769/// XML serialization — is a different language; convert it with owlready2 first.)
770pub fn parse_rdfxml(text: &str) -> Result<Vec<RawTriple>, IngestError> {
771 let mut out = Vec::new();
772 for r in oxrdfxml::RdfXmlParser::new().for_reader(text.as_bytes()) {
773 let t = r.map_err(|e| IngestError::RdfXml(e.to_string()))?;
774 out.push((
775 t.subject.to_string(),
776 t.predicate.to_string(),
777 t.object.to_string(),
778 ));
779 }
780 Ok(out)
781}
782
783/// Parse one text input by format name (`"nt"`, `"nq"`, `"ttl"`, `"trig"`, or
784/// `"rdfxml"`) into quads (triples land in the default graph).
785pub fn parse_statements(text: &str, format: &str) -> Result<Vec<RawQuad>, IngestError> {
786 parse_statements_audited(text, format, None)
787}
788
789/// [`parse_statements`] with an optional [`IriAudit`]. The line-based syntaxes
790/// audit as they parse (so `strict` fails on the offending line); the others are
791/// audited over the parsed quads, which is where `oxttl` has already had its say.
792pub fn parse_statements_audited(
793 text: &str,
794 format: &str,
795 audit: Option<&mut IriAudit>,
796) -> Result<Vec<RawQuad>, IngestError> {
797 parse_statements_audited_surface(text, format, audit, QuotedTripleSurface::default())
798}
799
800/// [`parse_statements_audited`] reading `<< … >>` in the given
801/// [`QuotedTripleSurface`]. This is the entry point `rete build` and
802/// `rete validate` use for everything but a streamed line-based input, and the
803/// one that turns "RDF 1.2 file, RDF-star reader" into an error that names the
804/// flag instead of a bare syntax complaint.
805pub fn parse_statements_audited_surface(
806 text: &str,
807 format: &str,
808 mut audit: Option<&mut IriAudit>,
809 surface: QuotedTripleSurface,
810) -> Result<Vec<RawQuad>, IngestError> {
811 let quads = match format {
812 "nq" => return parse_quads_audited(text, audit),
813 "nt" => {
814 return Ok(parse_audited(text, audit)?
815 .into_iter()
816 .map(|(s, p, o)| (s, p, o, None))
817 .collect())
818 }
819 // Straight to the quad path for both: going via `parse_turtle_surface`
820 // would build a `Vec<RawQuad>`, rebuild it as a `Vec<RawTriple>` to
821 // satisfy that function's signature, and rebuild it as quads again —
822 // three copies of every term String on the biggest input rete takes.
823 "ttl" | "trig" => parse_text_surface(text, format, surface)?,
824 "rdfxml" => parse_rdfxml(text)?
825 .into_iter()
826 .map(|(s, p, o)| (s, p, o, None))
827 .collect(),
828 // Inline contexts only here; `parse_jsonld_audited` takes a loader.
829 "jsonld" | "yamlld" => {
830 return parse_jsonld_audited(text, format == "yamlld", &JsonLdOptions::default(), audit)
831 }
832 "jelly" => {
833 return Err(IngestError::Jelly(
834 "Jelly is a binary format, so it cannot come in as text; \
835 use parse_jelly_bytes or stream_reader"
836 .into(),
837 ))
838 }
839 other => return Err(IngestError::UnknownFormat(other.to_string())),
840 };
841 for (s, p, o, g) in &quads {
842 audit_quad(audit.as_deref_mut(), s, p, o, g.as_deref(), || {
843 format.to_string()
844 })?;
845 }
846 Ok(quads)
847}
848
849/// Parse one JSON-LD (`yaml == false`) or YAML-LD document into quads. Named
850/// graphs stay named. Remote contexts come from `options`; with none, a document
851/// that needs one fails with [`IngestError::JsonLdRemoteContext`].
852pub fn parse_jsonld_audited(
853 text: &str,
854 yaml: bool,
855 options: &JsonLdOptions,
856 audit: Option<&mut IriAudit>,
857) -> Result<Vec<RawQuad>, IngestError> {
858 let mut out = Vec::new();
859 jsonld::parse(text, yaml, options, audit, &mut |q| out.push(q))?;
860 Ok(out)
861}
862
863/// Parse a whole Jelly file held in memory (`"jelly"` is binary, so it has no
864/// text entry point). The streaming path is [`stream_reader`] with `"jelly"`;
865/// this is for callers that already hold the bytes, such as the browser engine.
866pub fn parse_jelly_bytes(bytes: &[u8]) -> Result<Vec<RawQuad>, IngestError> {
867 parse_jelly_bytes_audited(bytes, None)
868}
869
870/// [`parse_jelly_bytes`] with an optional [`IriAudit`].
871pub fn parse_jelly_bytes_audited(
872 bytes: &[u8],
873 audit: Option<&mut IriAudit>,
874) -> Result<Vec<RawQuad>, IngestError> {
875 let mut out = Vec::new();
876 jelly::stream(bytes, audit, &mut |q| out.push(q))?;
877 Ok(out)
878}
879
880/// Take one term from the front of `s`, returning `(term, remainder)`.
881pub(crate) fn take_term(s: &str) -> Option<(String, &str)> {
882 let bytes = s.as_bytes();
883 let first = *bytes.first()?;
884 match first {
885 // A quoted triple / triple term, in EITHER surface — the inner terms are
886 // themselves terms (so this recurses; nesting works). `<<` starts with `<`
887 // and an IRI scan would stop at the first inner `>`, so it must be handled
888 // before the plain-IRI case.
889 // * RDF-star: `<< subject predicate object >>`
890 // * RDF 1.2: `<<( subject predicate object )>>` (triple term)
891 // Both re-emit the SAME canonical token `<<s p o>>`, so a file written in
892 // either surface — and a query written either way — dedupe and match. This
893 // makes RDF 1.2 N-Triples interoperable with the RDF-star we already store,
894 // with no format change and no dependency swap.
895 b'<' if bytes.get(1) == Some(&b'<') => {
896 // Distinguish `<<(` (RDF 1.2) from `<<` (RDF-star) by the char after `<<`.
897 let (inner, rdf12) = match s[2..].strip_prefix('(') {
898 Some(after) => (after.trim_start(), true),
899 None => (s[2..].trim_start(), false),
900 };
901 let (subj, r) = take_term(inner)?;
902 let (pred, r) = take_term(r.trim_start())?;
903 let (obj, r) = take_term(r.trim_start())?;
904 let r = r.trim_start();
905 let rest = if rdf12 {
906 r.strip_prefix(")>>")?
907 } else {
908 r.strip_prefix(">>")?
909 };
910 // Canonical surface = oxrdf's `Triple` Display: `<<s p o>>` (tight
911 // brackets, single spaces between components) — the same token for both
912 // input surfaces, so N-Triples-star, Turtle-star, and RDF 1.2 triple
913 // terms all resolve to one dictionary entry.
914 Some((format!("<<{subj} {pred} {obj}>>"), rest))
915 }
916 b'<' => {
917 // IRI ref: up to the closing '>'.
918 let end = s.find('>')?;
919 Some((s[..=end].to_string(), &s[end + 1..]))
920 }
921 b'_' => {
922 // Blank node: up to whitespace.
923 let end = s.find(char::is_whitespace).unwrap_or(s.len());
924 Some((s[..end].to_string(), &s[end..]))
925 }
926 b'"' => {
927 // Literal: closing unescaped quote, then optional ^^<dt> or @lang.
928 let mut i = 1;
929 let b = s.as_bytes();
930 while i < b.len() {
931 match b[i] {
932 b'\\' => i += 2, // skip escaped char
933 b'"' => break,
934 _ => i += 1,
935 }
936 }
937 if i >= b.len() {
938 return None; // unterminated
939 }
940 let mut end = i + 1; // past closing quote
941 if s[end..].starts_with("^^<") {
942 let close = s[end..].find('>')? + end;
943 // RDF 1.1: `"x"^^xsd:string` IS `"x"`; keep the one canonical
944 // spelling, so the two never become two dictionary terms and
945 // a plain query constant matches either input.
946 if &s[end + 3..close] == "http://www.w3.org/2001/XMLSchema#string" {
947 return Some((s[..end].to_string(), &s[close + 1..]));
948 }
949 end = close + 1;
950 } else if s[end..].starts_with('@') {
951 // Language tag: '@' then BCP-47 subtags `[a-zA-Z0-9-]+`. Stop at
952 // the first char that can't be part of a tag — normally the
953 // whitespace before the predicate, but ALSO the `>>` that closes
954 // a quoted triple when this literal is its object (`"x"@en>>`),
955 // where there is no separating whitespace.
956 let mut j = end + 1; // past '@'
957 while j < b.len() && (b[j].is_ascii_alphanumeric() || b[j] == b'-') {
958 j += 1;
959 }
960 end = j;
961 }
962 Some((s[..end].to_string(), &s[end..]))
963 }
964 _ => None,
965 }
966}
967
968/// Split a canonical quoted-triple token `<<s p o>>` (RDF-star) into its three
969/// component term tokens, or `None` if `t` is not a quoted triple. Reuses
970/// [`take_term`] for term-boundary scanning, so nested quoting parses correctly.
971/// The inverse of the `<<…>>` construction; used by the SUBJECT/PREDICATE/OBJECT
972/// SPARQL-star builtins.
973pub(crate) fn quoted_triple_parts(t: &str) -> Option<(String, String, String)> {
974 let inner = t.strip_prefix("<<")?.strip_suffix(">>")?.trim();
975 let (s, r) = take_term(inner)?;
976 let (p, r) = take_term(r.trim_start())?;
977 let (o, r) = take_term(r.trim_start())?;
978 if !r.trim().is_empty() {
979 return None;
980 }
981 Some((s, p, o))
982}
983
984/// Counts describing an assembled file, for status lines and UIs.
985///
986/// In the [`BuildStats`] a metadata callback receives, `statements` and
987/// `default_triples` are the counts **ingested** — the input multiset, before
988/// the index deduplicates it. In the `BuildStats` an assemble function
989/// *returns*, they are the counts actually **written** (see
990/// [`FinalCounts`]), so a status line reports what the file holds. Use
991/// [`FinalCounts`] rather than these when a payload must agree with the header.
992#[derive(Debug, Clone, Copy)]
993pub struct BuildStats {
994 /// Total statements, across the default graph and every named one.
995 pub statements: usize,
996 /// Statements that landed in the default graph.
997 pub default_triples: usize,
998 /// How many named graphs the input mentioned; one index is written per graph.
999 pub named_graphs: usize,
1000 /// Distinct terms in the shared dictionary.
1001 pub terms: usize,
1002 /// Levels in the community pyramid, or `0` when the build skipped it.
1003 pub pyramid_levels: u16,
1004 /// Whether the file got a label index. The record says so even when the
1005 /// caller (a binding) set no option for it.
1006 pub label_index: bool,
1007}
1008
1009/// How a build stores the **named graphs** (the default graph is unaffected).
1010#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
1011pub enum GraphLayout {
1012 /// One permutation index per graph, in the NAMED_GRAPHS section (kind 5):
1013 /// the graph IRI as text, then a full container per graph. What every rete
1014 /// before 0.4 writes and reads.
1015 #[default]
1016 PerGraph,
1017 /// The **quad index** (kind 10, SPEC.md §6.5): every named-graph quad
1018 /// stored once, the graph as a fourth column of graph-leading permutations
1019 /// in shared tiles, behind a sorted graph directory; graph names become
1020 /// dictionary terms. Far smaller and randomly addressable when graphs are
1021 /// many and small. Readers before 0.4.0 refuse these files (loudly — see
1022 /// `docs/compatibility.md`). Needs the 3- or 6-permutation set.
1023 Quad,
1024}
1025
1026/// One pass's named-graph quads, in the layout being built.
1027enum NamedQuads {
1028 PerGraph(std::collections::BTreeMap<String, Vec<(u32, u32, u32)>>),
1029 Quad {
1030 /// `(graph subject id, s, p, o)`.
1031 quads: Vec<[u32; 4]>,
1032 /// Distinct graphs seen (a bitset over subject ids).
1033 seen: Vec<u64>,
1034 graphs: usize,
1035 /// The last graph name and its id: N-Quads dumps keep a graph's
1036 /// statements together, so most quads skip the dictionary lookup.
1037 last: Option<(String, u32)>,
1038 },
1039}
1040
1041impl NamedQuads {
1042 fn new(layout: GraphLayout, dict: &Dictionary) -> Self {
1043 match layout {
1044 GraphLayout::PerGraph => NamedQuads::PerGraph(Default::default()),
1045 GraphLayout::Quad => NamedQuads::Quad {
1046 quads: Vec::new(),
1047 seen: vec![
1048 0u64;
1049 (dict.shared_count() as usize + dict.subject_only_count() as usize) / 64
1050 + 2
1051 ],
1052 graphs: 0,
1053 last: None,
1054 },
1055 }
1056 }
1057
1058 fn push(&mut self, dict: &Dictionary, graph: String, t: (u32, u32, u32)) {
1059 match self {
1060 NamedQuads::PerGraph(m) => m.entry(graph).or_default().push(t),
1061 NamedQuads::Quad {
1062 quads,
1063 seen,
1064 graphs,
1065 last,
1066 } => {
1067 let g = match last {
1068 Some((name, id)) if *name == graph => *id,
1069 _ => {
1070 let id = dict.subject_id(&graph).expect("observed graph name");
1071 *last = Some((graph, id));
1072 id
1073 }
1074 };
1075 let (w, b) = (g as usize / 64, g % 64);
1076 if let Some(word) = seen.get_mut(w) {
1077 if *word & (1 << b) == 0 {
1078 *word |= 1 << b;
1079 *graphs += 1;
1080 }
1081 }
1082 quads.push([g, t.0, t.1, t.2]);
1083 }
1084 }
1085 }
1086
1087 fn graph_count(&self) -> usize {
1088 match self {
1089 NamedQuads::PerGraph(m) => m.len(),
1090 NamedQuads::Quad { graphs, .. } => *graphs,
1091 }
1092 }
1093
1094 fn quad_count(&self) -> usize {
1095 match self {
1096 NamedQuads::PerGraph(m) => m.values().map(Vec::len).sum(),
1097 NamedQuads::Quad { quads, .. } => quads.len(),
1098 }
1099 }
1100}
1101
1102/// The **deduplicated** counts of the file being written — what the header
1103/// records, and therefore what any embedded metadata has to report.
1104///
1105/// The input multiset is not the file: every permutation index sorts and
1106/// deduplicates, so a harvest that pages with overlapping windows writes fewer
1107/// quads than it ingested. These counts are known only once the indexes exist,
1108/// which is why a payload that carries them is derived in two stages (see
1109/// [`IntoMetadata`]).
1110#[derive(Debug, Clone, Copy)]
1111pub struct FinalCounts {
1112 /// Unique triples in the default graph.
1113 pub default_triples: u64,
1114 /// Unique quads across the default graph and every named graph — exactly
1115 /// the header's `quad_count`.
1116 pub quads: u64,
1117}
1118
1119/// What a build's metadata callback may return.
1120///
1121/// A `Vec<u8>` is the payload verbatim — the common case (no metadata, or a
1122/// payload that carries no counts). [`DeferredMetadata`] is a payload that
1123/// *does* carry counts: the callback derives everything it needs while the
1124/// source quads are still resident, and returns a finalizer that the writer
1125/// calls with the [`FinalCounts`], which only exist once the indexes have
1126/// deduplicated the input. Without that second stage a card can only report the
1127/// ingested count, which over-reports any input containing duplicates.
1128pub trait IntoMetadata {
1129 /// Produce the metadata section payload for a file with these counts.
1130 fn into_metadata(self, counts: FinalCounts) -> Vec<u8>;
1131}
1132
1133impl IntoMetadata for Vec<u8> {
1134 fn into_metadata(self, _counts: FinalCounts) -> Vec<u8> {
1135 self
1136 }
1137}
1138
1139/// A metadata payload whose final form needs the written (deduplicated) counts.
1140/// See [`IntoMetadata`].
1141pub struct DeferredMetadata(Box<dyn FnOnce(FinalCounts) -> Vec<u8>>);
1142
1143impl DeferredMetadata {
1144 /// Defer `f` until the indexes are built and the true counts are known.
1145 pub fn new(f: impl FnOnce(FinalCounts) -> Vec<u8> + 'static) -> Self {
1146 DeferredMetadata(Box::new(f))
1147 }
1148
1149 /// No metadata section — byte-identical to a metadata-free build. Lets one
1150 /// `match` arm opt out while the other defers, without a type mismatch.
1151 pub fn none() -> Self {
1152 DeferredMetadata::new(|_| Vec::new())
1153 }
1154}
1155
1156impl IntoMetadata for DeferredMetadata {
1157 fn into_metadata(self, counts: FinalCounts) -> Vec<u8> {
1158 (self.0)(counts)
1159 }
1160}
1161
1162/// [`assemble_dataset`] for the writers that finish as format `0x06` (the
1163/// bindings and the browser): named graphs go in the quad index, the only
1164/// layout `0x06` has. Everything else is `assemble_dataset`'s.
1165pub fn assemble_dataset_quad(quads: Vec<RawQuad>, metadata: &[u8]) -> (Vec<u8>, BuildStats) {
1166 let blob = metadata.to_vec();
1167 assemble_dataset_with_quad(quads, move |_, _| blob)
1168}
1169
1170/// [`assemble_dataset_with`] with the named graphs in the quad index.
1171pub fn assemble_dataset_with_quad<M: IntoMetadata>(
1172 quads: Vec<RawQuad>,
1173 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1174) -> (Vec<u8>, BuildStats) {
1175 assemble_dataset_with_opts_algo_quad(quads, true, false, None, PyramidAlgo::Louvain, metadata)
1176}
1177
1178/// [`assemble_dataset_with_opts_algo`] for the `0x06` writers, with **the layout
1179/// `rete build --label-index` writes**: the named graphs in the quad index, the
1180/// pyramid block carrying only the community material, the SCHEMA section and
1181/// the label index. The browser engine and the Python, R and Java clients all
1182/// assemble through this, so one input and one set of options is one content
1183/// hash whichever front end built it (the browser and Java, having no zstd
1184/// encoder, write their sections uncompressed and so differ in bytes). The label
1185/// index is on because label prefix search is what a graph built in memory is
1186/// opened to do (the playground, the MCP server), and the legacy label block it
1187/// replaces is gone from the community-only pyramid block.
1188pub fn assemble_dataset_with_opts_algo_quad<M: IntoMetadata>(
1189 quads: Vec<RawQuad>,
1190 with_pyramid: bool,
1191 with_text_index: impl crate::text_index::IntoTextIndex,
1192 type_override: Option<&str>,
1193 algo: PyramidAlgo,
1194 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1195) -> (Vec<u8>, BuildStats) {
1196 let text_index = crate::text_index::IntoTextIndex::into_text_index(with_text_index);
1197 let opts = AssembleOptions {
1198 with_pyramid,
1199 with_text_index: text_index.is_some(),
1200 text_index_scope: text_index.as_ref(),
1201 type_override,
1202 algo,
1203 graph_layout: GraphLayout::Quad,
1204 label_index: true,
1205 ..AssembleOptions::sectioned()
1206 };
1207 assemble_dataset_opts(quads, &opts, metadata)
1208}
1209
1210/// Assemble a complete `.rete` file image from parsed quads: one shared
1211/// dictionary, the default-graph index, one index per named graph, and the
1212/// community pyramid. `metadata` is the opaque metadata-section payload (the
1213/// CLI puts a JSON Dataset Card there); pass `&[]` for none — that is
1214/// byte-identical to a metadata-free build.
1215pub fn assemble_dataset(quads: Vec<RawQuad>, metadata: &[u8]) -> (Vec<u8>, BuildStats) {
1216 let blob = metadata.to_vec();
1217 assemble_dataset_with(quads, move |_, _| blob)
1218}
1219
1220/// Like [`assemble_dataset`], but the metadata payload is derived from the
1221/// [`BuildStats`] and the source quads while they are still resident — for
1222/// metadata that describes the graph (the Dataset Card). Returning an empty
1223/// `Vec` is byte-identical to a metadata-free build; return a
1224/// [`DeferredMetadata`] for a payload that must also carry the file's
1225/// deduplicated counts, which exist only once the indexes are built.
1226pub fn assemble_dataset_with<M: IntoMetadata>(
1227 quads: Vec<RawQuad>,
1228 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1229) -> (Vec<u8>, BuildStats) {
1230 assemble_dataset_with_opts(quads, true, false, None, metadata)
1231}
1232
1233/// Like [`assemble_dataset_with`], but `with_pyramid = false` skips the Louvain
1234/// community pyramid entirely — no pyramid section is written (header length 0).
1235/// SPARQL / SHACL / triple / reachability queries don't use the pyramid, so a
1236/// pyramid-less file is fully queryable and markedly smaller (the pyramid is the
1237/// largest section on highly-connected graphs). Only the community / summary /
1238/// progressive paths need it.
1239pub fn assemble_dataset_with_opts<M: IntoMetadata>(
1240 quads: Vec<RawQuad>,
1241 with_pyramid: bool,
1242 with_text_index: impl crate::text_index::IntoTextIndex,
1243 type_override: Option<&str>,
1244 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1245) -> (Vec<u8>, BuildStats) {
1246 assemble_dataset_with_opts_algo(
1247 quads,
1248 with_pyramid,
1249 with_text_index,
1250 type_override,
1251 PyramidAlgo::Louvain,
1252 metadata,
1253 )
1254}
1255
1256/// Like [`assemble_dataset_with_opts`], but selects the community [`PyramidAlgo`]
1257/// (the in-memory build path for `rete build --pyramid-algo …`).
1258#[allow(clippy::too_many_arguments)]
1259pub fn assemble_dataset_with_opts_algo<M: IntoMetadata>(
1260 quads: Vec<RawQuad>,
1261 with_pyramid: bool,
1262 with_text_index: impl crate::text_index::IntoTextIndex,
1263 type_override: Option<&str>,
1264 algo: PyramidAlgo,
1265 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1266) -> (Vec<u8>, BuildStats) {
1267 assemble_dataset_with_perms(
1268 quads,
1269 with_pyramid,
1270 with_text_index,
1271 type_override,
1272 algo,
1273 crate::index::PermSet::ALL,
1274 metadata,
1275 )
1276}
1277
1278/// Like [`assemble_dataset_with_opts_algo`], but writes only the permutations in
1279/// `perms` (`rete build --permutations 3`). Every other byte of the file is
1280/// unchanged; [`crate::index::PermSet::ALL`] reproduces the default build
1281/// exactly, down to the header byte.
1282#[allow(clippy::too_many_arguments)]
1283pub fn assemble_dataset_with_perms<M: IntoMetadata>(
1284 quads: Vec<RawQuad>,
1285 with_pyramid: bool,
1286 with_text_index: impl crate::text_index::IntoTextIndex,
1287 type_override: Option<&str>,
1288 algo: PyramidAlgo,
1289 perms: crate::index::PermSet,
1290 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1291) -> (Vec<u8>, BuildStats) {
1292 assemble_dataset_with_layout(
1293 quads,
1294 with_pyramid,
1295 with_text_index,
1296 type_override,
1297 algo,
1298 perms,
1299 GraphLayout::PerGraph,
1300 metadata,
1301 )
1302}
1303
1304/// Like [`assemble_dataset_with_perms`], storing the named graphs in `graphs`
1305/// layout. [`GraphLayout::PerGraph`] reproduces it exactly.
1306///
1307/// # Panics
1308///
1309/// With [`GraphLayout::Quad`] and a permutation set other than
1310/// [`PermSet::CORE`](crate::index::PermSet::CORE) or
1311/// [`PermSet::ALL`](crate::index::PermSet::ALL), when the input has named graphs.
1312#[allow(clippy::too_many_arguments)]
1313pub fn assemble_dataset_with_layout<M: IntoMetadata>(
1314 quads: Vec<RawQuad>,
1315 with_pyramid: bool,
1316 with_text_index: impl crate::text_index::IntoTextIndex,
1317 type_override: Option<&str>,
1318 algo: PyramidAlgo,
1319 perms: crate::index::PermSet,
1320 graphs: GraphLayout,
1321 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1322) -> (Vec<u8>, BuildStats) {
1323 let text_index = crate::text_index::IntoTextIndex::into_text_index(with_text_index);
1324 let opts = AssembleOptions {
1325 with_pyramid,
1326 with_text_index: text_index.is_some(),
1327 text_index_scope: text_index.as_ref(),
1328 type_override,
1329 algo,
1330 perms,
1331 graph_layout: graphs,
1332 ..AssembleOptions::legacy()
1333 };
1334 assemble_dataset_opts(quads, &opts, metadata)
1335}
1336
1337/// Everything an in-memory build decides besides its input. The fields the
1338/// older `assemble_*` entry points take positionally, plus the sections added
1339/// after the generation-1 freeze.
1340#[derive(Debug, Clone, Copy)]
1341pub struct AssembleOptions<'a> {
1342 /// Build the community pyramid (kind 4).
1343 pub with_pyramid: bool,
1344 /// Build the full-text index (kind 6, or kind 11 when its scope leaves
1345 /// out some predicates).
1346 pub with_text_index: bool,
1347 /// The scope of that index (`--text-index-graphs`, `--text-index-include`,
1348 /// `--text-index-exclude`); `None` is the default scope, the only one
1349 /// before 0.4.0: the default graph, every predicate.
1350 pub text_index_scope: Option<&'a crate::text_index::TextIndexScope>,
1351 /// The type predicate for the schema (e.g. `wdt:P31`); `rdf:type` if `None`.
1352 pub type_override: Option<&'a str>,
1353 /// The community algorithm of the pyramid.
1354 pub algo: PyramidAlgo,
1355 /// Which index permutations to write.
1356 pub perms: crate::index::PermSet,
1357 /// How the named graphs are stored: one index per graph (kind 5) or the
1358 /// quad index (kind 10, `rete build --named-graphs quad`).
1359 pub graph_layout: GraphLayout,
1360 /// What the pyramid-meta carries besides the summary.
1361 pub pyramid_layout: crate::file::PyramidLayout,
1362 /// Write the LABEL_INDEX section (kind 8, `rete build --label-index`).
1363 pub label_index: bool,
1364 /// Write the SCHEMA section (kind 9; `rete build` writes it unless
1365 /// `--no-schema`).
1366 pub schema: bool,
1367}
1368
1369impl AssembleOptions<'_> {
1370 /// What every `assemble_*` entry point wrote before kinds 8 and 9 existed:
1371 /// a Louvain pyramid with the combined pyramid-meta, no text index, all six
1372 /// permutations, one index per named graph, no LABEL_INDEX or SCHEMA
1373 /// section.
1374 pub fn legacy() -> Self {
1375 AssembleOptions {
1376 with_pyramid: true,
1377 with_text_index: false,
1378 text_index_scope: None,
1379 type_override: None,
1380 algo: PyramidAlgo::Louvain,
1381 perms: crate::index::PermSet::ALL,
1382 graph_layout: GraphLayout::PerGraph,
1383 pyramid_layout: crate::file::PyramidLayout::Combined,
1384 label_index: false,
1385 schema: false,
1386 }
1387 }
1388
1389 /// What `rete build` writes: the pyramid-meta carries only the community
1390 /// material, the schema has its own section, and the label index is
1391 /// opt-in.
1392 pub fn sectioned() -> Self {
1393 AssembleOptions {
1394 pyramid_layout: crate::file::PyramidLayout::CommunityOnly,
1395 schema: true,
1396 ..Self::legacy()
1397 }
1398 }
1399}
1400
1401/// Assemble a file image from parsed quads under explicit [`AssembleOptions`].
1402///
1403/// # Panics
1404///
1405/// With [`GraphLayout::Quad`] and a permutation set other than
1406/// [`PermSet::CORE`](crate::index::PermSet::CORE) or
1407/// [`PermSet::ALL`](crate::index::PermSet::ALL), when the input has named graphs.
1408pub fn assemble_dataset_opts<M: IntoMetadata>(
1409 quads: Vec<RawQuad>,
1410 opts: &AssembleOptions<'_>,
1411 metadata: impl FnOnce(&BuildStats, &[RawQuad]) -> M,
1412) -> (Vec<u8>, BuildStats) {
1413 use crate::build_progress::{self as progress, Stat, Ticker, Unit};
1414 use crate::build_trace::Phase;
1415 let graphs = opts.graph_layout;
1416 let n = Some(quads.len() as u64);
1417 progress::begin(Phase::Dictionary, n, Unit::Statements);
1418 let mut ticker = Ticker::new();
1419 let mut db = DictionaryBuilder::new();
1420 for (s, p, o, g) in &quads {
1421 db.observe(s, p, o);
1422 if let (GraphLayout::Quad, Some(g)) = (graphs, g) {
1423 db.observe_graph(g);
1424 }
1425 ticker.tick(1);
1426 }
1427 ticker.flush();
1428 let dict = db.build();
1429 crate::build_trace::lap(Phase::Dictionary);
1430 progress::set_stat(Stat::Terms, dict.term_count().into());
1431
1432 progress::begin(Phase::Encode, n, Unit::Statements);
1433 let mut default_triples = Vec::new();
1434 let mut named = NamedQuads::new(graphs, &dict);
1435 for (s, p, o, g) in &quads {
1436 let t = dict.encode(s, p, o).expect("observed term");
1437 match g {
1438 None => default_triples.push(t),
1439 Some(graph) => named.push(&dict, graph.clone(), t),
1440 }
1441 ticker.tick(1);
1442 }
1443 drop(ticker);
1444 crate::build_trace::lap(Phase::Encode);
1445
1446 // Derive the metadata blob (the Dataset Card) from the raw quads NOW, while
1447 // they are resident — then DROP them before the memory-heavy pyramid + index
1448 // phases. On a big build the string quads are the largest working set (every
1449 // term an owned String, heavily duplicated) and are fully redundant with the
1450 // dictionary + id-triples once encoded, so freeing them here cuts peak RAM by
1451 // their whole size. `pyramid_levels` is not known yet (0 in the callback);
1452 // it is filled into the returned `stats` by `finish_assembly`, and no metadata
1453 // callback depends on it (the card derives from the quads + term/graph counts).
1454 let stats = BuildStats {
1455 statements: quads.len(),
1456 default_triples: default_triples.len(),
1457 named_graphs: named.graph_count(),
1458 terms: dict.term_count() as usize,
1459 pyramid_levels: 0,
1460 label_index: opts.label_index,
1461 };
1462 let pending = metadata(&stats, &quads);
1463 crate::build_trace::lap(crate::build_trace::Phase::CardDerive);
1464 drop(quads);
1465
1466 finish_assembly(dict, default_triples, named, opts, pending, stats)
1467}
1468
1469/// **Two-pass, low-RAM** assembly: build a `.rete` by **streaming** the input(s)
1470/// twice instead of holding every parsed quad in memory. `stream` is invoked
1471/// **twice** and MUST replay the exact same quads in the same order each time —
1472/// pass 1 observes every term into the dictionary; pass 2 encodes them to
1473/// id-triples. The raw string quads (every term an owned String, heavily
1474/// duplicated — by far the largest working set on a big graph) are **never
1475/// collected**, so peak RAM is bounded by the dictionary + id-triples + index
1476/// rather than the string-quad multiset. Output is **byte-identical** to
1477/// [`assemble_dataset_with_opts`] on the same quads (same dictionary, same
1478/// id-triples in file order, same downstream pipeline).
1479///
1480/// The metadata callback derives the Dataset Card from the built dictionary +
1481/// default-graph id-triples (resolving terms through the dictionary), since the
1482/// raw quads were never retained. `stream` propagates parse/IO errors.
1483pub fn assemble_dataset_streaming<S, M: IntoMetadata>(
1484 stream: S,
1485 with_pyramid: bool,
1486 with_text_index: impl crate::text_index::IntoTextIndex,
1487 type_override: Option<&str>,
1488 metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1489) -> Result<(Vec<u8>, BuildStats), IngestError>
1490where
1491 S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1492{
1493 assemble_dataset_streaming_algo(
1494 stream,
1495 with_pyramid,
1496 with_text_index,
1497 type_override,
1498 PyramidAlgo::Louvain,
1499 metadata,
1500 )
1501}
1502
1503/// Like [`assemble_dataset_streaming`], but selects the community [`PyramidAlgo`]
1504/// (the streaming, low-RAM build path for `rete build --pyramid-algo …`).
1505#[allow(clippy::too_many_arguments)]
1506pub fn assemble_dataset_streaming_algo<S, M: IntoMetadata>(
1507 stream: S,
1508 with_pyramid: bool,
1509 with_text_index: impl crate::text_index::IntoTextIndex,
1510 type_override: Option<&str>,
1511 algo: PyramidAlgo,
1512 metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1513) -> Result<(Vec<u8>, BuildStats), IngestError>
1514where
1515 S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1516{
1517 assemble_dataset_streaming_with_perms(
1518 stream,
1519 with_pyramid,
1520 with_text_index,
1521 type_override,
1522 algo,
1523 crate::index::PermSet::ALL,
1524 metadata,
1525 )
1526}
1527
1528/// Like [`assemble_dataset_streaming_algo`], but writes only the permutations in
1529/// `perms` — the two-pass low-RAM twin of [`assemble_dataset_with_perms`].
1530#[allow(clippy::too_many_arguments)]
1531pub fn assemble_dataset_streaming_with_perms<S, M: IntoMetadata>(
1532 stream: S,
1533 with_pyramid: bool,
1534 with_text_index: impl crate::text_index::IntoTextIndex,
1535 type_override: Option<&str>,
1536 algo: PyramidAlgo,
1537 perms: crate::index::PermSet,
1538 metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1539) -> Result<(Vec<u8>, BuildStats), IngestError>
1540where
1541 S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1542{
1543 assemble_dataset_streaming_with_layout(
1544 stream,
1545 with_pyramid,
1546 with_text_index,
1547 type_override,
1548 algo,
1549 perms,
1550 GraphLayout::PerGraph,
1551 metadata,
1552 )
1553}
1554
1555/// Like [`assemble_dataset_streaming_with_perms`], storing the named graphs in
1556/// `graphs` layout — the two-pass low-RAM twin of
1557/// [`assemble_dataset_with_layout`]. [`GraphLayout::Quad`] with a permutation
1558/// set other than the 3- or 6-permutation one is an error.
1559#[allow(clippy::too_many_arguments)]
1560pub fn assemble_dataset_streaming_with_layout<S, M: IntoMetadata>(
1561 stream: S,
1562 with_pyramid: bool,
1563 with_text_index: impl crate::text_index::IntoTextIndex,
1564 type_override: Option<&str>,
1565 algo: PyramidAlgo,
1566 perms: crate::index::PermSet,
1567 graphs: GraphLayout,
1568 metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1569) -> Result<(Vec<u8>, BuildStats), IngestError>
1570where
1571 S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1572{
1573 let text_index = crate::text_index::IntoTextIndex::into_text_index(with_text_index);
1574 let opts = AssembleOptions {
1575 with_pyramid,
1576 with_text_index: text_index.is_some(),
1577 text_index_scope: text_index.as_ref(),
1578 type_override,
1579 algo,
1580 perms,
1581 graph_layout: graphs,
1582 ..AssembleOptions::legacy()
1583 };
1584 assemble_dataset_streaming_opts(stream, &opts, metadata)
1585}
1586
1587/// The two-pass low-RAM twin of [`assemble_dataset_opts`]. A quad-index layout
1588/// with a permutation set other than the 3- or 6-permutation one is an error.
1589pub fn assemble_dataset_streaming_opts<S, M: IntoMetadata>(
1590 mut stream: S,
1591 opts: &AssembleOptions<'_>,
1592 metadata: impl FnOnce(&BuildStats, &Dictionary, &[(u32, u32, u32)]) -> M,
1593) -> Result<(Vec<u8>, BuildStats), IngestError>
1594where
1595 S: FnMut(&mut dyn FnMut(RawQuad)) -> Result<(), IngestError>,
1596{
1597 let graphs = opts.graph_layout;
1598 if graphs == GraphLayout::Quad
1599 && crate::build_record::writes_legacy()
1600 && crate::quad::check_perms(opts.perms).is_err()
1601 {
1602 return Err(IngestError::Io(
1603 "the quad index needs the 3- or 6-permutation set".into(),
1604 ));
1605 }
1606 // Pass 1: observe every term (the dictionary dedups; the string quads are
1607 // freed line by line and never collected). With the quad index, graph
1608 // names are terms too. The stream sets each pass's progress total (the
1609 // input bytes it will read).
1610 use crate::build_progress::{self as progress, Stat, Unit};
1611 use crate::build_trace::Phase;
1612 progress::begin(Phase::ParseDictionary, None, Unit::None);
1613 let mut db = DictionaryBuilder::new();
1614 stream(&mut |(s, p, o, g)| {
1615 db.observe(&s, &p, &o);
1616 if let (GraphLayout::Quad, Some(g)) = (graphs, g) {
1617 db.observe_graph(&g);
1618 }
1619 })?;
1620 let dict = db.build();
1621 crate::build_trace::lap(Phase::ParseDictionary);
1622 progress::set_stat(Stat::Terms, dict.term_count().into());
1623
1624 // Pass 2: encode each quad to an id-triple, bucketing named graphs.
1625 progress::begin(Phase::ParseEncode, None, Unit::None);
1626 let mut default_triples: Vec<(u32, u32, u32)> = Vec::new();
1627 let mut named = NamedQuads::new(graphs, &dict);
1628 stream(&mut |(s, p, o, g)| {
1629 let t = dict.encode(&s, &p, &o).expect("observed term");
1630 match g {
1631 None => default_triples.push(t),
1632 Some(graph) => named.push(&dict, graph, t),
1633 }
1634 })?;
1635 crate::build_trace::lap(crate::build_trace::Phase::ParseEncode);
1636
1637 let statements = default_triples.len() + named.quad_count();
1638 let stats = BuildStats {
1639 statements,
1640 default_triples: default_triples.len(),
1641 named_graphs: named.graph_count(),
1642 terms: dict.term_count() as usize,
1643 pyramid_levels: 0,
1644 label_index: opts.label_index,
1645 };
1646 let pending = metadata(&stats, &dict, &default_triples);
1647 crate::build_trace::lap(crate::build_trace::Phase::CardDerive);
1648 Ok(finish_assembly(
1649 dict,
1650 default_triples,
1651 named,
1652 opts,
1653 pending,
1654 stats,
1655 ))
1656}
1657
1658/// The shared tail of every build path: from a finished dictionary + encoded
1659/// id-triples (default graph + named), build the community pyramid, the optional
1660/// full-text index, the permutation indexes, and serialize the file image. Takes
1661/// the id-triples **by value** so they are freed as the permutations consume them.
1662fn finish_assembly<M: IntoMetadata>(
1663 dict: Dictionary,
1664 mut default_triples: Vec<(u32, u32, u32)>,
1665 named: NamedQuads,
1666 opts: &AssembleOptions<'_>,
1667 pending: M,
1668 mut stats: BuildStats,
1669) -> (Vec<u8>, BuildStats) {
1670 use crate::build_progress::{self as progress, Unit};
1671 use crate::build_trace::Phase;
1672 let has_named = named.graph_count() > 0;
1673 let perms = opts.perms;
1674
1675 // The pyramid and the full-text index are built FIRST, while the dictionary and
1676 // the default id-triples are both still resident — both need the two together.
1677 // The pyramid sees the triples in input order (Louvain is order-sensitive).
1678 let (meta, levels) = if opts.with_pyramid {
1679 progress::begin(Phase::Pyramid, None, Unit::None);
1680 crate::file::build_pyramid_meta_layout(
1681 &dict,
1682 &default_triples,
1683 DEFAULT_TILE_BUDGET,
1684 opts.type_override,
1685 opts.algo,
1686 opts.pyramid_layout,
1687 )
1688 } else {
1689 (Vec::new(), 0)
1690 };
1691 stats.pyramid_levels = levels;
1692 if opts.with_pyramid {
1693 crate::build_trace::lap(crate::build_trace::Phase::Pyramid);
1694 }
1695
1696 // The full-text index over the literals of the requested scope: the
1697 // default graph's (the only scope before 0.4.0), optionally every named
1698 // graph's too, optionally only some predicates'.
1699 let with_text_index = opts
1700 .with_text_index
1701 .then(|| opts.text_index_scope.cloned().unwrap_or_default());
1702 // The named graphs' triples, for an index whose scope covers them: per
1703 // graph, or (in the quad layout) the quads' `(s, p, o)`, copied only when
1704 // the scope needs them; the index does not care which graph a word came
1705 // from.
1706 let quad_spo: Vec<(u32, u32, u32)>;
1707 let named_triples: Vec<&[(u32, u32, u32)]> = match &named {
1708 NamedQuads::PerGraph(m) => m.values().map(Vec::as_slice).collect(),
1709 NamedQuads::Quad { quads, .. } => {
1710 quad_spo = if with_text_index
1711 .as_ref()
1712 .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs)
1713 {
1714 quads.iter().map(|q| (q[1], q[2], q[3])).collect()
1715 } else {
1716 Vec::new()
1717 };
1718 vec![quad_spo.as_slice()]
1719 }
1720 };
1721 if with_text_index.is_some() {
1722 progress::begin(Phase::TextIndex, None, Unit::None);
1723 }
1724 let text_index = match &with_text_index {
1725 Some(scope) => crate::text_index::build_section(
1726 &dict,
1727 scope,
1728 &default_triples,
1729 named_triples.into_iter(),
1730 crate::file::writer_codec(),
1731 ),
1732 None => Vec::new(),
1733 };
1734 if with_text_index.is_some() {
1735 crate::build_trace::lap(crate::build_trace::Phase::TextIndex);
1736 }
1737 let text_index_partial = with_text_index
1738 .as_ref()
1739 .is_some_and(crate::text_index::TextIndexScope::restricts_predicates);
1740 let text_index_named = with_text_index
1741 .as_ref()
1742 .is_some_and(crate::text_index::TextIndexScope::covers_named_graphs);
1743
1744 // The LABEL_INDEX and SCHEMA sections are computed over the DEDUPLICATED
1745 // default graph in SPO order (and the schema's second pass in POS order),
1746 // the order every index build re-sorts into anyway — which is what lets
1747 // the memory-bounded build, reading its sorted permutation streams, write
1748 // the very same bytes. Both need the dictionary, so they run before it is
1749 // dropped. Without either section the triples stay in input order.
1750 let mut label_section = Vec::new();
1751 let mut schema_section = Vec::new();
1752 if opts.label_index || opts.schema {
1753 progress::begin(Phase::LabelsSchema, None, Unit::None);
1754 crate::schema_section::sort_dedup_spo(&mut default_triples);
1755 if opts.label_index {
1756 let entries = crate::label_index::entries_from_sorted_triples(&dict, &default_triples);
1757 label_section =
1758 crate::label_index::encode_section(&entries, crate::file::writer_codec());
1759 }
1760 if opts.schema {
1761 schema_section = crate::schema_section::compute_in_memory(
1762 &dict,
1763 &mut default_triples,
1764 opts.type_override,
1765 )
1766 .encode();
1767 }
1768 crate::build_trace::lap(crate::build_trace::Phase::LabelsSchema);
1769 }
1770
1771 // From here the dictionary is only needed for its own serialized bytes. Encode
1772 // it, capture the header term count, then DROP it before building the
1773 // permutation indexes — which work purely on id-triples and never touch the
1774 // dictionary. On a large graph this frees the single biggest resident structure
1775 // (the dictionary) right when the index sort needs the headroom. The output is
1776 // byte-for-byte identical to serializing the dictionary inline.
1777 let codec = crate::file::writer_codec();
1778 progress::begin(Phase::DictionarySection, None, Unit::None);
1779 let dict_container = crate::file::encode_dict_container(&dict, codec);
1780 let term_count = dict.term_count() as u64;
1781 let has_quoted_triples = dict.has_quoted_triples();
1782 let term_order = dict.order();
1783 drop(dict);
1784 crate::build_trace::lap(crate::build_trace::Phase::DictionarySection);
1785
1786 // Build each graph's six permutations one-at-a-time on a large graph (a single
1787 // permuted copy resident at a time, each sort still parallel) instead of all
1788 // six concurrently; below the threshold the faster all-permutations-parallel
1789 // build is used (its 6x transient copy is negligible there). `build_seq` is
1790 // byte-identical to `build`.
1791 // Every graph's triples, once per permutation (the quad index's quads
1792 // once, as it finishes).
1793 let (per_graph, quad_rows): (u64, u64) = match &named {
1794 NamedQuads::PerGraph(m) => (m.values().map(|t| t.len() as u64).sum(), 0),
1795 NamedQuads::Quad { quads, .. } => (0, quads.len() as u64),
1796 };
1797 let rows = (default_triples.len() as u64 + per_graph) * perms.len() as u64 + quad_rows;
1798 progress::begin(Phase::IndexPermutations, Some(rows), Unit::Records);
1799 let build_index = |triples: Vec<(u32, u32, u32)>| -> crate::GraphIndex {
1800 let n = triples.len();
1801 let b = GraphIndexBuilder::from_triples(triples)
1802 .with_perms(perms)
1803 .with_progress();
1804 if n > LOWMEM_TRIPLE_THRESHOLD {
1805 b.build_seq()
1806 } else {
1807 b.build()
1808 }
1809 };
1810 let def = build_index(default_triples);
1811 // The named graphs: one index per graph, or one quad index for all of them
1812 // (no per-graph index is ever built on that path).
1813 let (named_indexes, quad_index): (Vec<(String, crate::GraphIndex)>, _) = match named {
1814 NamedQuads::PerGraph(named) => (
1815 named
1816 .into_iter()
1817 .map(|(g, ts)| (g, build_index(ts)))
1818 .collect(),
1819 None,
1820 ),
1821 NamedQuads::Quad { quads, .. } => {
1822 let n = quads.len() as u64;
1823 // `0x07` derives every other order from GSPO, so the in-RAM image
1824 // carries that one; the legacy `0x05` output (a test hook) keeps
1825 // the file's whole permutation set.
1826 let q = if crate::build_record::writes_legacy() {
1827 crate::quad::encode_quad_index(quads, perms, codec)
1828 } else {
1829 crate::quad::encode_quad_gspo(quads, codec)
1830 }
1831 .expect("the quad index needs the 3- or 6-permutation set");
1832 progress::advance(n);
1833 (Vec::new(), Some(q))
1834 }
1835 };
1836 let named_quads = match &quad_index {
1837 Some(q) => q.quad_count,
1838 None => named_indexes
1839 .iter()
1840 .map(|(_, ix)| ix.triple_count() as u64)
1841 .sum::<u64>(),
1842 };
1843 crate::build_trace::lap(crate::build_trace::Phase::IndexPermutations);
1844
1845 // Only now is the DEDUPLICATED size of the file known: every index sorts and
1846 // dedups, so an input containing duplicate statements (a harvest that pages
1847 // with overlapping windows, a merge of overlapping dumps) writes fewer quads
1848 // than it ingested. `write_dataset_from_parts` stamps exactly this sum into
1849 // the header, so a metadata payload finalized with it cannot disagree with
1850 // the header — which is what an embedded Dataset Card used to do.
1851 let counts = FinalCounts {
1852 default_triples: def.triple_count() as u64,
1853 quads: def.triple_count() as u64 + named_quads,
1854 };
1855 let blob = pending.into_metadata(counts);
1856 progress::begin(Phase::Serialize, None, Unit::None);
1857 // Report what was WRITTEN, not what was read: the returned stats drive the
1858 // CLI's "wrote …: N quads" line, which described the file, not the input.
1859 stats.statements = counts.quads as usize;
1860 stats.default_triples = counts.default_triples as usize;
1861
1862 let bytes = crate::file::write_dataset_from_parts(
1863 &dict_container,
1864 term_count,
1865 &def,
1866 match &quad_index {
1867 Some(q) => crate::file::NamedPart::Quad(q),
1868 None => crate::file::NamedPart::PerGraph(&named_indexes),
1869 },
1870 has_named,
1871 has_quoted_triples,
1872 &meta,
1873 levels,
1874 &blob,
1875 &text_index,
1876 text_index_partial,
1877 text_index_named,
1878 codec,
1879 term_order,
1880 );
1881 // Kinds 8 and 9 go after everything else: they are outside the content
1882 // hash, so appending them moves nothing and changes no hash.
1883 let bytes = if label_section.is_empty() && schema_section.is_empty() {
1884 bytes
1885 } else {
1886 crate::file::append_sections(
1887 &bytes,
1888 &[
1889 (crate::header::SectionKind::LabelIndex, &label_section),
1890 (crate::header::SectionKind::Schema, &schema_section),
1891 ],
1892 )
1893 .expect("appending to an image this function just wrote")
1894 };
1895 crate::build_trace::lap(crate::build_trace::Phase::Serialize);
1896 (bytes, stats)
1897}
1898
1899/// Above this default-graph triple count, the index is built one permutation at a
1900/// time (low peak RAM) instead of all six in parallel. Chosen so typical builds
1901/// keep the faster parallel path while multi-hundred-million-triple graphs stay
1902/// within a bounded memory budget.
1903const LOWMEM_TRIPLE_THRESHOLD: usize = 30_000_000;
1904
1905#[cfg(test)]
1906mod tests {
1907 use super::*;
1908 use crate::Rete;
1909
1910 #[test]
1911 fn parses_iris_bnodes_literals() {
1912 let input = r#"
1913 # a comment
1914 <http://ex/Alice> <http://ex/knows> <http://ex/Bob> .
1915 <http://ex/Alice> <http://ex/age> "30"^^<http://www.w3.org/2001/XMLSchema#integer> .
1916 <http://ex/Bob> <http://ex/label> "Bob"@en .
1917 _:b0 <http://ex/p> "plain" .
1918 "#;
1919 let t = parse(input).unwrap();
1920 assert_eq!(t.len(), 4);
1921 assert_eq!(t[0].0, "<http://ex/Alice>");
1922 assert_eq!(t[0].2, "<http://ex/Bob>");
1923 assert_eq!(t[1].2, "\"30\"^^<http://www.w3.org/2001/XMLSchema#integer>");
1924 assert_eq!(t[2].2, "\"Bob\"@en");
1925 assert_eq!(t[3].0, "_:b0");
1926 assert_eq!(t[3].2, "\"plain\"");
1927 }
1928
1929 #[test]
1930 fn literal_with_spaces_and_escaped_quote() {
1931 let input = r#"<http://ex/s> <http://ex/p> "a \"quoted\" phrase here" ."#;
1932 let t = parse(input).unwrap();
1933 assert_eq!(t.len(), 1);
1934 assert_eq!(t[0].2, r#""a \"quoted\" phrase here""#);
1935 }
1936
1937 /// RDF-star: a quoted triple whose object is a **language-tagged literal**
1938 /// sits directly against the closing `>>` with no separating whitespace
1939 /// (`"name"@fr>>`). The language-tag scan must stop at `>` — a regression
1940 /// guard for the greedy scan-to-whitespace that swallowed `@fr>>` as the tag.
1941 #[test]
1942 fn quoted_triple_langtagged_object() {
1943 let s = r#"<<<http://ex/sp> <http://ex/name> "Hirondelle rustique"@fr>>"#;
1944 let (tok, rest) = take_term(s).unwrap();
1945 assert_eq!(
1946 tok,
1947 r#"<<<http://ex/sp> <http://ex/name> "Hirondelle rustique"@fr>>"#
1948 );
1949 assert_eq!(rest, "");
1950 // and it round-trips through a full annotation line
1951 let line = r#"<<<http://ex/sp> <http://ex/name> "Oreneta vulgar"@ca>> <http://purl.org/dc/terms/source> "Catalogue of Life" ."#;
1952 let t = parse(line).unwrap();
1953 assert_eq!(t.len(), 1);
1954 assert_eq!(
1955 t[0].0,
1956 r#"<<<http://ex/sp> <http://ex/name> "Oreneta vulgar"@ca>>"#
1957 );
1958 assert_eq!(t[0].2, r#""Catalogue of Life""#);
1959 // a plain lang-tagged object still terminates at whitespace
1960 assert_eq!(take_term(r#""x"@pt-BR ."#).unwrap().0, r#""x"@pt-BR"#);
1961 }
1962
1963 /// RDF 1.2 triple-term surface `<<( s p o )>>` parses to the SAME canonical
1964 /// token as the RDF-star `<< s p o >>`, so the two are interoperable (a query
1965 /// written either way matches data ingested either way).
1966 #[test]
1967 fn rdf12_triple_term_same_token_as_rdf_star() {
1968 let rdf12 = r#"<http://ex/r> <http://ex/p> <<( <http://ex/s> <http://ex/q> "o" )>> ."#;
1969 let star = r#"<http://ex/r> <http://ex/p> << <http://ex/s> <http://ex/q> "o" >> ."#;
1970 let a = parse(rdf12).unwrap();
1971 let b = parse(star).unwrap();
1972 assert_eq!(a.len(), 1);
1973 assert_eq!(a[0].2, r#"<<<http://ex/s> <http://ex/q> "o">>"#);
1974 assert_eq!(
1975 a[0].2, b[0].2,
1976 "RDF 1.2 and RDF-star must dedupe to one token"
1977 );
1978 // take_term consumes exactly the triple term (nothing left dangling).
1979 let (tok, rest) =
1980 take_term(r#"<<( <http://ex/s> <http://ex/q> <http://ex/o> )>>"#).unwrap();
1981 assert_eq!(tok, "<<<http://ex/s> <http://ex/q> <http://ex/o>>>");
1982 assert_eq!(rest, "");
1983 }
1984
1985 #[test]
1986 fn rejects_missing_dot() {
1987 assert!(parse("<a> <b> <c>").is_err());
1988 }
1989
1990 #[test]
1991 fn parses_quads_with_and_without_graph() {
1992 let input = "<http://ex/a> <http://ex/p> <http://ex/b> .\n\
1993 <http://ex/a> <http://ex/p> <http://ex/c> <http://ex/g> .";
1994 let q = parse_quads(input).unwrap();
1995 assert_eq!(q.len(), 2);
1996 assert_eq!(q[0].3, None); // default graph
1997 assert_eq!(q[1].3, Some("<http://ex/g>".to_string()));
1998 }
1999
2000 #[test]
2001 fn parse_reader_matches_text_parse() {
2002 // The streaming reader must produce exactly the same quads as parsing the
2003 // whole text — including blank/comment skipping, a named graph, and CRLF.
2004 let nt = "<http://ex/a> <http://ex/p> <http://ex/b> .\r\n\
2005 # comment\n\
2006 \n\
2007 _:b0 <http://ex/q> \"lit\"@en .\n";
2008 let via_text = parse_statements(nt, "nt").unwrap();
2009 let via_reader = parse_reader(std::io::Cursor::new(nt), "nt", 0).unwrap();
2010 assert_eq!(via_text, via_reader);
2011
2012 let nq = "<http://ex/a> <http://ex/p> <http://ex/b> .\n\
2013 <http://ex/a> <http://ex/p> <http://ex/c> <http://ex/g> .\n";
2014 assert_eq!(
2015 parse_quads(nq).unwrap(),
2016 parse_reader(std::io::Cursor::new(nq), "nq", 0).unwrap()
2017 );
2018 // Turtle and TriG stream too — that is what lets the external build take a
2019 // multi-gigabyte `.ttl.gz` without expanding it to N-Triples first. Reading
2020 // a structured syntax from a reader must agree with parsing its whole text.
2021 let ttl = "@prefix ex: <http://ex/> .\nex:A ex:knows ex:B , ex:C .\n";
2022 assert_eq!(
2023 parse_statements(ttl, "ttl").unwrap(),
2024 parse_reader(std::io::Cursor::new(ttl), "ttl", 0).unwrap()
2025 );
2026 let trig = "@prefix ex: <http://ex/> .\nex:g { ex:A ex:knows ex:B . }\n";
2027 let streamed = parse_reader(std::io::Cursor::new(trig), "trig", 0).unwrap();
2028 assert_eq!(parse_statements(trig, "trig").unwrap(), streamed);
2029 assert_eq!(streamed[0].3.as_deref(), Some("<http://ex/g>"));
2030 // An unknown format is still a hard error, not a silent guess.
2031 assert!(parse_reader(std::io::Cursor::new(nt), "jsonld", 0).is_err());
2032 }
2033
2034 #[test]
2035 fn parse_statements_dispatches_by_format() {
2036 let nt = "<http://ex/a> <http://ex/p> <http://ex/b> .";
2037 assert_eq!(parse_statements(nt, "nt").unwrap().len(), 1);
2038 let ttl = "@prefix ex: <http://ex/> .\nex:A ex:knows ex:B , ex:C .";
2039 assert_eq!(parse_statements(ttl, "ttl").unwrap().len(), 2);
2040 // N-Triples is a subset of both Turtle and TriG, so the same text parses
2041 // under all three — and lands in the default graph under each.
2042 assert_eq!(parse_statements(nt, "trig").unwrap().len(), 1);
2043 assert!(parse_statements(nt, "trig").unwrap()[0].3.is_none());
2044 assert!(parse_statements(nt, "jsonld").is_err());
2045 }
2046
2047 /// A TriG graph block carries its graph name through as the same canonical
2048 /// token N-Quads would produce — so the two syntaxes agree on dictionary keys
2049 /// and a `.trig` shard federates with a `.nq` one.
2050 #[test]
2051 fn trig_graph_names_match_nquads() {
2052 let trig = "@prefix ex: <http://ex/> .\nex:g { ex:a ex:p ex:b . }\nex:a ex:p ex:c .\n";
2053 let nq = "<http://ex/a> <http://ex/p> <http://ex/b> <http://ex/g> .\n\
2054 <http://ex/a> <http://ex/p> <http://ex/c> .\n";
2055 let mut from_trig = parse_statements(trig, "trig").unwrap();
2056 let mut from_nq = parse_statements(nq, "nq").unwrap();
2057 from_trig.sort();
2058 from_nq.sort();
2059 assert_eq!(from_trig, from_nq);
2060 }
2061
2062 /// RDF/XML (how most OWL ontologies ship) parses to the same canonical tokens,
2063 /// including the abbreviated typed-node syntax and `rdf:resource` references.
2064 #[test]
2065 fn parses_rdfxml_owl() {
2066 let xml = r#"<?xml version="1.0"?>
2067 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
2068 xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
2069 xmlns:owl="http://www.w3.org/2002/07/owl#">
2070 <owl:Class rdf:about="http://ex/Dog">
2071 <rdfs:subClassOf rdf:resource="http://ex/Animal"/>
2072 <rdfs:label>Dog</rdfs:label>
2073 </owl:Class>
2074 </rdf:RDF>"#;
2075 let triples = parse_rdfxml(xml).unwrap();
2076 // rdf:type owl:Class, rdfs:subClassOf, rdfs:label = 3 triples.
2077 assert_eq!(triples.len(), 3);
2078 assert!(triples.iter().any(|(s, p, o)| s == "<http://ex/Dog>"
2079 && p == "<http://www.w3.org/2000/01/rdf-schema#subClassOf>"
2080 && o == "<http://ex/Animal>"));
2081 // Same data through the format dispatcher (triples → default graph).
2082 assert_eq!(parse_statements(xml, "rdfxml").unwrap().len(), 3);
2083 // Malformed XML is a clear RdfXml error, not a silent empty parse.
2084 assert!(matches!(
2085 parse_statements("<rdf:RDF><not closed", "rdfxml"),
2086 Err(IngestError::RdfXml(_))
2087 ));
2088 }
2089
2090 /// Regression coverage for RUSTSEC-2026-0195: namespace declarations from
2091 /// an untrusted RDF/XML document must not make parsing panic or exhaust the
2092 /// bounded test process.
2093 #[test]
2094 fn rdfxml_namespace_fanout_is_bounded() {
2095 let declarations = (0..2_000)
2096 .map(|i| format!(r#" xmlns:p{i}="http://example.test/{i}/""#))
2097 .collect::<String>();
2098 let xml = format!(
2099 r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"{declarations}/>"#
2100 );
2101 let result = parse_statements(&xml, "rdfxml");
2102 assert!(result.is_ok() || result.unwrap_err().to_string().contains("namespace"));
2103 }
2104
2105 /// Regression coverage for RUSTSEC-2026-0194: duplicate-name checking must
2106 /// complete for a large valid start tag without panicking or exhausting the
2107 /// bounded test process.
2108 #[test]
2109 fn rdfxml_attribute_fanout_completes() {
2110 let attributes = (0..2_000)
2111 .map(|i| format!(r#" p:a{i}="{i}""#))
2112 .collect::<String>();
2113 let xml = format!(
2114 r#"<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:p="http://example.test/"><rdf:Description rdf:about="http://example.test/s"{attributes}/></rdf:RDF>"#
2115 );
2116 assert!(parse_statements(&xml, "rdfxml").is_ok());
2117 }
2118
2119 /// Text in, queryable file image out — the whole in-memory build path the
2120 /// wasm `build()` binding uses, including a named graph.
2121 #[test]
2122 fn assemble_dataset_round_trips() {
2123 let text = "<http://ex/a> <http://ex/knows> <http://ex/b> .\n\
2124 <http://ex/b> <http://ex/knows> <http://ex/c> .\n\
2125 <http://ex/a> <http://ex/age> \"30\"^^<http://www.w3.org/2001/XMLSchema#integer> .\n\
2126 <http://ex/a> <http://ex/p> <http://ex/d> <http://ex/g1> .";
2127 let quads = parse_statements(text, "nq").unwrap();
2128 let (bytes, stats) = assemble_dataset(quads, &[]);
2129 assert_eq!(stats.statements, 4);
2130 assert_eq!(stats.default_triples, 3);
2131 assert_eq!(stats.named_graphs, 1);
2132 assert!(stats.terms >= 7);
2133
2134 let rete = Rete::open(&bytes).unwrap();
2135 assert_eq!(
2136 rete.query(None, Some("<http://ex/knows>"), None).len(),
2137 2,
2138 "default-graph pattern query"
2139 );
2140 assert_eq!(rete.graph_names(), vec!["<http://ex/g1>"]);
2141 let out = crate::eval_query(
2142 &rete,
2143 "SELECT ?x WHERE { ?x <http://ex/knows> ?y . ?y <http://ex/knows> ?z }",
2144 )
2145 .unwrap();
2146 match out {
2147 crate::QueryOutput::Select(_, rows) => assert_eq!(rows.len(), 1),
2148 other => panic!("expected select result, got {other:?}"),
2149 }
2150 }
2151
2152 /// The exact minimal input the playground Build tab uses: two triples, no
2153 /// literals, no rdf:type, no named graph — the smallest graph that still
2154 /// builds a community pyramid. (The wasm `build()` panic this guards was a
2155 /// `std::time::Instant::now()` in the pyramid timing path; native has a clock
2156 /// so this passes here, while the playground harness exercises the wasm path.)
2157 #[test]
2158 fn assemble_minimal_typeless_graph() {
2159 let text = "<http://ex/A> <http://ex/knows> <http://ex/B> .\n\
2160 <http://ex/B> <http://ex/knows> <http://ex/C> .\n";
2161 let quads = parse_statements(text, "nt").unwrap();
2162 let (bytes, stats) = assemble_dataset(quads, &[]);
2163 assert_eq!(stats.default_triples, 2);
2164 let rete = Rete::open(&bytes).unwrap();
2165 assert_eq!(rete.query(None, Some("<http://ex/knows>"), None).len(), 2);
2166 }
2167}