rete_wasm/lib.rs
1//! Browser bindings: load a `.rete` file's bytes (e.g. from `fetch`) and query
2//! it entirely in the browser — the same engine the native CLI uses, compiled
3//! to wasm. Results come back as JSON strings.
4
5use rete_core::{
6 batch_reach_serial, build_adjacency, build_dendrogram, choose_round_for_budget, eval_query,
7 eval_query_with_warnings, eval_select_communities, eval_sparql, project_graph, schema_classes,
8 schema_summary, summary_query_shape, tile_by_community, validate_shacl, BlockCacheReader,
9 ByteRange, CountingReader, DataGraph, Header, OffsetReader, QueryOpts, QueryOutput,
10 RangeReader, Rete, ReteGraph, ShaclShapes, SliceReader, SummaryQueryShape, SummaryView,
11 TermTriple, TripleProvenance, ValidationReport, DEFAULT_BLOCK, DEFAULT_TILE_BUDGET,
12};
13use std::rc::Rc;
14use wasm_bindgen::prelude::*;
15
16/// Version of Rete-owned JSON object envelopes exposed by the browser API.
17pub const JSON_SCHEMA_VERSION: u8 = 1;
18
19/// Module init: route Rust panics to `console.error` with their message and
20/// location. In release wasm a panic otherwise aborts as a bare
21/// `RuntimeError: unreachable` with no clue where — this turns that into a
22/// `rete-wasm panic: panicked at '…', src/…:line` line in the devtools console,
23/// so an intermittent first-query crash (e.g. a parser tripping on a flaky
24/// range read) can actually be diagnosed.
25#[wasm_bindgen(start)]
26pub fn __start() {
27 // The asyncify build must NOT format in the panic hook: the fmt machinery
28 // is asyncify-instrumented (everything that can reach panic_fmt is), so a
29 // panic raised while the instance is unwinding/rewinding recurses through
30 // garbage-returning fmt calls forever (stack overflow / null-function).
31 // Report the raw location pointers through a LEAF import instead — no
32 // formatting, no instrumented calls — then let panic=abort trap cleanly.
33 #[cfg(feature = "asyncify")]
34 std::panic::set_hook(Box::new(|info| {
35 if let Some(loc) = info.location() {
36 let f = loc.file();
37 unsafe { rete_panic_report(f.as_ptr(), f.len(), loc.line()) };
38 } else {
39 unsafe { rete_panic_report(std::ptr::null(), 0, 0) };
40 }
41 }));
42 #[cfg(not(feature = "asyncify"))]
43 std::panic::set_hook(Box::new(|info| {
44 web_sys::console::error_1(&JsValue::from_str(&format!("rete-wasm panic: {info}")));
45 }));
46}
47
48/// Header summary as JSON: `{ "quads": N, "terms": N, "pyramidLevels": N }`.
49#[wasm_bindgen]
50pub fn info(bytes: &[u8]) -> Result<String, JsValue> {
51 let rete = open(bytes)?;
52 let h = rete.header();
53 Ok(format!(
54 r#"{{"schemaVersion":1,"quads":{},"terms":{},"pyramidLevels":{},"namedGraphs":{}}}"#,
55 h.quad_count,
56 h.term_count,
57 h.pyramid_levels,
58 rete.named_graph_count()
59 ))
60}
61
62/// The named-graph IRIs of a dataset, as a JSON array.
63#[wasm_bindgen]
64pub fn graph_names(bytes: &[u8]) -> Result<String, JsValue> {
65 let rete = open(bytes)?;
66 serde_json::to_string(&rete.graph_names()).map_err(err)
67}
68
69/// Build a complete `.rete` file image from RDF text, entirely in the browser.
70///
71/// `format` is `"nt"` (N-Triples), `"nq"` (N-Quads; named graphs become a
72/// dataset), or `"ttl"` (Turtle). Returns the file bytes (a `Uint8Array`),
73/// ready to download or to hand straight back to the query functions. The wasm
74/// build has no zstd *encoder*, so sections are written uncompressed (codec
75/// `NONE`) — larger than a CLI build of the same data, but every reader
76/// accepts it; rebuild with `rete build` for a compressed file.
77#[wasm_bindgen]
78pub fn build(text: &str, format: &str) -> Result<Vec<u8>, JsValue> {
79 let quads = rete_core::ingest::parse_statements(text, format).map_err(err)?;
80 if quads.is_empty() {
81 return Err(js_error(
82 "no statements parsed (empty input or only comments)",
83 ));
84 }
85 let read = quads.len() as u64;
86 let (bytes, stats) = rete_core::ingest::assemble_dataset_quad(quads, &[]);
87 finish_build(
88 bytes,
89 "build",
90 format,
91 ("string", text.as_bytes()),
92 &stats,
93 read,
94 )
95}
96
97/// The file every browser build returns: the in-RAM `0x05` image laid out as
98/// `0x06` (the one step the CLI and every binding share,
99/// [`rete_core::build_record::finish_in_ram`]), with the build record beside
100/// the card: engine, options, the input (name, length, BLAKE3), counts, and
101/// `front_end: "wasm"`. The browser's clock is the only one there is, so the
102/// timestamp comes from it.
103fn finish_build(
104 bytes: Vec<u8>,
105 command: &str,
106 format: &str,
107 input: (&str, &[u8]),
108 stats: &rete_core::ingest::BuildStats,
109 read: u64,
110) -> Result<Vec<u8>, JsValue> {
111 use rete_core::build_record as br;
112 let built_at = js_sys::Date::new_0()
113 .to_iso_string()
114 .as_string()
115 .filter(|s| s.len() >= 19)
116 .map(|s| format!("{}Z", &s[..19]));
117 let record = br::to_json(
118 &br::Record {
119 front_end: br::front_end::WASM.into(),
120 front_end_version: env!("CARGO_PKG_VERSION").into(),
121 built_at,
122 options: br::Options {
123 command: command.into(),
124 format: Some(format.into()),
125 pyramid_algo: Some("louvain".into()),
126 ..Default::default()
127 },
128 inputs: vec![br::Input::from_bytes(input.0, format, input.1)],
129 statements_read: Some(read),
130 },
131 Some(stats),
132 );
133 br::finish_in_ram(bytes, Some(&record)).map_err(err)
134}
135
136/// [`build`], but the file carries a **Dataset Card** written from
137/// `card_json` — the same document `rete build --card-file` takes, validated
138/// by the same rules ([`rete_core::card::validate_curated_card`]), so a card
139/// authored in the browser is one the CLI would also have accepted.
140///
141/// What the browser can and cannot put in a card, stated plainly because the
142/// difference matters to whoever reads the file afterwards:
143///
144/// - **Curated fields travel in full** — title, description, licence, source,
145/// version, creators, publisher, DOI, citation, keywords, theme, the `extra`
146/// bag, everything on [`rete_core::card::CURATED_CARD_FIELDS`].
147/// - **The four counts are measured, not asserted**: `triple_count`,
148/// `quad_count`, `named_graph_count` and `term_count` come from the build's
149/// own [`BuildStats`](rete_core::ingest::BuildStats), and any values supplied
150/// for them would be ignored (they are not curated fields, so supplying them
151/// is already an error). `format_version` is stamped by the writer.
152/// - **The derived profile is NOT written.** Predicates, classes,
153/// vocabularies, datatypes, languages, class links, hubs, signals and the
154/// tiered starter-query library are absent. Their absence is honest absence:
155/// the card simply does not carry those keys, exactly as a `rete merge` card
156/// does not. Call [`build_with_derived_card`] instead to compute them here —
157/// this function stays curated-only so its bytes never change under a caller
158/// who did not ask for the extra passes.
159/// - **No build-info section** (kind 7) is written: its cost figures come from
160/// measuring the starter queries, and there are none to measure.
161///
162/// Pass an empty string for no card — byte-identical to [`build`].
163#[wasm_bindgen]
164pub fn build_with_card(text: &str, format: &str, card_json: &str) -> Result<Vec<u8>, JsValue> {
165 build_card_surface(
166 text,
167 format,
168 card_json,
169 rete_core::ingest::QuotedTripleSurface::default(),
170 )
171}
172
173/// [`build_with_card`], reading Turtle/TriG `<< … >>` in the named surface —
174/// the browser half of `rete build --quoted-triple-syntax`, same two values:
175///
176/// - `"rdf-star"` (or `""`): `<< s p o >>` is a quoted triple. Byte-identical
177/// to [`build_with_card`].
178/// - `"rdf12"`: the **RDF 1.2** reader. `<<( s p o )>>` is a triple term,
179/// `<< s p o >>` a reifier (a blank node `rdf:reifies` the triple term),
180/// `{| … |}` an annotation, `"…"@lang--dir` a directional literal.
181///
182/// The same Turtle file is two different graphs under the two values, and
183/// nothing in the bytes says which was meant, so the caller chooses. N-Triples
184/// and N-Quads ignore the choice (their reader takes both spellings). Anything
185/// else is refused by name.
186#[wasm_bindgen]
187pub fn build_with_card_syntax(
188 text: &str,
189 format: &str,
190 card_json: &str,
191 quoted_triple_syntax: &str,
192) -> Result<Vec<u8>, JsValue> {
193 let surface = if quoted_triple_syntax.is_empty() {
194 rete_core::ingest::QuotedTripleSurface::default()
195 } else {
196 rete_core::ingest::QuotedTripleSurface::parse(quoted_triple_syntax).ok_or_else(|| {
197 js_error(format!(
198 "unknown quoted-triple syntax {quoted_triple_syntax:?} (expected \"rdf-star\" or \"rdf12\")"
199 ))
200 })?
201 };
202 build_card_surface(text, format, card_json, surface)
203}
204
205fn build_card_surface(
206 text: &str,
207 format: &str,
208 card_json: &str,
209 surface: rete_core::ingest::QuotedTripleSurface,
210) -> Result<Vec<u8>, JsValue> {
211 let quads = rete_core::ingest::parse_statements_audited_surface(text, format, None, surface)
212 .map_err(err)?;
213 build_card_quads(quads, ("string", text.as_bytes()), format, card_json)
214}
215
216/// The tail of [`build_card_surface`], after the input is parsed: shared with
217/// [`build_bytes`], whose Jelly input has no text form.
218fn build_card_quads(
219 quads: Vec<rete_core::ingest::RawQuad>,
220 input: (&str, &[u8]),
221 format: &str,
222 card_json: &str,
223) -> Result<Vec<u8>, JsValue> {
224 if quads.is_empty() {
225 return Err(js_error(
226 "no statements parsed (empty input or only comments)",
227 ));
228 }
229 if card_json.trim().is_empty() {
230 let read = quads.len() as u64;
231 let (bytes, stats) = rete_core::ingest::assemble_dataset_quad(quads, &[]);
232 return finish_build(bytes, "build_with_card", format, input, &stats, read);
233 }
234 let curated = validated_card(card_json)?;
235 let read = quads.len() as u64;
236 // The counts are only known once the dictionary and indexes exist, so the
237 // card is serialized from inside the writer rather than handed in whole.
238 let (bytes, stats) = rete_core::ingest::assemble_dataset_with_quad(quads, move |stats, _| {
239 let card = rete_core::card::compose_curated_card(
240 curated,
241 stats.default_triples as u64,
242 stats.statements as u64,
243 stats.named_graphs as u64,
244 stats.terms as u64,
245 rete_core::LEGACY_FORMAT_VERSION,
246 );
247 serde_json::to_vec(&card).expect("card serializes")
248 });
249 finish_build(bytes, "build_with_card", format, input, &stats, read)
250}
251
252/// [`build_with_card_syntax`] for input held as **bytes**: any text format
253/// (UTF-8, as `text` would be) or `"jelly"`, the binary Jelly RDF stream
254/// format, which has no text form and so cannot go through the `&str`
255/// builders. Same card and quoted-triple rules, same file out. The Jelly
256/// reader is the one `rete build` uses ([`rete_core::ingest::parse_jelly_bytes`]);
257/// the whole file must be in memory, as every browser build is.
258#[wasm_bindgen]
259pub fn build_bytes(
260 bytes: &[u8],
261 format: &str,
262 card_json: &str,
263 quoted_triple_syntax: &str,
264) -> Result<Vec<u8>, JsValue> {
265 if format != "jelly" {
266 let text = std::str::from_utf8(bytes)
267 .map_err(|e| js_error(format!("input is not valid UTF-8 text: {e}")))?;
268 return build_with_card_syntax(text, format, card_json, quoted_triple_syntax);
269 }
270 let quads = rete_core::ingest::parse_jelly_bytes(bytes).map_err(err)?;
271 build_card_quads(quads, ("bytes", bytes), format, card_json)
272}
273
274/// [`build_with_card`], but the card also carries the **auto-derived profile**
275/// — the half a browser build used to have to do without (#152).
276///
277/// Predicates, classes, vocabularies, datatypes, languages, the class-link
278/// quotient, hubs, the affordance signals, and the tiered starter-query
279/// library are all computed here, by exactly the code `rete build --card`
280/// runs ([`rete_core::card::derive_card`]). On the same graph with the same
281/// curated document, the metadata section this writes is **byte-identical** to
282/// the CLI's.
283///
284/// Two honest differences remain, and neither is derivation:
285///
286/// - **Sections are uncompressed** (the wasm build has no zstd *encoder*), so
287/// the file is larger than a CLI build of the same graph. Every reader
288/// accepts it.
289/// - **The build record has no starter-query costs**: they come from *running*
290/// the starter queries, which is a benchmark, not a build. Everything else
291/// in the record is there (see `finish_build`).
292///
293/// # Why this is a separate function
294///
295/// Derivation walks the graph twice more. In a browser, on a paste the user is
296/// waiting on, that is a cost they should choose — so [`build_with_card`]
297/// keeps writing exactly the bytes it always has, and this is the opt-in.
298///
299/// Pass an empty string for `card_json` to derive a profile-only card with no
300/// curated fields (the equivalent of a bare `rete build --card`).
301#[wasm_bindgen]
302pub fn build_with_derived_card(
303 text: &str,
304 format: &str,
305 card_json: &str,
306) -> Result<Vec<u8>, JsValue> {
307 let quads = rete_core::ingest::parse_statements(text, format).map_err(err)?;
308 if quads.is_empty() {
309 return Err(js_error(
310 "no statements parsed (empty input or only comments)",
311 ));
312 }
313 // The typed curated half, held to the same rules `--card-file` is held to.
314 let curated = if card_json.trim().is_empty() {
315 rete_core::card::CardInput::default()
316 } else {
317 // Validate through the document validator first, so the wording a
318 // browser author sees is the wording `validate_card` shows them while
319 // they type, not a raw serde message.
320 validated_card(card_json)?;
321 rete_core::card::CardInput::from_json_str(card_json).map_err(js_error)?
322 };
323 let read = quads.len() as u64;
324 let (bytes, stats) =
325 rete_core::ingest::assemble_dataset_with_quad(quads, move |stats, quads| {
326 let card = rete_core::card::derive_card(
327 quads,
328 stats.terms as u64,
329 stats.named_graphs as u64,
330 curated,
331 );
332 // The counts the file actually holds are known only once the indexes
333 // have deduplicated the input — the same two-stage stamp the CLI uses.
334 rete_core::ingest::DeferredMetadata::new(move |counts| {
335 card.with_final_counts(counts).to_json_bytes()
336 })
337 });
338 finish_build(
339 bytes,
340 "build_with_derived_card",
341 format,
342 ("string", text.as_bytes()),
343 &stats,
344 read,
345 )
346}
347
348/// Check a curated card document without building anything — so an editor can
349/// report the **exact** error `rete build --card-file` would report, while the
350/// author is still typing. Returns the empty string when the document is
351/// valid, otherwise the error message.
352///
353/// Deliberately not a boolean: the wording is the useful part (a free-text
354/// `theme` is told to use `keywords`; a stray top-level key is told about the
355/// `extra` bag), and duplicating that wording in JavaScript is exactly how the
356/// two writers would drift apart again.
357#[wasm_bindgen]
358pub fn validate_card(card_json: &str) -> String {
359 check_card(card_json).err().unwrap_or_default()
360}
361
362/// Parse + validate a curated card document. The message stays a `String` all
363/// the way through — wrapping it in a JS `Error` first and unwrapping it later
364/// loses the wording, which is the part worth returning.
365fn check_card(card_json: &str) -> Result<serde_json::Value, String> {
366 let doc: serde_json::Value =
367 serde_json::from_str(card_json).map_err(|e| format!("card is not JSON: {e}"))?;
368 rete_core::card::validate_curated_card(&doc)
369}
370
371/// [`check_card`] as a JS exception, for the build path.
372fn validated_card(card_json: &str) -> Result<serde_json::Value, JsValue> {
373 check_card(card_json).map_err(js_error)
374}
375
376/// Evaluate a triple pattern; `null`/`undefined` positions are wildcards.
377/// Returns a JSON array of `[subject, predicate, object]` triples.
378#[wasm_bindgen]
379pub fn query_triples(
380 bytes: &[u8],
381 subject: Option<String>,
382 predicate: Option<String>,
383 object: Option<String>,
384) -> Result<String, JsValue> {
385 let rete = open(bytes)?;
386 let rows = rete.query(subject.as_deref(), predicate.as_deref(), object.as_deref());
387 serde_json::to_string(&rows).map_err(err)
388}
389
390/// Explain why each triple-pattern match is present in the `.rete` file.
391///
392/// `null`/`undefined` positions are wildcards. The JSON uses browser-facing
393/// camelCase fields:
394/// `{ "pattern", "resultCount", "results": [{ "terms", "ids", "provenance" }] }`.
395#[wasm_bindgen]
396pub fn why_triples(
397 bytes: &[u8],
398 subject: Option<String>,
399 predicate: Option<String>,
400 object: Option<String>,
401) -> Result<String, JsValue> {
402 why_triples_json(
403 bytes,
404 subject.as_deref(),
405 predicate.as_deref(),
406 object.as_deref(),
407 )
408 .map_err(js_error)
409}
410
411/// Native-testable implementation for [`why_triples`].
412pub fn why_triples_json(
413 bytes: &[u8],
414 subject: Option<&str>,
415 predicate: Option<&str>,
416 object: Option<&str>,
417) -> Result<String, String> {
418 use serde_json::json;
419
420 let rete = Rete::open(bytes).map_err(|e| e.to_string())?;
421 let results = rete.query_with_provenance(subject, predicate, object);
422 let out = json!({
423 "schemaVersion": JSON_SCHEMA_VERSION,
424 "pattern": {
425 "subject": subject,
426 "predicate": predicate,
427 "object": object,
428 },
429 "resultCount": results.len(),
430 "results": results.iter().map(provenance_json).collect::<Vec<_>>(),
431 });
432 serde_json::to_string(&out).map_err(|e| e.to_string())
433}
434
435/// Run a SPARQL SELECT; returns a JSON array of solution objects
436/// (`{ "var": "value", ... }`).
437#[wasm_bindgen]
438pub fn query_sparql(bytes: &[u8], query: &str) -> Result<String, JsValue> {
439 let rete = open(bytes)?;
440 let (_vars, solutions) = eval_sparql(&rete, query).map_err(err)?;
441 serde_json::to_string(&solutions).map_err(err)
442}
443
444/// The ontology profile (the semantic coarse graph), as JSON:
445/// `{ "classes": [["<iri>", count], ...],
446/// "relations": [["sClass","pred","oClass", count], ...] }`.
447/// The "overview first" payload — render it before fetching any detail.
448#[wasm_bindgen]
449pub fn schema(bytes: &[u8]) -> Result<String, JsValue> {
450 let rete = open(bytes)?;
451 let classes = schema_classes(&rete);
452 let relations = schema_summary(&rete);
453 serde_json::to_string(&serde_json::json!({
454 "schemaVersion": JSON_SCHEMA_VERSION,
455 "classes": classes,
456 "relations": relations,
457 }))
458 .map_err(err)
459}
460
461/// Read the **baked** schema pyramid (classes + class-level relations) straight
462/// from the file's schema block via a slice reader — no triple scan. The
463/// in-memory twin of [`schema_url`]: a cached big graph gets its schema from a
464/// few KB of the buffer instead of dumping every triple (seconds on a 150 MB
465/// file). Errors when the file carries no schema pyramid, so callers fall back
466/// to the scanning [`schema`].
467#[wasm_bindgen]
468pub fn schema_packed(bytes: &[u8]) -> Result<String, JsValue> {
469 use serde_json::json;
470 let (classes, relations) = rete_core::read_schema_summary_ranged(&SliceReader::new(bytes))
471 .map_err(err)?
472 .ok_or_else(|| js_error("file has no schema pyramid"))?;
473 let classes: Vec<serde_json::Value> = classes.iter().map(|(c, n)| json!([c, n])).collect();
474 let relations: Vec<serde_json::Value> = relations
475 .iter()
476 .map(|(s, p, o, n)| json!([s, p, o, n]))
477 .collect();
478 Ok(json!({
479 "schemaVersion": JSON_SCHEMA_VERSION,
480 "classes": classes,
481 "relations": relations,
482 })
483 .to_string())
484}
485
486/// The embedded **Dataset Card** — the file's own self-description (title,
487/// description, license, provenance, counts, example queries) as the JSON text
488/// it was written with, or `undefined` when the file carries none. Reads the
489/// metadata section straight out of the buffer.
490#[wasm_bindgen]
491pub fn card(bytes: &[u8]) -> Result<Option<String>, JsValue> {
492 let bytes = rete_core::read_metadata_ranged(&SliceReader::new(bytes)).map_err(err)?;
493 Ok(bytes.map(|b| String::from_utf8_lossy(&b).into_owned()))
494}
495
496/// The embedded **Dataset Card of a remote `.rete`**, in **two small range
497/// requests**: the header, then the metadata section it points at — never the
498/// dictionary, index, or pyramid. This is the index-free CARD tier: a client
499/// learns what a multi-gigabyte graph *is* for a few KB. `undefined` when the
500/// file carries no card. Worker-only (synchronous XHR).
501#[wasm_bindgen]
502pub fn card_url(url: &str) -> Result<Option<String>, JsValue> {
503 let reader = RemoteReader::open(url)?.view();
504 let bytes = rete_core::read_metadata_ranged(&reader).map_err(err)?;
505 Ok(bytes.map(|b| String::from_utf8_lossy(&b).into_owned()))
506}
507
508/// The Dataset Card **and the build record** of a remote `.rete`, in the same
509/// budget as the card alone: one header read, then **one coalesced range**
510/// covering both sections — the writer lays the kind-7 build-info immediately
511/// after the metadata precisely so this holds
512/// ([`rete_core::range::read_card_and_build_info_ranged`], pinned by a
513/// `rete-core` test). Reading the two separately would have made the CARD tier
514/// cost three requests instead of two, which is why there is one export rather
515/// than a second `build_info_url`.
516///
517/// JSON envelope:
518/// `{"schemaVersion":1,"card":<text|null>,"build":<text|null>,"text_index":{…}}`.
519/// `card` and `build` are the sections' **own bytes** as text, not a
520/// re-serialization — the card a client displays is the card the file holds.
521/// `text_index` is the one thing the file does *not* store about itself and this
522/// reader measures instead (see the private `text_index_json`). Worker-only
523/// (synchronous XHR).
524#[wasm_bindgen]
525pub fn card_and_build_url(url: &str) -> Result<String, JsValue> {
526 // The polyglot-aware view: base 0 for a plain .rete, shifted past the HTML
527 // shell for a polyglot — so the card of an embedded graph is read with the
528 // same two requests as a standalone file's.
529 let reader = RemoteReader::open(url)?.view();
530 card_build_from(&reader)
531}
532
533/// [`card_and_build_url`] for an image already in memory — no I/O at all.
534#[wasm_bindgen]
535pub fn card_and_build(bytes: &[u8]) -> Result<String, JsValue> {
536 card_build_from(&SliceReader::new(bytes))
537}
538
539/// The shared body of the two: header (which the card read needs anyway), the
540/// coalesced card + build-info range, and the measured text-index signal — so
541/// the CARD tier stays at one header read plus one coalesced range, plus the
542/// ≤10-byte probe only when there is an index to measure.
543fn card_build_from<R: RangeReader>(reader: &R) -> Result<String, JsValue> {
544 let (header, card, build) =
545 rete_core::read_card_and_build_info_with_header(reader).map_err(err)?;
546 let token_table = rete_core::read_text_index_token_table_len_ranged(reader, &header);
547 Ok(card_build_envelope(
548 card,
549 build,
550 text_index_json(&header, token_table),
551 ))
552}
553
554/// Whether a `.rete` carries a **full-text (TEXT_INDEX) section**, measured from
555/// the section directory in the 1 KiB header the caller has just read:
556/// `{"present":bool,"bytes":N,"token_table_bytes":N}` (the last two only when
557/// there is an index).
558///
559/// This is not in the card, and deliberately so. A file built with
560/// `--text-index` answers `FILTER(CONTAINS(…))` by word lookup; one built
561/// without it answers the *same query with the same rows* by full scan, so the
562/// capability is invisible from the results — and a stored flag would be a claim
563/// that can outlive the section it describes. Measuring costs the header (already
564/// fetched) plus one ≤10-byte range read for the token-table length: the figure
565/// a first search actually pays, several times smaller than the whole section.
566///
567/// `token_table_bytes` is omitted when it was not measured — absent means
568/// "not measured here", never "zero".
569fn text_index_json(header: &Header, token_table_bytes: Option<u64>) -> serde_json::Value {
570 if header.text_index_len == 0 {
571 return serde_json::json!({ "present": false });
572 }
573 let mut v = serde_json::json!({
574 "present": true,
575 "bytes": header.text_index_len,
576 });
577 if let Some(tt) = token_table_bytes {
578 v.as_object_mut()
579 .expect("an object")
580 .insert("token_table_bytes".into(), serde_json::json!(tt));
581 }
582 v
583}
584
585/// The `{card, build, text_index}` envelope both readers return. Absent sections
586/// are `null`, never `""` or `{}`: a file built before build-info existed has no
587/// build record, and a reader must be able to tell that from one that recorded
588/// nothing. `text_index` is always an object — it is measured, so there is
589/// always an answer, including for a file with no card at all.
590fn card_build_envelope(
591 card: Option<Vec<u8>>,
592 build: Option<Vec<u8>>,
593 text_index: serde_json::Value,
594) -> String {
595 let text = |b: Option<Vec<u8>>| match b {
596 Some(b) if !b.is_empty() => {
597 serde_json::Value::String(String::from_utf8_lossy(&b).into_owned())
598 }
599 _ => serde_json::Value::Null,
600 };
601 serde_json::json!({
602 "schemaVersion": JSON_SCHEMA_VERSION,
603 "card": text(card),
604 "build": text(build),
605 "text_index": text_index,
606 })
607 .to_string()
608}
609
610/// The **true byte length of a remote `.rete`**, in 1–2 tiny range requests —
611/// derived from the file's *own* header (the issue-#95 probe: sections are
612/// back-to-back and the file ends with the 4-byte `RETE` footer), never from
613/// the transport's numbers, which may describe a compressed representation
614/// (GitHub Pages HEADs a 71 MB file as its 58 MB gzip) or be hidden from
615/// cross-origin JS entirely. This is how a UI can say what "download the whole
616/// file" actually costs **before** committing to it.
617/// JSON: `{ "schemaVersion": 1, "fileLength": <bytes> }`. Worker-only
618/// (synchronous XHR in the sync build).
619#[wasm_bindgen]
620pub fn file_len_url(url: &str) -> Result<String, JsValue> {
621 // The GRAPH's length: for a polyglot that is the appended `.rete`, not the
622 // web page wrapped around it — "download the whole file" must quote the
623 // bytes a `.rete` consumer would actually take.
624 let reader = RemoteReader::open(url)?;
625 Ok(format!(
626 r#"{{"schemaVersion":{},"fileLength":{}}}"#,
627 JSON_SCHEMA_VERSION,
628 reader.len()
629 ))
630}
631
632/// A `.rete` opened **once** and kept resident, so a client (the playground's
633/// cached/in-memory mode) can run many queries on a big file without re-copying
634/// the whole buffer into wasm and re-decoding its dictionary on every call. The
635/// methods mirror the free functions above but operate on the already-open
636/// [`Rete`]. The few index-free readers (`schema_packed`, `progressive_query`,
637/// `check_schema`) stay free functions — they read small ranges from the buffer
638/// and are called rarely (once at load / on demand), so a handle buys little.
639#[wasm_bindgen]
640pub struct Graph {
641 rete: Rc<Rete>,
642 file_len: usize,
643 /// The kind-7 build record's own bytes, lifted at open time. `Rete` keeps
644 /// the metadata section but not this one (it is outside the content hash
645 /// and no query needs it), and the handle does not retain the buffer — so
646 /// it is read once here rather than made unreachable.
647 build_info: Option<String>,
648}
649
650#[wasm_bindgen]
651impl Graph {
652 /// Open a `.rete` image and keep it resident for repeated querying.
653 #[wasm_bindgen(constructor)]
654 pub fn new(bytes: &[u8]) -> Result<Graph, JsValue> {
655 Ok(Graph {
656 rete: Rc::new(open(bytes)?),
657 file_len: bytes.len(),
658 build_info: rete_core::read_build_info(bytes)
659 .ok()
660 .flatten()
661 .filter(|b| !b.is_empty())
662 .map(|b| String::from_utf8_lossy(&b).into_owned()),
663 })
664 }
665
666 /// A **lazy, resumable cursor** over the quads of this graph — the streaming
667 /// export path. See [`QuadCursor`]; `graph` selects one graph (`""` = the
668 /// default graph), `None` streams the default graph followed by every named
669 /// graph. `s` / `p` / `o` optionally restrict the dump to a triple pattern,
670 /// which **prunes tiles** rather than filtering rows.
671 pub fn quads(
672 &self,
673 graph: Option<String>,
674 s: Option<String>,
675 p: Option<String>,
676 o: Option<String>,
677 ) -> QuadCursor {
678 QuadCursor::start(self.rete.clone(), graph, s, p, o)
679 }
680
681 /// See [`info`].
682 pub fn info(&self) -> String {
683 let h = self.rete.header();
684 format!(
685 r#"{{"schemaVersion":1,"quads":{},"terms":{},"pyramidLevels":{},"namedGraphs":{}}}"#,
686 h.quad_count,
687 h.term_count,
688 h.pyramid_levels,
689 self.rete.named_graph_count()
690 )
691 }
692
693 /// See [`graph_names`].
694 pub fn graph_names(&self) -> Result<String, JsValue> {
695 serde_json::to_string(&self.rete.graph_names()).map_err(err)
696 }
697
698 /// See [`query`].
699 pub fn query(&self, query: &str, format: &str) -> Result<String, JsValue> {
700 query_json(&self.rete, query, format, "")
701 }
702
703 /// As [`query`], with OWL 2 QL entailment on (`rdfs:subClassOf` /
704 /// `subPropertyOf` / `domain` / `range` reasoning by query rewriting).
705 pub fn query_reasoned(&self, query: &str, format: &str) -> Result<String, JsValue> {
706 query_json_reasoned(&self.rete, query, format, "")
707 }
708
709 /// As [`query`], with explicit opt-in toggles: `reason` (OWL 2 QL
710 /// entailment) and `union_default` (union default graph — a pattern
711 /// outside `GRAPH` matches the merge of the default graph and every named
712 /// graph, the Virtuoso / GraphDB / Jena TDB mode; non-standard, so plain
713 /// [`Graph::query`] never does this).
714 pub fn query_opts(
715 &self,
716 query: &str,
717 format: &str,
718 reason: bool,
719 union_default: bool,
720 ) -> Result<String, JsValue> {
721 query_json_with(&self.rete, query, format, "", reason, union_default)
722 }
723
724 /// Run a SELECT to a **result handle**: the rows stay dictionary ids and
725 /// [`QueryRows::page`] decodes a window on demand. See [`QueryRows`].
726 pub fn query_rows(
727 &self,
728 query: &str,
729 reason: bool,
730 union_default: bool,
731 ) -> Result<QueryRows, JsValue> {
732 QueryRows::run(&self.rete, query, reason, union_default, false)
733 }
734
735 /// Plan running `query` as chunks split by the values of `var` — see
736 /// `rete_core::partition_plan`. JSON `{var, form, keys_query,
737 /// keys_from_where, offset, limit, dedupe}`, or an error saying why the
738 /// split would change the answer.
739 pub fn partition_plan(&self, query: &str, var: &str) -> Result<String, JsValue> {
740 partition_plan_json(&self.rete, query, var)
741 }
742
743 /// One chunk of a split query: the solutions whose `var` is one of `keys`
744 /// (term tokens, as the plan's keys query returned them), without the
745 /// query's top-level OFFSET/LIMIT. Same envelope as [`Graph::query`].
746 pub fn query_partition(
747 &self,
748 query: &str,
749 format: &str,
750 reason: bool,
751 var: &str,
752 keys: Vec<String>,
753 ) -> Result<String, JsValue> {
754 query_partition_json(&self.rete, query, format, reason, var, &keys)
755 }
756
757 /// See [`query_triples`].
758 pub fn query_triples(
759 &self,
760 subject: Option<String>,
761 predicate: Option<String>,
762 object: Option<String>,
763 ) -> Result<String, JsValue> {
764 let rows = self
765 .rete
766 .query(subject.as_deref(), predicate.as_deref(), object.as_deref());
767 serde_json::to_string(&rows).map_err(err)
768 }
769
770 /// See [`prefix_search`].
771 pub fn prefix_search(&self, prefix: &str, limit: usize) -> Result<String, JsValue> {
772 prefix_search_json(&self.rete, prefix, limit)
773 }
774
775 /// See [`text_search`].
776 pub fn text_search(
777 &self,
778 words: Vec<String>,
779 contains_prefix: Option<String>,
780 limit: usize,
781 ) -> Result<String, JsValue> {
782 text_search_json(&self.rete, &words, contains_prefix.as_deref(), limit)
783 }
784
785 /// Byte length of the TEXT_INDEX section, `0` when the file has none — a
786 /// header read, never a fault. The UI asks this to decide whether to offer
787 /// full-text search at all, and to state the cost before the first one.
788 /// `f64` because the section outgrows `u32`: causenet's is 1.88 GB.
789 pub fn text_index_len(&self) -> f64 {
790 self.rete.header().text_index_len as f64
791 }
792
793 /// Byte length of the TEXT_INDEX's leading **token table** — what a first
794 /// [`Graph::text_search_one`] actually faults, and therefore the only honest
795 /// number to quote as its cost. [`Graph::text_index_len`] is the whole
796 /// section, postings blob included, and overstates it 6.5× on
797 /// `epfl-infoscience` (195 MB section, 29 MB token table); the postings are
798 /// only ever fetched one list at a time. `0` when the file has no text index
799 /// or the length could not be read — the caller must then say nothing about
800 /// a token table rather than pass the section length off as one.
801 /// `f64` for the same reason as the section length: causenet's table is
802 /// 1.88 GB.
803 pub fn text_index_token_table_len(&self) -> f64 {
804 self.rete.text_index_token_table_len().unwrap_or(0) as f64
805 }
806
807 /// [`Graph::text_search`] from ONE phrase: whitespace splits it into words
808 /// and **every** word must match (AND), like `rete search --contains a b`.
809 /// One string in, one string out — that is what the remote twin's
810 /// hand-marshaled asyncify path can carry (a JS array marshaled raw is what
811 /// traps), and the UI is a single text box either way. Same JSON envelope.
812 pub fn text_search_one(&self, phrase: &str, limit: usize) -> Result<String, JsValue> {
813 let words: Vec<String> = phrase.split_whitespace().map(str::to_owned).collect();
814 text_search_json(&self.rete, &words, None, limit)
815 }
816
817 /// See [`why_triples`].
818 pub fn why_triples(
819 &self,
820 subject: Option<String>,
821 predicate: Option<String>,
822 object: Option<String>,
823 ) -> Result<String, JsValue> {
824 use serde_json::json;
825 let results = self.rete.query_with_provenance(
826 subject.as_deref(),
827 predicate.as_deref(),
828 object.as_deref(),
829 );
830 let out = json!({
831 "schemaVersion": JSON_SCHEMA_VERSION,
832 "pattern": { "subject": subject, "predicate": predicate, "object": object },
833 "resultCount": results.len(),
834 "results": results.iter().map(provenance_json).collect::<Vec<_>>(),
835 });
836 serde_json::to_string(&out).map_err(err)
837 }
838
839 /// See [`schema`] — the live (scanning) profile; prefer `schema_packed` when
840 /// the file carries a pyramid.
841 pub fn schema(&self) -> Result<String, JsValue> {
842 serde_json::to_string(&serde_json::json!({
843 "schemaVersion": JSON_SCHEMA_VERSION,
844 "classes": schema_classes(&self.rete),
845 "relations": schema_summary(&self.rete),
846 }))
847 .map_err(err)
848 }
849
850 /// The card's `signals.quoted_triples` for this file, from its header: does
851 /// it hold quoted triples, and in which surfaces can an export write them.
852 pub fn quoted_triples_signal(&self) -> String {
853 quoted_triples_signal_json(self.rete.header())
854 }
855
856 /// See [`card`] — the Dataset Card of the resident file.
857 pub fn card(&self) -> Option<String> {
858 self.rete
859 .metadata()
860 .map(|b| String::from_utf8_lossy(b).into_owned())
861 }
862
863 /// See [`card_and_build`] — the card and the build record of the resident
864 /// file, in the same envelope the remote path returns, so one caller
865 /// handles both sources.
866 pub fn card_and_build(&self) -> String {
867 let card = self
868 .rete
869 .metadata()
870 .filter(|b| !b.is_empty())
871 .map(|b| b.to_vec());
872 // A resident open already decoded the whole TEXT_INDEX section, so both
873 // figures are free here.
874 let text_index =
875 text_index_json(self.rete.header(), self.rete.text_index_token_table_len());
876 card_build_envelope(
877 card,
878 self.build_info.as_ref().map(|s| s.as_bytes().to_vec()),
879 text_index,
880 )
881 }
882
883 /// See [`query_communities`].
884 pub fn query_communities(&self, query: &str, round: Option<usize>) -> Result<String, JsValue> {
885 query_communities_value(&self.rete, query, round)
886 }
887
888 /// See [`reach`].
889 pub fn reach(&self, predicate: &str, seeds: &str, reverse: bool) -> Result<String, JsValue> {
890 reach_rete(&self.rete, predicate, seeds, reverse)
891 }
892
893 /// See [`shacl`].
894 pub fn shacl(
895 &self,
896 shapes_turtle: &str,
897 graph: Option<String>,
898 format: &str,
899 ) -> Result<String, JsValue> {
900 shacl_rete(&self.rete, shapes_turtle, graph.as_deref(), format)
901 }
902
903 /// See [`reason`].
904 pub fn reason(&self, graph: Option<String>) -> String {
905 let base = self.rete.dump(graph.as_deref());
906 reasoning_json(&rete_core::reason(&base), None)
907 }
908
909 /// See [`pyramid_tree`].
910 pub fn pyramid_tree(&self) -> Result<String, JsValue> {
911 pyramid_tree_value(&self.rete)
912 }
913
914 /// See [`file_layout`].
915 pub fn file_layout(&self) -> Result<String, JsValue> {
916 use serde_json::json;
917 let segments: Vec<serde_json::Value> = self
918 .rete
919 .file_layout()
920 .iter()
921 .map(|s| json!({ "kind": s.kind, "label": s.label, "offset": s.offset, "len": s.len }))
922 .collect();
923 serde_json::to_string(&json!({
924 "schemaVersion": JSON_SCHEMA_VERSION,
925 "fileLength": self.file_len,
926 "segments": segments,
927 }))
928 .map_err(err)
929 }
930}
931
932/// A **lazy, resumable cursor** over the quads of an open `.rete` — the engine
933/// side of `for await (const [s, p, o, g] of graph.dump())` in the JS client.
934///
935/// # Why a cursor and not a callback
936///
937/// [`Rete::dump_each`] already streams in constant memory, but a Rust callback
938/// cannot be *paused* to hand control back to JavaScript: to feed a JS iterator
939/// it would have to buffer every quad first, which is exactly the `Vec` that
940/// [`Rete::dump`] builds and that OOMs on a large file. This wraps
941/// [`Rete::query_batch`] instead, so the scan can be suspended between calls and
942/// resumed in place — the whole resume state is one opaque `u64`, never a
943/// whole-graph materialization anywhere in the pipeline.
944///
945/// # Why batched (and not one call per quad)
946///
947/// Each wasm→JS call costs far more than decoding a triple, and every returned
948/// `String` becomes a fresh JS string. Pulling one quad per call would make the
949/// boundary the bottleneck; pulling *all* of them would reintroduce the `Vec`.
950/// So the JS wrapper asks for `DUMP_BATCH` quads at a time and yields them one
951/// by one — bounded, amortized, and lazy. Memory is O(batch), not O(graph).
952///
953/// # Cost model
954///
955/// The dictionary is **not** prefetched whole: each batch faults only the
956/// chunks its own terms live in, so taking five quads off the front costs five
957/// quads' worth of dictionary rather than all of it. Index tiles fault in as the
958/// scan advances and stay resident, and so do dictionary chunks, so an
959/// **unfiltered** dump driven to the end still ends up fetching essentially the
960/// whole file — that is inherent in exporting a graph.
961///
962/// A **filtered** cursor (a bound `s` / `p` / `o`) is a different shape: the
963/// scan routes to the one permutation that sorts on the bound prefix and drops
964/// every tile whose synopsis proves it cannot match, from the tile directory,
965/// *without fetching it*. On `cordis.rete` (801 MB, six named graphs) dumping
966/// one predicate of one graph reads 16 MB where the unfiltered dump of that
967/// graph reads 376 MB. Peak memory is O(faulted dictionary + index), never
968/// O(quads), either way.
969/// A SELECT answered to dictionary ids: the **result handle** of
970/// [`Graph::query_rows`] and [`RemoteGraph::query_rows`].
971///
972/// `query` evaluates a SELECT and decodes every term of every row to JSON; a
973/// page of 25 rows of a 650,000-row result still costs the whole result's
974/// dictionary chunks (181 MB for a class of the nanopub dump). The handle keeps
975/// the rows as ids, with the exact row count, and `page(offset, limit)` decodes
976/// one window, as one batch of chunk reads. What the query itself needs is
977/// decoded as before (the keys of an ORDER BY, the inputs of a FILTER); the
978/// other columns wait for the page. `free()` it when done.
979#[wasm_bindgen]
980pub struct QueryRows {
981 rete: Rc<Rete>,
982 rows: rete_core::SelectRows,
983 remote: bool,
984}
985
986impl QueryRows {
987 fn run(
988 rete: &Rc<Rete>,
989 query: &str,
990 reason: bool,
991 union_default: bool,
992 remote: bool,
993 ) -> Result<QueryRows, JsValue> {
994 let rows = rete_core::eval_select_rows(
995 rete,
996 query,
997 QueryOpts {
998 reason,
999 union_default_graph: union_default,
1000 },
1001 )
1002 .map_err(err)?;
1003 Ok(QueryRows {
1004 rete: rete.clone(),
1005 rows,
1006 remote,
1007 })
1008 }
1009}
1010
1011#[wasm_bindgen]
1012impl QueryRows {
1013 /// The projected variables, a JSON array (for `SELECT *`, the variables
1014 /// some row binds, in name order).
1015 pub fn vars(&self) -> String {
1016 serde_json::to_string(self.rows.vars()).unwrap_or_else(|_| "[]".to_string())
1017 }
1018
1019 /// The exact number of rows.
1020 pub fn len(&self) -> f64 {
1021 self.rows.len() as f64
1022 }
1023
1024 /// Rows `offset..offset + limit` (cut at the end), in the envelope of
1025 /// `query(..., "json")`: `{"kind":"select","vars":[…],"rows":[{var:term}]}`
1026 /// plus `"offset"` and `"total"`. Calling it again, or for another window,
1027 /// reuses the dictionary chunks already read.
1028 pub fn page(&self, offset: usize, limit: usize) -> Result<String, JsValue> {
1029 if self.remote {
1030 self.rete.reset_load_failures();
1031 }
1032 let page = self.rows.page(&self.rete, offset, limit);
1033 if self.remote {
1034 incomplete_guard(&self.rete, "query")?;
1035 }
1036 let start = offset.min(self.rows.len());
1037 let out = QueryOutput::Select(self.rows.vars().to_vec(), page);
1038 let mut extra = format!(
1039 r#","schemaVersion":{JSON_SCHEMA_VERSION},"offset":{start},"total":{}"#,
1040 self.rows.len()
1041 );
1042 // As `query`: a `warnings` member only when there is something to say.
1043 if !self.rows.warnings().is_empty() {
1044 extra.push_str(r#","warnings":"#);
1045 extra.push_str(&rete_core::warnings_json(self.rows.warnings()));
1046 }
1047 Ok(rete_core::results_envelope_json(&out, &extra))
1048 }
1049}
1050
1051#[wasm_bindgen]
1052pub struct QuadCursor {
1053 /// Graph slots not yet streamed; `None` = the default graph.
1054 pending: std::vec::IntoIter<Option<String>>,
1055 /// The graph being streamed (`None` = the default graph), and how far into
1056 /// it we got. The resume state is one opaque `u64` — see `Rete::query_batch`.
1057 current: Option<Option<String>>,
1058 cursor: u64,
1059 /// The triple pattern this dump is restricted to; all-`None` = every quad.
1060 /// Held as owned tokens because the cursor outlives every call.
1061 s: Option<String>,
1062 p: Option<String>,
1063 o: Option<String>,
1064 /// Whether the live graph's last batch was its final one.
1065 slot_done: bool,
1066 /// Triples resolved by the last `query_batch` that the caller has not taken
1067 /// yet. A batch ends on a group boundary, so this drains before the next.
1068 buffered: std::vec::IntoIter<TermTriple>,
1069 rete: Rc<Rete>,
1070 /// `next_nquads` respells quoted triples into the RDF 1.2 triple-term
1071 /// surface `<<( s p o )>>`. Off until [`QuadCursor::set_quoted_triple_syntax`]
1072 /// asks for `"rdf12"`, and off whatever is asked when the header says the
1073 /// file holds no quoted triple — so such a dump is byte-for-byte unchanged.
1074 rdf12: bool,
1075}
1076
1077/// Quads per `next_batch` / `next_nquads` call, and the JS wrapper's default.
1078///
1079/// 10 000 keeps the per-call boundary cost negligible (one call and ~40 000 JS
1080/// string allocations amortize a 10 000-triple decode) while bounding the
1081/// transient buffer at roughly 10 000 × ~120 B ≈ 1.2 MB — small next to the
1082/// dictionary any dump already needs resident, and flat no matter how many
1083/// billions of quads follow it.
1084const DUMP_BATCH: usize = 10_000;
1085
1086impl QuadCursor {
1087 /// `graph`: `None` = the default graph followed by every named graph;
1088 /// `Some("")` = the default graph only; `Some(name)` = that named graph
1089 /// (bare IRI or `<iri>` token — both are accepted).
1090 ///
1091 /// `s` / `p` / `o` are canonical N-Triples term tokens; an empty string is
1092 /// treated as unbound, so a JS caller can pass `""` for "no filter" without
1093 /// a separate sentinel. A bound term the dictionary does not know yields an
1094 /// empty dump without touching the index.
1095 fn start(
1096 rete: Rc<Rete>,
1097 graph: Option<String>,
1098 s: Option<String>,
1099 p: Option<String>,
1100 o: Option<String>,
1101 ) -> QuadCursor {
1102 let slots: Vec<Option<String>> = match graph {
1103 None => std::iter::once(None)
1104 .chain(rete.graph_names().iter().map(|g| Some((*g).to_string())))
1105 .collect(),
1106 Some(g) if g.is_empty() => vec![None],
1107 Some(g) => vec![Some(canonical_graph_name(&rete, g))],
1108 };
1109 // The incompleteness verdict is PER DUMP: clear the sticky flags so a
1110 // failure from an earlier query does not condemn this one (and so
1111 // `finish()` reports only what happened while streaming).
1112 rete.reset_load_failures();
1113 let term = |t: Option<String>| t.filter(|t| !t.is_empty());
1114 QuadCursor {
1115 pending: slots.into_iter(),
1116 current: None,
1117 cursor: 0,
1118 s: term(s),
1119 p: term(p),
1120 o: term(o),
1121 slot_done: false,
1122 buffered: Vec::new().into_iter(),
1123 rete,
1124 rdf12: false,
1125 }
1126 }
1127
1128 /// Pull at most `max` quads, handing each to `sink` as
1129 /// `(s, p, o, graph)` where `graph` is `None` for the default graph.
1130 /// Returns how many were emitted; fewer than `max` means the stream ended.
1131 ///
1132 /// The scan is resumable rather than held open: each refill calls
1133 /// `Rete::query_batch`, whose entire resume state is one opaque `u64`. That
1134 /// is what lets this struct be `'static` — required by `#[wasm_bindgen]` —
1135 /// without a self-referential iterator borrowing the `Rete` beside it.
1136 fn pull<F: FnMut(&str, &str, &str, Option<&str>)>(&mut self, max: usize, mut sink: F) -> usize {
1137 let mut emitted = 0;
1138 while emitted < max {
1139 // 1. Hand out what the last batch already resolved.
1140 if let Some((s, p, o)) = self.buffered.next() {
1141 let graph = self.current.as_ref().and_then(|g| g.as_deref());
1142 sink(&s, &p, &o, graph);
1143 emitted += 1;
1144 continue;
1145 }
1146 // 2. Buffer empty. Open the next graph slot if none is live…
1147 let Some(slot) = self.current.clone() else {
1148 match self.pending.next() {
1149 Some(next) => {
1150 self.current = Some(next);
1151 self.cursor = 0;
1152 self.slot_done = false;
1153 }
1154 None => break,
1155 }
1156 continue;
1157 };
1158 // 3. …or close the live one if its last batch is drained.
1159 if self.slot_done {
1160 self.current = None;
1161 continue;
1162 }
1163 // 4. Otherwise resume it. `query_batch` always advances the cursor
1164 // or reports done, so this cannot spin.
1165 let want = (max - emitted).max(DUMP_BATCH);
1166 let (triples, next, done) = self.rete.query_batch(
1167 slot.as_deref(),
1168 self.s.as_deref(),
1169 self.p.as_deref(),
1170 self.o.as_deref(),
1171 self.cursor,
1172 want,
1173 );
1174 self.cursor = next;
1175 self.slot_done = done;
1176 self.buffered = triples.into_iter();
1177 }
1178 emitted
1179 }
1180
1181 /// True once every selected graph has been streamed to its end.
1182 fn finished(&self) -> bool {
1183 self.current.is_none() && self.pending.len() == 0 && self.buffered.len() == 0
1184 }
1185
1186 /// Refuse to end a dump that silently lost bytes to a failed range fetch —
1187 /// a truncated export is worse than a failed one.
1188 fn guard(&self) -> Result<(), JsValue> {
1189 if self.finished() {
1190 incomplete_guard(&self.rete, "dump")?;
1191 }
1192 Ok(())
1193 }
1194}
1195
1196/// One statement's object token in the RDF 1.2 surface, or the reason it has
1197/// none — the same rule as `rete export --quoted-triple-syntax rdf12`
1198/// (`ttSubject ::= iri | BlankNode`: a triple term stands in object position
1199/// only), with the same hint.
1200fn rdf12_object<'a>(s: &str, p: &str, o: &'a str) -> Result<std::borrow::Cow<'a, str>, String> {
1201 use rete_core::terms::is_quoted_triple;
1202 if is_quoted_triple(s) || is_quoted_triple(p) {
1203 let slot = if is_quoted_triple(s) {
1204 "subject"
1205 } else {
1206 "predicate"
1207 };
1208 return Err(format!(
1209 "this graph has a quoted triple in the {slot} position of a statement, and RDF 1.2 \
1210 has no syntax for one there: a triple term may stand in OBJECT position only.\n\
1211 statement: {s} {p} {o} .\n\
1212 hint: quotedTripleSyntax \"rdf-star\" writes the RDF-star surface <<s p o>>, which \
1213 rete re-ingests losslessly"
1214 ));
1215 }
1216 rete_core::terms::rdf12_triple_term(o).ok_or_else(|| {
1217 format!(
1218 "this graph has a quoted triple nested in the SUBJECT of another quoted triple, and \
1219 RDF 1.2 has no syntax for one there.\n\
1220 statement: {s} {p} {o} .\n\
1221 hint: quotedTripleSyntax \"rdf-star\" writes the RDF-star surface <<s p o>>"
1222 )
1223 })
1224}
1225
1226/// The card's `signals.quoted_triples`, measured from the header exactly as
1227/// `rete card` does: `{"present":bool,"export_surfaces":[…],"export_default":…}`.
1228fn quoted_triples_signal_json(header: &Header) -> String {
1229 serde_json::to_string(&rete_core::card::QuotedTriplesSignal::probe(header))
1230 .expect("the signal serializes")
1231}
1232
1233#[wasm_bindgen]
1234impl QuadCursor {
1235 /// Up to `max` quads as a **flat** `string[]` of `[s, p, o, g, s, p, o, g, …]`
1236 /// N-Triples term tokens, `g` being `""` for the default graph. Flat because
1237 /// a nested array would allocate one JS array per quad for no gain — the
1238 /// caller slices it into tuples as it yields them.
1239 ///
1240 /// An empty array means the stream is finished; keep calling until you get
1241 /// one (that final call is what verifies no range fetch failed mid-dump).
1242 pub fn next_batch(&mut self, max: Option<usize>) -> Result<Vec<String>, JsValue> {
1243 let max = max.unwrap_or(DUMP_BATCH).max(1);
1244 let mut out: Vec<String> = Vec::with_capacity(max.min(DUMP_BATCH) * 4);
1245 self.pull(max, |s, p, o, g| {
1246 out.push(s.to_string());
1247 out.push(p.to_string());
1248 out.push(o.to_string());
1249 out.push(g.unwrap_or("").to_string());
1250 });
1251 self.guard()?;
1252 Ok(out)
1253 }
1254
1255 /// Up to `max` quads already serialized as N-Quads lines in **one** string —
1256 /// the `.rete` → Oxigraph / N-Quads-file path. One string crossing per batch
1257 /// instead of four per quad: no per-term JS string, no re-serialization in
1258 /// JavaScript, and the terms are already canonical N-Triples tokens, so the
1259 /// lines are emitted verbatim.
1260 ///
1261 /// An empty string means the stream is finished.
1262 pub fn next_nquads(&mut self, max: Option<usize>) -> Result<String, JsValue> {
1263 let max = max.unwrap_or(DUMP_BATCH).max(1);
1264 // ~120 B/quad is a typical N-Quads line; the Vec grows if it is wrong.
1265 let mut out = String::with_capacity(max.min(DUMP_BATCH) * 120);
1266 let rdf12 = self.rdf12;
1267 let mut refused: Option<String> = None;
1268 self.pull(max, |s, p, o, g| {
1269 if refused.is_some() {
1270 return;
1271 }
1272 let o = if rdf12 {
1273 match rdf12_object(s, p, o) {
1274 Ok(o) => o,
1275 Err(e) => {
1276 refused = Some(e);
1277 return;
1278 }
1279 }
1280 } else {
1281 std::borrow::Cow::Borrowed(o)
1282 };
1283 out.push_str(s);
1284 out.push(' ');
1285 out.push_str(p);
1286 out.push(' ');
1287 out.push_str(&o);
1288 if let Some(g) = g {
1289 out.push(' ');
1290 out.push_str(g);
1291 }
1292 out.push_str(" .\n");
1293 });
1294 if let Some(e) = refused {
1295 return Err(js_error(e));
1296 }
1297 self.guard()?;
1298 Ok(out)
1299 }
1300
1301 /// How [`next_nquads`](Self::next_nquads) spells a quoted triple — the
1302 /// values of `rete export --quoted-triple-syntax`:
1303 ///
1304 /// - `"rdf-star"` (or `""`; the cursor's initial state): `<<s p o>>`, the
1305 /// stored token, verbatim — what this cursor always wrote.
1306 /// - `"rdf12"`: the RDF 1.2 triple term `<<( s p o )>>`, nested terms too,
1307 /// which current RDF 1.2 parsers (Oxigraph 0.5, Jena 5) load. RDF 1.2
1308 /// admits a triple term in object position only, so a quoted triple in a
1309 /// statement's subject/predicate (or in another's subject) makes the next
1310 /// call fail by name instead of writing a line nothing parses.
1311 ///
1312 /// Anything else is refused. A file whose header records no quoted triple
1313 /// is written byte-for-byte the same under both.
1314 pub fn set_quoted_triple_syntax(&mut self, syntax: &str) -> Result<(), JsValue> {
1315 let surface = if syntax.is_empty() {
1316 rete_core::ingest::QuotedTripleSurface::RdfStar
1317 } else {
1318 rete_core::ingest::QuotedTripleSurface::parse(syntax).ok_or_else(|| {
1319 js_error(format!(
1320 "unknown quoted-triple syntax {syntax:?} (expected \"rdf12\" or \"rdf-star\")"
1321 ))
1322 })?
1323 };
1324 self.rdf12 = surface == rete_core::ingest::QuotedTripleSurface::Rdf12
1325 && self.rete.header().has_quoted_triples();
1326 Ok(())
1327 }
1328
1329 /// Whether every selected graph has been streamed to its end.
1330 pub fn done(&self) -> bool {
1331 self.finished()
1332 }
1333}
1334
1335/// Resolve a caller-supplied graph name to the token the file stores. Graph
1336/// names are canonical N-Triples terms (`<iri>`), but the JS client hands out
1337/// bare IRIs, so accept either and prefer an exact match.
1338fn canonical_graph_name(rete: &Rete, name: String) -> String {
1339 if rete.graph_names().iter().any(|g| *g == name) {
1340 return name;
1341 }
1342 if name.starts_with('<') || name.starts_with("_:") {
1343 return name;
1344 }
1345 format!("<{name}>")
1346}
1347
1348/// The wasm linear memory's current size in bytes — the engine's high-water
1349/// mark, since wasm memory grows but never shrinks. Exposed so a host can
1350/// *measure* the streaming-dump memory claim instead of trusting it: sample it
1351/// before and after a full [`QuadCursor`] drain and the growth stays flat
1352/// however many quads went by, where materializing them all does not.
1353#[wasm_bindgen]
1354pub fn heap_bytes() -> f64 {
1355 #[cfg(target_arch = "wasm32")]
1356 {
1357 (core::arch::wasm32::memory_size(0) as f64) * 65536.0
1358 }
1359 #[cfg(not(target_arch = "wasm32"))]
1360 {
1361 0.0
1362 }
1363}
1364
1365/// A remote `.rete` opened **once over HTTP range** and kept resident in the
1366/// worker, so repeated queries on the same URL reuse (a) the block cache — any
1367/// 64 KiB block fetched once is served from memory by [`BlockCacheReader`] — and
1368/// (b) the lazily faulted index tiles + decoded dictionary chunks that live
1369/// inside the resident [`Rete`]. The free [`sparql_url`] re-opens the file on
1370/// every call, so its block cache dies after one query; this handle keeps it.
1371/// The counting reader stays reachable so the worker can read cumulative
1372/// bytes/requests and show how little a cache-hit query actually fetched.
1373/// **Worker-only** (synchronous range-read XHR).
1374#[wasm_bindgen]
1375pub struct RemoteGraph {
1376 reader: RemoteReader,
1377 rete: Rc<Rete>,
1378}
1379
1380#[wasm_bindgen]
1381impl RemoteGraph {
1382 /// Open a remote `.rete` over HTTP range and keep it resident for repeated
1383 /// querying. The first query faults in the dictionary chunks + index tiles it
1384 /// needs; later queries on this handle reuse them and the block cache.
1385 #[wasm_bindgen(constructor)]
1386 pub fn new(url: &str) -> Result<RemoteGraph, JsValue> {
1387 let (reader, rete) = open_url(url)?;
1388 Ok(RemoteGraph {
1389 reader,
1390 rete: Rc::new(rete),
1391 })
1392 }
1393
1394 /// See [`Graph::quads`] — the SAME lazy cursor, over the lazily range-read
1395 /// remote handle.
1396 ///
1397 /// An **unfiltered** dump is not network-lazy and cannot be: it resolves
1398 /// every term and visits every tile, so it ends up fetching essentially the
1399 /// whole file (and what it faults stays resident). A **filtered** one is:
1400 /// pass `s` / `p` / `o` and the scan routes to one permutation, keeps only
1401 /// the tiles whose synopsis admits the bound components, and fetches those.
1402 /// On `cordis.rete` (801 MB) one predicate of one named graph costs 16 MB
1403 /// instead of 376 MB. To peek at an unfiltered graph, still prefer a `LIMIT`
1404 /// query. Worker-only in the browser, like every other read here.
1405 pub fn quads(
1406 &self,
1407 graph: Option<String>,
1408 s: Option<String>,
1409 p: Option<String>,
1410 o: Option<String>,
1411 ) -> QuadCursor {
1412 QuadCursor::start(self.rete.clone(), graph, s, p, o)
1413 }
1414
1415 /// `{ fileLength, bytes, requests, base }` — CUMULATIVE physical fetches
1416 /// since this session opened. The worker diffs successive calls to report a
1417 /// single query's traffic (a fully cached re-run adds ~0). `fileLength` is
1418 /// the **graph's** length and `base` the byte offset it starts at: `0` for an
1419 /// ordinary `.rete`, and the size of the HTML shell for a polyglot file.
1420 pub fn stats(&self) -> String {
1421 format!(
1422 r#"{{"schemaVersion":1,"fileLength":{},"bytes":{},"requests":{},"base":{}}}"#,
1423 self.reader.len(),
1424 self.reader.bytes_read(),
1425 self.reader.requests(),
1426 self.reader.base
1427 )
1428 }
1429
1430 /// The file's content hash (blake3-16, hex). The worker keys its session
1431 /// cache by this rather than the URL, so two URLs of the same file share the
1432 /// cache — and it's the stable key a future IndexedDB block store (L3) needs
1433 /// to survive page reloads.
1434 pub fn content_hash(&self) -> String {
1435 self.rete
1436 .header()
1437 .content_hash
1438 .iter()
1439 .map(|b| format!("{b:02x}"))
1440 .collect()
1441 }
1442
1443 /// See [`sparql_url`] — same query, but over the resident, cached handle.
1444 /// The incompleteness verdict is PER QUERY: reset the sticky failure flags
1445 /// first, so one transient fetch failure fails only the query it happened
1446 /// in — not every later query on this session (failed tiles/chunks are
1447 /// never cached, so they retry here).
1448 pub fn query(&self, query: &str, format: &str) -> Result<String, JsValue> {
1449 self.rete.reset_load_failures();
1450 let s = query_json(&self.rete, query, format, "")?;
1451 incomplete_guard(&self.rete, "query")?;
1452 Ok(s)
1453 }
1454
1455 /// As [`query`], with OWL 2 QL entailment on (reason over the ontology while
1456 /// reading only the bytes the rewritten query touches).
1457 pub fn query_reasoned(&self, query: &str, format: &str) -> Result<String, JsValue> {
1458 self.rete.reset_load_failures();
1459 let s = query_json_reasoned(&self.rete, query, format, "")?;
1460 incomplete_guard(&self.rete, "query")?;
1461 Ok(s)
1462 }
1463
1464 /// As [`query`], with explicit opt-in toggles — see [`Graph::query_opts`].
1465 /// With `union_default` on, a lazy remote read may fault the index tiles of
1466 /// every named graph the union touches (the merge is strictly opt-in).
1467 pub fn query_opts(
1468 &self,
1469 query: &str,
1470 format: &str,
1471 reason: bool,
1472 union_default: bool,
1473 ) -> Result<String, JsValue> {
1474 self.rete.reset_load_failures();
1475 let s = query_json_with(&self.rete, query, format, "", reason, union_default)?;
1476 incomplete_guard(&self.rete, "query")?;
1477 Ok(s)
1478 }
1479
1480 /// See [`Graph::query_rows`]. The query's reads happen here; each
1481 /// [`QueryRows::page`] then reads the dictionary chunks of its own terms.
1482 pub fn query_rows(
1483 &self,
1484 query: &str,
1485 reason: bool,
1486 union_default: bool,
1487 ) -> Result<QueryRows, JsValue> {
1488 self.rete.reset_load_failures();
1489 let rows = QueryRows::run(&self.rete, query, reason, union_default, true)?;
1490 incomplete_guard(&self.rete, "query")?;
1491 Ok(rows)
1492 }
1493
1494 /// See [`Graph::partition_plan`]. Reads the dictionary entries of the
1495 /// query's constants and the resident tile directory — no tile data.
1496 pub fn partition_plan(&self, query: &str, var: &str) -> Result<String, JsValue> {
1497 self.rete.reset_load_failures();
1498 let s = partition_plan_json(&self.rete, query, var)?;
1499 incomplete_guard(&self.rete, "query")?;
1500 Ok(s)
1501 }
1502
1503 /// See [`Graph::query_partition`].
1504 pub fn query_partition(
1505 &self,
1506 query: &str,
1507 format: &str,
1508 reason: bool,
1509 var: &str,
1510 keys: Vec<String>,
1511 ) -> Result<String, JsValue> {
1512 self.rete.reset_load_failures();
1513 let s = query_partition_json(&self.rete, query, format, reason, var, &keys)?;
1514 incomplete_guard(&self.rete, "query")?;
1515 Ok(s)
1516 }
1517
1518 /// See [`prefix_search`] — over the resident, cached remote handle. Faults the
1519 /// pyramid (where the label index lives) on the first call, then serves the
1520 /// search from memory.
1521 pub fn prefix_search(&self, prefix: &str, limit: usize) -> Result<String, JsValue> {
1522 prefix_search_json(&self.rete, prefix, limit)
1523 }
1524
1525 /// See [`text_search`] — over the resident remote handle. Faults the TEXT_INDEX
1526 /// token table on the first call, then fetches only the queried posting lists
1527 /// (never the whole postings blob), serving repeat searches from memory.
1528 pub fn text_search(
1529 &self,
1530 words: Vec<String>,
1531 contains_prefix: Option<String>,
1532 limit: usize,
1533 ) -> Result<String, JsValue> {
1534 text_search_json(&self.rete, &words, contains_prefix.as_deref(), limit)
1535 }
1536
1537 /// See [`Graph::text_index_len`] — read from the resident header, so it
1538 /// costs no fetch at all. Worth asking before [`RemoteGraph::text_search`]:
1539 /// it is the size of the section the first search starts pulling over the
1540 /// wire, so the UI can warn instead of surprising the user.
1541 pub fn text_index_len(&self) -> f64 {
1542 self.rete.header().text_index_len as f64
1543 }
1544
1545 /// See [`Graph::text_index_token_table_len`] — the figure to quote before
1546 /// [`RemoteGraph::text_search`], because it is what that first search pulls
1547 /// over the wire; the section length would promise the user several times
1548 /// the real bill.
1549 ///
1550 /// Unlike [`RemoteGraph::text_index_len`] this is not free: the token
1551 /// table's length lives in the section's first bytes, not the header, so it
1552 /// costs ONE ≤10-byte range read (memoized). Trivial next to the table it
1553 /// measures — but it *is* IO, so the asyncify path must drive this call
1554 /// rather than treat it as a header field.
1555 pub fn text_index_token_table_len(&self) -> f64 {
1556 self.rete.text_index_token_table_len().unwrap_or(0) as f64
1557 }
1558
1559 /// See [`Graph::text_search_one`] — over the resident remote handle, with
1560 /// the same token-table-then-posting-lists fault pattern as
1561 /// [`RemoteGraph::text_search`]. This is the shape the playground's raw
1562 /// asyncify glue drives: one string in, one string out, marshaled once.
1563 pub fn text_search_one(&self, phrase: &str, limit: usize) -> Result<String, JsValue> {
1564 let words: Vec<String> = phrase.split_whitespace().map(str::to_owned).collect();
1565 text_search_json(&self.rete, &words, None, limit)
1566 }
1567
1568 /// See [`Graph::quoted_triples_signal`] — from the header the open already
1569 /// holds, so no range read at all.
1570 pub fn quoted_triples_signal(&self) -> String {
1571 quoted_triples_signal_json(self.rete.header())
1572 }
1573
1574 /// See [`card_url`] — the Dataset Card, over the resident handle's reader
1575 /// (so the header range it already fetched is served from the block cache).
1576 pub fn card(&self) -> Result<Option<String>, JsValue> {
1577 let bytes = rete_core::read_metadata_ranged(&self.reader.view()).map_err(err)?;
1578 Ok(bytes.map(|b| String::from_utf8_lossy(&b).into_owned()))
1579 }
1580
1581 /// See [`card_and_build_url`] — card + build record over the resident
1582 /// handle's reader, still one coalesced range (and served from the block
1583 /// cache when the header range is already there).
1584 pub fn card_and_build(&self) -> Result<String, JsValue> {
1585 let (card, build) =
1586 rete_core::read_card_and_build_info_ranged(&self.reader.view()).map_err(err)?;
1587 // Header-only, deliberately: `token_table_bytes` would cost a range read
1588 // (see [`RemoteGraph::text_index_token_table_len`]), and this call must
1589 // stay IO-free beyond the card range so the asyncify driver does not
1590 // have to suspend inside it. A caller that wants the figure asks for it
1591 // through that export, which the glue does drive.
1592 let text_index = text_index_json(self.rete.header(), None);
1593 Ok(card_build_envelope(card, build, text_index))
1594 }
1595
1596 /// See [`schema_url`] — the **baked** schema pyramid over the resident
1597 /// handle. Deliberately never falls back to the scanning [`schema`]: that
1598 /// would drag the whole remote file across the wire.
1599 pub fn schema(&self) -> Result<String, JsValue> {
1600 use serde_json::json;
1601 let (classes, relations) = rete_core::read_schema_summary_ranged(&self.reader.view())
1602 .map_err(err)?
1603 .ok_or_else(|| js_error("file has no schema pyramid"))?;
1604 let classes: Vec<serde_json::Value> = classes.iter().map(|(c, n)| json!([c, n])).collect();
1605 let relations: Vec<serde_json::Value> = relations
1606 .iter()
1607 .map(|(s, p, o, n)| json!([s, p, o, n]))
1608 .collect();
1609 Ok(json!({
1610 "schemaVersion": JSON_SCHEMA_VERSION,
1611 "kind": "schema",
1612 "classes": classes,
1613 "relations": relations,
1614 })
1615 .to_string())
1616 }
1617
1618 /// See [`info`] — read from the resident header, no extra fetch.
1619 pub fn info(&self) -> String {
1620 let h = self.rete.header();
1621 format!(
1622 r#"{{"schemaVersion":1,"quads":{},"terms":{},"pyramidLevels":{},"namedGraphs":{}}}"#,
1623 h.quad_count,
1624 h.term_count,
1625 h.pyramid_levels,
1626 // The cheap count (the section's leading varint) — listing every
1627 // IRI would walk the whole named-graphs directory over the wire.
1628 self.rete.named_graph_count()
1629 )
1630 }
1631
1632 /// See [`graph_names`].
1633 pub fn graph_names(&self) -> Result<String, JsValue> {
1634 serde_json::to_string(&self.rete.graph_names()).map_err(err)
1635 }
1636
1637 /// See [`shacl_url`] — validate over the resident handle. The default graph
1638 /// validates lazily against the index (only the shapes' targets are
1639 /// fetched), so a shape over a huge remote file stays cheap.
1640 pub fn shacl(
1641 &self,
1642 shapes_turtle: &str,
1643 graph: Option<String>,
1644 format: &str,
1645 ) -> Result<String, JsValue> {
1646 self.rete.reset_load_failures();
1647 let out = shacl_rete(&self.rete, shapes_turtle, graph.as_deref(), format)?;
1648 incomplete_guard(&self.rete, "validation")?;
1649 Ok(out)
1650 }
1651}
1652
1653/// Serialize a label prefix search as `[{"label":…,"subject":…}]`.
1654fn prefix_search_json(rete: &Rete, prefix: &str, limit: usize) -> Result<String, JsValue> {
1655 use serde_json::json;
1656 let hits: Vec<serde_json::Value> = rete
1657 .prefix_search(prefix, limit)
1658 .into_iter()
1659 .map(|(label, subject)| json!({ "label": label, "subject": subject }))
1660 .collect();
1661 serde_json::to_string(&hits).map_err(err)
1662}
1663
1664/// Serialize a full-text (word/CONTAINS) search as `[{"subject":…}]`.
1665fn text_search_json(
1666 rete: &Rete,
1667 words: &[String],
1668 contains_prefix: Option<&str>,
1669 limit: usize,
1670) -> Result<String, JsValue> {
1671 use serde_json::json;
1672 let word_refs: Vec<&str> = words.iter().map(String::as_str).collect();
1673 let hits: Vec<serde_json::Value> = rete
1674 .text_search(&word_refs, contains_prefix, limit)
1675 .into_iter()
1676 .map(|subject| json!({ "subject": subject }))
1677 .collect();
1678 serde_json::to_string(&hits).map_err(err)
1679}
1680
1681/// Parse the fixed-size header and report the byte ranges a *progressive*
1682/// client needs for the overview — the dictionary and the pyramid summary — plus
1683/// the (large) index range it can skip, and the metadata (Dataset Card) range.
1684/// JSON: `{ "dictOffset","dictLen","pyramidOffset","pyramidLen","indexOffset",
1685/// "indexLen","metadataOffset","metadataLen" }`.
1686/// The browser fetches bytes `0..HEADER_LEN`, calls this, then range-fetches only the
1687/// dict + pyramid — never the index. A host with its own byte reader (Node over a
1688/// local file, say) can use `metadataOffset`/`metadataLen` to read just the card.
1689#[wasm_bindgen]
1690pub fn header_ranges(head: &[u8]) -> Result<String, JsValue> {
1691 let h = Header::from_bytes(head).map_err(err)?;
1692 Ok(format!(
1693 r#"{{"schemaVersion":1,"dictOffset":{},"dictLen":{},"pyramidOffset":{},"pyramidLen":{},"indexOffset":{},"indexLen":{},"metadataOffset":{},"metadataLen":{}}}"#,
1694 h.dictionary_offset,
1695 h.dictionary_len,
1696 h.pyramid_meta_offset,
1697 h.pyramid_meta_len,
1698 h.root_dir_offset,
1699 h.root_dir_len,
1700 h.metadata_offset,
1701 h.metadata_len,
1702 ))
1703}
1704
1705/// Build the coarse-graph overview from a buffer in which only the header,
1706/// dictionary, and pyramid-summary ranges are populated — the index region may be
1707/// absent (zero-filled), because the summary path provably never reads it (see
1708/// the `ranged` test in rete-core). Returns JSON:
1709/// `{ "round", "communities", "predicateTotals": [["<iri>", count], ...] }`.
1710/// This is the "overview first, drill down later" payload, fetched in ~3 ranges.
1711#[wasm_bindgen]
1712pub fn summary_overview(bytes: &[u8]) -> Result<String, JsValue> {
1713 let view = SummaryView::open_ranged(&SliceReader::new(bytes))
1714 .map_err(err)?
1715 .ok_or_else(|| js_error("file has no pyramid summary"))?;
1716 serde_json::to_string(&serde_json::json!({
1717 "schemaVersion": JSON_SCHEMA_VERSION,
1718 "round": view.round,
1719 "communities": view.community_count(),
1720 "predicateTotals": view.predicate_totals(),
1721 }))
1722 .map_err(err)
1723}
1724
1725/// Answer a conservative subset of SPARQL exactly from the pyramid summary,
1726/// without opening the triple index. Unsupported query shapes return an error
1727/// instead of silently falling back to a full scan.
1728#[wasm_bindgen]
1729pub fn progressive_query(bytes: &[u8], query: &str) -> Result<String, JsValue> {
1730 progressive_query_json(bytes, query).map_err(js_error)
1731}
1732
1733/// Native-testable implementation for [`progressive_query`].
1734pub fn progressive_query_json(bytes: &[u8], query: &str) -> Result<String, String> {
1735 use serde_json::json;
1736
1737 let shape = summary_query_shape(query)
1738 .map_err(|e| e.to_string())?
1739 .ok_or_else(|| "query is not exactly answerable from the summary".to_string())?;
1740
1741 let reader = CountingReader::new(SliceReader::new(bytes));
1742 let file_bytes = reader.len();
1743 let view = SummaryView::open_ranged(&reader)
1744 .map_err(|e| e.to_string())?
1745 .ok_or_else(|| "file has no pyramid summary".to_string())?;
1746
1747 let value = match shape {
1748 SummaryQueryShape::PredicateCount {
1749 predicate,
1750 variable,
1751 } => {
1752 let count = u64::from(view.predicate_total(&predicate));
1753 select_count_response(
1754 &variable,
1755 count,
1756 progressive_meta(
1757 &reader,
1758 file_bytes,
1759 "predicate_count",
1760 Some(&predicate),
1761 json!(count),
1762 ),
1763 )
1764 }
1765 SummaryQueryShape::TripleCount { variable } => {
1766 let count = summary_total(&view);
1767 select_count_response(
1768 &variable,
1769 count,
1770 progressive_meta(&reader, file_bytes, "triple_count", None, json!(count)),
1771 )
1772 }
1773 SummaryQueryShape::PredicateTotals {
1774 predicate_variable,
1775 count_variable,
1776 } => {
1777 let totals = view.predicate_totals();
1778 select_predicate_totals_response(
1779 &predicate_variable,
1780 &count_variable,
1781 &totals,
1782 progressive_meta(
1783 &reader,
1784 file_bytes,
1785 "predicate_totals",
1786 None,
1787 json!(&totals),
1788 ),
1789 )
1790 }
1791 SummaryQueryShape::PredicateList { variable } => {
1792 let predicates = predicate_list(&view);
1793 select_predicate_list_response(
1794 &variable,
1795 &predicates,
1796 progressive_meta(
1797 &reader,
1798 file_bytes,
1799 "predicate_list",
1800 None,
1801 json!(&predicates),
1802 ),
1803 )
1804 }
1805 SummaryQueryShape::PredicateDistinctCount { variable } => {
1806 let count = predicate_count(&view);
1807 select_count_response(
1808 &variable,
1809 count,
1810 progressive_meta(
1811 &reader,
1812 file_bytes,
1813 "predicate_distinct_count",
1814 None,
1815 json!(count),
1816 ),
1817 )
1818 }
1819 SummaryQueryShape::TripleExists => {
1820 let exists = summary_total(&view) > 0;
1821 json!({
1822 "schemaVersion": JSON_SCHEMA_VERSION,
1823 "kind": "ask",
1824 "boolean": exists,
1825 "progressive": progressive_meta(
1826 &reader,
1827 file_bytes,
1828 "triple_exists",
1829 None,
1830 json!(exists),
1831 ),
1832 })
1833 }
1834 SummaryQueryShape::PredicateExists { predicate } => {
1835 let exists = view.predicate_total(&predicate) > 0;
1836 json!({
1837 "schemaVersion": JSON_SCHEMA_VERSION,
1838 "kind": "ask",
1839 "boolean": exists,
1840 "progressive": progressive_meta(
1841 &reader,
1842 file_bytes,
1843 "predicate_exists",
1844 Some(&predicate),
1845 json!(exists),
1846 ),
1847 })
1848 }
1849 _ => return Err("summary query shape is not supported by this WASM build".to_string()),
1850 };
1851
1852 serde_json::to_string(&value).map_err(|e| e.to_string())
1853}
1854
1855/// Run any SPARQL form (SELECT / ASK / CONSTRUCT / DESCRIBE) and serialize the
1856/// result for the playground. Returns a JSON string the page parses; a `"kind"`
1857/// field (`"select"|"ask"|"construct"`) tells the UI how to render it.
1858///
1859/// `format` controls the construct serialization and the select fallback:
1860/// - SELECT → `{ "kind":"select", "vars":[...], "rows":[ {var:value,...} ] }`
1861/// (`format` is ignored for SELECT — the table view is always available).
1862/// - ASK → `{ "kind":"ask", "boolean": true|false }`.
1863/// - CONSTRUCT/DESCRIBE →
1864/// - `format=="ttl"` → `{ "kind":"construct", "format":"ttl", "text": "<turtle>" }`
1865/// - `format=="jsonld"` → `{ "kind":"construct", "format":"jsonld", "text": "<json-ld>" }`
1866/// - otherwise (`table`/`json`) → `{ "kind":"construct", "triples": [[s,p,o], ...] }`.
1867#[wasm_bindgen]
1868pub fn query(bytes: &[u8], query: &str, format: &str) -> Result<String, JsValue> {
1869 let rete = open(bytes)?;
1870 query_json(&rete, query, format, "")
1871}
1872
1873/// Prefix-search the label index of an embedded `.rete` image: the subjects whose
1874/// label starts with `prefix` (case-insensitive), as `[{"label":…,"subject":…}]`,
1875/// capped at `limit`. Answers from the LABEL_INDEX section, else from an older
1876/// file's pyramid-meta label block, with no literal scan. Empty array without one.
1877#[wasm_bindgen]
1878pub fn prefix_search(bytes: &[u8], prefix: &str, limit: usize) -> Result<String, JsValue> {
1879 let rete = open(bytes)?;
1880 prefix_search_json(&rete, prefix, limit)
1881}
1882
1883/// Full-text (word/CONTAINS) search over an embedded `.rete` image: the subjects
1884/// whose literals contain **every** word in `words` (whole-word, case-insensitive
1885/// — AND), optionally also a word starting with `contains_prefix`, as
1886/// `[{"subject":…}]`, capped at `limit`. Answers from the TEXT_INDEX section.
1887/// Empty array when the file has none (`build --text-index`).
1888#[wasm_bindgen]
1889pub fn text_search(
1890 bytes: &[u8],
1891 words: Vec<String>,
1892 contains_prefix: Option<String>,
1893 limit: usize,
1894) -> Result<String, JsValue> {
1895 let rete = open(bytes)?;
1896 text_search_json(&rete, &words, contains_prefix.as_deref(), limit)
1897}
1898
1899/// Evaluate any SPARQL form against an open [`Rete`] and serialize the playground
1900/// JSON envelope (shared by [`query`] and [`sparql_url`]). `extra` is a raw JSON
1901/// fragment of additional object members appended before the closing brace (e.g.
1902/// `,"remote":{…}` for [`sparql_url`]); pass `""` for none.
1903///
1904/// This writes the JSON **directly into a `String`** rather than building an
1905/// intermediate `serde_json::Value` tree and stringifying it: on a large SELECT
1906/// the tree path allocates ~25× the payload (a `BTreeMap` per row + two String
1907/// clones per cell) and costs more than the query itself; writing direct cuts the
1908/// serialization peak heap ~13× and the time ~10× (see `rete-bench --query-mem`).
1909fn query_json(rete: &Rete, query: &str, format: &str, extra: &str) -> Result<String, JsValue> {
1910 query_json_opt(rete, query, format, extra, false)
1911}
1912
1913/// As [`query_json`], but with OWL 2 QL entailment on (the query is rewritten so
1914/// the answer includes ontology-entailed solutions, computed over the raw data).
1915fn query_json_reasoned(
1916 rete: &Rete,
1917 query: &str,
1918 format: &str,
1919 extra: &str,
1920) -> Result<String, JsValue> {
1921 query_json_opt(rete, query, format, extra, true)
1922}
1923
1924fn query_json_opt(
1925 rete: &Rete,
1926 query: &str,
1927 format: &str,
1928 extra: &str,
1929 reason: bool,
1930) -> Result<String, JsValue> {
1931 query_json_with(rete, query, format, extra, reason, false)
1932}
1933
1934/// The full-options variant: `reason` (OWL 2 QL) and `union_default` (the
1935/// opt-in union-default-graph mode — a pattern outside GRAPH matches the merge
1936/// of the default graph and every named graph; see `QueryOpts`).
1937fn query_json_with(
1938 rete: &Rete,
1939 query: &str,
1940 format: &str,
1941 extra: &str,
1942 reason: bool,
1943 union_default: bool,
1944) -> Result<String, JsValue> {
1945 let (out, warnings) = eval_query_with_warnings(
1946 rete,
1947 query,
1948 QueryOpts {
1949 reason,
1950 union_default_graph: union_default,
1951 },
1952 )
1953 .map_err(err)?;
1954 write_with_warnings(&out, &warnings, format, extra)
1955}
1956
1957/// The envelope plus, when there are any, the expression type errors FILTER
1958/// absorbed (CONTAINS on an IRI, …): an optional `warnings` member, present
1959/// only when there is one, so an envelope without diagnostics is
1960/// byte-identical to before.
1961fn write_with_warnings(
1962 out: &QueryOutput,
1963 warnings: &[rete_core::QueryWarning],
1964 format: &str,
1965 extra: &str,
1966) -> Result<String, JsValue> {
1967 if warnings.is_empty() {
1968 return write_query_json(out, format, extra).map_err(js_error);
1969 }
1970 let mut extra = extra.to_string();
1971 extra.push_str(r#","warnings":"#);
1972 extra.push_str(&rete_core::warnings_json(warnings));
1973 write_query_json(out, format, &extra).map_err(js_error)
1974}
1975
1976/// [`rete_core::partition_plan`] as JSON.
1977fn partition_plan_json(rete: &Rete, query: &str, var: &str) -> Result<String, JsValue> {
1978 let plan = rete_core::partition_plan(rete, query, var).map_err(err)?;
1979 serde_json::to_string(&plan).map_err(err)
1980}
1981
1982/// One chunk of a split query ([`rete_core::eval_query_partitioned`]), as the
1983/// envelope [`query_json_with`] writes.
1984fn query_partition_json(
1985 rete: &Rete,
1986 query: &str,
1987 format: &str,
1988 reason: bool,
1989 var: &str,
1990 keys: &[String],
1991) -> Result<String, JsValue> {
1992 let opts = QueryOpts {
1993 reason,
1994 union_default_graph: false,
1995 };
1996 let (out, warnings) =
1997 rete_core::eval_query_partitioned(rete, query, opts, var, keys).map_err(err)?;
1998 write_with_warnings(&out, &warnings, format, "")
1999}
2000
2001/// Serialize an already-evaluated [`QueryOutput`] into the playground JSON
2002/// envelope. SELECT / ASK / `CONSTRUCT`-as-triples go through the shared,
2003/// host-tested `rete_core::results_envelope_json` (the allocation-lean direct
2004/// writer); a `CONSTRUCT` requested as Turtle / JSON-LD wraps the rendered text
2005/// (those serializers live here).
2006///
2007/// `format` may carry the quoted-triple surface the way `rete export` takes
2008/// `--quoted-triple-syntax`: `"ttl"` writes RDF 1.2 triple terms
2009/// `<<( s p o )>>` (the default, as on the CLI), and
2010/// `"ttl;quotedTripleSyntax=rdf-star"` the RDF-star surface `<<s p o>>`. An
2011/// `Err` is a refusal or a bad format, never a partly-written document.
2012fn write_query_json(out: &QueryOutput, format: &str, extra: &str) -> Result<String, String> {
2013 let mut versioned_extra = format!(r#","schemaVersion":{JSON_SCHEMA_VERSION}"#);
2014 versioned_extra.push_str(extra);
2015 if let QueryOutput::Construct(triples) = out {
2016 let (base, syntax) = construct_format(format)?;
2017 let text = match base {
2018 "ttl" => Some(("ttl", to_turtle(triples, syntax)?)),
2019 "jsonld" => Some(("jsonld", to_jsonld(triples)?)),
2020 _ => None,
2021 };
2022 if let Some((fmt, text)) = text {
2023 let mut s = String::from(r#"{"kind":"construct","format":""#);
2024 s.push_str(fmt);
2025 s.push_str(r#"","text":"#);
2026 rete_core::push_json_string(&mut s, &text);
2027 s.push_str(&versioned_extra);
2028 s.push('}');
2029 return Ok(s);
2030 }
2031 }
2032 Ok(rete_core::results_envelope_json(out, &versioned_extra))
2033}
2034
2035/// Split a CONSTRUCT `format` into its base (`ttl`, `jsonld`, …) and the
2036/// quoted-triple surface named by an optional `;quotedTripleSyntax=<value>`
2037/// parameter — the values of `rete export --quoted-triple-syntax`, with the
2038/// same default, `rdf12`. A parameter rete does not know is refused rather
2039/// than ignored, so a misspelling cannot silently write the other surface.
2040fn construct_format(
2041 format: &str,
2042) -> Result<(&str, rete_core::ingest::QuotedTripleSurface), String> {
2043 use rete_core::ingest::QuotedTripleSurface;
2044 let mut parts = format.split(';');
2045 let base = parts.next().unwrap_or("").trim();
2046 let mut syntax = QuotedTripleSurface::Rdf12;
2047 for param in parts {
2048 let (key, value) = param.split_once('=').unwrap_or((param, ""));
2049 match key.trim() {
2050 "quotedTripleSyntax" => {
2051 syntax = QuotedTripleSurface::parse(value.trim()).ok_or_else(|| {
2052 format!(
2053 "unknown quotedTripleSyntax {:?} in format {format:?} (expected \"rdf12\" \
2054 or \"rdf-star\")",
2055 value.trim()
2056 )
2057 })?;
2058 }
2059 other => {
2060 return Err(format!(
2061 "unknown format parameter {other:?} in {format:?} (the one parameter is \
2062 quotedTripleSyntax=rdf12|rdf-star)"
2063 ))
2064 }
2065 }
2066 }
2067 Ok((base, syntax))
2068}
2069
2070/// URL scheme naming a `.rete` that is **already in the page** — a `File` the
2071/// user picked, or any `Blob` — instead of one on a server.
2072///
2073/// A local open used to be `file.arrayBuffer()`: the whole file into a JS
2074/// `ArrayBuffer`, copied again into wasm linear memory, then `Rete::open`
2075/// decoding every dictionary chunk up front. That is ~6× the file size resident
2076/// before a single row is answered, which on wasm32 is a hard wall — so a user
2077/// could query a 16.8 GB graph over HTTP and not open their own 500 MB file
2078/// from disk (issue #102).
2079///
2080/// The fix is not a second reader. [`XhrRangeReader`] already *is* the lazy
2081/// reader — length, header-window cache, batching, block cache, polyglot base,
2082/// progress reporting — and only its bottom transport was HTTP-shaped. A URL
2083/// with this scheme swaps that transport for `Blob.slice()` +
2084/// [`web_sys::FileReaderSync`], so every `*_url` entry point ([`sparql_url`],
2085/// [`card_url`], [`schema_url`], [`RemoteGraph`], …) reads a local file exactly
2086/// as lazily as a remote one, over the same code.
2087///
2088/// `FileReaderSync` is genuinely synchronous and exists **only in workers** —
2089/// the same constraint sync XHR already imposes on this reader, so it fits the
2090/// contract with no new suspension points. That is also why the **asyncify**
2091/// build needs nothing extra: a local read never suspends, so it never reaches
2092/// `env.rete_fetch_ranges`.
2093///
2094/// The blob is registered by JS through [`register_local_file`], which owns the
2095/// URL→`Blob` map for this wasm instance.
2096const LOCAL_SCHEME: &str = "rete-local:";
2097
2098fn is_local_url(url: &str) -> bool {
2099 url.starts_with(LOCAL_SCHEME)
2100}
2101
2102thread_local! {
2103 /// URL → `Blob` for this wasm instance. Thread-local because a `Blob` is a
2104 /// JS handle and wasm32 is single-threaded here; the experimental `threads`
2105 /// build would simply not find a blob registered on another thread, and the
2106 /// read fails with the message below rather than silently reading nothing.
2107 static LOCAL_BLOBS: std::cell::RefCell<std::collections::HashMap<String, web_sys::Blob>> =
2108 std::cell::RefCell::new(std::collections::HashMap::new());
2109}
2110
2111/// Register a local `File`/`Blob` under a `rete-local:…` URL, so every `*_url`
2112/// entry point can range-read it.
2113///
2114/// **Worker-only** (the read uses `FileReaderSync`), and the caller mints the
2115/// URL: a worker can be torn down and rebuilt — the playground does that on a
2116/// wasm trap, an engine switch, or a phone memory reclaim — and the page must be
2117/// able to re-register the same file under the same URL so a resident session
2118/// key stays stable. Re-registering an existing URL replaces the blob.
2119#[wasm_bindgen]
2120pub fn register_local_file(url: &str, blob: &web_sys::Blob) -> Result<(), JsValue> {
2121 if !is_local_url(url) {
2122 return Err(js_error(format!(
2123 "a local file URL must start with `{LOCAL_SCHEME}` (got {url})"
2124 )));
2125 }
2126 LOCAL_BLOBS.with(|map| map.borrow_mut().insert(url.to_string(), blob.clone()));
2127 Ok(())
2128}
2129
2130/// Drop a registration made by [`register_local_file`]. Releases this wasm
2131/// instance's reference to the `Blob`; any open handle over it stops working.
2132#[wasm_bindgen]
2133pub fn forget_local_file(url: &str) -> bool {
2134 LOCAL_BLOBS.with(|map| map.borrow_mut().remove(url).is_some())
2135}
2136
2137fn local_blob(url: &str) -> std::io::Result<web_sys::Blob> {
2138 LOCAL_BLOBS
2139 .with(|map| map.borrow().get(url).cloned())
2140 .ok_or_else(|| {
2141 std::io::Error::other(format!(
2142 "no local file is registered as {url} — the page must call \
2143 register_local_file(url, file) in THIS worker before opening it"
2144 ))
2145 })
2146}
2147
2148/// The local transport: one `Blob.slice()` per range, read synchronously.
2149///
2150/// Batched like the HTTP one (the engine coalesces ranges before it gets here),
2151/// but with no round trip to pay for — a slice is a view over bytes the browser
2152/// already has on disk, so the only real cost is the copy of what was asked for.
2153fn read_local_ranges(url: &str, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
2154 let (offsets, lens, total) = checked_async_layout(ranges)?;
2155 if total == 0 {
2156 return Ok(ranges.iter().map(|_| Vec::new()).collect());
2157 }
2158 let blob = local_blob(url)?;
2159 let size = blob.size();
2160 let reader = web_sys::FileReaderSync::new().map_err(|e| {
2161 std::io::Error::other(format!(
2162 "FileReaderSync is unavailable ({e:?}) — a local .rete is read from a Web Worker only, \
2163 never the main thread"
2164 ))
2165 })?;
2166 let mut out = Vec::with_capacity(ranges.len());
2167 for (&offset, &len) in offsets.iter().zip(&lens) {
2168 let start = offset as f64;
2169 let end = start + f64::from(len);
2170 if end > size {
2171 return Err(std::io::Error::new(
2172 std::io::ErrorKind::UnexpectedEof,
2173 format!("range {offset}..{end} is past the end of {url} ({size} bytes)"),
2174 ));
2175 }
2176 let slice = blob
2177 .slice_with_f64_and_f64(start, end)
2178 .map_err(|e| std::io::Error::other(format!("Blob.slice failed on {url}: {e:?}")))?;
2179 let buffer = reader
2180 .read_as_array_buffer(&slice)
2181 .map_err(|e| std::io::Error::other(format!("FileReaderSync failed on {url}: {e:?}")))?;
2182 let bytes = js_sys::Uint8Array::new(&buffer).to_vec();
2183 if bytes.len() != len as usize {
2184 return Err(std::io::Error::other(format!(
2185 "short local read: got {} of {len} bytes at offset {offset} from {url}",
2186 bytes.len()
2187 )));
2188 }
2189 out.push(bytes);
2190 }
2191 // One batch is one "request" to the UI, matching what the asyncify HTTP
2192 // reader reports for its concurrent batch — so the playground's range
2193 // inspector and `stats()` describe local and remote reads on one scale.
2194 report_progress(total, ranges);
2195 Ok(out)
2196}
2197
2198/// HTTP `Range` reader over **synchronous** XMLHttpRequest — the bridge that
2199/// lets the lazily-faulting `Rete::open_ranged_lazy` run in the browser with
2200/// the synchronous engine untouched. Browsers permit sync XHR with a binary
2201/// response **only inside Web Workers**: call [`sparql_url`] from a worker,
2202/// never the main thread (where the browser throws).
2203///
2204/// Also the **local** reader: a `rete-local:` URL (see [`LOCAL_SCHEME`]) swaps
2205/// the transport for `Blob.slice()` + `FileReaderSync` and changes nothing else.
2206struct XhrRangeReader {
2207 url: String,
2208 len: u64,
2209 /// The resource's first [`rete_core::HEADER_LEN`] bytes, cached once.
2210 ///
2211 /// Opening a `.rete` reads this window several times over — the sync length
2212 /// probe reads it, [`polyglot_base`](Self::polyglot_base) needs it, and every
2213 /// `read_*_ranged` helper starts by reading the header — and the bytes cannot
2214 /// change under us: the file is immutable for the session, and a host that
2215 /// served different bytes for the same range would already have broken every
2216 /// other read. So the first reader to pay for it fills this in and the rest
2217 /// are served from memory, which is what keeps the CARD tier at its
2218 /// advertised two requests.
2219 head: std::sync::OnceLock<Vec<u8>>,
2220 /// The **whole** resource, when the host answered a range request with
2221 /// `200` and the full body: it ignores `Range` (no `Accept-Ranges`, a proxy
2222 /// that strips it, a CDN that serves a cached full object). That body has
2223 /// already crossed the network by the time we see it, and every later range
2224 /// request would download it again, so it is kept and every read is served
2225 /// from it — the file then behaves like an in-memory one. Only bodies up to
2226 /// [`WHOLE_BODY_CAP`] are kept; a larger one is refused with the reason.
2227 whole: std::sync::OnceLock<Vec<u8>>,
2228}
2229
2230/// The largest full-body answer to a range request that [`XhrRangeReader`]
2231/// keeps and serves reads from. The playground loads a local file whole below
2232/// 128 MiB; past this cap a range-ignoring host is refused by name rather than
2233/// risking the wasm32 heap on a body the reader never asked for.
2234const WHOLE_BODY_CAP: u64 = 256 << 20;
2235
2236/// The largest remote file [`XhrRangeReader`] reads whole at open, the same
2237/// 8 MiB as the native reader's `RETE_HTTP_WHOLE_MB` default.
2238const WHOLE_FILE_MAX: u64 = 8 << 20;
2239
2240/// One read served from a whole-resource body (see [`XhrRangeReader::whole`]),
2241/// bounds-checked: the body is whatever the host sent, so a read past its end
2242/// is a clean error, never a panic.
2243fn slice_whole(whole: &[u8], offset: u64, len: u64, url: &str) -> std::io::Result<Vec<u8>> {
2244 let end = offset
2245 .checked_add(len)
2246 .filter(|&end| end <= whole.len() as u64)
2247 .ok_or_else(|| {
2248 std::io::Error::other(format!(
2249 "range {offset}+{len} is past the end of the {} bytes {url} sent",
2250 whole.len()
2251 ))
2252 })?;
2253 Ok(whole[offset as usize..end as usize].to_vec())
2254}
2255
2256/// Whether a `200` full body to a range request can stand in for the file:
2257/// not empty, within [`WHOLE_BODY_CAP`], and — when its first KiB is a `.rete`
2258/// header — exactly the length that header derives (anything else is a
2259/// truncated file or a representation whose bytes are not the file's).
2260#[cfg_attr(feature = "asyncify", allow(dead_code))] // the async build keeps whole bodies on the JS side
2261fn accept_whole_body(body_len: u64, derived: Option<u64>, url: &str) -> Result<(), String> {
2262 if body_len == 0 {
2263 return Err(format!("probe {url}: 200 with an empty body"));
2264 }
2265 if body_len > WHOLE_BODY_CAP {
2266 return Err(format!(
2267 "the host ignored the Range header and sent all {body_len} bytes of {url} \
2268 (status 200 instead of 206 Partial Content); a full body over {} MiB is \
2269 not kept in memory — host the file where HTTP range requests work",
2270 WHOLE_BODY_CAP >> 20
2271 ));
2272 }
2273 match derived {
2274 Some(d) if d != body_len => Err(format!(
2275 "the host ignored the Range header and sent {body_len} bytes of {url}, \
2276 but the file's own header says it is {d} bytes — the file is truncated, \
2277 or the host sends a different representation"
2278 )),
2279 _ => Ok(()),
2280 }
2281}
2282
2283// Asyncify variant only (feature = "asyncify"): one dedicated async import in
2284// module "env". The worker provides it as a `Promise.all` of `fetch` over the N
2285// ranges, writing the spans concatenated into `dst` and returning the total
2286// bytes. `wasm-opt --asyncify --pass-arg=asyncify-imports@env.rete_fetch_ranges`
2287// instruments every fn that can reach it to SUSPEND the wasm while the worker
2288// awaits the fetches, then RESUME — concurrent reads with no SAB / cross-origin
2289// isolation. Proven in dev/asyncify-wbg-probe (wasm-bindgen × Asyncify).
2290#[cfg(feature = "asyncify")]
2291#[link(wasm_import_module = "env")]
2292extern "C" {
2293 fn rete_fetch_ranges(
2294 url_ptr: *const u8,
2295 url_len: usize,
2296 offs_ptr: *const u64,
2297 lens_ptr: *const u32,
2298 n: usize,
2299 dst_ptr: *mut u8,
2300 ) -> usize;
2301 /// Async length probe (a `bytes=0-0` fetch; reads the total from
2302 /// `Content-Range`). Writes the u64 length to `out_ptr`, returns 1 on
2303 /// success. Lets the asyncify build open a file with NO sync XHR at all.
2304 fn rete_file_len(url_ptr: *const u8, url_len: usize, out_ptr: *mut u64) -> usize;
2305 /// LEAF panic reporter (deliberately NOT in asyncify-imports): the panic
2306 /// hook passes the raw `Location` pointers so the host can log file:line
2307 /// without any formatting — see `__start`.
2308 fn rete_panic_report(file_ptr: *const u8, file_len: usize, line: u32);
2309}
2310
2311fn checked_async_layout(ranges: &[(u64, u64)]) -> std::io::Result<(Vec<u64>, Vec<u32>, usize)> {
2312 let offs: Vec<u64> = ranges.iter().map(|&(offset, _)| offset).collect();
2313 let lens: Vec<u32> = ranges
2314 .iter()
2315 .map(|&(_, len)| {
2316 u32::try_from(len).map_err(|_| {
2317 std::io::Error::new(
2318 std::io::ErrorKind::InvalidInput,
2319 format!("range length {len} exceeds the wasm32 u32 length type"),
2320 )
2321 })
2322 })
2323 .collect::<std::io::Result<_>>()?;
2324 let total = lens.iter().try_fold(0usize, |sum, &len| {
2325 sum.checked_add(len as usize).ok_or_else(|| {
2326 std::io::Error::new(
2327 std::io::ErrorKind::InvalidInput,
2328 "range length sum overflow",
2329 )
2330 })
2331 })?;
2332 let _total_u32 = u32::try_from(total).map_err(|_| {
2333 std::io::Error::new(
2334 std::io::ErrorKind::InvalidInput,
2335 "range response exceeds the wasm32 address space",
2336 )
2337 })?;
2338 Ok((offs, lens, total))
2339}
2340
2341fn split_range_response(
2342 ranges: &[(u64, u64)],
2343 dst: Vec<u8>,
2344 got: usize,
2345 source: &str,
2346) -> std::io::Result<Vec<Vec<u8>>> {
2347 let (_, lens, total) = checked_async_layout(ranges)?;
2348 if dst.len() != total {
2349 return Err(std::io::Error::other(format!(
2350 "async fetch destination has {} bytes, expected {total} for {source}",
2351 dst.len()
2352 )));
2353 }
2354 if got != total {
2355 return Err(std::io::Error::other(format!(
2356 "async fetch returned {got} of {total} bytes for {source}"
2357 )));
2358 }
2359
2360 let mut out = Vec::with_capacity(ranges.len());
2361 let mut pos = 0usize;
2362 for len in lens {
2363 let end = pos + len as usize;
2364 out.push(dst[pos..end].to_vec());
2365 pos = end;
2366 }
2367 Ok(out)
2368}
2369
2370#[cfg(test)]
2371mod async_range_tests {
2372 use super::{checked_async_layout, split_range_response};
2373
2374 #[test]
2375 fn rejects_a_range_larger_than_the_wasm32_length_type() {
2376 let error = checked_async_layout(&[(0, u64::from(u32::MAX) + 1)]).unwrap_err();
2377 assert!(error.to_string().contains("u32"));
2378 }
2379
2380 #[test]
2381 fn rejects_a_total_larger_than_the_wasm32_address_space() {
2382 let error = checked_async_layout(&[(0, u64::from(u32::MAX)), (0, 1)]).unwrap_err();
2383 assert!(error.to_string().contains("address space"));
2384 }
2385
2386 #[test]
2387 fn accepts_no_ranges_and_preserves_zero_length_ranges() {
2388 let (offs, lens, total) = checked_async_layout(&[]).unwrap();
2389 assert!(offs.is_empty());
2390 assert!(lens.is_empty());
2391 assert_eq!(total, 0);
2392
2393 let (_, lens, total) = checked_async_layout(&[(7, 0), (9, 0)]).unwrap();
2394 assert_eq!(lens, vec![0, 0]);
2395 assert_eq!(total, 0);
2396 }
2397
2398 #[test]
2399 fn rejects_a_short_javascript_response() {
2400 let error = split_range_response(&[(0, 4)], vec![1, 2, 3, 4], 3, "fixture").unwrap_err();
2401 assert!(error.to_string().contains("returned 3 of 4 bytes"));
2402 }
2403
2404 #[test]
2405 fn only_the_local_scheme_routes_to_the_blob_transport() {
2406 use super::is_local_url;
2407 assert!(is_local_url("rete-local:1/mirbase.rete"));
2408 // Everything the reader has ever been given before must stay on HTTP —
2409 // a URL that merely CONTAINS the token is not a local file.
2410 assert!(!is_local_url("https://example.org/rete-local:1/x.rete"));
2411 assert!(!is_local_url(
2412 "https://data.graphplaza.com/mirbase/mirbase.rete"
2413 ));
2414 assert!(!is_local_url("file:///tmp/x.rete"));
2415 assert!(!is_local_url(""));
2416 }
2417
2418 // A host that ignores Range sends the whole file with a 200. The reader keeps
2419 // that body and serves reads as slices of it; these pin the two decisions
2420 // that have no browser in them: which bodies are kept, and the slicing.
2421 #[test]
2422 fn a_whole_body_is_kept_only_when_it_can_be_the_file() {
2423 use super::{accept_whole_body, WHOLE_BODY_CAP};
2424 let url = "https://host/f.rete";
2425 assert!(accept_whole_body(4096, Some(4096), url).is_ok());
2426 // Not a .rete header (a polyglot, say): its own length is all there is.
2427 assert!(accept_whole_body(4096, None, url).is_ok());
2428 assert!(accept_whole_body(WHOLE_BODY_CAP, None, url).is_ok());
2429 let empty = accept_whole_body(0, None, url).unwrap_err();
2430 assert!(empty.contains("empty body"), "{empty}");
2431 // Too big to hold: refused by name, with the cap, not kept.
2432 let big = accept_whole_body(WHOLE_BODY_CAP + 1, Some(WHOLE_BODY_CAP + 1), url).unwrap_err();
2433 assert!(
2434 big.contains("ignored the Range header") && big.contains("256 MiB"),
2435 "{big}"
2436 );
2437 // A body that is not the length the file's header derives is not the file.
2438 let short = accept_whole_body(4000, Some(4096), url).unwrap_err();
2439 assert!(short.contains("header says it is 4096 bytes"), "{short}");
2440 }
2441
2442 #[test]
2443 fn reads_from_a_whole_body_are_bounds_checked_slices() {
2444 use super::slice_whole;
2445 let body: Vec<u8> = (0..=255u8).collect();
2446 assert_eq!(slice_whole(&body, 10, 3, "u").unwrap(), vec![10, 11, 12]);
2447 assert_eq!(
2448 slice_whole(&body, 250, 6, "u").unwrap(),
2449 vec![250, 251, 252, 253, 254, 255]
2450 );
2451 assert!(slice_whole(&body, 0, 0, "u").unwrap().is_empty());
2452 let past = slice_whole(&body, 250, 7, "u").unwrap_err();
2453 assert!(
2454 past.to_string().contains("past the end of the 256 bytes"),
2455 "{past}"
2456 );
2457 // An overflowing offset is an error, never a wrapped index.
2458 assert!(slice_whole(&body, u64::MAX, 2, "u").is_err());
2459 }
2460}
2461
2462#[cfg(feature = "asyncify")]
2463impl XhrRangeReader {
2464 /// Fetch all `ranges` through the async import in one suspend/resume, then
2465 /// split the concatenated bytes back into per-range buffers.
2466 fn read_ranges_async(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
2467 let (offs, lens, total) = checked_async_layout(ranges)?;
2468 if total == 0 {
2469 return Ok(ranges.iter().map(|_| Vec::new()).collect());
2470 }
2471 let mut dst = vec![0u8; total];
2472 if offs.len() != ranges.len() || lens.len() != ranges.len() || dst.len() != total {
2473 return Err(std::io::Error::other(
2474 "async range layout does not match its allocated buffers",
2475 ));
2476 }
2477 // SAFETY: self.url, offs, and lens own their pointer/length pairs for
2478 // this call; dst owns exactly total writable bytes. The imported
2479 // function contract writes at most that concatenated length and returns
2480 // before any of these allocations leave scope.
2481 let got = unsafe {
2482 rete_fetch_ranges(
2483 self.url.as_ptr(),
2484 self.url.len(),
2485 offs.as_ptr(),
2486 lens.as_ptr(),
2487 ranges.len(),
2488 dst.as_mut_ptr(),
2489 )
2490 };
2491 let out = split_range_response(ranges, dst, got, &self.url)?;
2492 report_progress(total, ranges);
2493 Ok(out)
2494 }
2495}
2496
2497impl XhrRangeReader {
2498 /// Probe the resource length. HTTP's own signals are unreliable
2499 /// cross-origin — a host may advertise the size of a **compressed**
2500 /// representation in `Content-Length` (GitHub Pages HEADs a 71 MB `.rete`
2501 /// as its 58 MB gzip; `Content-Encoding` is not CORS-safelisted, so JS
2502 /// cannot even see the lie) while range requests address the identity
2503 /// bytes, and may hide `Content-Range` by omitting
2504 /// `Access-Control-Expose-Headers` (GitHub Pages, Zenodo). So the probe
2505 /// reads the file's OWN first KiB — the `.rete` header, whose section
2506 /// directory pins the exact length (issue #95) — and only falls back to
2507 /// the transport's numbers for a resource that isn't a `.rete`.
2508 fn open(url: &str) -> Result<Self, JsValue> {
2509 let r = Self::open_probe(url)?;
2510 r.fetch_small_whole();
2511 Ok(r)
2512 }
2513
2514 /// A remote file of at most [`WHOLE_FILE_MAX`] is read whole right after
2515 /// the length probe: one more request, and no later read leaves memory
2516 /// (2 round trips per query on a 7 MB catalog file, instead of 11–33). A
2517 /// failure leaves the reader ranged. Local files are never read whole here.
2518 fn fetch_small_whole(&self) {
2519 if is_local_url(&self.url) || self.whole.get().is_some() || self.len == 0 {
2520 return;
2521 }
2522 if self.len > WHOLE_FILE_MAX {
2523 return;
2524 }
2525 // The probe's window already holds a file this small: no request.
2526 if let Some(h) = self.head.get().filter(|h| h.len() as u64 >= self.len) {
2527 let _ = self.whole.set(h[..self.len as usize].to_vec());
2528 return;
2529 }
2530 if let Ok(body) = self.fetch_at(0, self.len) {
2531 if body.len() as u64 == self.len {
2532 let _ = self.whole.set(body);
2533 }
2534 }
2535 }
2536
2537 fn open_probe(url: &str) -> Result<Self, JsValue> {
2538 // A LOCAL blob needs no probe at all: its length is a property, not a
2539 // network fact, and none of the reasons the HTTP probe exists (a
2540 // compressing host, a hidden Content-Range, a 405 on HEAD) can apply.
2541 // The header window is faulted on first use like any other range, so
2542 // opening a local file costs exactly ONE read before the engine starts.
2543 // Placed above both build variants because it is the same either way.
2544 if is_local_url(url) {
2545 // `Blob.size` is an f64: reject NaN/∞/0 explicitly rather than
2546 // letting `as u64` saturate one of them into a plausible length.
2547 let len = local_blob(url).map_err(|e| js_error(e.to_string()))?.size();
2548 if !len.is_finite() || len < 1.0 {
2549 return Err(js_error(format!("{url} is an empty local file")));
2550 }
2551 return Ok(Self {
2552 url: url.to_string(),
2553 len: len as u64,
2554 head: std::sync::OnceLock::new(),
2555 whole: std::sync::OnceLock::new(),
2556 });
2557 }
2558 // Asyncify build: probe the length via the async import — no sync XHR.
2559 // The JS side (`__reteDoLen` in scripts/build_playground.py) applies
2560 // this same derive-from-the-header-then-validate-the-footer strategy.
2561 #[cfg(feature = "asyncify")]
2562 {
2563 let mut len: u64 = 0;
2564 // SAFETY: url owns url.as_ptr() for exactly url.len() bytes and len
2565 // owns one writable u64. The import writes only that value and
2566 // returns before either borrowed allocation leaves scope.
2567 let ok = unsafe { rete_file_len(url.as_ptr(), url.len(), &mut len as *mut u64) };
2568 if ok == 0 || len == 0 {
2569 return Err(js_error(format!("could not determine length of {url}")));
2570 }
2571 // The async length probe reads the window on the JS side and only
2572 // hands back the length, so the first `head_window()` pays for it.
2573 // (A range-ignoring host's full body is kept on the JS side too:
2574 // `__reteDoLen` stores it and `__reteDoFetch` slices from it.)
2575 Ok(Self {
2576 url: url.to_string(),
2577 len,
2578 head: std::sync::OnceLock::new(),
2579 whole: std::sync::OnceLock::new(),
2580 })
2581 }
2582 #[cfg(not(feature = "asyncify"))]
2583 {
2584 // Hugging Face's Space gateway is intermittently flaky on the length
2585 // probe (a 200 with no Content-Length, a chunked response with neither
2586 // header, …). A fresh attempt usually lands on a healthy response,
2587 // so retry a few times before surfacing the error.
2588 let mut last = format!("could not determine length of {url}");
2589 for _ in 0..4 {
2590 match Self::probe_len(url) {
2591 Ok(probe) => {
2592 // The probe is the open's first read (64 KiB, or the
2593 // whole of a small file): report it like any other, so a
2594 // query over a file that small still shows its read.
2595 let got = probe.whole.as_ref().map_or(probe.head.len(), Vec::len);
2596 if got > 0 {
2597 report_progress(got, &[(0, got as u64)]);
2598 }
2599 let cached = std::sync::OnceLock::new();
2600 if !probe.head.is_empty() {
2601 let _ = cached.set(probe.head);
2602 }
2603 let whole = std::sync::OnceLock::new();
2604 if let Some(body) = probe.whole {
2605 let _ = whole.set(body);
2606 }
2607 return Ok(Self {
2608 url: url.to_string(),
2609 len: probe.len,
2610 head: cached,
2611 whole,
2612 });
2613 }
2614 // A refusal that a retry would only repeat — at the cost of
2615 // downloading a whole body again — is final.
2616 Err(e) if !e.retry => return Err(js_error(e.msg)),
2617 Err(e) => last = e.msg,
2618 }
2619 }
2620 Err(js_error(last))
2621 }
2622 }
2623
2624 /// The cached first [`rete_core::HEADER_LEN`] bytes, fetching them once if
2625 /// nobody has yet. `None` only when the resource cannot be read at all — the
2626 /// open that follows reports the real error.
2627 fn head_window(&self) -> Option<&[u8]> {
2628 if let Some(head) = self.head.get() {
2629 return Some(head);
2630 }
2631 let head = self
2632 .fetch_at(0, rete_core::v06::FIRST_READ.min(self.len))
2633 .ok()?;
2634 let _ = self.head.set(head);
2635 self.head.get().map(Vec::as_slice)
2636 }
2637
2638 /// Where the `.rete` starts inside this resource: `0` for an ordinary file,
2639 /// and the size of the HTML shell for a **polyglot** — a file that is both a
2640 /// web page and a graph, whose first bytes carry a `RETE-BASE:` marker naming
2641 /// the offset. Reads only the header window, which the open that follows
2642 /// needs anyway and now gets from [`head_window`](Self::head_window).
2643 fn polyglot_base(&self) -> u64 {
2644 let Some(head) = self.head_window() else {
2645 return 0;
2646 };
2647 if head.starts_with(&rete_core::MAGIC) {
2648 return 0;
2649 }
2650 rete_core::detect_polyglot_base(head).unwrap_or(0)
2651 }
2652
2653 /// A HEAD length probe: `Content-Length` is CORS-safelisted (readable
2654 /// cross-origin with no `access-control-expose-headers` entry needed).
2655 /// **Last-resort fallback only**: on a transparently-compressing host this
2656 /// is the size of the gzip representation, not the file (issue #95), which
2657 /// is exactly why [`probe_len`](Self::probe_len) prefers the file's own
2658 /// header. Returns `None` on a non-2xx (some hosts 405 HEAD) or a
2659 /// missing/zero length.
2660 #[cfg(not(feature = "asyncify"))]
2661 fn head_len(url: &str) -> Option<u64> {
2662 let xhr = web_sys::XmlHttpRequest::new().ok()?;
2663 xhr.open_with_async("HEAD", url, false).ok()?;
2664 xhr.send().ok()?;
2665 let status = xhr.status().ok()?;
2666 if !(200..300).contains(&status) {
2667 return None;
2668 }
2669 xhr.get_response_header("Content-Length")
2670 .ok()
2671 .flatten()
2672 .and_then(|v| v.parse::<u64>().ok())
2673 .filter(|&n| n > 0)
2674 }
2675
2676 /// One ranged `GET bytes=first-last`. The total is `None` when
2677 /// `Content-Range` is absent, hidden by CORS (no
2678 /// `Access-Control-Expose-Headers` entry: GitHub Pages, Zenodo), or
2679 /// `bytes a-b/*`.
2680 ///
2681 /// A `200` is a host that ignored `Range` and sent the whole resource. Its
2682 /// size is read from the response itself, and the body is copied into wasm
2683 /// memory only when it is small enough to keep ([`WHOLE_BODY_CAP`]): a
2684 /// multi-GB body must not be pulled across into the wasm32 heap just to be
2685 /// refused.
2686 #[cfg(not(feature = "asyncify"))]
2687 fn ranged_get(url: &str, first: u64, last: u64) -> Result<Ranged, String> {
2688 let err = |m: &str| m.to_string();
2689 let xhr = web_sys::XmlHttpRequest::new().map_err(|_| err("xhr"))?;
2690 xhr.open_with_async("GET", url, false)
2691 .map_err(|_| err("open"))?;
2692 xhr.set_response_type(web_sys::XmlHttpRequestResponseType::Arraybuffer);
2693 xhr.set_request_header("Range", &format!("bytes={first}-{last}"))
2694 .map_err(|_| err("range header"))?;
2695 xhr.send()
2696 .map_err(|_| format!("probe {url}: network error"))?;
2697 let status = xhr.status().map_err(|_| err("status"))?;
2698 // `Content-Range: bytes 0-1023/12345` — the part after `/` is the total.
2699 let total = xhr
2700 .get_response_header("Content-Range")
2701 .ok()
2702 .flatten()
2703 .and_then(|v| {
2704 v.rsplit('/')
2705 .next()
2706 .and_then(|t| t.trim().parse::<u64>().ok())
2707 });
2708 let array = xhr.response().ok().map(|r| js_sys::Uint8Array::new(&r));
2709 let full_len = array.as_ref().map(|a| u64::from(a.length())).unwrap_or(0);
2710 if status == 200 && full_len > WHOLE_BODY_CAP {
2711 // Keep only the requested window (the header parse still wants it).
2712 let end = (last.saturating_add(1)).min(full_len) as u32;
2713 let body = array
2714 .map(|a| a.subarray(first.min(full_len) as u32, end).to_vec())
2715 .unwrap_or_default();
2716 return Ok(Ranged {
2717 status,
2718 total: None,
2719 body,
2720 full_len,
2721 });
2722 }
2723 let body = array.map(|a| a.to_vec()).unwrap_or_default();
2724 Ok(Ranged {
2725 status,
2726 total,
2727 body,
2728 full_len,
2729 })
2730 }
2731
2732 /// One length probe. Reads the resource's first KiB over a ranged `GET`;
2733 /// if that is a `.rete` header, the length is **derived from the file
2734 /// itself** (`Header::expected_file_len`: sections are back-to-back and
2735 /// the file ends with the 4-byte `RETE` footer) and, unless a visible
2736 /// `Content-Range` already confirms it, **validated** by reading those 4
2737 /// footer bytes — one extra tiny request, only on hosts whose headers are
2738 /// unusable. Non-`.rete` resources fall back to `Content-Range`, then to
2739 /// HEAD's `Content-Length` (CORS-safelisted, so readable on a host that
2740 /// hides `Content-Range`). A 206's `Content-Length` is never believed (it
2741 /// is the size of the partial body, and taking it as the file size made
2742 /// every later read "range out of bounds").
2743 ///
2744 /// A host that ignores `Range` answers with `200` and the whole file. That
2745 /// body is kept (up to [`WHOLE_BODY_CAP`]) and the reader serves every read
2746 /// from it; a larger one is refused, finally, so the open does not retry
2747 /// and download it again.
2748 ///
2749 /// Returns the length **and the head window it read** — the caller keeps it
2750 /// so polyglot detection costs no second request.
2751 #[cfg(not(feature = "asyncify"))]
2752 fn probe_len(url: &str) -> Result<Probe, ProbeFail> {
2753 let Ranged {
2754 status,
2755 total: cr_total,
2756 body,
2757 full_len,
2758 } = Self::ranged_get(url, 0, rete_core::v06::FIRST_READ - 1).map_err(ProbeFail::retry)?;
2759 // The probe IS the first read: 64 KiB, the header plus — on a 0x06 file
2760 // — the root, the card and, on a small file, every leaf directory. Kept
2761 // whole, so every later read inside it costs nothing. (A Range-ignoring
2762 // host may have sent the whole file here.)
2763 let head =
2764 |body: &[u8]| body[..body.len().min(rete_core::v06::FIRST_READ as usize)].to_vec();
2765 let derived_of = |body: &[u8]| {
2766 rete_core::Header::from_bytes(body)
2767 .ok()
2768 .and_then(|h| h.expected_file_len())
2769 };
2770 if status == 200 {
2771 accept_whole_body(full_len, derived_of(&body), url).map_err(ProbeFail::final_)?;
2772 // The whole file crossed the network: say so, or the open's byte
2773 // count would claim a bounded read (docs/architecture.md).
2774 report_progress(body.len(), &[(0, full_len)]);
2775 return Ok(Probe {
2776 len: full_len,
2777 head: head(&body),
2778 whole: Some(body),
2779 });
2780 }
2781 if status != 206 {
2782 return Err(ProbeFail::retry(format!("probe {url}: status {status}")));
2783 }
2784
2785 // A `.rete` self-describes its length; that is the only signal a
2786 // compressing/CORS-hiding host cannot skew (issue #95).
2787 if let Some(derived) = derived_of(&body) {
2788 if cr_total == Some(derived) {
2789 // transport agrees — no extra request
2790 return Ok(Probe {
2791 len: derived,
2792 head: head(&body),
2793 whole: None,
2794 });
2795 }
2796 // Ask the file: its last 4 bytes are the `RETE` footer. A 206
2797 // with exactly those bytes proves `derived` addresses real
2798 // identity bytes; anything else means the file is truncated
2799 // or the host's ranges don't address the file's bytes.
2800 let tail = Self::ranged_get(url, derived - 4, derived - 1).map_err(ProbeFail::retry)?;
2801 if tail.status == 206 && tail.body == rete_core::MAGIC {
2802 return Ok(Probe {
2803 len: derived,
2804 head: head(&body),
2805 whole: None,
2806 });
2807 }
2808 if tail.status == 200 {
2809 // Ranges honoured for the head, ignored for the tail: the
2810 // body is the whole file, so keep it like any other 200.
2811 accept_whole_body(tail.full_len, Some(derived), url).map_err(ProbeFail::final_)?;
2812 if tail.body.ends_with(&rete_core::MAGIC) {
2813 report_progress(tail.body.len(), &[(0, tail.full_len)]);
2814 return Ok(Probe {
2815 len: derived,
2816 head: head(&body),
2817 whole: Some(tail.body),
2818 });
2819 }
2820 }
2821 return Err(ProbeFail::retry(format!(
2822 "length probe disagrees with the file: its header derives \
2823 {derived} bytes but the host{} has no RETE footer at \
2824 {}..{} (tail probe: status {}) — the file is truncated, \
2825 or the host serves ranges over a compressed representation",
2826 match cr_total {
2827 Some(t) => format!(" reports {t} bytes and"),
2828 None => String::from(" hides Content-Range and"),
2829 },
2830 derived - 4,
2831 derived,
2832 tail.status,
2833 )));
2834 }
2835 if body.starts_with(&[0x1f, 0x8b]) {
2836 // The ranged body is a slice of a GZIP stream: the host applied
2837 // `Content-Encoding` to the range response, so byte offsets do not
2838 // address the file's bytes at all. No length can fix that.
2839 return Err(ProbeFail::retry(format!(
2840 "the host serves HTTP ranges over a gzip-compressed representation \
2841 of {url}; range offsets cannot address the file's bytes"
2842 )));
2843 }
2844
2845 // Not a `.rete` header: fall back to the transport's signals. A POLYGLOT
2846 // lands here — its byte 0 is `<`, so the header parse above failed — and the
2847 // head we keep is exactly what `polyglot_base` needs to find the graph.
2848 if let Some(total) = cr_total {
2849 return Ok(Probe {
2850 len: total,
2851 head: head(&body),
2852 whole: None,
2853 });
2854 }
2855 if let Some(len) = Self::head_len(url) {
2856 return Ok(Probe {
2857 len,
2858 head: head(&body),
2859 whole: None,
2860 });
2861 }
2862 Err(ProbeFail::retry(format!(
2863 "could not determine length of {url}"
2864 )))
2865 }
2866}
2867
2868/// One ranged response (see [`XhrRangeReader::ranged_get`]).
2869#[cfg(not(feature = "asyncify"))]
2870struct Ranged {
2871 status: u16,
2872 /// From a readable `Content-Range`; `None` when hidden or absent.
2873 total: Option<u64>,
2874 /// The body — for a `200` over [`WHOLE_BODY_CAP`], only the requested window.
2875 body: Vec<u8>,
2876 /// The byte length of the whole response body as the browser received it.
2877 full_len: u64,
2878}
2879
2880/// What a length probe learnt: the length, the header window, and — when the
2881/// host ignored `Range` — the whole file.
2882#[cfg(not(feature = "asyncify"))]
2883struct Probe {
2884 len: u64,
2885 head: Vec<u8>,
2886 whole: Option<Vec<u8>>,
2887}
2888
2889/// A failed probe, and whether another attempt could change the answer.
2890#[cfg(not(feature = "asyncify"))]
2891struct ProbeFail {
2892 msg: String,
2893 retry: bool,
2894}
2895
2896#[cfg(not(feature = "asyncify"))]
2897impl ProbeFail {
2898 fn retry(msg: String) -> Self {
2899 Self { msg, retry: true }
2900 }
2901 fn final_(msg: String) -> Self {
2902 Self { msg, retry: false }
2903 }
2904}
2905
2906impl RangeReader for XhrRangeReader {
2907 fn len(&self) -> u64 {
2908 self.len
2909 }
2910
2911 /// The asyncify build fires its batched ranges as one concurrent
2912 /// `Promise.all` of fetches; the sync build is serial XHR unless the
2913 /// opt-in COI fetch-worker pool (`globalThis.reteReadMany`) is installed.
2914 /// 16 matches the pool size and the CLI's thread fan-out.
2915 fn concurrency(&self) -> usize {
2916 // A local blob has no round trip to hide, so the planner may probe as
2917 // freely as the concurrent HTTP readers do — and a wider batch means
2918 // fewer crossings of the wasm↔JS boundary, which is the only real cost
2919 // here. Same number as the asyncify reader and the CLI's fan-out.
2920 // A range-ignoring host's whole body is memory now, the same case.
2921 if is_local_url(&self.url) || self.whole.get().is_some() {
2922 return 16;
2923 }
2924 #[cfg(feature = "asyncify")]
2925 {
2926 16
2927 }
2928 #[cfg(not(feature = "asyncify"))]
2929 {
2930 let has_pool =
2931 js_sys::Reflect::get(&js_sys::global(), &JsValue::from_str("reteReadMany"))
2932 .map(|h| h.is_function())
2933 .unwrap_or(false);
2934 if has_pool {
2935 16
2936 } else {
2937 1
2938 }
2939 }
2940 }
2941
2942 /// The cached header window (or a whole body) is in memory: a block cache
2943 /// above this reader serves reads inside it without a fetch.
2944 fn resident_prefix(&self) -> u64 {
2945 if self.whole.get().is_some() {
2946 return self.len;
2947 }
2948 self.head.get().map_or(0, |h| h.len() as u64)
2949 }
2950
2951 /// Reads that land entirely inside the cached header window are answered
2952 /// from memory: opening a file re-reads those bytes several times (length
2953 /// probe, polyglot marker, then every `read_*_ranged` helper's own header
2954 /// read) and they are immutable for the session, so paying for them once is
2955 /// the difference between the CARD tier costing two requests and four.
2956 fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
2957 if len == 0 {
2958 return Ok(Vec::new());
2959 }
2960 if let Some(whole) = self.whole.get() {
2961 return slice_whole(whole, offset, len, &self.url);
2962 }
2963 if let Some(head) = self.head.get() {
2964 if let Some(end) = offset.checked_add(len) {
2965 if end <= head.len() as u64 {
2966 return Ok(head[offset as usize..end as usize].to_vec());
2967 }
2968 }
2969 }
2970 self.fetch_at(offset, len)
2971 }
2972
2973 /// Synchronous XHR can't run two requests at once on one thread, so the
2974 /// engine's sequential faults serialize their round trips. If the page
2975 /// installs `globalThis.reteReadMany(url, offsets, lens)` — a fetch-worker
2976 /// pool that fetches the ranges in parallel and blocks (via SAB/Atomics)
2977 /// until done — use it; it returns one buffer with the spans concatenated
2978 /// in order, or `null`/throws if it can't (no cross-origin isolation, a
2979 /// non-206, a short read). On `null` we fall back to the sequential reads
2980 /// below, which keep the rigorous per-range validation.
2981 fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
2982 // Local: the whole batch is sliced in one call, in either build.
2983 if is_local_url(&self.url) {
2984 return read_local_ranges(&self.url, ranges);
2985 }
2986 // Asyncify build: fetch the whole batch concurrently in one suspend.
2987 #[cfg(feature = "asyncify")]
2988 {
2989 self.read_ranges_async(ranges)
2990 }
2991 #[cfg(not(feature = "asyncify"))]
2992 {
2993 let (_, _, total) = checked_async_layout(ranges)?;
2994 // The host ignored Range and its whole body is in memory: every
2995 // range is a slice of it, and the pool would only download it again.
2996 if self.whole.get().is_some() {
2997 return ranges.iter().map(|&(o, l)| self.read_at(o, l)).collect();
2998 }
2999 if ranges.len() > 1 {
3000 if let Some(buf) = self.read_many_via_pool(ranges) {
3001 if buf.len() == total {
3002 return split_range_response(ranges, buf, total, "globalThis.reteReadMany");
3003 }
3004 }
3005 }
3006 ranges.iter().map(|&(o, l)| self.read_at(o, l)).collect()
3007 }
3008 }
3009}
3010
3011/// The raw transport, below the header cache in [`RangeReader::read_at`].
3012impl XhrRangeReader {
3013 /// One real range request. Never consults the cached header window — this is
3014 /// what fills it.
3015 fn fetch_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
3016 // Local: one `Blob.slice()` + `FileReaderSync`, in either build.
3017 if is_local_url(&self.url) {
3018 return read_local_ranges(&self.url, &[(offset, len)])?
3019 .pop()
3020 .ok_or_else(|| std::io::Error::other("local read returned no range"));
3021 }
3022 // Asyncify build: a single read is a 1-range async fetch (suspends).
3023 #[cfg(feature = "asyncify")]
3024 {
3025 Ok(self
3026 .read_ranges_async(&[(offset, len)])?
3027 .into_iter()
3028 .next()
3029 .unwrap())
3030 }
3031 #[cfg(not(feature = "asyncify"))]
3032 {
3033 let js = |e: JsValue| std::io::Error::other(format!("XHR error: {e:?}"));
3034 let xhr = web_sys::XmlHttpRequest::new().map_err(js)?;
3035 xhr.open_with_async("GET", &self.url, false).map_err(js)?;
3036 xhr.set_response_type(web_sys::XmlHttpRequestResponseType::Arraybuffer);
3037 let end = offset
3038 .checked_add(len - 1)
3039 .ok_or_else(|| std::io::Error::other("HTTP range end overflow"))?;
3040 xhr.set_request_header("Range", &format!("bytes={offset}-{end}"))
3041 .map_err(js)?;
3042 xhr.send().map_err(js)?;
3043 let status = xhr.status().map_err(js)?;
3044 // A 200 means the server ignored Range and sent the whole body. It is
3045 // never mis-sliced as if it were the range: when it is exactly the
3046 // file (the length the probe established) and small enough to keep,
3047 // it is kept and this and every later read become slices of it;
3048 // otherwise it is refused loudly, as the CLI's HttpRangeReader does.
3049 if status == 200 {
3050 let array = js_sys::Uint8Array::new(&xhr.response().map_err(js)?);
3051 let got = u64::from(array.length());
3052 if got == self.len && accept_whole_body(got, None, &self.url).is_ok() {
3053 let whole = array.to_vec();
3054 let out = slice_whole(&whole, offset, len, &self.url)?;
3055 // Report the whole body: it is what crossed the network.
3056 report_progress(whole.len(), &[(0, got)]);
3057 let _ = self.whole.set(whole);
3058 return Ok(out);
3059 }
3060 return Err(std::io::Error::other(format!(
3061 "server ignored Range (status 200 with {got} bytes, expected 206 Partial \
3062 Content) for {}; the host must support HTTP range requests",
3063 self.url
3064 )));
3065 }
3066 if status != 206 {
3067 return Err(std::io::Error::other(format!(
3068 "server ignored Range (status {status}, expected 206 Partial Content) for {}; \
3069 the host must support HTTP range requests",
3070 self.url
3071 )));
3072 }
3073 let resp = xhr.response().map_err(js)?;
3074 let mut buf = js_sys::Uint8Array::new(&resp).to_vec();
3075 if (buf.len() as u64) < len {
3076 return Err(std::io::Error::other(format!(
3077 "short range response: got {} of {len} bytes at offset {offset} from {}",
3078 buf.len(),
3079 self.url
3080 )));
3081 }
3082 let requested = usize::try_from(len)
3083 .map_err(|_| std::io::Error::other("range exceeds wasm32 memory"))?;
3084 buf.truncate(requested);
3085 // Report this fetch to an optional progress hook so a worker can stream
3086 // live "N requests · M bytes" updates to the UI *during* the otherwise
3087 // opaque synchronous query (postMessage works mid-sync-call).
3088 report_progress(buf.len(), &[(offset, len)]);
3089 Ok(buf)
3090 }
3091 }
3092}
3093
3094/// Notify an optional `globalThis.reteProgress(bytes, spans, n)` hook of one
3095/// completed fetch (the worker forwards it to the UI and keeps a per-query
3096/// log). `spans` is a JS array of `"start-end"` byte-offset strings — capped at
3097/// [`PROGRESS_SPAN_CAP`] entries so a huge coalesced batch doesn't build a huge
3098/// array — and `n` is the TRUE span count. The offsets used to be dropped here
3099/// (only the byte count was forwarded), which left the playground's
3100/// "Range requests" inspector promising a `start-end` column it could never
3101/// fill. A no-op when unset.
3102const PROGRESS_SPAN_CAP: usize = 256;
3103
3104fn report_progress(bytes: usize, ranges: &[(u64, u64)]) {
3105 let g = js_sys::global();
3106 if let Ok(f) = js_sys::Reflect::get(&g, &JsValue::from_str("reteProgress")) {
3107 if let Ok(f) = f.dyn_into::<js_sys::Function>() {
3108 let spans = js_sys::Array::new();
3109 for &(offset, len) in ranges.iter().take(PROGRESS_SPAN_CAP) {
3110 let end = offset + len.saturating_sub(1);
3111 spans.push(&JsValue::from_str(&format!("{offset}-{end}")));
3112 }
3113 let _ = f.call3(
3114 &JsValue::NULL,
3115 &JsValue::from_f64(bytes as f64),
3116 &spans.into(),
3117 &JsValue::from_f64(ranges.len() as f64),
3118 );
3119 }
3120 }
3121}
3122
3123#[cfg(not(feature = "asyncify"))]
3124impl XhrRangeReader {
3125 /// Call the optional JS parallel-fetch hook. Returns the concatenated span
3126 /// bytes on full success, or `None` if the hook is absent, threw, or
3127 /// returned anything other than a `Uint8Array` (→ sequential fallback).
3128 fn read_many_via_pool(&self, ranges: &[(u64, u64)]) -> Option<Vec<u8>> {
3129 let global = js_sys::global();
3130 let hook = js_sys::Reflect::get(&global, &JsValue::from_str("reteReadMany")).ok()?;
3131 if !hook.is_function() {
3132 return None;
3133 }
3134 let hook = hook.dyn_into::<js_sys::Function>().ok()?;
3135 let (_, checked_lens, _) = checked_async_layout(ranges).ok()?;
3136 let count = u32::try_from(ranges.len()).ok()?;
3137 let offs = js_sys::Float64Array::new_with_length(count);
3138 let lens = js_sys::Float64Array::new_with_length(count);
3139 const MAX_SAFE_JS_INTEGER: u64 = (1_u64 << 53) - 1;
3140 for (i, (&(offset, _), &len)) in ranges.iter().zip(&checked_lens).enumerate() {
3141 if offset > MAX_SAFE_JS_INTEGER {
3142 return None;
3143 }
3144 let index = u32::try_from(i).ok()?;
3145 offs.set_index(index, offset as f64);
3146 lens.set_index(index, f64::from(len));
3147 }
3148 let res = hook
3149 .call3(&JsValue::NULL, &JsValue::from_str(&self.url), &offs, &lens)
3150 .ok()?;
3151 if res.is_null() || res.is_undefined() {
3152 return None;
3153 }
3154 Some(js_sys::Uint8Array::new(&res).to_vec())
3155 }
3156}
3157
3158/// The browser's [`rete_core::ServiceClient`]: a `SERVICE <endpoint> { … }`
3159/// block POSTs its sub-query to the endpoint over synchronous XHR (worker-only,
3160/// like every other read here — the engine is synchronous) and rete-core parses
3161/// the SPARQL JSON results. The endpoint must send CORS headers (the public
3162/// ones — Wikidata, DBpedia — do).
3163struct XhrServiceClient;
3164
3165impl rete_core::ServiceClient for XhrServiceClient {
3166 fn query(&self, endpoint: &str, query: &str) -> Result<Vec<rete_core::Binding>, String> {
3167 let e = |m: &str| format!("{endpoint}: {m}");
3168 let xhr = web_sys::XmlHttpRequest::new().map_err(|_| e("xhr"))?;
3169 xhr.open_with_async("POST", endpoint, false)
3170 .map_err(|_| e("open"))?;
3171 xhr.set_request_header("Content-Type", "application/x-www-form-urlencoded")
3172 .map_err(|_| e("header"))?;
3173 xhr.set_request_header("Accept", "application/sparql-results+json")
3174 .map_err(|_| e("header"))?;
3175 let body = format!(
3176 "query={}",
3177 String::from(js_sys::encode_uri_component(query))
3178 );
3179 xhr.send_with_opt_str(Some(&body))
3180 .map_err(|_| e("network error (endpoint down or no CORS)"))?;
3181 let status = xhr.status().map_err(|_| e("status"))?;
3182 if !(200..300).contains(&status) {
3183 return Err(e(&format!("HTTP {status}")));
3184 }
3185 let text = xhr
3186 .response_text()
3187 .map_err(|_| e("read"))?
3188 .unwrap_or_default();
3189 rete_core::parse_sparql_json_results(&text).map_err(|m| e(&m))
3190 }
3191}
3192
3193/// Run a SPARQL query against a **remote `.rete` URL**, fetching only the
3194/// byte ranges the query needs: header, dictionary chunk directories, tile
3195/// directories, then just the touched chunks and tiles (full scans coalesce
3196/// adjacent tiles into batched range reads). Returns the same JSON envelope
3197/// as [`query`], plus `"remote": { "fileLength", "bytes", "requests" }` —
3198/// how little of the file the query actually pulled.
3199///
3200/// **Worker-only.** This uses synchronous XHR (the engine is synchronous and
3201/// wasm cannot block on `fetch`); browsers allow that off the main thread
3202/// only, so call it from a Web Worker. A failed range fetch mid-query is an
3203/// error, never a silently incomplete result. The host must answer `Range`
3204/// requests with `206` (and send CORS headers if cross-origin).
3205/// Open a remote `.rete` lazily over HTTP range reads, returning the counting
3206/// reader (for byte/request stats) and the `Rete`. The seam every `*_url`
3207/// task shares with [`sparql_url`].
3208/// Auto-tune the block-cache size from the FILE SIZE — known for free at open
3209/// from the `Content-Range` of the single `bytes=0-0` request (one byte, no
3210/// download; it's what `stats().fileLength` reports). Remote reads are
3211/// round-trip-bound, so a bigger block means far fewer requests; benchmarked on
3212/// wikidata-1GB: 64 KiB = 262 reqs / 63 s, 256 KiB = 83 / 27 s, 512 KiB = 51 / 19 s.
3213/// Bigger files (bigger working sets + dictionaries) get bigger blocks as
3214/// multiples of the 64 KiB `DEFAULT_BLOCK`; small files keep over-fetch modest.
3215/// Per-query override comes via `RemoteGraph::with_block` (the Settings control).
3216fn auto_block(len: u64) -> u64 {
3217 const MB: u64 = 1 << 20;
3218 let mult: u64 = if len > 100 * MB {
3219 8 // 512 KiB — chebi, wikidata-100mb, ohm-full, wikidata-1GB, causenet
3220 } else if len > 10 * MB {
3221 4 // 256 KiB
3222 } else {
3223 2 // 128 KiB — small remotes (chemotion, getty-ulan)
3224 };
3225 mult * DEFAULT_BLOCK
3226}
3227
3228/// The physical range reader behind a remote open, plus the **base offset** of
3229/// the `.rete` inside the resource.
3230///
3231/// For an ordinary `.rete` the base is `0` and every method is a pass-through.
3232/// For a **polyglot** — one object that is simultaneously an HTML page and a
3233/// graph, byte 0 being `<` — the base is where the appended `.rete` starts, read
3234/// from the `RETE-BASE:` marker the page carries in its first
3235/// [`rete_core::HEADER_LEN`] bytes (written by
3236/// `experiments/polyglot/build_polyglot.py`). Every graph read then goes through
3237/// [`RemoteReader::view`], an [`OffsetReader`] that makes the embedded graph read
3238/// as if it began at byte 0 — so a polyglot is range-read exactly as lazily as a
3239/// plain file instead of being downloaded whole.
3240///
3241/// [`len`](Self::len) reports the **graph's** length (the shell excluded), while
3242/// `bytes_read`/`requests` stay the true wire cost, shell probe included.
3243#[derive(Clone)]
3244struct RemoteReader {
3245 counting: std::sync::Arc<CountingReader<XhrRangeReader>>,
3246 base: u64,
3247}
3248
3249impl RemoteReader {
3250 /// Open `url` and resolve its polyglot base — free on the sync build, where
3251 /// the length probe has already read the header window.
3252 fn open(url: &str) -> Result<Self, JsValue> {
3253 let xhr = XhrRangeReader::open(url)?;
3254 let base = xhr.polyglot_base();
3255 Ok(Self {
3256 counting: std::sync::Arc::new(CountingReader::new(xhr)),
3257 base,
3258 })
3259 }
3260
3261 /// The graph-relative reader: absolute for a plain `.rete`, shifted past the
3262 /// HTML shell for a polyglot.
3263 fn view(&self) -> OffsetReader<std::sync::Arc<CountingReader<XhrRangeReader>>> {
3264 OffsetReader::new(self.counting.clone(), self.base)
3265 }
3266
3267 /// Byte length of the embedded `.rete` (not of the enclosing resource).
3268 fn len(&self) -> u64 {
3269 self.counting.len().saturating_sub(self.base)
3270 }
3271
3272 fn bytes_read(&self) -> u64 {
3273 self.counting.bytes_read()
3274 }
3275
3276 fn requests(&self) -> u64 {
3277 self.counting.requests()
3278 }
3279}
3280
3281/// The default memory budget of a lazily opened file, in MiB.
3282const DEFAULT_MEMORY_BUDGET_MB: u32 = 1024;
3283
3284/// The memory budget of the files opened from now on, in MiB; 0 = no cap.
3285static MEMORY_BUDGET_MB: std::sync::atomic::AtomicU32 =
3286 std::sync::atomic::AtomicU32::new(DEFAULT_MEMORY_BUDGET_MB);
3287
3288/// Set the memory budget of every remote (lazily read) file opened from now
3289/// on, in MiB. It caps the caches of decoded dictionary chunks, decoded index
3290/// tiles and raw file blocks; what is evicted is read again when a query
3291/// touches it, so the budget trades speed for memory and never changes an
3292/// answer. `0` removes the cap. The default is 1024.
3293///
3294/// A wasm32 heap ends at 4 GiB and never shrinks. Without a cap a query over
3295/// ten million rows of a 2.5 GB file kept 2.3 GB of decoded chunks and tiles,
3296/// the next one added 1.7 GB of its own, and the engine trapped
3297/// (`unreachable`) at the ceiling. The cap does not cover a query's own
3298/// working state (a hash join's build side, `GROUP BY` groups).
3299#[wasm_bindgen]
3300pub fn set_memory_budget_mb(mb: u32) {
3301 MEMORY_BUDGET_MB.store(mb, std::sync::atomic::Ordering::Relaxed);
3302}
3303
3304/// The memory budget files are opened with, in MiB (0 = no cap): see
3305/// [`set_memory_budget_mb`].
3306#[wasm_bindgen]
3307pub fn memory_budget_mb() -> u32 {
3308 MEMORY_BUDGET_MB.load(std::sync::atomic::Ordering::Relaxed)
3309}
3310
3311/// The budget in bytes, `None` when there is no cap.
3312fn memory_budget() -> Option<u64> {
3313 match memory_budget_mb() {
3314 0 => None,
3315 mb => Some(u64::from(mb) << 20),
3316 }
3317}
3318
3319fn open_url(url: &str) -> Result<(RemoteReader, Rete), JsValue> {
3320 // `reader` counts the PHYSICAL fetches; a read-through block cache above it
3321 // turns the query's scattered range reads into a few aligned block fetches
3322 // (and reuses them) — working over any single-range backend (S3, a CDN),
3323 // not just a multi-range gateway. The block fetches still go through
3324 // `read_many`, so a multi-range host coalesces them further. The block size
3325 // is auto-tuned from the file size (known at open, no download).
3326 let reader = RemoteReader::open(url)?;
3327 let budget = memory_budget();
3328 let split = rete_core::split_memory_budget(budget.unwrap_or(u64::MAX));
3329 let cached = std::sync::Arc::new(
3330 BlockCacheReader::new(reader.counting.clone(), auto_block(reader.len()))
3331 .with_cache_cap(split.block_cache.min(rete_core::DEFAULT_CACHE_CAP)),
3332 );
3333 // The offset shim sits ABOVE the cache so blocks stay aligned to the
3334 // resource's absolute offsets (a plain file's base is 0 — a no-op).
3335 let mut rete = Rete::open_ranged_lazy(OffsetReader::new(cached, reader.base)).map_err(err)?;
3336 // The caches of decoded chunks and tiles get their share of the budget.
3337 // There is no disk to spill a `COUNT(DISTINCT)` to here: that stays in
3338 // memory whatever the budget.
3339 rete.set_memory_budget(budget);
3340 rete.set_spill_threshold(None);
3341 // SERVICE blocks federate to remote SPARQL endpoints over sync XHR.
3342 rete.set_service_client(Box::new(XhrServiceClient));
3343 Ok((reader, rete))
3344}
3345
3346/// Turn a partial lazy fetch into a hard error: a `*_url` task must never
3347/// return a result computed over silently-incomplete data.
3348fn incomplete_guard(rete: &Rete, what: &str) -> Result<(), JsValue> {
3349 if rete.index_incomplete() {
3350 return Err(js_error(format!(
3351 "a range fetch failed mid-{what}; refusing to return incomplete results"
3352 )));
3353 }
3354 Ok(())
3355}
3356
3357/// Triple-pattern provenance over a **remote** `.rete` URL (lazy range reads):
3358/// which permutation/section/byte-ranges answer the pattern. Worker-only.
3359#[wasm_bindgen]
3360pub fn why_url(
3361 url: &str,
3362 subject: Option<String>,
3363 predicate: Option<String>,
3364 object: Option<String>,
3365) -> Result<String, JsValue> {
3366 let (reader, rete) = open_url(url)?;
3367 let out = why_triples_rete(
3368 &rete,
3369 subject.as_deref(),
3370 predicate.as_deref(),
3371 object.as_deref(),
3372 )
3373 .map_err(js_error)?;
3374 incomplete_guard(&rete, "query")?;
3375 let _ = reader;
3376 Ok(out)
3377}
3378
3379/// Provenance over an already-open `Rete` (shared by [`why_triples`] in-memory
3380/// and [`why_url`] remote).
3381fn why_triples_rete(
3382 rete: &Rete,
3383 subject: Option<&str>,
3384 predicate: Option<&str>,
3385 object: Option<&str>,
3386) -> Result<String, String> {
3387 use serde_json::json;
3388 let results = rete.query_with_provenance(subject, predicate, object);
3389 let out = json!({
3390 "schemaVersion": JSON_SCHEMA_VERSION,
3391 "pattern": { "subject": subject, "predicate": predicate, "object": object },
3392 "resultCount": results.len(),
3393 "results": results.iter().map(provenance_json).collect::<Vec<_>>(),
3394 });
3395 serde_json::to_string(&out).map_err(|e| e.to_string())
3396}
3397
3398#[wasm_bindgen]
3399pub fn sparql_url(url: &str, query: &str, format: &str) -> Result<String, JsValue> {
3400 // Use the block-caching `open_url` seam so scattered range reads within this
3401 // one query that land in the same 64 KiB block fetch it once. (The resident
3402 // `RemoteGraph` extends this reuse across queries.)
3403 let (reader, rete) = open_url(url)?;
3404 let file_length = reader.len();
3405 // Evaluate first (this is what faults tiles), then refuse incomplete results.
3406 let out = eval_query(&rete, query).map_err(err)?;
3407 if rete.index_incomplete() {
3408 return Err(js_error(
3409 "a range fetch failed mid-query; refusing to return incomplete results",
3410 ));
3411 }
3412 // The `remote` member's values are plain numbers — no escaping needed.
3413 let extra = format!(
3414 r#","remote":{{"fileLength":{},"bytes":{},"requests":{}}}"#,
3415 file_length,
3416 reader.bytes_read(),
3417 reader.requests(),
3418 );
3419 write_query_json(&out, format, &extra).map_err(js_error)
3420}
3421
3422/// Evaluate a SELECT with the **community-split strategy**: every basic graph
3423/// pattern is decomposed into subject stars, each star is evaluated per
3424/// pyramid community (the members pushed in as a VALUES binding), and the
3425/// stars are recombined with global hash joins — so multi-hop joins work and
3426/// cross-community solutions survive. FILTER / UNION / OPTIONAL / MINUS
3427/// recurse; paths and GRAPH blocks evaluate globally inside the split; GROUP
3428/// BY / ORDER BY / LIMIT / DISTINCT run once on the merged rows. Answers are
3429/// identical to [`query`]'s. Refused only when nothing in the query can
3430/// split (no BGP with a variable subject) or for FROM / FROM NAMED. JSON:
3431/// the SELECT envelope plus `"communities": [{ "community", "subjects",
3432/// "rows" }, …]` (rows contributed per community across all split stars).
3433#[wasm_bindgen]
3434pub fn query_communities(
3435 bytes: &[u8],
3436 query: &str,
3437 round: Option<usize>,
3438) -> Result<String, JsValue> {
3439 query_communities_value(&open(bytes)?, query, round)
3440}
3441
3442/// [`query_communities`] against an already-open [`Rete`] (shared with [`Graph`]).
3443fn query_communities_value(
3444 rete: &Rete,
3445 query: &str,
3446 round: Option<usize>,
3447) -> Result<String, JsValue> {
3448 use serde_json::{json, Map, Value};
3449 let (mut vars, solutions, partials) =
3450 eval_select_communities(rete, query, round).map_err(err)?;
3451 if vars.is_empty() {
3452 let mut seen = std::collections::BTreeSet::new();
3453 for s in &solutions {
3454 for k in s.keys() {
3455 if seen.insert(k.clone()) {
3456 vars.push(k.clone());
3457 }
3458 }
3459 }
3460 }
3461 let rows: Vec<Value> = solutions
3462 .iter()
3463 .map(|s| {
3464 let mut obj = Map::new();
3465 for var in &vars {
3466 if let Some(term) = s.get(var) {
3467 obj.insert(var.clone(), Value::String(term.clone()));
3468 }
3469 }
3470 Value::Object(obj)
3471 })
3472 .collect();
3473 let communities: Vec<Value> = partials
3474 .iter()
3475 .map(|p| json!({ "community": p.community, "subjects": p.subjects, "rows": p.rows }))
3476 .collect();
3477 serde_json::to_string(&json!({
3478 "schemaVersion": JSON_SCHEMA_VERSION,
3479 "kind": "select",
3480 "vars": vars,
3481 "rows": rows,
3482 "communities": communities,
3483 }))
3484 .map_err(err)
3485}
3486
3487/// The file's byte layout, for the playground's byte-map view. JSON:
3488/// `{ "fileLength": N, "segments": [ { "kind", "label", "offset", "len" } ] }`
3489/// — segments sorted by offset; uncovered bytes are container framing.
3490#[wasm_bindgen]
3491pub fn file_layout(bytes: &[u8]) -> Result<String, JsValue> {
3492 use serde_json::json;
3493 let rete = open(bytes)?;
3494 let segments: Vec<serde_json::Value> = rete
3495 .file_layout()
3496 .iter()
3497 .map(|s| json!({ "kind": s.kind, "label": s.label, "offset": s.offset, "len": s.len }))
3498 .collect();
3499 serde_json::to_string(&json!({
3500 "schemaVersion": JSON_SCHEMA_VERSION,
3501 "fileLength": bytes.len(),
3502 "segments": segments,
3503 }))
3504 .map_err(err)
3505}
3506
3507/// The full community pyramid as a tree — the "cluster of clusters" view.
3508/// Per dendrogram round (index 0 = finest, last = coarsest), every community
3509/// with its member-node count, triple count (triples whose subject belongs to
3510/// it), and its parent community at the next-coarser round (`null` at the
3511/// top). JSON:
3512/// `{ "rounds": N, "levels": [ [ { "id", "nodes", "triples", "parent" } ] ] }`.
3513#[wasm_bindgen]
3514pub fn pyramid_tree(bytes: &[u8]) -> Result<String, JsValue> {
3515 pyramid_tree_value(&open(bytes)?)
3516}
3517
3518/// [`pyramid_tree`] against an already-open [`Rete`] (shared with [`Graph`]).
3519fn pyramid_tree_value(rete: &Rete) -> Result<String, JsValue> {
3520 use serde_json::json;
3521 use std::collections::BTreeMap;
3522 let dict = rete.dictionary();
3523 let ids = rete.match_ids((None, None, None));
3524 let g = project_graph(dict, &ids);
3525 let dend = build_dendrogram(&g);
3526 let rounds = dend.rounds();
3527 if rounds == 0 {
3528 return serde_json::to_string(&json!({
3529 "schemaVersion": JSON_SCHEMA_VERSION,
3530 "rounds": 0,
3531 "levels": [],
3532 }))
3533 .map_err(err);
3534 }
3535 let n = g.node_count();
3536 let mut levels = Vec::with_capacity(rounds);
3537 for r in 0..rounds {
3538 let mut nodes_per: BTreeMap<usize, usize> = BTreeMap::new();
3539 let mut rep: BTreeMap<usize, usize> = BTreeMap::new();
3540 for node in 0..n {
3541 let c = dend.base_community(node, r);
3542 *nodes_per.entry(c).or_default() += 1;
3543 rep.entry(c).or_insert(node);
3544 }
3545 let mut triples_per: BTreeMap<usize, usize> = BTreeMap::new();
3546 for &(s, _, _) in &ids {
3547 let c = dend.base_community(dict.subject_node(s) as usize, r);
3548 *triples_per.entry(c).or_default() += 1;
3549 }
3550 let level: Vec<serde_json::Value> = nodes_per
3551 .iter()
3552 .map(|(&c, &nodes)| {
3553 let parent = if r + 1 < rounds {
3554 json!(dend.base_community(rep[&c], r + 1))
3555 } else {
3556 json!(null)
3557 };
3558 json!({
3559 "id": c,
3560 "nodes": nodes,
3561 "triples": triples_per.get(&c).copied().unwrap_or(0),
3562 "parent": parent,
3563 })
3564 })
3565 .collect();
3566 levels.push(level);
3567 }
3568 serde_json::to_string(&json!({
3569 "schemaVersion": JSON_SCHEMA_VERSION,
3570 "rounds": rounds,
3571 "levels": levels,
3572 }))
3573 .map_err(err)
3574}
3575
3576/// Recompute the Louvain community decomposition and report, per community, its
3577/// member-subject count and triple count. Powers the "split by community"
3578/// strategy view in the playground. JSON:
3579/// `[{ "community": N, "size": M, "triples": K }, ...]`, ordered by community id.
3580///
3581/// `round` selects the dendrogram granularity; `None` chooses the round for the
3582/// default tile budget (the same choice the native build makes).
3583#[wasm_bindgen]
3584pub fn communities(bytes: &[u8], round: Option<usize>) -> Result<String, JsValue> {
3585 use std::collections::HashSet;
3586 let rete = open(bytes)?;
3587 let dict = rete.dictionary();
3588 let ids = rete.match_ids((None, None, None));
3589 let g = project_graph(dict, &ids);
3590 let dend = build_dendrogram(&g);
3591 let round =
3592 round.unwrap_or_else(|| choose_round_for_budget(dict, &ids, &dend, DEFAULT_TILE_BUDGET));
3593 let tiles = tile_by_community(dict, &ids, &dend, round);
3594 let arr: Vec<serde_json::Value> = tiles
3595 .iter()
3596 .map(|t| {
3597 let members: HashSet<u32> = t.triples.iter().map(|&(s, _, _)| s).collect();
3598 serde_json::json!({
3599 "community": t.community,
3600 "size": members.len(),
3601 "triples": t.triples.len(),
3602 })
3603 })
3604 .collect();
3605 serde_json::to_string(&arr).map_err(err)
3606}
3607
3608/// Multi-source transitive reachability over one relation, run **serially**
3609/// (the browser engine is single-threaded — the native CLI's
3610/// `rete reach --parallel` fans one task per seed). For each seed, the set of
3611/// nodes it transitively reaches; with `reverse`, the set that reaches it
3612/// (impact analysis).
3613///
3614/// - `predicate` — the relation IRI token, e.g. `<http://ex/dependsOn>`.
3615/// - `seeds` — a JSON array of seed IRI tokens (e.g. `["<http://ex/app>"]`); a
3616/// bare single IRI string is also accepted.
3617/// - `reverse` — traverse edges backward ("who reaches the seed?").
3618///
3619/// Returns a JSON array, one entry per seed in input order:
3620/// `[{ "seed":"<iri>", "reached":["<iri>",...], "count":N }, ...]`.
3621/// A seed not present in the graph yields `{ "seed":"...", "error":"not in graph" }`
3622/// instead of failing the whole call.
3623#[wasm_bindgen]
3624pub fn reach(bytes: &[u8], predicate: &str, seeds: &str, reverse: bool) -> Result<String, JsValue> {
3625 reach_rete(&open(bytes)?, predicate, seeds, reverse)
3626}
3627
3628/// Multi-source reachability over a **remote** `.rete` URL (lazy HTTP range
3629/// reads): builds adjacency for `predicate` by faulting only that predicate's
3630/// tiles, then BFS from each seed. Worker-only (synchronous XHR).
3631#[wasm_bindgen]
3632pub fn reach_url(
3633 url: &str,
3634 predicate: &str,
3635 seeds: &str,
3636 reverse: bool,
3637) -> Result<String, JsValue> {
3638 let (reader, rete) = open_url(url)?;
3639 let out = reach_rete(&rete, predicate, seeds, reverse)?;
3640 incomplete_guard(&rete, "reach")?;
3641 let _ = reader;
3642 Ok(out)
3643}
3644
3645fn reach_rete(rete: &Rete, predicate: &str, seeds: &str, reverse: bool) -> Result<String, JsValue> {
3646 use std::collections::HashMap;
3647 let dict = rete.dictionary();
3648
3649 // Accept either a JSON array of IRIs or a single bare IRI string.
3650 let seed_iris: Vec<String> = match serde_json::from_str::<Vec<String>>(seeds) {
3651 Ok(v) => v,
3652 Err(_) => match serde_json::from_str::<String>(seeds) {
3653 Ok(s) => vec![s],
3654 Err(_) => vec![seeds.to_string()],
3655 },
3656 };
3657
3658 // Adjacency in unified node space for the chosen direction.
3659 let adj: HashMap<u32, Vec<u32>> = if reverse {
3660 let mut m: HashMap<u32, Vec<u32>> = HashMap::new();
3661 for (s, o) in rete.predicate_pairs(predicate) {
3662 m.entry(o).or_default().push(s); // who points at o
3663 }
3664 m
3665 } else {
3666 build_adjacency(rete, predicate)
3667 };
3668
3669 // Resolve known seeds to node ids; remember unknowns to report inline.
3670 let mut known_idx: Vec<usize> = Vec::new();
3671 let mut seed_nodes: Vec<u32> = Vec::new();
3672 for (i, iri) in seed_iris.iter().enumerate() {
3673 if let Some(n) = dict.node_of_term(iri) {
3674 known_idx.push(i);
3675 seed_nodes.push(n);
3676 }
3677 }
3678
3679 let sets = batch_reach_serial(&adj, &seed_nodes);
3680
3681 // Map known-seed position -> its reach set, then assemble in input order.
3682 let mut by_input: HashMap<usize, &std::collections::BTreeSet<u32>> = HashMap::new();
3683 for (k, &i) in known_idx.iter().enumerate() {
3684 by_input.insert(i, &sets[k]);
3685 }
3686
3687 let arr: Vec<serde_json::Value> = seed_iris
3688 .iter()
3689 .enumerate()
3690 .map(|(i, iri)| match by_input.get(&i) {
3691 Some(set) => {
3692 let reached: Vec<String> = set.iter().filter_map(|&n| dict.node_term(n)).collect();
3693 serde_json::json!({
3694 "seed": iri,
3695 "count": reached.len(),
3696 "reached": reached,
3697 })
3698 }
3699 None => serde_json::json!({ "seed": iri, "error": "not in graph" }),
3700 })
3701 .collect();
3702
3703 serde_json::to_string(&arr).map_err(err)
3704}
3705
3706/// Validate a `.rete` graph against SHACL Core shapes written in Turtle.
3707///
3708/// The default graph is validated unless `graph` names a dataset graph IRI.
3709/// `format` is one of:
3710/// - `"json"`: structured validation report from rete-core
3711/// - `"ttl"`: Turtle validation report
3712/// - anything else: compact text report
3713///
3714/// A non-conformant graph returns a report; it is not a JS exception. Exceptions
3715/// are reserved for parse/open errors.
3716#[wasm_bindgen]
3717pub fn shacl(
3718 bytes: &[u8],
3719 shapes_turtle: &str,
3720 graph: Option<String>,
3721 format: &str,
3722) -> Result<String, JsValue> {
3723 shacl_rete(&open(bytes)?, shapes_turtle, graph.as_deref(), format)
3724}
3725
3726/// Validate a **remote** `.rete` graph (lazy HTTP range reads) against SHACL
3727/// Core shapes written in Turtle. Validating the **default** graph routes every
3728/// lookup as a range read ([`ReteGraph`]), so a targeted shape faults only the
3729/// tiles holding its targets — not the whole graph. A named graph (`graph`) still
3730/// materializes (the routed view is default-graph only). Worker-only (sync XHR).
3731#[wasm_bindgen]
3732pub fn shacl_url(
3733 url: &str,
3734 shapes_turtle: &str,
3735 graph: Option<String>,
3736 format: &str,
3737) -> Result<String, JsValue> {
3738 let (reader, rete) = open_url(url)?;
3739 let out = shacl_rete(&rete, shapes_turtle, graph.as_deref(), format)?;
3740 incomplete_guard(&rete, "validation")?;
3741 let _ = reader;
3742 Ok(out)
3743}
3744
3745fn shacl_rete(
3746 rete: &Rete,
3747 shapes_turtle: &str,
3748 graph: Option<&str>,
3749 format: &str,
3750) -> Result<String, JsValue> {
3751 let shapes = ShaclShapes::parse_turtle(shapes_turtle).map_err(err)?;
3752 // Default graph: validate over the index directly (lazy — fetch only targets).
3753 // A named graph still materializes (ReteGraph views the default graph only).
3754 let report = match graph {
3755 None => validate_shacl(&ReteGraph::new(rete), &shapes),
3756 Some(g) => validate_shacl(&DataGraph::from_rete(rete, Some(g)), &shapes),
3757 };
3758 Ok(format_report(&report, format))
3759}
3760
3761fn shacl_over(data: DataGraph, shapes_turtle: &str, format: &str) -> Result<String, JsValue> {
3762 let shapes = ShaclShapes::parse_turtle(shapes_turtle).map_err(err)?;
3763 Ok(format_report(&validate_shacl(&data, &shapes), format))
3764}
3765
3766fn format_report(report: &ValidationReport, format: &str) -> String {
3767 match format {
3768 "json" => report.to_json(),
3769 "ttl" => report.to_turtle(),
3770 _ => format_shacl_text(report),
3771 }
3772}
3773
3774/// **Chain a SPARQL subset, then SHACL over it.** Evaluates a `CONSTRUCT` over
3775/// the remote `.rete` (touching only the tiles its patterns need), then validates
3776/// the resulting subgraph against the Turtle `shapes`. Where [`shacl_url`] selects
3777/// by *shape target* (validate every Person…), this selects by an explicit
3778/// `CONSTRUCT` — "validate just the slice this query carves out, in place".
3779/// Worker-only (synchronous XHR).
3780#[wasm_bindgen]
3781pub fn shacl_construct_url(
3782 url: &str,
3783 construct: &str,
3784 shapes_turtle: &str,
3785 format: &str,
3786) -> Result<String, JsValue> {
3787 let (reader, rete) = open_url(url)?;
3788 let triples = match eval_query(&rete, construct).map_err(err)? {
3789 QueryOutput::Construct(t) => t,
3790 _ => {
3791 return Err(js_error(
3792 "the subset query must be a CONSTRUCT — it builds the subgraph SHACL validates",
3793 ))
3794 }
3795 };
3796 incomplete_guard(&rete, "query")?;
3797 let out = shacl_over(DataGraph::from_triples(triples), shapes_turtle, format)?;
3798 let _ = reader;
3799 Ok(out)
3800}
3801
3802// --- OWL RL / RDFS coherence checking ----------------------------------------
3803//
3804// The reasoner (`rete_core::reason`) is already slice-shaped, so these are near
3805// twins of the SHACL functions above: `reason`/`reason_url` materialize the whole
3806// graph (Tier-2, the complete check), while `reason_construct_url` validates only
3807// the slice a CONSTRUCT selects (Tier-1, selective range reads).
3808
3809/// The fixed, **selective** coherence subgraph for [`reason_construct_url`]
3810/// (Tier-1). A UNION of constant-predicate branches: each routes to exactly one
3811/// predicate's index tiles, so the client faults only `rdf:type` + the class/
3812/// equality T-Box predicates over one warm lazy cache — not the whole graph.
3813///
3814/// The T-Box branches are MANDATORY: the reasoner finds disjoint-class clashes
3815/// only *after* `subClassOf` type-propagation, so omitting `subClassOf`/
3816/// `disjointWith`/`sameAs` would silently miss propagation-dependent
3817/// contradictions. Property-characteristic axioms (Functional/Symmetric/
3818/// Transitive) ride inside the `rdf:type` slice. Instance-level
3819/// FunctionalProperty / domain / range clashes need the property's own
3820/// assertions — use [`reason_url`] (Tier-2) for those.
3821pub const COHERENCE_CONSTRUCT: &str = r#"
3822PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
3823PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
3824PREFIX owl: <http://www.w3.org/2002/07/owl#>
3825CONSTRUCT {
3826 ?x rdf:type ?c .
3827 ?sub rdfs:subClassOf ?sup .
3828 ?c1 owl:disjointWith ?c2 .
3829 ?s1 owl:sameAs ?s2 .
3830 ?f1 owl:differentFrom ?f2 .
3831}
3832WHERE {
3833 { ?x rdf:type ?c }
3834 UNION { ?sub rdfs:subClassOf ?sup }
3835 UNION { ?c1 owl:disjointWith ?c2 }
3836 UNION { ?s1 owl:sameAs ?s2 }
3837 UNION { ?f1 owl:differentFrom ?f2 }
3838}
3839"#;
3840
3841/// Build the playground JSON envelope for a [`rete_core::Reasoning`] result,
3842/// shared by the in-memory and remote entrypoints. `remote` is
3843/// `(fileLength, bytes, requests)` for the `*_url` variants, `None` in-memory.
3844/// JSON: `{ "kind":"reasoning", "coherent":bool, "inferredCount":N,
3845/// "inconsistencies":[{"kind","detail"}], ["remote":{...}] }`.
3846fn reasoning_json(r: &rete_core::Reasoning, remote: Option<(u64, u64, u64)>) -> String {
3847 use serde_json::json;
3848 let inconsistencies: Vec<serde_json::Value> = r
3849 .inconsistencies
3850 .iter()
3851 .map(|i| json!({ "kind": i.kind, "detail": i.detail }))
3852 .collect();
3853 let mut v = json!({
3854 "schemaVersion": JSON_SCHEMA_VERSION,
3855 "kind": "reasoning",
3856 "coherent": r.inconsistencies.is_empty(),
3857 "inferredCount": r.inferred.len(),
3858 "inconsistencies": inconsistencies,
3859 });
3860 if let (Some(obj), Some((file_length, bytes, requests))) = (v.as_object_mut(), remote) {
3861 obj.insert(
3862 "remote".to_string(),
3863 json!({ "fileLength": file_length, "bytes": bytes, "requests": requests }),
3864 );
3865 }
3866 v.to_string()
3867}
3868
3869/// Run the OWL RL / RDFS reasoner over an **in-memory** `.rete` and report the
3870/// inferred-triple count plus any incoherent points (logical contradictions).
3871/// `graph` selects a named graph; the default graph is used when omitted. This is
3872/// the complete (Tier-2) check — it materializes the whole graph. Returns the
3873/// `reasoning_json` envelope (no `remote` block).
3874#[wasm_bindgen]
3875pub fn reason(bytes: &[u8], graph: Option<String>) -> Result<String, JsValue> {
3876 let rete = open(bytes)?;
3877 let base = rete.dump(graph.as_deref());
3878 let result = rete_core::reason(&base);
3879 Ok(reasoning_json(&result, None))
3880}
3881
3882/// Run the reasoner over a **remote** `.rete` URL — the complete (Tier-2) check.
3883/// It materializes the whole graph, so this faults in the dataset's chunks/tiles
3884/// as it reads (≈ the whole file); use [`reason_construct_url`] for the cheaper
3885/// selective check. Worker-only (synchronous XHR). A failed range fetch mid-read
3886/// is an error, never a silently-incomplete (and thus possibly false "coherent")
3887/// result. JSON adds `"remote": { fileLength, bytes, requests }`.
3888#[wasm_bindgen]
3889pub fn reason_url(url: &str, graph: Option<String>) -> Result<String, JsValue> {
3890 let (reader, rete) = open_url(url)?;
3891 let base = rete.dump(graph.as_deref());
3892 let result = rete_core::reason(&base);
3893 incomplete_guard(&rete, "reasoning")?;
3894 let out = reasoning_json(
3895 &result,
3896 Some((reader.len(), reader.bytes_read(), reader.requests())),
3897 );
3898 let _ = reader;
3899 Ok(out)
3900}
3901
3902/// **Selective (Tier-1) coherence check** over a remote `.rete`: evaluate a
3903/// CONSTRUCT (touching only the tiles its constant-predicate patterns need), then
3904/// reason over just that subgraph. Pass [`COHERENCE_CONSTRUCT`] for the standard
3905/// class/equality coherence slice, or a custom CONSTRUCT to scope the check
3906/// further. Unlike [`reason_url`] (which materializes the whole graph), this
3907/// fetches only the slice the CONSTRUCT selects. Worker-only (synchronous XHR).
3908#[wasm_bindgen]
3909pub fn reason_construct_url(url: &str, construct: &str) -> Result<String, JsValue> {
3910 let (reader, rete) = open_url(url)?;
3911 let triples =
3912 match eval_query(&rete, construct).map_err(err)? {
3913 QueryOutput::Construct(t) => t,
3914 _ => return Err(js_error(
3915 "the subset query must be a CONSTRUCT — it builds the subgraph the reasoner checks",
3916 )),
3917 };
3918 incomplete_guard(&rete, "query")?;
3919 let result = rete_core::reason(&triples);
3920 let out = reasoning_json(
3921 &result,
3922 Some((reader.len(), reader.bytes_read(), reader.requests())),
3923 );
3924 let _ = reader;
3925 Ok(out)
3926}
3927
3928/// Build the Tier-0 schema-coherence JSON envelope from the incoherent points.
3929/// `remote` is `(fileLength, bytes, requests)` for the `*_url` variant.
3930fn schema_coherence_json(
3931 points: &[rete_core::Inconsistency],
3932 remote: Option<(u64, u64, u64)>,
3933) -> String {
3934 use serde_json::json;
3935 let schema_points: Vec<serde_json::Value> = points
3936 .iter()
3937 .map(|i| json!({ "kind": i.kind, "detail": i.detail }))
3938 .collect();
3939 let mut v = json!({
3940 "schemaVersion": JSON_SCHEMA_VERSION,
3941 "kind": "schemaCoherence",
3942 "coherent": points.is_empty(),
3943 "schemaPoints": schema_points,
3944 // The defining property: this answer never touched the index OR dictionary.
3945 "readsIndex": false,
3946 });
3947 if let (Some(obj), Some((file_length, bytes, requests))) = (v.as_object_mut(), remote) {
3948 obj.insert(
3949 "remote".to_string(),
3950 json!({ "fileLength": file_length, "bytes": bytes, "requests": requests }),
3951 );
3952 }
3953 v.to_string()
3954}
3955
3956/// **Index-free schema coherence (Tier-0)** over an in-memory `.rete`: read only
3957/// the header + pyramid-meta (never the dictionary or the triple index) and report
3958/// schema-level incoherent points (subClassOf cycles, unsatisfiable classes).
3959/// Errors if the file ships no schema pyramid.
3960#[wasm_bindgen]
3961pub fn check_schema(bytes: &[u8]) -> Result<String, JsValue> {
3962 let points = rete_core::read_schema_coherence_ranged(&SliceReader::new(bytes))
3963 .map_err(err)?
3964 .ok_or_else(|| js_error("file has no schema pyramid"))?;
3965 Ok(schema_coherence_json(&points, None))
3966}
3967
3968/// **Index-free schema coherence (Tier-0) over a remote `.rete` URL.** Reads only
3969/// TWO ranges — the header and the trailing schema block (the header records its
3970/// length) — never the dictionary, the community summary, or the triple index. So
3971/// it's a flat **~1–8 KB at any graph size** (8.1 KB of a 48.8 MB file; see
3972/// docs/BENCHMARK.md), making it the cheap "is the ontology coherent?" gate.
3973/// Worker-only (synchronous XHR); a failed range fetch is an error, never a false
3974/// "coherent".
3975#[wasm_bindgen]
3976pub fn check_schema_url(url: &str) -> Result<String, JsValue> {
3977 let reader = RemoteReader::open(url)?;
3978 let points = rete_core::read_schema_coherence_ranged(&reader.view())
3979 .map_err(err)?
3980 .ok_or_else(|| js_error("file has no schema pyramid"))?;
3981 Ok(schema_coherence_json(
3982 &points,
3983 Some((reader.len(), reader.bytes_read(), reader.requests())),
3984 ))
3985}
3986
3987/// The schema summary (classes + relations) read from the schema pyramid over
3988/// HTTP range — a Schema view of a remote graph without downloading it.
3989/// Worker-only (synchronous XHR). JSON:
3990/// `{ "kind":"schema", "classes":[[iri,count]], "relations":[[s,p,o,count]], "remote":{…} }`.
3991#[wasm_bindgen]
3992pub fn schema_url(url: &str) -> Result<String, JsValue> {
3993 use serde_json::json;
3994 let reader = RemoteReader::open(url)?;
3995 let (classes, relations) = rete_core::read_schema_summary_ranged(&reader.view())
3996 .map_err(err)?
3997 .ok_or_else(|| js_error("file has no schema pyramid"))?;
3998 let classes: Vec<serde_json::Value> = classes.iter().map(|(c, n)| json!([c, n])).collect();
3999 let relations: Vec<serde_json::Value> = relations
4000 .iter()
4001 .map(|(s, p, o, n)| json!([s, p, o, n]))
4002 .collect();
4003 Ok(json!({
4004 "schemaVersion": JSON_SCHEMA_VERSION,
4005 "kind": "schema",
4006 "classes": classes,
4007 "relations": relations,
4008 "remote": { "fileLength": reader.len(), "bytes": reader.bytes_read(), "requests": reader.requests() },
4009 })
4010 .to_string())
4011}
4012
4013fn format_shacl_text(report: &ValidationReport) -> String {
4014 if report.conforms {
4015 return "conforms: true\n".to_string();
4016 }
4017 let mut out = format!("conforms: false\nresults: {}\n", report.results.len());
4018 for result in &report.results {
4019 out.push_str("\n- focus: ");
4020 out.push_str(&result.focus_node);
4021 if let Some(value) = &result.value_node {
4022 out.push_str("\n value: ");
4023 out.push_str(value);
4024 }
4025 if let Some(path) = &result.result_path {
4026 out.push_str("\n path: ");
4027 out.push_str(path);
4028 }
4029 out.push_str("\n component: ");
4030 out.push_str(&result.source_constraint_component);
4031 out.push_str("\n severity: ");
4032 out.push_str(&result.severity.iri());
4033 for message in &result.messages {
4034 out.push_str("\n message: ");
4035 out.push_str(message);
4036 }
4037 out.push('\n');
4038 }
4039 out
4040}
4041
4042// --- EXPERIMENTAL: real browser-thread parallelism (feature = "threads") ------
4043//
4044// Only compiled when the `threads` feature is on (a nightly + build-std wasm
4045// build, served cross-origin-isolated). The default build never sees any of
4046// this, so it stays single-threaded and `file://`-able.
4047
4048/// Re-export wasm-bindgen-rayon's thread-pool initializer. JS MUST
4049/// `await initThreadPool(n)` once (after the module `init()`) before calling
4050/// [`reach_parallel`], or any rayon use will panic. Backed by Web Workers +
4051/// `SharedArrayBuffer`, so the page must be cross-origin isolated (COOP/COEP).
4052#[cfg(feature = "threads")]
4053pub use wasm_bindgen_rayon::init_thread_pool;
4054
4055/// Multi-source transitive reachability run on **real browser threads** via
4056/// `wasm-bindgen-rayon` — the threaded twin of [`reach`]. Same arguments, same
4057/// JSON result shape; the only difference is it calls
4058/// [`rete_core::parallel::batch_reach_parallel`] (one rayon task per seed)
4059/// instead of the serial loop. Use it to benchmark the 14× batch-reachability
4060/// workload against [`reach`] in the browser.
4061///
4062/// Requires `await initThreadPool(navigator.hardwareConcurrency)` first.
4063#[cfg(feature = "threads")]
4064#[wasm_bindgen]
4065pub fn reach_parallel(
4066 bytes: &[u8],
4067 predicate: &str,
4068 seeds: &str,
4069 reverse: bool,
4070) -> Result<String, JsValue> {
4071 use std::collections::HashMap;
4072 let rete = open(bytes)?;
4073 let dict = rete.dictionary();
4074
4075 // Accept either a JSON array of IRIs or a single bare IRI string (mirrors `reach`).
4076 let seed_iris: Vec<String> = match serde_json::from_str::<Vec<String>>(seeds) {
4077 Ok(v) => v,
4078 Err(_) => match serde_json::from_str::<String>(seeds) {
4079 Ok(s) => vec![s],
4080 Err(_) => vec![seeds.to_string()],
4081 },
4082 };
4083
4084 // Adjacency in unified node space for the chosen direction.
4085 let adj: HashMap<u32, Vec<u32>> = if reverse {
4086 let mut m: HashMap<u32, Vec<u32>> = HashMap::new();
4087 for (s, o) in rete.predicate_pairs(predicate) {
4088 m.entry(o).or_default().push(s); // who points at o
4089 }
4090 m
4091 } else {
4092 build_adjacency(&rete, predicate)
4093 };
4094
4095 // Resolve known seeds to node ids; remember unknowns to report inline.
4096 let mut known_idx: Vec<usize> = Vec::new();
4097 let mut seed_nodes: Vec<u32> = Vec::new();
4098 for (i, iri) in seed_iris.iter().enumerate() {
4099 if let Some(n) = dict.node_of_term(iri) {
4100 known_idx.push(i);
4101 seed_nodes.push(n);
4102 }
4103 }
4104
4105 // The one line that differs from serial `reach`: real-thread fan-out.
4106 let sets = rete_core::parallel::batch_reach_parallel(&adj, &seed_nodes);
4107
4108 // Map known-seed position -> its reach set, then assemble in input order.
4109 let mut by_input: HashMap<usize, &std::collections::BTreeSet<u32>> = HashMap::new();
4110 for (k, &i) in known_idx.iter().enumerate() {
4111 by_input.insert(i, &sets[k]);
4112 }
4113
4114 let arr: Vec<serde_json::Value> = seed_iris
4115 .iter()
4116 .enumerate()
4117 .map(|(i, iri)| match by_input.get(&i) {
4118 Some(set) => {
4119 let reached: Vec<String> = set.iter().filter_map(|&n| dict.node_term(n)).collect();
4120 serde_json::json!({
4121 "seed": iri,
4122 "count": reached.len(),
4123 "reached": reached,
4124 })
4125 }
4126 None => serde_json::json!({ "seed": iri, "error": "not in graph" }),
4127 })
4128 .collect();
4129
4130 serde_json::to_string(&arr).map_err(err)
4131}
4132
4133// --- CONSTRUCT serializers (mirrored from rete-cli, kept minimal) ------------
4134
4135const RDF_TYPE_IRI: &str = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
4136const XSD_INTEGER_IRI: &str = "http://www.w3.org/2001/XMLSchema#integer";
4137
4138fn summary_total(view: &SummaryView) -> u64 {
4139 view.summary.iter().map(|edge| u64::from(edge.count)).sum()
4140}
4141
4142fn predicate_list(view: &SummaryView) -> Vec<String> {
4143 view.predicate_totals()
4144 .into_iter()
4145 .map(|(predicate, _)| predicate)
4146 .collect()
4147}
4148
4149fn predicate_count(view: &SummaryView) -> u64 {
4150 view.predicate_totals().len() as u64
4151}
4152
4153fn select_count_response(
4154 variable: &str,
4155 count: u64,
4156 progressive: serde_json::Value,
4157) -> serde_json::Value {
4158 use serde_json::{json, Map, Value};
4159
4160 let mut row = Map::new();
4161 row.insert(variable.to_string(), Value::String(integer_literal(count)));
4162 json!({
4163 "schemaVersion": JSON_SCHEMA_VERSION,
4164 "kind": "select",
4165 "vars": [variable],
4166 "rows": [Value::Object(row)],
4167 "progressive": progressive,
4168 })
4169}
4170
4171fn select_predicate_list_response(
4172 variable: &str,
4173 predicates: &[String],
4174 progressive: serde_json::Value,
4175) -> serde_json::Value {
4176 use serde_json::{json, Map, Value};
4177
4178 let rows: Vec<Value> = predicates
4179 .iter()
4180 .map(|predicate| {
4181 let mut row = Map::new();
4182 row.insert(variable.to_string(), Value::String(predicate.clone()));
4183 Value::Object(row)
4184 })
4185 .collect();
4186
4187 json!({
4188 "schemaVersion": JSON_SCHEMA_VERSION,
4189 "kind": "select",
4190 "vars": [variable],
4191 "rows": rows,
4192 "progressive": progressive,
4193 })
4194}
4195
4196fn select_predicate_totals_response(
4197 predicate_variable: &str,
4198 count_variable: &str,
4199 totals: &[(String, u32)],
4200 progressive: serde_json::Value,
4201) -> serde_json::Value {
4202 use serde_json::{json, Map, Value};
4203
4204 let rows: Vec<Value> = totals
4205 .iter()
4206 .map(|(predicate, count)| {
4207 let mut row = Map::new();
4208 row.insert(
4209 predicate_variable.to_string(),
4210 Value::String(predicate.clone()),
4211 );
4212 row.insert(
4213 count_variable.to_string(),
4214 Value::String(integer_literal(u64::from(*count))),
4215 );
4216 Value::Object(row)
4217 })
4218 .collect();
4219
4220 json!({
4221 "schemaVersion": JSON_SCHEMA_VERSION,
4222 "kind": "select",
4223 "vars": [predicate_variable, count_variable],
4224 "rows": rows,
4225 "progressive": progressive,
4226 })
4227}
4228
4229fn progressive_meta<R: RangeReader>(
4230 reader: &CountingReader<R>,
4231 file_bytes: u64,
4232 query_shape: &str,
4233 predicate: Option<&str>,
4234 value: serde_json::Value,
4235) -> serde_json::Value {
4236 use serde_json::{json, Map, Value};
4237
4238 let mut meta = Map::new();
4239 meta.insert("stage".into(), json!("summary"));
4240 meta.insert("exact".into(), json!(true));
4241 meta.insert("readsIndex".into(), json!(false));
4242 meta.insert("queryShape".into(), json!(query_shape));
4243 meta.insert("value".into(), value);
4244 meta.insert("bytes".into(), json!(reader.bytes_read()));
4245 meta.insert("requests".into(), json!(reader.requests()));
4246 meta.insert("fileBytes".into(), json!(file_bytes));
4247 if let Some(predicate) = predicate {
4248 meta.insert("predicate".into(), json!(predicate));
4249 }
4250 Value::Object(meta)
4251}
4252
4253fn integer_literal(value: u64) -> String {
4254 format!("\"{}\"^^<{}>", value, XSD_INTEGER_IRI)
4255}
4256
4257fn provenance_json(m: &TripleProvenance) -> serde_json::Value {
4258 use serde_json::json;
4259
4260 let tile = match (&m.tile, m.tile_range) {
4261 (Some(id), Some(range)) => json!({
4262 "available": true,
4263 "id": id,
4264 "range": range_json(range),
4265 }),
4266 (Some(id), None) => json!({
4267 "available": true,
4268 "id": id,
4269 }),
4270 _ => json!({
4271 "available": false,
4272 "reason": "not_materialized",
4273 }),
4274 };
4275
4276 json!({
4277 "terms": {
4278 "subject": m.terms.0,
4279 "predicate": m.terms.1,
4280 "object": m.terms.2,
4281 },
4282 "ids": {
4283 "subject": m.ids.0,
4284 "predicate": m.ids.1,
4285 "object": m.ids.2,
4286 },
4287 "provenance": {
4288 "graph": m.graph.as_deref().unwrap_or("default"),
4289 "matchedPattern": {
4290 "subject": m.matched_pattern.0,
4291 "predicate": m.matched_pattern.1,
4292 "object": m.matched_pattern.2,
4293 },
4294 "indexPermutation": m.index_permutation.name(),
4295 "indexSection": m.index_permutation.section_index(),
4296 "dictionaryRange": range_json(m.dictionary_range),
4297 "indexRange": range_json(m.index_range),
4298 "indexSectionRange": range_json(m.index_section_range),
4299 "pyramidRange": m.pyramid_range.map(range_json),
4300 "tile": tile,
4301 },
4302 })
4303}
4304
4305fn range_json(range: ByteRange) -> serde_json::Value {
4306 serde_json::json!({
4307 "offset": range.offset,
4308 "len": range.len,
4309 "end": range.end(),
4310 })
4311}
4312
4313/// Serialize a triple list (canonical N-Triples tokens) to Turtle: group by
4314/// subject, sort predicates/objects, abbreviate `rdf:type` to `a`. The tokens are
4315/// already valid Turtle term syntax, so they pass through verbatim — except a
4316/// quoted triple, which `syntax` spells like `rete export` does:
4317///
4318/// - `rdf12` (the default): the RDF 1.2 triple term `<<( s p o )>>`, which
4319/// current Turtle 1.2 parsers read. Written as the stored `<< s p o >>`, the
4320/// same parsers read a *reifier* instead, one statement becoming two — the
4321/// silent misreading #262 fixed for the CLI. RDF 1.2 puts a triple term in
4322/// object position only, so a quoted triple as a subject is refused by name
4323/// ([`rdf12_object`]), not written.
4324/// - `rdf-star`: the stored token `<<s p o>>`, verbatim.
4325///
4326/// A graph with no quoted triple is written byte-for-byte the same either way.
4327fn to_turtle(
4328 triples: &[(String, String, String)],
4329 syntax: rete_core::ingest::QuotedTripleSurface,
4330) -> Result<String, String> {
4331 use std::borrow::Cow;
4332 use std::collections::BTreeMap;
4333 let rdf12 = syntax == rete_core::ingest::QuotedTripleSurface::Rdf12;
4334 let mut by_subject: BTreeMap<&str, BTreeMap<&str, Vec<Cow<'_, str>>>> = BTreeMap::new();
4335 for (s, p, o) in triples {
4336 let o = if rdf12 {
4337 rdf12_object(s, p, o)?
4338 } else {
4339 Cow::Borrowed(o.as_str())
4340 };
4341 by_subject
4342 .entry(s)
4343 .or_default()
4344 .entry(p)
4345 .or_default()
4346 .push(o);
4347 }
4348 let mut out = String::new();
4349 for (s, preds) in &by_subject {
4350 out.push_str(s);
4351 out.push('\n');
4352 let pred_count = preds.len();
4353 for (i, (p, objs)) in preds.iter().enumerate() {
4354 let pred = if *p == RDF_TYPE_IRI { "a" } else { p };
4355 let objects = objs.join(" , ");
4356 let terminator = if i + 1 == pred_count { " ." } else { " ;" };
4357 out.push_str(&format!(" {pred} {objects}{terminator}\n"));
4358 }
4359 out.push('\n');
4360 }
4361 Ok(out)
4362}
4363
4364/// Serialize a triple list to expanded JSON-LD (array of node objects keyed by
4365/// `@id`; literals as `@value` + optional `@type`/`@language`).
4366///
4367/// Expanded JSON-LD has no term kind for a quoted triple, so a result holding
4368/// one is refused, as `rete export --format jsonld` refuses (#262): written
4369/// anyway it came out as an `@id`, a document that loads cleanly and means
4370/// something else. The CLI refuses from the header flag before any work; here
4371/// the CONSTRUCT result is already in hand, so the check is on the result.
4372fn to_jsonld(triples: &[(String, String, String)]) -> Result<String, String> {
4373 use rete_core::terms::is_quoted_triple;
4374 use serde_json::{json, Map, Value};
4375 use std::collections::BTreeMap;
4376 if let Some((s, p, o)) = triples
4377 .iter()
4378 .find(|(s, p, o)| is_quoted_triple(s) || is_quoted_triple(p) || is_quoted_triple(o))
4379 {
4380 return Err(format!(
4381 "this CONSTRUCT result contains an RDF-star quoted triple, and expanded JSON-LD has \
4382 no term kind for one.\n\
4383 Written anyway, a quoted triple would be serialized as though it were an IRI — a \
4384 document that loads cleanly and means something else.\n\
4385 statement: {s} {p} {o} .\n\
4386 hint: format \"ttl\" writes it as an RDF 1.2 triple term <<( s p o )>> (add \
4387 \";quotedTripleSyntax=rdf-star\" for <<s p o>>), and the table output lists it as is."
4388 ));
4389 }
4390 let mut nodes: BTreeMap<String, BTreeMap<String, Vec<Value>>> = BTreeMap::new();
4391 for (s, p, o) in triples {
4392 let id = node_id(s);
4393 let pred = p
4394 .strip_prefix('<')
4395 .and_then(|x| x.strip_suffix('>'))
4396 .unwrap_or(p)
4397 .to_string();
4398 nodes
4399 .entry(id)
4400 .or_default()
4401 .entry(pred)
4402 .or_default()
4403 .push(object_to_jsonld(o));
4404 }
4405 let arr: Vec<Value> = nodes
4406 .into_iter()
4407 .map(|(id, preds)| {
4408 let mut obj = Map::new();
4409 obj.insert("@id".into(), json!(id));
4410 for (pred, vals) in preds {
4411 obj.insert(pred, Value::Array(vals));
4412 }
4413 Value::Object(obj)
4414 })
4415 .collect();
4416 Ok(serde_json::to_string_pretty(&Value::Array(arr)).unwrap_or_default())
4417}
4418
4419/// JSON-LD `@id` for a subject token: bare IRI for `<iri>`, `_:b` verbatim.
4420fn node_id(token: &str) -> String {
4421 token
4422 .strip_prefix('<')
4423 .and_then(|s| s.strip_suffix('>'))
4424 .map(str::to_string)
4425 .unwrap_or_else(|| token.to_string())
4426}
4427
4428/// Classify an object token into a JSON-LD value object.
4429fn object_to_jsonld(token: &str) -> serde_json::Value {
4430 use serde_json::{json, Map, Value};
4431 if let Some(iri) = token.strip_prefix('<').and_then(|s| s.strip_suffix('>')) {
4432 return json!({ "@id": iri });
4433 }
4434 if let Some(b) = token.strip_prefix("_:") {
4435 return json!({ "@id": format!("_:{b}") });
4436 }
4437 if token.starts_with('"') {
4438 let bytes = token.as_bytes();
4439 let mut i = 1;
4440 while i < bytes.len() {
4441 match bytes[i] {
4442 b'\\' => i += 2,
4443 b'"' => break,
4444 _ => i += 1,
4445 }
4446 }
4447 let value = unescape_nt(&token[1..i.min(token.len())]);
4448 let rest = token.get(i + 1..).unwrap_or("");
4449 let mut obj = Map::new();
4450 obj.insert("@value".into(), Value::String(value));
4451 if let Some(dt) = rest.strip_prefix("^^<").and_then(|s| s.strip_suffix('>')) {
4452 obj.insert("@type".into(), json!(dt));
4453 } else if let Some(lang) = rest.strip_prefix('@') {
4454 obj.insert("@language".into(), json!(lang));
4455 }
4456 return Value::Object(obj);
4457 }
4458 json!({ "@value": token })
4459}
4460
4461/// Resolve the N-Triples escape sequences in a literal's body to actual chars.
4462fn unescape_nt(s: &str) -> String {
4463 rete_core::terms::unescape_literal(s)
4464}
4465
4466fn open(bytes: &[u8]) -> Result<Rete, JsValue> {
4467 let mut rete = Rete::open(bytes).map_err(err)?;
4468 // SERVICE blocks federate to remote SPARQL endpoints over sync XHR.
4469 rete.set_service_client(Box::new(XhrServiceClient));
4470 Ok(rete)
4471}
4472
4473fn err<E: std::fmt::Display>(e: E) -> JsValue {
4474 js_error(e.to_string())
4475}
4476
4477fn js_error(message: impl AsRef<str>) -> JsValue {
4478 js_sys::Error::new(message.as_ref()).into()
4479}
4480
4481#[cfg(test)]
4482mod construct_writer_tests {
4483 use super::{construct_format, to_jsonld, to_turtle, write_query_json};
4484 use rete_core::ingest::QuotedTripleSurface::{Rdf12, RdfStar};
4485 use rete_core::QueryOutput;
4486
4487 const S: &str = "<http://ex/s>";
4488 const P: &str = "<http://ex/p>";
4489 const QT: &str = "<<<http://ex/a> <http://ex/b> <http://ex/c>>>";
4490
4491 fn t(s: &str, p: &str, o: &str) -> (String, String, String) {
4492 (s.to_string(), p.to_string(), o.to_string())
4493 }
4494
4495 /// A graph with no quoted triple, the shape every published file has.
4496 fn plain() -> Vec<(String, String, String)> {
4497 vec![
4498 t(S, P, "\"a > b << c\"@en"),
4499 t(
4500 S,
4501 "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>",
4502 "<http://ex/T>",
4503 ),
4504 t(S, P, "_:b0"),
4505 t(
4506 "_:b0",
4507 P,
4508 "\"5\"^^<http://www.w3.org/2001/XMLSchema#integer>",
4509 ),
4510 ]
4511 }
4512
4513 /// The writer as it was before this change, kept verbatim as the oracle
4514 /// for byte identity.
4515 fn legacy_turtle(triples: &[(String, String, String)]) -> String {
4516 use std::collections::BTreeMap;
4517 let mut by_subject: BTreeMap<&str, BTreeMap<&str, Vec<&str>>> = BTreeMap::new();
4518 for (s, p, o) in triples {
4519 by_subject
4520 .entry(s)
4521 .or_default()
4522 .entry(p)
4523 .or_default()
4524 .push(o);
4525 }
4526 let mut out = String::new();
4527 for (s, preds) in &by_subject {
4528 out.push_str(s);
4529 out.push('\n');
4530 let pred_count = preds.len();
4531 for (i, (p, objs)) in preds.iter().enumerate() {
4532 let pred = if *p == super::RDF_TYPE_IRI { "a" } else { p };
4533 let objects = objs.join(" , ");
4534 let terminator = if i + 1 == pred_count { " ." } else { " ;" };
4535 out.push_str(&format!(" {pred} {objects}{terminator}\n"));
4536 }
4537 out.push('\n');
4538 }
4539 out
4540 }
4541
4542 #[test]
4543 fn turtle_without_quoted_triples_is_byte_identical_under_both_surfaces() {
4544 let g = plain();
4545 let before = legacy_turtle(&g);
4546 assert_eq!(to_turtle(&g, Rdf12).unwrap(), before);
4547 assert_eq!(to_turtle(&g, RdfStar).unwrap(), before);
4548 }
4549
4550 #[test]
4551 fn turtle_writes_an_object_triple_term_in_the_rdf12_surface_by_default() {
4552 let g = vec![t(S, P, QT)];
4553 let out = to_turtle(&g, Rdf12).unwrap();
4554 assert!(
4555 out.contains("<<( <http://ex/a> <http://ex/b> <http://ex/c> )>>"),
4556 "{out}"
4557 );
4558 assert!(!out.contains("<<<"), "the legacy surface leaked: {out}");
4559 // Nested in the object slot: rewritten at depth too.
4560 let nested = format!("<<<http://ex/x> <http://ex/y> {QT}>>");
4561 let out = to_turtle(&[t(S, P, &nested)], Rdf12).unwrap();
4562 assert!(
4563 out.contains(
4564 "<<( <http://ex/x> <http://ex/y> <<( <http://ex/a> <http://ex/b> <http://ex/c> )>> )>>"
4565 ),
4566 "{out}"
4567 );
4568 }
4569
4570 #[test]
4571 fn turtle_output_reads_back_as_the_same_graph_under_rdf12() {
4572 // Through rete's own RDF 1.2 Turtle reader: one statement (a triple
4573 // term, not a reifier), carrying the stored token it started as. The
4574 // grouped `;` / `,` layout is exercised by a second predicate.
4575 let g = vec![
4576 t(S, P, QT),
4577 t(S, P, "<http://ex/o>"),
4578 t(S, "<http://ex/q>", QT),
4579 ];
4580 let out = to_turtle(&g, Rdf12).unwrap();
4581 let mut quads = rete_core::ingest::parse_statements_audited_surface(
4582 &out,
4583 "ttl",
4584 None,
4585 rete_core::ingest::QuotedTripleSurface::Rdf12,
4586 )
4587 .unwrap();
4588 quads.sort();
4589 let mut want: Vec<_> = g
4590 .iter()
4591 .map(|(s, p, o)| (s.clone(), p.clone(), o.clone()))
4592 .collect();
4593 want.sort();
4594 let got: Vec<_> = quads
4595 .iter()
4596 .map(|q| (q.0.clone(), q.1.clone(), q.2.clone()))
4597 .collect();
4598 assert_eq!(got, want, "{out}");
4599 }
4600
4601 #[test]
4602 fn turtle_rdf_star_keeps_the_stored_token() {
4603 let out = to_turtle(&[t(S, P, QT)], RdfStar).unwrap();
4604 assert_eq!(out, legacy_turtle(&[t(S, P, QT)]));
4605 assert!(out.contains(QT));
4606 }
4607
4608 #[test]
4609 fn turtle_refuses_a_subject_position_quoted_triple_by_name() {
4610 let e = to_turtle(&[t(QT, P, "<http://ex/o>")], Rdf12).unwrap_err();
4611 assert!(e.contains("quoted triple in the subject position"), "{e}");
4612 assert!(e.contains("OBJECT position only"), "{e}");
4613 assert!(e.contains("rdf-star"), "the hint names the escape: {e}");
4614 // …and one nested in another's subject.
4615 let inner_subject = format!("<<{QT} <http://ex/y> <http://ex/z>>>");
4616 let e = to_turtle(&[t(S, P, &inner_subject)], Rdf12).unwrap_err();
4617 assert!(e.contains("nested in the SUBJECT"), "{e}");
4618 // The RDF-star surface has a spelling for both.
4619 assert!(to_turtle(&[t(QT, P, "<http://ex/o>")], RdfStar).is_ok());
4620 }
4621
4622 #[test]
4623 fn jsonld_refuses_a_quoted_triple_in_any_position() {
4624 for g in [vec![t(S, P, QT)], vec![t(QT, P, "<http://ex/o>")]] {
4625 let e = to_jsonld(&g).unwrap_err();
4626 assert!(e.contains("expanded JSON-LD has no term kind"), "{e}");
4627 assert!(e.contains("<<( s p o )>>"), "{e}");
4628 }
4629 }
4630
4631 #[test]
4632 fn jsonld_without_quoted_triples_is_unchanged() {
4633 let out = to_jsonld(&plain()).unwrap();
4634 let v: serde_json::Value = serde_json::from_str(&out).unwrap();
4635 assert_eq!(v.as_array().unwrap().len(), 2);
4636 // A literal that merely contains `<<` is not a quoted triple.
4637 assert!(out.contains(r#""@value": "a > b << c""#), "{out}");
4638 }
4639
4640 #[test]
4641 fn the_format_parameter_mirrors_the_cli_flag() {
4642 assert_eq!(construct_format("ttl").unwrap(), ("ttl", Rdf12));
4643 assert_eq!(
4644 construct_format("ttl;quotedTripleSyntax=rdf-star").unwrap(),
4645 ("ttl", RdfStar)
4646 );
4647 assert_eq!(
4648 construct_format("ttl;quotedTripleSyntax=rdf12").unwrap(),
4649 ("ttl", Rdf12)
4650 );
4651 assert!(construct_format("ttl;quotedTripleSyntax=rdfstar")
4652 .unwrap_err()
4653 .contains("unknown quotedTripleSyntax"));
4654 assert!(construct_format("ttl;surface=rdf12")
4655 .unwrap_err()
4656 .contains("unknown format parameter"));
4657
4658 // End to end through the envelope.
4659 let out = QueryOutput::Construct(vec![t(S, P, QT)]);
4660 let env = write_query_json(&out, "ttl", "").unwrap();
4661 assert!(
4662 env.contains(r#""format":"ttl""#) && env.contains("<<( "),
4663 "{env}"
4664 );
4665 let env = write_query_json(&out, "ttl;quotedTripleSyntax=rdf-star", "").unwrap();
4666 assert!(env.contains("<<<http://ex/a>"), "{env}");
4667 assert!(write_query_json(&out, "jsonld", "").is_err());
4668 // The table form lists the stored token as before.
4669 let env = write_query_json(&out, "table", "").unwrap();
4670 assert!(env.contains("<<<http://ex/a>"), "{env}");
4671 }
4672}