use rete_core::{
batch_reach_serial, build_adjacency, build_dendrogram, choose_round_for_budget, eval_query,
eval_query_with_warnings, eval_select_communities, eval_sparql, project_graph, schema_classes,
schema_summary, summary_query_shape, tile_by_community, validate_shacl, BlockCacheReader,
ByteRange, CountingReader, DataGraph, Header, OffsetReader, QueryOpts, QueryOutput,
RangeReader, Rete, ReteGraph, ShaclShapes, SliceReader, SummaryQueryShape, SummaryView,
TermTriple, TripleProvenance, ValidationReport, DEFAULT_BLOCK, DEFAULT_TILE_BUDGET,
};
use std::rc::Rc;
use wasm_bindgen::prelude::*;
pub const JSON_SCHEMA_VERSION: u8 = 1;
#[wasm_bindgen(start)]
pub fn __start() {
#[cfg(feature = "asyncify")]
std::panic::set_hook(Box::new(|info| {
if let Some(loc) = info.location() {
let f = loc.file();
unsafe { rete_panic_report(f.as_ptr(), f.len(), loc.line()) };
} else {
unsafe { rete_panic_report(std::ptr::null(), 0, 0) };
}
}));
#[cfg(not(feature = "asyncify"))]
std::panic::set_hook(Box::new(|info| {
web_sys::console::error_1(&JsValue::from_str(&format!("rete-wasm panic: {info}")));
}));
}
#[wasm_bindgen]
pub fn info(bytes: &[u8]) -> Result<String, JsValue> {
let rete = open(bytes)?;
let h = rete.header();
Ok(format!(
r#"{{"schemaVersion":1,"quads":{},"terms":{},"pyramidLevels":{},"namedGraphs":{}}}"#,
h.quad_count,
h.term_count,
h.pyramid_levels,
rete.named_graph_count()
))
}
#[wasm_bindgen]
pub fn graph_names(bytes: &[u8]) -> Result<String, JsValue> {
let rete = open(bytes)?;
serde_json::to_string(&rete.graph_names()).map_err(err)
}
#[wasm_bindgen]
pub fn build(text: &str, format: &str) -> Result<Vec<u8>, JsValue> {
let quads = rete_core::ingest::parse_statements(text, format).map_err(err)?;
if quads.is_empty() {
return Err(js_error(
"no statements parsed (empty input or only comments)",
));
}
let (bytes, _stats) = rete_core::ingest::assemble_dataset(quads, &[]);
Ok(bytes)
}
#[wasm_bindgen]
pub fn build_with_card(text: &str, format: &str, card_json: &str) -> Result<Vec<u8>, JsValue> {
build_card_surface(
text,
format,
card_json,
rete_core::ingest::QuotedTripleSurface::default(),
)
}
#[wasm_bindgen]
pub fn build_with_card_syntax(
text: &str,
format: &str,
card_json: &str,
quoted_triple_syntax: &str,
) -> Result<Vec<u8>, JsValue> {
let surface = if quoted_triple_syntax.is_empty() {
rete_core::ingest::QuotedTripleSurface::default()
} else {
rete_core::ingest::QuotedTripleSurface::parse(quoted_triple_syntax).ok_or_else(|| {
js_error(format!(
"unknown quoted-triple syntax {quoted_triple_syntax:?} (expected \"rdf-star\" or \"rdf12\")"
))
})?
};
build_card_surface(text, format, card_json, surface)
}
fn build_card_surface(
text: &str,
format: &str,
card_json: &str,
surface: rete_core::ingest::QuotedTripleSurface,
) -> Result<Vec<u8>, JsValue> {
let quads = rete_core::ingest::parse_statements_audited_surface(text, format, None, surface)
.map_err(err)?;
if quads.is_empty() {
return Err(js_error(
"no statements parsed (empty input or only comments)",
));
}
if card_json.trim().is_empty() {
let (bytes, _stats) = rete_core::ingest::assemble_dataset(quads, &[]);
return Ok(bytes);
}
let curated = validated_card(card_json)?;
let (bytes, _stats) = rete_core::ingest::assemble_dataset_with(quads, move |stats, _| {
let card = rete_core::card::compose_curated_card(
curated,
stats.default_triples as u64,
stats.statements as u64,
stats.named_graphs as u64,
stats.terms as u64,
rete_core::CURRENT_FORMAT_VERSION,
);
serde_json::to_vec(&card).expect("card serializes")
});
Ok(bytes)
}
#[wasm_bindgen]
pub fn build_with_derived_card(
text: &str,
format: &str,
card_json: &str,
) -> Result<Vec<u8>, JsValue> {
let quads = rete_core::ingest::parse_statements(text, format).map_err(err)?;
if quads.is_empty() {
return Err(js_error(
"no statements parsed (empty input or only comments)",
));
}
let curated = if card_json.trim().is_empty() {
rete_core::card::CardInput::default()
} else {
validated_card(card_json)?;
rete_core::card::CardInput::from_json_str(card_json).map_err(js_error)?
};
let (bytes, _stats) = rete_core::ingest::assemble_dataset_with(quads, move |stats, quads| {
let card = rete_core::card::derive_card(
quads,
stats.terms as u64,
stats.named_graphs as u64,
curated,
);
rete_core::ingest::DeferredMetadata::new(move |counts| {
card.with_final_counts(counts).to_json_bytes()
})
});
Ok(bytes)
}
#[wasm_bindgen]
pub fn validate_card(card_json: &str) -> String {
check_card(card_json).err().unwrap_or_default()
}
fn check_card(card_json: &str) -> Result<serde_json::Value, String> {
let doc: serde_json::Value =
serde_json::from_str(card_json).map_err(|e| format!("card is not JSON: {e}"))?;
rete_core::card::validate_curated_card(&doc)
}
fn validated_card(card_json: &str) -> Result<serde_json::Value, JsValue> {
check_card(card_json).map_err(js_error)
}
#[wasm_bindgen]
pub fn query_triples(
bytes: &[u8],
subject: Option<String>,
predicate: Option<String>,
object: Option<String>,
) -> Result<String, JsValue> {
let rete = open(bytes)?;
let rows = rete.query(subject.as_deref(), predicate.as_deref(), object.as_deref());
serde_json::to_string(&rows).map_err(err)
}
#[wasm_bindgen]
pub fn why_triples(
bytes: &[u8],
subject: Option<String>,
predicate: Option<String>,
object: Option<String>,
) -> Result<String, JsValue> {
why_triples_json(
bytes,
subject.as_deref(),
predicate.as_deref(),
object.as_deref(),
)
.map_err(js_error)
}
pub fn why_triples_json(
bytes: &[u8],
subject: Option<&str>,
predicate: Option<&str>,
object: Option<&str>,
) -> Result<String, String> {
use serde_json::json;
let rete = Rete::open(bytes).map_err(|e| e.to_string())?;
let results = rete.query_with_provenance(subject, predicate, object);
let out = json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"pattern": {
"subject": subject,
"predicate": predicate,
"object": object,
},
"resultCount": results.len(),
"results": results.iter().map(provenance_json).collect::<Vec<_>>(),
});
serde_json::to_string(&out).map_err(|e| e.to_string())
}
#[wasm_bindgen]
pub fn query_sparql(bytes: &[u8], query: &str) -> Result<String, JsValue> {
let rete = open(bytes)?;
let (_vars, solutions) = eval_sparql(&rete, query).map_err(err)?;
serde_json::to_string(&solutions).map_err(err)
}
#[wasm_bindgen]
pub fn schema(bytes: &[u8]) -> Result<String, JsValue> {
let rete = open(bytes)?;
let classes = schema_classes(&rete);
let relations = schema_summary(&rete);
serde_json::to_string(&serde_json::json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"classes": classes,
"relations": relations,
}))
.map_err(err)
}
#[wasm_bindgen]
pub fn schema_packed(bytes: &[u8]) -> Result<String, JsValue> {
use serde_json::json;
let (classes, relations) = rete_core::read_schema_summary_ranged(&SliceReader::new(bytes))
.map_err(err)?
.ok_or_else(|| js_error("file has no schema pyramid"))?;
let classes: Vec<serde_json::Value> = classes.iter().map(|(c, n)| json!([c, n])).collect();
let relations: Vec<serde_json::Value> = relations
.iter()
.map(|(s, p, o, n)| json!([s, p, o, n]))
.collect();
Ok(json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"classes": classes,
"relations": relations,
})
.to_string())
}
#[wasm_bindgen]
pub fn card(bytes: &[u8]) -> Result<Option<String>, JsValue> {
let bytes = rete_core::read_metadata_ranged(&SliceReader::new(bytes)).map_err(err)?;
Ok(bytes.map(|b| String::from_utf8_lossy(&b).into_owned()))
}
#[wasm_bindgen]
pub fn card_url(url: &str) -> Result<Option<String>, JsValue> {
let reader = RemoteReader::open(url)?.view();
let bytes = rete_core::read_metadata_ranged(&reader).map_err(err)?;
Ok(bytes.map(|b| String::from_utf8_lossy(&b).into_owned()))
}
#[wasm_bindgen]
pub fn card_and_build_url(url: &str) -> Result<String, JsValue> {
let reader = RemoteReader::open(url)?.view();
card_build_from(&reader)
}
#[wasm_bindgen]
pub fn card_and_build(bytes: &[u8]) -> Result<String, JsValue> {
card_build_from(&SliceReader::new(bytes))
}
fn card_build_from<R: RangeReader>(reader: &R) -> Result<String, JsValue> {
let (header, card, build) =
rete_core::read_card_and_build_info_with_header(reader).map_err(err)?;
let token_table = rete_core::read_text_index_token_table_len_ranged(reader, &header);
Ok(card_build_envelope(
card,
build,
text_index_json(&header, token_table),
))
}
fn text_index_json(header: &Header, token_table_bytes: Option<u64>) -> serde_json::Value {
if header.text_index_len == 0 {
return serde_json::json!({ "present": false });
}
let mut v = serde_json::json!({
"present": true,
"bytes": header.text_index_len,
});
if let Some(tt) = token_table_bytes {
v.as_object_mut()
.expect("an object")
.insert("token_table_bytes".into(), serde_json::json!(tt));
}
v
}
fn card_build_envelope(
card: Option<Vec<u8>>,
build: Option<Vec<u8>>,
text_index: serde_json::Value,
) -> String {
let text = |b: Option<Vec<u8>>| match b {
Some(b) if !b.is_empty() => {
serde_json::Value::String(String::from_utf8_lossy(&b).into_owned())
}
_ => serde_json::Value::Null,
};
serde_json::json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"card": text(card),
"build": text(build),
"text_index": text_index,
})
.to_string()
}
#[wasm_bindgen]
pub fn file_len_url(url: &str) -> Result<String, JsValue> {
let reader = RemoteReader::open(url)?;
Ok(format!(
r#"{{"schemaVersion":{},"fileLength":{}}}"#,
JSON_SCHEMA_VERSION,
reader.len()
))
}
#[wasm_bindgen]
pub struct Graph {
rete: Rc<Rete>,
file_len: usize,
build_info: Option<String>,
}
#[wasm_bindgen]
impl Graph {
#[wasm_bindgen(constructor)]
pub fn new(bytes: &[u8]) -> Result<Graph, JsValue> {
Ok(Graph {
rete: Rc::new(open(bytes)?),
file_len: bytes.len(),
build_info: rete_core::read_build_info(bytes)
.ok()
.flatten()
.filter(|b| !b.is_empty())
.map(|b| String::from_utf8_lossy(&b).into_owned()),
})
}
pub fn quads(
&self,
graph: Option<String>,
s: Option<String>,
p: Option<String>,
o: Option<String>,
) -> QuadCursor {
QuadCursor::start(self.rete.clone(), graph, s, p, o)
}
pub fn info(&self) -> String {
let h = self.rete.header();
format!(
r#"{{"schemaVersion":1,"quads":{},"terms":{},"pyramidLevels":{},"namedGraphs":{}}}"#,
h.quad_count,
h.term_count,
h.pyramid_levels,
self.rete.named_graph_count()
)
}
pub fn graph_names(&self) -> Result<String, JsValue> {
serde_json::to_string(&self.rete.graph_names()).map_err(err)
}
pub fn query(&self, query: &str, format: &str) -> Result<String, JsValue> {
query_json(&self.rete, query, format, "")
}
pub fn query_reasoned(&self, query: &str, format: &str) -> Result<String, JsValue> {
query_json_reasoned(&self.rete, query, format, "")
}
pub fn query_opts(
&self,
query: &str,
format: &str,
reason: bool,
union_default: bool,
) -> Result<String, JsValue> {
query_json_with(&self.rete, query, format, "", reason, union_default)
}
pub fn query_triples(
&self,
subject: Option<String>,
predicate: Option<String>,
object: Option<String>,
) -> Result<String, JsValue> {
let rows = self
.rete
.query(subject.as_deref(), predicate.as_deref(), object.as_deref());
serde_json::to_string(&rows).map_err(err)
}
pub fn prefix_search(&self, prefix: &str, limit: usize) -> Result<String, JsValue> {
prefix_search_json(&self.rete, prefix, limit)
}
pub fn text_search(
&self,
words: Vec<String>,
contains_prefix: Option<String>,
limit: usize,
) -> Result<String, JsValue> {
text_search_json(&self.rete, &words, contains_prefix.as_deref(), limit)
}
pub fn text_index_len(&self) -> f64 {
self.rete.header().text_index_len as f64
}
pub fn text_index_token_table_len(&self) -> f64 {
self.rete.text_index_token_table_len().unwrap_or(0) as f64
}
pub fn text_search_one(&self, phrase: &str, limit: usize) -> Result<String, JsValue> {
let words: Vec<String> = phrase.split_whitespace().map(str::to_owned).collect();
text_search_json(&self.rete, &words, None, limit)
}
pub fn why_triples(
&self,
subject: Option<String>,
predicate: Option<String>,
object: Option<String>,
) -> Result<String, JsValue> {
use serde_json::json;
let results = self.rete.query_with_provenance(
subject.as_deref(),
predicate.as_deref(),
object.as_deref(),
);
let out = json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"pattern": { "subject": subject, "predicate": predicate, "object": object },
"resultCount": results.len(),
"results": results.iter().map(provenance_json).collect::<Vec<_>>(),
});
serde_json::to_string(&out).map_err(err)
}
pub fn schema(&self) -> Result<String, JsValue> {
serde_json::to_string(&serde_json::json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"classes": schema_classes(&self.rete),
"relations": schema_summary(&self.rete),
}))
.map_err(err)
}
pub fn quoted_triples_signal(&self) -> String {
quoted_triples_signal_json(self.rete.header())
}
pub fn card(&self) -> Option<String> {
self.rete
.metadata()
.map(|b| String::from_utf8_lossy(b).into_owned())
}
pub fn card_and_build(&self) -> String {
let card = self
.rete
.metadata()
.filter(|b| !b.is_empty())
.map(|b| b.to_vec());
let text_index =
text_index_json(self.rete.header(), self.rete.text_index_token_table_len());
card_build_envelope(
card,
self.build_info.as_ref().map(|s| s.as_bytes().to_vec()),
text_index,
)
}
pub fn query_communities(&self, query: &str, round: Option<usize>) -> Result<String, JsValue> {
query_communities_value(&self.rete, query, round)
}
pub fn reach(&self, predicate: &str, seeds: &str, reverse: bool) -> Result<String, JsValue> {
reach_rete(&self.rete, predicate, seeds, reverse)
}
pub fn shacl(
&self,
shapes_turtle: &str,
graph: Option<String>,
format: &str,
) -> Result<String, JsValue> {
shacl_rete(&self.rete, shapes_turtle, graph.as_deref(), format)
}
pub fn reason(&self, graph: Option<String>) -> String {
let base = self.rete.dump(graph.as_deref());
reasoning_json(&rete_core::reason(&base), None)
}
pub fn pyramid_tree(&self) -> Result<String, JsValue> {
pyramid_tree_value(&self.rete)
}
pub fn file_layout(&self) -> Result<String, JsValue> {
use serde_json::json;
let segments: Vec<serde_json::Value> = self
.rete
.file_layout()
.iter()
.map(|s| json!({ "kind": s.kind, "label": s.label, "offset": s.offset, "len": s.len }))
.collect();
serde_json::to_string(&json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"fileLength": self.file_len,
"segments": segments,
}))
.map_err(err)
}
}
#[wasm_bindgen]
pub struct QuadCursor {
pending: std::vec::IntoIter<Option<String>>,
current: Option<Option<String>>,
cursor: u64,
s: Option<String>,
p: Option<String>,
o: Option<String>,
slot_done: bool,
buffered: std::vec::IntoIter<TermTriple>,
rete: Rc<Rete>,
rdf12: bool,
}
const DUMP_BATCH: usize = 10_000;
impl QuadCursor {
fn start(
rete: Rc<Rete>,
graph: Option<String>,
s: Option<String>,
p: Option<String>,
o: Option<String>,
) -> QuadCursor {
let slots: Vec<Option<String>> = match graph {
None => std::iter::once(None)
.chain(rete.graph_names().iter().map(|g| Some((*g).to_string())))
.collect(),
Some(g) if g.is_empty() => vec![None],
Some(g) => vec![Some(canonical_graph_name(&rete, g))],
};
rete.reset_load_failures();
let term = |t: Option<String>| t.filter(|t| !t.is_empty());
QuadCursor {
pending: slots.into_iter(),
current: None,
cursor: 0,
s: term(s),
p: term(p),
o: term(o),
slot_done: false,
buffered: Vec::new().into_iter(),
rete,
rdf12: false,
}
}
fn pull<F: FnMut(&str, &str, &str, Option<&str>)>(&mut self, max: usize, mut sink: F) -> usize {
let mut emitted = 0;
while emitted < max {
if let Some((s, p, o)) = self.buffered.next() {
let graph = self.current.as_ref().and_then(|g| g.as_deref());
sink(&s, &p, &o, graph);
emitted += 1;
continue;
}
let Some(slot) = self.current.clone() else {
match self.pending.next() {
Some(next) => {
self.current = Some(next);
self.cursor = 0;
self.slot_done = false;
}
None => break,
}
continue;
};
if self.slot_done {
self.current = None;
continue;
}
let want = (max - emitted).max(DUMP_BATCH);
let (triples, next, done) = self.rete.query_batch(
slot.as_deref(),
self.s.as_deref(),
self.p.as_deref(),
self.o.as_deref(),
self.cursor,
want,
);
self.cursor = next;
self.slot_done = done;
self.buffered = triples.into_iter();
}
emitted
}
fn finished(&self) -> bool {
self.current.is_none() && self.pending.len() == 0 && self.buffered.len() == 0
}
fn guard(&self) -> Result<(), JsValue> {
if self.finished() {
incomplete_guard(&self.rete, "dump")?;
}
Ok(())
}
}
fn rdf12_object<'a>(s: &str, p: &str, o: &'a str) -> Result<std::borrow::Cow<'a, str>, String> {
use rete_core::terms::is_quoted_triple;
if is_quoted_triple(s) || is_quoted_triple(p) {
let slot = if is_quoted_triple(s) {
"subject"
} else {
"predicate"
};
return Err(format!(
"this graph has a quoted triple in the {slot} position of a statement, and RDF 1.2 \
has no syntax for one there: a triple term may stand in OBJECT position only.\n\
statement: {s} {p} {o} .\n\
hint: quotedTripleSyntax \"rdf-star\" writes the RDF-star surface <<s p o>>, which \
rete re-ingests losslessly"
));
}
rete_core::terms::rdf12_triple_term(o).ok_or_else(|| {
format!(
"this graph has a quoted triple nested in the SUBJECT of another quoted triple, and \
RDF 1.2 has no syntax for one there.\n\
statement: {s} {p} {o} .\n\
hint: quotedTripleSyntax \"rdf-star\" writes the RDF-star surface <<s p o>>"
)
})
}
fn quoted_triples_signal_json(header: &Header) -> String {
serde_json::to_string(&rete_core::card::QuotedTriplesSignal::probe(header))
.expect("the signal serializes")
}
#[wasm_bindgen]
impl QuadCursor {
pub fn next_batch(&mut self, max: Option<usize>) -> Result<Vec<String>, JsValue> {
let max = max.unwrap_or(DUMP_BATCH).max(1);
let mut out: Vec<String> = Vec::with_capacity(max.min(DUMP_BATCH) * 4);
self.pull(max, |s, p, o, g| {
out.push(s.to_string());
out.push(p.to_string());
out.push(o.to_string());
out.push(g.unwrap_or("").to_string());
});
self.guard()?;
Ok(out)
}
pub fn next_nquads(&mut self, max: Option<usize>) -> Result<String, JsValue> {
let max = max.unwrap_or(DUMP_BATCH).max(1);
let mut out = String::with_capacity(max.min(DUMP_BATCH) * 120);
let rdf12 = self.rdf12;
let mut refused: Option<String> = None;
self.pull(max, |s, p, o, g| {
if refused.is_some() {
return;
}
let o = if rdf12 {
match rdf12_object(s, p, o) {
Ok(o) => o,
Err(e) => {
refused = Some(e);
return;
}
}
} else {
std::borrow::Cow::Borrowed(o)
};
out.push_str(s);
out.push(' ');
out.push_str(p);
out.push(' ');
out.push_str(&o);
if let Some(g) = g {
out.push(' ');
out.push_str(g);
}
out.push_str(" .\n");
});
if let Some(e) = refused {
return Err(js_error(e));
}
self.guard()?;
Ok(out)
}
pub fn set_quoted_triple_syntax(&mut self, syntax: &str) -> Result<(), JsValue> {
let surface = if syntax.is_empty() {
rete_core::ingest::QuotedTripleSurface::RdfStar
} else {
rete_core::ingest::QuotedTripleSurface::parse(syntax).ok_or_else(|| {
js_error(format!(
"unknown quoted-triple syntax {syntax:?} (expected \"rdf12\" or \"rdf-star\")"
))
})?
};
self.rdf12 = surface == rete_core::ingest::QuotedTripleSurface::Rdf12
&& self.rete.header().has_quoted_triples();
Ok(())
}
pub fn done(&self) -> bool {
self.finished()
}
}
fn canonical_graph_name(rete: &Rete, name: String) -> String {
if rete.graph_names().iter().any(|g| *g == name) {
return name;
}
if name.starts_with('<') || name.starts_with("_:") {
return name;
}
format!("<{name}>")
}
#[wasm_bindgen]
pub fn heap_bytes() -> f64 {
#[cfg(target_arch = "wasm32")]
{
(core::arch::wasm32::memory_size(0) as f64) * 65536.0
}
#[cfg(not(target_arch = "wasm32"))]
{
0.0
}
}
#[wasm_bindgen]
pub struct RemoteGraph {
reader: RemoteReader,
rete: Rc<Rete>,
}
#[wasm_bindgen]
impl RemoteGraph {
#[wasm_bindgen(constructor)]
pub fn new(url: &str) -> Result<RemoteGraph, JsValue> {
let (reader, rete) = open_url(url)?;
Ok(RemoteGraph {
reader,
rete: Rc::new(rete),
})
}
pub fn quads(
&self,
graph: Option<String>,
s: Option<String>,
p: Option<String>,
o: Option<String>,
) -> QuadCursor {
QuadCursor::start(self.rete.clone(), graph, s, p, o)
}
pub fn stats(&self) -> String {
format!(
r#"{{"schemaVersion":1,"fileLength":{},"bytes":{},"requests":{},"base":{}}}"#,
self.reader.len(),
self.reader.bytes_read(),
self.reader.requests(),
self.reader.base
)
}
pub fn content_hash(&self) -> String {
self.rete
.header()
.content_hash
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
pub fn query(&self, query: &str, format: &str) -> Result<String, JsValue> {
self.rete.reset_load_failures();
let s = query_json(&self.rete, query, format, "")?;
incomplete_guard(&self.rete, "query")?;
Ok(s)
}
pub fn query_reasoned(&self, query: &str, format: &str) -> Result<String, JsValue> {
self.rete.reset_load_failures();
let s = query_json_reasoned(&self.rete, query, format, "")?;
incomplete_guard(&self.rete, "query")?;
Ok(s)
}
pub fn query_opts(
&self,
query: &str,
format: &str,
reason: bool,
union_default: bool,
) -> Result<String, JsValue> {
self.rete.reset_load_failures();
let s = query_json_with(&self.rete, query, format, "", reason, union_default)?;
incomplete_guard(&self.rete, "query")?;
Ok(s)
}
pub fn prefix_search(&self, prefix: &str, limit: usize) -> Result<String, JsValue> {
prefix_search_json(&self.rete, prefix, limit)
}
pub fn text_search(
&self,
words: Vec<String>,
contains_prefix: Option<String>,
limit: usize,
) -> Result<String, JsValue> {
text_search_json(&self.rete, &words, contains_prefix.as_deref(), limit)
}
pub fn text_index_len(&self) -> f64 {
self.rete.header().text_index_len as f64
}
pub fn text_index_token_table_len(&self) -> f64 {
self.rete.text_index_token_table_len().unwrap_or(0) as f64
}
pub fn text_search_one(&self, phrase: &str, limit: usize) -> Result<String, JsValue> {
let words: Vec<String> = phrase.split_whitespace().map(str::to_owned).collect();
text_search_json(&self.rete, &words, None, limit)
}
pub fn quoted_triples_signal(&self) -> String {
quoted_triples_signal_json(self.rete.header())
}
pub fn card(&self) -> Result<Option<String>, JsValue> {
let bytes = rete_core::read_metadata_ranged(&self.reader.view()).map_err(err)?;
Ok(bytes.map(|b| String::from_utf8_lossy(&b).into_owned()))
}
pub fn card_and_build(&self) -> Result<String, JsValue> {
let (card, build) =
rete_core::read_card_and_build_info_ranged(&self.reader.view()).map_err(err)?;
let text_index = text_index_json(self.rete.header(), None);
Ok(card_build_envelope(card, build, text_index))
}
pub fn schema(&self) -> Result<String, JsValue> {
use serde_json::json;
let (classes, relations) = rete_core::read_schema_summary_ranged(&self.reader.view())
.map_err(err)?
.ok_or_else(|| js_error("file has no schema pyramid"))?;
let classes: Vec<serde_json::Value> = classes.iter().map(|(c, n)| json!([c, n])).collect();
let relations: Vec<serde_json::Value> = relations
.iter()
.map(|(s, p, o, n)| json!([s, p, o, n]))
.collect();
Ok(json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "schema",
"classes": classes,
"relations": relations,
})
.to_string())
}
pub fn info(&self) -> String {
let h = self.rete.header();
format!(
r#"{{"schemaVersion":1,"quads":{},"terms":{},"pyramidLevels":{},"namedGraphs":{}}}"#,
h.quad_count,
h.term_count,
h.pyramid_levels,
self.rete.named_graph_count()
)
}
pub fn graph_names(&self) -> Result<String, JsValue> {
serde_json::to_string(&self.rete.graph_names()).map_err(err)
}
pub fn shacl(
&self,
shapes_turtle: &str,
graph: Option<String>,
format: &str,
) -> Result<String, JsValue> {
self.rete.reset_load_failures();
let out = shacl_rete(&self.rete, shapes_turtle, graph.as_deref(), format)?;
incomplete_guard(&self.rete, "validation")?;
Ok(out)
}
}
fn prefix_search_json(rete: &Rete, prefix: &str, limit: usize) -> Result<String, JsValue> {
use serde_json::json;
let hits: Vec<serde_json::Value> = rete
.prefix_search(prefix, limit)
.into_iter()
.map(|(label, subject)| json!({ "label": label, "subject": subject }))
.collect();
serde_json::to_string(&hits).map_err(err)
}
fn text_search_json(
rete: &Rete,
words: &[String],
contains_prefix: Option<&str>,
limit: usize,
) -> Result<String, JsValue> {
use serde_json::json;
let word_refs: Vec<&str> = words.iter().map(String::as_str).collect();
let hits: Vec<serde_json::Value> = rete
.text_search(&word_refs, contains_prefix, limit)
.into_iter()
.map(|subject| json!({ "subject": subject }))
.collect();
serde_json::to_string(&hits).map_err(err)
}
#[wasm_bindgen]
pub fn header_ranges(head: &[u8]) -> Result<String, JsValue> {
let h = Header::from_bytes(head).map_err(err)?;
Ok(format!(
r#"{{"schemaVersion":1,"dictOffset":{},"dictLen":{},"pyramidOffset":{},"pyramidLen":{},"indexOffset":{},"indexLen":{},"metadataOffset":{},"metadataLen":{}}}"#,
h.dictionary_offset,
h.dictionary_len,
h.pyramid_meta_offset,
h.pyramid_meta_len,
h.root_dir_offset,
h.root_dir_len,
h.metadata_offset,
h.metadata_len,
))
}
#[wasm_bindgen]
pub fn summary_overview(bytes: &[u8]) -> Result<String, JsValue> {
let view = SummaryView::open_ranged(&SliceReader::new(bytes))
.map_err(err)?
.ok_or_else(|| js_error("file has no pyramid summary"))?;
serde_json::to_string(&serde_json::json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"round": view.round,
"communities": view.community_count(),
"predicateTotals": view.predicate_totals(),
}))
.map_err(err)
}
#[wasm_bindgen]
pub fn progressive_query(bytes: &[u8], query: &str) -> Result<String, JsValue> {
progressive_query_json(bytes, query).map_err(js_error)
}
pub fn progressive_query_json(bytes: &[u8], query: &str) -> Result<String, String> {
use serde_json::json;
let shape = summary_query_shape(query)
.map_err(|e| e.to_string())?
.ok_or_else(|| "query is not exactly answerable from the summary".to_string())?;
let reader = CountingReader::new(SliceReader::new(bytes));
let file_bytes = reader.len();
let view = SummaryView::open_ranged(&reader)
.map_err(|e| e.to_string())?
.ok_or_else(|| "file has no pyramid summary".to_string())?;
let value = match shape {
SummaryQueryShape::PredicateCount {
predicate,
variable,
} => {
let count = u64::from(view.predicate_total(&predicate));
select_count_response(
&variable,
count,
progressive_meta(
&reader,
file_bytes,
"predicate_count",
Some(&predicate),
json!(count),
),
)
}
SummaryQueryShape::TripleCount { variable } => {
let count = summary_total(&view);
select_count_response(
&variable,
count,
progressive_meta(&reader, file_bytes, "triple_count", None, json!(count)),
)
}
SummaryQueryShape::PredicateTotals {
predicate_variable,
count_variable,
} => {
let totals = view.predicate_totals();
select_predicate_totals_response(
&predicate_variable,
&count_variable,
&totals,
progressive_meta(
&reader,
file_bytes,
"predicate_totals",
None,
json!(&totals),
),
)
}
SummaryQueryShape::PredicateList { variable } => {
let predicates = predicate_list(&view);
select_predicate_list_response(
&variable,
&predicates,
progressive_meta(
&reader,
file_bytes,
"predicate_list",
None,
json!(&predicates),
),
)
}
SummaryQueryShape::PredicateDistinctCount { variable } => {
let count = predicate_count(&view);
select_count_response(
&variable,
count,
progressive_meta(
&reader,
file_bytes,
"predicate_distinct_count",
None,
json!(count),
),
)
}
SummaryQueryShape::TripleExists => {
let exists = summary_total(&view) > 0;
json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "ask",
"boolean": exists,
"progressive": progressive_meta(
&reader,
file_bytes,
"triple_exists",
None,
json!(exists),
),
})
}
SummaryQueryShape::PredicateExists { predicate } => {
let exists = view.predicate_total(&predicate) > 0;
json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "ask",
"boolean": exists,
"progressive": progressive_meta(
&reader,
file_bytes,
"predicate_exists",
Some(&predicate),
json!(exists),
),
})
}
_ => return Err("summary query shape is not supported by this WASM build".to_string()),
};
serde_json::to_string(&value).map_err(|e| e.to_string())
}
#[wasm_bindgen]
pub fn query(bytes: &[u8], query: &str, format: &str) -> Result<String, JsValue> {
let rete = open(bytes)?;
query_json(&rete, query, format, "")
}
#[wasm_bindgen]
pub fn prefix_search(bytes: &[u8], prefix: &str, limit: usize) -> Result<String, JsValue> {
let rete = open(bytes)?;
prefix_search_json(&rete, prefix, limit)
}
#[wasm_bindgen]
pub fn text_search(
bytes: &[u8],
words: Vec<String>,
contains_prefix: Option<String>,
limit: usize,
) -> Result<String, JsValue> {
let rete = open(bytes)?;
text_search_json(&rete, &words, contains_prefix.as_deref(), limit)
}
fn query_json(rete: &Rete, query: &str, format: &str, extra: &str) -> Result<String, JsValue> {
query_json_opt(rete, query, format, extra, false)
}
fn query_json_reasoned(
rete: &Rete,
query: &str,
format: &str,
extra: &str,
) -> Result<String, JsValue> {
query_json_opt(rete, query, format, extra, true)
}
fn query_json_opt(
rete: &Rete,
query: &str,
format: &str,
extra: &str,
reason: bool,
) -> Result<String, JsValue> {
query_json_with(rete, query, format, extra, reason, false)
}
fn query_json_with(
rete: &Rete,
query: &str,
format: &str,
extra: &str,
reason: bool,
union_default: bool,
) -> Result<String, JsValue> {
let (out, warnings) = eval_query_with_warnings(
rete,
query,
QueryOpts {
reason,
union_default_graph: union_default,
},
)
.map_err(err)?;
if warnings.is_empty() {
return write_query_json(&out, format, extra).map_err(js_error);
}
let mut extra = extra.to_string();
extra.push_str(r#","warnings":"#);
extra.push_str(&rete_core::warnings_json(&warnings));
write_query_json(&out, format, &extra).map_err(js_error)
}
fn write_query_json(out: &QueryOutput, format: &str, extra: &str) -> Result<String, String> {
let mut versioned_extra = format!(r#","schemaVersion":{JSON_SCHEMA_VERSION}"#);
versioned_extra.push_str(extra);
if let QueryOutput::Construct(triples) = out {
let (base, syntax) = construct_format(format)?;
let text = match base {
"ttl" => Some(("ttl", to_turtle(triples, syntax)?)),
"jsonld" => Some(("jsonld", to_jsonld(triples)?)),
_ => None,
};
if let Some((fmt, text)) = text {
let mut s = String::from(r#"{"kind":"construct","format":""#);
s.push_str(fmt);
s.push_str(r#"","text":"#);
rete_core::push_json_string(&mut s, &text);
s.push_str(&versioned_extra);
s.push('}');
return Ok(s);
}
}
Ok(rete_core::results_envelope_json(out, &versioned_extra))
}
fn construct_format(
format: &str,
) -> Result<(&str, rete_core::ingest::QuotedTripleSurface), String> {
use rete_core::ingest::QuotedTripleSurface;
let mut parts = format.split(';');
let base = parts.next().unwrap_or("").trim();
let mut syntax = QuotedTripleSurface::Rdf12;
for param in parts {
let (key, value) = param.split_once('=').unwrap_or((param, ""));
match key.trim() {
"quotedTripleSyntax" => {
syntax = QuotedTripleSurface::parse(value.trim()).ok_or_else(|| {
format!(
"unknown quotedTripleSyntax {:?} in format {format:?} (expected \"rdf12\" \
or \"rdf-star\")",
value.trim()
)
})?;
}
other => {
return Err(format!(
"unknown format parameter {other:?} in {format:?} (the one parameter is \
quotedTripleSyntax=rdf12|rdf-star)"
))
}
}
}
Ok((base, syntax))
}
const LOCAL_SCHEME: &str = "rete-local:";
fn is_local_url(url: &str) -> bool {
url.starts_with(LOCAL_SCHEME)
}
thread_local! {
static LOCAL_BLOBS: std::cell::RefCell<std::collections::HashMap<String, web_sys::Blob>> =
std::cell::RefCell::new(std::collections::HashMap::new());
}
#[wasm_bindgen]
pub fn register_local_file(url: &str, blob: &web_sys::Blob) -> Result<(), JsValue> {
if !is_local_url(url) {
return Err(js_error(format!(
"a local file URL must start with `{LOCAL_SCHEME}` (got {url})"
)));
}
LOCAL_BLOBS.with(|map| map.borrow_mut().insert(url.to_string(), blob.clone()));
Ok(())
}
#[wasm_bindgen]
pub fn forget_local_file(url: &str) -> bool {
LOCAL_BLOBS.with(|map| map.borrow_mut().remove(url).is_some())
}
fn local_blob(url: &str) -> std::io::Result<web_sys::Blob> {
LOCAL_BLOBS
.with(|map| map.borrow().get(url).cloned())
.ok_or_else(|| {
std::io::Error::other(format!(
"no local file is registered as {url} — the page must call \
register_local_file(url, file) in THIS worker before opening it"
))
})
}
fn read_local_ranges(url: &str, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
let (offsets, lens, total) = checked_async_layout(ranges)?;
if total == 0 {
return Ok(ranges.iter().map(|_| Vec::new()).collect());
}
let blob = local_blob(url)?;
let size = blob.size();
let reader = web_sys::FileReaderSync::new().map_err(|e| {
std::io::Error::other(format!(
"FileReaderSync is unavailable ({e:?}) — a local .rete is read from a Web Worker only, \
never the main thread"
))
})?;
let mut out = Vec::with_capacity(ranges.len());
for (&offset, &len) in offsets.iter().zip(&lens) {
let start = offset as f64;
let end = start + f64::from(len);
if end > size {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
format!("range {offset}..{end} is past the end of {url} ({size} bytes)"),
));
}
let slice = blob
.slice_with_f64_and_f64(start, end)
.map_err(|e| std::io::Error::other(format!("Blob.slice failed on {url}: {e:?}")))?;
let buffer = reader
.read_as_array_buffer(&slice)
.map_err(|e| std::io::Error::other(format!("FileReaderSync failed on {url}: {e:?}")))?;
let bytes = js_sys::Uint8Array::new(&buffer).to_vec();
if bytes.len() != len as usize {
return Err(std::io::Error::other(format!(
"short local read: got {} of {len} bytes at offset {offset} from {url}",
bytes.len()
)));
}
out.push(bytes);
}
report_progress(total, ranges);
Ok(out)
}
struct XhrRangeReader {
url: String,
len: u64,
head: std::sync::OnceLock<Vec<u8>>,
}
#[cfg(feature = "asyncify")]
#[link(wasm_import_module = "env")]
extern "C" {
fn rete_fetch_ranges(
url_ptr: *const u8,
url_len: usize,
offs_ptr: *const u64,
lens_ptr: *const u32,
n: usize,
dst_ptr: *mut u8,
) -> usize;
fn rete_file_len(url_ptr: *const u8, url_len: usize, out_ptr: *mut u64) -> usize;
fn rete_panic_report(file_ptr: *const u8, file_len: usize, line: u32);
}
fn checked_async_layout(ranges: &[(u64, u64)]) -> std::io::Result<(Vec<u64>, Vec<u32>, usize)> {
let offs: Vec<u64> = ranges.iter().map(|&(offset, _)| offset).collect();
let lens: Vec<u32> = ranges
.iter()
.map(|&(_, len)| {
u32::try_from(len).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("range length {len} exceeds the wasm32 u32 length type"),
)
})
})
.collect::<std::io::Result<_>>()?;
let total = lens.iter().try_fold(0usize, |sum, &len| {
sum.checked_add(len as usize).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"range length sum overflow",
)
})
})?;
let _total_u32 = u32::try_from(total).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"range response exceeds the wasm32 address space",
)
})?;
Ok((offs, lens, total))
}
fn split_range_response(
ranges: &[(u64, u64)],
dst: Vec<u8>,
got: usize,
source: &str,
) -> std::io::Result<Vec<Vec<u8>>> {
let (_, lens, total) = checked_async_layout(ranges)?;
if dst.len() != total {
return Err(std::io::Error::other(format!(
"async fetch destination has {} bytes, expected {total} for {source}",
dst.len()
)));
}
if got != total {
return Err(std::io::Error::other(format!(
"async fetch returned {got} of {total} bytes for {source}"
)));
}
let mut out = Vec::with_capacity(ranges.len());
let mut pos = 0usize;
for len in lens {
let end = pos + len as usize;
out.push(dst[pos..end].to_vec());
pos = end;
}
Ok(out)
}
#[cfg(test)]
mod async_range_tests {
use super::{checked_async_layout, split_range_response};
#[test]
fn rejects_a_range_larger_than_the_wasm32_length_type() {
let error = checked_async_layout(&[(0, u64::from(u32::MAX) + 1)]).unwrap_err();
assert!(error.to_string().contains("u32"));
}
#[test]
fn rejects_a_total_larger_than_the_wasm32_address_space() {
let error = checked_async_layout(&[(0, u64::from(u32::MAX)), (0, 1)]).unwrap_err();
assert!(error.to_string().contains("address space"));
}
#[test]
fn accepts_no_ranges_and_preserves_zero_length_ranges() {
let (offs, lens, total) = checked_async_layout(&[]).unwrap();
assert!(offs.is_empty());
assert!(lens.is_empty());
assert_eq!(total, 0);
let (_, lens, total) = checked_async_layout(&[(7, 0), (9, 0)]).unwrap();
assert_eq!(lens, vec![0, 0]);
assert_eq!(total, 0);
}
#[test]
fn rejects_a_short_javascript_response() {
let error = split_range_response(&[(0, 4)], vec![1, 2, 3, 4], 3, "fixture").unwrap_err();
assert!(error.to_string().contains("returned 3 of 4 bytes"));
}
#[test]
fn only_the_local_scheme_routes_to_the_blob_transport() {
use super::is_local_url;
assert!(is_local_url("rete-local:1/mirbase.rete"));
assert!(!is_local_url("https://example.org/rete-local:1/x.rete"));
assert!(!is_local_url(
"https://data.graphplaza.com/mirbase/mirbase.rete"
));
assert!(!is_local_url("file:///tmp/x.rete"));
assert!(!is_local_url(""));
}
}
#[cfg(feature = "asyncify")]
impl XhrRangeReader {
fn read_ranges_async(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
let (offs, lens, total) = checked_async_layout(ranges)?;
if total == 0 {
return Ok(ranges.iter().map(|_| Vec::new()).collect());
}
let mut dst = vec![0u8; total];
if offs.len() != ranges.len() || lens.len() != ranges.len() || dst.len() != total {
return Err(std::io::Error::other(
"async range layout does not match its allocated buffers",
));
}
let got = unsafe {
rete_fetch_ranges(
self.url.as_ptr(),
self.url.len(),
offs.as_ptr(),
lens.as_ptr(),
ranges.len(),
dst.as_mut_ptr(),
)
};
let out = split_range_response(ranges, dst, got, &self.url)?;
report_progress(total, ranges);
Ok(out)
}
}
impl XhrRangeReader {
fn open(url: &str) -> Result<Self, JsValue> {
if is_local_url(url) {
let len = local_blob(url).map_err(|e| js_error(e.to_string()))?.size();
if !len.is_finite() || len < 1.0 {
return Err(js_error(format!("{url} is an empty local file")));
}
return Ok(Self {
url: url.to_string(),
len: len as u64,
head: std::sync::OnceLock::new(),
});
}
#[cfg(feature = "asyncify")]
{
let mut len: u64 = 0;
let ok = unsafe { rete_file_len(url.as_ptr(), url.len(), &mut len as *mut u64) };
if ok == 0 || len == 0 {
return Err(js_error(format!("could not determine length of {url}")));
}
Ok(Self {
url: url.to_string(),
len,
head: std::sync::OnceLock::new(),
})
}
#[cfg(not(feature = "asyncify"))]
{
let mut last = format!("could not determine length of {url}");
for _ in 0..4 {
match Self::probe_len(url) {
Ok((len, head)) => {
let cached = std::sync::OnceLock::new();
if !head.is_empty() {
let _ = cached.set(head);
}
return Ok(Self {
url: url.to_string(),
len,
head: cached,
});
}
Err(e) => last = e,
}
}
Err(js_error(last))
}
}
fn head_window(&self) -> Option<&[u8]> {
if let Some(head) = self.head.get() {
return Some(head);
}
let head = self.fetch_at(0, rete_core::HEADER_LEN as u64).ok()?;
let _ = self.head.set(head);
self.head.get().map(Vec::as_slice)
}
fn polyglot_base(&self) -> u64 {
let Some(head) = self.head_window() else {
return 0;
};
if head.starts_with(&rete_core::MAGIC) {
return 0;
}
rete_core::detect_polyglot_base(head).unwrap_or(0)
}
#[cfg(not(feature = "asyncify"))]
fn head_len(url: &str) -> Option<u64> {
let xhr = web_sys::XmlHttpRequest::new().ok()?;
xhr.open_with_async("HEAD", url, false).ok()?;
xhr.send().ok()?;
let status = xhr.status().ok()?;
if !(200..300).contains(&status) {
return None;
}
xhr.get_response_header("Content-Length")
.ok()
.flatten()
.and_then(|v| v.parse::<u64>().ok())
.filter(|&n| n > 0)
}
#[cfg(not(feature = "asyncify"))]
fn ranged_get(url: &str, first: u64, last: u64) -> Result<(u16, Option<u64>, Vec<u8>), String> {
let err = |m: &str| m.to_string();
let xhr = web_sys::XmlHttpRequest::new().map_err(|_| err("xhr"))?;
xhr.open_with_async("GET", url, false)
.map_err(|_| err("open"))?;
xhr.set_response_type(web_sys::XmlHttpRequestResponseType::Arraybuffer);
xhr.set_request_header("Range", &format!("bytes={first}-{last}"))
.map_err(|_| err("range header"))?;
xhr.send()
.map_err(|_| format!("probe {url}: network error"))?;
let status = xhr.status().map_err(|_| err("status"))?;
let total = xhr
.get_response_header("Content-Range")
.ok()
.flatten()
.and_then(|v| {
v.rsplit('/')
.next()
.and_then(|t| t.trim().parse::<u64>().ok())
});
let body = xhr
.response()
.ok()
.map(|r| js_sys::Uint8Array::new(&r).to_vec())
.unwrap_or_default();
Ok((status, total, body))
}
#[cfg(not(feature = "asyncify"))]
fn probe_len(url: &str) -> Result<(u64, Vec<u8>), String> {
let (status, cr_total, body) = Self::ranged_get(url, 0, rete_core::HEADER_LEN as u64 - 1)?;
let head = |body: &Vec<u8>| {
if body.len() <= rete_core::HEADER_LEN {
body.clone()
} else {
body[..rete_core::HEADER_LEN].to_vec()
}
};
if status == 200 {
if !body.is_empty() {
return Ok((body.len() as u64, head(&body)));
}
return Err(format!("probe {url}: 200 with an empty body"));
}
if status != 206 {
return Err(format!("probe {url}: status {status}"));
}
if let Ok(header) = rete_core::Header::from_bytes(&body) {
if let Some(derived) = header.expected_file_len() {
if cr_total == Some(derived) {
return Ok((derived, head(&body))); }
let (ts, _, tail) = Self::ranged_get(url, derived - 4, derived - 1)?;
if ts == 206 && tail == rete_core::MAGIC {
return Ok((derived, head(&body)));
}
return Err(format!(
"length probe disagrees with the file: its header derives \
{derived} bytes but the host{} has no RETE footer at \
{}..{} (tail probe: status {ts}) — the file is truncated, \
or the host serves ranges over a compressed representation",
match cr_total {
Some(t) => format!(" reports {t} bytes and"),
None => String::from(" hides Content-Range and"),
},
derived - 4,
derived,
));
}
}
if body.starts_with(&[0x1f, 0x8b]) {
return Err(format!(
"the host serves HTTP ranges over a gzip-compressed representation \
of {url}; range offsets cannot address the file's bytes"
));
}
if let Some(total) = cr_total {
return Ok((total, head(&body)));
}
if let Some(len) = Self::head_len(url) {
return Ok((len, head(&body)));
}
Err(format!("could not determine length of {url}"))
}
}
impl RangeReader for XhrRangeReader {
fn len(&self) -> u64 {
self.len
}
fn concurrency(&self) -> usize {
if is_local_url(&self.url) {
return 16;
}
#[cfg(feature = "asyncify")]
{
16
}
#[cfg(not(feature = "asyncify"))]
{
let has_pool =
js_sys::Reflect::get(&js_sys::global(), &JsValue::from_str("reteReadMany"))
.map(|h| h.is_function())
.unwrap_or(false);
if has_pool {
16
} else {
1
}
}
}
fn read_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
if len == 0 {
return Ok(Vec::new());
}
if let Some(head) = self.head.get() {
if let Some(end) = offset.checked_add(len) {
if end <= head.len() as u64 {
return Ok(head[offset as usize..end as usize].to_vec());
}
}
}
self.fetch_at(offset, len)
}
fn read_many(&self, ranges: &[(u64, u64)]) -> std::io::Result<Vec<Vec<u8>>> {
if is_local_url(&self.url) {
return read_local_ranges(&self.url, ranges);
}
#[cfg(feature = "asyncify")]
{
self.read_ranges_async(ranges)
}
#[cfg(not(feature = "asyncify"))]
{
let (_, _, total) = checked_async_layout(ranges)?;
if ranges.len() > 1 {
if let Some(buf) = self.read_many_via_pool(ranges) {
if buf.len() == total {
return split_range_response(ranges, buf, total, "globalThis.reteReadMany");
}
}
}
ranges.iter().map(|&(o, l)| self.read_at(o, l)).collect()
}
}
}
impl XhrRangeReader {
fn fetch_at(&self, offset: u64, len: u64) -> std::io::Result<Vec<u8>> {
if is_local_url(&self.url) {
return read_local_ranges(&self.url, &[(offset, len)])?
.pop()
.ok_or_else(|| std::io::Error::other("local read returned no range"));
}
#[cfg(feature = "asyncify")]
{
Ok(self
.read_ranges_async(&[(offset, len)])?
.into_iter()
.next()
.unwrap())
}
#[cfg(not(feature = "asyncify"))]
{
let js = |e: JsValue| std::io::Error::other(format!("XHR error: {e:?}"));
let xhr = web_sys::XmlHttpRequest::new().map_err(js)?;
xhr.open_with_async("GET", &self.url, false).map_err(js)?;
xhr.set_response_type(web_sys::XmlHttpRequestResponseType::Arraybuffer);
let end = offset
.checked_add(len - 1)
.ok_or_else(|| std::io::Error::other("HTTP range end overflow"))?;
xhr.set_request_header("Range", &format!("bytes={offset}-{end}"))
.map_err(js)?;
xhr.send().map_err(js)?;
let status = xhr.status().map_err(js)?;
if status != 206 {
return Err(std::io::Error::other(format!(
"server ignored Range (status {status}, expected 206 Partial Content) for {}; \
the host must support HTTP range requests",
self.url
)));
}
let resp = xhr.response().map_err(js)?;
let mut buf = js_sys::Uint8Array::new(&resp).to_vec();
if (buf.len() as u64) < len {
return Err(std::io::Error::other(format!(
"short range response: got {} of {len} bytes at offset {offset} from {}",
buf.len(),
self.url
)));
}
let requested = usize::try_from(len)
.map_err(|_| std::io::Error::other("range exceeds wasm32 memory"))?;
buf.truncate(requested);
report_progress(buf.len(), &[(offset, len)]);
Ok(buf)
}
}
}
const PROGRESS_SPAN_CAP: usize = 256;
fn report_progress(bytes: usize, ranges: &[(u64, u64)]) {
let g = js_sys::global();
if let Ok(f) = js_sys::Reflect::get(&g, &JsValue::from_str("reteProgress")) {
if let Ok(f) = f.dyn_into::<js_sys::Function>() {
let spans = js_sys::Array::new();
for &(offset, len) in ranges.iter().take(PROGRESS_SPAN_CAP) {
let end = offset + len.saturating_sub(1);
spans.push(&JsValue::from_str(&format!("{offset}-{end}")));
}
let _ = f.call3(
&JsValue::NULL,
&JsValue::from_f64(bytes as f64),
&spans.into(),
&JsValue::from_f64(ranges.len() as f64),
);
}
}
}
#[cfg(not(feature = "asyncify"))]
impl XhrRangeReader {
fn read_many_via_pool(&self, ranges: &[(u64, u64)]) -> Option<Vec<u8>> {
let global = js_sys::global();
let hook = js_sys::Reflect::get(&global, &JsValue::from_str("reteReadMany")).ok()?;
if !hook.is_function() {
return None;
}
let hook = hook.dyn_into::<js_sys::Function>().ok()?;
let (_, checked_lens, _) = checked_async_layout(ranges).ok()?;
let count = u32::try_from(ranges.len()).ok()?;
let offs = js_sys::Float64Array::new_with_length(count);
let lens = js_sys::Float64Array::new_with_length(count);
const MAX_SAFE_JS_INTEGER: u64 = (1_u64 << 53) - 1;
for (i, (&(offset, _), &len)) in ranges.iter().zip(&checked_lens).enumerate() {
if offset > MAX_SAFE_JS_INTEGER {
return None;
}
let index = u32::try_from(i).ok()?;
offs.set_index(index, offset as f64);
lens.set_index(index, f64::from(len));
}
let res = hook
.call3(&JsValue::NULL, &JsValue::from_str(&self.url), &offs, &lens)
.ok()?;
if res.is_null() || res.is_undefined() {
return None;
}
Some(js_sys::Uint8Array::new(&res).to_vec())
}
}
struct XhrServiceClient;
impl rete_core::ServiceClient for XhrServiceClient {
fn query(&self, endpoint: &str, query: &str) -> Result<Vec<rete_core::Binding>, String> {
let e = |m: &str| format!("{endpoint}: {m}");
let xhr = web_sys::XmlHttpRequest::new().map_err(|_| e("xhr"))?;
xhr.open_with_async("POST", endpoint, false)
.map_err(|_| e("open"))?;
xhr.set_request_header("Content-Type", "application/x-www-form-urlencoded")
.map_err(|_| e("header"))?;
xhr.set_request_header("Accept", "application/sparql-results+json")
.map_err(|_| e("header"))?;
let body = format!(
"query={}",
String::from(js_sys::encode_uri_component(query))
);
xhr.send_with_opt_str(Some(&body))
.map_err(|_| e("network error (endpoint down or no CORS)"))?;
let status = xhr.status().map_err(|_| e("status"))?;
if !(200..300).contains(&status) {
return Err(e(&format!("HTTP {status}")));
}
let text = xhr
.response_text()
.map_err(|_| e("read"))?
.unwrap_or_default();
rete_core::parse_sparql_json_results(&text).map_err(|m| e(&m))
}
}
fn auto_block(len: u64) -> u64 {
const MB: u64 = 1 << 20;
let mult: u64 = if len > 100 * MB {
8 } else if len > 10 * MB {
4 } else {
2 };
mult * DEFAULT_BLOCK
}
#[derive(Clone)]
struct RemoteReader {
counting: std::sync::Arc<CountingReader<XhrRangeReader>>,
base: u64,
}
impl RemoteReader {
fn open(url: &str) -> Result<Self, JsValue> {
let xhr = XhrRangeReader::open(url)?;
let base = xhr.polyglot_base();
Ok(Self {
counting: std::sync::Arc::new(CountingReader::new(xhr)),
base,
})
}
fn view(&self) -> OffsetReader<std::sync::Arc<CountingReader<XhrRangeReader>>> {
OffsetReader::new(self.counting.clone(), self.base)
}
fn len(&self) -> u64 {
self.counting.len().saturating_sub(self.base)
}
fn bytes_read(&self) -> u64 {
self.counting.bytes_read()
}
fn requests(&self) -> u64 {
self.counting.requests()
}
}
fn open_url(url: &str) -> Result<(RemoteReader, Rete), JsValue> {
let reader = RemoteReader::open(url)?;
let cached = std::sync::Arc::new(BlockCacheReader::new(
reader.counting.clone(),
auto_block(reader.len()),
));
let mut rete = Rete::open_ranged_lazy(OffsetReader::new(cached, reader.base)).map_err(err)?;
rete.set_service_client(Box::new(XhrServiceClient));
Ok((reader, rete))
}
fn incomplete_guard(rete: &Rete, what: &str) -> Result<(), JsValue> {
if rete.index_incomplete() {
return Err(js_error(format!(
"a range fetch failed mid-{what}; refusing to return incomplete results"
)));
}
Ok(())
}
#[wasm_bindgen]
pub fn why_url(
url: &str,
subject: Option<String>,
predicate: Option<String>,
object: Option<String>,
) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let out = why_triples_rete(
&rete,
subject.as_deref(),
predicate.as_deref(),
object.as_deref(),
)
.map_err(js_error)?;
incomplete_guard(&rete, "query")?;
let _ = reader;
Ok(out)
}
fn why_triples_rete(
rete: &Rete,
subject: Option<&str>,
predicate: Option<&str>,
object: Option<&str>,
) -> Result<String, String> {
use serde_json::json;
let results = rete.query_with_provenance(subject, predicate, object);
let out = json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"pattern": { "subject": subject, "predicate": predicate, "object": object },
"resultCount": results.len(),
"results": results.iter().map(provenance_json).collect::<Vec<_>>(),
});
serde_json::to_string(&out).map_err(|e| e.to_string())
}
#[wasm_bindgen]
pub fn sparql_url(url: &str, query: &str, format: &str) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let file_length = reader.len();
let out = eval_query(&rete, query).map_err(err)?;
if rete.index_incomplete() {
return Err(js_error(
"a range fetch failed mid-query; refusing to return incomplete results",
));
}
let extra = format!(
r#","remote":{{"fileLength":{},"bytes":{},"requests":{}}}"#,
file_length,
reader.bytes_read(),
reader.requests(),
);
write_query_json(&out, format, &extra).map_err(js_error)
}
#[wasm_bindgen]
pub fn query_communities(
bytes: &[u8],
query: &str,
round: Option<usize>,
) -> Result<String, JsValue> {
query_communities_value(&open(bytes)?, query, round)
}
fn query_communities_value(
rete: &Rete,
query: &str,
round: Option<usize>,
) -> Result<String, JsValue> {
use serde_json::{json, Map, Value};
let (mut vars, solutions, partials) =
eval_select_communities(rete, query, round).map_err(err)?;
if vars.is_empty() {
let mut seen = std::collections::BTreeSet::new();
for s in &solutions {
for k in s.keys() {
if seen.insert(k.clone()) {
vars.push(k.clone());
}
}
}
}
let rows: Vec<Value> = solutions
.iter()
.map(|s| {
let mut obj = Map::new();
for var in &vars {
if let Some(term) = s.get(var) {
obj.insert(var.clone(), Value::String(term.clone()));
}
}
Value::Object(obj)
})
.collect();
let communities: Vec<Value> = partials
.iter()
.map(|p| json!({ "community": p.community, "subjects": p.subjects, "rows": p.rows }))
.collect();
serde_json::to_string(&json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "select",
"vars": vars,
"rows": rows,
"communities": communities,
}))
.map_err(err)
}
#[wasm_bindgen]
pub fn file_layout(bytes: &[u8]) -> Result<String, JsValue> {
use serde_json::json;
let rete = open(bytes)?;
let segments: Vec<serde_json::Value> = rete
.file_layout()
.iter()
.map(|s| json!({ "kind": s.kind, "label": s.label, "offset": s.offset, "len": s.len }))
.collect();
serde_json::to_string(&json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"fileLength": bytes.len(),
"segments": segments,
}))
.map_err(err)
}
#[wasm_bindgen]
pub fn pyramid_tree(bytes: &[u8]) -> Result<String, JsValue> {
pyramid_tree_value(&open(bytes)?)
}
fn pyramid_tree_value(rete: &Rete) -> Result<String, JsValue> {
use serde_json::json;
use std::collections::BTreeMap;
let dict = rete.dictionary();
let ids = rete.match_ids((None, None, None));
let g = project_graph(dict, &ids);
let dend = build_dendrogram(&g);
let rounds = dend.rounds();
if rounds == 0 {
return serde_json::to_string(&json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"rounds": 0,
"levels": [],
}))
.map_err(err);
}
let n = g.node_count();
let mut levels = Vec::with_capacity(rounds);
for r in 0..rounds {
let mut nodes_per: BTreeMap<usize, usize> = BTreeMap::new();
let mut rep: BTreeMap<usize, usize> = BTreeMap::new();
for node in 0..n {
let c = dend.base_community(node, r);
*nodes_per.entry(c).or_default() += 1;
rep.entry(c).or_insert(node);
}
let mut triples_per: BTreeMap<usize, usize> = BTreeMap::new();
for &(s, _, _) in &ids {
let c = dend.base_community(dict.subject_node(s) as usize, r);
*triples_per.entry(c).or_default() += 1;
}
let level: Vec<serde_json::Value> = nodes_per
.iter()
.map(|(&c, &nodes)| {
let parent = if r + 1 < rounds {
json!(dend.base_community(rep[&c], r + 1))
} else {
json!(null)
};
json!({
"id": c,
"nodes": nodes,
"triples": triples_per.get(&c).copied().unwrap_or(0),
"parent": parent,
})
})
.collect();
levels.push(level);
}
serde_json::to_string(&json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"rounds": rounds,
"levels": levels,
}))
.map_err(err)
}
#[wasm_bindgen]
pub fn communities(bytes: &[u8], round: Option<usize>) -> Result<String, JsValue> {
use std::collections::HashSet;
let rete = open(bytes)?;
let dict = rete.dictionary();
let ids = rete.match_ids((None, None, None));
let g = project_graph(dict, &ids);
let dend = build_dendrogram(&g);
let round =
round.unwrap_or_else(|| choose_round_for_budget(dict, &ids, &dend, DEFAULT_TILE_BUDGET));
let tiles = tile_by_community(dict, &ids, &dend, round);
let arr: Vec<serde_json::Value> = tiles
.iter()
.map(|t| {
let members: HashSet<u32> = t.triples.iter().map(|&(s, _, _)| s).collect();
serde_json::json!({
"community": t.community,
"size": members.len(),
"triples": t.triples.len(),
})
})
.collect();
serde_json::to_string(&arr).map_err(err)
}
#[wasm_bindgen]
pub fn reach(bytes: &[u8], predicate: &str, seeds: &str, reverse: bool) -> Result<String, JsValue> {
reach_rete(&open(bytes)?, predicate, seeds, reverse)
}
#[wasm_bindgen]
pub fn reach_url(
url: &str,
predicate: &str,
seeds: &str,
reverse: bool,
) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let out = reach_rete(&rete, predicate, seeds, reverse)?;
incomplete_guard(&rete, "reach")?;
let _ = reader;
Ok(out)
}
fn reach_rete(rete: &Rete, predicate: &str, seeds: &str, reverse: bool) -> Result<String, JsValue> {
use std::collections::HashMap;
let dict = rete.dictionary();
let seed_iris: Vec<String> = match serde_json::from_str::<Vec<String>>(seeds) {
Ok(v) => v,
Err(_) => match serde_json::from_str::<String>(seeds) {
Ok(s) => vec![s],
Err(_) => vec![seeds.to_string()],
},
};
let adj: HashMap<u32, Vec<u32>> = if reverse {
let mut m: HashMap<u32, Vec<u32>> = HashMap::new();
for (s, o) in rete.predicate_pairs(predicate) {
m.entry(o).or_default().push(s); }
m
} else {
build_adjacency(rete, predicate)
};
let mut known_idx: Vec<usize> = Vec::new();
let mut seed_nodes: Vec<u32> = Vec::new();
for (i, iri) in seed_iris.iter().enumerate() {
if let Some(n) = dict.node_of_term(iri) {
known_idx.push(i);
seed_nodes.push(n);
}
}
let sets = batch_reach_serial(&adj, &seed_nodes);
let mut by_input: HashMap<usize, &std::collections::BTreeSet<u32>> = HashMap::new();
for (k, &i) in known_idx.iter().enumerate() {
by_input.insert(i, &sets[k]);
}
let arr: Vec<serde_json::Value> = seed_iris
.iter()
.enumerate()
.map(|(i, iri)| match by_input.get(&i) {
Some(set) => {
let reached: Vec<String> = set.iter().filter_map(|&n| dict.node_term(n)).collect();
serde_json::json!({
"seed": iri,
"count": reached.len(),
"reached": reached,
})
}
None => serde_json::json!({ "seed": iri, "error": "not in graph" }),
})
.collect();
serde_json::to_string(&arr).map_err(err)
}
#[wasm_bindgen]
pub fn shacl(
bytes: &[u8],
shapes_turtle: &str,
graph: Option<String>,
format: &str,
) -> Result<String, JsValue> {
shacl_rete(&open(bytes)?, shapes_turtle, graph.as_deref(), format)
}
#[wasm_bindgen]
pub fn shacl_url(
url: &str,
shapes_turtle: &str,
graph: Option<String>,
format: &str,
) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let out = shacl_rete(&rete, shapes_turtle, graph.as_deref(), format)?;
incomplete_guard(&rete, "validation")?;
let _ = reader;
Ok(out)
}
fn shacl_rete(
rete: &Rete,
shapes_turtle: &str,
graph: Option<&str>,
format: &str,
) -> Result<String, JsValue> {
let shapes = ShaclShapes::parse_turtle(shapes_turtle).map_err(err)?;
let report = match graph {
None => validate_shacl(&ReteGraph::new(rete), &shapes),
Some(g) => validate_shacl(&DataGraph::from_rete(rete, Some(g)), &shapes),
};
Ok(format_report(&report, format))
}
fn shacl_over(data: DataGraph, shapes_turtle: &str, format: &str) -> Result<String, JsValue> {
let shapes = ShaclShapes::parse_turtle(shapes_turtle).map_err(err)?;
Ok(format_report(&validate_shacl(&data, &shapes), format))
}
fn format_report(report: &ValidationReport, format: &str) -> String {
match format {
"json" => report.to_json(),
"ttl" => report.to_turtle(),
_ => format_shacl_text(report),
}
}
#[wasm_bindgen]
pub fn shacl_construct_url(
url: &str,
construct: &str,
shapes_turtle: &str,
format: &str,
) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let triples = match eval_query(&rete, construct).map_err(err)? {
QueryOutput::Construct(t) => t,
_ => {
return Err(js_error(
"the subset query must be a CONSTRUCT — it builds the subgraph SHACL validates",
))
}
};
incomplete_guard(&rete, "query")?;
let out = shacl_over(DataGraph::from_triples(triples), shapes_turtle, format)?;
let _ = reader;
Ok(out)
}
pub const COHERENCE_CONSTRUCT: &str = r#"
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>
CONSTRUCT {
?x rdf:type ?c .
?sub rdfs:subClassOf ?sup .
?c1 owl:disjointWith ?c2 .
?s1 owl:sameAs ?s2 .
?f1 owl:differentFrom ?f2 .
}
WHERE {
{ ?x rdf:type ?c }
UNION { ?sub rdfs:subClassOf ?sup }
UNION { ?c1 owl:disjointWith ?c2 }
UNION { ?s1 owl:sameAs ?s2 }
UNION { ?f1 owl:differentFrom ?f2 }
}
"#;
fn reasoning_json(r: &rete_core::Reasoning, remote: Option<(u64, u64, u64)>) -> String {
use serde_json::json;
let inconsistencies: Vec<serde_json::Value> = r
.inconsistencies
.iter()
.map(|i| json!({ "kind": i.kind, "detail": i.detail }))
.collect();
let mut v = json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "reasoning",
"coherent": r.inconsistencies.is_empty(),
"inferredCount": r.inferred.len(),
"inconsistencies": inconsistencies,
});
if let (Some(obj), Some((file_length, bytes, requests))) = (v.as_object_mut(), remote) {
obj.insert(
"remote".to_string(),
json!({ "fileLength": file_length, "bytes": bytes, "requests": requests }),
);
}
v.to_string()
}
#[wasm_bindgen]
pub fn reason(bytes: &[u8], graph: Option<String>) -> Result<String, JsValue> {
let rete = open(bytes)?;
let base = rete.dump(graph.as_deref());
let result = rete_core::reason(&base);
Ok(reasoning_json(&result, None))
}
#[wasm_bindgen]
pub fn reason_url(url: &str, graph: Option<String>) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let base = rete.dump(graph.as_deref());
let result = rete_core::reason(&base);
incomplete_guard(&rete, "reasoning")?;
let out = reasoning_json(
&result,
Some((reader.len(), reader.bytes_read(), reader.requests())),
);
let _ = reader;
Ok(out)
}
#[wasm_bindgen]
pub fn reason_construct_url(url: &str, construct: &str) -> Result<String, JsValue> {
let (reader, rete) = open_url(url)?;
let triples =
match eval_query(&rete, construct).map_err(err)? {
QueryOutput::Construct(t) => t,
_ => return Err(js_error(
"the subset query must be a CONSTRUCT — it builds the subgraph the reasoner checks",
)),
};
incomplete_guard(&rete, "query")?;
let result = rete_core::reason(&triples);
let out = reasoning_json(
&result,
Some((reader.len(), reader.bytes_read(), reader.requests())),
);
let _ = reader;
Ok(out)
}
fn schema_coherence_json(
points: &[rete_core::Inconsistency],
remote: Option<(u64, u64, u64)>,
) -> String {
use serde_json::json;
let schema_points: Vec<serde_json::Value> = points
.iter()
.map(|i| json!({ "kind": i.kind, "detail": i.detail }))
.collect();
let mut v = json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "schemaCoherence",
"coherent": points.is_empty(),
"schemaPoints": schema_points,
"readsIndex": false,
});
if let (Some(obj), Some((file_length, bytes, requests))) = (v.as_object_mut(), remote) {
obj.insert(
"remote".to_string(),
json!({ "fileLength": file_length, "bytes": bytes, "requests": requests }),
);
}
v.to_string()
}
#[wasm_bindgen]
pub fn check_schema(bytes: &[u8]) -> Result<String, JsValue> {
let points = rete_core::read_schema_coherence_ranged(&SliceReader::new(bytes))
.map_err(err)?
.ok_or_else(|| js_error("file has no schema pyramid"))?;
Ok(schema_coherence_json(&points, None))
}
#[wasm_bindgen]
pub fn check_schema_url(url: &str) -> Result<String, JsValue> {
let reader = RemoteReader::open(url)?;
let points = rete_core::read_schema_coherence_ranged(&reader.view())
.map_err(err)?
.ok_or_else(|| js_error("file has no schema pyramid"))?;
Ok(schema_coherence_json(
&points,
Some((reader.len(), reader.bytes_read(), reader.requests())),
))
}
#[wasm_bindgen]
pub fn schema_url(url: &str) -> Result<String, JsValue> {
use serde_json::json;
let reader = RemoteReader::open(url)?;
let (classes, relations) = rete_core::read_schema_summary_ranged(&reader.view())
.map_err(err)?
.ok_or_else(|| js_error("file has no schema pyramid"))?;
let classes: Vec<serde_json::Value> = classes.iter().map(|(c, n)| json!([c, n])).collect();
let relations: Vec<serde_json::Value> = relations
.iter()
.map(|(s, p, o, n)| json!([s, p, o, n]))
.collect();
Ok(json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "schema",
"classes": classes,
"relations": relations,
"remote": { "fileLength": reader.len(), "bytes": reader.bytes_read(), "requests": reader.requests() },
})
.to_string())
}
fn format_shacl_text(report: &ValidationReport) -> String {
if report.conforms {
return "conforms: true\n".to_string();
}
let mut out = format!("conforms: false\nresults: {}\n", report.results.len());
for result in &report.results {
out.push_str("\n- focus: ");
out.push_str(&result.focus_node);
if let Some(value) = &result.value_node {
out.push_str("\n value: ");
out.push_str(value);
}
if let Some(path) = &result.result_path {
out.push_str("\n path: ");
out.push_str(path);
}
out.push_str("\n component: ");
out.push_str(&result.source_constraint_component);
out.push_str("\n severity: ");
out.push_str(&result.severity.iri());
for message in &result.messages {
out.push_str("\n message: ");
out.push_str(message);
}
out.push('\n');
}
out
}
#[cfg(feature = "threads")]
pub use wasm_bindgen_rayon::init_thread_pool;
#[cfg(feature = "threads")]
#[wasm_bindgen]
pub fn reach_parallel(
bytes: &[u8],
predicate: &str,
seeds: &str,
reverse: bool,
) -> Result<String, JsValue> {
use std::collections::HashMap;
let rete = open(bytes)?;
let dict = rete.dictionary();
let seed_iris: Vec<String> = match serde_json::from_str::<Vec<String>>(seeds) {
Ok(v) => v,
Err(_) => match serde_json::from_str::<String>(seeds) {
Ok(s) => vec![s],
Err(_) => vec![seeds.to_string()],
},
};
let adj: HashMap<u32, Vec<u32>> = if reverse {
let mut m: HashMap<u32, Vec<u32>> = HashMap::new();
for (s, o) in rete.predicate_pairs(predicate) {
m.entry(o).or_default().push(s); }
m
} else {
build_adjacency(&rete, predicate)
};
let mut known_idx: Vec<usize> = Vec::new();
let mut seed_nodes: Vec<u32> = Vec::new();
for (i, iri) in seed_iris.iter().enumerate() {
if let Some(n) = dict.node_of_term(iri) {
known_idx.push(i);
seed_nodes.push(n);
}
}
let sets = rete_core::parallel::batch_reach_parallel(&adj, &seed_nodes);
let mut by_input: HashMap<usize, &std::collections::BTreeSet<u32>> = HashMap::new();
for (k, &i) in known_idx.iter().enumerate() {
by_input.insert(i, &sets[k]);
}
let arr: Vec<serde_json::Value> = seed_iris
.iter()
.enumerate()
.map(|(i, iri)| match by_input.get(&i) {
Some(set) => {
let reached: Vec<String> = set.iter().filter_map(|&n| dict.node_term(n)).collect();
serde_json::json!({
"seed": iri,
"count": reached.len(),
"reached": reached,
})
}
None => serde_json::json!({ "seed": iri, "error": "not in graph" }),
})
.collect();
serde_json::to_string(&arr).map_err(err)
}
const RDF_TYPE_IRI: &str = "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>";
const XSD_INTEGER_IRI: &str = "http://www.w3.org/2001/XMLSchema#integer";
fn summary_total(view: &SummaryView) -> u64 {
view.summary.iter().map(|edge| u64::from(edge.count)).sum()
}
fn predicate_list(view: &SummaryView) -> Vec<String> {
view.predicate_totals()
.into_iter()
.map(|(predicate, _)| predicate)
.collect()
}
fn predicate_count(view: &SummaryView) -> u64 {
view.predicate_totals().len() as u64
}
fn select_count_response(
variable: &str,
count: u64,
progressive: serde_json::Value,
) -> serde_json::Value {
use serde_json::{json, Map, Value};
let mut row = Map::new();
row.insert(variable.to_string(), Value::String(integer_literal(count)));
json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "select",
"vars": [variable],
"rows": [Value::Object(row)],
"progressive": progressive,
})
}
fn select_predicate_list_response(
variable: &str,
predicates: &[String],
progressive: serde_json::Value,
) -> serde_json::Value {
use serde_json::{json, Map, Value};
let rows: Vec<Value> = predicates
.iter()
.map(|predicate| {
let mut row = Map::new();
row.insert(variable.to_string(), Value::String(predicate.clone()));
Value::Object(row)
})
.collect();
json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "select",
"vars": [variable],
"rows": rows,
"progressive": progressive,
})
}
fn select_predicate_totals_response(
predicate_variable: &str,
count_variable: &str,
totals: &[(String, u32)],
progressive: serde_json::Value,
) -> serde_json::Value {
use serde_json::{json, Map, Value};
let rows: Vec<Value> = totals
.iter()
.map(|(predicate, count)| {
let mut row = Map::new();
row.insert(
predicate_variable.to_string(),
Value::String(predicate.clone()),
);
row.insert(
count_variable.to_string(),
Value::String(integer_literal(u64::from(*count))),
);
Value::Object(row)
})
.collect();
json!({
"schemaVersion": JSON_SCHEMA_VERSION,
"kind": "select",
"vars": [predicate_variable, count_variable],
"rows": rows,
"progressive": progressive,
})
}
fn progressive_meta<R: RangeReader>(
reader: &CountingReader<R>,
file_bytes: u64,
query_shape: &str,
predicate: Option<&str>,
value: serde_json::Value,
) -> serde_json::Value {
use serde_json::{json, Map, Value};
let mut meta = Map::new();
meta.insert("stage".into(), json!("summary"));
meta.insert("exact".into(), json!(true));
meta.insert("readsIndex".into(), json!(false));
meta.insert("queryShape".into(), json!(query_shape));
meta.insert("value".into(), value);
meta.insert("bytes".into(), json!(reader.bytes_read()));
meta.insert("requests".into(), json!(reader.requests()));
meta.insert("fileBytes".into(), json!(file_bytes));
if let Some(predicate) = predicate {
meta.insert("predicate".into(), json!(predicate));
}
Value::Object(meta)
}
fn integer_literal(value: u64) -> String {
format!("\"{}\"^^<{}>", value, XSD_INTEGER_IRI)
}
fn provenance_json(m: &TripleProvenance) -> serde_json::Value {
use serde_json::json;
let tile = match (&m.tile, m.tile_range) {
(Some(id), Some(range)) => json!({
"available": true,
"id": id,
"range": range_json(range),
}),
(Some(id), None) => json!({
"available": true,
"id": id,
}),
_ => json!({
"available": false,
"reason": "not_materialized",
}),
};
json!({
"terms": {
"subject": m.terms.0,
"predicate": m.terms.1,
"object": m.terms.2,
},
"ids": {
"subject": m.ids.0,
"predicate": m.ids.1,
"object": m.ids.2,
},
"provenance": {
"graph": m.graph.as_deref().unwrap_or("default"),
"matchedPattern": {
"subject": m.matched_pattern.0,
"predicate": m.matched_pattern.1,
"object": m.matched_pattern.2,
},
"indexPermutation": m.index_permutation.name(),
"indexSection": m.index_permutation.section_index(),
"dictionaryRange": range_json(m.dictionary_range),
"indexRange": range_json(m.index_range),
"indexSectionRange": range_json(m.index_section_range),
"pyramidRange": m.pyramid_range.map(range_json),
"tile": tile,
},
})
}
fn range_json(range: ByteRange) -> serde_json::Value {
serde_json::json!({
"offset": range.offset,
"len": range.len,
"end": range.end(),
})
}
fn to_turtle(
triples: &[(String, String, String)],
syntax: rete_core::ingest::QuotedTripleSurface,
) -> Result<String, String> {
use std::borrow::Cow;
use std::collections::BTreeMap;
let rdf12 = syntax == rete_core::ingest::QuotedTripleSurface::Rdf12;
let mut by_subject: BTreeMap<&str, BTreeMap<&str, Vec<Cow<'_, str>>>> = BTreeMap::new();
for (s, p, o) in triples {
let o = if rdf12 {
rdf12_object(s, p, o)?
} else {
Cow::Borrowed(o.as_str())
};
by_subject
.entry(s)
.or_default()
.entry(p)
.or_default()
.push(o);
}
let mut out = String::new();
for (s, preds) in &by_subject {
out.push_str(s);
out.push('\n');
let pred_count = preds.len();
for (i, (p, objs)) in preds.iter().enumerate() {
let pred = if *p == RDF_TYPE_IRI { "a" } else { p };
let objects = objs.join(" , ");
let terminator = if i + 1 == pred_count { " ." } else { " ;" };
out.push_str(&format!(" {pred} {objects}{terminator}\n"));
}
out.push('\n');
}
Ok(out)
}
fn to_jsonld(triples: &[(String, String, String)]) -> Result<String, String> {
use rete_core::terms::is_quoted_triple;
use serde_json::{json, Map, Value};
use std::collections::BTreeMap;
if let Some((s, p, o)) = triples
.iter()
.find(|(s, p, o)| is_quoted_triple(s) || is_quoted_triple(p) || is_quoted_triple(o))
{
return Err(format!(
"this CONSTRUCT result contains an RDF-star quoted triple, and expanded JSON-LD has \
no term kind for one.\n\
Written anyway, a quoted triple would be serialized as though it were an IRI — a \
document that loads cleanly and means something else.\n\
statement: {s} {p} {o} .\n\
hint: format \"ttl\" writes it as an RDF 1.2 triple term <<( s p o )>> (add \
\";quotedTripleSyntax=rdf-star\" for <<s p o>>), and the table output lists it as is."
));
}
let mut nodes: BTreeMap<String, BTreeMap<String, Vec<Value>>> = BTreeMap::new();
for (s, p, o) in triples {
let id = node_id(s);
let pred = p
.strip_prefix('<')
.and_then(|x| x.strip_suffix('>'))
.unwrap_or(p)
.to_string();
nodes
.entry(id)
.or_default()
.entry(pred)
.or_default()
.push(object_to_jsonld(o));
}
let arr: Vec<Value> = nodes
.into_iter()
.map(|(id, preds)| {
let mut obj = Map::new();
obj.insert("@id".into(), json!(id));
for (pred, vals) in preds {
obj.insert(pred, Value::Array(vals));
}
Value::Object(obj)
})
.collect();
Ok(serde_json::to_string_pretty(&Value::Array(arr)).unwrap_or_default())
}
fn node_id(token: &str) -> String {
token
.strip_prefix('<')
.and_then(|s| s.strip_suffix('>'))
.map(str::to_string)
.unwrap_or_else(|| token.to_string())
}
fn object_to_jsonld(token: &str) -> serde_json::Value {
use serde_json::{json, Map, Value};
if let Some(iri) = token.strip_prefix('<').and_then(|s| s.strip_suffix('>')) {
return json!({ "@id": iri });
}
if let Some(b) = token.strip_prefix("_:") {
return json!({ "@id": format!("_:{b}") });
}
if token.starts_with('"') {
let bytes = token.as_bytes();
let mut i = 1;
while i < bytes.len() {
match bytes[i] {
b'\\' => i += 2,
b'"' => break,
_ => i += 1,
}
}
let value = unescape_nt(&token[1..i.min(token.len())]);
let rest = token.get(i + 1..).unwrap_or("");
let mut obj = Map::new();
obj.insert("@value".into(), Value::String(value));
if let Some(dt) = rest.strip_prefix("^^<").and_then(|s| s.strip_suffix('>')) {
obj.insert("@type".into(), json!(dt));
} else if let Some(lang) = rest.strip_prefix('@') {
obj.insert("@language".into(), json!(lang));
}
return Value::Object(obj);
}
json!({ "@value": token })
}
fn unescape_nt(s: &str) -> String {
rete_core::terms::unescape_literal(s)
}
fn open(bytes: &[u8]) -> Result<Rete, JsValue> {
let mut rete = Rete::open(bytes).map_err(err)?;
rete.set_service_client(Box::new(XhrServiceClient));
Ok(rete)
}
fn err<E: std::fmt::Display>(e: E) -> JsValue {
js_error(e.to_string())
}
fn js_error(message: impl AsRef<str>) -> JsValue {
js_sys::Error::new(message.as_ref()).into()
}
#[cfg(test)]
mod construct_writer_tests {
use super::{construct_format, to_jsonld, to_turtle, write_query_json};
use rete_core::ingest::QuotedTripleSurface::{Rdf12, RdfStar};
use rete_core::QueryOutput;
const S: &str = "<http://ex/s>";
const P: &str = "<http://ex/p>";
const QT: &str = "<<<http://ex/a> <http://ex/b> <http://ex/c>>>";
fn t(s: &str, p: &str, o: &str) -> (String, String, String) {
(s.to_string(), p.to_string(), o.to_string())
}
fn plain() -> Vec<(String, String, String)> {
vec![
t(S, P, "\"a > b << c\"@en"),
t(
S,
"<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>",
"<http://ex/T>",
),
t(S, P, "_:b0"),
t(
"_:b0",
P,
"\"5\"^^<http://www.w3.org/2001/XMLSchema#integer>",
),
]
}
fn legacy_turtle(triples: &[(String, String, String)]) -> String {
use std::collections::BTreeMap;
let mut by_subject: BTreeMap<&str, BTreeMap<&str, Vec<&str>>> = BTreeMap::new();
for (s, p, o) in triples {
by_subject
.entry(s)
.or_default()
.entry(p)
.or_default()
.push(o);
}
let mut out = String::new();
for (s, preds) in &by_subject {
out.push_str(s);
out.push('\n');
let pred_count = preds.len();
for (i, (p, objs)) in preds.iter().enumerate() {
let pred = if *p == super::RDF_TYPE_IRI { "a" } else { p };
let objects = objs.join(" , ");
let terminator = if i + 1 == pred_count { " ." } else { " ;" };
out.push_str(&format!(" {pred} {objects}{terminator}\n"));
}
out.push('\n');
}
out
}
#[test]
fn turtle_without_quoted_triples_is_byte_identical_under_both_surfaces() {
let g = plain();
let before = legacy_turtle(&g);
assert_eq!(to_turtle(&g, Rdf12).unwrap(), before);
assert_eq!(to_turtle(&g, RdfStar).unwrap(), before);
}
#[test]
fn turtle_writes_an_object_triple_term_in_the_rdf12_surface_by_default() {
let g = vec![t(S, P, QT)];
let out = to_turtle(&g, Rdf12).unwrap();
assert!(
out.contains("<<( <http://ex/a> <http://ex/b> <http://ex/c> )>>"),
"{out}"
);
assert!(!out.contains("<<<"), "the legacy surface leaked: {out}");
let nested = format!("<<<http://ex/x> <http://ex/y> {QT}>>");
let out = to_turtle(&[t(S, P, &nested)], Rdf12).unwrap();
assert!(
out.contains(
"<<( <http://ex/x> <http://ex/y> <<( <http://ex/a> <http://ex/b> <http://ex/c> )>> )>>"
),
"{out}"
);
}
#[test]
fn turtle_output_reads_back_as_the_same_graph_under_rdf12() {
let g = vec![
t(S, P, QT),
t(S, P, "<http://ex/o>"),
t(S, "<http://ex/q>", QT),
];
let out = to_turtle(&g, Rdf12).unwrap();
let mut quads = rete_core::ingest::parse_statements_audited_surface(
&out,
"ttl",
None,
rete_core::ingest::QuotedTripleSurface::Rdf12,
)
.unwrap();
quads.sort();
let mut want: Vec<_> = g
.iter()
.map(|(s, p, o)| (s.clone(), p.clone(), o.clone()))
.collect();
want.sort();
let got: Vec<_> = quads
.iter()
.map(|q| (q.0.clone(), q.1.clone(), q.2.clone()))
.collect();
assert_eq!(got, want, "{out}");
}
#[test]
fn turtle_rdf_star_keeps_the_stored_token() {
let out = to_turtle(&[t(S, P, QT)], RdfStar).unwrap();
assert_eq!(out, legacy_turtle(&[t(S, P, QT)]));
assert!(out.contains(QT));
}
#[test]
fn turtle_refuses_a_subject_position_quoted_triple_by_name() {
let e = to_turtle(&[t(QT, P, "<http://ex/o>")], Rdf12).unwrap_err();
assert!(e.contains("quoted triple in the subject position"), "{e}");
assert!(e.contains("OBJECT position only"), "{e}");
assert!(e.contains("rdf-star"), "the hint names the escape: {e}");
let inner_subject = format!("<<{QT} <http://ex/y> <http://ex/z>>>");
let e = to_turtle(&[t(S, P, &inner_subject)], Rdf12).unwrap_err();
assert!(e.contains("nested in the SUBJECT"), "{e}");
assert!(to_turtle(&[t(QT, P, "<http://ex/o>")], RdfStar).is_ok());
}
#[test]
fn jsonld_refuses_a_quoted_triple_in_any_position() {
for g in [vec![t(S, P, QT)], vec![t(QT, P, "<http://ex/o>")]] {
let e = to_jsonld(&g).unwrap_err();
assert!(e.contains("expanded JSON-LD has no term kind"), "{e}");
assert!(e.contains("<<( s p o )>>"), "{e}");
}
}
#[test]
fn jsonld_without_quoted_triples_is_unchanged() {
let out = to_jsonld(&plain()).unwrap();
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
assert_eq!(v.as_array().unwrap().len(), 2);
assert!(out.contains(r#""@value": "a > b << c""#), "{out}");
}
#[test]
fn the_format_parameter_mirrors_the_cli_flag() {
assert_eq!(construct_format("ttl").unwrap(), ("ttl", Rdf12));
assert_eq!(
construct_format("ttl;quotedTripleSyntax=rdf-star").unwrap(),
("ttl", RdfStar)
);
assert_eq!(
construct_format("ttl;quotedTripleSyntax=rdf12").unwrap(),
("ttl", Rdf12)
);
assert!(construct_format("ttl;quotedTripleSyntax=rdfstar")
.unwrap_err()
.contains("unknown quotedTripleSyntax"));
assert!(construct_format("ttl;surface=rdf12")
.unwrap_err()
.contains("unknown format parameter"));
let out = QueryOutput::Construct(vec![t(S, P, QT)]);
let env = write_query_json(&out, "ttl", "").unwrap();
assert!(
env.contains(r#""format":"ttl""#) && env.contains("<<( "),
"{env}"
);
let env = write_query_json(&out, "ttl;quotedTripleSyntax=rdf-star", "").unwrap();
assert!(env.contains("<<<http://ex/a>"), "{env}");
assert!(write_query_json(&out, "jsonld", "").is_err());
let env = write_query_json(&out, "table", "").unwrap();
assert!(env.contains("<<<http://ex/a>"), "{env}");
}
}