rete_core/
lib.rs

1//! Core on-disk types for the Rete graph file format.
2//!
3//! See `docs/SPEC.md` for the full design. This crate owns the byte-level
4//! layout (header, directory, blocks) and is deliberately free of any I/O or
5//! query logic so it compiles cleanly to both native and `wasm32`.
6
7#![deny(rustdoc::broken_intra_doc_links)]
8#![warn(missing_docs)]
9
10/// The engine version, as published on crates.io.
11///
12/// Language bindings re-export this so a caller can always tell which engine is
13/// embedded in their wheel, npm package, or bundle. That is the question behind
14/// "does my install support feature X?", and it is not answerable from the
15/// binding's own version, which tracks the binding. Client versions match this
16/// crate's `MAJOR.MINOR` (see `scripts/sync_versions.py`) while their patch
17/// component moves independently for binding-only fixes.
18pub const VERSION: &str = env!("CARGO_PKG_VERSION");
19
20#[doc(hidden)]
21pub mod bgp;
22#[doc(hidden)]
23pub mod block_cache;
24pub mod build_progress;
25pub mod build_record;
26pub mod build_trace;
27#[doc(hidden)]
28pub mod card_derive;
29#[doc(hidden)]
30pub mod card_input;
31#[doc(hidden)]
32pub mod card_queries;
33#[doc(hidden)]
34pub mod chunk_cache;
35#[doc(hidden)]
36pub mod dict;
37#[doc(hidden)]
38pub mod dictionary;
39#[doc(hidden)]
40pub mod extbuild;
41mod extbuild_remap;
42mod extbuild_sections;
43mod extbuild_spill;
44mod extbuild_v06;
45mod extsort;
46#[doc(hidden)]
47pub mod file;
48#[doc(hidden)]
49pub mod geo;
50pub mod geo3;
51#[doc(hidden)]
52pub mod header;
53pub mod hll;
54#[doc(hidden)]
55pub mod index;
56
57pub mod ingest;
58pub mod iri;
59pub mod label_index;
60pub mod manifest;
61#[doc(hidden)]
62pub mod meta;
63#[cfg(feature = "parallel")]
64#[doc(hidden)]
65pub mod parallel;
66#[doc(hidden)]
67pub mod pyramid;
68mod quad;
69#[doc(hidden)]
70pub mod reach;
71#[doc(hidden)]
72pub mod reader;
73#[doc(hidden)]
74pub mod reason;
75pub mod recard;
76#[doc(hidden)]
77pub mod results;
78mod row;
79#[doc(hidden)]
80pub mod schema_pyramid;
81pub mod schema_section;
82#[doc(hidden)]
83pub mod service;
84#[doc(hidden)]
85pub mod shacl;
86#[doc(hidden)]
87pub mod sparql;
88/// The memory available to this process and the default query budget sized
89/// from it (native targets only).
90#[cfg(not(target_arch = "wasm32"))]
91pub mod sysmem;
92#[doc(hidden)]
93pub mod terms;
94#[doc(hidden)]
95pub mod text_index;
96mod text_index_spill;
97#[doc(hidden)]
98pub mod tiling;
99#[doc(hidden)]
100pub mod triples;
101pub mod v06;
102#[doc(hidden)]
103pub mod varint;
104pub mod write_opts;
105
106/// Stable file-format, reader, and in-memory build API.
107pub mod format {
108    pub use crate::file::{
109        append_sections, attach_build_info, content_hash_parts, plan_build_info, read_build_info,
110        replace_metadata, verify, BuildInfoPlan, ByteRange, DumpPlan, FileError, LayoutSegment,
111        PyramidLayout, Rete, TermTriple, TripleProvenance, CODEC_NONE, CODEC_ZSTD, RDF_TYPE,
112    };
113    pub use crate::header::{
114        Header, HeaderError, Section, SectionKind, CURRENT_FORMAT_VERSION, HEADER_LEN,
115        LEGACY_FORMAT_VERSION, MAGIC, MIN_STABLE_READ_VERSION,
116    };
117    pub use crate::index::ScanPlan;
118    pub use crate::ingest::{
119        assemble_dataset, assemble_dataset_with, assemble_dataset_with_layout,
120        assemble_dataset_with_opts, assemble_dataset_with_opts_algo, assemble_dataset_with_perms,
121        parse, parse_quads, parse_rdfxml, parse_statements, parse_statements_audited_surface,
122        parse_turtle, parse_turtle_surface, BuildStats, DeferredMetadata, FinalCounts, GraphLayout,
123        IngestError, IntoMetadata, QuotedTripleSurface, RawQuad, RawTriple,
124    };
125    #[cfg(not(target_arch = "wasm32"))]
126    pub use crate::recard::commit_card_append;
127    pub use crate::recard::{
128        card_append_hash, plan_card_append, plan_card_rewrite, rewrite_card, CardAppendPlan,
129        CardRewritePlan, REWRITE_CHUNK,
130    };
131}
132
133/// Stable **Dataset Card** API — both halves of a card, so that every writer
134/// (`rete build --card-file`, the browser builder, any language binding, any
135/// third-party tool) produces the same card from the same graph.
136///
137/// - The **curated** half ([`crate::card_input`]) is the write-time rules a
138///   hand-authored card document must satisfy: the reserved top level, the
139///   `theme` IRI requirement, the bounded `extra` bag.
140/// - The **derived** half ([`crate::card_derive`] + [`crate::card_queries`])
141///   is the profile computed from the
142///   graph's own statements — predicate/class histograms, vocabularies,
143///   datatypes, languages, the class-link quotient, hubs, affordance signals
144///   and the tiered starter-query library. [`card_derive::derive_card`] is the entry point
145///   for an in-memory quad slice; [`card_derive::derive_card_encoded`] for a streaming build
146///   that only has a dictionary + id-triples; [`card_derive::CardTripleSource`] for anything
147///   else.
148///
149/// Derivation is **opt-in**: it walks the graph a second time, so no build
150/// spends that pass unless the caller asks for a card.
151pub mod card {
152    pub use crate::card_derive::{
153        curated_counts_card, derive_card, derive_card_encoded, derive_card_from,
154        derive_card_streaming, load_card, CardInput, CardTripleSource, ClassLink, Coherence,
155        Creator, DatasetCard, ExampleQuery, LabelIndexSignal, PermutationsSignal, Publisher,
156        QuotedTriplesSignal, Signals, StreamReport, TextIndexSignal, Tier, CARD_TOP_N, GEO_ASWKT,
157        GEO_HASGEOMETRY, NS_TALLY_CAP, NS_TALLY_ENTRY_BYTES, O_LITERAL,
158        QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR,
159    };
160    pub use crate::card_input::{
161        canonicalize_json, check_description_len, compose_curated_card, json_depth,
162        normalize_description, normalize_extra, normalize_string_list, normalize_themes,
163        validate_curated_card, CARD_DESCRIPTION_MAX_BYTES, CARD_EXTRA_MAX_BYTES,
164        CARD_EXTRA_MAX_DEPTH, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES, CURATED_CARD_FIELDS,
165        UNKNOWN_FIELD_HINT,
166    };
167    pub use crate::card_queries::{
168        audit, audit_examples, claim_of, claim_of_query, example_query_id, generate, Claim,
169        Finding, Observation, Recorded, Substitution, Verdict, STARTER_MAX_BYTES,
170        STARTER_SCAN_BUDGET,
171    };
172}
173
174/// Stable SPARQL, graph-pattern, federation, and result API.
175pub mod query {
176    pub use crate::bgp::{Binding, PatternTerm, TriplePattern};
177    pub use crate::file::TripleProvenance;
178    pub use crate::results::{push_json_string, results_envelope_json};
179    pub use crate::service::{
180        parse_sparql_json_results, sparql_json_ask, sparql_json_results, ServiceClient,
181    };
182    pub use crate::sparql::{
183        eval_query, eval_query_partitioned, eval_query_reasoned, eval_query_with,
184        eval_query_with_warnings, eval_select_communities, eval_sparql, eval_sparql_reasoned,
185        partition_plan, query_predicates, routed_triple_pattern, summary_query_shape,
186        warnings_json, CommunityPartial, CommunitySelect, PartitionPlan, QueryOpts, QueryOutput,
187        QueryWarning, RoutedTriplePattern, SparqlError, SummaryQueryShape, WarningSeverity,
188    };
189}
190
191/// Stable byte-range and lazy-open API for local or remote `.rete` files.
192pub mod range {
193    pub use crate::block_cache::{auto_block, BlockCacheReader, DEFAULT_BLOCK, DEFAULT_CACHE_CAP};
194    pub use crate::file::{
195        read_card_and_build_info_ranged, read_card_and_build_info_with_header,
196        read_metadata_ranged, read_schema_coherence_ranged, read_schema_ranged,
197        read_schema_summary_ranged, read_text_index_token_table_len_ranged, ByteRange,
198        LayoutSegment, SearchView, SummaryView,
199    };
200    pub use crate::reader::{
201        detect_polyglot_base, BytesReader, CountingReader, OffsetReader, PrefixReader, RangeReader,
202        SliceReader, POLYGLOT_DIGITS, POLYGLOT_MARKER,
203    };
204}
205
206/// Stable integrity and SHACL validation API.
207pub mod validation {
208    pub use crate::file::verify;
209    pub use crate::shacl::{
210        validate_shacl, DataGraph, GraphView, ReteGraph, Severity, ShaclError, ShaclShapes,
211        ValidationReport, ValidationResult,
212    };
213}
214
215/// Stable RDFS/OWL reasoning and schema-coherence API.
216pub mod reasoning {
217    pub use crate::file::{
218        read_schema_coherence_ranged, schema_coherence, schema_summary, TermTriple,
219    };
220    pub use crate::reason::{reason, Inconsistency, Reasoning, REASON_RULESET};
221    pub use crate::sparql::{eval_query_reasoned, eval_sparql_reasoned};
222}
223
224#[doc(hidden)]
225pub use bgp::{eval_bgp, Binding, PatternTerm, TriplePattern};
226#[doc(hidden)]
227pub use block_cache::{auto_block, BlockCacheReader, DEFAULT_BLOCK, DEFAULT_CACHE_CAP};
228#[doc(hidden)]
229pub use dict::{DictSection, DictSectionBuilder};
230#[doc(hidden)]
231pub use dictionary::{Dictionary, DictionaryBuilder};
232#[doc(hidden)]
233pub use file::{
234    append_sections, attach_build_info, build_pyramid_meta, build_pyramid_meta_algo,
235    build_pyramid_meta_layout, build_pyramid_meta_with, compute_text_index, plan_build_info,
236    read_build_info, read_card_and_build_info_ranged, read_card_and_build_info_with_header,
237    read_metadata_ranged, read_schema_coherence_ranged, read_schema_ranged,
238    read_schema_summary_ranged, read_text_index_token_table_len_ranged, replace_metadata,
239    schema_classes, schema_coherence, schema_summary, spill_threshold_for, split_memory_budget,
240    verify, verify_ranged, write_dataset, write_dataset_v05, write_dataset_with_metadata,
241    write_dataset_with_metadata_v05, write_file, write_file_v05, BuildInfoPlan, ByteRange,
242    DumpPlan, FixedOverhead, LayoutSegment, MemoryBudget, PyramidLayout, Rete, SearchView,
243    SummaryView, TermTriple, TripleProvenance, CODEC_NONE, CODEC_ZSTD, DEFAULT_EXPORT_BUDGET_MB,
244    DEFAULT_QUERY_BUDGET_MB, DEFAULT_TILE_BUDGET, RDF_TYPE,
245};
246#[doc(hidden)]
247pub use header::{
248    Header, HeaderError, Section, SectionKind, CURRENT_FORMAT_VERSION, HEADER_LEN,
249    LEGACY_FORMAT_VERSION, MAGIC, MIN_STABLE_READ_VERSION,
250};
251#[doc(hidden)]
252pub use index::{GraphIndex, GraphIndexBuilder, IndexPermutation, Pattern, PermSet, ScanPlan};
253pub use ingest::GraphLayout;
254pub use iri::{
255    iri_content_defect, sanitize_iri_content, sanitize_term, term_defect, IriDefect, IriReport,
256};
257#[doc(hidden)]
258pub use label_index::{read_label_index_entries_ranged, LabelIndex};
259#[doc(hidden)]
260pub use meta::{
261    CharSet, ClassNode, ClassRelation, CommunityDescriptor, LabelEntry, LevelLinks, LevelRollup,
262    PredStat, PyramidMeta,
263};
264#[doc(hidden)]
265pub use pyramid::{
266    build_dendrogram, louvain_one_level, louvain_one_level_stats, project_graph, Dendrogram, Graph,
267    LevelStats, Partition, PyramidAlgo, MAX_LOUVAIN_PASSES,
268};
269#[doc(hidden)]
270pub use reach::{batch_reach_serial, build_adjacency, reach_one};
271#[doc(hidden)]
272pub use reader::{
273    detect_polyglot_base, BytesReader, CountingReader, OffsetReader, PrefixReader, RangeReader,
274    SliceReader, POLYGLOT_DIGITS, POLYGLOT_MARKER,
275};
276#[doc(hidden)]
277pub use reason::{reason, Inconsistency, Reasoning, REASON_RULESET};
278#[doc(hidden)]
279pub use results::{push_json_string, results_envelope_json};
280#[doc(hidden)]
281pub use schema_pyramid::build_schema_pyramid;
282#[doc(hidden)]
283pub use schema_section::SchemaMeta;
284#[doc(hidden)]
285pub use service::{parse_sparql_json_results, sparql_json_ask, sparql_json_results, ServiceClient};
286#[doc(hidden)]
287pub use shacl::{
288    validate_shacl, DataGraph, GraphView, ReteGraph, Severity, ShaclError, ShaclShapes,
289    ValidationReport, ValidationResult,
290};
291#[doc(hidden)]
292pub use sparql::{
293    eval_query, eval_query_partitioned, eval_query_reasoned, eval_query_with,
294    eval_query_with_warnings, eval_select_communities, eval_sparql, eval_sparql_reasoned,
295    parse_select, partition_plan, query_predicates, routed_triple_pattern, summary_query_shape,
296    warnings_json, Agg, CommunityPartial, CommunitySelect, FExpr, GraphTarget, GroupSpec, Op,
297    PartitionPlan, PathAst, Plan, QueryOpts, QueryOutput, QueryWarning, Rep, RoutedTriplePattern,
298    Select, SparqlError, SummaryQueryShape, WarningSeverity,
299};
300#[cfg(not(target_arch = "wasm32"))]
301pub use sysmem::{
302    available_memory, default_query_budget_mb, MIN_DEFAULT_QUERY_BUDGET_MB, QUERY_BUDGET_ENV,
303};
304#[doc(hidden)]
305pub use terms::{NodeId, ObjectId, PredicateId, SubjectId, TermToken};
306#[doc(hidden)]
307pub use text_index::{
308    tokenize, IntoTextIndex, TextIndex, TextIndexBuilder, TextIndexGraphs, TextIndexPredicates,
309    TextIndexScope,
310};
311#[doc(hidden)]
312pub use tiling::{choose_round_for_budget, summarize, tile_by_community, SuperEdge, Tile};
313#[doc(hidden)]
314pub use triples::{GroupDirectory, Triple, TripleBlock, TripleBlockBuilder, ZoneMap};
315
316// HTTP(S) range reads for native hosts (the `http` feature): the reader the
317// CLI and the Python and R clients share. Its docs are the module's own (an
318// outer doc here would move their intra-doc links to this scope).
319#[cfg(feature = "http")]
320pub mod http;
rete_core/lib.rs

rete_core/
lib.rs
