rete_core/
lib.rs

1//! Core on-disk types for the Rete graph file format.
2//!
3//! See `docs/SPEC.md` for the full design. This crate owns the byte-level
4//! layout (header, directory, blocks) and is deliberately free of any I/O or
5//! query logic so it compiles cleanly to both native and `wasm32`.
6
7#![deny(rustdoc::broken_intra_doc_links)]
8#![warn(missing_docs)]
9
10/// The engine version, as published on crates.io.
11///
12/// Language bindings re-export this so a caller can always tell which engine is
13/// embedded in their wheel, npm package, or bundle. That is the question behind
14/// "does my install support feature X?", and it is not answerable from the
15/// binding's own version, which tracks the binding. Client versions match this
16/// crate's `MAJOR.MINOR` (see `scripts/sync_versions.py`) while their patch
17/// component moves independently for binding-only fixes.
18pub const VERSION: &str = env!("CARGO_PKG_VERSION");
19
20#[doc(hidden)]
21pub mod bgp;
22#[doc(hidden)]
23pub mod block_cache;
24pub mod build_progress;
25pub mod build_record;
26pub mod build_trace;
27#[doc(hidden)]
28pub mod card_derive;
29#[doc(hidden)]
30pub mod card_input;
31#[doc(hidden)]
32pub mod card_queries;
33#[doc(hidden)]
34pub mod chunk_cache;
35#[doc(hidden)]
36pub mod dict;
37#[doc(hidden)]
38pub mod dictionary;
39#[doc(hidden)]
40pub mod extbuild;
41mod extbuild_remap;
42mod extbuild_sections;
43mod extbuild_spill;
44mod extbuild_v06;
45mod extsort;
46#[doc(hidden)]
47pub mod file;
48#[doc(hidden)]
49pub mod geo;
50pub mod geo3;
51#[doc(hidden)]
52pub mod guard;
53#[doc(hidden)]
54pub mod header;
55pub mod hll;
56#[doc(hidden)]
57pub mod index;
58
59pub mod ingest;
60pub mod iri;
61pub mod label_index;
62pub mod manifest;
63#[doc(hidden)]
64pub mod meta;
65#[cfg(feature = "parallel")]
66#[doc(hidden)]
67pub mod parallel;
68#[doc(hidden)]
69pub mod pyramid;
70mod quad;
71#[doc(hidden)]
72pub mod reach;
73#[doc(hidden)]
74pub mod reader;
75#[doc(hidden)]
76pub mod reason;
77pub mod recard;
78#[doc(hidden)]
79pub mod results;
80mod row;
81#[doc(hidden)]
82pub mod schema_pyramid;
83pub mod schema_section;
84#[doc(hidden)]
85pub mod service;
86#[doc(hidden)]
87pub mod shacl;
88#[doc(hidden)]
89pub mod sparql;
90/// The memory available to this process and the default query budget sized
91/// from it (native targets only).
92#[cfg(not(target_arch = "wasm32"))]
93pub mod sysmem;
94#[doc(hidden)]
95pub mod terms;
96#[doc(hidden)]
97pub mod text_index;
98mod text_index_spill;
99#[doc(hidden)]
100pub mod tiling;
101#[doc(hidden)]
102pub mod triples;
103pub mod v06;
104#[doc(hidden)]
105pub mod varint;
106pub mod write_opts;
107
108/// Stable file-format, reader, and in-memory build API.
109pub mod format {
110    pub use crate::file::{
111        append_sections, attach_build_info, content_hash_parts, plan_build_info, read_build_info,
112        replace_metadata, verify, BuildInfoPlan, ByteRange, DumpPlan, FileError, LayoutSegment,
113        PyramidLayout, Rete, TermTriple, TripleProvenance, CODEC_NONE, CODEC_ZSTD, RDF_TYPE,
114    };
115    pub use crate::header::{
116        Header, HeaderError, Section, SectionKind, CURRENT_FORMAT_VERSION, HEADER_LEN,
117        LEGACY_FORMAT_VERSION, MAGIC, MIN_STABLE_READ_VERSION,
118    };
119    pub use crate::index::ScanPlan;
120    pub use crate::ingest::{
121        assemble_dataset, assemble_dataset_with, assemble_dataset_with_layout,
122        assemble_dataset_with_opts, assemble_dataset_with_opts_algo, assemble_dataset_with_perms,
123        parse, parse_jelly_bytes, parse_jelly_bytes_audited, parse_quads, parse_rdfxml,
124        parse_statements, parse_statements_audited_surface, parse_turtle, parse_turtle_surface,
125        BuildStats, DeferredMetadata, FinalCounts, GraphLayout, IngestError, IntoMetadata,
126        QuotedTripleSurface, RawQuad, RawTriple,
127    };
128    #[cfg(not(target_arch = "wasm32"))]
129    pub use crate::recard::commit_card_append;
130    pub use crate::recard::{
131        card_append_hash, plan_card_append, plan_card_rewrite, rewrite_card, CardAppendPlan,
132        CardRewritePlan, REWRITE_CHUNK,
133    };
134}
135
136/// Stable **Dataset Card** API — both halves of a card, so that every writer
137/// (`rete build --card-file`, the browser builder, any language binding, any
138/// third-party tool) produces the same card from the same graph.
139///
140/// - The **curated** half ([`crate::card_input`]) is the write-time rules a
141///   hand-authored card document must satisfy: the reserved top level, the
142///   `theme` IRI requirement, the bounded `extra` bag.
143/// - The **derived** half ([`crate::card_derive`] + [`crate::card_queries`])
144///   is the profile computed from the
145///   graph's own statements — predicate/class histograms, vocabularies,
146///   datatypes, languages, the class-link quotient, hubs, affordance signals
147///   and the tiered starter-query library. [`card_derive::derive_card`] is the entry point
148///   for an in-memory quad slice; [`card_derive::derive_card_encoded`] for a streaming build
149///   that only has a dictionary + id-triples; [`card_derive::CardTripleSource`] for anything
150///   else.
151///
152/// Derivation is **opt-in**: it walks the graph a second time, so no build
153/// spends that pass unless the caller asks for a card.
154pub mod card {
155    pub use crate::card_derive::{
156        curated_counts_card, derive_card, derive_card_encoded, derive_card_from,
157        derive_card_streaming, load_card, CardInput, CardTripleSource, ClassLink, Coherence,
158        Creator, DatasetCard, ExampleQuery, LabelIndexSignal, PermutationsSignal, Publisher,
159        QuotedTriplesSignal, Signals, StreamReport, TextIndexSignal, Tier, CARD_TOP_N, GEO_ASWKT,
160        GEO_HASGEOMETRY, NS_TALLY_CAP, NS_TALLY_ENTRY_BYTES, O_LITERAL,
161        QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR,
162    };
163    pub use crate::card_input::{
164        canonicalize_json, check_description_len, compose_curated_card, json_depth,
165        normalize_description, normalize_extra, normalize_string_list, normalize_themes,
166        validate_curated_card, CARD_DESCRIPTION_MAX_BYTES, CARD_EXTRA_MAX_BYTES,
167        CARD_EXTRA_MAX_DEPTH, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES, CURATED_CARD_FIELDS,
168        UNKNOWN_FIELD_HINT,
169    };
170    pub use crate::card_queries::{
171        audit, audit_examples, claim_of, claim_of_query, example_query_id, generate, Claim,
172        Finding, Observation, Recorded, Substitution, Verdict, STARTER_MAX_BYTES,
173        STARTER_SCAN_BUDGET,
174    };
175}
176
177/// Stable SPARQL, graph-pattern, federation, and result API.
178pub mod query {
179    pub use crate::bgp::{Binding, PatternTerm, TriplePattern};
180    pub use crate::file::TripleProvenance;
181    pub use crate::results::{push_json_string, results_envelope_json};
182    pub use crate::service::{
183        parse_sparql_json_results, sparql_json_ask, sparql_json_results, ServiceClient,
184    };
185    pub use crate::sparql::{
186        eval_query, eval_query_partitioned, eval_query_reasoned, eval_query_with,
187        eval_query_with_warnings, eval_select_communities, eval_sparql, eval_sparql_reasoned,
188        partition_plan, query_predicates, routed_triple_pattern, summary_query_shape,
189        warnings_json, CommunityPartial, CommunitySelect, PartitionPlan, QueryOpts, QueryOutput,
190        QueryWarning, RoutedTriplePattern, SparqlError, SummaryQueryShape, WarningSeverity,
191    };
192}
193
194/// Stable byte-range and lazy-open API for local or remote `.rete` files.
195pub mod range {
196    pub use crate::block_cache::{auto_block, BlockCacheReader, DEFAULT_BLOCK, DEFAULT_CACHE_CAP};
197    pub use crate::file::{
198        read_card_and_build_info_ranged, read_card_and_build_info_with_header,
199        read_metadata_ranged, read_schema_coherence_ranged, read_schema_ranged,
200        read_schema_summary_ranged, read_text_index_token_table_len_ranged, ByteRange,
201        LayoutSegment, SearchView, SummaryView,
202    };
203    pub use crate::reader::{
204        detect_polyglot_base, BytesReader, CountingReader, OffsetReader, PrefixReader, RangeReader,
205        SliceReader, POLYGLOT_DIGITS, POLYGLOT_MARKER,
206    };
207}
208
209/// Stable integrity and SHACL validation API.
210pub mod validation {
211    pub use crate::file::verify;
212    pub use crate::shacl::{
213        validate_shacl, DataGraph, GraphView, ReteGraph, Severity, ShaclError, ShaclShapes,
214        ValidationReport, ValidationResult,
215    };
216}
217
218/// Stable RDFS/OWL reasoning and schema-coherence API.
219pub mod reasoning {
220    pub use crate::file::{
221        read_schema_coherence_ranged, schema_coherence, schema_summary, TermTriple,
222    };
223    pub use crate::reason::{reason, Inconsistency, Reasoning, REASON_RULESET};
224    pub use crate::sparql::{eval_query_reasoned, eval_sparql_reasoned};
225}
226
227#[doc(hidden)]
228pub use bgp::{eval_bgp, Binding, PatternTerm, TriplePattern};
229#[doc(hidden)]
230pub use block_cache::{auto_block, BlockCacheReader, DEFAULT_BLOCK, DEFAULT_CACHE_CAP};
231#[doc(hidden)]
232pub use dict::{DictSection, DictSectionBuilder};
233#[doc(hidden)]
234pub use dictionary::{Dictionary, DictionaryBuilder};
235#[doc(hidden)]
236pub use file::{
237    append_sections, attach_build_info, build_pyramid_meta, build_pyramid_meta_algo,
238    build_pyramid_meta_layout, build_pyramid_meta_with, compute_text_index, plan_build_info,
239    read_build_info, read_card_and_build_info_ranged, read_card_and_build_info_with_header,
240    read_metadata_ranged, read_schema_coherence_ranged, read_schema_ranged,
241    read_schema_summary_ranged, read_text_index_token_table_len_ranged, replace_metadata,
242    schema_classes, schema_coherence, schema_summary, spill_threshold_for, split_memory_budget,
243    verify, verify_ranged, write_dataset, write_dataset_v05, write_dataset_with_metadata,
244    write_dataset_with_metadata_v05, write_file, write_file_v05, BuildInfoPlan, ByteRange,
245    DumpPlan, FixedOverhead, LayoutSegment, MemoryBudget, PyramidLayout, Rete, SearchView,
246    SummaryView, TermTriple, TripleProvenance, CODEC_NONE, CODEC_ZSTD, DEFAULT_EXPORT_BUDGET_MB,
247    DEFAULT_QUERY_BUDGET_MB, DEFAULT_TILE_BUDGET, RDF_TYPE,
248};
249#[doc(hidden)]
250pub use header::{
251    Header, HeaderError, Section, SectionKind, CURRENT_FORMAT_VERSION, HEADER_LEN,
252    LEGACY_FORMAT_VERSION, MAGIC, MIN_STABLE_READ_VERSION,
253};
254#[doc(hidden)]
255pub use index::{GraphIndex, GraphIndexBuilder, IndexPermutation, Pattern, PermSet, ScanPlan};
256pub use ingest::GraphLayout;
257pub use iri::{
258    iri_content_defect, sanitize_iri_content, sanitize_term, term_defect, IriDefect, IriReport,
259};
260#[doc(hidden)]
261pub use label_index::{read_label_index_entries_ranged, LabelIndex};
262#[doc(hidden)]
263pub use meta::{
264    CharSet, ClassNode, ClassRelation, CommunityDescriptor, LabelEntry, LevelLinks, LevelRollup,
265    PredStat, PyramidMeta,
266};
267#[doc(hidden)]
268pub use pyramid::{
269    build_dendrogram, louvain_one_level, louvain_one_level_stats, project_graph, Dendrogram, Graph,
270    LevelStats, Partition, PyramidAlgo, MAX_LOUVAIN_PASSES,
271};
272#[doc(hidden)]
273pub use reach::{batch_reach_serial, build_adjacency, reach_one};
274#[doc(hidden)]
275pub use reader::{
276    detect_polyglot_base, BytesReader, CountingReader, OffsetReader, PrefixReader, RangeReader,
277    SliceReader, POLYGLOT_DIGITS, POLYGLOT_MARKER,
278};
279#[doc(hidden)]
280pub use reason::{reason, Inconsistency, Reasoning, REASON_RULESET};
281#[doc(hidden)]
282pub use results::{push_json_string, results_envelope_json};
283#[doc(hidden)]
284pub use schema_pyramid::build_schema_pyramid;
285#[doc(hidden)]
286pub use schema_section::SchemaMeta;
287#[doc(hidden)]
288pub use service::{parse_sparql_json_results, sparql_json_ask, sparql_json_results, ServiceClient};
289#[doc(hidden)]
290pub use shacl::{
291    validate_shacl, DataGraph, GraphView, ReteGraph, Severity, ShaclError, ShaclShapes,
292    ValidationReport, ValidationResult,
293};
294#[doc(hidden)]
295pub use sparql::{
296    eval_query, eval_query_partitioned, eval_query_reasoned, eval_query_with,
297    eval_query_with_warnings, eval_select_communities, eval_sparql, eval_sparql_reasoned,
298    parse_select, partition_plan, query_predicates, routed_triple_pattern, summary_query_shape,
299    warnings_json, Agg, CommunityPartial, CommunitySelect, FExpr, GraphTarget, GroupSpec, Op,
300    PartitionPlan, PathAst, Plan, QueryOpts, QueryOutput, QueryWarning, Rep, RoutedTriplePattern,
301    Select, SparqlError, SummaryQueryShape, WarningSeverity,
302};
303#[cfg(not(target_arch = "wasm32"))]
304pub use sysmem::{
305    available_memory, default_query_budget_mb, MIN_DEFAULT_QUERY_BUDGET_MB, QUERY_BUDGET_ENV,
306};
307#[doc(hidden)]
308pub use terms::{NodeId, ObjectId, PredicateId, SubjectId, TermToken};
309#[doc(hidden)]
310pub use text_index::{
311    tokenize, IntoTextIndex, TextIndex, TextIndexBuilder, TextIndexGraphs, TextIndexPredicates,
312    TextIndexScope,
313};
314#[doc(hidden)]
315pub use tiling::{choose_round_for_budget, summarize, tile_by_community, SuperEdge, Tile};
316#[doc(hidden)]
317pub use triples::{GroupDirectory, Triple, TripleBlock, TripleBlockBuilder, ZoneMap};
318
319// HTTP(S) range reads for native hosts (the `http` feature): the reader the
320// CLI and the Python and R clients share. Its docs are the module's own (an
321// outer doc here would move their intra-doc links to this scope).
322#[cfg(feature = "http")]
323pub mod http;
rete_core/lib.rs

rete_core/
lib.rs
