rete_core/
lib.rs

1//! Core on-disk types for the Rete graph file format.
2//!
3//! See `docs/SPEC.md` for the full design. This crate owns the byte-level
4//! layout (header, directory, blocks) and is deliberately free of any I/O or
5//! query logic so it compiles cleanly to both native and `wasm32`.
6
7#![deny(rustdoc::broken_intra_doc_links)]
8#![warn(missing_docs)]
9
10/// The engine version, as published on crates.io.
11///
12/// Language bindings re-export this so a caller can always tell which engine is
13/// embedded in their wheel, npm package, or bundle. That is the question behind
14/// "does my install support feature X?", and it is not answerable from the
15/// binding's own version, which tracks the binding. Client versions match this
16/// crate's `MAJOR.MINOR` (see `scripts/sync_versions.py`) while their patch
17/// component moves independently for binding-only fixes.
18pub const VERSION: &str = env!("CARGO_PKG_VERSION");
19
20#[doc(hidden)]
21pub mod bgp;
22#[doc(hidden)]
23pub mod block_cache;
24pub mod build_progress;
25pub mod build_trace;
26#[doc(hidden)]
27pub mod card_derive;
28#[doc(hidden)]
29pub mod card_input;
30#[doc(hidden)]
31pub mod card_queries;
32#[doc(hidden)]
33pub mod chunk_cache;
34#[doc(hidden)]
35pub mod dict;
36#[doc(hidden)]
37pub mod dictionary;
38#[doc(hidden)]
39pub mod extbuild;
40mod extbuild_remap;
41mod extbuild_sections;
42mod extbuild_spill;
43mod extsort;
44#[doc(hidden)]
45pub mod file;
46#[doc(hidden)]
47pub mod geo;
48pub mod geo3;
49#[doc(hidden)]
50pub mod header;
51#[doc(hidden)]
52pub mod index;
53
54pub mod ingest;
55pub mod iri;
56pub mod label_index;
57pub mod manifest;
58#[doc(hidden)]
59pub mod meta;
60#[cfg(feature = "parallel")]
61#[doc(hidden)]
62pub mod parallel;
63#[doc(hidden)]
64pub mod pyramid;
65mod quad;
66#[doc(hidden)]
67pub mod reach;
68#[doc(hidden)]
69pub mod reader;
70#[doc(hidden)]
71pub mod reason;
72pub mod recard;
73#[doc(hidden)]
74pub mod results;
75mod row;
76#[doc(hidden)]
77pub mod schema_pyramid;
78pub mod schema_section;
79#[doc(hidden)]
80pub mod service;
81#[doc(hidden)]
82pub mod shacl;
83#[doc(hidden)]
84pub mod sparql;
85/// The memory available to this process and the default query budget sized
86/// from it (native targets only).
87#[cfg(not(target_arch = "wasm32"))]
88pub mod sysmem;
89#[doc(hidden)]
90pub mod terms;
91#[doc(hidden)]
92pub mod text_index;
93mod text_index_spill;
94#[doc(hidden)]
95pub mod tiling;
96#[doc(hidden)]
97pub mod triples;
98#[doc(hidden)]
99pub mod varint;
100pub mod write_opts;
101
102/// Stable file-format, reader, and in-memory build API.
103pub mod format {
104    pub use crate::file::{
105        append_sections, attach_build_info, content_hash_parts, plan_build_info, read_build_info,
106        replace_metadata, verify, BuildInfoPlan, ByteRange, DumpPlan, FileError, LayoutSegment,
107        PyramidLayout, Rete, TermTriple, TripleProvenance, CODEC_NONE, CODEC_ZSTD, RDF_TYPE,
108    };
109    pub use crate::header::{
110        Header, HeaderError, Section, SectionKind, CURRENT_FORMAT_VERSION, HEADER_LEN, MAGIC,
111        MIN_STABLE_READ_VERSION,
112    };
113    pub use crate::index::ScanPlan;
114    pub use crate::ingest::{
115        assemble_dataset, assemble_dataset_with, assemble_dataset_with_layout,
116        assemble_dataset_with_opts, assemble_dataset_with_opts_algo, assemble_dataset_with_perms,
117        parse, parse_quads, parse_rdfxml, parse_statements, parse_statements_audited_surface,
118        parse_turtle, parse_turtle_surface, BuildStats, DeferredMetadata, FinalCounts, GraphLayout,
119        IngestError, IntoMetadata, QuotedTripleSurface, RawQuad, RawTriple,
120    };
121    #[cfg(not(target_arch = "wasm32"))]
122    pub use crate::recard::commit_card_append;
123    pub use crate::recard::{
124        card_append_hash, plan_card_append, plan_card_rewrite, rewrite_card, CardAppendPlan,
125        CardRewritePlan, REWRITE_CHUNK,
126    };
127}
128
129/// Stable **Dataset Card** API — both halves of a card, so that every writer
130/// (`rete build --card-file`, the browser builder, any language binding, any
131/// third-party tool) produces the same card from the same graph.
132///
133/// - The **curated** half ([`crate::card_input`]) is the write-time rules a
134///   hand-authored card document must satisfy: the reserved top level, the
135///   `theme` IRI requirement, the bounded `extra` bag.
136/// - The **derived** half ([`crate::card_derive`] + [`crate::card_queries`])
137///   is the profile computed from the
138///   graph's own statements — predicate/class histograms, vocabularies,
139///   datatypes, languages, the class-link quotient, hubs, affordance signals
140///   and the tiered starter-query library. [`card_derive::derive_card`] is the entry point
141///   for an in-memory quad slice; [`card_derive::derive_card_encoded`] for a streaming build
142///   that only has a dictionary + id-triples; [`card_derive::CardTripleSource`] for anything
143///   else.
144///
145/// Derivation is **opt-in**: it walks the graph a second time, so no build
146/// spends that pass unless the caller asks for a card.
147pub mod card {
148    pub use crate::card_derive::{
149        curated_counts_card, derive_card, derive_card_encoded, derive_card_from,
150        derive_card_streaming, load_card, CardInput, CardTripleSource, ClassLink, Coherence,
151        Creator, DatasetCard, ExampleQuery, LabelIndexSignal, PermutationsSignal, Publisher,
152        QuotedTriplesSignal, Signals, StreamReport, TextIndexSignal, Tier, CARD_TOP_N, GEO_ASWKT,
153        GEO_HASGEOMETRY, O_LITERAL, QUOTED_TRIPLE_SURFACE_RDF12, QUOTED_TRIPLE_SURFACE_RDF_STAR,
154    };
155    pub use crate::card_input::{
156        canonicalize_json, check_description_len, compose_curated_card, json_depth,
157        normalize_description, normalize_extra, normalize_string_list, normalize_themes,
158        validate_curated_card, CARD_DESCRIPTION_MAX_BYTES, CARD_EXTRA_MAX_BYTES,
159        CARD_EXTRA_MAX_DEPTH, CARD_EXTRA_MAX_KEYS, CARD_EXTRA_MAX_KEY_BYTES, CURATED_CARD_FIELDS,
160        UNKNOWN_FIELD_HINT,
161    };
162    pub use crate::card_queries::{
163        audit, audit_examples, claim_of, claim_of_query, example_query_id, generate, Claim,
164        Finding, Observation, Recorded, Substitution, Verdict, STARTER_MAX_BYTES,
165        STARTER_SCAN_BUDGET,
166    };
167}
168
169/// Stable SPARQL, graph-pattern, federation, and result API.
170pub mod query {
171    pub use crate::bgp::{Binding, PatternTerm, TriplePattern};
172    pub use crate::file::TripleProvenance;
173    pub use crate::results::{push_json_string, results_envelope_json};
174    pub use crate::service::{
175        parse_sparql_json_results, sparql_json_ask, sparql_json_results, ServiceClient,
176    };
177    pub use crate::sparql::{
178        eval_query, eval_query_partitioned, eval_query_reasoned, eval_query_with,
179        eval_query_with_warnings, eval_select_communities, eval_sparql, eval_sparql_reasoned,
180        partition_plan, query_predicates, routed_triple_pattern, summary_query_shape,
181        warnings_json, CommunityPartial, CommunitySelect, PartitionPlan, QueryOpts, QueryOutput,
182        QueryWarning, RoutedTriplePattern, SparqlError, SummaryQueryShape, WarningSeverity,
183    };
184}
185
186/// Stable byte-range and lazy-open API for local or remote `.rete` files.
187pub mod range {
188    pub use crate::block_cache::{auto_block, BlockCacheReader, DEFAULT_BLOCK, DEFAULT_CACHE_CAP};
189    pub use crate::file::{
190        read_card_and_build_info_ranged, read_card_and_build_info_with_header,
191        read_metadata_ranged, read_schema_coherence_ranged, read_schema_ranged,
192        read_schema_summary_ranged, read_text_index_token_table_len_ranged, ByteRange,
193        LayoutSegment, SearchView, SummaryView,
194    };
195    pub use crate::reader::{
196        detect_polyglot_base, CountingReader, OffsetReader, RangeReader, SliceReader,
197        POLYGLOT_DIGITS, POLYGLOT_MARKER,
198    };
199}
200
201/// Stable integrity and SHACL validation API.
202pub mod validation {
203    pub use crate::file::verify;
204    pub use crate::shacl::{
205        validate_shacl, DataGraph, GraphView, ReteGraph, Severity, ShaclError, ShaclShapes,
206        ValidationReport, ValidationResult,
207    };
208}
209
210/// Stable RDFS/OWL reasoning and schema-coherence API.
211pub mod reasoning {
212    pub use crate::file::{
213        read_schema_coherence_ranged, schema_coherence, schema_summary, TermTriple,
214    };
215    pub use crate::reason::{reason, Inconsistency, Reasoning, REASON_RULESET};
216    pub use crate::sparql::{eval_query_reasoned, eval_sparql_reasoned};
217}
218
219#[doc(hidden)]
220pub use bgp::{eval_bgp, Binding, PatternTerm, TriplePattern};
221#[doc(hidden)]
222pub use block_cache::{auto_block, BlockCacheReader, DEFAULT_BLOCK, DEFAULT_CACHE_CAP};
223#[doc(hidden)]
224pub use dict::{DictSection, DictSectionBuilder};
225#[doc(hidden)]
226pub use dictionary::{Dictionary, DictionaryBuilder};
227#[doc(hidden)]
228pub use file::{
229    append_sections, attach_build_info, build_pyramid_meta, build_pyramid_meta_algo,
230    build_pyramid_meta_layout, build_pyramid_meta_with, compute_text_index, plan_build_info,
231    read_build_info, read_card_and_build_info_ranged, read_card_and_build_info_with_header,
232    read_metadata_ranged, read_schema_coherence_ranged, read_schema_ranged,
233    read_schema_summary_ranged, read_text_index_token_table_len_ranged, replace_metadata,
234    schema_classes, schema_coherence, schema_summary, spill_threshold_for, split_memory_budget,
235    verify, verify_ranged, write_dataset, write_dataset_with_metadata, write_file, BuildInfoPlan,
236    ByteRange, DumpPlan, FixedOverhead, LayoutSegment, MemoryBudget, PyramidLayout, Rete,
237    SearchView, SummaryView, TermTriple, TripleProvenance, CODEC_NONE, CODEC_ZSTD,
238    DEFAULT_EXPORT_BUDGET_MB, DEFAULT_QUERY_BUDGET_MB, DEFAULT_TILE_BUDGET, RDF_TYPE,
239};
240#[doc(hidden)]
241pub use header::{
242    Header, HeaderError, Section, SectionKind, CURRENT_FORMAT_VERSION, HEADER_LEN, MAGIC,
243    MIN_STABLE_READ_VERSION,
244};
245#[doc(hidden)]
246pub use index::{GraphIndex, GraphIndexBuilder, IndexPermutation, Pattern, PermSet, ScanPlan};
247pub use ingest::GraphLayout;
248pub use iri::{
249    iri_content_defect, sanitize_iri_content, sanitize_term, term_defect, IriDefect, IriReport,
250};
251#[doc(hidden)]
252pub use label_index::{read_label_index_entries_ranged, LabelIndex};
253#[doc(hidden)]
254pub use meta::{
255    CharSet, ClassNode, ClassRelation, CommunityDescriptor, LabelEntry, LevelLinks, LevelRollup,
256    PredStat, PyramidMeta,
257};
258#[doc(hidden)]
259pub use pyramid::{
260    build_dendrogram, louvain_one_level, louvain_one_level_stats, project_graph, Dendrogram, Graph,
261    LevelStats, Partition, PyramidAlgo, MAX_LOUVAIN_PASSES,
262};
263#[doc(hidden)]
264pub use reach::{batch_reach_serial, build_adjacency, reach_one};
265#[doc(hidden)]
266pub use reader::{
267    detect_polyglot_base, CountingReader, OffsetReader, RangeReader, SliceReader, POLYGLOT_DIGITS,
268    POLYGLOT_MARKER,
269};
270#[doc(hidden)]
271pub use reason::{reason, Inconsistency, Reasoning, REASON_RULESET};
272#[doc(hidden)]
273pub use results::{push_json_string, results_envelope_json};
274#[doc(hidden)]
275pub use schema_pyramid::build_schema_pyramid;
276#[doc(hidden)]
277pub use schema_section::SchemaMeta;
278#[doc(hidden)]
279pub use service::{parse_sparql_json_results, sparql_json_ask, sparql_json_results, ServiceClient};
280#[doc(hidden)]
281pub use shacl::{
282    validate_shacl, DataGraph, GraphView, ReteGraph, Severity, ShaclError, ShaclShapes,
283    ValidationReport, ValidationResult,
284};
285#[doc(hidden)]
286pub use sparql::{
287    eval_query, eval_query_partitioned, eval_query_reasoned, eval_query_with,
288    eval_query_with_warnings, eval_select_communities, eval_sparql, eval_sparql_reasoned,
289    parse_select, partition_plan, query_predicates, routed_triple_pattern, summary_query_shape,
290    warnings_json, Agg, CommunityPartial, CommunitySelect, FExpr, GraphTarget, GroupSpec, Op,
291    PartitionPlan, PathAst, Plan, QueryOpts, QueryOutput, QueryWarning, Rep, RoutedTriplePattern,
292    Select, SparqlError, SummaryQueryShape, WarningSeverity,
293};
294#[cfg(not(target_arch = "wasm32"))]
295pub use sysmem::{
296    available_memory, default_query_budget_mb, MIN_DEFAULT_QUERY_BUDGET_MB, QUERY_BUDGET_ENV,
297};
298#[doc(hidden)]
299pub use terms::{NodeId, ObjectId, PredicateId, SubjectId, TermToken};
300#[doc(hidden)]
301pub use text_index::{
302    tokenize, IntoTextIndex, TextIndex, TextIndexBuilder, TextIndexGraphs, TextIndexPredicates,
303    TextIndexScope,
304};
305#[doc(hidden)]
306pub use tiling::{choose_round_for_budget, summarize, tile_by_community, SuperEdge, Tile};
307#[doc(hidden)]
308pub use triples::{GroupDirectory, Triple, TripleBlock, TripleBlockBuilder, ZoneMap};
309
310// HTTP(S) range reads for native hosts (the `http` feature): the reader the
311// CLI and the Python and R clients share. Its docs are the module's own (an
312// outer doc here would move their intra-doc links to this scope).
313#[cfg(feature = "http")]
314pub mod http;
rete_core/lib.rs

rete_core/
lib.rs
