sochdb_query/
lib.rs

1// Copyright 2025 Sushanth (https://github.com/sushanthpy)
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! SochDB Query Engine
16//!
17//! SOCH-QL query language for TOON-native data.
18//!
19//! ## SOCH-QL
20//!
21//! SQL-like query language that returns results in TOON format:
22//!
23//! ```text
24//! SELECT id,name FROM users WHERE score > 80
25//! → users[2]{id,name}:
26//!   1,Alice
27//!   3,Charlie
28//! ```
29//!
30//! ## Query Execution Pipeline (Task 6)
31//!
32//! ```text
33//! parse(sql) → SochQuery → validate → plan → execute → SochTable
34//! ```
35//!
36//! Token reduction: 40-60% vs JSON (66% for typical queries)
37//!
38//! ## CONTEXT SELECT (LLM-Native)
39//!
40//! Priority-based context aggregation for LLM consumption:
41//!
42//! ```text
43//! CONTEXT SELECT
44//!   FROM session('abc123')
45//!   WITH TOKEN_LIMIT 4000
46//!   SECTIONS (
47//!     USER {query, preferences} PRIORITY 1,
48//!     HISTORY {recent} PRIORITY 2,
49//!     KNOWLEDGE {docs} PRIORITY 3
50//!   )
51//! ```
52
53pub mod agent_context;
54pub mod bm25_filtered; // Task 6: BM25 filter pushdown via posting-set intersection
55pub mod calc;
56pub mod candidate_gate; // Task 4: Unified candidate gate interface
57pub mod capability_token; // Task 8: Capability tokens + ACLs
58pub mod context_query;
59pub mod cost_optimizer; // Cost-based query optimizer (Task 6)
60pub mod embedding_provider; // Task 2: Automatic embedding generation
61pub mod exact_token_counter; // Task 6: BPE-accurate token counting
62pub mod filter_ir; // Task 1: Canonical Filter IR (CNF/DNF)
63pub mod filtered_vector_search; // Task 5: Filter-aware vector search with selectivity fallback
64pub mod hybrid_retrieval; // Task 3: Vector + BM25 + RRF fusion
65pub mod memory_compaction; // Task 5: Hierarchical memory compaction
66pub mod metadata_index; // Task 3: Metadata index primitives (bitmap + range)
67pub mod namespace; // Task 2: Namespace-scoped query API
68pub mod optimizer_integration;
69pub mod plugin_table;
70pub mod query_optimizer;
71pub mod semantic_triggers; // Task 7: Vector percolator triggers
72pub mod simd_filter; // SIMD vectorized query filters (mm.md Task 5.3)
73pub mod sql; // SQL-92 compatible query engine with SochDB extensions
74pub mod streaming_context; // Task 1: Streaming context generation
75pub mod temporal_decay; // Task 4: Recency-biased scoring
76pub mod token_budget;
77pub mod soch_ql;
78pub mod soch_ql_executor;
79pub mod unified_fusion; // Task 7: Hybrid fusion that never post-filters
80
81pub use agent_context::{
82    AgentContext, AgentPermissions, AuditEntry, AuditOperation, AuditResult, ContextError,
83    ContextValue, DbPermissions, FsPermissions, OperationBudget, PendingWrite, ResourceType,
84    SessionId, SessionManager, TransactionScope,
85};
86pub use calc::{
87    BinaryOp, CalcError, Evaluator, Expr, Parser as CalcParser, RowContext, UnaryOp, calculate,
88    parse_expr,
89};
90pub use context_query::{
91    ContextQueryError, ContextQueryParser, ContextQueryResult, ContextSection, ContextSelectQuery,
92    HnswVectorIndex, SectionPriority, SectionResult, SimpleVectorIndex, VectorIndex,
93    VectorIndexStats, VectorSearchResult,
94};
95pub use optimizer_integration::{
96    CacheStats, ExecutionPlan, ExecutionStep, OptimizedExecutor, OptimizedQueryPlan, PlanCache,
97    StorageBackend, TableStats,
98};
99pub use plugin_table::{
100    PluginVirtualTable, VirtualColumnDef, VirtualColumnType, VirtualFilter, VirtualRow,
101    VirtualTable, VirtualTableError, VirtualTableRegistry, VirtualTableSchema, VirtualTableStats,
102};
103pub use sql::{
104    BinaryOperator, ColumnDef as SqlColumnDef, CreateTableStmt, DeleteStmt, DropTableStmt,
105    Expr as SqlExpr, InsertStmt, JoinType, Lexer, OrderByItem as SqlOrderBy, Parser as SqlParser,
106    SelectStmt, Span, SqlError, SqlResult, Statement, Token, TokenKind, UnaryOperator, UpdateStmt,
107};
108pub use token_budget::{
109    BudgetAllocation, BudgetSection, TokenBudgetConfig, TokenBudgetEnforcer, TokenEstimator,
110    TokenEstimatorConfig, truncate_rows, truncate_to_tokens,
111};
112pub use soch_ql::{
113    ColumnDef, ColumnType, ComparisonOp, Condition, CreateTableQuery, InsertQuery, LogicalOp,
114    OrderBy, ParseError, SelectQuery, SortDirection, SochQlParser, SochQuery, SochResult,
115    SochValue, WhereClause,
116};
117pub use soch_ql_executor::{
118    KeyRange, Predicate, PredicateCondition, QueryPlan, TokenReductionStats, SochQlExecutor,
119    estimate_token_reduction, execute_sochql,
120};
121
122// Task 1: Streaming context generation
123pub use streaming_context::{
124    RollingBudget, SectionChunk, StreamingConfig, StreamingContextExecutor, StreamingContextIter,
125};
126
127// Task 2: Automatic embedding generation
128pub use embedding_provider::{
129    CachedEmbeddingProvider, EmbeddingError, EmbeddingProvider, EmbeddingVectorIndex,
130    MockEmbeddingProvider,
131};
132
133// Task 3: Hybrid retrieval pipeline
134pub use hybrid_retrieval::{
135    FusionMethod, HybridQuery, HybridQueryExecutor, LexicalIndex, MetadataFilter,
136};
137
138// Task 4: Temporal decay scoring
139pub use temporal_decay::{
140    DecayCurve, TemporalDecayConfig, TemporalScorer, TemporallyDecayedResult,
141};
142
143// Task 5: Memory compaction
144pub use memory_compaction::{
145    Abstraction, CompactionStats, Episode, ExtractiveSummarizer, HierarchicalMemory, Summary,
146    Summarizer,
147};
148
149// Task 6: Exact token counting
150pub use exact_token_counter::{
151    ExactBudgetEnforcer, ExactTokenCounter, HeuristicTokenCounter, TokenCounter,
152};
153
154// Task 7: Semantic triggers
155pub use semantic_triggers::{
156    EscalationLevel, EventSource, LogLevel, SemanticTrigger, TriggerAction, TriggerBuilder,
157    TriggerError, TriggerEvent, TriggerIndex, TriggerMatch, TriggerStats,
158};
159
160// ============================================================================
161// Canonical Filter IR + Pushdown Contract (mm.md Tasks 1-8)
162// ============================================================================
163
164// Task 1: Canonical Filter IR (CNF/DNF with typed atoms)
165pub use filter_ir::{
166    AuthCapabilities, AuthScope, Disjunction, FilterAtom, FilterBuilder, FilterIR, FilterValue,
167    FilteredExecutor,
168};
169
170// Task 2: Namespace-Scoped Query API (mandatory namespace)
171pub use namespace::{
172    Namespace, NamespaceError, NamespaceScope, QueryRequest, ScopedQuery,
173};
174
175// Task 3: Metadata Index Primitives (bitmap + range accessors)
176pub use metadata_index::{
177    ConcurrentMetadataIndex, EqualityIndex, MetadataIndex, PostingSet, RangeIndex,
178};
179
180// Task 4: Unified Candidate Gate Interface
181pub use candidate_gate::{
182    AllowedBitmap, AllowedSet, CandidateGate, ExecutionStrategy,
183};
184
185// Task 5: Filter-Aware Vector Search with selectivity-driven fallback
186pub use filtered_vector_search::{
187    FilterAwareSearch, FilteredSearchConfig, FilteredSearchResult, FilteredSearchStrategy,
188    FilteredVectorStore, ScoredResult,
189};
190
191// Task 6: BM25 Filter Pushdown via posting-set intersection
192pub use bm25_filtered::{
193    Bm25Params, DisjunctiveBm25Executor, FilteredBm25Executor, FilteredPhraseExecutor,
194    InvertedIndex, PositionalIndex, PositionalPosting, PostingList,
195};
196
197// Task 7: Hybrid Fusion That Never Post-Filters
198pub use unified_fusion::{
199    Bm25Executor, Bm25QuerySpec, FilteredCandidates, FusionConfig, FusionEngine,
200    FusionMethod as UnifiedFusionMethod, FusionResult, Modality, UnifiedHybridExecutor,
201    UnifiedHybridQuery, VectorExecutor, VectorQuerySpec,
202};
203
204// Task 8: Capability Tokens + ACLs
205pub use capability_token::{
206    AclTagIndex, CapabilityToken, TokenBuilder, TokenCapabilities, TokenError, TokenSigner,
207    TokenValidator,
208};
sochdb_query/lib.rs

sochdb_query/
lib.rs
