sochdb_vector/
lib.rs

1//! Streaming Elimination Vector Search Engine
2//!
3//! A silicon-sympathetic, CPU-first ANN design that avoids pointer chasing,
4//! supports updates, and makes recall a controllable knob.
5//!
6//! # Architecture
7//!
8//! The engine uses two complementary views for candidate generation:
9//! - **RDF (Rare-Dominant Fingerprint)**: IR-style inverted lists for precision
10//! - **BPS (Block Projection Sketch)**: Dense-friendly streaming scans for recall
11//!
12//! Final ranking uses int8 dot products with outlier-aware correction.
13//!
14//! # SIMD Acceleration
15//!
16//! The engine uses pure Rust SIMD implementations for critical operations:
17//! - BPS scans: AVX2 (32x) / AVX512 (64x) / NEON (16x) speedup
18//! - int8 dot products: 8x speedup on AVX2
19//! - Visibility checks: 4x speedup on AVX2 / 2x on NEON
20//!
21//! All SIMD code is written in Rust using `core::arch` intrinsics,
22//! enabling cross-function inlining and eliminating FFI overhead.
23//!
24//! See [`simd`] module for the pure Rust implementations and
25//! [`dispatch`] module for runtime CPU detection and kernel dispatch.
26
27pub mod bmi2_paths; // P1: BMI2 fast paths with PEXT/PDEP for bit packing/unpacking
28pub mod catalog;
29pub mod compaction; // P1: Per-shard compaction isolation
30pub mod config;
31pub mod dispatch;
32pub mod error;
33pub mod filter;
34pub mod filter_pushdown; // P2: Filter/projection pushdown plugin API
35pub mod jit_ir; // P2: JIT compilation for filter expressions
36pub mod lsm;
37pub mod numa_alloc; // P1: NUMA-aware memory allocation
38pub mod outlier_encoding; // P1: Optimized outlier encoding with bitvector/sorted list hybrid
39pub mod query;
40pub mod rerank;
41pub mod rotation;
42pub mod search_plan; // P1: Quantization-aware search planning with SLA optimization
43pub mod segment;
44pub mod shard_topology; // P1: Shard-first ANN routing to minimize fan-out
45pub mod simd;
46pub mod types;
47
48// Performance optimization modules (jj.txt optimizations)
49pub mod async_lsm; // Non-Blocking LSM Sealing with WAL durability
50pub mod async_rotation; // Async Rotation Pipeline (background Walsh-Hadamard)
51pub mod batch_segment_writer; // True Batch SegmentWriter Ingest with contiguous API
52pub mod lazy_segment;
53pub mod simd_hadamard; // SIMD-Accelerated Walsh-Hadamard Transform // Lazy BPS/RDF/Rerank Construction (build-on-first-query)
54
55// Hybrid search and multi-vector modules (Task 4-6)
56pub mod bm25; // BM25 scoring for lexical search
57pub mod hybrid; // Hybrid search with RRF fusion
58pub mod inverted_index; // Inverted index for keyword search
59pub mod multi_vector; // Multi-vector documents with aggregation
60pub mod tombstones; // Tombstone-based logical deletion
61
62// Task implementations for operationalized vector search
63pub mod compressed_routing; // Task 4: Routing cache-resident
64pub mod cost_model; // Task 1: Cost model budgets
65pub mod filter_indexing; // Task 7: Cardinality-aware filter indexing
66pub mod guarantee_ladder; // Task 2: Guarantee ladder modes
67pub mod hot_path_layout;
68pub mod list_bounds; // Task 3: Cosine/Dot list bounds
69pub mod portable_simd; // Task 6: Portable SIMD kernels
70pub mod quantization_calibration; // Task 5: Quantization error calibration
71pub mod query_telemetry; // Task 10: Per-query telemetry
72pub mod segment_compaction; // Task 9: Drift-resilient compaction
73pub mod ssd_rerank; // Task 8: SSD rerank executor // Task 16: Vector hot-path layout
74pub mod wasm_rerank; // In-engine WASM rerank plugin interface
75
76pub use wasm_rerank::{
77    ProvenanceRerankOutput, RerankCandidate, RerankResult, WasmRerankPlugin, attach_provenance,
78};
79
80// Re-export main types
81pub use config::EngineConfig;
82pub use error::{Error, Result};
83pub use query::QueryEngine;
84pub use search_plan::{
85    OptimizationMode, PipelineStage, SearchPlan, SearchPlanner, SearchSLA, StageQuantLevel,
86};
87pub use segment::Segment;
88pub use shard_topology::{Centroid, RoutingDecision, ShardRouter, ShardTopology, TopologyConfig};
89pub use types::*;
90
91// Re-export SIMD dispatch for cross-crate usage
92pub use dispatch::{
93    BpsScanDispatcher, CpuFeatures, DotI8Dispatcher, SimdLevel, VisibilityDispatcher, cpu_features,
94    dispatch_info, simd_available, simd_level,
95};
96
97// Re-export hybrid search types
98pub use bm25::{BM25Config, BM25Scorer, BM25Stats, tokenize, tokenize_minimal};
99pub use hybrid::{ComponentScores, HybridSearchEngine, RRFConfig, RRFFusion, SearchResult};
100pub use inverted_index::{InvertedIndex, InvertedIndexBuilder, Posting, PostingList};
101pub use multi_vector::{
102    AggregationMethod, DocumentScore, MultiVectorAggregator, MultiVectorConfig,
103    MultiVectorDocument, MultiVectorError, MultiVectorMapping,
104};
105pub use tombstones::{TombstoneError, TombstoneFilter, TombstoneManager};
sochdb_vector/lib.rs

sochdb_vector/
lib.rs
