oneai_vector/
lib.rs

1//! OneAI default retrieval backend stack.
2//!
3//! Implements the retrieval-backend traits declared in [`oneai-core::traits`]
4//! (`VectorBackend`, `KeywordBackend`, `RetrievalBackend`, `RerankerProvider`)
5//! using the P0-spike-verified library set (2026-07-26):
6//!
7//! | Backend | Crate | Feature | Role |
8//! |---|---|---|---|
9//! | [`InMemoryVectorBackend`] | (std only) | always | brute-force cosine, tests/tiny sets |
10//! | [`SqliteVecBackend`] | `sqlite-vec` | `sqlite` | exact KNN via `vec0`, mobile/small |
11//! | [`UsearchBackend`] | `usearch` | `usearch` | HNSW + mmap + pre-filter |
12//! | [`TantivyBm25Backend`] | `tantivy` + `tantivy-jieba` | `tantivy` | CJK BM25 |
13//! | [`BgeM3Embedder`] | `ort` | `ort` | local BGE-M3 dense embeddings |
14//! | [`BgeRerankerOnnx`] | `ort` | `ort` | local `bge-reranker-v2-m3` cross-encoder |
15//! | [`StandardRetrievalPipeline`] | (composes the above) | `default` | BM25+dense → RRF → rerank |
16//!
17//! ## The non-negotiable default pipeline
18//!
19//! [`StandardRetrievalPipeline`] runs BM25 + dense → RRF(k=60) → rerank
20//! (top-150 → top-K), which Anthropic's "Contextual Retrieval" evaluation
21//! showed cuts top-20 retrieval failure rate by 67%. An app that implements
22//! its own [`RetrievalBackend`] (e.g. Qdrant, which does hybrid natively)
23//! bypasses this pipeline — the trait docs in `oneai-core` say explicitly
24//! that skipping the keyword/sparse leg degrades Chinese short queries and
25//! unique-identifier lookups, so dropping BM25 is an informed decision.
26//!
27//! ## Feature flags
28//!
29//! - `default = ["sqlite", "usearch", "tantivy"]` — the storage backends +
30//!   BM25 + the pipeline all build without network access. The pipeline can
31//!   take any `EmbeddingService` (the ones in `oneai-rag`, or `BgeM3Embedder`
32//!   when `ort` is on).
33//! - `ort` — local ONNX embedder + reranker. Off by default because
34//!   `ort-sys` downloads a prebuilt `libonnxruntime` blob and the embedder
35//!   needs ONNX model files at runtime. Enable with `--features ort`.
36//!
37//! ## Design principle
38//!
39//! These implementations must NOT leak storage-internal types — an app that
40//! brings Qdrant/Milvus/pgvector/Elasticsearch implements against the public
41//! `oneai-core` trait surface, not against these internals. Backends here are
42//! the *reference* embedded stack.
43//!
44//! `unsafe` is confined to the `sqlite-vec` static-registration shim (a single
45//! FFI transmute required to register `sqlite3_vec_init` as a SQLite auto
46//! extension; every other backend is pure safe Rust over its crate's API).
47
48use std::collections::HashMap;
49
50use oneai_core::Metadata;
51
52pub mod fusion;
53pub mod in_memory;
54pub mod pipeline;
55
56#[cfg(feature = "sqlite")]
57pub mod sqlite_vec;
58
59#[cfg(feature = "usearch")]
60pub mod usearch_backend;
61
62#[cfg(feature = "tantivy")]
63pub mod tantivy_bm25;
64
65#[cfg(feature = "ort")]
66pub mod bge_m3;
67#[cfg(feature = "ort")]
68pub mod bge_reranker;
69
70pub use fusion::{dbsf_fuse, rrf_fuse};
71pub use in_memory::InMemoryVectorBackend;
72pub use pipeline::{
73    StandardRetrievalPipeline, StandardRetrievalPipelineBuilder, StandardRetrievalPipelineConfig,
74};
75
76#[cfg(feature = "ort")]
77pub use bge_m3::BgeM3Embedder;
78#[cfg(feature = "ort")]
79pub use bge_reranker::BgeRerankerOnnx;
80#[cfg(feature = "sqlite")]
81pub use sqlite_vec::SqliteVecBackend;
82#[cfg(feature = "tantivy")]
83pub use tantivy_bm25::TantivyBm25Backend;
84#[cfg(feature = "usearch")]
85pub use usearch_backend::UsearchBackend;
86
87// ─── shared helpers ─────────────────────────────────────────────────────────
88
89/// Cosine similarity between two equal-length vectors. Both must be non-empty
90/// and the same length; returns 0.0 for a zero-norm query (degenerate, no
91/// direction) rather than dividing by ~0.
92pub fn cosine(a: &[f32], b: &[f32]) -> f32 {
93    if a.is_empty() || a.len() != b.len() {
94        return 0.0;
95    }
96    let mut dot = 0.0f32;
97    let mut na = 0.0f32;
98    let mut nb = 0.0f32;
99    for (x, y) in a.iter().zip(b.iter()) {
100        dot += x * y;
101        na += x * x;
102        nb += y * y;
103    }
104    let denom = (na.sqrt() * nb.sqrt()).max(1e-12);
105    if !denom.is_finite() || denom <= 1e-12 {
106        return 0.0;
107    }
108    dot / denom
109}
110
111/// Merge two metadata maps, `b` taking precedence over `a` (used when fusing
112/// legs that each carry metadata for the same id).
113pub(crate) fn merge_meta(a: &Metadata, b: &Metadata) -> Metadata {
114    let mut out = a.clone();
115    for (k, v) in b {
116        out.insert(k.clone(), v.clone());
117    }
118    out
119}
120
121/// Convert a metadata map to a JSON string for storage in single-column backends.
122pub(crate) fn meta_to_json(meta: &Metadata) -> String {
123    serde_json::to_string(&meta).unwrap_or_else(|_| "{}".into())
124}
125
126/// Parse a metadata map from a JSON string; empty/invalid → empty map.
127pub(crate) fn meta_from_json(s: &str) -> Metadata {
128    serde_json::from_str::<HashMap<String, String>>(s).unwrap_or_default()
129}
oneai_vector/lib.rs

oneai_vector/
lib.rs
