semtree_rag/lib.rs
1//! **RAG pipeline for code: index, search, and inject codebase context into
2//! LLM prompts - fully on-device, no daemon, no API key.**
3//!
4//! `semtree-rag` is the composable core behind the [`semtree`] CLI. It wires a
5//! pluggable [`Embedder`](semtree_embed::Embedder) and
6//! [`VectorStore`](semtree_store::VectorStore) into a full pipeline -
7//! parse → chunk → embed → index → search - that you can drop into your own
8//! tool, an MCP server, or an LLM context provider.
9//!
10//! # Pipeline at a glance
11//!
12//! | Stage | Building blocks |
13//! |-------|-----------------|
14//! | Index a directory | [`Indexer`] + [`ChunkRegistry`] |
15//! | Search (vector / BM25 / fused) | [`HybridSearcher`] over [`SearchEngine`] + [`LexicalIndex`] |
16//! | Build an LLM prompt | [`ContextBuilder`] → [`ContextWindow`] |
17//! | Incremental re-index | [`FileManifest`] |
18//!
19//! # Example
20//!
21//! ```no_run
22//! use std::sync::Arc;
23//! use semtree_embed::fastembed::FastEmbedder;
24//! use semtree_store::usearch::UsearchStore;
25//! use semtree_rag::{ChunkRegistry, HybridSearcher, Indexer, LexicalIndex, SearchEngine, SearchMode};
26//!
27//! # async fn run() -> Result<(), Box<dyn std::error::Error>> {
28//! // Backends are traits - swap FastEmbedder for OpenAI, UsearchStore for Qdrant.
29//! let embedder = Arc::new(FastEmbedder::new()?);
30//! let store = Arc::new(UsearchStore::new(384)?);
31//!
32//! // Index: parse -> chunk -> embed -> store. The registry keeps chunk metadata.
33//! let mut registry = ChunkRegistry::default();
34//! Indexer::new(embedder.clone(), store.clone())
35//! .index_dir(std::path::Path::new("./src"), &mut registry, None, |_, _| {})
36//! .await?;
37//!
38//! // Search: fuse vector similarity with BM25 keyword matching.
39//! let engine = SearchEngine::new(embedder, store);
40//! let lexical = LexicalIndex::from_chunks(registry.iter());
41//! let hits = HybridSearcher::new(engine, lexical)
42//! .search("how is authentication handled", 5, SearchMode::Hybrid)
43//! .await?;
44//! # Ok(())
45//! # }
46//! ```
47//!
48//! A complete, runnable version lives in [`examples/build_your_own.rs`].
49//!
50//! [`semtree`]: https://crates.io/crates/semtree
51//! [`examples/build_your_own.rs`]: https://github.com/rustkit-ai/semtree/blob/main/crates/semtree-rag/examples/build_your_own.rs
52
53mod context;
54mod error;
55mod hybrid;
56mod indexer;
57mod lexical;
58mod manifest;
59mod registry;
60mod search;
61
62pub use context::{ContextBuilder, ContextSnippet, ContextWindow};
63pub use error::RagError;
64pub use hybrid::{HybridSearcher, SearchMode};
65pub use indexer::{collect_indexable_files, Indexer};
66pub use lexical::LexicalIndex;
67pub use manifest::FileManifest;
68pub use registry::ChunkRegistry;
69pub use search::SearchEngine;