sift_core/
lib.rs

1//! Composable indexed code search engine.
2//!
3//! `sift-core` builds on-disk indexes over codebases and uses them to narrow
4//! candidate files before running the full regex engine. The index layer is
5//! designed for multiple coexisting index types: each type independently
6//! narrows candidates, and the [`Indexes`] registry intersects their results.
7//!
8//! Today the shipped index type is a trigram index ([`TrigramIndex`]), which
9//! records overlapping 3-byte sequences and achieves up to 60x speedup over
10//! ripgrep on indexed queries. Additional index types (AST indexes, dependency
11//! graphs, vector indexes) can be added by extending the [`IndexKind`] and
12//! [`Index`] enums.
13//!
14//! # Architecture
15//!
16//! ```text
17//! IndexStore::build(kinds) -> snapshot with index artifacts
18//! Indexes::open(sift_dir)  -> registry of opened indexes
19//! QueryPlanner::candidates -> intersect index candidate sets
20//! SearchQuery::run          -> regex scan over narrowed candidates
21//! ```
22//!
23//! **Walking:** [`WalkBuilder`] from the [`ignore`] crate.
24
25pub mod candidate;
26pub mod grep;
27pub use grep::GrepRun;
28mod index;
29pub mod query;
30pub mod search;
31
32pub use candidate::Candidate;
33
34pub use search::{SearchError, SearchOutcome, SearchQuery};
35
36pub use ignore::{Walk, WalkBuilder};
37
38pub use index::config::IndexWalkConfig;
39pub use index::meta::StoreMeta;
40pub use index::store::IndexStore;
41pub use index::trigram::{TrigramIndex, TrigramIndexError};
42pub use index::{
43    CorpusKind, CorpusMeta, CorpusSpec, FileId, FilterMeta, Index, IndexConfig, IndexError,
44    IndexId, IndexKind, Indexes, PlanMode, QueryPlanOutput, WalkMeta,
45};
46
47pub use query::{CandidateRequirement, QueryFlags, QueryPlanner, QuerySpec, UnindexedStrategy};
48
49use thiserror::Error;
50
51pub const SIFT_DIR: &str = ".sift";
52pub const FILES_BIN: &str = "files.bin";
53pub const LEXICON_BIN: &str = "lexicon.bin";
54pub const POSTINGS_BIN: &str = "postings.bin";
55pub const TRIGRAMS_BIN: &str = "trigrams.bin";
56
57/// Top-level umbrella error for all core operations.
58#[derive(Debug, Error)]
59pub enum Error {
60    #[error(transparent)]
61    Index(#[from] IndexError),
62
63    #[error(transparent)]
64    Search(#[from] SearchError),
65
66    #[error("IO error: {0}")]
67    Io(#[from] std::io::Error),
68
69    #[error("ignore walk error: {0}")]
70    Ignore(#[from] ignore::Error),
71
72    #[error("regex error: {0}")]
73    Regex(#[from] Box<regex_automata::meta::BuildError>),
74}
75
76impl From<crate::index::trigram::TrigramIndexError> for Error {
77    fn from(e: crate::index::trigram::TrigramIndexError) -> Self {
78        Self::Index(IndexError::Trigram(e))
79    }
80}
81
82pub type Result<T> = std::result::Result<T, Error>;
83
84#[cfg(test)]
85mod tests {
86    use super::*;
87    use crate::search::{CaseMode, SearchOptions, VisibilityConfig};
88    use std::fs;
89    use tempfile::TempDir;
90
91    fn build_trigram_in_tmp(tmp: &TempDir, corpus_path: &std::path::Path) -> TrigramIndex {
92        let sift_dir = tmp.path().join(".sift");
93        let trigram_dir = sift_dir.join("trigram");
94        let config = IndexConfig {
95            corpus: CorpusSpec {
96                root: corpus_path,
97                kind: CorpusKind::Directory,
98                follow_links: false,
99                include_paths: &[],
100                exclude_paths: &[],
101            },
102            walk: IndexWalkConfig::new(false),
103            visibility: VisibilityConfig::default(),
104        };
105        TrigramIndex::build(&config, &trigram_dir, &[]).expect("build index")
106    }
107
108    fn build_index_in_tmp(tmp: &TempDir, corpus_path: &std::path::Path) -> Index {
109        Index::Trigram(build_trigram_in_tmp(tmp, corpus_path))
110    }
111
112    #[test]
113    fn build_open_search_finds_line() {
114        let tmp = TempDir::new().expect("create temp dir");
115        let corpus = tmp.path().join("corpus");
116        fs::create_dir_all(corpus.join("src")).expect("create src dir");
117        fs::write(corpus.join("src/lib.rs"), "fn hello() {\n  let x = 1;\n}\n")
118            .expect("write test file");
119
120        let tri = build_trigram_in_tmp(&tmp, &corpus);
121        assert!(
122            tri.file_path(FileId::new(0)).is_some(),
123            "should have indexed files"
124        );
125        let index = Index::Trigram(tri);
126
127        let pat = vec![r"let\s+x".to_string()];
128        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
129        let hits = q.collect_index_matches(&index).expect("search");
130        assert_eq!(hits.len(), 1);
131        assert!(hits[0].file.ends_with("src/lib.rs"));
132        assert_eq!(hits[0].line, 2);
133    }
134
135    #[test]
136    fn indexed_search_matches_walk_search_for_literal() {
137        let tmp = TempDir::new().expect("create temp dir");
138        let corpus = tmp.path().join("corpus");
139        fs::create_dir_all(corpus.join("a")).expect("create dir a");
140        fs::create_dir_all(corpus.join("b")).expect("create dir b");
141        fs::write(corpus.join("a/x.txt"), "alpha beta\n").expect("write file a");
142        fs::write(corpus.join("b/y.txt"), "gamma delta\n").expect("write file b");
143
144        let index = build_index_in_tmp(&tmp, &corpus);
145
146        let pat = vec!["beta".to_string()];
147        let opts = SearchOptions::default();
148        let q = SearchQuery::new(&pat, opts).expect("compile search");
149        let naive = q.collect_walk_matches(&corpus).expect("walk search");
150        let hits = q.collect_index_matches(&index).expect("index search");
151        assert_eq!(hits, naive);
152    }
153
154    #[test]
155    fn full_scan_parallel_candidate_path_finds_all_files() {
156        let tmp = TempDir::new().expect("create temp dir");
157        let corpus = tmp.path().join("corpus");
158        fs::create_dir_all(corpus.join("d")).expect("create dir");
159        let n_files = 8;
160        for i in 0..n_files {
161            fs::write(
162                corpus.join("d").join(format!("f{i}.txt")),
163                format!("line {i} needle\n"),
164            )
165            .expect("write file");
166        }
167
168        let tri = build_trigram_in_tmp(&tmp, &corpus);
169        for i in 0..n_files {
170            assert!(
171                tri.file_path(FileId::new(i)).is_some(),
172                "file {i} should be indexed"
173            );
174        }
175        let index = Index::Trigram(tri);
176
177        let pat = vec!["needle".to_string()];
178        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
179        let hits = q.collect_index_matches(&index).expect("search");
180        assert_eq!(hits.len(), n_files);
181    }
182
183    #[test]
184    fn index_and_walk_return_same_matches_for_literal_pattern() {
185        let tmp = TempDir::new().expect("create temp dir");
186        let corpus = tmp.path().join("corpus");
187        fs::create_dir_all(corpus.join("keep")).expect("create keep dir");
188        fs::write(corpus.join("keep/a.txt"), "one\ntwo beta\n").expect("write file a");
189        fs::write(corpus.join("keep/b.txt"), "three\n").expect("write file b");
190
191        let index = build_index_in_tmp(&tmp, &corpus);
192
193        let pat = vec!["one".to_string(), "three".to_string()];
194        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
195        let mut from_index = q.collect_index_matches(&index).expect("index search");
196        let mut from_walk = q.collect_walk_matches(&corpus).expect("walk search");
197        from_index.sort_by(|a, b| (&a.file, a.line, &a.text).cmp(&(&b.file, b.line, &b.text)));
198        from_walk.sort_by(|a, b| (&a.file, a.line, &a.text).cmp(&(&b.file, b.line, &b.text)));
199        assert_eq!(from_index, from_walk);
200    }
201
202    #[test]
203    fn multi_pattern_search_matches_either() {
204        let tmp = TempDir::new().expect("create temp dir");
205        let corpus = tmp.path().join("corpus");
206        fs::create_dir_all(&corpus).expect("create corpus");
207        fs::write(corpus.join("a.txt"), "hello world\nfoo bar\n").expect("write file");
208
209        let index = build_index_in_tmp(&tmp, &corpus);
210
211        let pat = vec!["hello".to_string(), "foo".to_string()];
212        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
213        let hits = q.collect_index_matches(&index).expect("search");
214        assert_eq!(hits.len(), 2);
215    }
216
217    #[test]
218    fn case_insensitive_search_matches_variants() {
219        let tmp = TempDir::new().expect("create temp dir");
220        let corpus = tmp.path().join("corpus");
221        fs::create_dir_all(&corpus).expect("create corpus");
222        fs::write(corpus.join("a.txt"), "Hello WORLD\n").expect("write file");
223
224        let index = build_index_in_tmp(&tmp, &corpus);
225
226        let pat = vec!["hello".to_string()];
227        let opts = SearchOptions {
228            case_mode: CaseMode::Insensitive,
229            ..SearchOptions::default()
230        };
231        let q = SearchQuery::new(&pat, opts).expect("compile search");
232        let hits = q.collect_index_matches(&index).expect("search");
233        assert_eq!(hits.len(), 1);
234    }
235}
sift_core/lib.rs

sift_core/
lib.rs
