sift_core/
lib.rs

1//! Fast indexed regex search over codebases — core engine.
2//!
3//! **Walking:** [`WalkBuilder`] from the [`ignore`] crate.
4
5pub mod candidate;
6pub mod grep;
7mod index;
8pub(crate) mod query;
9mod search;
10
11pub use candidate::Candidate;
12
13pub use search::{
14    BinaryMode, CandidateFilter, CandidateFilterConfig, CaseMode, ColorChoice, ColumnLimit,
15    ColumnOverflow, FilenameMode, GlobConfig, HiddenMode, IgnoreConfig, IgnoreSources,
16    LineStyleFlags, LinkTraversal, Match, MatchEmissionMode, OutputEmission, PassthruMode,
17    PathDisplay, PatternCompiler, RecordTerminator, SearchError, SearchLineStyle, SearchMatchFlags,
18    SearchMode, SearchOptions, SearchOutcome, SearchOutput, SearchOutputFormat, SearchQuery,
19    SearchRecordStyle, SearchSeparators, SearchStats, TypeDef, VisibilityConfig, WalkOptions,
20    ZeroCountMode, discover_files,
21};
22
23pub use ignore::{Walk, WalkBuilder};
24
25pub use index::meta::StoreMeta;
26pub use index::store::IndexStore;
27pub use index::trigram::{TrigramIndex, TrigramIndexError};
28pub use index::{
29    CorpusKind, CorpusSpec, FileId, Index, IndexConfig, IndexError, IndexId, IndexKind, Indexes,
30    PlanMode, QueryPlanOutput,
31};
32
33pub use query::{QueryFlags, QuerySpec};
34
35use thiserror::Error;
36
37pub const SIFT_DIR: &str = ".sift";
38pub const FILES_BIN: &str = "files.bin";
39pub const LEXICON_BIN: &str = "lexicon.bin";
40pub const POSTINGS_BIN: &str = "postings.bin";
41pub const TRIGRAMS_BIN: &str = "trigrams.bin";
42
43/// Top-level umbrella error for all core operations.
44#[derive(Debug, Error)]
45pub enum Error {
46    #[error(transparent)]
47    Index(#[from] IndexError),
48
49    #[error(transparent)]
50    Search(#[from] SearchError),
51
52    #[error("IO error: {0}")]
53    Io(#[from] std::io::Error),
54
55    #[error("ignore walk error: {0}")]
56    Ignore(#[from] ignore::Error),
57
58    #[error("regex error: {0}")]
59    Regex(#[from] Box<regex_automata::meta::BuildError>),
60}
61
62impl From<crate::index::trigram::TrigramIndexError> for Error {
63    fn from(e: crate::index::trigram::TrigramIndexError) -> Self {
64        Self::Index(IndexError::Trigram(e))
65    }
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[cfg(test)]
71mod tests {
72    use super::*;
73    use std::fs;
74    use tempfile::TempDir;
75
76    fn build_trigram_in_tmp(tmp: &TempDir, corpus_path: &std::path::Path) -> TrigramIndex {
77        let sift_dir = tmp.path().join(".sift");
78        let trigram_dir = sift_dir.join("trigram");
79        let config = IndexConfig {
80            corpus: CorpusSpec {
81                root: corpus_path,
82                kind: CorpusKind::Directory,
83                follow_links: false,
84                include_paths: &[],
85                exclude_paths: &[],
86            },
87            visibility: VisibilityConfig::default(),
88        };
89        TrigramIndex::build(&config, &trigram_dir).expect("build index")
90    }
91
92    fn build_index_in_tmp(tmp: &TempDir, corpus_path: &std::path::Path) -> Index {
93        Index::Trigram(build_trigram_in_tmp(tmp, corpus_path))
94    }
95
96    #[test]
97    fn build_open_search_finds_line() {
98        let tmp = TempDir::new().expect("create temp dir");
99        let corpus = tmp.path().join("corpus");
100        fs::create_dir_all(corpus.join("src")).expect("create src dir");
101        fs::write(corpus.join("src/lib.rs"), "fn hello() {\n  let x = 1;\n}\n")
102            .expect("write test file");
103
104        let tri = build_trigram_in_tmp(&tmp, &corpus);
105        assert!(
106            tri.file_path(FileId::new(0)).is_some(),
107            "should have indexed files"
108        );
109        let index = Index::Trigram(tri);
110
111        let pat = vec![r"let\s+x".to_string()];
112        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
113        let hits = q.collect_index_matches(&index).expect("search");
114        assert_eq!(hits.len(), 1);
115        assert!(hits[0].file.ends_with("src/lib.rs"));
116        assert_eq!(hits[0].line, 2);
117    }
118
119    #[test]
120    fn indexed_search_matches_walk_search_for_literal() {
121        let tmp = TempDir::new().expect("create temp dir");
122        let corpus = tmp.path().join("corpus");
123        fs::create_dir_all(corpus.join("a")).expect("create dir a");
124        fs::create_dir_all(corpus.join("b")).expect("create dir b");
125        fs::write(corpus.join("a/x.txt"), "alpha beta\n").expect("write file a");
126        fs::write(corpus.join("b/y.txt"), "gamma delta\n").expect("write file b");
127
128        let index = build_index_in_tmp(&tmp, &corpus);
129
130        let pat = vec!["beta".to_string()];
131        let opts = SearchOptions::default();
132        let q = SearchQuery::new(&pat, opts).expect("compile search");
133        let naive = q.collect_walk_matches(&corpus).expect("walk search");
134        let hits = q.collect_index_matches(&index).expect("index search");
135        assert_eq!(hits, naive);
136    }
137
138    #[test]
139    fn full_scan_parallel_candidate_path_finds_all_files() {
140        let tmp = TempDir::new().expect("create temp dir");
141        let corpus = tmp.path().join("corpus");
142        fs::create_dir_all(corpus.join("d")).expect("create dir");
143        let n_files = 8;
144        for i in 0..n_files {
145            fs::write(
146                corpus.join("d").join(format!("f{i}.txt")),
147                format!("line {i} needle\n"),
148            )
149            .expect("write file");
150        }
151
152        let tri = build_trigram_in_tmp(&tmp, &corpus);
153        for i in 0..n_files {
154            assert!(
155                tri.file_path(FileId::new(i)).is_some(),
156                "file {i} should be indexed"
157            );
158        }
159        let index = Index::Trigram(tri);
160
161        let pat = vec!["needle".to_string()];
162        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
163        let hits = q.collect_index_matches(&index).expect("search");
164        assert_eq!(hits.len(), n_files);
165    }
166
167    #[test]
168    fn full_scan_uses_files_bin_same_hits_as_fresh_walk() {
169        let tmp = TempDir::new().expect("create temp dir");
170        let corpus = tmp.path().join("corpus");
171        fs::create_dir_all(corpus.join("keep")).expect("create keep dir");
172        fs::write(corpus.join("keep/a.txt"), "one\ntwo beta\n").expect("write file a");
173        fs::write(corpus.join("keep/b.txt"), "three\n").expect("write file b");
174
175        let index = build_index_in_tmp(&tmp, &corpus);
176
177        let pat = vec![".*".to_string()];
178        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
179        let mut from_index = q.collect_index_matches(&index).expect("index search");
180        let mut from_walk = q.collect_walk_matches(&corpus).expect("walk search");
181        from_index.sort_by(|a, b| (&a.file, a.line, &a.text).cmp(&(&b.file, b.line, &b.text)));
182        from_walk.sort_by(|a, b| (&a.file, a.line, &a.text).cmp(&(&b.file, b.line, &b.text)));
183        assert_eq!(from_index, from_walk);
184    }
185
186    #[test]
187    fn multi_pattern_search_matches_either() {
188        let tmp = TempDir::new().expect("create temp dir");
189        let corpus = tmp.path().join("corpus");
190        fs::create_dir_all(&corpus).expect("create corpus");
191        fs::write(corpus.join("a.txt"), "hello world\nfoo bar\n").expect("write file");
192
193        let index = build_index_in_tmp(&tmp, &corpus);
194
195        let pat = vec!["hello".to_string(), "foo".to_string()];
196        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
197        let hits = q.collect_index_matches(&index).expect("search");
198        assert_eq!(hits.len(), 2);
199    }
200
201    #[test]
202    fn case_insensitive_search_matches_variants() {
203        let tmp = TempDir::new().expect("create temp dir");
204        let corpus = tmp.path().join("corpus");
205        fs::create_dir_all(&corpus).expect("create corpus");
206        fs::write(corpus.join("a.txt"), "Hello WORLD\n").expect("write file");
207
208        let index = build_index_in_tmp(&tmp, &corpus);
209
210        let pat = vec!["hello".to_string()];
211        let opts = SearchOptions {
212            case_mode: CaseMode::Insensitive,
213            ..SearchOptions::default()
214        };
215        let q = SearchQuery::new(&pat, opts).expect("compile search");
216        let hits = q.collect_index_matches(&index).expect("search");
217        assert_eq!(hits.len(), 1);
218    }
219}
sift_core/lib.rs

sift_core/
lib.rs
