sift_core/
lib.rs

1//! Fast indexed regex search over codebases — core engine.
2//!
3//! **Walking:** [`WalkBuilder`] from the [`ignore`] crate.
4
5pub mod candidate;
6pub mod grep;
7pub use grep::GrepRun;
8mod index;
9pub mod query;
10mod search;
11
12pub use candidate::Candidate;
13
14pub use search::{
15    BinaryMode, CandidateFilter, CandidateFilterConfig, CaseMode, ColorChoice, ColumnLimit,
16    ColumnOverflow, FilenameMode, GlobConfig, HiddenMode, IgnoreConfig, IgnoreSources,
17    LineStyleFlags, LinkTraversal, Match, MatchEmissionMode, OutputEmission, PassthruMode,
18    PathDisplay, PatternCompiler, RecordTerminator, SearchCollection, SearchError, SearchLineStyle,
19    SearchMatchFlags, SearchMode, SearchOptions, SearchOutcome, SearchOutput, SearchOutputFormat,
20    SearchQuery, SearchRecordStyle, SearchSeparators, SearchStats, TypeDef, VisibilityConfig,
21    WalkOptions, ZeroCountMode,
22};
23
24pub use ignore::{Walk, WalkBuilder};
25
26pub use index::config::WalkOptions as IndexWalkOptions;
27pub use index::meta::StoreMeta;
28pub use index::store::IndexStore;
29pub use index::trigram::{TrigramIndex, TrigramIndexError};
30pub use index::{
31    CorpusKind, CorpusMeta, CorpusSpec, FileId, FilterMeta, Index, IndexConfig, IndexError,
32    IndexId, IndexKind, Indexes, PlanMode, QueryPlanOutput, WalkMeta,
33};
34
35pub use query::{CandidateRequirement, QueryFlags, QueryPlanner, QuerySpec};
36
37use thiserror::Error;
38
39pub const SIFT_DIR: &str = ".sift";
40pub const FILES_BIN: &str = "files.bin";
41pub const LEXICON_BIN: &str = "lexicon.bin";
42pub const POSTINGS_BIN: &str = "postings.bin";
43pub const TRIGRAMS_BIN: &str = "trigrams.bin";
44
45/// Top-level umbrella error for all core operations.
46#[derive(Debug, Error)]
47pub enum Error {
48    #[error(transparent)]
49    Index(#[from] IndexError),
50
51    #[error(transparent)]
52    Search(#[from] SearchError),
53
54    #[error("IO error: {0}")]
55    Io(#[from] std::io::Error),
56
57    #[error("ignore walk error: {0}")]
58    Ignore(#[from] ignore::Error),
59
60    #[error("regex error: {0}")]
61    Regex(#[from] Box<regex_automata::meta::BuildError>),
62}
63
64impl From<crate::index::trigram::TrigramIndexError> for Error {
65    fn from(e: crate::index::trigram::TrigramIndexError) -> Self {
66        Self::Index(IndexError::Trigram(e))
67    }
68}
69
70pub type Result<T> = std::result::Result<T, Error>;
71
72#[cfg(test)]
73mod tests {
74    use super::*;
75    use std::fs;
76    use tempfile::TempDir;
77
78    fn build_trigram_in_tmp(tmp: &TempDir, corpus_path: &std::path::Path) -> TrigramIndex {
79        let sift_dir = tmp.path().join(".sift");
80        let trigram_dir = sift_dir.join("trigram");
81        let config = IndexConfig {
82            corpus: CorpusSpec {
83                root: corpus_path,
84                kind: CorpusKind::Directory,
85                follow_links: false,
86                include_paths: &[],
87                exclude_paths: &[],
88            },
89            walk: IndexWalkOptions::new(false),
90            visibility: VisibilityConfig::default(),
91        };
92        TrigramIndex::build(&config, &trigram_dir, &[]).expect("build index")
93    }
94
95    fn build_index_in_tmp(tmp: &TempDir, corpus_path: &std::path::Path) -> Index {
96        Index::Trigram(build_trigram_in_tmp(tmp, corpus_path))
97    }
98
99    #[test]
100    fn build_open_search_finds_line() {
101        let tmp = TempDir::new().expect("create temp dir");
102        let corpus = tmp.path().join("corpus");
103        fs::create_dir_all(corpus.join("src")).expect("create src dir");
104        fs::write(corpus.join("src/lib.rs"), "fn hello() {\n  let x = 1;\n}\n")
105            .expect("write test file");
106
107        let tri = build_trigram_in_tmp(&tmp, &corpus);
108        assert!(
109            tri.file_path(FileId::new(0)).is_some(),
110            "should have indexed files"
111        );
112        let index = Index::Trigram(tri);
113
114        let pat = vec![r"let\s+x".to_string()];
115        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
116        let hits = q.collect_index_matches(&index).expect("search");
117        assert_eq!(hits.len(), 1);
118        assert!(hits[0].file.ends_with("src/lib.rs"));
119        assert_eq!(hits[0].line, 2);
120    }
121
122    #[test]
123    fn indexed_search_matches_walk_search_for_literal() {
124        let tmp = TempDir::new().expect("create temp dir");
125        let corpus = tmp.path().join("corpus");
126        fs::create_dir_all(corpus.join("a")).expect("create dir a");
127        fs::create_dir_all(corpus.join("b")).expect("create dir b");
128        fs::write(corpus.join("a/x.txt"), "alpha beta\n").expect("write file a");
129        fs::write(corpus.join("b/y.txt"), "gamma delta\n").expect("write file b");
130
131        let index = build_index_in_tmp(&tmp, &corpus);
132
133        let pat = vec!["beta".to_string()];
134        let opts = SearchOptions::default();
135        let q = SearchQuery::new(&pat, opts).expect("compile search");
136        let naive = q.collect_walk_matches(&corpus).expect("walk search");
137        let hits = q.collect_index_matches(&index).expect("index search");
138        assert_eq!(hits, naive);
139    }
140
141    #[test]
142    fn full_scan_parallel_candidate_path_finds_all_files() {
143        let tmp = TempDir::new().expect("create temp dir");
144        let corpus = tmp.path().join("corpus");
145        fs::create_dir_all(corpus.join("d")).expect("create dir");
146        let n_files = 8;
147        for i in 0..n_files {
148            fs::write(
149                corpus.join("d").join(format!("f{i}.txt")),
150                format!("line {i} needle\n"),
151            )
152            .expect("write file");
153        }
154
155        let tri = build_trigram_in_tmp(&tmp, &corpus);
156        for i in 0..n_files {
157            assert!(
158                tri.file_path(FileId::new(i)).is_some(),
159                "file {i} should be indexed"
160            );
161        }
162        let index = Index::Trigram(tri);
163
164        let pat = vec!["needle".to_string()];
165        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
166        let hits = q.collect_index_matches(&index).expect("search");
167        assert_eq!(hits.len(), n_files);
168    }
169
170    #[test]
171    fn index_and_walk_return_same_matches_for_literal_pattern() {
172        let tmp = TempDir::new().expect("create temp dir");
173        let corpus = tmp.path().join("corpus");
174        fs::create_dir_all(corpus.join("keep")).expect("create keep dir");
175        fs::write(corpus.join("keep/a.txt"), "one\ntwo beta\n").expect("write file a");
176        fs::write(corpus.join("keep/b.txt"), "three\n").expect("write file b");
177
178        let index = build_index_in_tmp(&tmp, &corpus);
179
180        let pat = vec!["one".to_string(), "three".to_string()];
181        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
182        let mut from_index = q.collect_index_matches(&index).expect("index search");
183        let mut from_walk = q.collect_walk_matches(&corpus).expect("walk search");
184        from_index.sort_by(|a, b| (&a.file, a.line, &a.text).cmp(&(&b.file, b.line, &b.text)));
185        from_walk.sort_by(|a, b| (&a.file, a.line, &a.text).cmp(&(&b.file, b.line, &b.text)));
186        assert_eq!(from_index, from_walk);
187    }
188
189    #[test]
190    fn multi_pattern_search_matches_either() {
191        let tmp = TempDir::new().expect("create temp dir");
192        let corpus = tmp.path().join("corpus");
193        fs::create_dir_all(&corpus).expect("create corpus");
194        fs::write(corpus.join("a.txt"), "hello world\nfoo bar\n").expect("write file");
195
196        let index = build_index_in_tmp(&tmp, &corpus);
197
198        let pat = vec!["hello".to_string(), "foo".to_string()];
199        let q = SearchQuery::new(&pat, SearchOptions::default()).expect("compile search");
200        let hits = q.collect_index_matches(&index).expect("search");
201        assert_eq!(hits.len(), 2);
202    }
203
204    #[test]
205    fn case_insensitive_search_matches_variants() {
206        let tmp = TempDir::new().expect("create temp dir");
207        let corpus = tmp.path().join("corpus");
208        fs::create_dir_all(&corpus).expect("create corpus");
209        fs::write(corpus.join("a.txt"), "Hello WORLD\n").expect("write file");
210
211        let index = build_index_in_tmp(&tmp, &corpus);
212
213        let pat = vec!["hello".to_string()];
214        let opts = SearchOptions {
215            case_mode: CaseMode::Insensitive,
216            ..SearchOptions::default()
217        };
218        let q = SearchQuery::new(&pat, opts).expect("compile search");
219        let hits = q.collect_index_matches(&index).expect("search");
220        assert_eq!(hits.len(), 1);
221    }
222}
sift_core/lib.rs

sift_core/
lib.rs
