Skip to main content

treesearch/engine/
indexer.rs

1//! Optional directory indexer with independently selectable progress reporting.
2//!
3//! Pipeline:
4//!   File Discovery (ignore crate) → Parallel Parse (rayon) → Batch Insert (FTS5)
5//!                                    ↓
6//!                            mtime/size fingerprinting for incremental index
7
8use std::collections::HashMap;
9use std::fs;
10use std::path::{Path, PathBuf};
11use std::time::Instant;
12
13use anyhow::{Context, Result};
14use ignore::WalkBuilder;
15#[cfg(feature = "progress")]
16use indicatif::{ProgressBar, ProgressStyle};
17use rayon::prelude::*;
18use tracing::{info, warn};
19
20use crate::config::TreeSearchConfig;
21use crate::document::Document;
22use crate::engine::fts::FTS5Index;
23use crate::parser::ParserRegistry;
24
25/// File fingerprint for incremental indexing.
26fn file_fingerprint(path: &Path) -> Option<String> {
27    let meta = fs::metadata(path).ok()?;
28    let mtime = meta
29        .modified()
30        .ok()?
31        .duration_since(std::time::UNIX_EPOCH)
32        .ok()?
33        .as_nanos();
34    let size = meta.len();
35    Some(format!("{}:{}", mtime, size))
36}
37
38/// Discover files respecting .gitignore and .treesearchignore.
39pub fn discover_files(
40    root: &Path,
41    config: &TreeSearchConfig,
42    follow_symlinks: bool,
43) -> Vec<PathBuf> {
44    discover_files_with_status(root, config, follow_symlinks).files
45}
46
47/// One bounded discovery result plus whether the whole tree was observed.
48struct FileDiscovery {
49    files: Vec<PathBuf>,
50    complete: bool,
51}
52
53/// Discovers supported files and records whether pruning evidence is complete.
54fn discover_files_with_status(
55    root: &Path,
56    config: &TreeSearchConfig,
57    follow_symlinks: bool,
58) -> FileDiscovery {
59    if config.max_dir_files == 0 {
60        return FileDiscovery {
61            files: Vec::new(),
62            complete: false,
63        };
64    }
65    let parser_registry = ParserRegistry::new();
66
67    let mut walker_builder = WalkBuilder::new(root);
68    walker_builder
69        .follow_links(follow_symlinks)
70        .hidden(true) // skip hidden files
71        .git_ignore(true)
72        .git_global(true)
73        .git_exclude(true)
74        .add_custom_ignore_filename(".treesearchignore")
75        .max_depth(None)
76        .sort_by_file_path(|left, right| left.cmp(right));
77    let walker = walker_builder.build();
78
79    let mut files: Vec<PathBuf> = Vec::new();
80    let mut complete = true;
81    for entry in walker {
82        let entry = match entry {
83            Ok(e) => e,
84            Err(e) => {
85                warn!("Walk error: {}", e);
86                complete = false;
87                continue;
88            }
89        };
90        if !entry.file_type().map(|ft| ft.is_file()).unwrap_or(false) {
91            continue;
92        }
93        let path = entry.into_path();
94        if parser_registry.supports(&path) {
95            files.push(path);
96        }
97        if files.len() >= config.max_dir_files {
98            warn!(
99                "Reached max_dir_files limit ({}), stopping discovery",
100                config.max_dir_files
101            );
102            complete = false;
103            break;
104        }
105    }
106    FileDiscovery { files, complete }
107}
108
109/// Index a directory into an FTS5 database.
110pub fn index_directory(
111    root: &Path,
112    fts_index: &FTS5Index,
113    config: &TreeSearchConfig,
114    follow_symlinks: bool,
115    show_progress: bool,
116) -> Result<IndexStats> {
117    let start = Instant::now();
118
119    // Discover files
120    let discovery = discover_files_with_status(root, config, follow_symlinks);
121    let files = discovery.files;
122    if files.is_empty() {
123        if discovery.complete {
124            let existing_meta = fts_index.get_all_index_meta()?;
125            for old_path in existing_meta.keys() {
126                if let Some(doc_id) = fts_index.get_doc_id_by_source_path(old_path)? {
127                    fts_index.delete_document(&doc_id)?;
128                }
129            }
130        }
131        info!("No supported files found in {:?}", root);
132        return Ok(IndexStats {
133            files_found: 0,
134            files_indexed: 0,
135            files_skipped: 0,
136            files_failed: 0,
137            nodes_indexed: 0,
138            duration_ms: start.elapsed().as_millis() as u64,
139        });
140    }
141
142    // Check existing fingerprints for incremental indexing
143    let existing_meta = fts_index.get_all_index_meta()?;
144
145    // Filter to files that need (re-)indexing
146    let mut to_index: Vec<PathBuf> = Vec::new();
147    let mut skipped = 0usize;
148    for file in &files {
149        let path_str = file.to_string_lossy().to_string();
150        let fp = file_fingerprint(file);
151        match (&fp, existing_meta.get(&path_str)) {
152            (Some(new_fp), Some(old_fp)) if new_fp == old_fp => {
153                skipped += 1;
154            }
155            _ => {
156                to_index.push(file.clone());
157            }
158        }
159    }
160
161    info!(
162        "Discovered {} files, {} unchanged (skipping), {} to index",
163        files.len(),
164        skipped,
165        to_index.len()
166    );
167
168    if to_index.is_empty() {
169        return Ok(IndexStats {
170            files_found: files.len(),
171            files_indexed: 0,
172            files_skipped: skipped,
173            files_failed: 0,
174            nodes_indexed: 0,
175            duration_ms: start.elapsed().as_millis() as u64,
176        });
177    }
178
179    #[cfg(not(feature = "progress"))]
180    let _ = show_progress;
181
182    #[cfg(feature = "progress")]
183    let pb = if show_progress {
184        let pb = ProgressBar::new(to_index.len() as u64);
185        pb.set_style(
186            ProgressStyle::default_bar()
187                .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos}/{len} {msg}")
188                .unwrap()
189                .progress_chars("█▓░"),
190        );
191        Some(pb)
192    } else {
193        None
194    };
195
196    // Parallel parsing in a request-local pool, rather than Rayon global defaults.
197    let parser_registry = ParserRegistry::new();
198    let pool = rayon::ThreadPoolBuilder::new()
199        .num_threads(effective_concurrency(
200            config.max_concurrency,
201            to_index.len(),
202        ))
203        .build()
204        .context("build bounded parser thread pool")?;
205    let parse_results: Vec<(PathBuf, Result<Document>)> = pool.install(|| {
206        to_index
207            .par_iter()
208            .map(|path| {
209                let result = parse_file(&parser_registry, path);
210                #[cfg(feature = "progress")]
211                if let Some(ref pb) = pb {
212                    pb.inc(1);
213                }
214                (path.clone(), result)
215            })
216            .collect()
217    });
218
219    #[cfg(feature = "progress")]
220    if let Some(ref pb) = pb {
221        pb.finish_with_message("Parsing complete");
222    }
223
224    // Batch insert into FTS5 (single-threaded for SQLite)
225    let mut indexed = 0usize;
226    let mut failed = 0usize;
227    let mut total_nodes = 0usize;
228    let mut new_meta: HashMap<String, String> = HashMap::new();
229
230    for (path, result) in parse_results {
231        match result {
232            Ok(doc) => {
233                let path_str = path.to_string_lossy().to_string();
234                match fts_index.index_document(&doc, false) {
235                    Ok(count) => {
236                        total_nodes += count;
237                        indexed += 1;
238                        if let Some(fp) = file_fingerprint(&path) {
239                            new_meta.insert(path_str, fp);
240                        }
241                    }
242                    Err(e) => {
243                        warn!("Index error for {:?}: {}", path, e);
244                        failed += 1;
245                    }
246                }
247            }
248            Err(e) => {
249                warn!("Parse error for {:?}: {}", path, e);
250                failed += 1;
251            }
252        }
253    }
254
255    // Batch commit fingerprints
256    fts_index.commit()?;
257    if !new_meta.is_empty() {
258        fts_index.set_index_meta_batch(&new_meta)?;
259    }
260
261    let mut pruned = 0;
262    if discovery.complete {
263        // Only a complete walk proves that a previously indexed path was deleted.
264        let all_paths: std::collections::HashSet<String> = files
265            .iter()
266            .map(|f| f.to_string_lossy().to_string())
267            .collect();
268        for old_path in existing_meta.keys() {
269            if !all_paths.contains(old_path) {
270                if let Some(doc_id) = fts_index.get_doc_id_by_source_path(old_path)? {
271                    fts_index.delete_document(&doc_id)?;
272                    pruned += 1;
273                }
274            }
275        }
276    } else {
277        warn!("Skipping stale-document pruning because discovery was incomplete");
278    }
279    if pruned > 0 {
280        info!("Pruned {} deleted documents from index", pruned);
281    }
282
283    let duration = start.elapsed().as_millis() as u64;
284    info!(
285        "Indexed {} files ({} nodes) in {}ms ({} failed, {} skipped)",
286        indexed, total_nodes, duration, failed, skipped
287    );
288
289    Ok(IndexStats {
290        files_found: files.len(),
291        files_indexed: indexed,
292        files_skipped: skipped,
293        files_failed: failed,
294        nodes_indexed: total_nodes,
295        duration_ms: duration,
296    })
297}
298
299/// Clamps parser parallelism to a non-zero value and the current work size.
300fn effective_concurrency(configured: usize, work_items: usize) -> usize {
301    configured.max(1).min(work_items.max(1))
302}
303
304/// Parse a single file using the parser registry.
305fn parse_file(registry: &ParserRegistry, path: &Path) -> Result<Document> {
306    let content = fs::read_to_string(path).with_context(|| format!("Failed to read {:?}", path))?;
307    registry
308        .parse_content(path, &content)?
309        .ok_or_else(|| anyhow::anyhow!("No parser found for {:?}", path))
310}
311
312/// Index statistics.
313#[derive(Debug, Clone)]
314pub struct IndexStats {
315    pub files_found: usize,
316    pub files_indexed: usize,
317    pub files_skipped: usize,
318    pub files_failed: usize,
319    pub nodes_indexed: usize,
320    pub duration_ms: u64,
321}
322
323impl std::fmt::Display for IndexStats {
324    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
325        write!(
326            f,
327            "Indexed: {} files ({} nodes) in {}ms | {} skipped, {} failed",
328            self.files_indexed,
329            self.nodes_indexed,
330            self.duration_ms,
331            self.files_skipped,
332            self.files_failed,
333        )
334    }
335}
336
337#[cfg(test)]
338mod tests {
339    use super::{discover_files_with_status, effective_concurrency};
340    use crate::config::TreeSearchConfig;
341
342    #[test]
343    fn parser_concurrency_is_bounded_by_policy_and_work() {
344        for (configured, work_items, expected) in [(0, 0, 1), (0, 5, 1), (2, 5, 2), (20, 5, 5)] {
345            assert_eq!(
346                effective_concurrency(configured, work_items),
347                expected,
348                "configured={configured}, work_items={work_items}"
349            );
350        }
351    }
352
353    #[cfg(feature = "parser-markdown")]
354    #[test]
355    fn bounded_discovery_is_marked_incomplete() {
356        let directory = tempfile::tempdir().unwrap();
357        std::fs::write(directory.path().join("a.md"), "# A").unwrap();
358        std::fs::write(directory.path().join("b.md"), "# B").unwrap();
359        let config = TreeSearchConfig {
360            max_dir_files: 1,
361            ..TreeSearchConfig::default()
362        };
363
364        let discovery = discover_files_with_status(directory.path(), &config, false);
365        assert_eq!(discovery.files.len(), 1);
366        assert!(!discovery.complete);
367        assert!(discovery.files[0].ends_with("a.md"));
368    }
369
370    #[cfg(feature = "parser-markdown")]
371    #[test]
372    fn incomplete_discovery_never_prunes_and_complete_empty_walk_does() {
373        use super::index_directory;
374        use crate::engine::fts::FTS5Index;
375
376        let directory = tempfile::tempdir().unwrap();
377        let first = directory.path().join("a.md");
378        let second = directory.path().join("b.md");
379        std::fs::write(&first, "# A").unwrap();
380        std::fs::write(&second, "# B").unwrap();
381        let index = FTS5Index::new(None, None).unwrap();
382        let full_config = TreeSearchConfig {
383            max_dir_files: 10,
384            ..TreeSearchConfig::default()
385        };
386        index_directory(directory.path(), &index, &full_config, false, false).unwrap();
387        assert_eq!(index.get_stats().unwrap().document_count, 2);
388
389        let bounded_config = TreeSearchConfig {
390            max_dir_files: 1,
391            ..full_config.clone()
392        };
393        index_directory(directory.path(), &index, &bounded_config, false, false).unwrap();
394        assert_eq!(index.get_stats().unwrap().document_count, 2);
395
396        std::fs::remove_file(first).unwrap();
397        std::fs::remove_file(second).unwrap();
398        index_directory(directory.path(), &index, &full_config, false, false).unwrap();
399        assert_eq!(index.get_stats().unwrap().document_count, 0);
400    }
401}