1use std::collections::HashMap;
9use std::fs;
10use std::path::{Path, PathBuf};
11use std::time::Instant;
12
13use anyhow::{Context, Result};
14use ignore::WalkBuilder;
15#[cfg(feature = "progress")]
16use indicatif::{ProgressBar, ProgressStyle};
17use rayon::prelude::*;
18use tracing::{info, warn};
19
20use crate::config::TreeSearchConfig;
21use crate::document::Document;
22use crate::engine::fts::FTS5Index;
23use crate::parser::ParserRegistry;
24
25fn file_fingerprint(path: &Path) -> Option<String> {
27 let meta = fs::metadata(path).ok()?;
28 let mtime = meta
29 .modified()
30 .ok()?
31 .duration_since(std::time::UNIX_EPOCH)
32 .ok()?
33 .as_nanos();
34 let size = meta.len();
35 Some(format!("{}:{}", mtime, size))
36}
37
38pub fn discover_files(
40 root: &Path,
41 config: &TreeSearchConfig,
42 follow_symlinks: bool,
43) -> Vec<PathBuf> {
44 discover_files_with_status(root, config, follow_symlinks).files
45}
46
47struct FileDiscovery {
49 files: Vec<PathBuf>,
50 complete: bool,
51}
52
53fn discover_files_with_status(
55 root: &Path,
56 config: &TreeSearchConfig,
57 follow_symlinks: bool,
58) -> FileDiscovery {
59 if config.max_dir_files == 0 {
60 return FileDiscovery {
61 files: Vec::new(),
62 complete: false,
63 };
64 }
65 let parser_registry = ParserRegistry::new();
66
67 let mut walker_builder = WalkBuilder::new(root);
68 walker_builder
69 .follow_links(follow_symlinks)
70 .hidden(true) .git_ignore(true)
72 .git_global(true)
73 .git_exclude(true)
74 .add_custom_ignore_filename(".treesearchignore")
75 .max_depth(None)
76 .sort_by_file_path(|left, right| left.cmp(right));
77 let walker = walker_builder.build();
78
79 let mut files: Vec<PathBuf> = Vec::new();
80 let mut complete = true;
81 for entry in walker {
82 let entry = match entry {
83 Ok(e) => e,
84 Err(e) => {
85 warn!("Walk error: {}", e);
86 complete = false;
87 continue;
88 }
89 };
90 if !entry.file_type().map(|ft| ft.is_file()).unwrap_or(false) {
91 continue;
92 }
93 let path = entry.into_path();
94 if parser_registry.supports(&path) {
95 files.push(path);
96 }
97 if files.len() >= config.max_dir_files {
98 warn!(
99 "Reached max_dir_files limit ({}), stopping discovery",
100 config.max_dir_files
101 );
102 complete = false;
103 break;
104 }
105 }
106 FileDiscovery { files, complete }
107}
108
109pub fn index_directory(
111 root: &Path,
112 fts_index: &FTS5Index,
113 config: &TreeSearchConfig,
114 follow_symlinks: bool,
115 show_progress: bool,
116) -> Result<IndexStats> {
117 let start = Instant::now();
118
119 let discovery = discover_files_with_status(root, config, follow_symlinks);
121 let files = discovery.files;
122 if files.is_empty() {
123 if discovery.complete {
124 let existing_meta = fts_index.get_all_index_meta()?;
125 for old_path in existing_meta.keys() {
126 if let Some(doc_id) = fts_index.get_doc_id_by_source_path(old_path)? {
127 fts_index.delete_document(&doc_id)?;
128 }
129 }
130 }
131 info!("No supported files found in {:?}", root);
132 return Ok(IndexStats {
133 files_found: 0,
134 files_indexed: 0,
135 files_skipped: 0,
136 files_failed: 0,
137 nodes_indexed: 0,
138 duration_ms: start.elapsed().as_millis() as u64,
139 });
140 }
141
142 let existing_meta = fts_index.get_all_index_meta()?;
144
145 let mut to_index: Vec<PathBuf> = Vec::new();
147 let mut skipped = 0usize;
148 for file in &files {
149 let path_str = file.to_string_lossy().to_string();
150 let fp = file_fingerprint(file);
151 match (&fp, existing_meta.get(&path_str)) {
152 (Some(new_fp), Some(old_fp)) if new_fp == old_fp => {
153 skipped += 1;
154 }
155 _ => {
156 to_index.push(file.clone());
157 }
158 }
159 }
160
161 info!(
162 "Discovered {} files, {} unchanged (skipping), {} to index",
163 files.len(),
164 skipped,
165 to_index.len()
166 );
167
168 if to_index.is_empty() {
169 return Ok(IndexStats {
170 files_found: files.len(),
171 files_indexed: 0,
172 files_skipped: skipped,
173 files_failed: 0,
174 nodes_indexed: 0,
175 duration_ms: start.elapsed().as_millis() as u64,
176 });
177 }
178
179 #[cfg(not(feature = "progress"))]
180 let _ = show_progress;
181
182 #[cfg(feature = "progress")]
183 let pb = if show_progress {
184 let pb = ProgressBar::new(to_index.len() as u64);
185 pb.set_style(
186 ProgressStyle::default_bar()
187 .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos}/{len} {msg}")
188 .unwrap()
189 .progress_chars("█▓░"),
190 );
191 Some(pb)
192 } else {
193 None
194 };
195
196 let parser_registry = ParserRegistry::new();
198 let pool = rayon::ThreadPoolBuilder::new()
199 .num_threads(effective_concurrency(
200 config.max_concurrency,
201 to_index.len(),
202 ))
203 .build()
204 .context("build bounded parser thread pool")?;
205 let parse_results: Vec<(PathBuf, Result<Document>)> = pool.install(|| {
206 to_index
207 .par_iter()
208 .map(|path| {
209 let result = parse_file(&parser_registry, path);
210 #[cfg(feature = "progress")]
211 if let Some(ref pb) = pb {
212 pb.inc(1);
213 }
214 (path.clone(), result)
215 })
216 .collect()
217 });
218
219 #[cfg(feature = "progress")]
220 if let Some(ref pb) = pb {
221 pb.finish_with_message("Parsing complete");
222 }
223
224 let mut indexed = 0usize;
226 let mut failed = 0usize;
227 let mut total_nodes = 0usize;
228 let mut new_meta: HashMap<String, String> = HashMap::new();
229
230 for (path, result) in parse_results {
231 match result {
232 Ok(doc) => {
233 let path_str = path.to_string_lossy().to_string();
234 match fts_index.index_document(&doc, false) {
235 Ok(count) => {
236 total_nodes += count;
237 indexed += 1;
238 if let Some(fp) = file_fingerprint(&path) {
239 new_meta.insert(path_str, fp);
240 }
241 }
242 Err(e) => {
243 warn!("Index error for {:?}: {}", path, e);
244 failed += 1;
245 }
246 }
247 }
248 Err(e) => {
249 warn!("Parse error for {:?}: {}", path, e);
250 failed += 1;
251 }
252 }
253 }
254
255 fts_index.commit()?;
257 if !new_meta.is_empty() {
258 fts_index.set_index_meta_batch(&new_meta)?;
259 }
260
261 let mut pruned = 0;
262 if discovery.complete {
263 let all_paths: std::collections::HashSet<String> = files
265 .iter()
266 .map(|f| f.to_string_lossy().to_string())
267 .collect();
268 for old_path in existing_meta.keys() {
269 if !all_paths.contains(old_path) {
270 if let Some(doc_id) = fts_index.get_doc_id_by_source_path(old_path)? {
271 fts_index.delete_document(&doc_id)?;
272 pruned += 1;
273 }
274 }
275 }
276 } else {
277 warn!("Skipping stale-document pruning because discovery was incomplete");
278 }
279 if pruned > 0 {
280 info!("Pruned {} deleted documents from index", pruned);
281 }
282
283 let duration = start.elapsed().as_millis() as u64;
284 info!(
285 "Indexed {} files ({} nodes) in {}ms ({} failed, {} skipped)",
286 indexed, total_nodes, duration, failed, skipped
287 );
288
289 Ok(IndexStats {
290 files_found: files.len(),
291 files_indexed: indexed,
292 files_skipped: skipped,
293 files_failed: failed,
294 nodes_indexed: total_nodes,
295 duration_ms: duration,
296 })
297}
298
299fn effective_concurrency(configured: usize, work_items: usize) -> usize {
301 configured.max(1).min(work_items.max(1))
302}
303
304fn parse_file(registry: &ParserRegistry, path: &Path) -> Result<Document> {
306 let content = fs::read_to_string(path).with_context(|| format!("Failed to read {:?}", path))?;
307 registry
308 .parse_content(path, &content)?
309 .ok_or_else(|| anyhow::anyhow!("No parser found for {:?}", path))
310}
311
312#[derive(Debug, Clone)]
314pub struct IndexStats {
315 pub files_found: usize,
316 pub files_indexed: usize,
317 pub files_skipped: usize,
318 pub files_failed: usize,
319 pub nodes_indexed: usize,
320 pub duration_ms: u64,
321}
322
323impl std::fmt::Display for IndexStats {
324 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
325 write!(
326 f,
327 "Indexed: {} files ({} nodes) in {}ms | {} skipped, {} failed",
328 self.files_indexed,
329 self.nodes_indexed,
330 self.duration_ms,
331 self.files_skipped,
332 self.files_failed,
333 )
334 }
335}
336
337#[cfg(test)]
338mod tests {
339 use super::{discover_files_with_status, effective_concurrency};
340 use crate::config::TreeSearchConfig;
341
342 #[test]
343 fn parser_concurrency_is_bounded_by_policy_and_work() {
344 for (configured, work_items, expected) in [(0, 0, 1), (0, 5, 1), (2, 5, 2), (20, 5, 5)] {
345 assert_eq!(
346 effective_concurrency(configured, work_items),
347 expected,
348 "configured={configured}, work_items={work_items}"
349 );
350 }
351 }
352
353 #[cfg(feature = "parser-markdown")]
354 #[test]
355 fn bounded_discovery_is_marked_incomplete() {
356 let directory = tempfile::tempdir().unwrap();
357 std::fs::write(directory.path().join("a.md"), "# A").unwrap();
358 std::fs::write(directory.path().join("b.md"), "# B").unwrap();
359 let config = TreeSearchConfig {
360 max_dir_files: 1,
361 ..TreeSearchConfig::default()
362 };
363
364 let discovery = discover_files_with_status(directory.path(), &config, false);
365 assert_eq!(discovery.files.len(), 1);
366 assert!(!discovery.complete);
367 assert!(discovery.files[0].ends_with("a.md"));
368 }
369
370 #[cfg(feature = "parser-markdown")]
371 #[test]
372 fn incomplete_discovery_never_prunes_and_complete_empty_walk_does() {
373 use super::index_directory;
374 use crate::engine::fts::FTS5Index;
375
376 let directory = tempfile::tempdir().unwrap();
377 let first = directory.path().join("a.md");
378 let second = directory.path().join("b.md");
379 std::fs::write(&first, "# A").unwrap();
380 std::fs::write(&second, "# B").unwrap();
381 let index = FTS5Index::new(None, None).unwrap();
382 let full_config = TreeSearchConfig {
383 max_dir_files: 10,
384 ..TreeSearchConfig::default()
385 };
386 index_directory(directory.path(), &index, &full_config, false, false).unwrap();
387 assert_eq!(index.get_stats().unwrap().document_count, 2);
388
389 let bounded_config = TreeSearchConfig {
390 max_dir_files: 1,
391 ..full_config.clone()
392 };
393 index_directory(directory.path(), &index, &bounded_config, false, false).unwrap();
394 assert_eq!(index.get_stats().unwrap().document_count, 2);
395
396 std::fs::remove_file(first).unwrap();
397 std::fs::remove_file(second).unwrap();
398 index_directory(directory.path(), &index, &full_config, false, false).unwrap();
399 assert_eq!(index.get_stats().unwrap().document_count, 0);
400 }
401}