1use parking_lot::Mutex;
33use serde::{Deserialize, Serialize};
34use std::cmp::Reverse;
35use std::collections::BinaryHeap;
36use std::num::NonZero;
37use std::path::Path;
38use std::sync::Arc;
39use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
40use tokio::sync::RwLock;
41
42use rayon::prelude::*;
43use vtcode_commons::StringId;
44
45pub struct FileIndex {
50 files: Vec<StringId>,
51 directories: Vec<StringId>,
52 interner: Arc<Mutex<vtcode_commons::StringInterner>>,
53 last_built: std::time::Instant,
54}
55
56fn build_parallel_walker(
58 search_directory: &Path,
59 exclude: &[String],
60 threads: usize,
61 respect_gitignore: bool,
62 follow_links: bool,
63) -> anyhow::Result<ignore::WalkParallel> {
64 let mut walk_builder = ignore::WalkBuilder::new(search_directory);
65 vtcode_commons::walk::apply_defaults(&mut walk_builder);
66
67 walk_builder.threads(threads);
69 walk_builder.follow_links(follow_links);
70 walk_builder.require_git(false); if !respect_gitignore {
73 walk_builder
74 .git_ignore(false)
75 .git_global(false)
76 .git_exclude(false)
77 .ignore(false)
78 .parents(false);
79 }
80
81 if !exclude.is_empty() {
82 let mut override_builder = ignore::overrides::OverrideBuilder::new(search_directory);
83 for exclude_pattern in exclude {
84 let pattern = format!("!{exclude_pattern}");
85 override_builder.add(&pattern)?;
86 }
87 walk_builder.overrides(override_builder.build()?);
88 }
89
90 Ok(walk_builder.build_parallel())
91}
92
93impl FileIndex {
94 fn build_from_directory(
97 search_directory: &Path,
98 exclude: &[String],
99 respect_gitignore: bool,
100 threads: usize,
101 ) -> anyhow::Result<Self> {
102 let walker = build_parallel_walker(search_directory, exclude, threads, respect_gitignore, true)?;
103
104 let files_arc = Arc::new(Mutex::new(Vec::new()));
106 let dirs_arc = Arc::new(Mutex::new(Vec::new()));
107
108 walker.run(|| {
109 let files_clone = files_arc.clone();
110 let dirs_clone = dirs_arc.clone();
111 let search_dir = search_directory.to_path_buf();
112
113 Box::new(move |result| {
114 let entry = match result {
115 Ok(e) => e,
116 Err(_) => return ignore::WalkState::Continue,
117 };
118
119 if let Some(rel_path) = entry.path().strip_prefix(&search_dir).ok().and_then(|p| p.to_str())
121 && !rel_path.is_empty()
122 {
123 if entry.path().is_dir() {
124 dirs_clone.lock().push(rel_path.to_string());
125 } else {
126 files_clone.lock().push(rel_path.to_string());
127 }
128 }
129
130 ignore::WalkState::Continue
131 })
132 });
133
134 let files = Arc::try_unwrap(files_arc)
135 .map_err(|arc| {
136 anyhow::anyhow!("failed to unwrap files arc, {} references remain", Arc::strong_count(&arc))
137 })?
138 .into_inner();
139 let directories = Arc::try_unwrap(dirs_arc)
140 .map_err(|arc| anyhow::anyhow!("failed to unwrap dirs arc, {} references remain", Arc::strong_count(&arc)))?
141 .into_inner();
142
143 let mut interner = vtcode_commons::StringInterner::new();
144 let interned_files: Vec<StringId> = files.iter().map(|s| interner.intern(s)).collect();
145 let interned_dirs: Vec<StringId> = directories.iter().map(|s| interner.intern(s)).collect();
146
147 Ok(Self {
148 files: interned_files,
149 directories: interned_dirs,
150 interner: Arc::new(Mutex::new(interner)),
151 last_built: std::time::Instant::now(),
152 })
153 }
154
155 fn query(
158 &self,
159 pattern_text: &str,
160 limit: usize,
161 match_type_filter: Option<MatchType>,
162 ) -> Vec<(u32, StringId, MatchType)> {
163 let mut heaps = Vec::new();
168
169 if match_type_filter.is_none_or(|t| t == MatchType::File) {
170 heaps.push(score_paths_top_k(&self.files, &self.interner, limit, pattern_text, MatchType::File));
171 }
172
173 if match_type_filter.is_none_or(|t| t == MatchType::Directory) {
174 heaps.push(score_paths_top_k(&self.directories, &self.interner, limit, pattern_text, MatchType::Directory));
175 }
176
177 merge_top_k(heaps, &self.interner, limit)
178 .into_sorted_vec()
179 .into_iter()
180 .map(|Reverse(item)| item)
181 .collect()
182 }
183}
184
185fn score_paths_top_k(
193 paths: &[StringId],
194 interner: &Arc<Mutex<vtcode_commons::StringInterner>>,
195 limit: usize,
196 pattern_text: &str,
197 match_type: MatchType,
198) -> BinaryHeap<Reverse<(u32, StringId, MatchType)>> {
199 const CHUNK: usize = 1024;
200
201 if paths.len() <= CHUNK {
204 let mut list = BestMatchesList::new(limit, pattern_text, interner);
205 for &path_id in paths {
206 let path_opt = interner.lock().get(path_id).map(|s| s.to_string());
207 if let Some(path) = path_opt {
208 list.record_match(&path, match_type);
209 }
210 }
211 return list.matches;
212 }
213
214 let heaps: Vec<_> = paths
215 .par_chunks(CHUNK)
216 .map_init(
217 || BestMatchesList::new(limit, pattern_text, interner),
218 |list, chunk| {
219 for &path_id in chunk {
220 let path_opt = interner.lock().get(path_id).map(|s| s.to_string());
221 if let Some(path) = path_opt {
222 list.record_match(&path, match_type);
223 }
224 }
225 std::mem::take(&mut list.matches)
226 },
227 )
228 .collect();
229
230 merge_top_k(heaps, interner, limit)
231}
232
233fn merge_top_k(
239 heaps: Vec<BinaryHeap<Reverse<(u32, StringId, MatchType)>>>,
240 _interner: &Arc<Mutex<vtcode_commons::StringInterner>>,
241 limit: usize,
242) -> BinaryHeap<Reverse<(u32, StringId, MatchType)>> {
243 let mut merged = BinaryHeap::with_capacity(limit);
244 for heap in heaps {
245 for Reverse(item) in heap.into_vec() {
246 push_top_match(&mut merged, limit, item.0, item.1, item.2);
247 }
248 }
249 merged
250}
251
252pub struct FileIndexCache {
254 cache: Arc<RwLock<Option<Arc<FileIndex>>>>,
255 search_directory: std::path::PathBuf,
256 exclude: Vec<String>,
257 respect_gitignore: bool,
258 threads: usize,
259}
260
261impl FileIndexCache {
262 pub fn new(
263 search_directory: std::path::PathBuf,
264 exclude: impl IntoIterator<Item = String>,
265 respect_gitignore: bool,
266 threads: usize,
267 ) -> Self {
268 Self {
269 cache: Arc::new(RwLock::new(None)),
270 search_directory,
271 exclude: exclude.into_iter().collect(),
272 respect_gitignore,
273 threads,
274 }
275 }
276
277 pub async fn get_or_build(&self) -> anyhow::Result<Arc<FileIndex>> {
279 {
281 let guard = self.cache.read().await;
282 if let Some(index) = guard.as_ref() {
283 if index.last_built.elapsed() < std::time::Duration::from_secs(300) {
285 return Ok(Arc::clone(index));
286 }
287 }
288 }
289
290 let index = Arc::new(FileIndex::build_from_directory(
292 &self.search_directory,
293 &self.exclude,
294 self.respect_gitignore,
295 self.threads,
296 )?);
297
298 {
300 let mut guard = self.cache.write().await;
301 *guard = Some(Arc::clone(&index));
302 }
303 Ok(index)
304 }
305
306 pub fn refresh_background(&self) -> Option<Arc<FileIndex>> {
309 let search_directory = self.search_directory.clone();
311 let exclude = self.exclude.clone();
312 let respect_gitignore = self.respect_gitignore;
313 let threads = self.threads;
314 let cache = self.cache.clone();
315
316 tokio::spawn(async move {
317 match FileIndex::build_from_directory(&search_directory, &exclude, respect_gitignore, threads) {
318 Ok(new_index) => {
319 let mut guard = cache.write().await;
320 *guard = Some(Arc::new(new_index));
321 }
322 Err(e) => {
323 tracing::error!("failed to rebuild file index: {e}");
324 }
325 }
326 });
327
328 let guard = self.cache.blocking_read();
330 guard.as_ref().map(Arc::clone)
331 }
332
333 pub fn update_file(&self, path: &str, is_added: bool) {
336 let mut guard = self.cache.blocking_write();
337 let Some(existing) = guard.take() else { return };
338
339 let mut index = Arc::try_unwrap(existing).unwrap_or_else(|arc| (*arc).clone());
340 let path_id = index.interner.lock().intern(path);
341 if is_added {
342 if Path::new(path).is_dir() {
343 index.directories.push(path_id);
344 } else {
345 index.files.push(path_id);
346 }
347 } else {
348 index.files.retain(|&p| p != path_id);
349 index.directories.retain(|&p| p != path_id);
350 }
351 index.last_built = std::time::Instant::now();
352 *guard = Some(Arc::new(index));
353 }
354
355 pub async fn index_age(&self) -> Option<std::time::Duration> {
357 let guard = self.cache.read().await;
358 guard.as_ref().map(|idx| idx.last_built.elapsed())
359 }
360}
361
362impl Clone for FileIndex {
364 fn clone(&self) -> Self {
365 Self {
366 files: self.files.clone(),
367 directories: self.directories.clone(),
368 interner: self.interner.clone(),
369 last_built: self.last_built,
370 }
371 }
372}
373
374#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
382#[serde(rename_all = "lowercase")]
383pub enum MatchType {
384 File,
385 Directory,
386}
387
388#[derive(Debug, Clone, Serialize, Deserialize)]
389pub struct FileMatch {
390 pub score: u32,
391 pub path: String,
392 pub match_type: MatchType,
393 #[serde(skip_serializing_if = "Option::is_none")]
394 pub indices: Option<Vec<u32>>,
395}
396
397#[derive(Debug)]
399pub struct FileSearchResults {
400 pub matches: Vec<FileMatch>,
401 pub total_match_count: usize,
402}
403
404pub struct FileSearchConfig {
406 pub pattern_text: String,
407 pub limit: NonZero<usize>,
408 pub search_directory: std::path::PathBuf,
409 pub exclude: Vec<String>,
410 pub threads: NonZero<usize>,
411 pub cancel_flag: Arc<AtomicBool>,
412 pub compute_indices: bool,
413 pub respect_gitignore: bool,
414}
415
416pub use vtcode_commons::paths::file_name_from_path;
417
418struct BestMatchesList {
423 matches: BinaryHeap<Reverse<(u32, StringId, MatchType)>>,
424 limit: usize,
425 matcher: nucleo_matcher::Matcher,
426 haystack_buf: Vec<char>,
427 pattern: PatternStorage,
429 interner: Arc<Mutex<vtcode_commons::StringInterner>>,
430}
431
432enum PatternStorage {
434 Ascii(Vec<u8>),
436 Unicode(Vec<char>),
438}
439
440impl BestMatchesList {
441 fn new(limit: usize, pattern_text: &str, interner: &Arc<Mutex<vtcode_commons::StringInterner>>) -> Self {
442 let pattern = if pattern_text.is_ascii() {
446 PatternStorage::Ascii(pattern_text.to_ascii_lowercase().into_bytes())
447 } else {
448 PatternStorage::Unicode(pattern_text.to_lowercase().chars().collect())
449 };
450
451 Self {
452 matches: BinaryHeap::new(),
453 limit,
454 matcher: nucleo_matcher::Matcher::new(nucleo_matcher::Config::DEFAULT),
455 haystack_buf: Vec::with_capacity(256),
456 pattern,
457 interner: interner.clone(),
458 }
459 }
460
461 fn record_match(&mut self, path: &str, match_type: MatchType) -> bool {
466 let haystack = nucleo_matcher::Utf32Str::new(path, &mut self.haystack_buf);
468 let needle = match &self.pattern {
469 PatternStorage::Ascii(bytes) => nucleo_matcher::Utf32Str::Ascii(bytes),
470 PatternStorage::Unicode(chars) => nucleo_matcher::Utf32Str::Unicode(chars),
471 };
472 let Some(score) = self.matcher.fuzzy_match(haystack, needle) else {
473 return false;
474 };
475
476 let path_id = self.interner.lock().intern(path);
477 push_top_match(&mut self.matches, self.limit, score as u32, path_id, match_type);
478 true
479 }
480}
481
482fn push_top_match(
483 matches: &mut BinaryHeap<Reverse<(u32, StringId, MatchType)>>,
484 limit: usize,
485 score: u32,
486 path: StringId,
487 match_type: MatchType,
488) -> bool {
489 let candidate = (score, path, match_type);
490 if matches.len() < limit {
491 matches.push(Reverse(candidate));
492 return true;
493 }
494
495 let Some(minimum) = matches.peek().map(|entry| &entry.0) else {
496 return false;
497 };
498
499 if &candidate <= minimum {
500 return false;
501 }
502
503 matches.pop();
504 matches.push(Reverse(candidate));
505 true
506}
507
508pub async fn run_with_index(
522 config: FileSearchConfig,
523 index_cache: &FileIndexCache,
524) -> anyhow::Result<FileSearchResults> {
525 let limit = config.limit.get();
526 let cancel_flag = &config.cancel_flag;
527 let compute_indices = config.compute_indices;
528
529 let index = index_cache.get_or_build().await?;
531
532 if cancel_flag.load(Ordering::Relaxed) {
534 return Ok(FileSearchResults { matches: Vec::new(), total_match_count: 0 });
535 }
536
537 let index_for_results = index.clone();
540 let matched_paths = tokio::task::spawn_blocking({
541 let pattern_text = config.pattern_text.clone();
542 move || Ok::<_, anyhow::Error>(index.query(&pattern_text, limit, None))
543 })
544 .await??;
545
546 let total_match_count = matched_paths.len();
547
548 let matches = matched_paths
550 .into_iter()
551 .filter_map(|(score, path_id, match_type)| {
552 let path = index_for_results.interner.lock().get(path_id)?.to_string();
553 Some(FileMatch {
554 score,
555 path,
556 match_type,
557 indices: if compute_indices { Some(Vec::new()) } else { None },
558 })
559 })
560 .collect();
561
562 Ok(FileSearchResults { matches, total_match_count })
563}
564
565pub fn run(config: FileSearchConfig) -> anyhow::Result<FileSearchResults> {
575 run_with_policy(config, true, false)
576}
577
578pub fn run_bounded_no_follow(config: FileSearchConfig) -> anyhow::Result<FileSearchResults> {
584 run_bounded_no_follow_with_visit(config, |_| {})
585}
586
587fn run_bounded_no_follow_with_visit(
588 config: FileSearchConfig,
589 mut visit: impl FnMut(&Path),
590) -> anyhow::Result<FileSearchResults> {
591 let limit = config.limit.get();
592 let search_directory = &config.search_directory;
593 let mut walk_builder = ignore::WalkBuilder::new(search_directory);
594 vtcode_commons::walk::apply_defaults(&mut walk_builder);
595 walk_builder
596 .follow_links(false)
597 .require_git(false)
598 .sort_by_file_path(|left, right| left.cmp(right));
599
600 if !config.respect_gitignore {
601 walk_builder
602 .git_ignore(false)
603 .git_global(false)
604 .git_exclude(false)
605 .ignore(false)
606 .parents(false);
607 }
608
609 if !config.exclude.is_empty() {
610 let mut override_builder = ignore::overrides::OverrideBuilder::new(search_directory);
611 for exclude_pattern in &config.exclude {
612 override_builder.add(&format!("!{exclude_pattern}"))?;
613 }
614 walk_builder.overrides(override_builder.build()?);
615 }
616
617 let interner = Arc::new(Mutex::new(vtcode_commons::StringInterner::new()));
618 let mut matches = BestMatchesList::new(limit, &config.pattern_text, &interner);
619 let mut matching_count = 0usize;
620 for result in walk_builder.build() {
621 if config.cancel_flag.load(Ordering::Relaxed) {
622 break;
623 }
624 let entry = match result {
625 Ok(entry) => entry,
626 Err(_) => continue,
627 };
628 visit(entry.path());
629 if !entry.file_type().is_some_and(|file_type| file_type.is_file()) {
630 continue;
631 }
632 let Some(relative_path) = entry
633 .path()
634 .strip_prefix(search_directory)
635 .ok()
636 .and_then(|path| path.to_str())
637 .filter(|path| !path.is_empty())
638 else {
639 continue;
640 };
641 if matches.record_match(relative_path, MatchType::File) {
642 matching_count += 1;
643 if matching_count >= limit {
644 break;
645 }
646 }
647 }
648
649 let interner_guard = interner.lock();
650 let matches = matches
651 .matches
652 .into_sorted_vec()
653 .into_iter()
654 .filter_map(|Reverse((score, path_id, match_type))| {
655 let path = interner_guard.get(path_id)?.to_string();
656 Some(FileMatch {
657 score,
658 path,
659 match_type,
660 indices: config.compute_indices.then(Vec::new),
661 })
662 })
663 .collect();
664
665 Ok(FileSearchResults {
666 matches,
667 total_match_count: matching_count + usize::from(matching_count >= limit),
670 })
671}
672
673fn run_with_policy(
674 config: FileSearchConfig,
675 follow_links: bool,
676 files_only: bool,
677) -> anyhow::Result<FileSearchResults> {
678 let limit = config.limit.get();
679 let search_directory = &config.search_directory;
680 let exclude = &config.exclude;
681 let threads = config.threads.get();
682 let cancel_flag = &config.cancel_flag;
683 let compute_indices = config.compute_indices;
684 let respect_gitignore = config.respect_gitignore;
685
686 let walker = build_parallel_walker(search_directory, exclude, threads, respect_gitignore, follow_links)?;
687
688 let interner = Arc::new(Mutex::new(vtcode_commons::StringInterner::new()));
689
690 let best_matchers_per_worker: Vec<Arc<Mutex<BestMatchesList>>> = (0..threads)
693 .map(|_| Arc::new(Mutex::new(BestMatchesList::new(limit, &config.pattern_text, &interner))))
694 .collect();
695
696 let interner_for_merge = interner.clone();
697 let total_match_count = Arc::new(AtomicUsize::new(0));
698
699 let worker_counter = AtomicUsize::new(0);
702 let worker_count = best_matchers_per_worker.len();
703 walker.run(|| {
704 let worker_id = worker_counter.fetch_add(1, Ordering::Relaxed) % worker_count;
705 let best_list = best_matchers_per_worker[worker_id].clone();
706 let cancel_flag_clone = cancel_flag.clone();
707 let total_match_count_clone = total_match_count.clone();
708 let _interner = interner.clone();
709
710 Box::new(move |result| {
711 if cancel_flag_clone.load(Ordering::Relaxed) {
713 return ignore::WalkState::Quit;
714 }
715
716 let entry = match result {
717 Ok(e) => e,
718 Err(_) => return ignore::WalkState::Continue,
719 };
720
721 let relative_path = entry.path().strip_prefix(search_directory).ok().and_then(|p| p.to_str());
723
724 let path_to_match = match relative_path {
725 Some(p) if !p.is_empty() => p,
726 _ => return ignore::WalkState::Continue, };
728
729 let match_type = if entry.path().is_dir() {
730 MatchType::Directory
731 } else {
732 MatchType::File
733 };
734
735 if files_only && match_type == MatchType::Directory {
736 return ignore::WalkState::Continue;
737 }
738
739 {
741 let mut list = best_list.lock();
742 if list.record_match(path_to_match, match_type) {
743 total_match_count_clone.fetch_add(1, Ordering::Relaxed);
744 }
745 }
746
747 ignore::WalkState::Continue
748 })
749 });
750
751 let worker_heaps: Vec<BinaryHeap<Reverse<(u32, StringId, MatchType)>>> = best_matchers_per_worker
753 .into_iter()
754 .map(|arc| std::mem::take(&mut arc.lock().matches))
755 .collect();
756 let merged_matches = merge_top_k(worker_heaps, &interner_for_merge, limit);
757
758 let interner_guard = interner_for_merge.lock();
760 let matches = merged_matches
761 .into_sorted_vec()
762 .into_iter()
763 .filter_map(|Reverse((score, path_id, match_type))| {
764 let path = interner_guard.get(path_id)?.to_string();
765 Some(FileMatch {
766 score,
767 path,
768 match_type,
769 indices: if compute_indices { Some(Vec::new()) } else { None },
770 })
771 })
772 .collect();
773
774 Ok(FileSearchResults {
775 matches,
776 total_match_count: total_match_count.load(Ordering::Relaxed),
777 })
778}
779
780#[cfg(test)]
781mod tests {
782 use super::{FileSearchConfig, run_bounded_no_follow, run_bounded_no_follow_with_visit};
783 use std::num::NonZero;
784 use std::sync::Arc;
785 use std::sync::atomic::AtomicBool;
786 use tempfile::TempDir;
787
788 fn bounded_paths(workspace: &std::path::Path) -> Vec<String> {
789 run_bounded_no_follow(FileSearchConfig {
790 pattern_text: "widget".to_string(),
791 limit: NonZero::new(2).expect("non-zero limit"),
792 search_directory: workspace.to_path_buf(),
793 exclude: Vec::new(),
794 threads: NonZero::new(4).expect("non-zero threads"),
795 cancel_flag: Arc::new(AtomicBool::new(false)),
796 compute_indices: false,
797 respect_gitignore: true,
798 })
799 .expect("bounded path search")
800 .matches
801 .into_iter()
802 .map(|candidate| candidate.path)
803 .collect()
804 }
805
806 #[test]
807 fn bounded_path_selection_is_stable_across_repeated_walks() {
808 let workspace = TempDir::new().expect("workspace");
809 for directory in ["z", "a", "m", "b", "y"] {
810 let directory = workspace.path().join(directory);
811 std::fs::create_dir(&directory).expect("fixture directory");
812 std::fs::write(directory.join("widget.rs"), "fn widget() {}\n").expect("fixture source");
813 }
814
815 let expected = bounded_paths(workspace.path());
816 assert_eq!(expected.len(), 2);
817 for _ in 0..20 {
818 assert_eq!(bounded_paths(workspace.path()), expected);
819 }
820 }
821
822 #[test]
823 fn bounded_path_selection_is_the_sorted_prefix_and_stops_early() {
824 let workspace = TempDir::new().expect("workspace");
825 for directory in ["z", "a", "m", "b", "y"] {
826 let directory = workspace.path().join(directory);
827 std::fs::create_dir(&directory).expect("fixture directory");
828 std::fs::write(directory.join("widget.rs"), "fn widget() {}\n").expect("fixture source");
829 }
830 let mut visited = Vec::new();
831
832 let results = run_bounded_no_follow_with_visit(
833 FileSearchConfig {
834 pattern_text: "widget".to_string(),
835 limit: NonZero::new(2).expect("non-zero limit"),
836 search_directory: workspace.path().to_path_buf(),
837 exclude: Vec::new(),
838 threads: NonZero::new(4).expect("non-zero threads"),
839 cancel_flag: Arc::new(AtomicBool::new(false)),
840 compute_indices: false,
841 respect_gitignore: true,
842 },
843 |path| visited.push(path.to_path_buf()),
844 )
845 .expect("bounded path search");
846 let mut paths = results.matches.into_iter().map(|candidate| candidate.path).collect::<Vec<_>>();
847 paths.sort();
848
849 assert_eq!(paths, vec!["a/widget.rs", "b/widget.rs"]);
850 assert!(visited.len() < 11, "the bounded route must stop before traversing the complete fixture tree");
851 assert_eq!(results.total_match_count, 3);
852 }
853}