1pub mod filter;
7pub mod open_index;
8pub mod result;
9pub mod zero_hint;
10
11pub use filter::{LiteralPattern, QueryFilter, prepare_literal_pattern, substring_hint_text};
12pub use zero_hint::{ExcludedReason, ZeroHint};
13
14use anyhow::{Context, Result};
15use regex::Regex;
16
17use crate::cache::CacheManager;
18use crate::models::{
19 IndexPath, IndexStatus, IndexStatusReport, IndexWarning, Language, QueryResponse, SearchResult,
20 Span, SymbolKind,
21};
22use crate::output;
23use crate::parsers::ParserFactory;
24use std::sync::atomic::{AtomicUsize, Ordering};
25use std::sync::{Arc, OnceLock};
26
27use open_index::OpenIndex;
28
29pub fn invalidate_caches(workspace_root: &std::path::Path) {
33 open_index::invalidate(&workspace_root.join(crate::cache::CACHE_DIR));
34 status_cache::invalidate(workspace_root);
35}
36
37struct Internal {
40 results: Vec<SearchResult>,
41 total_count: usize,
45 total_is_exact: bool,
47 approx_total: Option<usize>,
49 substring_only: Option<usize>,
52 excluded_by_default: usize,
55 counted: Option<(usize, usize)>,
58 candidates_us: u64,
60 index_path: IndexPath,
62 warnings: Vec<String>,
64}
65
66#[derive(Debug, Clone)]
68struct CandidateStats {
69 candidates_us: u64,
70 substring_only: Option<usize>,
71 excluded_by_default: usize,
73 counted: Option<(usize, usize)>,
75 exhausted: bool,
77 approx_total: Option<usize>,
79 index_path: IndexPath,
81 warnings: Vec<String>,
84}
85
86impl Default for CandidateStats {
87 fn default() -> Self {
88 Self {
89 candidates_us: 0,
90 substring_only: None,
91 excluded_by_default: 0,
92 counted: None,
93 exhausted: true,
94 approx_total: None,
95 index_path: IndexPath::Trigram,
96 warnings: Vec::new(),
97 }
98 }
99}
100
101enum LineSet {
103 Only(Vec<u32>),
105 All,
107}
108
109struct FileFilter {
112 language: Option<Language>,
113 include: Option<globset::GlobSet>,
114 exclude: Option<globset::GlobSet>,
115 file_pattern: Option<String>,
116 exclude_text: bool,
117 include_locks: bool,
118 include_generated: bool,
119}
120
121impl FileFilter {
122 fn from_filter(filter: &QueryFilter) -> Self {
123 Self {
124 language: filter.language,
125 include: result::build_glob_set(&filter.glob_patterns, "glob"),
126 exclude: result::build_glob_set(&filter.exclude_patterns, "exclude"),
127 file_pattern: filter.file_pattern.clone(),
128 exclude_text: filter.exclude_text,
129 include_locks: filter.include_locks,
130 include_generated: filter.include_generated,
131 }
132 }
133
134 fn excluded_by_default(&self, lang: Language) -> bool {
136 filter::excluded_by_default(
137 lang,
138 self.language,
139 self.include_locks,
140 self.include_generated,
141 )
142 }
143
144 fn path_admits(&self, path: &str) -> bool {
146 if let Some(set) = &self.include
147 && !set.is_match(path)
148 {
149 return false;
150 }
151 if let Some(set) = &self.exclude
152 && set.is_match(path)
153 {
154 return false;
155 }
156 if let Some(needle) = &self.file_pattern
157 && !path.contains(needle.as_str())
158 {
159 return false;
160 }
161 true
162 }
163
164 fn accept(&self, path: &str, lang: Language) -> bool {
165 filter::tier_admits(
166 lang,
167 self.language,
168 self.exclude_text,
169 self.include_locks,
170 self.include_generated,
171 ) && self.path_admits(path)
172 }
173}
174
175struct VerifyOutcome {
177 results: Vec<SearchResult>,
179 exhausted: bool,
181 estimated_total: Option<usize>,
185 substring_only: usize,
188 excluded_by_default: usize,
191 counted: Option<(usize, usize)>,
193}
194
195enum LineMatcher {
200 WordBoundary(Regex, String),
203 Contains(String),
206 Regex(Regex, &'static str),
209}
210
211impl LineMatcher {
212 fn new(pattern: &str, filter: &QueryFilter) -> Result<Self> {
213 if filter.use_regex {
214 let label = if filter.ignore_case && filter.rewritten_from.is_some() {
215 "text_match"
216 } else {
217 "regex_match"
218 };
219 return match Regex::new(pattern) {
220 Ok(re) => Ok(Self::Regex(re, label)),
221 Err(e) => {
222 log::error!("Invalid regex pattern '{}': {}", pattern, e);
223 anyhow::bail!("Invalid regex pattern '{}': {}", pattern, e);
224 }
225 };
226 }
227 if filter.use_contains {
228 return Ok(Self::Contains(pattern.to_string()));
229 }
230 match Regex::new(&format!(r"\b{}\b", regex::escape(pattern))) {
231 Ok(re) => Ok(Self::WordBoundary(re, pattern.to_string())),
232 Err(_) => {
233 log::debug!(
234 "Word boundary regex failed for pattern '{}', falling back to substring",
235 pattern
236 );
237 Ok(Self::Contains(pattern.to_string()))
238 }
239 }
240 }
241
242 fn is_word_boundary(&self) -> bool {
243 matches!(self, Self::WordBoundary(..))
244 }
245
246 #[inline]
247 fn is_match(&self, line: &str) -> bool {
248 match self {
249 Self::WordBoundary(re, _) | Self::Regex(re, _) => re.is_match(line),
250 Self::Contains(p) => line.contains(p.as_str()),
251 }
252 }
253
254 #[inline]
256 fn find(&self, line: &str) -> Option<usize> {
257 match self {
258 Self::WordBoundary(re, _) | Self::Regex(re, _) => re.find(line).map(|m| m.start()),
259 Self::Contains(p) => line.find(p.as_str()),
260 }
261 }
262
263 fn literal(&self) -> Option<&str> {
265 match self {
266 Self::WordBoundary(_, p) | Self::Contains(p) => Some(p.as_str()),
267 Self::Regex(..) => None,
268 }
269 }
270
271 fn kind_label(&self) -> &'static str {
273 match self {
274 Self::Regex(_, label) => label,
275 _ => "text_match",
276 }
277 }
278}
279
280pub const ESTIMATE_SAMPLE_FILES: usize = 32;
292pub const ESTIMATE_FINISH_LINES: usize = 128;
295pub const ESTIMATE_PER_FILE_LINES: usize = 16;
298
299fn verify_files_streaming(
300 open: &OpenIndex,
301 files: Vec<(u32, LineSet)>,
302 matcher: &LineMatcher,
303 file_filter: &FileFilter,
304 paths_only: bool,
305 budget: Option<usize>,
306 count_only: bool,
307) -> VerifyOutcome {
308 use rayon::prelude::*;
309
310 let content = &open.content;
311
312 let mut excluded_by_default = 0usize;
315 let mut accepted: Vec<(u32, String, Language, LineSet)> = files
316 .into_iter()
317 .filter_map(|(file_id, lines)| {
318 let path = content.get_file_path(file_id)?;
319 let lang = Language::from_path(path);
320 let path_str = path.to_string_lossy().into_owned();
321 if file_filter.accept(&path_str, lang) {
322 Some((file_id, path_str, lang, lines))
323 } else {
324 if file_filter.path_admits(&path_str) && file_filter.excluded_by_default(lang) {
328 excluded_by_default += 1;
329 }
330 None
331 }
332 })
333 .collect();
334 accepted.sort_by(|a, b| a.1.cmp(&b.1));
335
336 let count_substring_only = matcher.is_word_boundary();
337 let substring_only = AtomicUsize::new(0);
338 let kind_label = matcher.kind_label();
339
340 let verify_one = |file_id: u32, path: &str, lang: Language, lines: &LineSet| {
341 let Ok(text) = content.get_file_content(file_id) else {
342 return Vec::new();
343 };
344 let all_lines: Vec<&str> = text.lines().collect();
345 let mut out = Vec::new();
346
347 let mut check = |line_no: usize, line: &str| -> bool {
348 let Some(offset) = matcher.find(line) else {
349 if count_substring_only
350 && let Some(p) = matcher.literal()
351 && line.contains(p)
352 {
353 substring_only.fetch_add(1, Ordering::Relaxed);
354 }
355 return false;
356 };
357 out.push(SearchResult {
358 path: path.to_string(),
359 lang,
360 kind: SymbolKind::Unknown(kind_label.to_string()),
361 symbol: None,
362 span: Span {
363 start_line: line_no,
364 end_line: line_no,
365 },
366 preview: crate::parsers::preview::line_preview(line, offset),
370 dependencies: None,
371 });
372 true
373 };
374
375 match lines {
376 LineSet::Only(nos) => {
377 let mut last = 0u32;
378 for &line_no in nos {
379 if line_no == last {
380 continue;
381 }
382 last = line_no;
383 let idx = line_no as usize;
384 if idx == 0 || idx > all_lines.len() {
385 log::debug!(
386 "Line {} out of bounds (file has {} lines)",
387 line_no,
388 all_lines.len()
389 );
390 continue;
391 }
392 if check(idx, all_lines[idx - 1]) && paths_only {
393 break;
394 }
395 }
396 }
397 LineSet::All => {
398 for (i, line) in all_lines.iter().enumerate() {
399 if check(i + 1, line) && paths_only {
400 break;
401 }
402 }
403 }
404 }
405 out
406 };
407
408 if count_only && budget.is_none() {
413 let count_one = |file_id: u32, lines: &LineSet| -> usize {
414 let Ok(text) = content.get_file_content(file_id) else {
415 return 0;
416 };
417 let mut hits = 0usize;
418 let mut check = |line: &str| {
419 if matcher.find(line).is_some() {
420 hits += 1;
421 true
422 } else {
423 if count_substring_only
424 && let Some(p) = matcher.literal()
425 && line.contains(p)
426 {
427 substring_only.fetch_add(1, Ordering::Relaxed);
428 }
429 false
430 }
431 };
432 match lines {
433 LineSet::Only(nos) => {
434 let all_lines: Vec<&str> = text.lines().collect();
435 let mut last = 0u32;
436 for &line_no in nos {
437 if line_no == last {
438 continue;
439 }
440 last = line_no;
441 let idx = line_no as usize;
442 if idx == 0 || idx > all_lines.len() {
443 continue;
444 }
445 if check(all_lines[idx - 1]) && paths_only {
446 break;
447 }
448 }
449 }
450 LineSet::All => {
451 for line in text.lines() {
452 if check(line) && paths_only {
453 break;
454 }
455 }
456 }
457 }
458 hits
459 };
460 let (lines, files_hit) = open.pool().install(|| {
461 accepted
462 .par_iter()
463 .map(|(file_id, _, _, lines)| {
464 let n = count_one(*file_id, lines);
465 (n, usize::from(n > 0))
466 })
467 .reduce(|| (0, 0), |a, b| (a.0 + b.0, a.1 + b.1))
468 });
469 return VerifyOutcome {
470 results: Vec::new(),
471 exhausted: true,
472 estimated_total: None,
473 substring_only: substring_only.into_inner(),
474 excluded_by_default,
475 counted: Some((if paths_only { files_hit } else { lines }, files_hit)),
476 };
477 }
478
479 let mut results: Vec<SearchResult> = Vec::new();
480 let mut next = 0usize;
481 let mut chunk = if budget.is_some() {
485 16usize
486 } else {
487 accepted.len().max(1)
488 };
489 let mut exhausted = true;
490
491 while next < accepted.len() {
492 if let Some(b) = budget
493 && results.len() >= b
494 {
495 exhausted = false;
496 break;
497 }
498 let end = (next + chunk).min(accepted.len());
499 let slice = &accepted[next..end];
500 let round: Vec<Vec<SearchResult>> = open.pool().install(|| {
501 slice
502 .par_iter()
503 .map(|(file_id, path, lang, lines)| verify_one(*file_id, path, *lang, lines))
504 .collect()
505 });
506 for r in round {
507 results.extend(r);
508 }
509 next = end;
510 chunk = (chunk * 2).min(1024);
511 }
512
513 let mut estimated_total = None;
517 if !exhausted {
518 let remaining = &accepted[next..];
519 let unit = |lines: &LineSet| -> Option<usize> {
522 match lines {
523 _ if paths_only => Some(1),
524 LineSet::Only(v) => Some(v.len()),
525 LineSet::All => None,
526 }
527 };
528 let remaining_units = remaining
529 .iter()
530 .try_fold(0usize, |acc, (_, _, _, l)| unit(l).map(|u| acc + u));
531
532 let finish = remaining.len() <= ESTIMATE_SAMPLE_FILES
533 || remaining_units.is_some_and(|u| u <= ESTIMATE_FINISH_LINES);
534 if finish {
535 let round: Vec<Vec<SearchResult>> = open.pool().install(|| {
536 remaining
537 .par_iter()
538 .map(|(file_id, path, lang, lines)| verify_one(*file_id, path, *lang, lines))
539 .collect()
540 });
541 for r in round {
542 results.extend(r);
543 }
544 exhausted = true;
545 } else if let Some(units) = remaining_units {
546 let stride = remaining.len() / ESTIMATE_SAMPLE_FILES;
547 let picks: Vec<&(u32, String, Language, LineSet)> = (0..ESTIMATE_SAMPLE_FILES)
548 .map(|i| &remaining[i * stride])
549 .collect();
550 let (hits, sampled_units) = open.pool().install(|| {
551 picks
552 .par_iter()
553 .map(|(file_id, path, lang, lines)| {
554 let trimmed = match lines {
555 LineSet::Only(v) if !paths_only => {
556 LineSet::Only(v[..v.len().min(ESTIMATE_PER_FILE_LINES)].to_vec())
557 }
558 LineSet::Only(v) => LineSet::Only(v.clone()),
559 LineSet::All => LineSet::All,
560 };
561 let u = unit(&trimmed).unwrap_or(0);
562 (verify_one(*file_id, path, *lang, &trimmed).len(), u)
565 })
566 .reduce(|| (0usize, 0usize), |a, b| (a.0 + b.0, a.1 + b.1))
567 });
568 let rate = hits as f64 / sampled_units.max(1) as f64;
569 estimated_total = Some(results.len() + (rate * units as f64).round() as usize);
570 }
571 }
572
573 VerifyOutcome {
574 results,
575 exhausted,
576 estimated_total,
577 substring_only: substring_only.into_inner(),
578 excluded_by_default,
579 counted: None,
580 }
581}
582
583pub struct QueryEngine {
585 cache: CacheManager,
586 open: OnceLock<Arc<OpenIndex>>,
589}
590
591impl QueryEngine {
592 pub fn new(cache: CacheManager) -> Self {
594 Self {
595 cache,
596 open: OnceLock::new(),
597 }
598 }
599
600 fn open_index(&self) -> Result<Arc<OpenIndex>> {
605 if let Some(open) = self.open.get() {
606 return Ok(Arc::clone(open));
607 }
608 let open = open_index::get_or_open(&self.cache)?;
609 let _ = self.open.set(Arc::clone(&open));
610 Ok(open)
611 }
612
613 fn load_dependencies(&self, results: &mut [SearchResult], include_deps: bool) -> Result<()> {
616 if !include_deps || results.is_empty() {
617 return Ok(());
618 }
619
620 log::debug!("Loading dependencies for {} results", results.len());
621
622 let workspace_root = self
626 .cache
627 .path()
628 .parent()
629 .ok_or_else(|| anyhow::anyhow!("Cache path has no parent"))?;
630 let cache_for_deps = CacheManager::new(workspace_root);
631 let dep_index = crate::dependency::DependencyIndex::new(cache_for_deps);
632
633 for result in results {
635 let normalized_path = result.path.strip_prefix("./").unwrap_or(&result.path);
637
638 match self.cache.get_file_id(normalized_path) {
640 Ok(Some(file_id)) => {
641 log::debug!("Found file_id={} for path={}", file_id, result.path);
642 match dep_index.get_dependencies_info(file_id) {
644 Ok(dep_infos) => {
645 log::debug!(
646 "Loaded {} dependencies for file_id={}",
647 dep_infos.len(),
648 file_id
649 );
650 if !dep_infos.is_empty() {
651 result.dependencies = Some(dep_infos);
652 }
653 }
654 Err(e) => {
655 log::warn!("Failed to get dependencies for file_id={}: {}", file_id, e);
656 }
657 }
658 }
659 Ok(None) => {
660 log::warn!("No file_id found for path: {}", result.path);
661 }
662 Err(e) => {
663 log::warn!("Failed to get file_id for path {}: {}", result.path, e);
664 }
665 }
666 }
667
668 Ok(())
669 }
670
671 fn group_and_load_dependencies(
674 &self,
675 results: Vec<SearchResult>,
676 include_deps: bool,
677 context_lines: usize,
678 ) -> Result<Vec<crate::models::FileGroupedResult>> {
679 use crate::models::{FileGroupedResult, MatchResult};
680 use std::collections::HashMap;
681
682 if results.is_empty() {
683 return Ok(Vec::new());
684 }
685
686 let mut grouped: HashMap<String, Vec<SearchResult>> = HashMap::new();
688 for result in results {
689 grouped.entry(result.path.clone()).or_default().push(result);
690 }
691
692 let dep_index = if include_deps {
694 let workspace_root = self
695 .cache
696 .path()
697 .parent()
698 .ok_or_else(|| anyhow::anyhow!("Cache path has no parent"))?;
699 let cache_for_deps = CacheManager::new(workspace_root);
700 Some(crate::dependency::DependencyIndex::new(cache_for_deps))
701 } else {
702 None
703 };
704
705 let open_opt = self.open_index().ok();
707 let content_reader_opt = open_opt.as_deref().map(|o| &o.content);
708
709 let mut file_results: Vec<FileGroupedResult> = grouped
711 .into_iter()
712 .map(|(path, file_matches)| {
713 let language = file_matches.first().map(|r| r.lang).unwrap_or_default();
715
716 let dependencies = if let Some(dep_idx) = &dep_index {
718 let normalized_path = path.strip_prefix("./").unwrap_or(&path);
719 match self.cache.get_file_id(normalized_path) {
720 Ok(Some(file_id)) => match dep_idx.get_dependencies_info(file_id) {
721 Ok(dep_infos) if !dep_infos.is_empty() => {
722 log::debug!(
723 "Loaded {} dependencies for file: {}",
724 dep_infos.len(),
725 path
726 );
727 Some(dep_infos)
728 }
729 Ok(_) => None,
730 Err(e) => {
731 log::warn!("Failed to get dependencies for {}: {}", path, e);
732 None
733 }
734 },
735 Ok(None) => {
736 log::warn!("No file_id found for path: {}", path);
737 None
738 }
739 Err(e) => {
740 log::warn!("Failed to get file_id for path {}: {}", path, e);
741 None
742 }
743 }
744 } else {
745 None
746 };
747
748 let file_id_for_context = open_opt.as_deref().and_then(|o| o.file_id_for(&path));
750 log::debug!(
751 "Context extraction: file={}, file_id={:?}, content_reader={}",
752 path,
753 file_id_for_context,
754 content_reader_opt.is_some()
755 );
756
757 let matches: Vec<MatchResult> = file_matches
759 .into_iter()
760 .map(|r| {
761 let (context_before, context_after) = if context_lines > 0 {
763 if let (Some(reader), Some(fid)) =
764 (&content_reader_opt, file_id_for_context)
765 {
766 let result = reader
767 .get_context_by_line(fid, r.span.start_line, context_lines)
768 .unwrap_or_else(|e| {
769 log::warn!(
770 "Failed to extract context for {}:{}: {}",
771 path,
772 r.span.start_line,
773 e
774 );
775 (vec![], vec![])
776 });
777 log::debug!(
778 "Extracted context for {}:{} - before: {}, after: {}",
779 path,
780 r.span.start_line,
781 result.0.len(),
782 result.1.len()
783 );
784 result
785 } else {
786 if content_reader_opt.is_none() {
787 log::debug!(
788 "No ContentReader available for context extraction"
789 );
790 }
791 if file_id_for_context.is_none() {
792 log::debug!("No file_id found for {}", path);
793 }
794 (vec![], vec![])
795 }
796 } else {
797 (vec![], vec![])
798 };
799
800 MatchResult {
801 kind: r.kind,
802 symbol: r.symbol,
803 span: r.span,
804 preview: r.preview,
805 context_before,
806 context_after,
807 }
808 })
809 .collect();
810
811 FileGroupedResult {
812 path,
813 language,
814 dependencies,
815 matches,
816 }
817 })
818 .collect();
819
820 file_results.sort_by(|a, b| a.path.cmp(&b.path));
822
823 Ok(file_results)
824 }
825
826 pub fn search_with_metadata(
831 &self,
832 pattern: &str,
833 filter: QueryFilter,
834 ) -> Result<QueryResponse> {
835 log::info!(
836 "Executing query with metadata: pattern='{}', filter={:?}",
837 pattern,
838 filter
839 );
840
841 let started = std::time::Instant::now();
842
843 if !self.cache.exists() {
845 return Err(crate::errors::ReflexError::IndexNotFound.into());
846 }
847
848 self.open_index()?;
853 let open_us = started.elapsed().as_micros() as u64;
854
855 let prepared = prepare_literal_pattern(pattern, &filter);
859 let mut filter = filter;
860 if prepared.use_regex && !filter.use_regex {
861 log::debug!(
862 "Pattern {:?} rewritten to regex {:?}",
863 pattern,
864 prepared.effective
865 );
866 filter.rewritten_from = Some(pattern.to_string());
867 }
868 filter.use_regex = prepared.use_regex;
869
870 let status_started = std::time::Instant::now();
876 let (searched, search_done, snapshot) = std::thread::scope(|s| {
877 let status = s.spawn(|| {
878 let snapshot = status_cache::snapshot(&self.cache);
879 (snapshot, status_started.elapsed())
880 });
881 let searched = self.search_internal(&prepared.effective, filter.clone());
882 let search_done = started.elapsed();
884 let snapshot = status
885 .join()
886 .unwrap_or_else(|panic| std::panic::resume_unwind(panic));
887 (searched, search_done, snapshot)
888 });
889 let Internal {
890 results,
891 total_count: total,
892 total_is_exact,
893 approx_total,
894 substring_only,
895 excluded_by_default,
896 counted,
897 candidates_us,
898 index_path,
899 warnings: engine_warnings,
900 } = searched?;
901 let (snapshot, status_compute) = snapshot;
902 let snapshot = snapshot?;
903
904 let scope: Vec<String> = results.iter().map(|r| r.path.clone()).collect();
908 let report = Self::report_from(&snapshot, Some(&scope));
909 let (status, can_trust_results, warning) =
910 (report.status, report.can_trust_results, report.warning);
911 let status_done = started.elapsed();
912
913 use crate::models::PaginationInfo;
915 let pagination = PaginationInfo {
916 total: total_is_exact.then_some(total),
918 count: results.len(),
919 offset: filter.offset.unwrap_or(0),
920 limit: filter.limit,
921 has_more: !total_is_exact || total > filter.offset.unwrap_or(0) + results.len(),
922 total_is_exact,
923 approx_total,
924 };
925
926 let grouped_results = self.group_and_load_dependencies(
929 results,
930 filter.include_dependencies,
931 filter.context_lines,
932 )?;
933
934 let total_elapsed = started.elapsed();
935 let timings = crate::models::QueryTimings {
936 index_path,
937 open_us,
938 candidates_us,
939 verify_us: (search_done.as_micros() as u64).saturating_sub(open_us + candidates_us),
940 status_us: (status_done - search_done).as_micros() as u64,
941 status_compute_us: status_compute.as_micros() as u64,
942 group_us: (total_elapsed - status_done).as_micros() as u64,
943 total_us: total_elapsed.as_micros() as u64,
944 };
945 log::debug!("Query timings for '{}': {:?}", pattern, timings);
946
947 let substring_hint_count = if total == 0 { substring_only } else { None };
951 let excluded_by_default =
952 (total == 0 && excluded_by_default > 0).then_some(excluded_by_default);
953 let zero = if total == 0 {
954 let config = self.cache.load_index_config().unwrap_or_default();
955 self.open_index().ok().and_then(|open| {
956 zero_hint::explain_zero(
957 &self.cache.workspace_root(),
958 &config,
959 &open,
960 &filter,
961 pattern,
962 substring_only,
963 excluded_by_default.unwrap_or(0),
964 prepared.warning.is_some(),
965 )
966 })
967 } else {
968 None
969 };
970 let (hint, excluded_reason) = match zero {
971 Some(z) => (Some(z.text), Some(z.reason)),
972 None => (None, None),
973 };
974
975 Ok(QueryResponse {
976 ai_instruction: None, status,
978 can_trust_results,
979 warning,
980 pagination,
981 results: grouped_results,
982 substring_hint_count,
983 warnings: prepared
984 .warning
985 .into_iter()
986 .chain(engine_warnings)
987 .collect(),
988 hint,
989 excluded_reason,
990 excluded_by_default,
991 file_count: counted.map(|(_, files)| files),
992 timings: filter.collect_timings.then_some(timings),
993 })
994 }
995
996 pub fn search(&self, pattern: &str, filter: QueryFilter) -> Result<Vec<SearchResult>> {
1001 log::info!(
1002 "Executing query: pattern='{}', filter={:?}",
1003 pattern,
1004 filter
1005 );
1006
1007 if !self.cache.exists() {
1009 return Err(crate::errors::ReflexError::IndexNotFound.into());
1010 }
1011
1012 self.open_index()?;
1014
1015 self.check_index_freshness(&filter)?;
1017
1018 let prepared = prepare_literal_pattern(pattern, &filter);
1021 let mut filter = filter;
1022 if prepared.use_regex && !filter.use_regex {
1023 filter.rewritten_from = Some(pattern.to_string());
1024 }
1025 filter.use_regex = prepared.use_regex;
1026
1027 let mut results = self
1029 .search_internal(&prepared.effective, filter.clone())?
1030 .results;
1031
1032 self.load_dependencies(&mut results, filter.include_dependencies)?;
1034
1035 Ok(results)
1036 }
1037
1038 fn search_internal(&self, pattern: &str, filter: QueryFilter) -> Result<Internal> {
1041 use std::time::{Duration, Instant};
1042
1043 let start_time = Instant::now();
1045 let timeout = if filter.timeout_secs > 0 {
1046 Some(Duration::from_secs(filter.timeout_secs))
1047 } else {
1048 None
1049 };
1050
1051 let is_keyword_query = if filter.symbols_mode || filter.kind.is_some() {
1065 pattern.is_empty() || ParserFactory::get_all_keywords().contains(&pattern)
1066 } else {
1067 false
1068 };
1069
1070 let mut filter = filter.clone(); if is_keyword_query
1075 && filter.kind.is_none()
1076 && let Some(inferred_kind) = Self::keyword_to_kind(pattern)
1077 {
1078 log::info!(
1079 "Keyword '{}' mapped to kind {:?} (auto-inferred)",
1080 pattern,
1081 inferred_kind
1082 );
1083 filter.kind = Some(inferred_kind);
1084 }
1085
1086 let guarded_literal = filter.rewritten_from.as_deref();
1100 if !filter.force && (!filter.use_regex || guarded_literal.is_some()) && !is_keyword_query {
1101 let total_files = self.open_index()?.file_count();
1104 let pattern = guarded_literal.unwrap_or(pattern);
1105 let pattern_len = pattern.chars().count();
1106
1107 let large_index_threshold = filter.test_large_index_threshold.unwrap_or(20_000);
1112 let short_pattern_threshold = filter.test_short_pattern_threshold.unwrap_or(4);
1113
1114 if total_files > large_index_threshold && pattern_len < short_pattern_threshold {
1115 anyhow::bail!(
1116 "Query too broad - would be expensive to execute on this large index\n\
1117 \n\
1118 This index contains {} files, and pattern '{}' ({} characters) is too short for efficient searching.\n\
1119 On large codebases, short patterns can take 10-30+ seconds to complete.\n\
1120 \n\
1121 This query could:\n\
1122 • Hang for an extended period before returning results\n\
1123 • Return thousands of results\n\
1124 • Flood LLM context windows with excessive data\n\
1125 • Fail entirely\n\
1126 \n\
1127 Suggestions to narrow the query:\n\
1128 • Use a longer, more specific pattern (4+ characters recommended for large indexes)\n\
1129 • Add a language filter: --lang <language>\n\
1130 • Add a file filter: --glob <pattern> or --file <path>\n\
1131 • Use --force to bypass this check if you really need all results\n\
1132 \n\
1133 To force execution anyway:\n\
1134 rfx query \"{}\" --force",
1135 total_files,
1136 pattern,
1137 pattern_len,
1138 pattern
1139 );
1140 }
1141 }
1142
1143 let budget = if filter.symbols_mode
1149 || filter.kind.is_some()
1150 || filter.use_ast
1151 || is_keyword_query
1152 || filter.require_exact_total
1153 {
1154 None
1155 } else {
1156 filter
1157 .limit
1158 .map(|limit| filter.offset.unwrap_or(0).saturating_add(limit))
1159 };
1160
1161 let mut substring_only = None;
1163 let mut excluded_by_default = 0usize;
1164 let mut counted = None;
1165 let mut candidates_us = 0u64;
1166 let mut total_is_exact = true;
1167 let mut approx_total = None;
1168 let index_path;
1169 let engine_warnings;
1170 let mut results = if is_keyword_query {
1171 index_path = IndexPath::Scan;
1173 engine_warnings = Vec::new();
1174 if let Some(lang) = filter.language {
1177 log::info!(
1178 "Keyword query detected for '{}' - scanning all {:?} files (bypassing trigram search)",
1179 pattern,
1180 lang
1181 );
1182 } else {
1183 log::info!(
1184 "Keyword query detected for '{}' - scanning all files (bypassing trigram search)",
1185 pattern
1186 );
1187 }
1188 self.get_all_language_files(&filter)?
1189 } else {
1190 let (candidates, stats) = if filter.use_regex {
1191 self.get_regex_candidates(pattern, &filter, timeout.as_ref(), &start_time, budget)?
1193 } else {
1194 self.get_trigram_candidates(pattern, &filter, budget)?
1196 };
1197 substring_only = stats.substring_only;
1198 excluded_by_default = stats.excluded_by_default;
1199 counted = stats.counted;
1200 candidates_us = stats.candidates_us;
1201 total_is_exact = stats.exhausted;
1202 approx_total = (!stats.exhausted).then_some(stats.approx_total).flatten();
1203 index_path = stats.index_path;
1204 engine_warnings = stats.warnings;
1205 candidates
1206 };
1207
1208 if !is_keyword_query && let Some(lang) = filter.language {
1214 let before_count = results.len();
1215 results.retain(|r| r.lang == lang);
1216 log::debug!(
1217 "Language filter ({:?}): reduced {} candidates to {} candidates",
1218 lang,
1219 before_count,
1220 results.len()
1221 );
1222 }
1223
1224 if !filter.glob_patterns.is_empty() || !filter.exclude_patterns.is_empty() {
1228 let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
1230
1231 let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
1233
1234 let before_count = results.len();
1236 results.retain(|r| {
1237 let included = if let Some(ref matcher) = include_matcher {
1239 matcher.is_match(&r.path)
1240 } else {
1241 true };
1243
1244 let excluded = if let Some(ref matcher) = exclude_matcher {
1246 matcher.is_match(&r.path)
1247 } else {
1248 false };
1250
1251 included && !excluded
1252 });
1253 log::debug!(
1254 "Glob filter: reduced {} candidates to {} candidates",
1255 before_count,
1256 results.len()
1257 );
1258 }
1259
1260 if let Some(timeout_duration) = timeout
1262 && start_time.elapsed() > timeout_duration
1263 {
1264 anyhow::bail!(
1265 "Query timeout exceeded ({} seconds).\n\
1266 \n\
1267 The query took too long to complete. Try one of these approaches:\n\
1268 • Use a more specific search pattern (longer patterns = faster search)\n\
1269 • Add a language filter with --lang to narrow the search space\n\
1270 • Add a file filter with --file to search specific directories\n\
1271 • Increase the timeout with --timeout <seconds>\n\
1272 \n\
1273 Example: rfx query \"{}\" --lang rust --timeout 60",
1274 filter.timeout_secs,
1275 pattern
1276 );
1277 }
1278
1279 if !filter.force {
1282 let candidate_count = results.len();
1283 let pattern_len = pattern.chars().count();
1284
1285 let is_short_pattern = pattern_len < 3 && !filter.use_regex && !is_keyword_query;
1288
1289 let is_broad_ast =
1292 filter.use_ast && filter.glob_patterns.is_empty() && candidate_count >= 100;
1293
1294 let threshold = if filter.use_ast && filter.glob_patterns.is_empty() {
1301 100 } else if filter.use_ast {
1303 10_000 } else if is_keyword_query {
1305 20_000 } else {
1307 50_000 };
1309
1310 let has_many_candidates = candidate_count > threshold
1311 && (filter.symbols_mode || filter.kind.is_some() || filter.use_ast);
1312
1313 if is_short_pattern || has_many_candidates || is_broad_ast {
1314 let reason = if is_short_pattern {
1315 format!(
1316 "Pattern '{}' is too short ({} characters). Short patterns bypass trigram optimization and require scanning many files.",
1317 pattern, pattern_len
1318 )
1319 } else if is_broad_ast {
1320 format!(
1321 "AST query without --glob restriction will scan the entire codebase ({} files). AST queries are SLOW (500ms-10s+).",
1322 candidate_count
1323 )
1324 } else if is_keyword_query {
1325 format!(
1326 "Keyword query '{}' matched {} files. This query scans all files of the target language, which will take significant time and produce excessive results.",
1327 pattern, candidate_count
1328 )
1329 } else {
1330 format!(
1331 "Query matched {} files. Parsing this many files with --symbols or --kind will take significant time and produce excessive results.",
1332 candidate_count
1333 )
1334 };
1335
1336 let suggestions = if is_short_pattern {
1337 vec![
1338 "• Use a longer, more specific pattern (3+ characters recommended)",
1339 "• Add a language filter: --lang <language>",
1340 "• Add a file path filter: --file <path> or --glob <pattern>",
1341 "• Use --force to bypass this check if you really need all results",
1342 ]
1343 } else if is_broad_ast {
1344 vec![
1345 "• Add --glob to restrict AST query to specific files: --glob 'src/**/*.rs'",
1346 "• Use --symbols instead (10-100x faster in 95% of cases)",
1347 "• Use --force to bypass this check if you need a full codebase scan",
1348 ]
1349 } else if is_keyword_query {
1350 vec![
1351 "• Add a language filter to reduce files scanned: --lang <language>",
1352 "• Add glob patterns to search specific directories: --glob 'src/**/*.rs'",
1353 "• Add --kind to filter to specific symbol types: --kind function",
1354 "• Use a more specific pattern instead of a keyword",
1355 "• Use --force to bypass this check if you need all results",
1356 ]
1357 } else {
1358 vec![
1359 "• Add a language filter to reduce candidate set: --lang <language>",
1360 "• Add glob patterns to search specific directories: --glob 'src/**/*.rs'",
1361 "• Use a more specific search pattern",
1362 "• Use --force to bypass this check if you need all results",
1363 ]
1364 };
1365
1366 let mut cmd_flags = String::new();
1368 if filter.symbols_mode {
1369 cmd_flags.push_str("--symbols ");
1370 }
1371 if let Some(ref lang) = filter.language {
1372 cmd_flags.push_str(&format!("--lang {:?} ", lang));
1373 }
1374 if let Some(ref kind) = filter.kind {
1375 cmd_flags.push_str(&format!("--kind {:?} ", kind));
1376 }
1377 if filter.use_ast {
1378 cmd_flags.push_str("--ast ");
1379 }
1380
1381 anyhow::bail!(
1382 "Query too broad - would be expensive to execute\n\
1383 \n\
1384 {}\n\
1385 \n\
1386 This query could:\n\
1387 • Hang for an extended period before returning results\n\
1388 • Return thousands of results\n\
1389 • Flood LLM context windows with excessive data\n\
1390 • Fail entirely\n\
1391 \n\
1392 Suggestions to narrow the query:\n\
1393 {}\n\
1394 \n\
1395 To force execution anyway:\n\
1396 rfx query \"{}\" --force {}",
1397 reason,
1398 suggestions.join("\n "),
1399 pattern,
1400 cmd_flags
1401 );
1402 }
1403 }
1404
1405 if filter.symbols_mode || filter.kind.is_some() || filter.use_ast {
1408 results.sort_by(|a, b| {
1409 a.path
1410 .cmp(&b.path)
1411 .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1412 });
1413
1414 let candidate_count = results.len();
1416 if candidate_count > 1000 && !filter.suppress_output {
1417 output::warn(&format!(
1418 "Pattern '{}' matched {} files - parsing may take some time. Consider using --file, --glob, or a more specific pattern to narrow the search.",
1419 pattern, candidate_count
1420 ));
1421 } else if candidate_count > 100 {
1422 log::info!(
1423 "Parsing {} candidate files for symbol extraction",
1424 candidate_count
1425 );
1426 }
1427 }
1428
1429 if filter.use_ast {
1431 results = self.enrich_with_ast(results, pattern, filter.language)?;
1433 } else if filter.symbols_mode || filter.kind.is_some() {
1434 results = self.enrich_with_symbols(results, pattern, &filter, is_keyword_query)?;
1436 }
1437
1438 if filter.symbols_mode || filter.kind.is_some() {
1447 let mut seen = std::collections::HashSet::<(String, usize, Option<String>)>::new();
1448 results.retain(|r| seen.insert((r.path.clone(), r.span.start_line, r.symbol.clone())));
1449 }
1450
1451 if let Some(ref kind) = filter.kind {
1454 results.retain(|r| {
1455 if matches!(kind, SymbolKind::Function) {
1456 matches!(r.kind, SymbolKind::Function | SymbolKind::Method)
1458 } else {
1459 r.kind == *kind
1460 }
1461 });
1462 }
1463
1464 if let Some(ref file_pattern) = filter.file_pattern {
1466 results.retain(|r| r.path.contains(file_pattern));
1467 }
1468
1469 if filter.exact && filter.symbols_mode {
1471 results.retain(|r| r.symbol.as_deref() == Some(pattern));
1472 }
1473
1474 if filter.expand {
1477 if let Ok(open) = self.open_index() {
1479 let content_reader = &open.content;
1480 for result in &mut results {
1481 if result.span.start_line < result.span.end_line {
1483 if let Some(file_id) = open.file_id_for(&result.path) {
1485 if let Ok(content) = content_reader.get_file_content(file_id) {
1487 let lines: Vec<&str> = content.lines().collect();
1488 let start_idx = result.span.start_line.saturating_sub(1);
1489 let end_idx = result.span.end_line.min(lines.len());
1490
1491 if start_idx < end_idx {
1492 let full_body = lines[start_idx..end_idx].join("\n");
1493 result.preview =
1497 crate::parsers::preview::expand_preview(&full_body);
1498 }
1499 }
1500 }
1501 }
1502 }
1503 }
1504 }
1505
1506 if filter.paths_only {
1508 use std::collections::HashSet;
1509 let mut seen_paths = HashSet::new();
1510 results.retain(|r| seen_paths.insert(r.path.clone()));
1511 }
1512
1513 results.retain(|r| {
1517 filter::tier_admits(
1518 r.lang,
1519 filter.language,
1520 filter.exclude_text,
1521 filter.include_locks,
1522 filter.include_generated,
1523 )
1524 });
1525
1526 results.sort_by(|a, b| {
1528 a.path
1529 .cmp(&b.path)
1530 .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1531 });
1532
1533 let total_count = counted.map(|(lines, _)| lines).unwrap_or(results.len());
1537
1538 if let Some(offset) = filter.offset {
1540 if offset < results.len() {
1541 results = results.into_iter().skip(offset).collect();
1542 } else {
1543 results.clear();
1545 }
1546 }
1547
1548 if let Some(limit) = filter.limit {
1550 results.truncate(limit);
1551 }
1552
1553 log::info!(
1554 "Query returned {} results (total before pagination: {})",
1555 results.len(),
1556 total_count
1557 );
1558
1559 Ok(Internal {
1560 results,
1561 total_count,
1562 total_is_exact,
1563 approx_total,
1564 substring_only,
1565 excluded_by_default,
1566 counted,
1567 candidates_us,
1568 index_path,
1569 warnings: engine_warnings,
1570 })
1571 }
1572
1573 pub fn find_symbol(&self, name: &str) -> Result<Vec<SearchResult>> {
1575 let filter = QueryFilter {
1576 symbols_mode: true,
1577 ..Default::default()
1578 };
1579 self.search(name, filter)
1580 }
1581
1582 pub fn search_ast(&self, pattern: &str, lang: Option<Language>) -> Result<Vec<SearchResult>> {
1584 let filter = QueryFilter {
1585 language: lang,
1586 use_ast: true,
1587 ..Default::default()
1588 };
1589
1590 self.search(pattern, filter)
1591 }
1592
1593 pub fn search_ast_all_files(
1614 &self,
1615 ast_pattern: &str,
1616 filter: QueryFilter,
1617 ) -> Result<Vec<SearchResult>> {
1618 log::info!(
1619 "Executing AST query on all files: pattern='{}', filter={:?}",
1620 ast_pattern,
1621 filter
1622 );
1623
1624 let lang = filter.language.ok_or_else(|| anyhow::anyhow!(
1626 "Language must be specified for AST pattern matching. Use --lang to specify the language.\n\
1627 \n\
1628 Example: rfx query \"(function_definition) @fn\" --ast --lang python"
1629 ))?;
1630
1631 if !self.cache.exists() {
1633 return Err(crate::errors::ReflexError::IndexNotFound.into());
1634 }
1635
1636 self.check_index_freshness(&filter)?;
1638
1639 let open = self.open_index()?;
1641 let content_reader = &open.content;
1642
1643 let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
1645
1646 let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
1647
1648 let mut candidates: Vec<SearchResult> = Vec::new();
1650
1651 for file_id in 0..content_reader.file_count() {
1652 let file_path = match content_reader.get_file_path(file_id as u32) {
1653 Some(p) => p,
1654 None => continue,
1655 };
1656
1657 let detected_lang = Language::from_path(file_path);
1659
1660 if detected_lang != lang {
1662 continue;
1663 }
1664
1665 let file_path_str = file_path.to_string_lossy().to_string();
1666
1667 let included = include_matcher
1669 .as_ref()
1670 .is_none_or(|m| m.is_match(&file_path_str));
1671 let excluded = exclude_matcher
1672 .as_ref()
1673 .is_some_and(|m| m.is_match(&file_path_str));
1674
1675 if !included || excluded {
1676 continue;
1677 }
1678
1679 candidates.push(SearchResult {
1681 path: file_path_str,
1682 lang: detected_lang,
1683 span: Span {
1684 start_line: 1,
1685 end_line: 1,
1686 },
1687 symbol: None,
1688 kind: SymbolKind::Unknown("ast_query".to_string()),
1689 preview: String::new(),
1690 dependencies: None,
1691 });
1692 }
1693
1694 log::info!(
1695 "AST query scanning {} files for language {:?}",
1696 candidates.len(),
1697 lang
1698 );
1699
1700 if !filter.force && filter.glob_patterns.is_empty() && candidates.len() >= 100 {
1703 anyhow::bail!(
1704 "Query too broad - would be expensive to execute\n\
1705 \n\
1706 AST query without --glob restriction will scan the ENTIRE codebase ({} files). AST queries are SLOW (500ms-10s+).\n\
1707 \n\
1708 This query could:\n\
1709 • Hang for an extended period before returning results\n\
1710 • Return thousands of results\n\
1711 • Flood LLM context windows with excessive data\n\
1712 • Fail entirely\n\
1713 \n\
1714 Suggestions to narrow the query:\n\
1715 • Add --glob to restrict AST query to specific files: --glob 'src/**/*.rs'\n\
1716 • Use --symbols instead (10-100x faster in 95% of cases)\n\
1717 • Use --force to bypass this check if you need a full codebase scan\n\
1718 \n\
1719 To force execution anyway:\n\
1720 rfx query \"{}\" --force --ast --lang {:?}",
1721 candidates.len(),
1722 ast_pattern,
1723 lang
1724 );
1725 }
1726
1727 if candidates.is_empty() {
1728 if !filter.suppress_output {
1729 output::warn(&format!(
1730 "No files found for language {:?}. Check your language filter or glob patterns.",
1731 lang
1732 ));
1733 }
1734 return Ok(Vec::new());
1735 }
1736
1737 let mut results = self.enrich_with_ast(candidates, ast_pattern, filter.language)?;
1740
1741 log::debug!("AST query found {} matches before filtering", results.len());
1742
1743 if let Some(ref kind) = filter.kind {
1747 results.retain(|r| {
1748 if matches!(kind, SymbolKind::Function) {
1749 matches!(r.kind, SymbolKind::Function | SymbolKind::Method)
1750 } else {
1751 r.kind == *kind
1752 }
1753 });
1754 }
1755
1756 if filter.expand
1760 && let Ok(open) = self.open_index()
1761 {
1762 let content_reader = &open.content;
1763 {
1764 for result in &mut results {
1765 if result.span.start_line < result.span.end_line
1766 && let Some(file_id) = open.file_id_for(&result.path)
1767 && let Ok(content) = content_reader.get_file_content(file_id)
1768 {
1769 let lines: Vec<&str> = content.lines().collect();
1770 let start_idx = result.span.start_line.saturating_sub(1);
1771 let end_idx = result.span.end_line.min(lines.len());
1772
1773 if start_idx < end_idx {
1774 let full_body = lines[start_idx..end_idx].join("\n");
1775 result.preview = crate::parsers::preview::expand_preview(&full_body);
1777 }
1778 }
1779 }
1780 }
1781 }
1782
1783 if filter.paths_only {
1785 use std::collections::HashSet;
1786 let mut seen_paths = HashSet::new();
1787 results.retain(|r| seen_paths.insert(r.path.clone()));
1788 }
1789
1790 results.retain(|r| {
1794 filter::tier_admits(
1795 r.lang,
1796 filter.language,
1797 filter.exclude_text,
1798 filter.include_locks,
1799 filter.include_generated,
1800 )
1801 });
1802
1803 results.sort_by(|a, b| {
1805 a.path
1806 .cmp(&b.path)
1807 .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1808 });
1809
1810 if let Some(offset) = filter.offset {
1812 if offset < results.len() {
1813 results = results.into_iter().skip(offset).collect();
1814 } else {
1815 results.clear();
1816 }
1817 }
1818
1819 if let Some(limit) = filter.limit {
1821 results.truncate(limit);
1822 }
1823
1824 log::info!("AST query returned {} results", results.len());
1825
1826 self.load_dependencies(&mut results, filter.include_dependencies)?;
1828
1829 Ok(results)
1830 }
1831
1832 pub fn search_ast_with_text_filter(
1844 &self,
1845 text_pattern: &str,
1846 ast_pattern: &str,
1847 filter: QueryFilter,
1848 ) -> Result<Vec<SearchResult>> {
1849 log::info!(
1850 "Executing AST query with text filter: text='{}', ast='{}', filter={:?}",
1851 text_pattern,
1852 ast_pattern,
1853 filter
1854 );
1855
1856 if !self.cache.exists() {
1858 return Err(crate::errors::ReflexError::IndexNotFound.into());
1859 }
1860
1861 self.check_index_freshness(&filter)?;
1863
1864 use std::time::{Duration, Instant};
1866 let start_time = Instant::now();
1867 let timeout = if filter.timeout_secs > 0 {
1868 Some(Duration::from_secs(filter.timeout_secs))
1869 } else {
1870 None
1871 };
1872
1873 let candidates = if filter.use_regex {
1875 self.get_regex_candidates(text_pattern, &filter, timeout.as_ref(), &start_time, None)?
1876 .0
1877 } else {
1878 self.get_trigram_candidates(text_pattern, &filter, None)?.0
1879 };
1880
1881 log::debug!("Phase 1 found {} candidate locations", candidates.len());
1882
1883 let mut results = self.enrich_with_ast(candidates, ast_pattern, filter.language)?;
1885
1886 log::debug!("Phase 2 AST matching found {} results", results.len());
1887
1888 if let Some(lang) = filter.language {
1890 results.retain(|r| r.lang == lang);
1891 }
1892
1893 if let Some(ref kind) = filter.kind {
1894 results.retain(|r| {
1895 if matches!(kind, SymbolKind::Function) {
1896 matches!(r.kind, SymbolKind::Function | SymbolKind::Method)
1897 } else {
1898 r.kind == *kind
1899 }
1900 });
1901 }
1902
1903 if let Some(ref file_pattern) = filter.file_pattern {
1904 results.retain(|r| r.path.contains(file_pattern));
1905 }
1906
1907 if !filter.glob_patterns.is_empty() || !filter.exclude_patterns.is_empty() {
1909 let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
1910
1911 let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
1912
1913 results.retain(|r| {
1914 let included = include_matcher.as_ref().is_none_or(|m| m.is_match(&r.path));
1915 let excluded = exclude_matcher
1916 .as_ref()
1917 .is_some_and(|m| m.is_match(&r.path));
1918 included && !excluded
1919 });
1920 }
1921
1922 if filter.exact && filter.symbols_mode {
1923 results.retain(|r| r.symbol.as_deref() == Some(text_pattern));
1924 }
1925
1926 if filter.expand
1928 && let Ok(open) = self.open_index()
1929 {
1930 let content_reader = &open.content;
1931 {
1932 for result in &mut results {
1933 if result.span.start_line < result.span.end_line
1934 && let Some(file_id) = open.file_id_for(&result.path)
1935 && let Ok(content) = content_reader.get_file_content(file_id)
1936 {
1937 let lines: Vec<&str> = content.lines().collect();
1938 let start_idx = result.span.start_line.saturating_sub(1);
1939 let end_idx = result.span.end_line.min(lines.len());
1940
1941 if start_idx < end_idx {
1942 let full_body = lines[start_idx..end_idx].join("\n");
1943 result.preview = crate::parsers::preview::expand_preview(&full_body);
1945 }
1946 }
1947 }
1948 }
1949 }
1950
1951 results.retain(|r| {
1955 filter::tier_admits(
1956 r.lang,
1957 filter.language,
1958 filter.exclude_text,
1959 filter.include_locks,
1960 filter.include_generated,
1961 )
1962 });
1963
1964 results.sort_by(|a, b| {
1966 a.path
1967 .cmp(&b.path)
1968 .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1969 });
1970
1971 if let Some(offset) = filter.offset {
1973 if offset < results.len() {
1974 results = results.into_iter().skip(offset).collect();
1975 } else {
1976 results.clear();
1977 }
1978 }
1979
1980 if let Some(limit) = filter.limit {
1982 results.truncate(limit);
1983 }
1984
1985 log::info!("AST query returned {} results", results.len());
1986
1987 Ok(results)
1988 }
1989
1990 pub fn list_by_kind(&self, kind: SymbolKind) -> Result<Vec<SearchResult>> {
1992 let filter = QueryFilter {
1993 kind: Some(kind),
1994 symbols_mode: true,
1995 ..Default::default()
1996 };
1997
1998 self.search("*", filter)
1999 }
2000
2001 fn enrich_with_symbols(
2022 &self,
2023 candidates: Vec<SearchResult>,
2024 pattern: &str,
2025 filter: &QueryFilter,
2026 keyword_query: bool,
2027 ) -> Result<Vec<SearchResult>> {
2028 use rayon::prelude::*;
2029 use std::collections::{HashMap, HashSet};
2030
2031 let open = self.open_index()?;
2033 let content_reader = &open.content;
2034
2035 let mut files_by_path: HashMap<String, Vec<SearchResult>> = HashMap::new();
2037 let mut skipped_unsupported = 0;
2038
2039 for candidate in candidates {
2040 if !candidate.lang.is_supported() {
2042 skipped_unsupported += 1;
2043 continue;
2044 }
2045
2046 files_by_path
2047 .entry(candidate.path.clone())
2048 .or_default()
2049 .push(candidate);
2050 }
2051
2052 let total_files = files_by_path.len();
2053 log::debug!(
2054 "Processing {} candidate files for symbol enrichment (skipped {} unsupported language files)",
2055 total_files,
2056 skipped_unsupported
2057 );
2058
2059 if total_files > 1000 && !filter.suppress_output {
2061 output::warn(&format!(
2062 "Pattern '{}' matched {} files. This may take some time to parse. Consider using a more specific pattern or adding --lang/--file filters to narrow the search.",
2063 pattern, total_files
2064 ));
2065 }
2066
2067 let pool = open.pool();
2069
2070 let files_to_skip: HashSet<String> = if keyword_query || filter.use_regex {
2083 HashSet::new()
2084 } else {
2085 pool.install(|| {
2086 files_by_path
2087 .par_iter()
2088 .filter_map(|(file_path, cands)| {
2089 let lang = Language::from_path(std::path::Path::new(file_path));
2090 let line_filter = crate::line_filter::get_filter(lang)?;
2091 let file_id = open.file_id_for(file_path)?;
2092 let content = content_reader.get_file_content(file_id).ok()?;
2093 let all_lines: Vec<&str> = content.lines().collect();
2094
2095 let mut wanted: Vec<usize> =
2096 cands.iter().map(|c| c.span.start_line).collect();
2097 wanted.sort_unstable();
2098 wanted.dedup();
2099
2100 let mut saw_occurrence = false;
2101 for line_no in wanted {
2102 let Some(line) = line_no.checked_sub(1).and_then(|i| all_lines.get(i))
2103 else {
2104 continue;
2105 };
2106 let mut search_start = 0;
2107 while let Some(pos) = line[search_start..].find(pattern) {
2108 saw_occurrence = true;
2109 let at = search_start + pos;
2110 if !line_filter.is_in_comment(line, at)
2111 && !line_filter.is_in_string(line, at)
2112 {
2113 return None;
2115 }
2116 search_start = at + pattern.len();
2117 }
2118 }
2119 saw_occurrence.then(|| {
2121 log::debug!(
2122 "Pre-filter: Skipping {} (all matches in comments/strings)",
2123 file_path
2124 );
2125 file_path.clone()
2126 })
2127 })
2128 .collect()
2129 })
2130 };
2131
2132 let files_to_process: Vec<String> = files_by_path
2133 .keys()
2134 .filter(|p| !files_to_skip.contains(p.as_str()))
2135 .cloned()
2136 .collect();
2137
2138 log::debug!(
2139 "Pre-filter: Skipped {} files where all matches are in comments/strings (parsing {} files)",
2140 files_to_skip.len(),
2141 files_to_process.len()
2142 );
2143
2144 let root = self.cache.workspace_root();
2149 let branch = crate::git::read_head_branch(&root).unwrap_or_else(|| "_default".to_string());
2150 let mut conn = open.meta_conn()?;
2151 let rows =
2152 crate::cache::CacheManager::branch_file_rows_on(&conn, &branch, &files_to_process)
2153 .context("Failed to load file hashes")?;
2154 log::debug!(
2155 "Loaded {} file rows for branch '{}' for symbol cache lookups",
2156 rows.len(),
2157 branch
2158 );
2159
2160 let file_lookup_tuples: Vec<(i64, String, String)> = files_to_process
2161 .iter()
2162 .filter_map(|path| {
2163 let (id, hash) = rows.get(path)?;
2164 Some((*id, hash.clone(), path.clone()))
2165 })
2166 .collect();
2167
2168 let batch_results = crate::symbol_cache::SymbolCache::batch_get_with_kind_on(
2169 &conn,
2170 &file_lookup_tuples,
2171 filter.kind.clone(),
2172 )
2173 .context("Failed to batch read symbol cache")?;
2174
2175 let id_to_path: HashMap<i64, &str> = rows
2176 .iter()
2177 .map(|(path, (id, _))| (*id, path.as_str()))
2178 .collect();
2179
2180 let mut cached_symbols: HashMap<String, Vec<SearchResult>> = HashMap::new();
2181 for (file_id, symbols) in batch_results {
2182 if let Some(file_path) = id_to_path.get(&file_id) {
2183 cached_symbols.insert(file_path.to_string(), symbols);
2184 }
2185 }
2186
2187 let files_needing_parse: Vec<String> = files_to_process
2189 .iter()
2190 .filter(|p| !cached_symbols.contains_key(p.as_str()))
2191 .cloned()
2192 .collect();
2193
2194 log::debug!(
2195 "Symbol cache: {} hits, {} need parsing",
2196 cached_symbols.len(),
2197 files_needing_parse.len()
2198 );
2199
2200 type ParsedFile = (Vec<SearchResult>, Option<(i64, String)>);
2205 let parsed: Vec<ParsedFile> = pool.install(|| {
2206 files_needing_parse
2207 .par_iter()
2208 .map(|file_path| {
2209 let Some(file_id) = open.file_id_for(file_path) else {
2210 log::warn!("Could not find file_id for path: {}", file_path);
2211 return (Vec::new(), None);
2212 };
2213 let content = match content_reader.get_file_content(file_id) {
2214 Ok(c) => c,
2215 Err(e) => {
2216 log::warn!("Failed to read file {}: {}", file_path, e);
2217 return (Vec::new(), None);
2218 }
2219 };
2220 let lang = Language::from_path(std::path::Path::new(file_path));
2221 let symbols = match ParserFactory::parse(file_path, content, lang) {
2222 Ok(symbols) => {
2223 log::debug!("Parsed {} symbols from {}", symbols.len(), file_path);
2224 symbols
2225 }
2226 Err(e) => {
2227 log::debug!("Failed to parse {}: {}", file_path, e);
2228 Vec::new()
2229 }
2230 };
2231 let key = rows.get(file_path).map(|(id, hash)| (*id, hash.clone()));
2232 (symbols, key)
2233 })
2234 .collect()
2235 });
2236
2237 let mut parsed_symbols: Vec<SearchResult> = Vec::new();
2238 let mut to_cache: Vec<(i64, String, Vec<SearchResult>)> = Vec::new();
2239 for (symbols, key) in parsed {
2240 if let Some((id, hash)) = key {
2241 to_cache.push((id, hash, symbols.clone()));
2242 }
2243 parsed_symbols.extend(symbols);
2244 }
2245 if let Err(e) = crate::symbol_cache::SymbolCache::batch_set_by_id_on(&mut conn, &to_cache) {
2247 log::debug!(
2248 "Failed to cache symbols for {} files: {}",
2249 to_cache.len(),
2250 e
2251 );
2252 }
2253 drop(conn);
2254
2255 let mut all_symbols: Vec<SearchResult> = Vec::new();
2257
2258 for symbols in cached_symbols.values() {
2260 all_symbols.extend_from_slice(symbols);
2261 }
2262
2263 all_symbols.extend(parsed_symbols);
2265
2266 let is_keyword_query = {
2274 let lang_to_check = if let Some(lang) = filter.language {
2276 vec![lang]
2279 } else {
2280 let mut langs: Vec<Language> =
2284 all_symbols.iter().map(|s| s.lang).collect::<Vec<_>>();
2285 langs.sort_by(|a, b| format!("{:?}", a).cmp(&format!("{:?}", b))); langs.dedup(); langs
2288 };
2289
2290 lang_to_check
2292 .iter()
2293 .any(|lang| ParserFactory::get_keywords(*lang).contains(&pattern))
2294 };
2295
2296 let filtered: Vec<SearchResult> = if is_keyword_query {
2299 log::info!(
2300 "Pattern '{}' is a language keyword - listing all symbols (kind filtering will be applied in Phase 3)",
2301 pattern
2302 );
2303 all_symbols
2304 } else if filter.use_regex {
2305 use std::collections::{HashMap, HashSet};
2311 let mut candidate_lines: HashMap<String, HashSet<usize>> = HashMap::new();
2312 for candidate in &files_by_path {
2313 for cand in candidate.1 {
2314 candidate_lines
2315 .entry(candidate.0.clone())
2316 .or_default()
2317 .insert(cand.span.start_line);
2318 }
2319 }
2320
2321 all_symbols
2323 .into_iter()
2324 .filter(|sym| {
2325 if let Some(lines) = candidate_lines.get(&sym.path) {
2326 for line in sym.span.start_line..=sym.span.end_line {
2328 if lines.contains(&line) {
2329 return true;
2330 }
2331 }
2332 }
2333 false
2334 })
2335 .collect()
2336 } else if filter.use_contains {
2337 all_symbols
2339 .into_iter()
2340 .filter(|sym| sym.symbol.as_deref().is_some_and(|s| s.contains(pattern)))
2341 .collect()
2342 } else {
2343 all_symbols
2345 .into_iter()
2346 .filter(|sym| sym.symbol.as_deref() == Some(pattern))
2347 .collect()
2348 };
2349
2350 log::info!(
2351 "Symbol enrichment found {} matches for pattern '{}'",
2352 filtered.len(),
2353 pattern
2354 );
2355
2356 Ok(filtered)
2357 }
2358
2359 fn enrich_with_ast(
2378 &self,
2379 candidates: Vec<SearchResult>,
2380 ast_pattern: &str,
2381 language: Option<Language>,
2382 ) -> Result<Vec<SearchResult>> {
2383 let lang = language.ok_or_else(|| anyhow::anyhow!(
2385 "Language must be specified for AST pattern matching. Use --lang to specify the language."
2386 ))?;
2387
2388 let open = self.open_index()?;
2390 let content_reader = &open.content;
2391
2392 use std::collections::HashMap;
2394 let mut file_contents: HashMap<String, String> = HashMap::new();
2395
2396 for candidate in &candidates {
2397 if file_contents.contains_key(&candidate.path) {
2398 continue;
2399 }
2400
2401 let file_id = match open.file_id_for(&candidate.path) {
2403 Some(id) => id,
2404 None => {
2405 log::warn!("Could not find file_id for path: {}", candidate.path);
2406 continue;
2407 }
2408 };
2409
2410 let content = match content_reader.get_file_content(file_id) {
2412 Ok(c) => c,
2413 Err(e) => {
2414 log::warn!("Failed to read file {}: {}", candidate.path, e);
2415 continue;
2416 }
2417 };
2418
2419 file_contents.insert(candidate.path.clone(), content.to_string());
2420 }
2421
2422 log::debug!(
2423 "Executing AST query on {} candidate files with language {:?}",
2424 file_contents.len(),
2425 lang
2426 );
2427
2428 let results =
2430 crate::ast_query::execute_ast_query(candidates, ast_pattern, lang, &file_contents)?;
2431
2432 log::info!(
2433 "AST query found {} matches for pattern '{}'",
2434 results.len(),
2435 ast_pattern
2436 );
2437
2438 Ok(results)
2439 }
2440
2441 fn keyword_to_kind(keyword: &str) -> Option<SymbolKind> {
2449 filter::keyword_to_kind(keyword)
2450 }
2451
2452 fn get_all_language_files(&self, filter: &QueryFilter) -> Result<Vec<SearchResult>> {
2460 let open = self.open_index()?;
2465 let content_reader = &open.content;
2466
2467 let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
2469
2470 let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
2471
2472 let mut candidates: Vec<SearchResult> = Vec::new();
2474
2475 for file_id in 0..content_reader.file_count() {
2476 let file_path = match content_reader.get_file_path(file_id as u32) {
2477 Some(p) => p,
2478 None => continue,
2479 };
2480
2481 let detected_lang = Language::from_path(file_path);
2483
2484 if let Some(lang) = filter.language
2486 && detected_lang != lang
2487 {
2488 continue;
2489 }
2490
2491 let file_path_str = file_path.to_string_lossy().to_string();
2492
2493 let included = include_matcher
2495 .as_ref()
2496 .is_none_or(|m| m.is_match(&file_path_str));
2497 let excluded = exclude_matcher
2498 .as_ref()
2499 .is_some_and(|m| m.is_match(&file_path_str));
2500
2501 if !included || excluded {
2502 continue;
2503 }
2504
2505 if let Some(ref file_pattern) = filter.file_pattern
2507 && !file_path_str.contains(file_pattern)
2508 {
2509 continue;
2510 }
2511
2512 candidates.push(SearchResult {
2515 path: file_path_str,
2516 lang: detected_lang,
2517 span: Span {
2518 start_line: 1,
2519 end_line: 1,
2520 },
2521 symbol: None,
2522 kind: SymbolKind::Unknown("keyword_query".to_string()),
2523 preview: String::new(),
2524 dependencies: None,
2525 });
2526 }
2527
2528 if let Some(lang) = filter.language {
2529 log::info!(
2530 "Keyword query will scan {} {:?} files for symbol extraction",
2531 candidates.len(),
2532 lang
2533 );
2534 } else {
2535 log::info!(
2536 "Keyword query will scan {} files (all languages) for symbol extraction",
2537 candidates.len()
2538 );
2539 }
2540
2541 Ok(candidates)
2542 }
2543
2544 fn get_trigram_candidates(
2550 &self,
2551 pattern: &str,
2552 filter: &QueryFilter,
2553 budget: Option<usize>,
2554 ) -> Result<(Vec<SearchResult>, CandidateStats)> {
2555 let open = self.open_index()?;
2556 let trigram_index = &open.trigrams;
2557
2558 if pattern.chars().count() < 3 {
2562 log::info!(
2563 "Pattern '{}' is shorter than 3 chars — trigram index cannot be used, \
2564 falling back to linear scan",
2565 pattern
2566 );
2567 let results = self.linear_scan_candidates(pattern, filter, &open)?;
2568 let stats = CandidateStats {
2569 index_path: IndexPath::Scan,
2570 ..CandidateStats::default()
2571 };
2572 return Ok((results, stats));
2573 }
2574
2575 let candidates_started = std::time::Instant::now();
2577 let candidates = trigram_index.search_candidates(pattern);
2578 let candidates_us = candidates_started.elapsed().as_micros() as u64;
2579 log::debug!(
2580 "Found {} candidate locations from trigram search in {} us",
2581 candidates.len(),
2582 candidates_us
2583 );
2584
2585 let mut files: Vec<(u32, LineSet)> = Vec::new();
2588 for loc in candidates {
2589 match files.last_mut() {
2590 Some((id, LineSet::Only(lines))) if *id == loc.file_id => lines.push(loc.line_no),
2591 _ => files.push((loc.file_id, LineSet::Only(vec![loc.line_no]))),
2592 }
2593 }
2594 log::debug!("Scanning {} files with trigram matches", files.len());
2595
2596 let matcher = LineMatcher::new(pattern, filter)?;
2598 let file_filter = FileFilter::from_filter(filter);
2599
2600 let outcome = verify_files_streaming(
2601 &open,
2602 files,
2603 &matcher,
2604 &file_filter,
2605 filter.paths_only,
2606 budget,
2607 filter.count_only_line_search(),
2608 );
2609
2610 let stats = CandidateStats {
2611 candidates_us,
2612 substring_only: matcher.is_word_boundary().then_some(outcome.substring_only),
2613 excluded_by_default: outcome.excluded_by_default,
2614 counted: outcome.counted,
2615 exhausted: outcome.exhausted,
2616 approx_total: outcome.estimated_total,
2617 index_path: IndexPath::Trigram,
2618 warnings: Vec::new(),
2619 };
2620 Ok((outcome.results, stats))
2621 }
2622
2623 fn linear_scan_candidates(
2630 &self,
2631 pattern: &str,
2632 filter: &QueryFilter,
2633 open: &OpenIndex,
2634 ) -> Result<Vec<SearchResult>> {
2635 use rayon::prelude::*;
2636
2637 let content_reader = &open.content;
2638 let pattern_owned = pattern.to_string();
2639 let file_count = content_reader.file_count();
2640 let matcher = LineMatcher::new(pattern, filter)?;
2641
2642 let results: Vec<SearchResult> = open.pool().install(|| {
2643 (0..file_count as u32)
2644 .collect::<Vec<_>>()
2645 .par_iter()
2646 .flat_map(|&file_id| {
2647 let file_path = match content_reader.get_file_path(file_id) {
2648 Some(p) => p.to_path_buf(),
2649 None => return Vec::new(),
2650 };
2651 let content = match content_reader.get_file_content(file_id) {
2652 Ok(c) => c,
2653 Err(_) => return Vec::new(),
2654 };
2655
2656 let file_path_str = file_path.to_string_lossy().to_string();
2657 let lang = Language::from_path(&file_path);
2658
2659 let mut seen_lines = std::collections::HashSet::new();
2660 let mut file_results = Vec::new();
2661
2662 for (line_idx, line) in content.lines().enumerate() {
2663 let line_no = line_idx + 1;
2664 if seen_lines.contains(&line_no) {
2665 continue;
2666 }
2667
2668 if !matcher.is_match(line) {
2669 continue;
2670 }
2671
2672 seen_lines.insert(line_no);
2673 file_results.push(SearchResult {
2674 path: file_path_str.clone(),
2675 lang,
2676 kind: SymbolKind::Unknown("text_match".to_string()),
2677 symbol: None,
2678 span: Span {
2679 start_line: line_no,
2680 end_line: line_no,
2681 },
2682 preview: crate::parsers::preview::line_preview(
2686 line,
2687 line.find(pattern_owned.as_str()).unwrap_or(0),
2688 ),
2689 dependencies: None,
2690 });
2691 }
2692
2693 file_results
2694 })
2695 .collect()
2696 });
2697
2698 log::info!(
2699 "Linear scan (short pattern '{}') found {} results across {} files",
2700 pattern,
2701 results.len(),
2702 file_count
2703 );
2704 Ok(results)
2705 }
2706
2707 fn get_regex_candidates(
2728 &self,
2729 pattern: &str,
2730 filter: &QueryFilter,
2731 timeout: Option<&std::time::Duration>,
2732 start_time: &std::time::Instant,
2733 budget: Option<usize>,
2734 ) -> Result<(Vec<SearchResult>, CandidateStats)> {
2735 let matcher = LineMatcher::new(pattern, filter)
2737 .with_context(|| format!("Invalid regex pattern: {}", pattern))?;
2738
2739 if let Some(timeout_duration) = timeout
2741 && start_time.elapsed() > *timeout_duration
2742 {
2743 anyhow::bail!(
2744 "Query timeout exceeded ({} seconds) during regex compilation",
2745 timeout_duration.as_secs()
2746 );
2747 }
2748
2749 use crate::regex_trigrams::extract_literals;
2752 let literals = extract_literals(pattern);
2753 let unfoldable = literals
2754 .iter()
2755 .any(|l| l.case_insensitive && !l.text.is_ascii());
2756
2757 let open = self.open_index()?;
2758 let candidates_started = std::time::Instant::now();
2759
2760 let mut warnings = Vec::new();
2761 let index_path;
2762 let files: Vec<(u32, LineSet)> = if literals.is_empty() || unfoldable {
2763 index_path = IndexPath::Scan;
2765 let short_literal = filter
2768 .rewritten_from
2769 .as_deref()
2770 .is_some_and(|p| p.chars().count() < 3);
2771 if !short_literal {
2772 let text = if unfoldable {
2773 format!(
2774 "Regex pattern '{}' has a non-ASCII case-insensitive literal, which the trigram index cannot fold; falling back to full content scan. This may be slow on large codebases.",
2775 pattern
2776 )
2777 } else {
2778 format!(
2779 "Regex pattern '{}' has no literals (≥3 chars), falling back to full content scan. This may be slow on large codebases. Consider using patterns with literal text.",
2780 pattern
2781 )
2782 };
2783 if !filter.suppress_output {
2784 output::warn(&text);
2785 }
2786 warnings.push(text);
2787 }
2788 (0..open.content.file_count() as u32)
2789 .map(|id| (id, LineSet::All))
2790 .collect()
2791 } else {
2792 index_path = IndexPath::Trigram;
2793 log::debug!(
2794 "Using {} literals to narrow regex search candidates",
2795 literals.len()
2796 );
2797
2798 let mut locations: Vec<crate::trigram::FileLocation> = Vec::new();
2803 let mut sources = 0usize;
2804 let mut need_exotic = false;
2807 for literal in &literals {
2808 let found = if literal.case_insensitive {
2809 if literal
2810 .text
2811 .bytes()
2812 .any(|b| matches!(b.to_ascii_lowercase(), b'k' | b's'))
2813 {
2814 need_exotic = true;
2815 }
2816 open.trigrams
2817 .search_candidates_fold(literal.text.as_bytes())
2818 } else {
2819 open.trigrams.search_candidates(&literal.text)
2820 };
2821 log::debug!(
2822 "Literal '{}' (ci={}) found on {} candidate lines",
2823 literal.text,
2824 literal.case_insensitive,
2825 found.len()
2826 );
2827 locations.extend(found);
2828 sources += 1;
2829 }
2830 if need_exotic {
2831 locations.extend(open.trigrams.exotic_fold_lines());
2832 sources += 1;
2833 }
2834 if sources > 1 {
2835 locations.sort_unstable();
2836 locations.dedup();
2837 }
2838 let mut files: Vec<(u32, LineSet)> = Vec::new();
2840 for loc in locations {
2841 match files.last_mut() {
2842 Some((id, LineSet::Only(lines))) if *id == loc.file_id => {
2843 lines.push(loc.line_no)
2844 }
2845 _ => files.push((loc.file_id, LineSet::Only(vec![loc.line_no]))),
2846 }
2847 }
2848 files
2849 };
2850 let candidates_us = candidates_started.elapsed().as_micros() as u64;
2851 log::debug!(
2852 "Regex candidates: {} files in {} us",
2853 files.len(),
2854 candidates_us
2855 );
2856
2857 let file_filter = FileFilter::from_filter(filter);
2858 let outcome = verify_files_streaming(
2859 &open,
2860 files,
2861 &matcher,
2862 &file_filter,
2863 filter.paths_only,
2864 budget,
2865 filter.count_only_line_search(),
2866 );
2867
2868 log::info!(
2869 "Regex search found {} matches for pattern '{}'",
2870 outcome.results.len(),
2871 pattern
2872 );
2873 let stats = CandidateStats {
2874 candidates_us,
2875 substring_only: None,
2876 excluded_by_default: outcome.excluded_by_default,
2877 counted: outcome.counted,
2878 exhausted: outcome.exhausted,
2879 approx_total: outcome.estimated_total,
2880 index_path,
2881 warnings,
2882 };
2883 Ok((outcome.results, stats))
2884 }
2885
2886 pub fn get_index_status(&self) -> Result<(IndexStatus, bool, Option<IndexWarning>)> {
2891 self.index_status_for(None)
2892 }
2893
2894 pub fn fresh_index_status(&self) -> Result<(IndexStatus, bool, Option<IndexWarning>)> {
2899 let r = self.fresh_index_report()?;
2900 Ok((r.status, r.can_trust_results, r.warning))
2901 }
2902
2903 pub fn fresh_index_report(&self) -> Result<IndexStatusReport> {
2906 status_cache::invalidate(&self.cache.workspace_root());
2907 self.index_report_for(None)
2908 }
2909
2910 pub fn index_status_for(
2922 &self,
2923 scope: Option<&[String]>,
2924 ) -> Result<(IndexStatus, bool, Option<IndexWarning>)> {
2925 let r = self.index_report_for(scope)?;
2926 Ok((r.status, r.can_trust_results, r.warning))
2927 }
2928
2929 pub fn index_report_for(&self, scope: Option<&[String]>) -> Result<IndexStatusReport> {
2931 let snapshot = status_cache::snapshot(&self.cache)?;
2934 Ok(Self::report_from(&snapshot, scope))
2935 }
2936
2937 fn report_from(
2942 snapshot: &status_cache::Snapshot,
2943 scope: Option<&[String]>,
2944 ) -> IndexStatusReport {
2945 if let Some((status, can_trust_results, warning)) = snapshot.decided.clone() {
2946 return IndexStatusReport {
2947 status,
2948 can_trust_results,
2949 warning,
2950 details: snapshot.details.clone(),
2951 };
2952 }
2953 let changes = &snapshot.changes;
2954 let details = snapshot.details.clone();
2955
2956 if changes.is_empty() {
2957 return IndexStatusReport {
2958 status: IndexStatus::Fresh,
2959 can_trust_results: true,
2960 warning: None,
2961 details,
2962 };
2963 }
2964
2965 let hits_results = match scope {
2968 None | Some([]) => false,
2969 Some(paths) => changes.any(|changed| {
2970 paths
2971 .iter()
2972 .any(|p| p == changed || p.ends_with(changed) || changed.ends_with(p.as_str()))
2973 }),
2974 };
2975
2976 let reason = {
2977 let mut parts = Vec::new();
2978 if changes.modified_count > 0 {
2979 parts.push(format!("{} modified", changes.modified_count));
2980 }
2981 if changes.added_count > 0 {
2982 parts.push(format!("{} added", changes.added_count));
2983 }
2984 if changes.deleted_count > 0 {
2985 parts.push(format!("{} deleted", changes.deleted_count));
2986 }
2987 let impact = if hits_results {
2988 " — including a file these results came from"
2989 } else if changes.deleted_count > 0 {
2990 " — deleted files still produce hits at their old lines"
2991 } else {
2992 " — these results may not reflect them"
2993 };
2994 format!(
2995 "Files changed since the index was built ({}){}",
2996 parts.join(", "),
2997 impact
2998 )
2999 };
3000
3001 let some_if_any = |v: Vec<String>| if v.is_empty() { None } else { Some(v) };
3002 let warning = IndexWarning {
3003 reason,
3004 action_required: "index_project".to_string(),
3005 files_modified: some_if_any(changes.modified.clone()),
3006 files_added: some_if_any(changes.added.clone()),
3007 files_deleted: some_if_any(changes.deleted.clone()),
3008 changed_count: Some(
3009 changes.modified_count + changes.added_count + changes.deleted_count,
3010 ),
3011 truncated: changes.truncated,
3012 details: details.clone(),
3013 };
3014
3015 IndexStatusReport {
3019 status: IndexStatus::Stale,
3020 can_trust_results: false,
3021 warning: Some(warning),
3022 details,
3023 }
3024 }
3025
3026 fn check_index_freshness(&self, filter: &QueryFilter) -> Result<()> {
3033 let root = self.cache.workspace_root();
3034
3035 if crate::git::is_git_repo(&root) {
3037 if !crate::git::is_git_available() {
3038 static WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
3039 if !filter.suppress_output {
3040 WARNED.get_or_init(|| {
3041 output::warn("⚠️ git binary not found in PATH; index freshness checks disabled for this session.");
3042 });
3043 }
3044 return Ok(());
3045 }
3046 if let Ok(current_branch) = crate::git::get_current_branch(&root) {
3047 if !self.cache.branch_exists(¤t_branch).unwrap_or(false) {
3049 if !filter.suppress_output {
3050 output::warn(&format!(
3051 "⚠️ WARNING: Index not found for branch '{}'. Run 'rfx index' to index this branch.",
3052 current_branch
3053 ));
3054 }
3055 return Ok(());
3056 }
3057
3058 if let (Ok(current_commit), Ok(branch_info)) = (
3060 crate::git::get_current_commit(&root),
3061 self.cache.get_branch_info(¤t_branch),
3062 ) {
3063 if branch_info.commit_sha != current_commit {
3064 if !filter.suppress_output {
3065 output::warn(&format!(
3066 "⚠️ WARNING: Index may be stale (commit changed: {} → {}). Consider running 'rfx index'.",
3067 &branch_info.commit_sha[..7],
3068 ¤t_commit[..7]
3069 ));
3070 }
3071 return Ok(());
3072 }
3073
3074 if let Ok(branch_files) = self.cache.get_branch_files(¤t_branch) {
3077 let mut checked = 0;
3078 let mut changed = 0;
3079 const SAMPLE_SIZE: usize = 10;
3080
3081 for (path, _indexed_hash) in branch_files.iter().take(SAMPLE_SIZE) {
3082 checked += 1;
3083 let file_path = std::path::Path::new(path);
3084
3085 if let Ok(metadata) = std::fs::metadata(file_path)
3087 && let Ok(modified) = metadata.modified()
3088 {
3089 let indexed_time = branch_info.last_indexed;
3090 let file_time = modified
3091 .duration_since(std::time::UNIX_EPOCH)
3092 .unwrap_or_default()
3093 .as_secs()
3094 as i64;
3095
3096 if file_time > indexed_time {
3098 changed += 1;
3103 }
3104 }
3105 }
3106
3107 if changed > 0 && !filter.suppress_output {
3108 output::warn(&format!(
3109 "⚠️ WARNING: {} of {} sampled files changed since indexing. Consider running 'rfx index'.",
3110 changed, checked
3111 ));
3112 }
3113 }
3114 }
3115 }
3116 }
3117
3118 Ok(())
3119 }
3120}
3121
3122#[allow(clippy::too_many_arguments)]
3127pub fn generate_ai_instruction(
3128 result_count: usize,
3129 total_count: usize,
3130 has_more: bool,
3131 symbols_mode: bool,
3132 paths_only: bool,
3133 use_ast: bool,
3134 use_regex: bool,
3135 language_filter: bool,
3136 glob_filter: bool,
3137 exact_mode: bool,
3138) -> Option<String> {
3139 if result_count == 0 {
3141 return Some(
3142 "No results found. Consider these alternatives: 1) Check pattern spelling, 2) Remove --kind or --lang filters to broaden search, 3) Try partial match or related term, 4) Use search_regex tool for pattern matching with special characters or complex patterns."
3143 .to_string()
3144 );
3145 }
3146
3147 if total_count >= 500 {
3149 return Some(format!(
3150 "Query too broad: {} results found. STOP. Do not list results. Refine search automatically by adding filters: kind parameter (Function/Struct/Class), lang parameter (rust/python/etc), or glob parameter (['src/**/*.rs']). Call search_code again with appropriate filters.",
3151 total_count
3152 ));
3153 }
3154
3155 if has_more {
3163 return Some(format!(
3164 "Showing {} of {} results — {} more available. This is a partial answer. To finish a find-all task, call again with offset={} (raise limit up to 500 to get the rest in one call), or use mode=\"count\" first if you only need the total.",
3165 result_count,
3166 total_count,
3167 total_count.saturating_sub(result_count),
3168 result_count
3169 ));
3170 }
3171
3172 if result_count == 1 && symbols_mode {
3174 return Some(
3175 "Found 1 precise result. Respond concisely: '[symbol] at [path]:[line]'.".to_string(),
3176 );
3177 }
3178
3179 if (2..=10).contains(&result_count) && symbols_mode {
3181 return Some(format!(
3182 "Found {} precise results (definitions only, not usages). List locations concisely: '[symbol] at [path]:[line]' for each result.",
3183 result_count
3184 ));
3185 }
3186
3187 if (101..500).contains(&total_count) {
3189 return Some(format!(
3190 "Found {} results - this is broad. Suggest refining search with: kind parameter (Function/Struct/Class/etc), lang parameter (rust/python/etc), or glob parameter to narrow file scope.",
3191 total_count
3192 ));
3193 }
3194
3195 if result_count >= 100 && !symbols_mode {
3197 return Some(format!(
3198 "Found {} results in full-text search mode (includes definitions AND all usages). Consider using symbols=true parameter to filter to definitions only. This typically reduces results by 80-90%.",
3199 result_count
3200 ));
3201 }
3202
3203 if paths_only {
3205 return Some(format!(
3206 "Found {} unique files (paths-only mode - no code content included). Next step: Use Read tool on specific files that look relevant based on their paths.",
3207 result_count
3208 ));
3209 }
3210
3211 if use_ast {
3213 return Some(format!(
3214 "Found {} results using AST pattern matching. These are structure-based matches using Tree-sitter patterns, not text search.",
3215 result_count
3216 ));
3217 }
3218
3219 if use_regex && result_count >= 100 {
3221 return Some(format!(
3222 "Found {} results using regex pattern matching. Regex matches are expansive. Consider using exact text search or symbols mode for more precise results.",
3223 result_count
3224 ));
3225 }
3226
3227 if language_filter && result_count <= 5 {
3229 return Some(format!(
3230 "Found {} results with language filter active. Results are limited to this language only. Remove lang parameter if you want to search all languages.",
3231 result_count
3232 ));
3233 }
3234
3235 if glob_filter && result_count <= 10 {
3237 return Some(format!(
3238 "Found {} results with glob filter active. Results are limited to matching paths. Remove glob parameter to search entire codebase.",
3239 result_count
3240 ));
3241 }
3242
3243 if exact_mode && result_count <= 5 {
3245 return Some(format!(
3246 "Found {} results in exact match mode. Only exact symbol name matches are included. Remove exact parameter to allow substring matching.",
3247 result_count
3248 ));
3249 }
3250
3251 None
3253}
3254
3255#[cfg(test)]
3256mod tests {
3257 use super::*;
3258 use crate::indexer::Indexer;
3259 use crate::models::IndexConfig;
3260 use std::fs;
3261 use tempfile::TempDir;
3262
3263 #[test]
3266 fn test_query_engine_creation() {
3267 let temp = TempDir::new().unwrap();
3268 let cache = CacheManager::new(temp.path());
3269 let engine = QueryEngine::new(cache);
3270
3271 assert!(engine.cache.path().ends_with(".reflex"));
3272 }
3273
3274 #[test]
3275 fn test_filter_modes() {
3276 let filter_fulltext = QueryFilter::default();
3278 assert!(!filter_fulltext.symbols_mode);
3279
3280 let filter_symbols = QueryFilter {
3281 symbols_mode: true,
3282 ..Default::default()
3283 };
3284 assert!(filter_symbols.symbols_mode);
3285
3286 let filter_with_kind = QueryFilter {
3288 kind: Some(SymbolKind::Function),
3289 symbols_mode: true,
3290 ..Default::default()
3291 };
3292 assert!(filter_with_kind.symbols_mode);
3293 }
3294
3295 #[test]
3298 fn test_fulltext_search() {
3299 let temp = TempDir::new().unwrap();
3300 let project = temp.path().join("project");
3301 fs::create_dir(&project).unwrap();
3302
3303 fs::write(
3305 project.join("main.rs"),
3306 "fn main() {\n println!(\"hello\");\n}",
3307 )
3308 .unwrap();
3309 fs::write(project.join("lib.rs"), "pub fn hello() {}").unwrap();
3310
3311 let cache = CacheManager::new(&project);
3313 let indexer = Indexer::new(cache, IndexConfig::default());
3314 indexer.index(&project, false).unwrap();
3315
3316 let cache = CacheManager::new(&project);
3318 let engine = QueryEngine::new(cache);
3319 let filter = QueryFilter::default(); let results = engine.search("hello", filter).unwrap();
3321
3322 assert!(results.len() >= 2);
3324 assert!(results.iter().any(|r| r.path.contains("main.rs")));
3325 assert!(results.iter().any(|r| r.path.contains("lib.rs")));
3326 }
3327
3328 #[test]
3329 fn test_symbol_search() {
3330 let temp = TempDir::new().unwrap();
3331 let project = temp.path().join("project");
3332 fs::create_dir(&project).unwrap();
3333
3334 fs::write(
3336 project.join("main.rs"),
3337 "fn greet() {}\nfn main() {\n greet();\n}",
3338 )
3339 .unwrap();
3340
3341 let cache = CacheManager::new(&project);
3343 let indexer = Indexer::new(cache, IndexConfig::default());
3344 indexer.index(&project, false).unwrap();
3345
3346 let cache = CacheManager::new(&project);
3347
3348 let engine = QueryEngine::new(cache);
3350 let filter = QueryFilter {
3351 symbols_mode: true,
3352 ..Default::default()
3353 };
3354 let results = engine.search("greet", filter).unwrap();
3355
3356 assert!(!results.is_empty());
3358 assert!(results.iter().any(|r| r.kind == SymbolKind::Function));
3359 }
3360
3361 #[test]
3362 fn test_regex_search() {
3363 let temp = TempDir::new().unwrap();
3364 let project = temp.path().join("project");
3365 fs::create_dir(&project).unwrap();
3366
3367 fs::write(
3368 project.join("main.rs"),
3369 "fn test1() {}\nfn test2() {}\nfn other() {}",
3370 )
3371 .unwrap();
3372
3373 let cache = CacheManager::new(&project);
3374 let indexer = Indexer::new(cache, IndexConfig::default());
3375 indexer.index(&project, false).unwrap();
3376
3377 let cache = CacheManager::new(&project);
3378
3379 let engine = QueryEngine::new(cache);
3380 let filter = QueryFilter {
3381 use_regex: true,
3382 ..Default::default()
3383 };
3384 let results = engine.search(r"fn test\d", filter).unwrap();
3385
3386 assert_eq!(results.len(), 2);
3388 assert!(results.iter().all(|r| r.preview.contains("test")));
3389 }
3390
3391 #[test]
3394 fn test_language_filter() {
3395 let temp = TempDir::new().unwrap();
3396 let project = temp.path().join("project");
3397 fs::create_dir(&project).unwrap();
3398
3399 fs::write(project.join("main.rs"), "fn main() {}").unwrap();
3400 fs::write(project.join("main.js"), "function main() {}").unwrap();
3401
3402 let cache = CacheManager::new(&project);
3403 let indexer = Indexer::new(cache, IndexConfig::default());
3404 indexer.index(&project, false).unwrap();
3405
3406 let cache = CacheManager::new(&project);
3407
3408 let engine = QueryEngine::new(cache);
3409
3410 let filter = QueryFilter {
3412 language: Some(Language::Rust),
3413 ..Default::default()
3414 };
3415 let results = engine.search("main", filter).unwrap();
3416
3417 assert!(results.iter().all(|r| r.lang == Language::Rust));
3418 assert!(results.iter().all(|r| r.path.ends_with(".rs")));
3419 }
3420
3421 #[test]
3422 fn test_kind_filter() {
3423 let temp = TempDir::new().unwrap();
3424 let project = temp.path().join("project");
3425 fs::create_dir(&project).unwrap();
3426
3427 fs::write(
3428 project.join("main.rs"),
3429 "struct Point {}\nfn main() {}\nimpl Point { fn new() {} }",
3430 )
3431 .unwrap();
3432
3433 let cache = CacheManager::new(&project);
3434 let indexer = Indexer::new(cache, IndexConfig::default());
3435 indexer.index(&project, false).unwrap();
3436
3437 let cache = CacheManager::new(&project);
3438
3439 let engine = QueryEngine::new(cache);
3440
3441 let filter = QueryFilter {
3443 symbols_mode: true,
3444 kind: Some(SymbolKind::Function),
3445 use_contains: true, ..Default::default()
3447 };
3448 let results = engine.search("mai", filter).unwrap();
3450
3451 assert!(!results.is_empty(), "Should find at least one result");
3453 assert!(
3454 results.iter().any(|r| r.symbol.as_deref() == Some("main")),
3455 "Should find 'main' function"
3456 );
3457 }
3458
3459 #[test]
3460 fn test_file_pattern_filter() {
3461 let temp = TempDir::new().unwrap();
3462 let project = temp.path().join("project");
3463 fs::create_dir_all(project.join("src")).unwrap();
3464 fs::create_dir_all(project.join("tests")).unwrap();
3465
3466 fs::write(project.join("src/lib.rs"), "fn foo() {}").unwrap();
3467 fs::write(project.join("tests/test.rs"), "fn foo() {}").unwrap();
3468
3469 let cache = CacheManager::new(&project);
3470 let indexer = Indexer::new(cache, IndexConfig::default());
3471 indexer.index(&project, false).unwrap();
3472
3473 let cache = CacheManager::new(&project);
3474
3475 let engine = QueryEngine::new(cache);
3476
3477 let filter = QueryFilter {
3479 file_pattern: Some("src/".to_string()),
3480 ..Default::default()
3481 };
3482 let results = engine.search("foo", filter).unwrap();
3483
3484 assert!(results.iter().all(|r| r.path.contains("src/")));
3485 assert!(!results.iter().any(|r| r.path.contains("tests/")));
3486 }
3487
3488 #[test]
3489 fn test_limit_filter() {
3490 let temp = TempDir::new().unwrap();
3491 let project = temp.path().join("project");
3492 fs::create_dir(&project).unwrap();
3493
3494 let content = (0..20)
3496 .map(|i| format!("fn test{}() {{}}", i))
3497 .collect::<Vec<_>>()
3498 .join("\n");
3499 fs::write(project.join("main.rs"), content).unwrap();
3500
3501 let cache = CacheManager::new(&project);
3502 let indexer = Indexer::new(cache, IndexConfig::default());
3503 indexer.index(&project, false).unwrap();
3504
3505 let cache = CacheManager::new(&project);
3506
3507 let engine = QueryEngine::new(cache);
3508
3509 let filter = QueryFilter {
3511 limit: Some(5),
3512 use_contains: true, ..Default::default()
3514 };
3515 let results = engine.search("test", filter).unwrap();
3516
3517 assert_eq!(results.len(), 5);
3518 }
3519
3520 #[test]
3521 fn test_exact_match_filter() {
3522 let temp = TempDir::new().unwrap();
3523 let project = temp.path().join("project");
3524 fs::create_dir(&project).unwrap();
3525
3526 fs::write(
3527 project.join("main.rs"),
3528 "fn test() {}\nfn test_helper() {}\nfn other_test() {}",
3529 )
3530 .unwrap();
3531
3532 let cache = CacheManager::new(&project);
3533 let indexer = Indexer::new(cache, IndexConfig::default());
3534 indexer.index(&project, false).unwrap();
3535
3536 let cache = CacheManager::new(&project);
3537
3538 let engine = QueryEngine::new(cache);
3539
3540 let filter = QueryFilter {
3542 symbols_mode: true,
3543 exact: true,
3544 ..Default::default()
3545 };
3546 let results = engine.search("test", filter).unwrap();
3547
3548 assert_eq!(results.len(), 1);
3550 assert_eq!(results[0].symbol.as_deref(), Some("test"));
3551 }
3552
3553 #[test]
3556 fn test_expand_mode() {
3557 let temp = TempDir::new().unwrap();
3558 let project = temp.path().join("project");
3559 fs::create_dir(&project).unwrap();
3560
3561 fs::write(
3562 project.join("main.rs"),
3563 "fn greet() {\n println!(\"Hello\");\n println!(\"World\");\n}",
3564 )
3565 .unwrap();
3566
3567 let cache = CacheManager::new(&project);
3568 let indexer = Indexer::new(cache, IndexConfig::default());
3569 indexer.index(&project, false).unwrap();
3570
3571 let cache = CacheManager::new(&project);
3572
3573 let engine = QueryEngine::new(cache);
3574
3575 let filter = QueryFilter {
3577 symbols_mode: true,
3578 expand: true,
3579 ..Default::default()
3580 };
3581 let results = engine.search("greet", filter).unwrap();
3582
3583 assert!(!results.is_empty());
3585 let result = &results[0];
3586 assert!(result.preview.contains("println"));
3587 }
3588
3589 #[test]
3592 fn test_search_empty_index() {
3593 let temp = TempDir::new().unwrap();
3594 let project = temp.path().join("project");
3595 fs::create_dir(&project).unwrap();
3596
3597 let cache = CacheManager::new(&project);
3598 let indexer = Indexer::new(cache, IndexConfig::default());
3599 indexer.index(&project, false).unwrap();
3600
3601 let cache = CacheManager::new(&project);
3602
3603 let engine = QueryEngine::new(cache);
3604 let filter = QueryFilter::default();
3605 let results = engine.search("nonexistent", filter).unwrap();
3606
3607 assert_eq!(results.len(), 0);
3608 }
3609
3610 #[test]
3611 fn test_search_no_index() {
3612 let temp = TempDir::new().unwrap();
3613 let project = temp.path().join("project");
3614 fs::create_dir(&project).unwrap();
3615
3616 let cache = CacheManager::new(&project);
3617 let engine = QueryEngine::new(cache);
3618 let filter = QueryFilter::default();
3619
3620 assert!(engine.search("test", filter).is_err());
3622 }
3623
3624 #[test]
3625 fn test_search_special_characters() {
3626 let temp = TempDir::new().unwrap();
3627 let project = temp.path().join("project");
3628 fs::create_dir(&project).unwrap();
3629
3630 fs::write(project.join("main.rs"), "let x = 42;\nlet y = x + 1;").unwrap();
3631
3632 let cache = CacheManager::new(&project);
3633 let indexer = Indexer::new(cache, IndexConfig::default());
3634 indexer.index(&project, false).unwrap();
3635
3636 let cache = CacheManager::new(&project);
3637
3638 let engine = QueryEngine::new(cache);
3639 let filter = QueryFilter::default();
3640
3641 let results = engine.search("x + ", filter).unwrap();
3643 assert!(!results.is_empty());
3644 }
3645
3646 #[test]
3647 fn test_search_unicode() {
3648 let temp = TempDir::new().unwrap();
3649 let project = temp.path().join("project");
3650 fs::create_dir(&project).unwrap();
3651
3652 fs::write(project.join("main.rs"), "// 你好世界\nfn main() {}").unwrap();
3653
3654 let cache = CacheManager::new(&project);
3655 let indexer = Indexer::new(cache, IndexConfig::default());
3656 indexer.index(&project, false).unwrap();
3657
3658 let cache = CacheManager::new(&project);
3659
3660 let engine = QueryEngine::new(cache);
3661 let filter = QueryFilter {
3662 use_contains: true, force: true, ..Default::default()
3665 };
3666
3667 let results = engine.search("你好", filter).unwrap();
3669 assert!(!results.is_empty());
3670 }
3671
3672 #[test]
3673 fn test_case_sensitive_search() {
3674 let temp = TempDir::new().unwrap();
3675 let project = temp.path().join("project");
3676 fs::create_dir(&project).unwrap();
3677
3678 fs::write(project.join("main.rs"), "fn Test() {}\nfn test() {}").unwrap();
3679
3680 let cache = CacheManager::new(&project);
3681 let indexer = Indexer::new(cache, IndexConfig::default());
3682 indexer.index(&project, false).unwrap();
3683
3684 let cache = CacheManager::new(&project);
3685
3686 let engine = QueryEngine::new(cache);
3687 let filter = QueryFilter::default();
3688
3689 let results = engine.search("Test", filter).unwrap();
3691 assert!(results.iter().any(|r| r.preview.contains("Test()")));
3692 }
3693
3694 #[test]
3697 fn test_results_sorted_deterministically() {
3698 let temp = TempDir::new().unwrap();
3699 let project = temp.path().join("project");
3700 fs::create_dir(&project).unwrap();
3701
3702 fs::write(project.join("a.rs"), "fn test() {}").unwrap();
3703 fs::write(project.join("z.rs"), "fn test() {}").unwrap();
3704 fs::write(project.join("m.rs"), "fn test() {}\nfn test2() {}").unwrap();
3705
3706 let cache = CacheManager::new(&project);
3707 let indexer = Indexer::new(cache, IndexConfig::default());
3708 indexer.index(&project, false).unwrap();
3709
3710 let cache = CacheManager::new(&project);
3711
3712 let engine = QueryEngine::new(cache);
3713 let filter = QueryFilter::default();
3714
3715 let results1 = engine.search("test", filter.clone()).unwrap();
3717 let results2 = engine.search("test", filter.clone()).unwrap();
3718 let results3 = engine.search("test", filter).unwrap();
3719
3720 assert_eq!(results1.len(), results2.len());
3722 assert_eq!(results1.len(), results3.len());
3723
3724 for i in 0..results1.len() {
3725 assert_eq!(results1[i].path, results2[i].path);
3726 assert_eq!(results1[i].path, results3[i].path);
3727 assert_eq!(results1[i].span.start_line, results2[i].span.start_line);
3728 assert_eq!(results1[i].span.start_line, results3[i].span.start_line);
3729 }
3730
3731 for i in 0..results1.len().saturating_sub(1) {
3733 let curr = &results1[i];
3734 let next = &results1[i + 1];
3735 assert!(
3736 curr.path < next.path
3737 || (curr.path == next.path && curr.span.start_line <= next.span.start_line)
3738 );
3739 }
3740 }
3741
3742 #[test]
3745 fn test_multiple_filters_combined() {
3746 let temp = TempDir::new().unwrap();
3747 let project = temp.path().join("project");
3748 fs::create_dir_all(project.join("src")).unwrap();
3749
3750 fs::write(project.join("src/main.rs"), "fn test() {}\nstruct Test {}").unwrap();
3751 fs::write(project.join("src/lib.rs"), "fn test() {}").unwrap();
3752 fs::write(project.join("test.js"), "function test() {}").unwrap();
3753
3754 let cache = CacheManager::new(&project);
3755 let indexer = Indexer::new(cache, IndexConfig::default());
3756 indexer.index(&project, false).unwrap();
3757
3758 let cache = CacheManager::new(&project);
3759
3760 let engine = QueryEngine::new(cache);
3761
3762 let filter = QueryFilter {
3764 language: Some(Language::Rust),
3765 kind: Some(SymbolKind::Function),
3766 file_pattern: Some("src/main".to_string()),
3767 symbols_mode: true,
3768 ..Default::default()
3769 };
3770 let results = engine.search("test", filter).unwrap();
3771
3772 assert_eq!(results.len(), 1);
3774 assert!(results[0].path.contains("src/main.rs"));
3775 assert_eq!(results[0].kind, SymbolKind::Function);
3776 }
3777
3778 #[test]
3781 fn test_find_symbol_helper() {
3782 let temp = TempDir::new().unwrap();
3783 let project = temp.path().join("project");
3784 fs::create_dir(&project).unwrap();
3785
3786 fs::write(project.join("main.rs"), "fn greet() {}").unwrap();
3787
3788 let cache = CacheManager::new(&project);
3789 let indexer = Indexer::new(cache, IndexConfig::default());
3790 indexer.index(&project, false).unwrap();
3791
3792 let cache = CacheManager::new(&project);
3793
3794 let engine = QueryEngine::new(cache);
3795 let results = engine.find_symbol("greet").unwrap();
3796
3797 assert!(!results.is_empty());
3798 assert_eq!(results[0].kind, SymbolKind::Function);
3799 }
3800
3801 #[test]
3802 fn test_list_by_kind_helper() {
3803 let temp = TempDir::new().unwrap();
3804 let project = temp.path().join("project");
3805 fs::create_dir(&project).unwrap();
3806
3807 fs::write(
3808 project.join("main.rs"),
3809 "struct Point {}\nfn test() {}\nstruct Line {}",
3810 )
3811 .unwrap();
3812
3813 let cache = CacheManager::new(&project);
3814 let indexer = Indexer::new(cache, IndexConfig::default());
3815 indexer.index(&project, false).unwrap();
3816
3817 let cache = CacheManager::new(&project);
3818
3819 let engine = QueryEngine::new(cache);
3820
3821 let filter = QueryFilter {
3823 kind: Some(SymbolKind::Struct),
3824 symbols_mode: true,
3825 use_contains: true, ..Default::default()
3827 };
3828 let results = engine.search("oin", filter).unwrap();
3829
3830 assert!(!results.is_empty(), "Should find at least Point struct");
3832 assert!(results.iter().all(|r| r.kind == SymbolKind::Struct));
3833 assert!(results.iter().any(|r| r.symbol.as_deref() == Some("Point")));
3834 }
3835
3836 #[test]
3839 fn test_search_with_metadata() {
3840 let temp = TempDir::new().unwrap();
3841 let project = temp.path().join("project");
3842 fs::create_dir(&project).unwrap();
3843
3844 fs::write(project.join("main.rs"), "fn test() {}").unwrap();
3845
3846 let cache = CacheManager::new(&project);
3847 let indexer = Indexer::new(cache, IndexConfig::default());
3848 indexer.index(&project, false).unwrap();
3849
3850 let cache = CacheManager::new(&project);
3851
3852 let engine = QueryEngine::new(cache);
3853 let filter = QueryFilter::default();
3854 let response = engine.search_with_metadata("test", filter).unwrap();
3855
3856 assert!(!response.results.is_empty());
3858 }
3860
3861 #[test]
3864 fn test_search_across_languages() {
3865 let temp = TempDir::new().unwrap();
3866 let project = temp.path().join("project");
3867 fs::create_dir(&project).unwrap();
3868
3869 fs::write(project.join("main.rs"), "fn greet() {}").unwrap();
3870 fs::write(project.join("main.ts"), "function greet() {}").unwrap();
3871 fs::write(project.join("main.py"), "def greet(): pass").unwrap();
3872
3873 let cache = CacheManager::new(&project);
3874 let indexer = Indexer::new(cache, IndexConfig::default());
3875 indexer.index(&project, false).unwrap();
3876
3877 let cache = CacheManager::new(&project);
3878
3879 let engine = QueryEngine::new(cache);
3880 let filter = QueryFilter::default();
3881 let results = engine.search("greet", filter).unwrap();
3882
3883 assert!(results.len() >= 3);
3885 assert!(results.iter().any(|r| r.lang == Language::Rust));
3886 assert!(results.iter().any(|r| r.lang == Language::TypeScript));
3887 assert!(results.iter().any(|r| r.lang == Language::Python));
3888 }
3889}
3890
3891mod status_cache {
3907 use crate::cache::{CacheManager, FileFingerprint};
3908 use crate::git::WorktreeChanges;
3909 use crate::indexer::{Indexer, PathPolicy, looks_binary};
3910 use crate::models::{IndexConfig, IndexStatus, IndexWarning, IndexWarningDetails, Language};
3911 use anyhow::Result;
3912 use std::collections::{HashMap, HashSet};
3913 use std::path::{Path, PathBuf};
3914 use std::sync::{Arc, Mutex, OnceLock};
3915 use std::time::{Duration, Instant};
3916
3917 const DEFAULT_TTL_MS: u64 = 1_000;
3918
3919 pub type Verdict = (IndexStatus, bool, Option<IndexWarning>);
3920
3921 pub struct Snapshot {
3924 pub decided: Option<Verdict>,
3927 pub details: Option<IndexWarningDetails>,
3929 pub changes: WorktreeChanges,
3931 }
3932
3933 impl Snapshot {
3934 fn fresh() -> Self {
3935 Self {
3936 decided: Some((IndexStatus::Fresh, true, None)),
3937 details: None,
3938 changes: WorktreeChanges::default(),
3939 }
3940 }
3941 }
3942
3943 struct Entry {
3944 computed_at: Instant,
3945 snapshot: Arc<Snapshot>,
3946 }
3947
3948 fn ttl() -> Duration {
3949 static TTL: OnceLock<Duration> = OnceLock::new();
3950 *TTL.get_or_init(|| {
3951 let ms = std::env::var("REFLEX_FRESHNESS_TTL_MS")
3952 .ok()
3953 .and_then(|v| v.trim().parse::<u64>().ok())
3954 .unwrap_or(DEFAULT_TTL_MS);
3955 Duration::from_millis(ms)
3956 })
3957 }
3958
3959 fn store() -> &'static Mutex<HashMap<PathBuf, Entry>> {
3960 static STORE: OnceLock<Mutex<HashMap<PathBuf, Entry>>> = OnceLock::new();
3961 STORE.get_or_init(|| Mutex::new(HashMap::new()))
3962 }
3963
3964 fn key(root: &Path) -> PathBuf {
3965 root.canonicalize().unwrap_or_else(|_| root.to_path_buf())
3966 }
3967
3968 fn indexable_with(path: &str, policy: &crate::indexer::PathPolicy) -> bool {
3971 crate::indexer::Indexer::is_indexable_path_with(Path::new(path), Some(policy))
3972 }
3973
3974 pub fn snapshot(cache: &CacheManager) -> Result<Arc<Snapshot>> {
3976 let root = cache.workspace_root();
3977 let key = key(&root);
3978 let ttl = ttl();
3979
3980 if !ttl.is_zero()
3981 && let Ok(map) = store().lock()
3982 && let Some(entry) = map.get(&key)
3983 && entry.computed_at.elapsed() < ttl
3984 {
3985 return Ok(Arc::clone(&entry.snapshot));
3986 }
3987
3988 let snapshot = Arc::new(compute(cache, &root)?);
3989
3990 if !ttl.is_zero()
3991 && let Ok(mut map) = store().lock()
3992 {
3993 map.insert(
3994 key,
3995 Entry {
3996 computed_at: Instant::now(),
3997 snapshot: Arc::clone(&snapshot),
3998 },
3999 );
4000 }
4001
4002 Ok(snapshot)
4003 }
4004
4005 enum Probe {
4007 Git(HashSet<String>),
4010 Walk,
4012 }
4013
4014 fn compute(cache: &CacheManager, root: &Path) -> Result<Snapshot> {
4024 let is_git = crate::git::is_git_repo(root) && crate::git::is_git_available();
4025 let current_branch = if is_git {
4026 crate::git::read_head_branch(root).or_else(|| crate::git::get_current_branch(root).ok())
4027 } else {
4028 None
4029 };
4030
4031 let reads = match cache.status_reads(current_branch.as_deref()) {
4033 Ok(r) => r,
4034 Err(e) => {
4035 log::debug!("Could not read index status from meta.db: {}", e);
4036 return Ok(Snapshot::fresh());
4037 }
4038 };
4039
4040 if !reads.schema_ok {
4045 let reason = match reads.owner {
4046 Some((v, sha)) if v != env!("CARGO_PKG_VERSION") => {
4047 let sha = sha
4048 .map(|s| format!(" (sha {})", &s[..s.len().min(7)]))
4049 .unwrap_or_default();
4050 format!(
4051 "This .reflex/ was written by reflex {}{}; this binary is reflex {}. \
4052 Results come from a cache format this version does not fully \
4053 understand.",
4054 v,
4055 sha,
4056 env!("CARGO_PKG_VERSION")
4057 )
4058 }
4059 _ => "The index was built with a different cache format and needs \
4062 rebuilding. Results may be incomplete until then."
4063 .to_string(),
4064 };
4065
4066 let warning = IndexWarning::new(reason, "index_project");
4067 return Ok(Snapshot {
4068 decided: Some((IndexStatus::Stale, false, Some(warning))),
4069 details: None,
4070 changes: WorktreeChanges::default(),
4071 });
4072 }
4073
4074 let config = cache.load_index_config().unwrap_or_else(|e| {
4075 log::debug!("Using default index config for freshness: {}", e);
4076 IndexConfig::default()
4077 });
4078 let policy = PathPolicy::from_config(root, &config);
4079
4080 let info = reads.branch_info.as_ref();
4081 let mut details = IndexWarningDetails {
4082 current_branch: current_branch.clone(),
4083 indexed_branch: info.map(|i| i.branch.clone()),
4084 current_commit: None,
4085 indexed_commit: info.map(|i| i.commit_sha.clone()),
4086 indexed_at: info.map(|i| i.last_indexed),
4087 checked_by: None,
4088 };
4089
4090 let probe = if current_branch.is_some() {
4091 let current_commit = crate::git::get_current_commit(root).ok();
4092 details.current_commit = current_commit.clone();
4093 let indexed_commit = info.map(|i| i.commit_sha.as_str());
4094 match git_candidates(
4095 root,
4096 &reads.dirty_at_index,
4097 indexed_commit,
4098 current_commit.as_deref(),
4099 ) {
4100 Ok(set) => Probe::Git(set),
4101 Err(e) => {
4102 log::debug!("git could not name changed paths ({}); walking the tree", e);
4103 Probe::Walk
4104 }
4105 }
4106 } else {
4107 Probe::Walk
4108 };
4109
4110 let mut changes = match probe {
4111 Probe::Git(candidates) => {
4112 let keys: Vec<&str> = candidates.iter().map(String::as_str).collect();
4113 let fingerprints = cache.fingerprints_for(&keys)?;
4114 let mut out = WorktreeChanges::default();
4115 for path in &candidates {
4116 classify_one(
4117 root,
4118 &config,
4119 &policy,
4120 path,
4121 fingerprints.get(path),
4122 &mut out,
4123 );
4124 }
4125 details.checked_by = Some("git".to_string());
4126 out
4127 }
4128 Probe::Walk => {
4129 let fingerprints = fingerprints_all(cache)?;
4130 details.checked_by = Some("walk".to_string());
4131 walk_changes(root, &config, &policy, &fingerprints)
4132 }
4133 };
4134 changes.sort();
4135
4136 Ok(Snapshot {
4137 decided: None,
4138 details: Some(details),
4139 changes,
4140 })
4141 }
4142
4143 fn git_candidates(
4155 root: &Path,
4156 dirty_at_index: &[String],
4157 indexed_commit: Option<&str>,
4158 current_commit: Option<&str>,
4159 ) -> Result<HashSet<String>> {
4160 let mut set = crate::git::changed_paths(root)?;
4161 set.extend(dirty_at_index.iter().cloned());
4162 match (indexed_commit, current_commit) {
4163 (Some(indexed), Some(current)) if indexed != current => {
4164 if indexed == "unknown" {
4165 anyhow::bail!("the index does not record a commit");
4166 }
4167 set.extend(crate::git::diff_names(root, indexed, current)?);
4168 }
4169 (None, _) | (_, None) => anyhow::bail!("no commit to diff against"),
4170 _ => {}
4171 }
4172 Ok(set)
4173 }
4174
4175 fn fingerprints_all(cache: &CacheManager) -> Result<Arc<HashMap<String, FileFingerprint>>> {
4178 match super::open_index::get_or_open(cache) {
4179 Ok(open) => open.fingerprints(cache),
4180 Err(_) => Ok(Arc::new(cache.load_fingerprints()?)),
4181 }
4182 }
4183
4184 fn content_matches(full: &Path, md: &std::fs::Metadata, fp: &FileFingerprint) -> bool {
4190 if fp.stat_matches(md) {
4191 return true;
4192 }
4193 match std::fs::read(full) {
4194 Ok(bytes) => blake3::hash(&bytes).to_hex().as_str() == fp.hash,
4195 Err(_) => false,
4196 }
4197 }
4198
4199 fn classify_one(
4201 root: &Path,
4202 config: &IndexConfig,
4203 policy: &PathPolicy,
4204 path: &str,
4205 fp: Option<&FileFingerprint>,
4206 out: &mut WorktreeChanges,
4207 ) {
4208 if !indexable_with(path, policy) {
4209 return;
4210 }
4211 let full = root.join(path);
4212 let md = if config.follow_symlinks {
4213 std::fs::metadata(&full)
4214 } else {
4215 std::fs::symlink_metadata(&full)
4216 };
4217 match (md, fp) {
4218 (Err(_), Some(_)) => out.push_deleted(path),
4219 (Err(_), None) => {}
4220 (Ok(md), _) if !md.is_file() => {
4222 if fp.is_some() {
4223 out.push_deleted(path);
4224 }
4225 }
4226 (Ok(md), _) if md.len() > config.max_file_size as u64 => {
4227 if fp.is_some() {
4230 out.push_modified(path);
4231 }
4232 }
4233 (Ok(_), None) => {
4236 if !Language::from_path(Path::new(path)).is_code() && looks_binary(&full) {
4237 return;
4238 }
4239 out.push_added(path);
4240 }
4241 (Ok(md), Some(fp)) => {
4242 if !content_matches(&full, &md, fp) {
4243 out.push_modified(path);
4244 }
4245 }
4246 }
4247 }
4248
4249 fn walk_changes(
4255 root: &Path,
4256 config: &IndexConfig,
4257 policy: &PathPolicy,
4258 fingerprints: &HashMap<String, FileFingerprint>,
4259 ) -> WorktreeChanges {
4260 let threads = std::thread::available_parallelism()
4261 .map(|n| n.get())
4262 .unwrap_or(1)
4263 .min(8);
4264 let mut builder = Indexer::walk_builder(root, config, policy);
4265 builder.threads(threads);
4266
4267 let seen: Mutex<Vec<(String, std::fs::Metadata)>> = Mutex::new(Vec::new());
4268 let max_size = config.max_file_size as u64;
4269 builder.build_parallel().run(|| {
4270 let seen = &seen;
4271 Box::new(move |entry| {
4272 let Ok(entry) = entry else {
4273 return ignore::WalkState::Continue;
4274 };
4275 if !entry.file_type().map(|t| t.is_file()).unwrap_or(false) {
4276 return ignore::WalkState::Continue;
4277 }
4278 let path = entry.path();
4279 if policy.classify(path).is_none() {
4280 return ignore::WalkState::Continue;
4281 }
4282 let Ok(md) = entry.metadata() else {
4283 return ignore::WalkState::Continue;
4284 };
4285 if md.len() > max_size {
4286 return ignore::WalkState::Continue;
4287 }
4288 let rel = path
4289 .strip_prefix(root)
4290 .unwrap_or(path)
4291 .to_string_lossy()
4292 .replace('\\', "/");
4293 if let Ok(mut v) = seen.lock() {
4294 v.push((rel, md));
4295 }
4296 ignore::WalkState::Continue
4297 })
4298 });
4299
4300 let seen = seen.into_inner().unwrap_or_default();
4301 let mut out = WorktreeChanges::default();
4302 let mut present: HashSet<&str> = HashSet::with_capacity(seen.len());
4303 for (rel, md) in &seen {
4304 present.insert(rel.as_str());
4305 match fingerprints.get(rel) {
4306 None => {
4307 let full = root.join(rel);
4308 if !Language::from_path(&full).is_code() && looks_binary(&full) {
4309 continue;
4310 }
4311 out.push_added(rel);
4312 }
4313 Some(fp) => {
4314 if !content_matches(&root.join(rel), md, fp) {
4315 out.push_modified(rel);
4316 }
4317 }
4318 }
4319 }
4320 for path in fingerprints.keys() {
4321 if !present.contains(path.as_str()) {
4322 out.push_deleted(path);
4323 }
4324 }
4325 out
4326 }
4327
4328 pub fn invalidate(root: &Path) {
4334 let key = key(root);
4335 if let Ok(mut map) = store().lock() {
4336 map.remove(&key);
4337 }
4338 }
4339}