Skip to main content

reflex/query/
mod.rs

1//! Query engine for searching indexed code
2//!
3//! The query engine loads the memory-mapped cache and executes
4//! deterministic searches based on lexical, structural, or symbol patterns.
5
6pub mod filter;
7pub mod open_index;
8pub mod result;
9pub mod zero_hint;
10
11pub use filter::{LiteralPattern, QueryFilter, prepare_literal_pattern, substring_hint_text};
12pub use zero_hint::{ExcludedReason, ZeroHint};
13
14use anyhow::{Context, Result};
15use regex::Regex;
16
17use crate::cache::CacheManager;
18use crate::models::{
19    IndexPath, IndexStatus, IndexStatusReport, IndexWarning, Language, QueryResponse, SearchResult,
20    Span, SymbolKind,
21};
22use crate::output;
23use crate::parsers::ParserFactory;
24use std::sync::atomic::{AtomicUsize, Ordering};
25use std::sync::{Arc, OnceLock};
26
27use open_index::OpenIndex;
28
29/// Drop every process-wide memo for `workspace_root`: the shared open-index
30/// handle and the freshness snapshot. Called around index writes so no query in
31/// this process keeps serving from replaced files or a stale verdict.
32pub fn invalidate_caches(workspace_root: &std::path::Path) {
33    open_index::invalidate(&workspace_root.join(crate::cache::CACHE_DIR));
34    status_cache::invalidate(workspace_root);
35}
36
37/// What `search_internal` hands back: the page plus the bookkeeping the public
38/// wrappers turn into pagination, hints and timings.
39struct Internal {
40    results: Vec<SearchResult>,
41    /// Matches before offset/limit. Exact when `total_is_exact`; otherwise the
42    /// number verified before the page filled, which the public wrappers must
43    /// NOT report as a total.
44    total_count: usize,
45    /// False when verification stopped early (list mode with a limit).
46    total_is_exact: bool,
47    /// Sample-based estimate of the total when it is not exact.
48    approx_total: Option<usize>,
49    /// Candidate lines that contained the pattern only as a substring
50    /// (whole-identifier searches only), for the zero-result hint.
51    substring_only: Option<usize>,
52    /// Candidate files left out because they are lock/generated files nobody
53    /// asked for, for the zero-result hint.
54    excluded_by_default: usize,
55    /// Count-only mode: `(matching lines, files with a match)`; `results` is empty
56    /// and `total_count` is the line count.
57    counted: Option<(usize, usize)>,
58    /// Time spent in trigram lookup and intersection.
59    candidates_us: u64,
60    /// Trigram index or full scan.
61    index_path: IndexPath,
62    /// Engine warnings from the candidate pass.
63    warnings: Vec<String>,
64}
65
66/// Bookkeeping from a candidate pass (trigram or regex).
67#[derive(Debug, Clone)]
68struct CandidateStats {
69    candidates_us: u64,
70    substring_only: Option<usize>,
71    /// See `Internal::excluded_by_default`.
72    excluded_by_default: usize,
73    /// Count-only mode: `(matching lines, files with a match)`.
74    counted: Option<(usize, usize)>,
75    /// Every candidate was verified.
76    exhausted: bool,
77    /// Sample-based estimate of the total, only when not `exhausted`.
78    approx_total: Option<usize>,
79    /// Trigram index or full scan.
80    index_path: IndexPath,
81    /// What the engine wants the caller to know about the candidate pass (a
82    /// regex that had to scan), for `QueryResponse.warnings`.
83    warnings: Vec<String>,
84}
85
86impl Default for CandidateStats {
87    fn default() -> Self {
88        Self {
89            candidates_us: 0,
90            substring_only: None,
91            excluded_by_default: 0,
92            counted: None,
93            exhausted: true,
94            approx_total: None,
95            index_path: IndexPath::Trigram,
96            warnings: Vec::new(),
97        }
98    }
99}
100
101/// Which lines of a candidate file to verify.
102enum LineSet {
103    /// Only these 1-based lines (from the trigram intersection), ascending.
104    Only(Vec<u32>),
105    /// Every line: the pattern gave the index nothing to narrow on.
106    All,
107}
108
109/// Per-file filters, applied INSIDE the verification loop so that early
110/// termination counts only files the caller will actually receive.
111struct FileFilter {
112    language: Option<Language>,
113    include: Option<globset::GlobSet>,
114    exclude: Option<globset::GlobSet>,
115    file_pattern: Option<String>,
116    exclude_text: bool,
117    include_locks: bool,
118    include_generated: bool,
119}
120
121impl FileFilter {
122    fn from_filter(filter: &QueryFilter) -> Self {
123        Self {
124            language: filter.language,
125            include: result::build_glob_set(&filter.glob_patterns, "glob"),
126            exclude: result::build_glob_set(&filter.exclude_patterns, "exclude"),
127            file_pattern: filter.file_pattern.clone(),
128            exclude_text: filter.exclude_text,
129            include_locks: filter.include_locks,
130            include_generated: filter.include_generated,
131        }
132    }
133
134    /// The file is a lock/generated file nobody asked for. Counted, not searched.
135    fn excluded_by_default(&self, lang: Language) -> bool {
136        filter::excluded_by_default(
137            lang,
138            self.language,
139            self.include_locks,
140            self.include_generated,
141        )
142    }
143
144    /// The path filters alone: `glob`, `exclude`, `file`.
145    fn path_admits(&self, path: &str) -> bool {
146        if let Some(set) = &self.include
147            && !set.is_match(path)
148        {
149            return false;
150        }
151        if let Some(set) = &self.exclude
152            && set.is_match(path)
153        {
154            return false;
155        }
156        if let Some(needle) = &self.file_pattern
157            && !path.contains(needle.as_str())
158        {
159            return false;
160        }
161        true
162    }
163
164    fn accept(&self, path: &str, lang: Language) -> bool {
165        filter::tier_admits(
166            lang,
167            self.language,
168            self.exclude_text,
169            self.include_locks,
170            self.include_generated,
171        ) && self.path_admits(path)
172    }
173}
174
175/// What a streaming verification pass produced.
176struct VerifyOutcome {
177    /// In `(path, line)` order.
178    results: Vec<SearchResult>,
179    /// Every candidate file was verified.
180    exhausted: bool,
181    /// Estimate of the full match count, only when not `exhausted`: the page's
182    /// verified matches plus the sampled hit rate over the remaining candidates.
183    /// `None` when a remaining file had `LineSet::All` (no line count to scale).
184    estimated_total: Option<usize>,
185    /// Whole-identifier searches only: candidate lines holding the pattern as a
186    /// substring but not as a whole identifier.
187    substring_only: usize,
188    /// Candidate files left out because they are lock/generated files and nothing
189    /// asked for them.
190    excluded_by_default: usize,
191    /// Count-only mode: `(matching lines, files with a match)`; `results` is empty.
192    counted: Option<(usize, usize)>,
193}
194
195/// The per-line predicate for one query, built once.
196///
197/// The whole-identifier check used to compile `\b…\b` for every candidate line;
198/// on a common word that compile cost more than the match itself.
199enum LineMatcher {
200    /// Whole-identifier match (`\b…\b`), the default. Keeps the literal so the
201    /// zero-result hint can count substring-only lines.
202    WordBoundary(Regex, String),
203    /// Substring match (`contains: true`), also the fallback when the
204    /// word-boundary regex cannot be built.
205    Contains(String),
206    /// User-supplied regex, with the `kind` label its matches carry: a regex the
207    /// engine built from an `ignore_case` literal still reports `text_match`.
208    Regex(Regex, &'static str),
209}
210
211impl LineMatcher {
212    fn new(pattern: &str, filter: &QueryFilter) -> Result<Self> {
213        if filter.use_regex {
214            let label = if filter.ignore_case && filter.rewritten_from.is_some() {
215                "text_match"
216            } else {
217                "regex_match"
218            };
219            return match Regex::new(pattern) {
220                Ok(re) => Ok(Self::Regex(re, label)),
221                Err(e) => {
222                    log::error!("Invalid regex pattern '{}': {}", pattern, e);
223                    anyhow::bail!("Invalid regex pattern '{}': {}", pattern, e);
224                }
225            };
226        }
227        if filter.use_contains {
228            return Ok(Self::Contains(pattern.to_string()));
229        }
230        match Regex::new(&format!(r"\b{}\b", regex::escape(pattern))) {
231            Ok(re) => Ok(Self::WordBoundary(re, pattern.to_string())),
232            Err(_) => {
233                log::debug!(
234                    "Word boundary regex failed for pattern '{}', falling back to substring",
235                    pattern
236                );
237                Ok(Self::Contains(pattern.to_string()))
238            }
239        }
240    }
241
242    fn is_word_boundary(&self) -> bool {
243        matches!(self, Self::WordBoundary(..))
244    }
245
246    #[inline]
247    fn is_match(&self, line: &str) -> bool {
248        match self {
249            Self::WordBoundary(re, _) | Self::Regex(re, _) => re.is_match(line),
250            Self::Contains(p) => line.contains(p.as_str()),
251        }
252    }
253
254    /// Byte offset of the first match, to window the preview on it.
255    #[inline]
256    fn find(&self, line: &str) -> Option<usize> {
257        match self {
258            Self::WordBoundary(re, _) | Self::Regex(re, _) => re.find(line).map(|m| m.start()),
259            Self::Contains(p) => line.find(p.as_str()),
260        }
261    }
262
263    /// The literal text of a whole-identifier or substring matcher (regex: none).
264    fn literal(&self) -> Option<&str> {
265        match self {
266            Self::WordBoundary(_, p) | Self::Contains(p) => Some(p.as_str()),
267            Self::Regex(..) => None,
268        }
269    }
270
271    /// The `kind` label a text match carries in results.
272    fn kind_label(&self) -> &'static str {
273        match self {
274            Self::Regex(_, label) => label,
275            _ => "text_match",
276        }
277    }
278}
279
280/// Verify candidate files in path order, in parallel, stopping once `budget`
281/// results exist.
282///
283/// Files are processed in rounds of growing chunks (16, 32, … 1024). Inside a
284/// round rayon's indexed `map().collect()` keeps file order, and lines within a
285/// file are ascending, so the concatenation is already in `(path, line)` order:
286/// the first page is complete and correctly ordered the moment the budget is met,
287/// without verifying the rest. With `budget: None` every file is verified.
288/// Files sampled to estimate the total after a page fills early. Spread evenly over
289/// the remaining candidates (path order), so one directory of dense hits does not
290/// set the rate for the whole repo.
291pub const ESTIMATE_SAMPLE_FILES: usize = 32;
292/// Remaining candidate lines at or below which the search just finishes instead of
293/// sampling: verifying 128 lines costs less than reasoning about them.
294pub const ESTIMATE_FINISH_LINES: usize = 128;
295/// Candidate lines verified per sampled file, so a single minified bundle with a
296/// thousand candidate lines cannot dominate the sample.
297pub const ESTIMATE_PER_FILE_LINES: usize = 16;
298
299fn verify_files_streaming(
300    open: &OpenIndex,
301    files: Vec<(u32, LineSet)>,
302    matcher: &LineMatcher,
303    file_filter: &FileFilter,
304    paths_only: bool,
305    budget: Option<usize>,
306    count_only: bool,
307) -> VerifyOutcome {
308    use rayon::prelude::*;
309
310    let content = &open.content;
311
312    // Resolve path + language once per file, drop files the filters reject, and
313    // sort by the path string the results will carry.
314    let mut excluded_by_default = 0usize;
315    let mut accepted: Vec<(u32, String, Language, LineSet)> = files
316        .into_iter()
317        .filter_map(|(file_id, lines)| {
318            let path = content.get_file_path(file_id)?;
319            let lang = Language::from_path(path);
320            let path_str = path.to_string_lossy().into_owned();
321            if file_filter.accept(&path_str, lang) {
322                Some((file_id, path_str, lang, lines))
323            } else {
324                // Counted only when the tier gate is the SOLE reason: a lock file
325                // the `file` filter already rejected must not make the zero-result
326                // hint blame lock files (the 2.0.0 field test's wrong hint).
327                if file_filter.path_admits(&path_str) && file_filter.excluded_by_default(lang) {
328                    excluded_by_default += 1;
329                }
330                None
331            }
332        })
333        .collect();
334    accepted.sort_by(|a, b| a.1.cmp(&b.1));
335
336    let count_substring_only = matcher.is_word_boundary();
337    let substring_only = AtomicUsize::new(0);
338    let kind_label = matcher.kind_label();
339
340    let verify_one = |file_id: u32, path: &str, lang: Language, lines: &LineSet| {
341        let Ok(text) = content.get_file_content(file_id) else {
342            return Vec::new();
343        };
344        let all_lines: Vec<&str> = text.lines().collect();
345        let mut out = Vec::new();
346
347        let mut check = |line_no: usize, line: &str| -> bool {
348            let Some(offset) = matcher.find(line) else {
349                if count_substring_only
350                    && let Some(p) = matcher.literal()
351                    && line.contains(p)
352                {
353                    substring_only.fetch_add(1, Ordering::Relaxed);
354                }
355                return false;
356            };
357            out.push(SearchResult {
358                path: path.to_string(),
359                lang,
360                kind: SymbolKind::Unknown(kind_label.to_string()),
361                symbol: None,
362                span: Span {
363                    start_line: line_no,
364                    end_line: line_no,
365                },
366                // Bounded, and WINDOWED on the match: on a minified bundle this
367                // line is the whole 1.45 MB file, and the first 512 bytes of it
368                // would tell the caller nothing.
369                preview: crate::parsers::preview::line_preview(line, offset),
370                dependencies: None,
371            });
372            true
373        };
374
375        match lines {
376            LineSet::Only(nos) => {
377                let mut last = 0u32;
378                for &line_no in nos {
379                    if line_no == last {
380                        continue;
381                    }
382                    last = line_no;
383                    let idx = line_no as usize;
384                    if idx == 0 || idx > all_lines.len() {
385                        log::debug!(
386                            "Line {} out of bounds (file has {} lines)",
387                            line_no,
388                            all_lines.len()
389                        );
390                        continue;
391                    }
392                    if check(idx, all_lines[idx - 1]) && paths_only {
393                        break;
394                    }
395                }
396            }
397            LineSet::All => {
398                for (i, line) in all_lines.iter().enumerate() {
399                    if check(i + 1, line) && paths_only {
400                        break;
401                    }
402                }
403            }
404        }
405        out
406    };
407
408    // Count-only: no `SearchResult`, no preview, no path clone, no grouping —
409    // one parallel pass that returns two numbers. `(?i)kubernetes --count` on the
410    // Kubernetes checkout spent most of its ~220 ms materialising and grouping
411    // 114k rows it then threw away.
412    if count_only && budget.is_none() {
413        let count_one = |file_id: u32, lines: &LineSet| -> usize {
414            let Ok(text) = content.get_file_content(file_id) else {
415                return 0;
416            };
417            let mut hits = 0usize;
418            let mut check = |line: &str| {
419                if matcher.find(line).is_some() {
420                    hits += 1;
421                    true
422                } else {
423                    if count_substring_only
424                        && let Some(p) = matcher.literal()
425                        && line.contains(p)
426                    {
427                        substring_only.fetch_add(1, Ordering::Relaxed);
428                    }
429                    false
430                }
431            };
432            match lines {
433                LineSet::Only(nos) => {
434                    let all_lines: Vec<&str> = text.lines().collect();
435                    let mut last = 0u32;
436                    for &line_no in nos {
437                        if line_no == last {
438                            continue;
439                        }
440                        last = line_no;
441                        let idx = line_no as usize;
442                        if idx == 0 || idx > all_lines.len() {
443                            continue;
444                        }
445                        if check(all_lines[idx - 1]) && paths_only {
446                            break;
447                        }
448                    }
449                }
450                LineSet::All => {
451                    for line in text.lines() {
452                        if check(line) && paths_only {
453                            break;
454                        }
455                    }
456                }
457            }
458            hits
459        };
460        let (lines, files_hit) = open.pool().install(|| {
461            accepted
462                .par_iter()
463                .map(|(file_id, _, _, lines)| {
464                    let n = count_one(*file_id, lines);
465                    (n, usize::from(n > 0))
466                })
467                .reduce(|| (0, 0), |a, b| (a.0 + b.0, a.1 + b.1))
468        });
469        return VerifyOutcome {
470            results: Vec::new(),
471            exhausted: true,
472            estimated_total: None,
473            substring_only: substring_only.into_inner(),
474            excluded_by_default,
475            counted: Some((if paths_only { files_hit } else { lines }, files_hit)),
476        };
477    }
478
479    let mut results: Vec<SearchResult> = Vec::new();
480    let mut next = 0usize;
481    // The doubling rounds exist so a page can stop early. Without a budget every
482    // file is verified anyway, and each round boundary is a full pool drain that
483    // waits on its slowest file; one round keeps every thread busy to the end.
484    let mut chunk = if budget.is_some() {
485        16usize
486    } else {
487        accepted.len().max(1)
488    };
489    let mut exhausted = true;
490
491    while next < accepted.len() {
492        if let Some(b) = budget
493            && results.len() >= b
494        {
495            exhausted = false;
496            break;
497        }
498        let end = (next + chunk).min(accepted.len());
499        let slice = &accepted[next..end];
500        let round: Vec<Vec<SearchResult>> = open.pool().install(|| {
501            slice
502                .par_iter()
503                .map(|(file_id, path, lang, lines)| verify_one(*file_id, path, *lang, lines))
504                .collect()
505        });
506        for r in round {
507            results.extend(r);
508        }
509        next = end;
510        chunk = (chunk * 2).min(1024);
511    }
512
513    // The page is full but candidates remain. Either finish (cheap) or estimate the
514    // total from a spread sample, so the caller gets a number that is close rather
515    // than a posting-list upper bound that ran 2x high in the field.
516    let mut estimated_total = None;
517    if !exhausted {
518        let remaining = &accepted[next..];
519        // One "unit" is what a result counts: a line normally, a file in paths mode
520        // (`verify_one` stops at the first hit per file there).
521        let unit = |lines: &LineSet| -> Option<usize> {
522            match lines {
523                _ if paths_only => Some(1),
524                LineSet::Only(v) => Some(v.len()),
525                LineSet::All => None,
526            }
527        };
528        let remaining_units = remaining
529            .iter()
530            .try_fold(0usize, |acc, (_, _, _, l)| unit(l).map(|u| acc + u));
531
532        let finish = remaining.len() <= ESTIMATE_SAMPLE_FILES
533            || remaining_units.is_some_and(|u| u <= ESTIMATE_FINISH_LINES);
534        if finish {
535            let round: Vec<Vec<SearchResult>> = open.pool().install(|| {
536                remaining
537                    .par_iter()
538                    .map(|(file_id, path, lang, lines)| verify_one(*file_id, path, *lang, lines))
539                    .collect()
540            });
541            for r in round {
542                results.extend(r);
543            }
544            exhausted = true;
545        } else if let Some(units) = remaining_units {
546            let stride = remaining.len() / ESTIMATE_SAMPLE_FILES;
547            let picks: Vec<&(u32, String, Language, LineSet)> = (0..ESTIMATE_SAMPLE_FILES)
548                .map(|i| &remaining[i * stride])
549                .collect();
550            let (hits, sampled_units) = open.pool().install(|| {
551                picks
552                    .par_iter()
553                    .map(|(file_id, path, lang, lines)| {
554                        let trimmed = match lines {
555                            LineSet::Only(v) if !paths_only => {
556                                LineSet::Only(v[..v.len().min(ESTIMATE_PER_FILE_LINES)].to_vec())
557                            }
558                            LineSet::Only(v) => LineSet::Only(v.clone()),
559                            LineSet::All => LineSet::All,
560                        };
561                        let u = unit(&trimmed).unwrap_or(0);
562                        // Sample hits are counted, never appended: the page must stay
563                        // a contiguous prefix in path order.
564                        (verify_one(*file_id, path, *lang, &trimmed).len(), u)
565                    })
566                    .reduce(|| (0usize, 0usize), |a, b| (a.0 + b.0, a.1 + b.1))
567            });
568            let rate = hits as f64 / sampled_units.max(1) as f64;
569            estimated_total = Some(results.len() + (rate * units as f64).round() as usize);
570        }
571    }
572
573    VerifyOutcome {
574        results,
575        exhausted,
576        estimated_total,
577        substring_only: substring_only.into_inner(),
578        excluded_by_default,
579        counted: None,
580    }
581}
582
583/// Manages query execution against the index
584pub struct QueryEngine {
585    cache: CacheManager,
586    /// The open index for this engine's lifetime, resolved on first use so every
587    /// phase of one query reads the same files.
588    open: OnceLock<Arc<OpenIndex>>,
589}
590
591impl QueryEngine {
592    /// Create a new query engine with the given cache manager
593    pub fn new(cache: CacheManager) -> Self {
594        Self {
595            cache,
596            open: OnceLock::new(),
597        }
598    }
599
600    /// The shared open-index handle (see [`open_index`]).
601    ///
602    /// Typed `CacheCorrupted` on a short or garbled store, so the MCP layer can
603    /// rebuild once and retry.
604    fn open_index(&self) -> Result<Arc<OpenIndex>> {
605        if let Some(open) = self.open.get() {
606            return Ok(Arc::clone(open));
607        }
608        let open = open_index::get_or_open(&self.cache)?;
609        let _ = self.open.set(Arc::clone(&open));
610        Ok(open)
611    }
612
613    /// Load dependencies for search results if requested (legacy - per result)
614    /// Deprecated: Use group_and_load_dependencies for file-level grouping
615    fn load_dependencies(&self, results: &mut [SearchResult], include_deps: bool) -> Result<()> {
616        if !include_deps || results.is_empty() {
617            return Ok(());
618        }
619
620        log::debug!("Loading dependencies for {} results", results.len());
621
622        // Create dependency index
623        // Note: We need to pass the workspace root, not the cache directory
624        // The cache path is .reflex/, so its parent is the workspace root (.)
625        let workspace_root = self
626            .cache
627            .path()
628            .parent()
629            .ok_or_else(|| anyhow::anyhow!("Cache path has no parent"))?;
630        let cache_for_deps = CacheManager::new(workspace_root);
631        let dep_index = crate::dependency::DependencyIndex::new(cache_for_deps);
632
633        // Load dependencies for each result
634        for result in results {
635            // Normalize path: strip leading "./" if present
636            let normalized_path = result.path.strip_prefix("./").unwrap_or(&result.path);
637
638            // Get file_id from database by path
639            match self.cache.get_file_id(normalized_path) {
640                Ok(Some(file_id)) => {
641                    log::debug!("Found file_id={} for path={}", file_id, result.path);
642                    // Get dependencies for this file
643                    match dep_index.get_dependencies_info(file_id) {
644                        Ok(dep_infos) => {
645                            log::debug!(
646                                "Loaded {} dependencies for file_id={}",
647                                dep_infos.len(),
648                                file_id
649                            );
650                            if !dep_infos.is_empty() {
651                                result.dependencies = Some(dep_infos);
652                            }
653                        }
654                        Err(e) => {
655                            log::warn!("Failed to get dependencies for file_id={}: {}", file_id, e);
656                        }
657                    }
658                }
659                Ok(None) => {
660                    log::warn!("No file_id found for path: {}", result.path);
661                }
662                Err(e) => {
663                    log::warn!("Failed to get file_id for path {}: {}", result.path, e);
664                }
665            }
666        }
667
668        Ok(())
669    }
670
671    /// Group search results by file and load dependencies at file level
672    /// Returns file-grouped results with dependencies populated once per file
673    fn group_and_load_dependencies(
674        &self,
675        results: Vec<SearchResult>,
676        include_deps: bool,
677        context_lines: usize,
678    ) -> Result<Vec<crate::models::FileGroupedResult>> {
679        use crate::models::{FileGroupedResult, MatchResult};
680        use std::collections::HashMap;
681
682        if results.is_empty() {
683            return Ok(Vec::new());
684        }
685
686        // Group results by file path (preserving language from first match)
687        let mut grouped: HashMap<String, Vec<SearchResult>> = HashMap::new();
688        for result in results {
689            grouped.entry(result.path.clone()).or_default().push(result);
690        }
691
692        // Create dependency index if needed
693        let dep_index = if include_deps {
694            let workspace_root = self
695                .cache
696                .path()
697                .parent()
698                .ok_or_else(|| anyhow::anyhow!("Cache path has no parent"))?;
699            let cache_for_deps = CacheManager::new(workspace_root);
700            Some(crate::dependency::DependencyIndex::new(cache_for_deps))
701        } else {
702            None
703        };
704
705        // The shared index handle, for extracting context lines
706        let open_opt = self.open_index().ok();
707        let content_reader_opt = open_opt.as_deref().map(|o| &o.content);
708
709        // Convert to FileGroupedResult and load dependencies
710        let mut file_results: Vec<FileGroupedResult> = grouped
711            .into_iter()
712            .map(|(path, file_matches)| {
713                // Capture language from first match (all matches in a file share the same language)
714                let language = file_matches.first().map(|r| r.lang).unwrap_or_default();
715
716                // Load dependencies for this file (once per file, not per result)
717                let dependencies = if let Some(dep_idx) = &dep_index {
718                    let normalized_path = path.strip_prefix("./").unwrap_or(&path);
719                    match self.cache.get_file_id(normalized_path) {
720                        Ok(Some(file_id)) => match dep_idx.get_dependencies_info(file_id) {
721                            Ok(dep_infos) if !dep_infos.is_empty() => {
722                                log::debug!(
723                                    "Loaded {} dependencies for file: {}",
724                                    dep_infos.len(),
725                                    path
726                                );
727                                Some(dep_infos)
728                            }
729                            Ok(_) => None,
730                            Err(e) => {
731                                log::warn!("Failed to get dependencies for {}: {}", path, e);
732                                None
733                            }
734                        },
735                        Ok(None) => {
736                            log::warn!("No file_id found for path: {}", path);
737                            None
738                        }
739                        Err(e) => {
740                            log::warn!("Failed to get file_id for path {}: {}", path, e);
741                            None
742                        }
743                    }
744                } else {
745                    None
746                };
747
748                // Array file id (not the database id) for context extraction
749                let file_id_for_context = open_opt.as_deref().and_then(|o| o.file_id_for(&path));
750                log::debug!(
751                    "Context extraction: file={}, file_id={:?}, content_reader={}",
752                    path,
753                    file_id_for_context,
754                    content_reader_opt.is_some()
755                );
756
757                // Convert SearchResults to MatchResults (strip path and dependencies) and extract context
758                let matches: Vec<MatchResult> = file_matches
759                    .into_iter()
760                    .map(|r| {
761                        // Extract context lines if requested (0 = disabled)
762                        let (context_before, context_after) = if context_lines > 0 {
763                            if let (Some(reader), Some(fid)) =
764                                (&content_reader_opt, file_id_for_context)
765                            {
766                                let result = reader
767                                    .get_context_by_line(fid, r.span.start_line, context_lines)
768                                    .unwrap_or_else(|e| {
769                                        log::warn!(
770                                            "Failed to extract context for {}:{}: {}",
771                                            path,
772                                            r.span.start_line,
773                                            e
774                                        );
775                                        (vec![], vec![])
776                                    });
777                                log::debug!(
778                                    "Extracted context for {}:{} - before: {}, after: {}",
779                                    path,
780                                    r.span.start_line,
781                                    result.0.len(),
782                                    result.1.len()
783                                );
784                                result
785                            } else {
786                                if content_reader_opt.is_none() {
787                                    log::debug!(
788                                        "No ContentReader available for context extraction"
789                                    );
790                                }
791                                if file_id_for_context.is_none() {
792                                    log::debug!("No file_id found for {}", path);
793                                }
794                                (vec![], vec![])
795                            }
796                        } else {
797                            (vec![], vec![])
798                        };
799
800                        MatchResult {
801                            kind: r.kind,
802                            symbol: r.symbol,
803                            span: r.span,
804                            preview: r.preview,
805                            context_before,
806                            context_after,
807                        }
808                    })
809                    .collect();
810
811                FileGroupedResult {
812                    path,
813                    language,
814                    dependencies,
815                    matches,
816                }
817            })
818            .collect();
819
820        // Sort by path for deterministic output
821        file_results.sort_by(|a, b| a.path.cmp(&b.path));
822
823        Ok(file_results)
824    }
825
826    /// Execute a query and return matching results with index metadata
827    ///
828    /// This is the preferred method for programmatic/JSON output as it includes
829    /// index freshness information that AI agents can use to decide whether to re-index.
830    pub fn search_with_metadata(
831        &self,
832        pattern: &str,
833        filter: QueryFilter,
834    ) -> Result<QueryResponse> {
835        log::info!(
836            "Executing query with metadata: pattern='{}', filter={:?}",
837            pattern,
838            filter
839        );
840
841        let started = std::time::Instant::now();
842
843        // Ensure cache exists
844        if !self.cache.exists() {
845            return Err(crate::errors::ReflexError::IndexNotFound.into());
846        }
847
848        // Open (or reuse) the index. A short or garbled store surfaces as a typed
849        // `CacheCorrupted`, so the MCP layer can auto-rebuild once and retry. This
850        // replaces the per-query `validate()`, whose `PRAGMA quick_check` walked the
851        // whole database on every call.
852        self.open_index()?;
853        let open_us = started.elapsed().as_micros() as u64;
854
855        // A whole-identifier pattern containing brackets can never match; run it as an
856        // escaped regex and say so. This lives here, not in the surfaces, so the CLI,
857        // MCP and HTTP paths cannot disagree about it (1.7.2 fixed only MCP).
858        let prepared = prepare_literal_pattern(pattern, &filter);
859        let mut filter = filter;
860        if prepared.use_regex && !filter.use_regex {
861            log::debug!(
862                "Pattern {:?} rewritten to regex {:?}",
863                pattern,
864                prepared.effective
865            );
866            filter.rewritten_from = Some(pattern.to_string());
867        }
868        filter.use_regex = prepared.use_regex;
869
870        // The freshness snapshot runs on its own thread while the search runs. It
871        // costs a few git spawns (or a tree walk outside git) and depends on nothing
872        // the search produces; only the WORDING is scoped to the result paths, and
873        // that is derived after both are done. On the CLI, where the memo is cold
874        // every time, this takes the status phase off the zero-hit floor.
875        let status_started = std::time::Instant::now();
876        let (searched, search_done, snapshot) = std::thread::scope(|s| {
877            let status = s.spawn(|| {
878                let snapshot = status_cache::snapshot(&self.cache);
879                (snapshot, status_started.elapsed())
880            });
881            let searched = self.search_internal(&prepared.effective, filter.clone());
882            // Measured before the join, so the status wait is not charged to verify.
883            let search_done = started.elapsed();
884            let snapshot = status
885                .join()
886                .unwrap_or_else(|panic| std::panic::resume_unwind(panic));
887            (searched, search_done, snapshot)
888        });
889        let Internal {
890            results,
891            total_count: total,
892            total_is_exact,
893            approx_total,
894            substring_only,
895            excluded_by_default,
896            counted,
897            candidates_us,
898            index_path,
899            warnings: engine_warnings,
900        } = searched?;
901        let (snapshot, status_compute) = snapshot;
902        let snapshot = snapshot?;
903
904        // Scoped to the result paths: the index being behind is reported honestly as
905        // `stale` either way, and a stale index is never trusted; the scope only
906        // sharpens the reason ("including a file these results came from").
907        let scope: Vec<String> = results.iter().map(|r| r.path.clone()).collect();
908        let report = Self::report_from(&snapshot, Some(&scope));
909        let (status, can_trust_results, warning) =
910            (report.status, report.can_trust_results, report.warning);
911        let status_done = started.elapsed();
912
913        // Build pagination metadata
914        use crate::models::PaginationInfo;
915        let pagination = PaginationInfo {
916            // Never the verified-so-far number: that is not a total.
917            total: total_is_exact.then_some(total),
918            count: results.len(),
919            offset: filter.offset.unwrap_or(0),
920            limit: filter.limit,
921            has_more: !total_is_exact || total > filter.offset.unwrap_or(0) + results.len(),
922            total_is_exact,
923            approx_total,
924        };
925
926        // Always use grouped format (group results by file)
927        // Dependencies are loaded only when include_dependencies is true
928        let grouped_results = self.group_and_load_dependencies(
929            results,
930            filter.include_dependencies,
931            filter.context_lines,
932        )?;
933
934        let total_elapsed = started.elapsed();
935        let timings = crate::models::QueryTimings {
936            index_path,
937            open_us,
938            candidates_us,
939            verify_us: (search_done.as_micros() as u64).saturating_sub(open_us + candidates_us),
940            status_us: (status_done - search_done).as_micros() as u64,
941            status_compute_us: status_compute.as_micros() as u64,
942            group_us: (total_elapsed - status_done).as_micros() as u64,
943            total_us: total_elapsed.as_micros() as u64,
944        };
945        log::debug!("Query timings for '{}': {:?}", pattern, timings);
946
947        // Why nothing came back: one reason, chosen from the filter in a fixed
948        // order (see `zero_hint`). Computed only on a zero, so its disk and
949        // config reads cost nothing on the hot path.
950        let substring_hint_count = if total == 0 { substring_only } else { None };
951        let excluded_by_default =
952            (total == 0 && excluded_by_default > 0).then_some(excluded_by_default);
953        let zero = if total == 0 {
954            let config = self.cache.load_index_config().unwrap_or_default();
955            self.open_index().ok().and_then(|open| {
956                zero_hint::explain_zero(
957                    &self.cache.workspace_root(),
958                    &config,
959                    &open,
960                    &filter,
961                    pattern,
962                    substring_only,
963                    excluded_by_default.unwrap_or(0),
964                    prepared.warning.is_some(),
965                )
966            })
967        } else {
968            None
969        };
970        let (hint, excluded_reason) = match zero {
971            Some(z) => (Some(z.text), Some(z.reason)),
972            None => (None, None),
973        };
974
975        Ok(QueryResponse {
976            ai_instruction: None, // AI instruction is generated by CLI/MCP layer, not here
977            status,
978            can_trust_results,
979            warning,
980            pagination,
981            results: grouped_results,
982            substring_hint_count,
983            warnings: prepared
984                .warning
985                .into_iter()
986                .chain(engine_warnings)
987                .collect(),
988            hint,
989            excluded_reason,
990            excluded_by_default,
991            file_count: counted.map(|(_, files)| files),
992            timings: filter.collect_timings.then_some(timings),
993        })
994    }
995
996    /// Execute a query and return matching results (legacy method)
997    ///
998    /// This method prints warnings to stderr and returns just the results.
999    /// For programmatic use, prefer `search_with_metadata()`.
1000    pub fn search(&self, pattern: &str, filter: QueryFilter) -> Result<Vec<SearchResult>> {
1001        log::info!(
1002            "Executing query: pattern='{}', filter={:?}",
1003            pattern,
1004            filter
1005        );
1006
1007        // Ensure cache exists
1008        if !self.cache.exists() {
1009            return Err(crate::errors::ReflexError::IndexNotFound.into());
1010        }
1011
1012        // Open (or reuse) the index; corruption surfaces as a typed error.
1013        self.open_index()?;
1014
1015        // Show non-blocking warnings about branch state and staleness
1016        self.check_index_freshness(&filter)?;
1017
1018        // Same bracket rewrite as `search_with_metadata`; this surface has nowhere to
1019        // report it, but it must not return a different answer.
1020        let prepared = prepare_literal_pattern(pattern, &filter);
1021        let mut filter = filter;
1022        if prepared.use_regex && !filter.use_regex {
1023            filter.rewritten_from = Some(pattern.to_string());
1024        }
1025        filter.use_regex = prepared.use_regex;
1026
1027        // Execute the search (discard total count - legacy method doesn't use it)
1028        let mut results = self
1029            .search_internal(&prepared.effective, filter.clone())?
1030            .results;
1031
1032        // Load dependencies if requested
1033        self.load_dependencies(&mut results, filter.include_dependencies)?;
1034
1035        Ok(results)
1036    }
1037
1038    /// Internal search implementation (used by both search methods)
1039    /// Returns (results, total_count) where total_count is the count before offset/limit
1040    fn search_internal(&self, pattern: &str, filter: QueryFilter) -> Result<Internal> {
1041        use std::time::{Duration, Instant};
1042
1043        // Start timeout timer if configured
1044        let start_time = Instant::now();
1045        let timeout = if filter.timeout_secs > 0 {
1046            Some(Duration::from_secs(filter.timeout_secs))
1047        } else {
1048            None
1049        };
1050
1051        // KEYWORD DETECTION (early): Check if this is a keyword query that should scan ALL files
1052        // When a user searches for a language keyword (like "class", "function") with --symbols or --kind,
1053        // we interpret it as "list all symbols of that type" and should scan ALL files,
1054        // not just the first 100 candidates from trigram search.
1055        //
1056        // Requirements for keyword query mode:
1057        // 1. Symbol mode active (--symbols or --kind)
1058        // 2. Pattern matches a keyword in ANY supported language
1059        //
1060        // Note: --lang is optional. If specified, language filtering happens naturally in Phase 2/3.
1061        // Empty pattern in symbol mode means "list all symbols of the requested kind" —
1062        // treat it like a keyword query so we scan all files instead of failing the
1063        // broad-query guard or returning zero trigram matches.
1064        let is_keyword_query = if filter.symbols_mode || filter.kind.is_some() {
1065            pattern.is_empty() || ParserFactory::get_all_keywords().contains(&pattern)
1066        } else {
1067            false
1068        };
1069
1070        // KEYWORD-TO-KIND MAPPING: If user searches for a keyword without --kind, infer the kind
1071        // Example: "class" → SymbolKind::Class, "function" → SymbolKind::Function
1072        // This ensures keyword queries return only the relevant symbol type
1073        let mut filter = filter.clone(); // Clone so we can modify it
1074        if is_keyword_query
1075            && filter.kind.is_none()
1076            && let Some(inferred_kind) = Self::keyword_to_kind(pattern)
1077        {
1078            log::info!(
1079                "Keyword '{}' mapped to kind {:?} (auto-inferred)",
1080                pattern,
1081                inferred_kind
1082            );
1083            filter.kind = Some(inferred_kind);
1084        }
1085
1086        // EARLY BROAD QUERY DETECTION (Index Size Check)
1087        // This check happens BEFORE the expensive trigram search to prevent hangs on large indexes
1088        // For very large codebases (like Linux kernel with 62K files), even valid 3-char trigrams
1089        // like "get" can take 10-30+ seconds to search. This early check prevents that hang.
1090        //
1091        // Criteria for early blocking:
1092        // 1. Large index (> 20,000 files) AND
1093        // 2. Short pattern (< 4 chars) AND
1094        // 3. Not using regex (regex has its own trigram extraction) — a regex the
1095        //    engine built from a literal (`ignore_case`, brackets) is judged as
1096        //    that literal AND
1097        // 4. Not a keyword query (keywords are intentionally broad) AND
1098        // 5. Not forced by --force flag
1099        let guarded_literal = filter.rewritten_from.as_deref();
1100        if !filter.force && (!filter.use_regex || guarded_literal.is_some()) && !is_keyword_query {
1101            // Index-wide file count from the open handle. `cache.stats()` here used
1102            // to open SQLite and spawn git on every query.
1103            let total_files = self.open_index()?.file_count();
1104            let pattern = guarded_literal.unwrap_or(pattern);
1105            let pattern_len = pattern.chars().count();
1106
1107            // Thresholds for early blocking:
1108            // - Large index: 20,000+ files (approximately where performance degrades significantly)
1109            // - Short pattern: < 4 chars (3-char trigrams are borderline, < 4 catches edge cases)
1110            // Test overrides allow reducing thresholds for integration tests without creating 20K+ files
1111            let large_index_threshold = filter.test_large_index_threshold.unwrap_or(20_000);
1112            let short_pattern_threshold = filter.test_short_pattern_threshold.unwrap_or(4);
1113
1114            if total_files > large_index_threshold && pattern_len < short_pattern_threshold {
1115                anyhow::bail!(
1116                    "Query too broad - would be expensive to execute on this large index\n\
1117                     \n\
1118                     This index contains {} files, and pattern '{}' ({} characters) is too short for efficient searching.\n\
1119                     On large codebases, short patterns can take 10-30+ seconds to complete.\n\
1120                     \n\
1121                     This query could:\n\
1122                     • Hang for an extended period before returning results\n\
1123                     • Return thousands of results\n\
1124                     • Flood LLM context windows with excessive data\n\
1125                     • Fail entirely\n\
1126                     \n\
1127                     Suggestions to narrow the query:\n\
1128                     • Use a longer, more specific pattern (4+ characters recommended for large indexes)\n\
1129                     • Add a language filter: --lang <language>\n\
1130                     • Add a file filter: --glob <pattern> or --file <path>\n\
1131                     • Use --force to bypass this check if you really need all results\n\
1132                     \n\
1133                     To force execution anyway:\n\
1134                     rfx query \"{}\" --force",
1135                    total_files,
1136                    pattern,
1137                    pattern_len,
1138                    pattern
1139                );
1140            }
1141        }
1142
1143        // Early termination applies to plain list-mode text/regex searches: the
1144        // page is complete once `offset + limit` results exist, in order, and the
1145        // total is reported as inexact. Symbol and AST searches dedup and filter
1146        // after enrichment, so they verify everything; count and no-limit callers
1147        // (`mode:"count"`, `list_locations`, `--count`) pass no limit and also do.
1148        let budget = if filter.symbols_mode
1149            || filter.kind.is_some()
1150            || filter.use_ast
1151            || is_keyword_query
1152            || filter.require_exact_total
1153        {
1154            None
1155        } else {
1156            filter
1157                .limit
1158                .map(|limit| filter.offset.unwrap_or(0).saturating_add(limit))
1159        };
1160
1161        // PHASE 1: Get initial candidates (choose search strategy)
1162        let mut substring_only = None;
1163        let mut excluded_by_default = 0usize;
1164        let mut counted = None;
1165        let mut candidates_us = 0u64;
1166        let mut total_is_exact = true;
1167        let mut approx_total = None;
1168        let index_path;
1169        let engine_warnings;
1170        let mut results = if is_keyword_query {
1171            // A keyword query bypasses the index on purpose.
1172            index_path = IndexPath::Scan;
1173            engine_warnings = Vec::new();
1174            // KEYWORD QUERY MODE: Scan all files (or files of target language if --lang specified)
1175            // This ensures we find ALL classes/functions/etc, not just those in the first 100 trigram matches
1176            if let Some(lang) = filter.language {
1177                log::info!(
1178                    "Keyword query detected for '{}' - scanning all {:?} files (bypassing trigram search)",
1179                    pattern,
1180                    lang
1181                );
1182            } else {
1183                log::info!(
1184                    "Keyword query detected for '{}' - scanning all files (bypassing trigram search)",
1185                    pattern
1186                );
1187            }
1188            self.get_all_language_files(&filter)?
1189        } else {
1190            let (candidates, stats) = if filter.use_regex {
1191                // Regex pattern search with trigram optimization
1192                self.get_regex_candidates(pattern, &filter, timeout.as_ref(), &start_time, budget)?
1193            } else {
1194                // Standard trigram-based full-text search
1195                self.get_trigram_candidates(pattern, &filter, budget)?
1196            };
1197            substring_only = stats.substring_only;
1198            excluded_by_default = stats.excluded_by_default;
1199            counted = stats.counted;
1200            candidates_us = stats.candidates_us;
1201            total_is_exact = stats.exhausted;
1202            approx_total = (!stats.exhausted).then_some(stats.approx_total).flatten();
1203            index_path = stats.index_path;
1204            engine_warnings = stats.warnings;
1205            candidates
1206        };
1207
1208        // EARLY LANGUAGE FILTER: Apply language filtering BEFORE broad query check
1209        // This ensures we only parse files matching the language filter in Phase 2
1210        // Critical for non-keyword queries to work correctly with accurate candidate counts
1211        //
1212        // Skip for keyword queries - those candidates are already pre-filtered by language
1213        if !is_keyword_query && let Some(lang) = filter.language {
1214            let before_count = results.len();
1215            results.retain(|r| r.lang == lang);
1216            log::debug!(
1217                "Language filter ({:?}): reduced {} candidates to {} candidates",
1218                lang,
1219                before_count,
1220                results.len()
1221            );
1222        }
1223
1224        // EARLY GLOB PATTERN FILTER: Apply glob/exclude filtering BEFORE broad query check
1225        // This ensures candidate count reflects actual files that will be parsed
1226        // Critical for queries like: rfx query "index" --symbols --glob "src/**/*.rs"
1227        if !filter.glob_patterns.is_empty() || !filter.exclude_patterns.is_empty() {
1228            // Build include matcher (if patterns specified)
1229            let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
1230
1231            // Build exclude matcher (if patterns specified)
1232            let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
1233
1234            // Apply filters
1235            let before_count = results.len();
1236            results.retain(|r| {
1237                // If include patterns specified, path must match at least one
1238                let included = if let Some(ref matcher) = include_matcher {
1239                    matcher.is_match(&r.path)
1240                } else {
1241                    true // No include patterns = include all
1242                };
1243
1244                // If exclude patterns specified, path must NOT match any
1245                let excluded = if let Some(ref matcher) = exclude_matcher {
1246                    matcher.is_match(&r.path)
1247                } else {
1248                    false // No exclude patterns = exclude none
1249                };
1250
1251                included && !excluded
1252            });
1253            log::debug!(
1254                "Glob filter: reduced {} candidates to {} candidates",
1255                before_count,
1256                results.len()
1257            );
1258        }
1259
1260        // Check timeout after Phase 1
1261        if let Some(timeout_duration) = timeout
1262            && start_time.elapsed() > timeout_duration
1263        {
1264            anyhow::bail!(
1265                "Query timeout exceeded ({} seconds).\n\
1266                     \n\
1267                     The query took too long to complete. Try one of these approaches:\n\
1268                     • Use a more specific search pattern (longer patterns = faster search)\n\
1269                     • Add a language filter with --lang to narrow the search space\n\
1270                     • Add a file filter with --file to search specific directories\n\
1271                     • Increase the timeout with --timeout <seconds>\n\
1272                     \n\
1273                     Example: rfx query \"{}\" --lang rust --timeout 60",
1274                filter.timeout_secs,
1275                pattern
1276            );
1277        }
1278
1279        // BROAD QUERY DETECTION: Check if query is too expensive BEFORE parsing
1280        // This protects LLM users from accidentally running expensive queries that flood context windows
1281        if !filter.force {
1282            let candidate_count = results.len();
1283            let pattern_len = pattern.chars().count();
1284
1285            // Condition 1: Pattern too short (< 3 chars can't use trigram optimization efficiently)
1286            // Exception: Allow short keyword queries (e.g., "fn", "if") since they scan all language files
1287            let is_short_pattern = pattern_len < 3 && !filter.use_regex && !is_keyword_query;
1288
1289            // Condition 2: AST query without glob restriction on large codebases
1290            // Allow on small codebases (< 100 files) but require glob for larger ones
1291            let is_broad_ast =
1292                filter.use_ast && filter.glob_patterns.is_empty() && candidate_count >= 100;
1293
1294            // Condition 3: Query-type-aware threshold for symbol/AST parsing
1295            // Different thresholds based on actual performance characteristics:
1296            // - AST without glob: 100 files (allow small codebases, block large ones)
1297            // - AST with glob: 10,000 files (~5 seconds max)
1298            // - Keyword queries: 20,000 files (~3 seconds max) - scan all files of language
1299            // - Trigram-filtered symbols: 50,000 files (~5 seconds max) - very fast due to trigram filtering
1300            let threshold = if filter.use_ast && filter.glob_patterns.is_empty() {
1301                100 // AST without glob - allow small codebases
1302            } else if filter.use_ast {
1303                10_000 // AST with glob restriction
1304            } else if is_keyword_query {
1305                20_000 // Keyword queries (e.g., "class", "function")
1306            } else {
1307                50_000 // Trigram-filtered symbol queries
1308            };
1309
1310            let has_many_candidates = candidate_count > threshold
1311                && (filter.symbols_mode || filter.kind.is_some() || filter.use_ast);
1312
1313            if is_short_pattern || has_many_candidates || is_broad_ast {
1314                let reason = if is_short_pattern {
1315                    format!(
1316                        "Pattern '{}' is too short ({} characters). Short patterns bypass trigram optimization and require scanning many files.",
1317                        pattern, pattern_len
1318                    )
1319                } else if is_broad_ast {
1320                    format!(
1321                        "AST query without --glob restriction will scan the entire codebase ({} files). AST queries are SLOW (500ms-10s+).",
1322                        candidate_count
1323                    )
1324                } else if is_keyword_query {
1325                    format!(
1326                        "Keyword query '{}' matched {} files. This query scans all files of the target language, which will take significant time and produce excessive results.",
1327                        pattern, candidate_count
1328                    )
1329                } else {
1330                    format!(
1331                        "Query matched {} files. Parsing this many files with --symbols or --kind will take significant time and produce excessive results.",
1332                        candidate_count
1333                    )
1334                };
1335
1336                let suggestions = if is_short_pattern {
1337                    vec![
1338                        "• Use a longer, more specific pattern (3+ characters recommended)",
1339                        "• Add a language filter: --lang <language>",
1340                        "• Add a file path filter: --file <path> or --glob <pattern>",
1341                        "• Use --force to bypass this check if you really need all results",
1342                    ]
1343                } else if is_broad_ast {
1344                    vec![
1345                        "• Add --glob to restrict AST query to specific files: --glob 'src/**/*.rs'",
1346                        "• Use --symbols instead (10-100x faster in 95% of cases)",
1347                        "• Use --force to bypass this check if you need a full codebase scan",
1348                    ]
1349                } else if is_keyword_query {
1350                    vec![
1351                        "• Add a language filter to reduce files scanned: --lang <language>",
1352                        "• Add glob patterns to search specific directories: --glob 'src/**/*.rs'",
1353                        "• Add --kind to filter to specific symbol types: --kind function",
1354                        "• Use a more specific pattern instead of a keyword",
1355                        "• Use --force to bypass this check if you need all results",
1356                    ]
1357                } else {
1358                    vec![
1359                        "• Add a language filter to reduce candidate set: --lang <language>",
1360                        "• Add glob patterns to search specific directories: --glob 'src/**/*.rs'",
1361                        "• Use a more specific search pattern",
1362                        "• Use --force to bypass this check if you need all results",
1363                    ]
1364                };
1365
1366                // Build the command snippet showing current flags
1367                let mut cmd_flags = String::new();
1368                if filter.symbols_mode {
1369                    cmd_flags.push_str("--symbols ");
1370                }
1371                if let Some(ref lang) = filter.language {
1372                    cmd_flags.push_str(&format!("--lang {:?} ", lang));
1373                }
1374                if let Some(ref kind) = filter.kind {
1375                    cmd_flags.push_str(&format!("--kind {:?} ", kind));
1376                }
1377                if filter.use_ast {
1378                    cmd_flags.push_str("--ast ");
1379                }
1380
1381                anyhow::bail!(
1382                    "Query too broad - would be expensive to execute\n\
1383                     \n\
1384                     {}\n\
1385                     \n\
1386                     This query could:\n\
1387                     • Hang for an extended period before returning results\n\
1388                     • Return thousands of results\n\
1389                     • Flood LLM context windows with excessive data\n\
1390                     • Fail entirely\n\
1391                     \n\
1392                     Suggestions to narrow the query:\n\
1393                     {}\n\
1394                     \n\
1395                     To force execution anyway:\n\
1396                     rfx query \"{}\" --force {}",
1397                    reason,
1398                    suggestions.join("\n             "),
1399                    pattern,
1400                    cmd_flags
1401                );
1402            }
1403        }
1404
1405        // DETERMINISTIC SORTING: Sort candidates early for deterministic results
1406        // This ensures results are always returned in the same order
1407        if filter.symbols_mode || filter.kind.is_some() || filter.use_ast {
1408            results.sort_by(|a, b| {
1409                a.path
1410                    .cmp(&b.path)
1411                    .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1412            });
1413
1414            // Warn if many candidates need parsing (helps users refine queries)
1415            let candidate_count = results.len();
1416            if candidate_count > 1000 && !filter.suppress_output {
1417                output::warn(&format!(
1418                    "Pattern '{}' matched {} files - parsing may take some time. Consider using --file, --glob, or a more specific pattern to narrow the search.",
1419                    pattern, candidate_count
1420                ));
1421            } else if candidate_count > 100 {
1422                log::info!(
1423                    "Parsing {} candidate files for symbol extraction",
1424                    candidate_count
1425                );
1426            }
1427        }
1428
1429        // PHASE 2: Enrich with symbol information or AST pattern matching (if needed)
1430        if filter.use_ast {
1431            // AST pattern matching: Execute Tree-sitter query on candidate files
1432            results = self.enrich_with_ast(results, pattern, filter.language)?;
1433        } else if filter.symbols_mode || filter.kind.is_some() {
1434            // Symbol enrichment: Parse candidate files and extract symbol definitions
1435            results = self.enrich_with_symbols(results, pattern, &filter, is_keyword_query)?;
1436        }
1437
1438        // PHASE 3: Apply post-enrichment filters
1439        // Note: Language and glob filters are applied in Phase 1 (before broad query check)
1440        // Only kind, file_pattern, and exact filters are applied here
1441
1442        // Deduplicate symbols: the same source location can be emitted as both
1443        // Function and Method by some parsers.  Keep the first hit for each
1444        // (path, start_line, symbol_name) triple so --kind function doesn't
1445        // return the same definition twice.
1446        if filter.symbols_mode || filter.kind.is_some() {
1447            let mut seen = std::collections::HashSet::<(String, usize, Option<String>)>::new();
1448            results.retain(|r| seen.insert((r.path.clone(), r.span.start_line, r.symbol.clone())));
1449        }
1450
1451        // Apply kind filter (only relevant for symbol searches)
1452        // Special case: --kind function also includes methods (methods are functions in classes)
1453        if let Some(ref kind) = filter.kind {
1454            results.retain(|r| {
1455                if matches!(kind, SymbolKind::Function) {
1456                    // When searching for functions, also include methods
1457                    matches!(r.kind, SymbolKind::Function | SymbolKind::Method)
1458                } else {
1459                    r.kind == *kind
1460                }
1461            });
1462        }
1463
1464        // Apply file path filter (substring match)
1465        if let Some(ref file_pattern) = filter.file_pattern {
1466            results.retain(|r| r.path.contains(file_pattern));
1467        }
1468
1469        // Apply exact name filter (only for symbol searches)
1470        if filter.exact && filter.symbols_mode {
1471            results.retain(|r| r.symbol.as_deref() == Some(pattern));
1472        }
1473
1474        // Expand symbol bodies if requested
1475        // Works for both symbol-mode and regex searches (if regex matched a symbol definition)
1476        if filter.expand {
1477            // Fetch full symbol bodies from the shared content store
1478            if let Ok(open) = self.open_index() {
1479                let content_reader = &open.content;
1480                for result in &mut results {
1481                    // Only expand if the result has a meaningful span (not just a single line)
1482                    if result.span.start_line < result.span.end_line {
1483                        // Find the file_id for this result's path
1484                        if let Some(file_id) = open.file_id_for(&result.path) {
1485                            // Fetch the full span content
1486                            if let Ok(content) = content_reader.get_file_content(file_id) {
1487                                let lines: Vec<&str> = content.lines().collect();
1488                                let start_idx = result.span.start_line.saturating_sub(1);
1489                                let end_idx = result.span.end_line.min(lines.len());
1490
1491                                if start_idx < end_idx {
1492                                    let full_body = lines[start_idx..end_idx].join("\n");
1493                                    // Expand shows more than a preview, but not an
1494                                    // unbounded amount: on a minified file this is
1495                                    // the entire bundle.
1496                                    result.preview =
1497                                        crate::parsers::preview::expand_preview(&full_body);
1498                                }
1499                            }
1500                        }
1501                    }
1502                }
1503            }
1504        }
1505
1506        // Step 4: Deduplicate by path if paths-only mode
1507        if filter.paths_only {
1508            use std::collections::HashSet;
1509            let mut seen_paths = HashSet::new();
1510            results.retain(|r| seen_paths.insert(r.path.clone()));
1511        }
1512
1513        // Drop text-tier results when the caller asked for code only. Applied BEFORE
1514        // the total is captured, so pagination counts what the caller will actually
1515        // receive rather than what was found and then discarded.
1516        results.retain(|r| {
1517            filter::tier_admits(
1518                r.lang,
1519                filter.language,
1520                filter.exclude_text,
1521                filter.include_locks,
1522                filter.include_generated,
1523            )
1524        });
1525
1526        // Step 5: Sort results deterministically (by path, then line number)
1527        results.sort_by(|a, b| {
1528            a.path
1529                .cmp(&b.path)
1530                .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1531        });
1532
1533        // Capture total count AFTER all filtering but BEFORE pagination (offset/limit)
1534        // This is the total number of results the user can paginate through. In
1535        // count-only mode nothing was materialised and the verifier's count is it.
1536        let total_count = counted.map(|(lines, _)| lines).unwrap_or(results.len());
1537
1538        // Step 5.5: Apply offset (pagination)
1539        if let Some(offset) = filter.offset {
1540            if offset < results.len() {
1541                results = results.into_iter().skip(offset).collect();
1542            } else {
1543                // Offset beyond results - return empty
1544                results.clear();
1545            }
1546        }
1547
1548        // Step 6: Apply limit
1549        if let Some(limit) = filter.limit {
1550            results.truncate(limit);
1551        }
1552
1553        log::info!(
1554            "Query returned {} results (total before pagination: {})",
1555            results.len(),
1556            total_count
1557        );
1558
1559        Ok(Internal {
1560            results,
1561            total_count,
1562            total_is_exact,
1563            approx_total,
1564            substring_only,
1565            excluded_by_default,
1566            counted,
1567            candidates_us,
1568            index_path,
1569            warnings: engine_warnings,
1570        })
1571    }
1572
1573    /// Search for symbols by exact name match
1574    pub fn find_symbol(&self, name: &str) -> Result<Vec<SearchResult>> {
1575        let filter = QueryFilter {
1576            symbols_mode: true,
1577            ..Default::default()
1578        };
1579        self.search(name, filter)
1580    }
1581
1582    /// Search using a Tree-sitter AST pattern
1583    pub fn search_ast(&self, pattern: &str, lang: Option<Language>) -> Result<Vec<SearchResult>> {
1584        let filter = QueryFilter {
1585            language: lang,
1586            use_ast: true,
1587            ..Default::default()
1588        };
1589
1590        self.search(pattern, filter)
1591    }
1592
1593    /// Execute AST query on all indexed files (no trigram filtering)
1594    ///
1595    /// WARNING: This method scans the entire codebase (500ms-2s+).
1596    /// In 95% of cases, use --symbols instead which is 10-100x faster.
1597    ///
1598    /// # Algorithm
1599    /// 1. Get all indexed files for the specified language
1600    /// 2. Apply glob/exclude filters to reduce file set
1601    /// 3. Load file contents for all matching files
1602    /// 4. Execute AST query pattern using Tree-sitter
1603    /// 5. Apply remaining filters and return results
1604    ///
1605    /// # Performance
1606    /// - Parses entire codebase (not just trigram candidates)
1607    /// - Expected: 500ms-2s for medium codebases, 2-10s for large codebases
1608    /// - Use --glob to limit scope for better performance
1609    ///
1610    /// # Requirements
1611    /// - Language must be specified (AST queries are language-specific)
1612    /// - AST pattern must be valid S-expression syntax
1613    pub fn search_ast_all_files(
1614        &self,
1615        ast_pattern: &str,
1616        filter: QueryFilter,
1617    ) -> Result<Vec<SearchResult>> {
1618        log::info!(
1619            "Executing AST query on all files: pattern='{}', filter={:?}",
1620            ast_pattern,
1621            filter
1622        );
1623
1624        // Require language for AST queries
1625        let lang = filter.language.ok_or_else(|| anyhow::anyhow!(
1626            "Language must be specified for AST pattern matching. Use --lang to specify the language.\n\
1627             \n\
1628             Example: rfx query \"(function_definition) @fn\" --ast --lang python"
1629        ))?;
1630
1631        // Ensure cache exists
1632        if !self.cache.exists() {
1633            return Err(crate::errors::ReflexError::IndexNotFound.into());
1634        }
1635
1636        // Show non-blocking warnings about branch state and staleness
1637        self.check_index_freshness(&filter)?;
1638
1639        // The shared content store
1640        let open = self.open_index()?;
1641        let content_reader = &open.content;
1642
1643        // Build glob matchers ONCE before file iteration (performance optimization)
1644        let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
1645
1646        let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
1647
1648        // Get all files matching the language and glob filters
1649        let mut candidates: Vec<SearchResult> = Vec::new();
1650
1651        for file_id in 0..content_reader.file_count() {
1652            let file_path = match content_reader.get_file_path(file_id as u32) {
1653                Some(p) => p,
1654                None => continue,
1655            };
1656
1657            // Detect language from file extension
1658            let detected_lang = Language::from_path(file_path);
1659
1660            // Filter by language
1661            if detected_lang != lang {
1662                continue;
1663            }
1664
1665            let file_path_str = file_path.to_string_lossy().to_string();
1666
1667            // Apply glob/exclude filters BEFORE loading content (performance optimization)
1668            let included = include_matcher
1669                .as_ref()
1670                .is_none_or(|m| m.is_match(&file_path_str));
1671            let excluded = exclude_matcher
1672                .as_ref()
1673                .is_some_and(|m| m.is_match(&file_path_str));
1674
1675            if !included || excluded {
1676                continue;
1677            }
1678
1679            // Create a dummy candidate for this file (AST query will replace it)
1680            candidates.push(SearchResult {
1681                path: file_path_str,
1682                lang: detected_lang,
1683                span: Span {
1684                    start_line: 1,
1685                    end_line: 1,
1686                },
1687                symbol: None,
1688                kind: SymbolKind::Unknown("ast_query".to_string()),
1689                preview: String::new(),
1690                dependencies: None,
1691            });
1692        }
1693
1694        log::info!(
1695            "AST query scanning {} files for language {:?}",
1696            candidates.len(),
1697            lang
1698        );
1699
1700        // BROAD QUERY DETECTION: Block large AST queries without glob restriction
1701        // Allow small codebases (<100 files) but require --glob for larger ones
1702        if !filter.force && filter.glob_patterns.is_empty() && candidates.len() >= 100 {
1703            anyhow::bail!(
1704                "Query too broad - would be expensive to execute\n\
1705                 \n\
1706                 AST query without --glob restriction will scan the ENTIRE codebase ({} files). AST queries are SLOW (500ms-10s+).\n\
1707                 \n\
1708                 This query could:\n\
1709                 • Hang for an extended period before returning results\n\
1710                 • Return thousands of results\n\
1711                 • Flood LLM context windows with excessive data\n\
1712                 • Fail entirely\n\
1713                 \n\
1714                 Suggestions to narrow the query:\n\
1715                 • Add --glob to restrict AST query to specific files: --glob 'src/**/*.rs'\n\
1716                 • Use --symbols instead (10-100x faster in 95% of cases)\n\
1717                 • Use --force to bypass this check if you need a full codebase scan\n\
1718                 \n\
1719                 To force execution anyway:\n\
1720                 rfx query \"{}\" --force --ast --lang {:?}",
1721                candidates.len(),
1722                ast_pattern,
1723                lang
1724            );
1725        }
1726
1727        if candidates.is_empty() {
1728            if !filter.suppress_output {
1729                output::warn(&format!(
1730                    "No files found for language {:?}. Check your language filter or glob patterns.",
1731                    lang
1732                ));
1733            }
1734            return Ok(Vec::new());
1735        }
1736
1737        // Execute the AST query on all candidate files
1738        // This will load file contents and parse them with tree-sitter
1739        let mut results = self.enrich_with_ast(candidates, ast_pattern, filter.language)?;
1740
1741        log::debug!("AST query found {} matches before filtering", results.len());
1742
1743        // Apply remaining filters (same as search_internal Phase 3)
1744
1745        // Apply kind filter
1746        if let Some(ref kind) = filter.kind {
1747            results.retain(|r| {
1748                if matches!(kind, SymbolKind::Function) {
1749                    matches!(r.kind, SymbolKind::Function | SymbolKind::Method)
1750                } else {
1751                    r.kind == *kind
1752                }
1753            });
1754        }
1755
1756        // Note: exact filter doesn't make sense for AST queries (pattern is S-expression, not symbol name)
1757
1758        // Expand symbol bodies if requested
1759        if filter.expand
1760            && let Ok(open) = self.open_index()
1761        {
1762            let content_reader = &open.content;
1763            {
1764                for result in &mut results {
1765                    if result.span.start_line < result.span.end_line
1766                        && let Some(file_id) = open.file_id_for(&result.path)
1767                        && let Ok(content) = content_reader.get_file_content(file_id)
1768                    {
1769                        let lines: Vec<&str> = content.lines().collect();
1770                        let start_idx = result.span.start_line.saturating_sub(1);
1771                        let end_idx = result.span.end_line.min(lines.len());
1772
1773                        if start_idx < end_idx {
1774                            let full_body = lines[start_idx..end_idx].join("\n");
1775                            // Bounded: on a minified file this body is the whole bundle.
1776                            result.preview = crate::parsers::preview::expand_preview(&full_body);
1777                        }
1778                    }
1779                }
1780            }
1781        }
1782
1783        // Deduplicate by path if paths-only mode
1784        if filter.paths_only {
1785            use std::collections::HashSet;
1786            let mut seen_paths = HashSet::new();
1787            results.retain(|r| seen_paths.insert(r.path.clone()));
1788        }
1789
1790        // Drop text-tier results when the caller asked for code only. Applied BEFORE
1791        // the total is captured, so pagination counts what the caller will actually
1792        // receive rather than what was found and then discarded.
1793        results.retain(|r| {
1794            filter::tier_admits(
1795                r.lang,
1796                filter.language,
1797                filter.exclude_text,
1798                filter.include_locks,
1799                filter.include_generated,
1800            )
1801        });
1802
1803        // Sort results deterministically
1804        results.sort_by(|a, b| {
1805            a.path
1806                .cmp(&b.path)
1807                .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1808        });
1809
1810        // Apply offset (pagination)
1811        if let Some(offset) = filter.offset {
1812            if offset < results.len() {
1813                results = results.into_iter().skip(offset).collect();
1814            } else {
1815                results.clear();
1816            }
1817        }
1818
1819        // Apply limit
1820        if let Some(limit) = filter.limit {
1821            results.truncate(limit);
1822        }
1823
1824        log::info!("AST query returned {} results", results.len());
1825
1826        // Load dependencies if requested
1827        self.load_dependencies(&mut results, filter.include_dependencies)?;
1828
1829        Ok(results)
1830    }
1831
1832    /// Search using AST pattern with separate text pattern for trigram filtering
1833    ///
1834    /// This allows efficient AST queries by:
1835    /// 1. Using text_pattern for Phase 1 trigram filtering (narrows to candidate files)
1836    /// 2. Using ast_pattern for Phase 2 AST matching (structure-aware filtering)
1837    ///
1838    /// # Example
1839    /// ```ignore
1840    /// // Find async functions: trigram search for "fn ", AST match for function_item
1841    /// engine.search_ast_with_text_filter("fn ", "(function_item (async))", filter)?;
1842    /// ```
1843    pub fn search_ast_with_text_filter(
1844        &self,
1845        text_pattern: &str,
1846        ast_pattern: &str,
1847        filter: QueryFilter,
1848    ) -> Result<Vec<SearchResult>> {
1849        log::info!(
1850            "Executing AST query with text filter: text='{}', ast='{}', filter={:?}",
1851            text_pattern,
1852            ast_pattern,
1853            filter
1854        );
1855
1856        // Ensure cache exists
1857        if !self.cache.exists() {
1858            return Err(crate::errors::ReflexError::IndexNotFound.into());
1859        }
1860
1861        // Show non-blocking warnings about branch state and staleness
1862        self.check_index_freshness(&filter)?;
1863
1864        // Start timeout timer if configured
1865        use std::time::{Duration, Instant};
1866        let start_time = Instant::now();
1867        let timeout = if filter.timeout_secs > 0 {
1868            Some(Duration::from_secs(filter.timeout_secs))
1869        } else {
1870            None
1871        };
1872
1873        // PHASE 1: Get initial candidates using text pattern (trigram search)
1874        let candidates = if filter.use_regex {
1875            self.get_regex_candidates(text_pattern, &filter, timeout.as_ref(), &start_time, None)?
1876                .0
1877        } else {
1878            self.get_trigram_candidates(text_pattern, &filter, None)?.0
1879        };
1880
1881        log::debug!("Phase 1 found {} candidate locations", candidates.len());
1882
1883        // PHASE 2: Execute AST query on candidates
1884        let mut results = self.enrich_with_ast(candidates, ast_pattern, filter.language)?;
1885
1886        log::debug!("Phase 2 AST matching found {} results", results.len());
1887
1888        // PHASE 3: Apply filters
1889        if let Some(lang) = filter.language {
1890            results.retain(|r| r.lang == lang);
1891        }
1892
1893        if let Some(ref kind) = filter.kind {
1894            results.retain(|r| {
1895                if matches!(kind, SymbolKind::Function) {
1896                    matches!(r.kind, SymbolKind::Function | SymbolKind::Method)
1897                } else {
1898                    r.kind == *kind
1899                }
1900            });
1901        }
1902
1903        if let Some(ref file_pattern) = filter.file_pattern {
1904            results.retain(|r| r.path.contains(file_pattern));
1905        }
1906
1907        // Apply glob pattern filters (same logic as in search_internal)
1908        if !filter.glob_patterns.is_empty() || !filter.exclude_patterns.is_empty() {
1909            let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
1910
1911            let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
1912
1913            results.retain(|r| {
1914                let included = include_matcher.as_ref().is_none_or(|m| m.is_match(&r.path));
1915                let excluded = exclude_matcher
1916                    .as_ref()
1917                    .is_some_and(|m| m.is_match(&r.path));
1918                included && !excluded
1919            });
1920        }
1921
1922        if filter.exact && filter.symbols_mode {
1923            results.retain(|r| r.symbol.as_deref() == Some(text_pattern));
1924        }
1925
1926        // Expand symbol bodies if requested
1927        if filter.expand
1928            && let Ok(open) = self.open_index()
1929        {
1930            let content_reader = &open.content;
1931            {
1932                for result in &mut results {
1933                    if result.span.start_line < result.span.end_line
1934                        && let Some(file_id) = open.file_id_for(&result.path)
1935                        && let Ok(content) = content_reader.get_file_content(file_id)
1936                    {
1937                        let lines: Vec<&str> = content.lines().collect();
1938                        let start_idx = result.span.start_line.saturating_sub(1);
1939                        let end_idx = result.span.end_line.min(lines.len());
1940
1941                        if start_idx < end_idx {
1942                            let full_body = lines[start_idx..end_idx].join("\n");
1943                            // Bounded: on a minified file this body is the whole bundle.
1944                            result.preview = crate::parsers::preview::expand_preview(&full_body);
1945                        }
1946                    }
1947                }
1948            }
1949        }
1950
1951        // Drop text-tier results when the caller asked for code only. Applied BEFORE
1952        // the total is captured, so pagination counts what the caller will actually
1953        // receive rather than what was found and then discarded.
1954        results.retain(|r| {
1955            filter::tier_admits(
1956                r.lang,
1957                filter.language,
1958                filter.exclude_text,
1959                filter.include_locks,
1960                filter.include_generated,
1961            )
1962        });
1963
1964        // Sort results deterministically
1965        results.sort_by(|a, b| {
1966            a.path
1967                .cmp(&b.path)
1968                .then_with(|| a.span.start_line.cmp(&b.span.start_line))
1969        });
1970
1971        // Apply offset (pagination)
1972        if let Some(offset) = filter.offset {
1973            if offset < results.len() {
1974                results = results.into_iter().skip(offset).collect();
1975            } else {
1976                results.clear();
1977            }
1978        }
1979
1980        // Apply limit
1981        if let Some(limit) = filter.limit {
1982            results.truncate(limit);
1983        }
1984
1985        log::info!("AST query returned {} results", results.len());
1986
1987        Ok(results)
1988    }
1989
1990    /// List all symbols of a specific kind
1991    pub fn list_by_kind(&self, kind: SymbolKind) -> Result<Vec<SearchResult>> {
1992        let filter = QueryFilter {
1993            kind: Some(kind),
1994            symbols_mode: true,
1995            ..Default::default()
1996        };
1997
1998        self.search("*", filter)
1999    }
2000
2001    /// Enrich text match candidates with symbol information by parsing files
2002    ///
2003    /// Takes a list of text match candidates and extracts symbol information at those locations.
2004    ///
2005    /// # Algorithm
2006    /// 1. Group candidates by file_id for efficient processing
2007    /// 2. Parse each file with tree-sitter to extract ALL symbols
2008    /// 3. Filter symbols based on matching strategy:
2009    ///    - If use_regex=true: Extract symbols whose line spans overlap with candidate locations
2010    ///    - If use_contains=true: Filter symbols by substring match on symbol name
2011    ///    - Default: Filter symbols by exact name match
2012    /// 4. Return filtered symbol results
2013    ///
2014    /// # Performance
2015    /// Only parses files that have text matches, so typically 10-100 files
2016    /// instead of the entire codebase (62K+ files).
2017    ///
2018    /// # Optimizations
2019    /// 1. Language filtering: Skips files with unsupported languages (no parsers)
2020    /// 2. Parallel processing: Uses Rayon to parse files concurrently across CPU cores
2021    fn enrich_with_symbols(
2022        &self,
2023        candidates: Vec<SearchResult>,
2024        pattern: &str,
2025        filter: &QueryFilter,
2026        keyword_query: bool,
2027    ) -> Result<Vec<SearchResult>> {
2028        use rayon::prelude::*;
2029        use std::collections::{HashMap, HashSet};
2030
2031        // The shared index handle (content store + file-id map + meta.db connection)
2032        let open = self.open_index()?;
2033        let content_reader = &open.content;
2034
2035        // Group candidates by file, filtering out unsupported languages
2036        let mut files_by_path: HashMap<String, Vec<SearchResult>> = HashMap::new();
2037        let mut skipped_unsupported = 0;
2038
2039        for candidate in candidates {
2040            // Skip files with unsupported languages (no parser available)
2041            if !candidate.lang.is_supported() {
2042                skipped_unsupported += 1;
2043                continue;
2044            }
2045
2046            files_by_path
2047                .entry(candidate.path.clone())
2048                .or_default()
2049                .push(candidate);
2050        }
2051
2052        let total_files = files_by_path.len();
2053        log::debug!(
2054            "Processing {} candidate files for symbol enrichment (skipped {} unsupported language files)",
2055            total_files,
2056            skipped_unsupported
2057        );
2058
2059        // Warn if pattern is very broad (may take time to parse all files)
2060        if total_files > 1000 && !filter.suppress_output {
2061            output::warn(&format!(
2062                "Pattern '{}' matched {} files. This may take some time to parse. Consider using a more specific pattern or adding --lang/--file filters to narrow the search.",
2063                pattern, total_files
2064            ));
2065        }
2066
2067        // Parse on the shared query pool, sized from `[performance] parallel_threads`
2068        let pool = open.pool();
2069
2070        // PHASE 2a: pre-filter — skip files where EVERY occurrence of the pattern is
2071        // inside a comment or a string literal, so tree-sitter never parses them.
2072        //
2073        // Only the candidate lines are examined: the trigram pass already verified
2074        // that those are the lines holding the pattern, so scanning the whole file
2075        // (as this did before 2.0.0, serially) found nothing more. A definition
2076        // line is always a candidate line, so a file that defines the symbol is
2077        // never skipped. Two cases keep the old outcome exactly:
2078        // * keyword queries carry dummy line-1 candidates from
2079        //   `get_all_language_files`, so there is nothing to examine;
2080        // * a regex pattern's source text never occurs literally, so the old
2081        //   `line.find(pattern)` skipped nothing.
2082        let files_to_skip: HashSet<String> = if keyword_query || filter.use_regex {
2083            HashSet::new()
2084        } else {
2085            pool.install(|| {
2086                files_by_path
2087                    .par_iter()
2088                    .filter_map(|(file_path, cands)| {
2089                        let lang = Language::from_path(std::path::Path::new(file_path));
2090                        let line_filter = crate::line_filter::get_filter(lang)?;
2091                        let file_id = open.file_id_for(file_path)?;
2092                        let content = content_reader.get_file_content(file_id).ok()?;
2093                        let all_lines: Vec<&str> = content.lines().collect();
2094
2095                        let mut wanted: Vec<usize> =
2096                            cands.iter().map(|c| c.span.start_line).collect();
2097                        wanted.sort_unstable();
2098                        wanted.dedup();
2099
2100                        let mut saw_occurrence = false;
2101                        for line_no in wanted {
2102                            let Some(line) = line_no.checked_sub(1).and_then(|i| all_lines.get(i))
2103                            else {
2104                                continue;
2105                            };
2106                            let mut search_start = 0;
2107                            while let Some(pos) = line[search_start..].find(pattern) {
2108                                saw_occurrence = true;
2109                                let at = search_start + pos;
2110                                if !line_filter.is_in_comment(line, at)
2111                                    && !line_filter.is_in_string(line, at)
2112                                {
2113                                    // In code: this file must be parsed.
2114                                    return None;
2115                                }
2116                                search_start = at + pattern.len();
2117                            }
2118                        }
2119                        // Only skip when there WAS an occurrence and none was in code.
2120                        saw_occurrence.then(|| {
2121                            log::debug!(
2122                                "Pre-filter: Skipping {} (all matches in comments/strings)",
2123                                file_path
2124                            );
2125                            file_path.clone()
2126                        })
2127                    })
2128                    .collect()
2129            })
2130        };
2131
2132        let files_to_process: Vec<String> = files_by_path
2133            .keys()
2134            .filter(|p| !files_to_skip.contains(p.as_str()))
2135            .cloned()
2136            .collect();
2137
2138        log::debug!(
2139            "Pre-filter: Skipped {} files where all matches are in comments/strings (parsing {} files)",
2140            files_to_skip.len(),
2141            files_to_process.len()
2142        );
2143
2144        // Symbol cache lookup, on the handle's shared connection.
2145        //
2146        // The branch comes from `.git/HEAD` (no subprocess); the hashes come from a
2147        // query restricted to the candidate paths (not the whole branch).
2148        let root = self.cache.workspace_root();
2149        let branch = crate::git::read_head_branch(&root).unwrap_or_else(|| "_default".to_string());
2150        let mut conn = open.meta_conn()?;
2151        let rows =
2152            crate::cache::CacheManager::branch_file_rows_on(&conn, &branch, &files_to_process)
2153                .context("Failed to load file hashes")?;
2154        log::debug!(
2155            "Loaded {} file rows for branch '{}' for symbol cache lookups",
2156            rows.len(),
2157            branch
2158        );
2159
2160        let file_lookup_tuples: Vec<(i64, String, String)> = files_to_process
2161            .iter()
2162            .filter_map(|path| {
2163                let (id, hash) = rows.get(path)?;
2164                Some((*id, hash.clone(), path.clone()))
2165            })
2166            .collect();
2167
2168        let batch_results = crate::symbol_cache::SymbolCache::batch_get_with_kind_on(
2169            &conn,
2170            &file_lookup_tuples,
2171            filter.kind.clone(),
2172        )
2173        .context("Failed to batch read symbol cache")?;
2174
2175        let id_to_path: HashMap<i64, &str> = rows
2176            .iter()
2177            .map(|(path, (id, _))| (*id, path.as_str()))
2178            .collect();
2179
2180        let mut cached_symbols: HashMap<String, Vec<SearchResult>> = HashMap::new();
2181        for (file_id, symbols) in batch_results {
2182            if let Some(file_path) = id_to_path.get(&file_id) {
2183                cached_symbols.insert(file_path.to_string(), symbols);
2184            }
2185        }
2186
2187        // Everything else — no row on this branch, or no (current-hash) cache entry.
2188        let files_needing_parse: Vec<String> = files_to_process
2189            .iter()
2190            .filter(|p| !cached_symbols.contains_key(p.as_str()))
2191            .cloned()
2192            .collect();
2193
2194        log::debug!(
2195            "Symbol cache: {} hits, {} need parsing",
2196            cached_symbols.len(),
2197            files_needing_parse.len()
2198        );
2199
2200        // Parse cache misses in parallel; cache writes are collected and committed in
2201        // ONE transaction afterwards, instead of one connection + INSERT per file
2202        // from inside the pool.
2203        /// Symbols parsed from one file, plus its `(file_id, hash)` cache key when known.
2204        type ParsedFile = (Vec<SearchResult>, Option<(i64, String)>);
2205        let parsed: Vec<ParsedFile> = pool.install(|| {
2206            files_needing_parse
2207                .par_iter()
2208                .map(|file_path| {
2209                    let Some(file_id) = open.file_id_for(file_path) else {
2210                        log::warn!("Could not find file_id for path: {}", file_path);
2211                        return (Vec::new(), None);
2212                    };
2213                    let content = match content_reader.get_file_content(file_id) {
2214                        Ok(c) => c,
2215                        Err(e) => {
2216                            log::warn!("Failed to read file {}: {}", file_path, e);
2217                            return (Vec::new(), None);
2218                        }
2219                    };
2220                    let lang = Language::from_path(std::path::Path::new(file_path));
2221                    let symbols = match ParserFactory::parse(file_path, content, lang) {
2222                        Ok(symbols) => {
2223                            log::debug!("Parsed {} symbols from {}", symbols.len(), file_path);
2224                            symbols
2225                        }
2226                        Err(e) => {
2227                            log::debug!("Failed to parse {}: {}", file_path, e);
2228                            Vec::new()
2229                        }
2230                    };
2231                    let key = rows.get(file_path).map(|(id, hash)| (*id, hash.clone()));
2232                    (symbols, key)
2233                })
2234                .collect()
2235        });
2236
2237        let mut parsed_symbols: Vec<SearchResult> = Vec::new();
2238        let mut to_cache: Vec<(i64, String, Vec<SearchResult>)> = Vec::new();
2239        for (symbols, key) in parsed {
2240            if let Some((id, hash)) = key {
2241                to_cache.push((id, hash, symbols.clone()));
2242            }
2243            parsed_symbols.extend(symbols);
2244        }
2245        // Best-effort: a failed cache write must never fail the query.
2246        if let Err(e) = crate::symbol_cache::SymbolCache::batch_set_by_id_on(&mut conn, &to_cache) {
2247            log::debug!(
2248                "Failed to cache symbols for {} files: {}",
2249                to_cache.len(),
2250                e
2251            );
2252        }
2253        drop(conn);
2254
2255        // Combine cached and parsed symbols
2256        let mut all_symbols: Vec<SearchResult> = Vec::new();
2257
2258        // Add all cached symbols
2259        for symbols in cached_symbols.values() {
2260            all_symbols.extend_from_slice(symbols);
2261        }
2262
2263        // Add all parsed symbols
2264        all_symbols.extend(parsed_symbols);
2265
2266        // KEYWORD DETECTION: Check if pattern is a language keyword (e.g., "class", "function")
2267        // If it matches a keyword AND symbols_mode is true, interpret as "list all symbols of that type"
2268        // rather than looking for a symbol literally named "class" or "function"
2269        //
2270        // IMPORTANT: Only check keywords for languages that will pass Phase 3 filtering.
2271        // If a language filter is specified, only check that language's keywords.
2272        // Otherwise, check all languages present in the symbol results.
2273        let is_keyword_query = {
2274            // Determine which language to check keywords for
2275            let lang_to_check = if let Some(lang) = filter.language {
2276                // Language filter specified - check that language only
2277                // This ensures keyword detection aligns with Phase 3 language filtering
2278                vec![lang]
2279            } else {
2280                // No language filter - check all languages that appear in the actual symbols
2281                // (not candidates, but the parsed symbols that made it through)
2282                // This handles mixed-language codebases correctly
2283                let mut langs: Vec<Language> =
2284                    all_symbols.iter().map(|s| s.lang).collect::<Vec<_>>();
2285                langs.sort_by(|a, b| format!("{:?}", a).cmp(&format!("{:?}", b))); // Deterministic ordering
2286                langs.dedup(); // Remove duplicates after sorting
2287                langs
2288            };
2289
2290            // Check if pattern matches a keyword in any of the relevant languages
2291            lang_to_check
2292                .iter()
2293                .any(|lang| ParserFactory::get_keywords(*lang).contains(&pattern))
2294        };
2295
2296        // If pattern is a keyword (like "class" or "function"), skip name-based filtering
2297        // and return all symbols (kind filtering happens in Phase 3)
2298        let filtered: Vec<SearchResult> = if is_keyword_query {
2299            log::info!(
2300                "Pattern '{}' is a language keyword - listing all symbols (kind filtering will be applied in Phase 3)",
2301                pattern
2302            );
2303            all_symbols
2304        } else if filter.use_regex {
2305            // For regex queries, candidates already matched content via regex in Phase 1.
2306            // Extract symbols whose line spans overlap with the candidate locations.
2307            // This ensures symbols are found at the locations where the regex matched.
2308
2309            // Build a map of (file_path, line_no) from candidates
2310            use std::collections::{HashMap, HashSet};
2311            let mut candidate_lines: HashMap<String, HashSet<usize>> = HashMap::new();
2312            for candidate in &files_by_path {
2313                for cand in candidate.1 {
2314                    candidate_lines
2315                        .entry(candidate.0.clone())
2316                        .or_default()
2317                        .insert(cand.span.start_line);
2318                }
2319            }
2320
2321            // Filter symbols whose spans overlap with candidate lines
2322            all_symbols
2323                .into_iter()
2324                .filter(|sym| {
2325                    if let Some(lines) = candidate_lines.get(&sym.path) {
2326                        // Check if symbol's line span overlaps with any candidate line
2327                        for line in sym.span.start_line..=sym.span.end_line {
2328                            if lines.contains(&line) {
2329                                return true;
2330                            }
2331                        }
2332                    }
2333                    false
2334                })
2335                .collect()
2336        } else if filter.use_contains {
2337            // Substring match (opt-in with --contains)
2338            all_symbols
2339                .into_iter()
2340                .filter(|sym| sym.symbol.as_deref().is_some_and(|s| s.contains(pattern)))
2341                .collect()
2342        } else {
2343            // Exact match (default)
2344            all_symbols
2345                .into_iter()
2346                .filter(|sym| sym.symbol.as_deref() == Some(pattern))
2347                .collect()
2348        };
2349
2350        log::info!(
2351            "Symbol enrichment found {} matches for pattern '{}'",
2352            filtered.len(),
2353            pattern
2354        );
2355
2356        Ok(filtered)
2357    }
2358
2359    /// Enrich text match candidates with AST pattern matching
2360    ///
2361    /// Takes a list of text match candidates and executes a Tree-sitter AST query
2362    /// on the candidate files, returning only matches that satisfy the AST pattern.
2363    ///
2364    /// # Algorithm
2365    /// 1. Extract unique file paths from candidates
2366    /// 2. Load file contents for each candidate file
2367    /// 3. Execute AST query pattern using Tree-sitter
2368    /// 4. Return AST matches
2369    ///
2370    /// # Performance
2371    /// Only parses files that have text matches, so typically 10-100 files
2372    /// instead of the entire codebase (62K+ files).
2373    ///
2374    /// # Requirements
2375    /// - Language must be specified (AST queries are language-specific)
2376    /// - AST pattern must be valid S-expression syntax
2377    fn enrich_with_ast(
2378        &self,
2379        candidates: Vec<SearchResult>,
2380        ast_pattern: &str,
2381        language: Option<Language>,
2382    ) -> Result<Vec<SearchResult>> {
2383        // Require language for AST queries
2384        let lang = language.ok_or_else(|| anyhow::anyhow!(
2385            "Language must be specified for AST pattern matching. Use --lang to specify the language."
2386        ))?;
2387
2388        // The shared index handle (content store + file-id map)
2389        let open = self.open_index()?;
2390        let content_reader = &open.content;
2391
2392        // Collect unique file paths from candidates and load their contents
2393        use std::collections::HashMap;
2394        let mut file_contents: HashMap<String, String> = HashMap::new();
2395
2396        for candidate in &candidates {
2397            if file_contents.contains_key(&candidate.path) {
2398                continue;
2399            }
2400
2401            // Find file_id for this path
2402            let file_id = match open.file_id_for(&candidate.path) {
2403                Some(id) => id,
2404                None => {
2405                    log::warn!("Could not find file_id for path: {}", candidate.path);
2406                    continue;
2407                }
2408            };
2409
2410            // Load file content
2411            let content = match content_reader.get_file_content(file_id) {
2412                Ok(c) => c,
2413                Err(e) => {
2414                    log::warn!("Failed to read file {}: {}", candidate.path, e);
2415                    continue;
2416                }
2417            };
2418
2419            file_contents.insert(candidate.path.clone(), content.to_string());
2420        }
2421
2422        log::debug!(
2423            "Executing AST query on {} candidate files with language {:?}",
2424            file_contents.len(),
2425            lang
2426        );
2427
2428        // Execute AST query using the ast_query module
2429        let results =
2430            crate::ast_query::execute_ast_query(candidates, ast_pattern, lang, &file_contents)?;
2431
2432        log::info!(
2433            "AST query found {} matches for pattern '{}'",
2434            results.len(),
2435            ast_pattern
2436        );
2437
2438        Ok(results)
2439    }
2440
2441    /// Map keyword patterns to SymbolKind for auto-inference
2442    ///
2443    /// When users search for keywords like "class" or "function" with --symbols,
2444    /// automatically infer the kind filter to return only symbols of that type.
2445    ///
2446    /// This makes keyword queries more intuitive: searching for "class" returns
2447    /// only classes, not all symbols.
2448    fn keyword_to_kind(keyword: &str) -> Option<SymbolKind> {
2449        filter::keyword_to_kind(keyword)
2450    }
2451
2452    /// Get all files matching the language filter (for keyword queries)
2453    ///
2454    /// This method bypasses trigram search and returns ALL files of the specified language.
2455    /// Used for keyword queries like "list all classes" where we need complete coverage,
2456    /// not just the first 100 candidates from a trigram search.
2457    ///
2458    /// Similar to `search_ast_all_files()` but works for symbol queries instead of AST queries.
2459    fn get_all_language_files(&self, filter: &QueryFilter) -> Result<Vec<SearchResult>> {
2460        // Language filter is optional - if not specified, scan all files
2461        // If specified, only scan files of that language
2462
2463        // The shared content store
2464        let open = self.open_index()?;
2465        let content_reader = &open.content;
2466
2467        // Build glob matchers if specified (for filtering)
2468        let include_matcher = result::build_glob_set(&filter.glob_patterns, "glob");
2469
2470        let exclude_matcher = result::build_glob_set(&filter.exclude_patterns, "exclude");
2471
2472        // Scan all files and filter by language + glob patterns
2473        let mut candidates: Vec<SearchResult> = Vec::new();
2474
2475        for file_id in 0..content_reader.file_count() {
2476            let file_path = match content_reader.get_file_path(file_id as u32) {
2477                Some(p) => p,
2478                None => continue,
2479            };
2480
2481            // Detect language from file extension
2482            let detected_lang = Language::from_path(file_path);
2483
2484            // Filter by language (if specified)
2485            if let Some(lang) = filter.language
2486                && detected_lang != lang
2487            {
2488                continue;
2489            }
2490
2491            let file_path_str = file_path.to_string_lossy().to_string();
2492
2493            // Apply glob/exclude filters
2494            let included = include_matcher
2495                .as_ref()
2496                .is_none_or(|m| m.is_match(&file_path_str));
2497            let excluded = exclude_matcher
2498                .as_ref()
2499                .is_some_and(|m| m.is_match(&file_path_str));
2500
2501            if !included || excluded {
2502                continue;
2503            }
2504
2505            // Apply file path filter if specified
2506            if let Some(ref file_pattern) = filter.file_pattern
2507                && !file_path_str.contains(file_pattern)
2508            {
2509                continue;
2510            }
2511
2512            // Create a dummy candidate for this file
2513            // Phase 2 (symbol enrichment) will parse it and extract actual symbols
2514            candidates.push(SearchResult {
2515                path: file_path_str,
2516                lang: detected_lang,
2517                span: Span {
2518                    start_line: 1,
2519                    end_line: 1,
2520                },
2521                symbol: None,
2522                kind: SymbolKind::Unknown("keyword_query".to_string()),
2523                preview: String::new(),
2524                dependencies: None,
2525            });
2526        }
2527
2528        if let Some(lang) = filter.language {
2529            log::info!(
2530                "Keyword query will scan {} {:?} files for symbol extraction",
2531                candidates.len(),
2532                lang
2533            );
2534        } else {
2535            log::info!(
2536                "Keyword query will scan {} files (all languages) for symbol extraction",
2537                candidates.len()
2538            );
2539        }
2540
2541        Ok(candidates)
2542    }
2543
2544    /// Get candidate results using trigram-based full-text search
2545    ///
2546    /// The trigram intersection yields exact candidate `(file, line)` pairs; only
2547    /// those lines are verified, in path order, in parallel, and — in list mode —
2548    /// only until `budget` results exist (see [`verify_files_streaming`]).
2549    fn get_trigram_candidates(
2550        &self,
2551        pattern: &str,
2552        filter: &QueryFilter,
2553        budget: Option<usize>,
2554    ) -> Result<(Vec<SearchResult>, CandidateStats)> {
2555        let open = self.open_index()?;
2556        let trigram_index = &open.trigrams;
2557
2558        // Patterns shorter than 3 chars have no trigrams, so the trigram index always
2559        // returns empty.  Fall back to a linear scan of the content store so that
2560        // --force (which bypasses the broad-query guard) still produces real results.
2561        if pattern.chars().count() < 3 {
2562            log::info!(
2563                "Pattern '{}' is shorter than 3 chars — trigram index cannot be used, \
2564                 falling back to linear scan",
2565                pattern
2566            );
2567            let results = self.linear_scan_candidates(pattern, filter, &open)?;
2568            let stats = CandidateStats {
2569                index_path: IndexPath::Scan,
2570                ..CandidateStats::default()
2571            };
2572            return Ok((results, stats));
2573        }
2574
2575        // Search using trigrams
2576        let candidates_started = std::time::Instant::now();
2577        let candidates = trigram_index.search_candidates(pattern);
2578        let candidates_us = candidates_started.elapsed().as_micros() as u64;
2579        log::debug!(
2580            "Found {} candidate locations from trigram search in {} us",
2581            candidates.len(),
2582            candidates_us
2583        );
2584
2585        // Group candidate lines by file. The intersection is sorted by
2586        // (file_id, line_no), so each file's lines arrive ascending.
2587        let mut files: Vec<(u32, LineSet)> = Vec::new();
2588        for loc in candidates {
2589            match files.last_mut() {
2590                Some((id, LineSet::Only(lines))) if *id == loc.file_id => lines.push(loc.line_no),
2591                _ => files.push((loc.file_id, LineSet::Only(vec![loc.line_no]))),
2592            }
2593        }
2594        log::debug!("Scanning {} files with trigram matches", files.len());
2595
2596        // One matcher for the whole query (see `LineMatcher`).
2597        let matcher = LineMatcher::new(pattern, filter)?;
2598        let file_filter = FileFilter::from_filter(filter);
2599
2600        let outcome = verify_files_streaming(
2601            &open,
2602            files,
2603            &matcher,
2604            &file_filter,
2605            filter.paths_only,
2606            budget,
2607            filter.count_only_line_search(),
2608        );
2609
2610        let stats = CandidateStats {
2611            candidates_us,
2612            substring_only: matcher.is_word_boundary().then_some(outcome.substring_only),
2613            excluded_by_default: outcome.excluded_by_default,
2614            counted: outcome.counted,
2615            exhausted: outcome.exhausted,
2616            approx_total: outcome.estimated_total,
2617            index_path: IndexPath::Trigram,
2618            warnings: Vec::new(),
2619        };
2620        Ok((outcome.results, stats))
2621    }
2622
2623    /// Linear scan fallback for patterns shorter than 3 characters.
2624    ///
2625    /// The trigram index requires 3-char n-grams; patterns like "fn" or "i" yield
2626    /// zero trigrams and therefore zero results.  This method scans every file in
2627    /// the content store directly using the same matching logic (word-boundary,
2628    /// contains, or regex) so short-pattern queries always return real results.
2629    fn linear_scan_candidates(
2630        &self,
2631        pattern: &str,
2632        filter: &QueryFilter,
2633        open: &OpenIndex,
2634    ) -> Result<Vec<SearchResult>> {
2635        use rayon::prelude::*;
2636
2637        let content_reader = &open.content;
2638        let pattern_owned = pattern.to_string();
2639        let file_count = content_reader.file_count();
2640        let matcher = LineMatcher::new(pattern, filter)?;
2641
2642        let results: Vec<SearchResult> = open.pool().install(|| {
2643            (0..file_count as u32)
2644                .collect::<Vec<_>>()
2645                .par_iter()
2646                .flat_map(|&file_id| {
2647                    let file_path = match content_reader.get_file_path(file_id) {
2648                        Some(p) => p.to_path_buf(),
2649                        None => return Vec::new(),
2650                    };
2651                    let content = match content_reader.get_file_content(file_id) {
2652                        Ok(c) => c,
2653                        Err(_) => return Vec::new(),
2654                    };
2655
2656                    let file_path_str = file_path.to_string_lossy().to_string();
2657                    let lang = Language::from_path(&file_path);
2658
2659                    let mut seen_lines = std::collections::HashSet::new();
2660                    let mut file_results = Vec::new();
2661
2662                    for (line_idx, line) in content.lines().enumerate() {
2663                        let line_no = line_idx + 1;
2664                        if seen_lines.contains(&line_no) {
2665                            continue;
2666                        }
2667
2668                        if !matcher.is_match(line) {
2669                            continue;
2670                        }
2671
2672                        seen_lines.insert(line_no);
2673                        file_results.push(SearchResult {
2674                            path: file_path_str.clone(),
2675                            lang,
2676                            kind: SymbolKind::Unknown("text_match".to_string()),
2677                            symbol: None,
2678                            span: Span {
2679                                start_line: line_no,
2680                                end_line: line_no,
2681                            },
2682                            // Bounded, and WINDOWED on the match: on a minified bundle
2683                            // this line is the whole 1.45 MB file, and the first 512
2684                            // bytes of it would tell the caller nothing.
2685                            preview: crate::parsers::preview::line_preview(
2686                                line,
2687                                line.find(pattern_owned.as_str()).unwrap_or(0),
2688                            ),
2689                            dependencies: None,
2690                        });
2691                    }
2692
2693                    file_results
2694                })
2695                .collect()
2696        });
2697
2698        log::info!(
2699            "Linear scan (short pattern '{}') found {} results across {} files",
2700            pattern,
2701            results.len(),
2702            file_count
2703        );
2704        Ok(results)
2705    }
2706
2707    /// Get candidate results using regex patterns with trigram optimization
2708    ///
2709    /// # Algorithm
2710    ///
2711    /// 1. Extract literal sequences (≥3 chars) from the regex pattern
2712    /// 2. For each literal, take the exact candidate `(file, line)` pairs from the
2713    ///    trigram index and UNION them
2714    /// 3. Verify the regex on those lines only, in path order, in parallel, with the
2715    ///    same early termination as literal search
2716    /// 4. With no usable literal (none ≥3 chars, or a case-insensitive flag), verify
2717    ///    every line of every file
2718    ///
2719    /// # Why candidate lines are sufficient
2720    ///
2721    /// Matching is per line, and every literal the extractor emits must appear
2722    /// verbatim in any match (the extractor drops the atom before a `?`, `*` or
2723    /// `{0,n}`). A matching line therefore contains at least one emitted literal
2724    /// in full, and the union over literals of their exact candidate lines is a
2725    /// superset of the matching lines. This is the same assumption the literal
2726    /// path makes; a capped posting list loses a file in both paths alike.
2727    fn get_regex_candidates(
2728        &self,
2729        pattern: &str,
2730        filter: &QueryFilter,
2731        timeout: Option<&std::time::Duration>,
2732        start_time: &std::time::Instant,
2733        budget: Option<usize>,
2734    ) -> Result<(Vec<SearchResult>, CandidateStats)> {
2735        // Step 1: Compile the regex (the filter carries the `kind` label)
2736        let matcher = LineMatcher::new(pattern, filter)
2737            .with_context(|| format!("Invalid regex pattern: {}", pattern))?;
2738
2739        // Check timeout before expensive operations
2740        if let Some(timeout_duration) = timeout
2741            && start_time.elapsed() > *timeout_duration
2742        {
2743            anyhow::bail!(
2744                "Query timeout exceeded ({} seconds) during regex compilation",
2745                timeout_duration.as_secs()
2746            );
2747        }
2748
2749        // Step 2: Extract literals from the regex. A literal under `(?i)` is looked
2750        // up under every ASCII case variant; a non-ASCII one cannot be folded.
2751        use crate::regex_trigrams::extract_literals;
2752        let literals = extract_literals(pattern);
2753        let unfoldable = literals
2754            .iter()
2755            .any(|l| l.case_insensitive && !l.text.is_ascii());
2756
2757        let open = self.open_index()?;
2758        let candidates_started = std::time::Instant::now();
2759
2760        let mut warnings = Vec::new();
2761        let index_path;
2762        let files: Vec<(u32, LineSet)> = if literals.is_empty() || unfoldable {
2763            // No usable literal - fall back to a full scan of every line.
2764            index_path = IndexPath::Scan;
2765            // A literal shorter than 3 chars scans silently on the case-sensitive
2766            // path too (`get_trigram_candidates`); `-i fn` must not warn either.
2767            let short_literal = filter
2768                .rewritten_from
2769                .as_deref()
2770                .is_some_and(|p| p.chars().count() < 3);
2771            if !short_literal {
2772                let text = if unfoldable {
2773                    format!(
2774                        "Regex pattern '{}' has a non-ASCII case-insensitive literal, which the trigram index cannot fold; falling back to full content scan. This may be slow on large codebases.",
2775                        pattern
2776                    )
2777                } else {
2778                    format!(
2779                        "Regex pattern '{}' has no literals (≥3 chars), falling back to full content scan. This may be slow on large codebases. Consider using patterns with literal text.",
2780                        pattern
2781                    )
2782                };
2783                if !filter.suppress_output {
2784                    output::warn(&text);
2785                }
2786                warnings.push(text);
2787            }
2788            (0..open.content.file_count() as u32)
2789                .map(|id| (id, LineSet::All))
2790                .collect()
2791        } else {
2792            index_path = IndexPath::Trigram;
2793            log::debug!(
2794                "Using {} literals to narrow regex search candidates",
2795                literals.len()
2796            );
2797
2798            // Union of each literal's exact candidate lines (alternation-safe).
2799            // Each source is sorted by (file, line) and key-unique, so a single
2800            // source needs no sort; several are merged with one sort + dedup. A
2801            // per-file BTreeMap here cost ~8 ms per 100k locations.
2802            let mut locations: Vec<crate::trigram::FileLocation> = Vec::new();
2803            let mut sources = 0usize;
2804            // Under Unicode case folding `k` and `s` also match the Kelvin sign
2805            // and the long s, which live on lines the ASCII fold cannot reach.
2806            let mut need_exotic = false;
2807            for literal in &literals {
2808                let found = if literal.case_insensitive {
2809                    if literal
2810                        .text
2811                        .bytes()
2812                        .any(|b| matches!(b.to_ascii_lowercase(), b'k' | b's'))
2813                    {
2814                        need_exotic = true;
2815                    }
2816                    open.trigrams
2817                        .search_candidates_fold(literal.text.as_bytes())
2818                } else {
2819                    open.trigrams.search_candidates(&literal.text)
2820                };
2821                log::debug!(
2822                    "Literal '{}' (ci={}) found on {} candidate lines",
2823                    literal.text,
2824                    literal.case_insensitive,
2825                    found.len()
2826                );
2827                locations.extend(found);
2828                sources += 1;
2829            }
2830            if need_exotic {
2831                locations.extend(open.trigrams.exotic_fold_lines());
2832                sources += 1;
2833            }
2834            if sources > 1 {
2835                locations.sort_unstable();
2836                locations.dedup();
2837            }
2838            // Group by file; lines arrive ascending within each file.
2839            let mut files: Vec<(u32, LineSet)> = Vec::new();
2840            for loc in locations {
2841                match files.last_mut() {
2842                    Some((id, LineSet::Only(lines))) if *id == loc.file_id => {
2843                        lines.push(loc.line_no)
2844                    }
2845                    _ => files.push((loc.file_id, LineSet::Only(vec![loc.line_no]))),
2846                }
2847            }
2848            files
2849        };
2850        let candidates_us = candidates_started.elapsed().as_micros() as u64;
2851        log::debug!(
2852            "Regex candidates: {} files in {} us",
2853            files.len(),
2854            candidates_us
2855        );
2856
2857        let file_filter = FileFilter::from_filter(filter);
2858        let outcome = verify_files_streaming(
2859            &open,
2860            files,
2861            &matcher,
2862            &file_filter,
2863            filter.paths_only,
2864            budget,
2865            filter.count_only_line_search(),
2866        );
2867
2868        log::info!(
2869            "Regex search found {} matches for pattern '{}'",
2870            outcome.results.len(),
2871            pattern
2872        );
2873        let stats = CandidateStats {
2874            candidates_us,
2875            substring_only: None,
2876            excluded_by_default: outcome.excluded_by_default,
2877            counted: outcome.counted,
2878            exhausted: outcome.exhausted,
2879            approx_total: outcome.estimated_total,
2880            index_path,
2881            warnings,
2882        };
2883        Ok((outcome.results, stats))
2884    }
2885
2886    /// Get index status for programmatic use (doesn't print warnings)
2887    ///
2888    /// Returns (status, can_trust_results, warning) tuple for JSON output.
2889    /// This is optimized for AI agents to detect staleness and auto-reindex.
2890    pub fn get_index_status(&self) -> Result<(IndexStatus, bool, Option<IndexWarning>)> {
2891        self.index_status_for(None)
2892    }
2893
2894    /// Index status read from the filesystem, never from the freshness memo.
2895    ///
2896    /// For `check_index_status`: an agent asking whether the index is current is
2897    /// exactly the caller that must not be told what was true a second ago.
2898    pub fn fresh_index_status(&self) -> Result<(IndexStatus, bool, Option<IndexWarning>)> {
2899        let r = self.fresh_index_report()?;
2900        Ok((r.status, r.can_trust_results, r.warning))
2901    }
2902
2903    /// [`Self::fresh_index_status`] with the branch/commit details attached, which
2904    /// are present even when fresh.
2905    pub fn fresh_index_report(&self) -> Result<IndexStatusReport> {
2906        status_cache::invalidate(&self.cache.workspace_root());
2907        self.index_report_for(None)
2908    }
2909
2910    /// Index status, with `scope` naming the files a caller's answer came from.
2911    ///
2912    /// `scope` does NOT soften `can_trust_results`. A stale index always yields
2913    /// `can_trust_results: false`, because the changes that matter most are the ones
2914    /// scoping cannot see: a newly created file produces no results to intersect
2915    /// with, and an empty result set has no scope at all — which is exactly the case
2916    /// where an agent concludes "no callers" and acts on it.
2917    ///
2918    /// Scope is used only to describe the impact, via
2919    /// [`IndexWarning::reason`], so a caller can tell "your results may be missing a
2920    /// file you just edited" from "a file in these results has changed".
2921    pub fn index_status_for(
2922        &self,
2923        scope: Option<&[String]>,
2924    ) -> Result<(IndexStatus, bool, Option<IndexWarning>)> {
2925        let r = self.index_report_for(scope)?;
2926        Ok((r.status, r.can_trust_results, r.warning))
2927    }
2928
2929    /// [`Self::index_status_for`] with details.
2930    pub fn index_report_for(&self, scope: Option<&[String]>) -> Result<IndexStatusReport> {
2931        // Everything that costs a subprocess or a database open is computed once per
2932        // TTL in `status_cache`; only the scope-dependent wording is built here.
2933        let snapshot = status_cache::snapshot(&self.cache)?;
2934        Ok(Self::report_from(&snapshot, scope))
2935    }
2936
2937    /// The verdict for one caller, from a snapshot already computed.
2938    ///
2939    /// Pure, so a caller can compute the snapshot on another thread while the
2940    /// search runs and derive the wording once both are done.
2941    fn report_from(
2942        snapshot: &status_cache::Snapshot,
2943        scope: Option<&[String]>,
2944    ) -> IndexStatusReport {
2945        if let Some((status, can_trust_results, warning)) = snapshot.decided.clone() {
2946            return IndexStatusReport {
2947                status,
2948                can_trust_results,
2949                warning,
2950                details: snapshot.details.clone(),
2951            };
2952        }
2953        let changes = &snapshot.changes;
2954        let details = snapshot.details.clone();
2955
2956        if changes.is_empty() {
2957            return IndexStatusReport {
2958                status: IndexStatus::Fresh,
2959                can_trust_results: true,
2960                warning: None,
2961                details,
2962            };
2963        }
2964
2965        // Whether a changed file is among the ones this answer came from. Used only
2966        // to sharpen the message — never to upgrade trust.
2967        let hits_results = match scope {
2968            None | Some([]) => false,
2969            Some(paths) => changes.any(|changed| {
2970                paths
2971                    .iter()
2972                    .any(|p| p == changed || p.ends_with(changed) || changed.ends_with(p.as_str()))
2973            }),
2974        };
2975
2976        let reason = {
2977            let mut parts = Vec::new();
2978            if changes.modified_count > 0 {
2979                parts.push(format!("{} modified", changes.modified_count));
2980            }
2981            if changes.added_count > 0 {
2982                parts.push(format!("{} added", changes.added_count));
2983            }
2984            if changes.deleted_count > 0 {
2985                parts.push(format!("{} deleted", changes.deleted_count));
2986            }
2987            let impact = if hits_results {
2988                " — including a file these results came from"
2989            } else if changes.deleted_count > 0 {
2990                " — deleted files still produce hits at their old lines"
2991            } else {
2992                " — these results may not reflect them"
2993            };
2994            format!(
2995                "Files changed since the index was built ({}){}",
2996                parts.join(", "),
2997                impact
2998            )
2999        };
3000
3001        let some_if_any = |v: Vec<String>| if v.is_empty() { None } else { Some(v) };
3002        let warning = IndexWarning {
3003            reason,
3004            action_required: "index_project".to_string(),
3005            files_modified: some_if_any(changes.modified.clone()),
3006            files_added: some_if_any(changes.added.clone()),
3007            files_deleted: some_if_any(changes.deleted.clone()),
3008            changed_count: Some(
3009                changes.modified_count + changes.added_count + changes.deleted_count,
3010            ),
3011            truncated: changes.truncated,
3012            details: details.clone(),
3013        };
3014
3015        // Stale means untrusted, with no exception. A search served from an index
3016        // that does not know about the caller's own edits cannot promise completeness,
3017        // and a silently-confident wrong answer is the failure this release fixes.
3018        IndexStatusReport {
3019            status: IndexStatus::Stale,
3020            can_trust_results: false,
3021            warning: Some(warning),
3022            details,
3023        }
3024    }
3025
3026    /// Check index freshness and show non-blocking warnings
3027    ///
3028    /// This performs lightweight checks to warn users if their index might be stale:
3029    /// 1. Branch mismatch: indexed different branch
3030    /// 2. Commit changed: HEAD moved since indexing
3031    /// 3. File changes: quick mtime check on sample of files (if available)
3032    fn check_index_freshness(&self, filter: &QueryFilter) -> Result<()> {
3033        let root = self.cache.workspace_root();
3034
3035        // Check git state if in a git repo
3036        if crate::git::is_git_repo(&root) {
3037            if !crate::git::is_git_available() {
3038                static WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
3039                if !filter.suppress_output {
3040                    WARNED.get_or_init(|| {
3041                        output::warn("⚠️  git binary not found in PATH; index freshness checks disabled for this session.");
3042                    });
3043                }
3044                return Ok(());
3045            }
3046            if let Ok(current_branch) = crate::git::get_current_branch(&root) {
3047                // Check if we're on a different branch than what was indexed
3048                if !self.cache.branch_exists(&current_branch).unwrap_or(false) {
3049                    if !filter.suppress_output {
3050                        output::warn(&format!(
3051                            "⚠️  WARNING: Index not found for branch '{}'. Run 'rfx index' to index this branch.",
3052                            current_branch
3053                        ));
3054                    }
3055                    return Ok(());
3056                }
3057
3058                // Branch exists - check if commit changed
3059                if let (Ok(current_commit), Ok(branch_info)) = (
3060                    crate::git::get_current_commit(&root),
3061                    self.cache.get_branch_info(&current_branch),
3062                ) {
3063                    if branch_info.commit_sha != current_commit {
3064                        if !filter.suppress_output {
3065                            output::warn(&format!(
3066                                "⚠️  WARNING: Index may be stale (commit changed: {} → {}). Consider running 'rfx index'.",
3067                                &branch_info.commit_sha[..7],
3068                                &current_commit[..7]
3069                            ));
3070                        }
3071                        return Ok(());
3072                    }
3073
3074                    // If commits match, do a quick file freshness check
3075                    // Sample up to 10 files to check for modifications (cheap mtime check)
3076                    if let Ok(branch_files) = self.cache.get_branch_files(&current_branch) {
3077                        let mut checked = 0;
3078                        let mut changed = 0;
3079                        const SAMPLE_SIZE: usize = 10;
3080
3081                        for (path, _indexed_hash) in branch_files.iter().take(SAMPLE_SIZE) {
3082                            checked += 1;
3083                            let file_path = std::path::Path::new(path);
3084
3085                            // Check if file exists and has been modified (mtime/size heuristic)
3086                            if let Ok(metadata) = std::fs::metadata(file_path)
3087                                && let Ok(modified) = metadata.modified()
3088                            {
3089                                let indexed_time = branch_info.last_indexed;
3090                                let file_time = modified
3091                                    .duration_since(std::time::UNIX_EPOCH)
3092                                    .unwrap_or_default()
3093                                    .as_secs()
3094                                    as i64;
3095
3096                                // If file modified after indexing, it might be stale
3097                                if file_time > indexed_time {
3098                                    // File modified after indexing - likely stale
3099                                    // Note: We skip hash verification for performance (mtime check is sufficient)
3100                                    // This may cause false positives if files were touched without changes,
3101                                    // but the warning is non-blocking and vastly better than slow queries
3102                                    changed += 1;
3103                                }
3104                            }
3105                        }
3106
3107                        if changed > 0 && !filter.suppress_output {
3108                            output::warn(&format!(
3109                                "⚠️  WARNING: {} of {} sampled files changed since indexing. Consider running 'rfx index'.",
3110                                changed, checked
3111                            ));
3112                        }
3113                    }
3114                }
3115            }
3116        }
3117
3118        Ok(())
3119    }
3120}
3121
3122/// Generate AI instruction based on query results
3123///
3124/// Provides context-aware guidance to AI agents on how to handle search results.
3125/// Uses priority-based logic to determine the most relevant instruction.
3126#[allow(clippy::too_many_arguments)]
3127pub fn generate_ai_instruction(
3128    result_count: usize,
3129    total_count: usize,
3130    has_more: bool,
3131    symbols_mode: bool,
3132    paths_only: bool,
3133    use_ast: bool,
3134    use_regex: bool,
3135    language_filter: bool,
3136    glob_filter: bool,
3137    exact_mode: bool,
3138) -> Option<String> {
3139    // Priority 1: No results
3140    if result_count == 0 {
3141        return Some(
3142            "No results found. Consider these alternatives: 1) Check pattern spelling, 2) Remove --kind or --lang filters to broaden search, 3) Try partial match or related term, 4) Use search_regex tool for pattern matching with special characters or complex patterns."
3143            .to_string()
3144        );
3145    }
3146
3147    // Priority 2: Query too broad (500+ results)
3148    if total_count >= 500 {
3149        return Some(format!(
3150            "Query too broad: {} results found. STOP. Do not list results. Refine search automatically by adding filters: kind parameter (Function/Struct/Class), lang parameter (rust/python/etc), or glob parameter (['src/**/*.rs']). Call search_code again with appropriate filters.",
3151            total_count
3152        ));
3153    }
3154
3155    // Priority 3: Paginated results
3156    //
3157    // REF-191: for autonomous find-all tasks this instruction must NOT tell the
3158    // agent to stop and ask the user — there is no user in an agent loop, and a
3159    // partial answer to "find every occurrence" is wrong. Instruct decisive
3160    // continuation: fetch the remaining page(s) via offset, or probe the total
3161    // cheaply with mode="count" first.
3162    if has_more {
3163        return Some(format!(
3164            "Showing {} of {} results — {} more available. This is a partial answer. To finish a find-all task, call again with offset={} (raise limit up to 500 to get the rest in one call), or use mode=\"count\" first if you only need the total.",
3165            result_count,
3166            total_count,
3167            total_count.saturating_sub(result_count),
3168            result_count
3169        ));
3170    }
3171
3172    // Priority 4: Single precise result (symbols mode)
3173    if result_count == 1 && symbols_mode {
3174        return Some(
3175            "Found 1 precise result. Respond concisely: '[symbol] at [path]:[line]'.".to_string(),
3176        );
3177    }
3178
3179    // Priority 5: Few precise results (symbols mode)
3180    if (2..=10).contains(&result_count) && symbols_mode {
3181        return Some(format!(
3182            "Found {} precise results (definitions only, not usages). List locations concisely: '[symbol] at [path]:[line]' for each result.",
3183            result_count
3184        ));
3185    }
3186
3187    // Priority 6: Many results (101-500)
3188    if (101..500).contains(&total_count) {
3189        return Some(format!(
3190            "Found {} results - this is broad. Suggest refining search with: kind parameter (Function/Struct/Class/etc), lang parameter (rust/python/etc), or glob parameter to narrow file scope.",
3191            total_count
3192        ));
3193    }
3194
3195    // Priority 7: Full-text mode with many results (suggest symbols mode)
3196    if result_count >= 100 && !symbols_mode {
3197        return Some(format!(
3198            "Found {} results in full-text search mode (includes definitions AND all usages). Consider using symbols=true parameter to filter to definitions only. This typically reduces results by 80-90%.",
3199            result_count
3200        ));
3201    }
3202
3203    // Priority 8: Paths-only mode
3204    if paths_only {
3205        return Some(format!(
3206            "Found {} unique files (paths-only mode - no code content included). Next step: Use Read tool on specific files that look relevant based on their paths.",
3207            result_count
3208        ));
3209    }
3210
3211    // Priority 9: AST query results
3212    if use_ast {
3213        return Some(format!(
3214            "Found {} results using AST pattern matching. These are structure-based matches using Tree-sitter patterns, not text search.",
3215            result_count
3216        ));
3217    }
3218
3219    // Priority 10: Regex with many results
3220    if use_regex && result_count >= 100 {
3221        return Some(format!(
3222            "Found {} results using regex pattern matching. Regex matches are expansive. Consider using exact text search or symbols mode for more precise results.",
3223            result_count
3224        ));
3225    }
3226
3227    // Priority 11: Language filter with few results
3228    if language_filter && result_count <= 5 {
3229        return Some(format!(
3230            "Found {} results with language filter active. Results are limited to this language only. Remove lang parameter if you want to search all languages.",
3231            result_count
3232        ));
3233    }
3234
3235    // Priority 12: Glob filter with few results
3236    if glob_filter && result_count <= 10 {
3237        return Some(format!(
3238            "Found {} results with glob filter active. Results are limited to matching paths. Remove glob parameter to search entire codebase.",
3239            result_count
3240        ));
3241    }
3242
3243    // Priority 13: Exact mode with few results
3244    if exact_mode && result_count <= 5 {
3245        return Some(format!(
3246            "Found {} results in exact match mode. Only exact symbol name matches are included. Remove exact parameter to allow substring matching.",
3247            result_count
3248        ));
3249    }
3250
3251    // Normal case (11-100 results, no special conditions) - no instruction
3252    None
3253}
3254
3255#[cfg(test)]
3256mod tests {
3257    use super::*;
3258    use crate::indexer::Indexer;
3259    use crate::models::IndexConfig;
3260    use std::fs;
3261    use tempfile::TempDir;
3262
3263    // ==================== Basic Tests ====================
3264
3265    #[test]
3266    fn test_query_engine_creation() {
3267        let temp = TempDir::new().unwrap();
3268        let cache = CacheManager::new(temp.path());
3269        let engine = QueryEngine::new(cache);
3270
3271        assert!(engine.cache.path().ends_with(".reflex"));
3272    }
3273
3274    #[test]
3275    fn test_filter_modes() {
3276        // Test that symbols_mode works as expected
3277        let filter_fulltext = QueryFilter::default();
3278        assert!(!filter_fulltext.symbols_mode);
3279
3280        let filter_symbols = QueryFilter {
3281            symbols_mode: true,
3282            ..Default::default()
3283        };
3284        assert!(filter_symbols.symbols_mode);
3285
3286        // Test that kind implies symbols_mode (handled in CLI layer)
3287        let filter_with_kind = QueryFilter {
3288            kind: Some(SymbolKind::Function),
3289            symbols_mode: true,
3290            ..Default::default()
3291        };
3292        assert!(filter_with_kind.symbols_mode);
3293    }
3294
3295    // ==================== Search Mode Tests ====================
3296
3297    #[test]
3298    fn test_fulltext_search() {
3299        let temp = TempDir::new().unwrap();
3300        let project = temp.path().join("project");
3301        fs::create_dir(&project).unwrap();
3302
3303        // Create test files
3304        fs::write(
3305            project.join("main.rs"),
3306            "fn main() {\n    println!(\"hello\");\n}",
3307        )
3308        .unwrap();
3309        fs::write(project.join("lib.rs"), "pub fn hello() {}").unwrap();
3310
3311        // Index the project
3312        let cache = CacheManager::new(&project);
3313        let indexer = Indexer::new(cache, IndexConfig::default());
3314        indexer.index(&project, false).unwrap();
3315
3316        // Search for "hello"
3317        let cache = CacheManager::new(&project);
3318        let engine = QueryEngine::new(cache);
3319        let filter = QueryFilter::default(); // full-text mode
3320        let results = engine.search("hello", filter).unwrap();
3321
3322        // Should find both occurrences (println and function name)
3323        assert!(results.len() >= 2);
3324        assert!(results.iter().any(|r| r.path.contains("main.rs")));
3325        assert!(results.iter().any(|r| r.path.contains("lib.rs")));
3326    }
3327
3328    #[test]
3329    fn test_symbol_search() {
3330        let temp = TempDir::new().unwrap();
3331        let project = temp.path().join("project");
3332        fs::create_dir(&project).unwrap();
3333
3334        // Create test file with function definition and call
3335        fs::write(
3336            project.join("main.rs"),
3337            "fn greet() {}\nfn main() {\n    greet();\n}",
3338        )
3339        .unwrap();
3340
3341        // Index
3342        let cache = CacheManager::new(&project);
3343        let indexer = Indexer::new(cache, IndexConfig::default());
3344        indexer.index(&project, false).unwrap();
3345
3346        let cache = CacheManager::new(&project);
3347
3348        // Symbol search (definitions only)
3349        let engine = QueryEngine::new(cache);
3350        let filter = QueryFilter {
3351            symbols_mode: true,
3352            ..Default::default()
3353        };
3354        let results = engine.search("greet", filter).unwrap();
3355
3356        // Should find only the definition, not the call
3357        assert!(!results.is_empty());
3358        assert!(results.iter().any(|r| r.kind == SymbolKind::Function));
3359    }
3360
3361    #[test]
3362    fn test_regex_search() {
3363        let temp = TempDir::new().unwrap();
3364        let project = temp.path().join("project");
3365        fs::create_dir(&project).unwrap();
3366
3367        fs::write(
3368            project.join("main.rs"),
3369            "fn test1() {}\nfn test2() {}\nfn other() {}",
3370        )
3371        .unwrap();
3372
3373        let cache = CacheManager::new(&project);
3374        let indexer = Indexer::new(cache, IndexConfig::default());
3375        indexer.index(&project, false).unwrap();
3376
3377        let cache = CacheManager::new(&project);
3378
3379        let engine = QueryEngine::new(cache);
3380        let filter = QueryFilter {
3381            use_regex: true,
3382            ..Default::default()
3383        };
3384        let results = engine.search(r"fn test\d", filter).unwrap();
3385
3386        // Should match test1 and test2 but not other
3387        assert_eq!(results.len(), 2);
3388        assert!(results.iter().all(|r| r.preview.contains("test")));
3389    }
3390
3391    // ==================== Filter Tests ====================
3392
3393    #[test]
3394    fn test_language_filter() {
3395        let temp = TempDir::new().unwrap();
3396        let project = temp.path().join("project");
3397        fs::create_dir(&project).unwrap();
3398
3399        fs::write(project.join("main.rs"), "fn main() {}").unwrap();
3400        fs::write(project.join("main.js"), "function main() {}").unwrap();
3401
3402        let cache = CacheManager::new(&project);
3403        let indexer = Indexer::new(cache, IndexConfig::default());
3404        indexer.index(&project, false).unwrap();
3405
3406        let cache = CacheManager::new(&project);
3407
3408        let engine = QueryEngine::new(cache);
3409
3410        // Filter to Rust only
3411        let filter = QueryFilter {
3412            language: Some(Language::Rust),
3413            ..Default::default()
3414        };
3415        let results = engine.search("main", filter).unwrap();
3416
3417        assert!(results.iter().all(|r| r.lang == Language::Rust));
3418        assert!(results.iter().all(|r| r.path.ends_with(".rs")));
3419    }
3420
3421    #[test]
3422    fn test_kind_filter() {
3423        let temp = TempDir::new().unwrap();
3424        let project = temp.path().join("project");
3425        fs::create_dir(&project).unwrap();
3426
3427        fs::write(
3428            project.join("main.rs"),
3429            "struct Point {}\nfn main() {}\nimpl Point { fn new() {} }",
3430        )
3431        .unwrap();
3432
3433        let cache = CacheManager::new(&project);
3434        let indexer = Indexer::new(cache, IndexConfig::default());
3435        indexer.index(&project, false).unwrap();
3436
3437        let cache = CacheManager::new(&project);
3438
3439        let engine = QueryEngine::new(cache);
3440
3441        // Filter to functions only (includes methods)
3442        let filter = QueryFilter {
3443            symbols_mode: true,
3444            kind: Some(SymbolKind::Function),
3445            use_contains: true, // "mai" is substring of "main"
3446            ..Default::default()
3447        };
3448        // Search for "mai" which should match "main" (tri gram pattern will def be in index)
3449        let results = engine.search("mai", filter).unwrap();
3450
3451        // Should find main function
3452        assert!(!results.is_empty(), "Should find at least one result");
3453        assert!(
3454            results.iter().any(|r| r.symbol.as_deref() == Some("main")),
3455            "Should find 'main' function"
3456        );
3457    }
3458
3459    #[test]
3460    fn test_file_pattern_filter() {
3461        let temp = TempDir::new().unwrap();
3462        let project = temp.path().join("project");
3463        fs::create_dir_all(project.join("src")).unwrap();
3464        fs::create_dir_all(project.join("tests")).unwrap();
3465
3466        fs::write(project.join("src/lib.rs"), "fn foo() {}").unwrap();
3467        fs::write(project.join("tests/test.rs"), "fn foo() {}").unwrap();
3468
3469        let cache = CacheManager::new(&project);
3470        let indexer = Indexer::new(cache, IndexConfig::default());
3471        indexer.index(&project, false).unwrap();
3472
3473        let cache = CacheManager::new(&project);
3474
3475        let engine = QueryEngine::new(cache);
3476
3477        // Filter to src/ only
3478        let filter = QueryFilter {
3479            file_pattern: Some("src/".to_string()),
3480            ..Default::default()
3481        };
3482        let results = engine.search("foo", filter).unwrap();
3483
3484        assert!(results.iter().all(|r| r.path.contains("src/")));
3485        assert!(!results.iter().any(|r| r.path.contains("tests/")));
3486    }
3487
3488    #[test]
3489    fn test_limit_filter() {
3490        let temp = TempDir::new().unwrap();
3491        let project = temp.path().join("project");
3492        fs::create_dir(&project).unwrap();
3493
3494        // Create file with many matches
3495        let content = (0..20)
3496            .map(|i| format!("fn test{}() {{}}", i))
3497            .collect::<Vec<_>>()
3498            .join("\n");
3499        fs::write(project.join("main.rs"), content).unwrap();
3500
3501        let cache = CacheManager::new(&project);
3502        let indexer = Indexer::new(cache, IndexConfig::default());
3503        indexer.index(&project, false).unwrap();
3504
3505        let cache = CacheManager::new(&project);
3506
3507        let engine = QueryEngine::new(cache);
3508
3509        // Limit to 5 results
3510        let filter = QueryFilter {
3511            limit: Some(5),
3512            use_contains: true, // "test" is substring of "test0", "test1", etc.
3513            ..Default::default()
3514        };
3515        let results = engine.search("test", filter).unwrap();
3516
3517        assert_eq!(results.len(), 5);
3518    }
3519
3520    #[test]
3521    fn test_exact_match_filter() {
3522        let temp = TempDir::new().unwrap();
3523        let project = temp.path().join("project");
3524        fs::create_dir(&project).unwrap();
3525
3526        fs::write(
3527            project.join("main.rs"),
3528            "fn test() {}\nfn test_helper() {}\nfn other_test() {}",
3529        )
3530        .unwrap();
3531
3532        let cache = CacheManager::new(&project);
3533        let indexer = Indexer::new(cache, IndexConfig::default());
3534        indexer.index(&project, false).unwrap();
3535
3536        let cache = CacheManager::new(&project);
3537
3538        let engine = QueryEngine::new(cache);
3539
3540        // Exact match for "test"
3541        let filter = QueryFilter {
3542            symbols_mode: true,
3543            exact: true,
3544            ..Default::default()
3545        };
3546        let results = engine.search("test", filter).unwrap();
3547
3548        // Should only match exactly "test", not "test_helper" or "other_test"
3549        assert_eq!(results.len(), 1);
3550        assert_eq!(results[0].symbol.as_deref(), Some("test"));
3551    }
3552
3553    // ==================== Expand Mode Tests ====================
3554
3555    #[test]
3556    fn test_expand_mode() {
3557        let temp = TempDir::new().unwrap();
3558        let project = temp.path().join("project");
3559        fs::create_dir(&project).unwrap();
3560
3561        fs::write(
3562            project.join("main.rs"),
3563            "fn greet() {\n    println!(\"Hello\");\n    println!(\"World\");\n}",
3564        )
3565        .unwrap();
3566
3567        let cache = CacheManager::new(&project);
3568        let indexer = Indexer::new(cache, IndexConfig::default());
3569        indexer.index(&project, false).unwrap();
3570
3571        let cache = CacheManager::new(&project);
3572
3573        let engine = QueryEngine::new(cache);
3574
3575        // Search with expand mode
3576        let filter = QueryFilter {
3577            symbols_mode: true,
3578            expand: true,
3579            ..Default::default()
3580        };
3581        let results = engine.search("greet", filter).unwrap();
3582
3583        // Should have full function body in preview
3584        assert!(!results.is_empty());
3585        let result = &results[0];
3586        assert!(result.preview.contains("println"));
3587    }
3588
3589    // ==================== Edge Cases ====================
3590
3591    #[test]
3592    fn test_search_empty_index() {
3593        let temp = TempDir::new().unwrap();
3594        let project = temp.path().join("project");
3595        fs::create_dir(&project).unwrap();
3596
3597        let cache = CacheManager::new(&project);
3598        let indexer = Indexer::new(cache, IndexConfig::default());
3599        indexer.index(&project, false).unwrap();
3600
3601        let cache = CacheManager::new(&project);
3602
3603        let engine = QueryEngine::new(cache);
3604        let filter = QueryFilter::default();
3605        let results = engine.search("nonexistent", filter).unwrap();
3606
3607        assert_eq!(results.len(), 0);
3608    }
3609
3610    #[test]
3611    fn test_search_no_index() {
3612        let temp = TempDir::new().unwrap();
3613        let project = temp.path().join("project");
3614        fs::create_dir(&project).unwrap();
3615
3616        let cache = CacheManager::new(&project);
3617        let engine = QueryEngine::new(cache);
3618        let filter = QueryFilter::default();
3619
3620        // Should fail when index doesn't exist
3621        assert!(engine.search("test", filter).is_err());
3622    }
3623
3624    #[test]
3625    fn test_search_special_characters() {
3626        let temp = TempDir::new().unwrap();
3627        let project = temp.path().join("project");
3628        fs::create_dir(&project).unwrap();
3629
3630        fs::write(project.join("main.rs"), "let x = 42;\nlet y = x + 1;").unwrap();
3631
3632        let cache = CacheManager::new(&project);
3633        let indexer = Indexer::new(cache, IndexConfig::default());
3634        indexer.index(&project, false).unwrap();
3635
3636        let cache = CacheManager::new(&project);
3637
3638        let engine = QueryEngine::new(cache);
3639        let filter = QueryFilter::default();
3640
3641        // Search for special characters
3642        let results = engine.search("x + ", filter).unwrap();
3643        assert!(!results.is_empty());
3644    }
3645
3646    #[test]
3647    fn test_search_unicode() {
3648        let temp = TempDir::new().unwrap();
3649        let project = temp.path().join("project");
3650        fs::create_dir(&project).unwrap();
3651
3652        fs::write(project.join("main.rs"), "// 你好世界\nfn main() {}").unwrap();
3653
3654        let cache = CacheManager::new(&project);
3655        let indexer = Indexer::new(cache, IndexConfig::default());
3656        indexer.index(&project, false).unwrap();
3657
3658        let cache = CacheManager::new(&project);
3659
3660        let engine = QueryEngine::new(cache);
3661        let filter = QueryFilter {
3662            use_contains: true, // Unicode word boundaries may not work as expected
3663            force: true,        // Bypass broad query detection for 2-char Unicode pattern
3664            ..Default::default()
3665        };
3666
3667        // Search for unicode characters
3668        let results = engine.search("你好", filter).unwrap();
3669        assert!(!results.is_empty());
3670    }
3671
3672    #[test]
3673    fn test_case_sensitive_search() {
3674        let temp = TempDir::new().unwrap();
3675        let project = temp.path().join("project");
3676        fs::create_dir(&project).unwrap();
3677
3678        fs::write(project.join("main.rs"), "fn Test() {}\nfn test() {}").unwrap();
3679
3680        let cache = CacheManager::new(&project);
3681        let indexer = Indexer::new(cache, IndexConfig::default());
3682        indexer.index(&project, false).unwrap();
3683
3684        let cache = CacheManager::new(&project);
3685
3686        let engine = QueryEngine::new(cache);
3687        let filter = QueryFilter::default();
3688
3689        // Search is case-sensitive
3690        let results = engine.search("Test", filter).unwrap();
3691        assert!(results.iter().any(|r| r.preview.contains("Test()")));
3692    }
3693
3694    // ==================== Determinism Tests ====================
3695
3696    #[test]
3697    fn test_results_sorted_deterministically() {
3698        let temp = TempDir::new().unwrap();
3699        let project = temp.path().join("project");
3700        fs::create_dir(&project).unwrap();
3701
3702        fs::write(project.join("a.rs"), "fn test() {}").unwrap();
3703        fs::write(project.join("z.rs"), "fn test() {}").unwrap();
3704        fs::write(project.join("m.rs"), "fn test() {}\nfn test2() {}").unwrap();
3705
3706        let cache = CacheManager::new(&project);
3707        let indexer = Indexer::new(cache, IndexConfig::default());
3708        indexer.index(&project, false).unwrap();
3709
3710        let cache = CacheManager::new(&project);
3711
3712        let engine = QueryEngine::new(cache);
3713        let filter = QueryFilter::default();
3714
3715        // Run search multiple times
3716        let results1 = engine.search("test", filter.clone()).unwrap();
3717        let results2 = engine.search("test", filter.clone()).unwrap();
3718        let results3 = engine.search("test", filter).unwrap();
3719
3720        // Results should be identical and sorted by path then line
3721        assert_eq!(results1.len(), results2.len());
3722        assert_eq!(results1.len(), results3.len());
3723
3724        for i in 0..results1.len() {
3725            assert_eq!(results1[i].path, results2[i].path);
3726            assert_eq!(results1[i].path, results3[i].path);
3727            assert_eq!(results1[i].span.start_line, results2[i].span.start_line);
3728            assert_eq!(results1[i].span.start_line, results3[i].span.start_line);
3729        }
3730
3731        // Verify sorting (path ascending, then line ascending)
3732        for i in 0..results1.len().saturating_sub(1) {
3733            let curr = &results1[i];
3734            let next = &results1[i + 1];
3735            assert!(
3736                curr.path < next.path
3737                    || (curr.path == next.path && curr.span.start_line <= next.span.start_line)
3738            );
3739        }
3740    }
3741
3742    // ==================== Combined Filter Tests ====================
3743
3744    #[test]
3745    fn test_multiple_filters_combined() {
3746        let temp = TempDir::new().unwrap();
3747        let project = temp.path().join("project");
3748        fs::create_dir_all(project.join("src")).unwrap();
3749
3750        fs::write(project.join("src/main.rs"), "fn test() {}\nstruct Test {}").unwrap();
3751        fs::write(project.join("src/lib.rs"), "fn test() {}").unwrap();
3752        fs::write(project.join("test.js"), "function test() {}").unwrap();
3753
3754        let cache = CacheManager::new(&project);
3755        let indexer = Indexer::new(cache, IndexConfig::default());
3756        indexer.index(&project, false).unwrap();
3757
3758        let cache = CacheManager::new(&project);
3759
3760        let engine = QueryEngine::new(cache);
3761
3762        // Combine language, kind, and file pattern filters
3763        let filter = QueryFilter {
3764            language: Some(Language::Rust),
3765            kind: Some(SymbolKind::Function),
3766            file_pattern: Some("src/main".to_string()),
3767            symbols_mode: true,
3768            ..Default::default()
3769        };
3770        let results = engine.search("test", filter).unwrap();
3771
3772        // Should only find the function in src/main.rs
3773        assert_eq!(results.len(), 1);
3774        assert!(results[0].path.contains("src/main.rs"));
3775        assert_eq!(results[0].kind, SymbolKind::Function);
3776    }
3777
3778    // ==================== Helper Method Tests ====================
3779
3780    #[test]
3781    fn test_find_symbol_helper() {
3782        let temp = TempDir::new().unwrap();
3783        let project = temp.path().join("project");
3784        fs::create_dir(&project).unwrap();
3785
3786        fs::write(project.join("main.rs"), "fn greet() {}").unwrap();
3787
3788        let cache = CacheManager::new(&project);
3789        let indexer = Indexer::new(cache, IndexConfig::default());
3790        indexer.index(&project, false).unwrap();
3791
3792        let cache = CacheManager::new(&project);
3793
3794        let engine = QueryEngine::new(cache);
3795        let results = engine.find_symbol("greet").unwrap();
3796
3797        assert!(!results.is_empty());
3798        assert_eq!(results[0].kind, SymbolKind::Function);
3799    }
3800
3801    #[test]
3802    fn test_list_by_kind_helper() {
3803        let temp = TempDir::new().unwrap();
3804        let project = temp.path().join("project");
3805        fs::create_dir(&project).unwrap();
3806
3807        fs::write(
3808            project.join("main.rs"),
3809            "struct Point {}\nfn test() {}\nstruct Line {}",
3810        )
3811        .unwrap();
3812
3813        let cache = CacheManager::new(&project);
3814        let indexer = Indexer::new(cache, IndexConfig::default());
3815        indexer.index(&project, false).unwrap();
3816
3817        let cache = CacheManager::new(&project);
3818
3819        let engine = QueryEngine::new(cache);
3820
3821        // Search for structs that contain "oin" (Point contains it, Line doesn't)
3822        let filter = QueryFilter {
3823            kind: Some(SymbolKind::Struct),
3824            symbols_mode: true,
3825            use_contains: true, // "oin" is substring of "Point"
3826            ..Default::default()
3827        };
3828        let results = engine.search("oin", filter).unwrap();
3829
3830        // Should find Point struct
3831        assert!(!results.is_empty(), "Should find at least Point struct");
3832        assert!(results.iter().all(|r| r.kind == SymbolKind::Struct));
3833        assert!(results.iter().any(|r| r.symbol.as_deref() == Some("Point")));
3834    }
3835
3836    // ==================== Metadata Tests ====================
3837
3838    #[test]
3839    fn test_search_with_metadata() {
3840        let temp = TempDir::new().unwrap();
3841        let project = temp.path().join("project");
3842        fs::create_dir(&project).unwrap();
3843
3844        fs::write(project.join("main.rs"), "fn test() {}").unwrap();
3845
3846        let cache = CacheManager::new(&project);
3847        let indexer = Indexer::new(cache, IndexConfig::default());
3848        indexer.index(&project, false).unwrap();
3849
3850        let cache = CacheManager::new(&project);
3851
3852        let engine = QueryEngine::new(cache);
3853        let filter = QueryFilter::default();
3854        let response = engine.search_with_metadata("test", filter).unwrap();
3855
3856        // Check metadata is present (status might be stale if run inside git repo)
3857        assert!(!response.results.is_empty());
3858        // Note: can_trust_results may be false if running in a git repo without branch index
3859    }
3860
3861    // ==================== Multi-language Tests ====================
3862
3863    #[test]
3864    fn test_search_across_languages() {
3865        let temp = TempDir::new().unwrap();
3866        let project = temp.path().join("project");
3867        fs::create_dir(&project).unwrap();
3868
3869        fs::write(project.join("main.rs"), "fn greet() {}").unwrap();
3870        fs::write(project.join("main.ts"), "function greet() {}").unwrap();
3871        fs::write(project.join("main.py"), "def greet(): pass").unwrap();
3872
3873        let cache = CacheManager::new(&project);
3874        let indexer = Indexer::new(cache, IndexConfig::default());
3875        indexer.index(&project, false).unwrap();
3876
3877        let cache = CacheManager::new(&project);
3878
3879        let engine = QueryEngine::new(cache);
3880        let filter = QueryFilter::default();
3881        let results = engine.search("greet", filter).unwrap();
3882
3883        // Should find greet in all three languages
3884        assert!(results.len() >= 3);
3885        assert!(results.iter().any(|r| r.lang == Language::Rust));
3886        assert!(results.iter().any(|r| r.lang == Language::TypeScript));
3887        assert!(results.iter().any(|r| r.lang == Language::Python));
3888    }
3889}
3890
3891/// Process-local memo for the freshness verdict's expensive inputs.
3892///
3893/// `search_with_metadata` runs the freshness check on EVERY query, and
3894/// `find_references` calls it two or three times per MCP request. One snapshot
3895/// costs one `meta.db` connection, two `git` spawns and one `git status
3896/// --porcelain`; before this module the same query ran three connections and
3897/// three spawns, and the memo covered only the last of them. Results are memoised
3898/// briefly, keyed by workspace root.
3899///
3900/// What this trades away: an edit landing less than the TTL before a search can be
3901/// reported fresh. Agent tool round-trips are seconds apart, and `check_index_status`
3902/// — the explicit probe an agent uses when it cares — always bypasses the cache.
3903/// Every index write in this process invalidates the memo.
3904///
3905/// `REFLEX_FRESHNESS_TTL_MS` overrides the window; `0` disables caching entirely.
3906mod status_cache {
3907    use crate::cache::{CacheManager, FileFingerprint};
3908    use crate::git::WorktreeChanges;
3909    use crate::indexer::{Indexer, PathPolicy, looks_binary};
3910    use crate::models::{IndexConfig, IndexStatus, IndexWarning, IndexWarningDetails, Language};
3911    use anyhow::Result;
3912    use std::collections::{HashMap, HashSet};
3913    use std::path::{Path, PathBuf};
3914    use std::sync::{Arc, Mutex, OnceLock};
3915    use std::time::{Duration, Instant};
3916
3917    const DEFAULT_TTL_MS: u64 = 1_000;
3918
3919    pub type Verdict = (IndexStatus, bool, Option<IndexWarning>);
3920
3921    /// What one freshness computation found. The verdict is derived per caller
3922    /// (`QueryEngine::index_report_for`), because its wording depends on scope.
3923    pub struct Snapshot {
3924        /// A verdict that does not depend on the working tree at all (a cache
3925        /// written by another build). `None` means: judge by `changes`.
3926        pub decided: Option<Verdict>,
3927        /// Branch/commit context for humans; present in git repos even when fresh.
3928        pub details: Option<IndexWarningDetails>,
3929        /// Files whose content differs from what the index holds.
3930        pub changes: WorktreeChanges,
3931    }
3932
3933    impl Snapshot {
3934        fn fresh() -> Self {
3935            Self {
3936                decided: Some((IndexStatus::Fresh, true, None)),
3937                details: None,
3938                changes: WorktreeChanges::default(),
3939            }
3940        }
3941    }
3942
3943    struct Entry {
3944        computed_at: Instant,
3945        snapshot: Arc<Snapshot>,
3946    }
3947
3948    fn ttl() -> Duration {
3949        static TTL: OnceLock<Duration> = OnceLock::new();
3950        *TTL.get_or_init(|| {
3951            let ms = std::env::var("REFLEX_FRESHNESS_TTL_MS")
3952                .ok()
3953                .and_then(|v| v.trim().parse::<u64>().ok())
3954                .unwrap_or(DEFAULT_TTL_MS);
3955            Duration::from_millis(ms)
3956        })
3957    }
3958
3959    fn store() -> &'static Mutex<HashMap<PathBuf, Entry>> {
3960        static STORE: OnceLock<Mutex<HashMap<PathBuf, Entry>>> = OnceLock::new();
3961        STORE.get_or_init(|| Mutex::new(HashMap::new()))
3962    }
3963
3964    fn key(root: &Path) -> PathBuf {
3965        root.canonicalize().unwrap_or_else(|_| root.to_path_buf())
3966    }
3967
3968    /// Only paths Reflex would index can make the index stale. Editing a README or
3969    /// anything under `target/` must not mark it permanently behind.
3970    fn indexable_with(path: &str, policy: &crate::indexer::PathPolicy) -> bool {
3971        crate::indexer::Indexer::is_indexable_path_with(Path::new(path), Some(policy))
3972    }
3973
3974    /// The memoised snapshot for the workspace `cache` belongs to.
3975    pub fn snapshot(cache: &CacheManager) -> Result<Arc<Snapshot>> {
3976        let root = cache.workspace_root();
3977        let key = key(&root);
3978        let ttl = ttl();
3979
3980        if !ttl.is_zero()
3981            && let Ok(map) = store().lock()
3982            && let Some(entry) = map.get(&key)
3983            && entry.computed_at.elapsed() < ttl
3984        {
3985            return Ok(Arc::clone(&entry.snapshot));
3986        }
3987
3988        let snapshot = Arc::new(compute(cache, &root)?);
3989
3990        if !ttl.is_zero()
3991            && let Ok(mut map) = store().lock()
3992        {
3993            map.insert(
3994                key,
3995                Entry {
3996                    computed_at: Instant::now(),
3997                    snapshot: Arc::clone(&snapshot),
3998                },
3999            );
4000        }
4001
4002        Ok(snapshot)
4003    }
4004
4005    /// Where the candidate paths for a content check come from.
4006    enum Probe {
4007        /// `git status`, the dirty-at-index set and, when HEAD moved, the commit
4008        /// diff. Each path is then confirmed against its indexed fingerprint.
4009        Git(HashSet<String>),
4010        /// No git, or git could not answer: walk the tree and compare every file.
4011        Walk,
4012    }
4013
4014    /// Compare the working tree to what the index holds.
4015    ///
4016    /// Until 2.0.0 the baseline was the indexed COMMIT: any path `git status`
4017    /// listed made the index stale, so a dirty tree could never be fresh however
4018    /// often it was re-indexed, and an agent's session is dirty from its first edit
4019    /// to its last. The baseline is now the per-file fingerprint the indexer wrote
4020    /// (`size`, `mtime_ns`, blake3): a path git calls modified whose bytes the index
4021    /// already holds is not stale, and a commit of already-indexed content moves
4022    /// HEAD without changing a single file's verdict.
4023    fn compute(cache: &CacheManager, root: &Path) -> Result<Snapshot> {
4024        let is_git = crate::git::is_git_repo(root) && crate::git::is_git_available();
4025        let current_branch = if is_git {
4026            crate::git::read_head_branch(root).or_else(|| crate::git::get_current_branch(root).ok())
4027        } else {
4028            None
4029        };
4030
4031        // One connection for the schema hash, the branch row and the dirty set.
4032        let reads = match cache.status_reads(current_branch.as_deref()) {
4033            Ok(r) => r,
4034            Err(e) => {
4035                log::debug!("Could not read index status from meta.db: {}", e);
4036                return Ok(Snapshot::fresh());
4037            }
4038        };
4039
4040        // A cache written by a different Reflex build. `validate()` no longer bails on
4041        // this (that turned a version skew into a rebuild stampede — see cache.rs), so
4042        // reads still work; they are simply not to be trusted, and the message names
4043        // who owns the cache so the user can pick a side.
4044        if !reads.schema_ok {
4045            let reason = match reads.owner {
4046                Some((v, sha)) if v != env!("CARGO_PKG_VERSION") => {
4047                    let sha = sha
4048                        .map(|s| format!(" (sha {})", &s[..s.len().min(7)]))
4049                        .unwrap_or_default();
4050                    format!(
4051                        "This .reflex/ was written by reflex {}{}; this binary is reflex {}. \
4052                         Results come from a cache format this version does not fully \
4053                         understand.",
4054                        v,
4055                        sha,
4056                        env!("CARGO_PKG_VERSION")
4057                    )
4058                }
4059                // Same version, or unstamped: the cache format changed under it. A
4060                // reindex brings it up to date; results until then may be partial.
4061                _ => "The index was built with a different cache format and needs \
4062                      rebuilding. Results may be incomplete until then."
4063                    .to_string(),
4064            };
4065
4066            let warning = IndexWarning::new(reason, "index_project");
4067            return Ok(Snapshot {
4068                decided: Some((IndexStatus::Stale, false, Some(warning))),
4069                details: None,
4070                changes: WorktreeChanges::default(),
4071            });
4072        }
4073
4074        let config = cache.load_index_config().unwrap_or_else(|e| {
4075            log::debug!("Using default index config for freshness: {}", e);
4076            IndexConfig::default()
4077        });
4078        let policy = PathPolicy::from_config(root, &config);
4079
4080        let info = reads.branch_info.as_ref();
4081        let mut details = IndexWarningDetails {
4082            current_branch: current_branch.clone(),
4083            indexed_branch: info.map(|i| i.branch.clone()),
4084            current_commit: None,
4085            indexed_commit: info.map(|i| i.commit_sha.clone()),
4086            indexed_at: info.map(|i| i.last_indexed),
4087            checked_by: None,
4088        };
4089
4090        let probe = if current_branch.is_some() {
4091            let current_commit = crate::git::get_current_commit(root).ok();
4092            details.current_commit = current_commit.clone();
4093            let indexed_commit = info.map(|i| i.commit_sha.as_str());
4094            match git_candidates(
4095                root,
4096                &reads.dirty_at_index,
4097                indexed_commit,
4098                current_commit.as_deref(),
4099            ) {
4100                Ok(set) => Probe::Git(set),
4101                Err(e) => {
4102                    log::debug!("git could not name changed paths ({}); walking the tree", e);
4103                    Probe::Walk
4104                }
4105            }
4106        } else {
4107            Probe::Walk
4108        };
4109
4110        let mut changes = match probe {
4111            Probe::Git(candidates) => {
4112                let keys: Vec<&str> = candidates.iter().map(String::as_str).collect();
4113                let fingerprints = cache.fingerprints_for(&keys)?;
4114                let mut out = WorktreeChanges::default();
4115                for path in &candidates {
4116                    classify_one(
4117                        root,
4118                        &config,
4119                        &policy,
4120                        path,
4121                        fingerprints.get(path),
4122                        &mut out,
4123                    );
4124                }
4125                details.checked_by = Some("git".to_string());
4126                out
4127            }
4128            Probe::Walk => {
4129                let fingerprints = fingerprints_all(cache)?;
4130                details.checked_by = Some("walk".to_string());
4131                walk_changes(root, &config, &policy, &fingerprints)
4132            }
4133        };
4134        changes.sort();
4135
4136        Ok(Snapshot {
4137            decided: None,
4138            details: Some(details),
4139            changes,
4140        })
4141    }
4142
4143    /// Paths whose content MAY differ from the index, in a git repository.
4144    ///
4145    /// Three sources, each necessary:
4146    /// * `git status` — edits, additions and deletions since HEAD;
4147    /// * the paths that were dirty when the index was written — an edit indexed
4148    ///   and then reverted with `git checkout -- f` is clean to git but differs
4149    ///   from the index;
4150    /// * `git diff --name-only indexed..HEAD` when HEAD moved — a commit, pull or
4151    ///   checkout changes tracked files without `git status` ever listing them.
4152    ///
4153    /// A path in none of the three has the bytes the index was built from.
4154    fn git_candidates(
4155        root: &Path,
4156        dirty_at_index: &[String],
4157        indexed_commit: Option<&str>,
4158        current_commit: Option<&str>,
4159    ) -> Result<HashSet<String>> {
4160        let mut set = crate::git::changed_paths(root)?;
4161        set.extend(dirty_at_index.iter().cloned());
4162        match (indexed_commit, current_commit) {
4163            (Some(indexed), Some(current)) if indexed != current => {
4164                if indexed == "unknown" {
4165                    anyhow::bail!("the index does not record a commit");
4166                }
4167                set.extend(crate::git::diff_names(root, indexed, current)?);
4168            }
4169            (None, _) | (_, None) => anyhow::bail!("no commit to diff against"),
4170            _ => {}
4171        }
4172        Ok(set)
4173    }
4174
4175    /// Every indexed fingerprint, memoised on the open index handle so a walk pays
4176    /// the table read once per index version.
4177    fn fingerprints_all(cache: &CacheManager) -> Result<Arc<HashMap<String, FileFingerprint>>> {
4178        match super::open_index::get_or_open(cache) {
4179            Ok(open) => open.fingerprints(cache),
4180            Err(_) => Ok(Arc::new(cache.load_fingerprints()?)),
4181        }
4182    }
4183
4184    /// Whether the file at `full` still has the bytes the index holds.
4185    ///
4186    /// A matching `(size, mtime_ns)` is proof enough; otherwise the file is hashed.
4187    /// That second step is what makes "reindexed but uncommitted" fresh: git lists
4188    /// the path, the mtime moved, and the hash says the index already has it.
4189    fn content_matches(full: &Path, md: &std::fs::Metadata, fp: &FileFingerprint) -> bool {
4190        if fp.stat_matches(md) {
4191            return true;
4192        }
4193        match std::fs::read(full) {
4194            Ok(bytes) => blake3::hash(&bytes).to_hex().as_str() == fp.hash,
4195            Err(_) => false,
4196        }
4197    }
4198
4199    /// Sort one candidate path into `out`, or leave it out when the indexer would.
4200    fn classify_one(
4201        root: &Path,
4202        config: &IndexConfig,
4203        policy: &PathPolicy,
4204        path: &str,
4205        fp: Option<&FileFingerprint>,
4206        out: &mut WorktreeChanges,
4207    ) {
4208        if !indexable_with(path, policy) {
4209            return;
4210        }
4211        let full = root.join(path);
4212        let md = if config.follow_symlinks {
4213            std::fs::metadata(&full)
4214        } else {
4215            std::fs::symlink_metadata(&full)
4216        };
4217        match (md, fp) {
4218            (Err(_), Some(_)) => out.push_deleted(path),
4219            (Err(_), None) => {}
4220            // A directory or an unfollowed symlink is nothing the indexer holds.
4221            (Ok(md), _) if !md.is_file() => {
4222                if fp.is_some() {
4223                    out.push_deleted(path);
4224                }
4225            }
4226            (Ok(md), _) if md.len() > config.max_file_size as u64 => {
4227                // Grown past the limit: the content differs and the next index
4228                // drops it. Not indexed before either → nothing to report.
4229                if fp.is_some() {
4230                    out.push_modified(path);
4231                }
4232            }
4233            // A new non-code file is sniffed the way the indexer sniffs it: a
4234            // binary is never indexed, so it can never be missing from the index.
4235            (Ok(_), None) => {
4236                if !Language::from_path(Path::new(path)).is_code() && looks_binary(&full) {
4237                    return;
4238                }
4239                out.push_added(path);
4240            }
4241            (Ok(md), Some(fp)) => {
4242                if !content_matches(&full, &md, fp) {
4243                    out.push_modified(path);
4244                }
4245            }
4246        }
4247    }
4248
4249    /// Compare every file the indexer would walk against the fingerprint table.
4250    ///
4251    /// Outside git this is the only way to know; the walk is parallel and the
4252    /// hash is computed only for files whose stat does not match, so on a warm
4253    /// cache it is a few milliseconds per thousand files.
4254    fn walk_changes(
4255        root: &Path,
4256        config: &IndexConfig,
4257        policy: &PathPolicy,
4258        fingerprints: &HashMap<String, FileFingerprint>,
4259    ) -> WorktreeChanges {
4260        let threads = std::thread::available_parallelism()
4261            .map(|n| n.get())
4262            .unwrap_or(1)
4263            .min(8);
4264        let mut builder = Indexer::walk_builder(root, config, policy);
4265        builder.threads(threads);
4266
4267        let seen: Mutex<Vec<(String, std::fs::Metadata)>> = Mutex::new(Vec::new());
4268        let max_size = config.max_file_size as u64;
4269        builder.build_parallel().run(|| {
4270            let seen = &seen;
4271            Box::new(move |entry| {
4272                let Ok(entry) = entry else {
4273                    return ignore::WalkState::Continue;
4274                };
4275                if !entry.file_type().map(|t| t.is_file()).unwrap_or(false) {
4276                    return ignore::WalkState::Continue;
4277                }
4278                let path = entry.path();
4279                if policy.classify(path).is_none() {
4280                    return ignore::WalkState::Continue;
4281                }
4282                let Ok(md) = entry.metadata() else {
4283                    return ignore::WalkState::Continue;
4284                };
4285                if md.len() > max_size {
4286                    return ignore::WalkState::Continue;
4287                }
4288                let rel = path
4289                    .strip_prefix(root)
4290                    .unwrap_or(path)
4291                    .to_string_lossy()
4292                    .replace('\\', "/");
4293                if let Ok(mut v) = seen.lock() {
4294                    v.push((rel, md));
4295                }
4296                ignore::WalkState::Continue
4297            })
4298        });
4299
4300        let seen = seen.into_inner().unwrap_or_default();
4301        let mut out = WorktreeChanges::default();
4302        let mut present: HashSet<&str> = HashSet::with_capacity(seen.len());
4303        for (rel, md) in &seen {
4304            present.insert(rel.as_str());
4305            match fingerprints.get(rel) {
4306                None => {
4307                    let full = root.join(rel);
4308                    if !Language::from_path(&full).is_code() && looks_binary(&full) {
4309                        continue;
4310                    }
4311                    out.push_added(rel);
4312                }
4313                Some(fp) => {
4314                    if !content_matches(&root.join(rel), md, fp) {
4315                        out.push_modified(rel);
4316                    }
4317                }
4318            }
4319        }
4320        for path in fingerprints.keys() {
4321            if !present.contains(path.as_str()) {
4322                out.push_deleted(path);
4323            }
4324        }
4325        out
4326    }
4327
4328    /// Drop any memo for `root`, so the next read is fresh.
4329    ///
4330    /// Used by `check_index_status`, the explicit probe: an agent that asks whether
4331    /// the index is current must never be answered from a cache. Also called around
4332    /// every index write.
4333    pub fn invalidate(root: &Path) {
4334        let key = key(root);
4335        if let Ok(mut map) = store().lock() {
4336            map.remove(&key);
4337        }
4338    }
4339}